feat: add integration test harness, workflow cancellation, and provider send gating

drizzle/0007_workflow_status_canceled.sql

@@ -0,0 +1,3 @@
+-- WorkflowRepository.cancelInstance writes 'canceled', which the enum never
+-- had, so every cancel failed with invalid_text_representation.
+ALTER TYPE "public"."workflow_status" ADD VALUE IF NOT EXISTS 'canceled';

drizzle/meta/_journal.json

@@ -50,6 +50,13 @@
       "when": 1790600000000,
       "tag": "0006_message_logs_redundant_indexes",
       "breakpoints": true
+    },
+    {
+      "idx": 7,
+      "version": "7",
+      "when": 1790700000000,
+      "tag": "0007_workflow_status_canceled",
+      "breakpoints": true
     }
   ]
 }

packages/mcp/src/tools.ts

@@ -302,24 +302,17 @@ export function registerTools(server: McpServer, api: NotifkitApi): void {
         if (seen.has(target)) continue;
         seen.add(target);
 
-        let userRecord: Record<string, unknown>;
-        if (channel === "email") {
-          userRecord = { id: `email:${target}`, email: target };
-        } else if (channel === "sms") {
-          userRecord = { id: `phone:${target}`, phone: target };
-        } else if (channel === "push") {
-          userRecord = { id: `push:${target}`, pushToken: target };
-        } else if (channel === "telegram") {
-          userRecord = { id: `telegram:${target}` };
-        } else if (channel === "discord") {
-          userRecord = { id: `discord:${target}` };
-        } else if (channel === "whatsapp") {
-          userRecord = { id: `whatsapp:${target}`, phone: target };
-        } else if (channel === "slack") {
-          userRecord = { id: `slack:${target}` };
-        } else {
-          userRecord = { id: `webhook:${target}` };
-        }
+        // Every channel needs a contact on that channel to be reached. Chat
+        // channels used to get a bare user record, and WhatsApp a phone
+        // number filed as an SMS contact, so those campaigns sent nothing.
+        const userRecord: Record<string, unknown> =
+          channel === "email"
+            ? { id: `email:${target}`, email: target }
+            : channel === "sms"
+              ? { id: `phone:${target}`, phone: target }
+              : channel === "push"
+                ? { id: `push:${target}`, pushToken: target }
+                : { id: `${channel}:${target}`, contacts: [{ channel, target }] };
         recipients.push(userRecord);
       }
 

packages/provider-discord/src/index.ts

@@ -66,7 +66,11 @@ export class DiscordTransport implements Transport {
     const subject = content.subject;
     // Discord messages have no subject line of their own, so a template that
     // renders one is bolded onto the first line rather than dropped.
-    const text = subject ? `**${subject}**\n${body}` : body;
+    // Either part may be missing: a subject-only template must not post
+    // "undefined" underneath it.
+    const text = [subject ? `**${subject}**` : undefined, body]
+      .filter((part) => typeof part === "string" && part)
+      .join("\n");
 
     // A template may carry its own sender, so one webhook can post as
     // different bots for different alert types.

packages/provider-resend/src/index.ts

@@ -73,7 +73,10 @@ export class ResendTransport implements Transport {
         from,
         to,
         subject: subject ?? "Notification",
-        html: htmlBody ?? `<p>${body}</p>`,
+        // A text-only template gets an HTML part built from its text. The text
+        // was rendered unescaped (it is plain text), so it is escaped here —
+        // otherwise a variable holding markup becomes live HTML in the email.
+        html: htmlBody ?? (typeof body === "string" ? textToHtml(body) : undefined),
         text: body,
         ...(replyTo ? { replyTo } : {}),
         ...(headers && Object.keys(headers).length > 0 ? { headers } : {}),
@@ -249,3 +252,13 @@ export class ResendTransport implements Transport {
     return [];
   }
 }
+
+function textToHtml(text: string): string {
+  const escaped = text
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&#039;");
+  return `<p>${escaped.replace(/\r?\n/g, "<br>")}</p>`;
+}

packages/provider-telegram/src/index.ts

@@ -55,7 +55,12 @@ export class TelegramTransport implements Transport {
     const subject = content.subject;
     // Telegram messages have no subject line of their own, so a template
     // that renders one gets folded into the first line rather than dropped.
-    const text = subject ? `${subject}\n\n${body}` : body;
+    // Either part may be missing: a subject-only template must not send the
+    // word "undefined" underneath it.
+    const text = [subject, body].filter((part) => typeof part === "string" && part).join("\n\n");
+    if (!text) {
+      return { success: false, error: "No message text on task" };
+    }
     const parseMode = typeof content.parseMode === "string" ? content.parseMode : this.parseMode;
 
     try {

src/client.ts

@@ -22,6 +22,7 @@ import type {
   SuppressionRecord,
   ProjectRecord,
   ProjectKeyRecord,
+  CreatedProjectKeyRecord,
   SystemHealthRecord,
   SystemMetricsRecord,
   DLQMessageRecord,
@@ -61,10 +62,17 @@ export class NotifkitClient {
       return undefined as T;
     }
 
-    const data = await res.json();
+    const text = await res.text();
+    let data: unknown;
+    try {
+      data = text ? JSON.parse(text) : undefined;
+    } catch {
+      // Not JSON: an HTML error page, or a proxy in front of the API.
+      data = undefined;
+    }
     if (!res.ok) {
-      const errorMsg =
-        (data as any).message || (data as any).error || `Request failed with status ${res.status}`;
+      const body = data as { message?: string; error?: string } | undefined;
+      const errorMsg = body?.message || body?.error || `Request failed with status ${res.status}`;
       throw new Error(errorMsg);
     }
     return data as T;
@@ -285,7 +293,7 @@ export class NotifkitClient {
   async createProjectKey(
     id: string,
     input?: { role?: "admin" | "read_only" },
-  ): Promise<ProjectKeyRecord> {
+  ): Promise<CreatedProjectKeyRecord> {
     return this.request(`/v1/projects/${encodeURIComponent(id)}/keys`, "POST", input || {});
   }
 
@@ -472,7 +480,9 @@ export class NotifkitClient {
   }
 
   /** Replay a dead-letter queue message. */
-  async replayDLQMessage(id: string): Promise<{ success: boolean; replayedId: string }> {
+  async replayDLQMessage(
+    id: string,
+  ): Promise<{ success: boolean; replayedId: string; stream: string }> {
     return this.request("/v1/dlq/replay", "POST", { id });
   }
 

src/contracts/events/notification-ai-pending.ts

@@ -14,5 +14,17 @@ export const NotificationAiPendingPayloadSchema = z.object({
   aiPrompts: z.record(z.string(), z.string()),
   scheduledAt: z.string().datetime().optional(),
   fallbackChain: z.array(NotificationChannelSchema).optional(),
+  /**
+   * The task id the engine assigned. Stable across replays, so a re-run of
+   * this event dispatches the same task — which delivery's lease then sends
+   * once — rather than a fresh one.
+   */
+  taskId: z.string().min(1).optional(),
+  /** The contact address the engine resolved and cleared against suppressions. */
+  destination: z.string().min(1).optional(),
+  /** Headers for the provider, e.g. one-click unsubscribe. */
+  deliveryHeaders: z.record(z.string(), z.string()).optional(),
+  campaignId: z.string().min(1).max(128).optional(),
+  workflowInstanceId: z.string().uuid().optional(),
 });
 export type NotificationAiPendingPayload = z.infer<typeof NotificationAiPendingPayloadSchema>;

src/contracts/events/notification-dispatched.ts

@@ -45,8 +45,15 @@ export const NotificationDispatchedPayloadSchema = z.object({
   deliveryOptions: DeliveryOptionsSchema,
   fallbackChain: z.array(NotificationChannelSchema).optional(),
   throttleAttemptCount: z.number().int().nonnegative().optional(),
+  /**
+   * Provider attempts that already failed. Delivery re-queues a failed send
+   * through the scheduler until this reaches `deliveryOptions.maxAttempts`.
+   */
+  deliveryAttemptCount: z.number().int().nonnegative().optional(),
   /** Campaign this message belongs to, carried from the originating request. */
   campaignId: z.string().min(1).max(128).optional(),
+  /** Workflow instance whose step sent this, recorded on the delivery log. */
+  workflowInstanceId: z.string().uuid().optional(),
 });
 /** A `notification.dispatched` task as it travels on the outbound stream. */
 export type DispatchedTaskPayload = z.infer<typeof NotificationDispatchedPayloadSchema>;

src/contracts/events/notification-enriched.ts

@@ -50,5 +50,13 @@ export const NotificationEnrichedPayloadSchema = z.object({
   destination: z.string().min(1).optional(),
   /** Campaign this message belongs to, carried from the originating request. */
   campaignId: z.string().min(1).max(128).optional(),
+  /** Workflow instance whose step sent this, carried to the delivery log. */
+  workflowInstanceId: z.string().uuid().optional(),
+  /**
+   * Set on delivery's channel fallback. The notification was counted against
+   * the user's throttle when it was first sent; counting it again per channel
+   * it falls back through would drop the very send meant to reach them.
+   */
+  throttleCounted: z.boolean().optional(),
 });
 export type NotificationEnrichedPayload = z.infer<typeof NotificationEnrichedPayloadSchema>;

src/contracts/events/notification-requested.ts

@@ -32,5 +32,7 @@ export const NotificationRequestedPayloadSchema = z.object({
    * reported on afterwards. Carried unchanged to the delivery log.
    */
   campaignId: z.string().min(1).max(128).optional(),
+  /** Set by the workflow worker on a step's notify, carried to the delivery log. */
+  workflowInstanceId: z.string().uuid().optional(),
 });
 export type NotificationRequestedPayload = z.infer<typeof NotificationRequestedPayloadSchema>;

src/contracts/sdk.ts

@@ -110,12 +110,12 @@ export const TemplateSchema = z.object({
   content: z.record(z.string(), z.unknown()),
   aiPrompts: z.record(z.string(), z.string()).optional(),
 });
-export type TemplateInput = z.infer<typeof TemplateSchema>;
+export type TemplateInput = z.input<typeof TemplateSchema>;
 
 export const SyncTemplatesSchema = z.object({
   templates: z.array(TemplateSchema).min(1),
 });
-export type SyncTemplatesInput = z.infer<typeof SyncTemplatesSchema>;
+export type SyncTemplatesInput = z.input<typeof SyncTemplatesSchema>;
 
 // ─── notify() ─────────────────────────────────────────────────────────────────
 
@@ -216,7 +216,7 @@ export const TriggerWorkflowSchema = z.object({
   input: z.record(z.string(), z.unknown()).optional(),
   user: z.union([z.string().min(1), InlineUserSchema]).optional(),
 });
-export type TriggerWorkflowInput = z.infer<typeof TriggerWorkflowSchema>;
+export type TriggerWorkflowInput = z.input<typeof TriggerWorkflowSchema>;
 
 // ─── Events ───────────────────────────────────────────────────────────────────
 
@@ -241,6 +241,12 @@ export const WorkflowStepSchema = z.discriminatedUnion("action", [
     options: z
       .object({
         timeout: z.string().optional(), // e.g. "24h"
+        /**
+         * Fields the event must carry to wake this instance, e.g.
+         * `{ userId: "u1" }`. Without it any event of that name in the project
+         * wakes every instance waiting on it.
+         */
+        match: z.record(z.string(), z.unknown()).optional(),
       })
       .optional(),
   }),
@@ -251,7 +257,7 @@ export const CreateWorkflowSchema = z.object({
   name: z.string().min(1),
   steps: z.array(WorkflowStepSchema).min(1),
 });
-export type CreateWorkflowInput = z.infer<typeof CreateWorkflowSchema>;
+export type CreateWorkflowInput = z.input<typeof CreateWorkflowSchema>;
 
 // ─── Projects ─────────────────────────────────────────────────────────────────
 
@@ -291,7 +297,8 @@ export interface UserResponse extends UserProfileResponse {
 
 export interface UserDetailResponse extends UserResponse {
   contacts: UserContactResponse[];
-  recentLogs?: unknown[];
+  /** The 50 most recent delivery-log rows for this user's addresses. */
+  logs: NotificationLogRecord[];
 }
 
 export interface WorkflowDefinitionRecord {
@@ -302,30 +309,53 @@ export interface WorkflowDefinitionRecord {
   updatedAt?: string;
 }
 
-export interface WorkflowInstanceRecord {
+export interface WorkflowStepRecord {
   id: string;
-  name: string;
-  status: string;
-  currentStepIndex: number;
-  input?: Record<string, unknown>;
-  output?: Record<string, unknown>;
+  instanceId: string;
+  stepIndex: string;
+  action: "notify" | "wait" | "waitForEvent" | "run" | string;
+  output: unknown;
+  error: string | null;
   createdAt: string;
-  updatedAt: string;
 }
 
+export interface WorkflowWaiterRecord {
+  id: string;
+  instanceId: string;
+  eventName: string;
+  matchCriteria: Record<string, unknown>;
+  expiresAt: string;
+  createdAt: string;
+}
+
+export interface WorkflowInstanceRecord {
+  id: string;
+  projectId: string;
+  name: string;
+  status: "pending" | "running" | "completed" | "failed" | "canceled";
+  input?: Record<string, unknown> | null;
+  createdAt: string;
+  updatedAt: string;
+  steps: WorkflowStepRecord[];
+  waiters: WorkflowWaiterRecord[];
+}
+
+/** One row of the delivery log, as the API returns it. */
 export interface NotificationLogRecord {
   id: string;
-  notificationId: string;
-  taskId?: string;
+  projectId: string;
+  taskId: string;
+  providerMessageId: string | null;
+  templateId: string | null;
+  workflowInstanceId: string | null;
   channel: string;
-  target?: string;
+  attempt: number;
+  /** `dispatched`, `attempt`, or an engagement event such as `opened`. */
+  kind: string;
   status: string;
-  templateId?: string;
-  campaign?: string;
-  createdAt: string;
-  dispatchedAt?: string;
-  error?: string;
-  metadata?: Record<string, unknown>;
+  campaignId: string | null;
+  metadata: Record<string, unknown> | null;
+  timestamp: string;
 }
 
 export interface TemplateRecordResponse {
@@ -338,9 +368,12 @@ export interface TemplateRecordResponse {
 
 export interface SuppressionRecord {
   id: string;
+  projectId: string;
   channel: string;
   target: string;
   reason: string;
+  source: string | null;
+  taskId: string | null;
   createdAt: string;
 }
 
@@ -353,30 +386,45 @@ export interface ProjectRecord {
   createdAt: string;
 }
 
+/** A key as listed: the secret itself is only ever returned once, on creation. */
 export interface ProjectKeyRecord {
   id: string;
-  apiKey: string;
-  role: string;
+  role: "admin" | "read_only";
   createdAt: string;
 }
 
+export interface CreatedProjectKeyRecord {
+  id: string;
+  apiKey: string;
+  role: "admin" | "read_only";
+}
+
 export interface SystemHealthRecord {
-  status: string;
-  timestamp: string;
-  redis?: { status: string };
-  db?: { status: string };
-  workers?: Record<string, unknown>;
+  status: "healthy" | "degraded";
+  redis: { ok: boolean; latencyMs: number };
+  db: { ok: boolean; latencyMs: number };
+  /** Each worker's last heartbeat, or `{ status: "unknown" }` when none arrived. */
+  workers: Record<string, Record<string, unknown>>;
 }
 
 export interface SystemMetricsRecord {
-  queues: Record<string, number>;
-  throughput?: Record<string, unknown>;
+  /** Unconsumed backlog per stream. Empty for project-scoped callers. */
+  streams: Record<string, number>;
+  deliveryStats: {
+    total: number;
+    delivered: number;
+    failed: number;
+    successRate: number;
+  };
 }
 
 export interface DLQMessageRecord {
   id: string;
-  stream: string;
+  eventType: string;
   payload: Record<string, unknown>;
-  error?: string;
+  /** Why the message was dead-lettered. */
+  error: string;
   timestamp: string;
+  /** The stream it failed on, where a replay puts it back. */
+  originalStream?: string;
 }

src/db/schema.ts

@@ -270,6 +270,7 @@ export const workflowStatusEnum = pgEnum("workflow_status", [
   "running",
   "completed",
   "failed",
+  "canceled",
 ]);
 
 export const workflowDefinitions = pgTable(

src/queue/index.ts

@@ -96,12 +96,14 @@ export class StreamProducer {
   }
 
   async publish(partial: Omit<StreamEvent, "id" | "timestamp">): Promise<string> {
-    const event: StreamEvent = {
+    return this.enqueue({
       ...partial,
       id: crypto.randomUUID(),
       timestamp: new Date().toISOString(),
-    };
+    });
+  }
 
+  private enqueue(event: StreamEvent): Promise<string> {
     return new Promise<string>((resolve, reject) => {
       this.pendingQueue.push({ event, resolve, reject });
       if (this.pendingQueue.length >= 100) {
@@ -209,13 +211,15 @@ export class StreamProducer {
   ): Promise<{ messageIds: string[]; eventIds: string[] }> {
     if (partials.length === 0) return { messageIds: [], eventIds: [] };
 
+    // The ids returned must be the ids the events are written with: callers
+    // record them as notification ids.
     const eventIds: string[] = [];
     const promises: Promise<string>[] = [];
 
     for (const partial of partials) {
       const id = crypto.randomUUID();
       eventIds.push(id);
-      promises.push(this.publish(partial));
+      promises.push(this.enqueue({ ...partial, id, timestamp: new Date().toISOString() }));
     }
 
     const messageIds = await Promise.all(promises);
@@ -472,7 +476,7 @@ export class StreamConsumer {
    * against this consumer's stream so a replay lands where it would have been
    * read from.
    */
-  async deadLetter(event: StreamEvent): Promise<void> {
+  async deadLetter(event: StreamEvent, reason?: string): Promise<void> {
     if (!this.dlqStream) {
       this.logger?.error(
         { eventId: event.id },
@@ -486,12 +490,17 @@ export class StreamConsumer {
       "data",
       JSON.stringify({
         ...event,
-        dlq: { originalStream: this.streams[0]!, ackedAt: new Date().toISOString() },
+        dlq: { originalStream: this.streams[0]!, ackedAt: new Date().toISOString(), reason },
       }),
     );
   }
 
-  async nack(messageId: string, event: StreamEvent, stream?: string): Promise<void> {
+  async nack(
+    messageId: string,
+    event: StreamEvent,
+    stream?: string,
+    reason?: string,
+  ): Promise<void> {
     const s = stream ?? this.streams[0]!;
     if (this.dlqStream) {
       // Sequential rather than pipelined, and in this order: the ack is what
@@ -506,7 +515,7 @@ export class StreamConsumer {
         "data",
         JSON.stringify({
           ...event,
-          dlq: { originalStream: s, ackedAt: new Date().toISOString() },
+          dlq: { originalStream: s, ackedAt: new Date().toISOString(), reason },
         }),
       );
 

src/rate-limiter/index.ts

@@ -308,24 +308,25 @@ export class ApiRateLimiter {
     }
 
     // 2. Slow Path: tokens are depleted, block on the in-flight lease or start one.
-    // Multiple concurrent requests racing here all join the *same* promise so
-    // only one Redis round trip happens.  After it resolves each waiter tries to
-    // claim exactly one token — if they lose the race they're rate-limited.
-    if (!state.inFlightLease) {
-      this.refill(projectId, state, limitRpm, nowMs, currentBucket).catch(() => {});
-    }
+    // Concurrent requests racing here all join the *same* promise so one Redis
+    // round trip serves them. A lease only grants `batchSize` tokens, so a
+    // burst larger than that leases again rather than rejecting requests the
+    // window still has room for: a request is refused only once Redis itself
+    // reports the window spent.
+    for (;;) {
+      if (!state.inFlightLease) {
+        this.refill(projectId, state, limitRpm, nowMs, currentBucket).catch(() => {});
+      }
+      const granted = state.inFlightLease ? await state.inFlightLease : 0;
 
-    if (state.inFlightLease) {
-      await state.inFlightLease;
+      // Only claim a token if the lease came back for the same bucket we waited on.
+      if (state.bucketIndex !== currentBucket) return false;
+      if (state.tokens > 0) {
+        state.tokens--;
+        return true;
+      }
+      if (granted <= 0) return false;
     }
-
-    // Only claim a token if the lease came back for the same bucket we waited on.
-    if (state.bucketIndex === currentBucket && state.tokens > 0) {
-      state.tokens--;
-      return true;
-    }
-
-    return false;
   }
 
   private async refill(

src/repositories/index.ts

@@ -22,6 +22,8 @@ import {
   adminUsers,
 } from "@/db/schema.js";
 
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
 // ─── Domain types ───────────────────────────────────────────────────────────
 
 export interface UserProfile {
@@ -310,6 +312,12 @@ export class UserRepository {
   ): Promise<void> {
     if (usersList.length === 0) return;
 
+    // ON CONFLICT DO UPDATE rejects the whole statement when two proposed rows
+    // hit the same key, so an import naming one user twice failed outright.
+    // The last occurrence wins, as it would had they been sent one by one.
+    const uniqueUsers = [...new Map(usersList.map((u) => [u.userId, u])).values()];
+    const winners = new Set(uniqueUsers);
+
     let attempts = 0;
     while (attempts < 3) {
       try {
@@ -317,7 +325,7 @@ export class UserRepository {
           await tx
             .insert(users)
             .values(
-              usersList.map((u) => ({
+              uniqueUsers.map((u) => ({
                 projectId,
                 externalId: u.userId,
                 attributes: {
@@ -381,7 +389,12 @@ export class UserRepository {
               }
             }
 
-            if (u.preferences?.quietHours && u.preferences.quietHours.length > 0) {
+            // Quiet hours are replaced, not merged, so only the winning row counts.
+            if (
+              winners.has(u) &&
+              u.preferences?.quietHours &&
+              u.preferences.quietHours.length > 0
+            ) {
               for (const window of u.preferences.quietHours) {
                 quietHoursInserts.push({
                   userId: internalId,
@@ -438,7 +451,7 @@ export class UserRepository {
               });
           }
 
-          const usersWithQuietHours = usersList.filter(
+          const usersWithQuietHours = uniqueUsers.filter(
             (u) => u.preferences?.quietHours !== undefined,
           );
           const internalIdsToClearQuietHours = usersWithQuietHours
@@ -485,7 +498,8 @@ export class UserRepository {
     const attrs = {
       language: patch.language ?? existing.language,
       timezone: patch.timezone ?? existing.timezone,
-      email: patch.email ?? existing.email,
+      // `null` clears the email; only an absent field keeps the old one.
+      email: patch.email !== undefined ? patch.email : existing.email,
     };
 
     await this.db.transaction(async (tx) => {
@@ -588,11 +602,14 @@ export class UserRepository {
   ): Promise<{ users: UserProfile[]; nextCursor: string | null }> {
     const conditions = [eq(users.projectId, projectId)];
 
-    if (cursor) {
-      const cursorDate = new Date(parseInt(cursor, 10));
-      if (!isNaN(cursorDate.getTime())) {
-        conditions.push(drizzleSql`${users.createdAt} < ${cursorDate.toISOString()}`);
-      }
+    // The cursor is the id of the last row returned. Paging on created_at
+    // alone skipped rows: a bulk import writes every user in one transaction,
+    // so they share one created_at, and the millisecond cursor could not tell
+    // them apart. (created_at, id) is unique and ordered.
+    if (cursor && UUID_RE.test(cursor)) {
+      conditions.push(
+        drizzleSql`(${users.createdAt}, ${users.id}) < (SELECT c.created_at, c.id FROM ${users} c WHERE c.id = ${cursor})`,
+      );
     }
 
     if (filters?.language) {
@@ -626,7 +643,7 @@ export class UserRepository {
       .select()
       .from(users)
       .where(and(...conditions))
-      .orderBy(desc(users.createdAt))
+      .orderBy(desc(users.createdAt), desc(users.id))
       .limit(limit);
 
     const items = rows.map((r) => {
@@ -640,8 +657,7 @@ export class UserRepository {
       };
     });
 
-    const nextCursor =
-      items.length === limit ? items[items.length - 1]!.createdAt.toString() : null;
+    const nextCursor = rows.length === limit ? rows[rows.length - 1]!.id : null;
     return { users: items, nextCursor };
   }
 
@@ -794,6 +810,25 @@ export class ContactRepository {
         ),
       );
 
+    // Per-address topic opt-outs (a user who wants marketing at home but not at
+    // work). Without them the pipeline cannot honour an opt-out it accepted.
+    const topicsByContact = new Map<string, Record<string, boolean>>();
+    if (rows.length > 0) {
+      const topicRows = await this.db
+        .select()
+        .from(contactTopicPreferences)
+        .where(
+          inArray(
+            contactTopicPreferences.contactId,
+            rows.map((r) => r.id),
+          ),
+        );
+      for (const t of topicRows) {
+        if (!topicsByContact.has(t.contactId)) topicsByContact.set(t.contactId, {});
+        topicsByContact.get(t.contactId)![t.topic] = t.enabled;
+      }
+    }
+
     const fetchedByUser = new Map<string, UserContact[]>();
     for (const id of missingIds) {
       fetchedByUser.set(id, []);
@@ -801,12 +836,13 @@ export class ContactRepository {
 
     for (const row of rows) {
       const contacts = fetchedByUser.get(row.userId) ?? [];
+      const topics = topicsByContact.get(row.id);
       contacts.push({
         id: row.id,
         userId: row.userId,
         channel: row.channel as ContactChannel,
         target: row.target,
-        preferences: {},
+        preferences: topics ? { topics } : {},
         active: row.enabled,
       });
       fetchedByUser.set(row.userId, contacts);
@@ -897,7 +933,29 @@ export class ContactRepository {
 
     const idMap = new Map(internalUserIdRows.map((r) => [r.externalId, r.id]));
 
-    const validContacts = contactsList.filter((c) => idMap.has(c.userId));
+    // The same address can arrive twice — as `email` and again in `contacts`
+    // — and ON CONFLICT DO UPDATE rejects a statement that touches one row
+    // twice. Keep one entry per address, merging their topic preferences.
+    const byAddress = new Map<string, (typeof contactsList)[number]>();
+    for (const c of contactsList) {
+      if (!idMap.has(c.userId)) continue;
+      const key = `${c.userId}\u0000${c.channel}\u0000${c.target}`;
+      const seen = byAddress.get(key);
+      byAddress.set(
+        key,
+        seen
+          ? {
+              ...c,
+              preferences: {
+                ...seen.preferences,
+                ...c.preferences,
+                topics: { ...seen.preferences?.topics, ...c.preferences?.topics },
+              },
+            }
+          : c,
+      );
+    }
+    const validContacts = [...byAddress.values()];
     if (validContacts.length === 0) return;
 
     const inserted = await this.db
@@ -1118,6 +1176,7 @@ export class ProjectRepository {
       await tx.delete(suppressions).where(eq(suppressions.projectId, id));
       await tx.delete(messageLogs).where(eq(messageLogs.projectId, id));
       await tx.delete(workflowInstances).where(eq(workflowInstances.projectId, id));
+      await tx.delete(workflowDefinitions).where(eq(workflowDefinitions.projectId, id));
 
       const result = await tx.delete(projects).where(eq(projects.id, id)).returning();
       return result.length > 0;
@@ -1234,7 +1293,7 @@ export class WorkflowRepository {
     return await this.db.transaction(async (tx) => {
       const result = await tx
         .update(workflowInstances)
-        .set({ status: "canceled" as any })
+        .set({ status: "canceled" })
         .where(
           and(
             eq(workflowInstances.id, instanceId),

src/services/ai/main.ts

@@ -1,4 +1,3 @@
-import { randomUUID } from "node:crypto";
 import { loadEnv, readBaseConfig } from "@/index.js";
 import type { Logger } from "@/index.js";
 import type { RedisClient, Redis } from "@/index.js";
@@ -31,6 +30,7 @@ import {
   createStreamConsumer,
   createWorkerRuntime,
   shutdownWorker,
+  uniqueConsumerId,
 } from "@/workers/bootstrap.js";
 
 // ─── Bootstrap ─────────────────────────────────────────────────────────────
@@ -128,11 +128,22 @@ export class AiWorker extends BaseWorker {
 
     const pending = payloadResult.data as NotificationAiPendingPayload;
 
-    // Idempotency
-    const idempotencyKey = `${pending.enrichedEventId}:${pending.recipientId}:${pending.channel}:ai`;
+    // Stable across replays: the engine's task id when it sent one. A fresh id
+    // per attempt made a replay after a crash a second, unrelated task that
+    // delivery's lease could not recognise — the recipient got it twice.
+    const taskId =
+      pending.taskId ?? `${pending.enrichedEventId}:${pending.recipientId}:${pending.channel}`;
+
+    // Idempotency. The lease has to outlast every generation this message may
+    // run, or a second worker takes it over mid-call and bills the prompts again.
+    const idempotencyKey = `${taskId}:ai`;
+    const aiConfig = getAiConfig();
+    const maxPrompts = aiConfig.maxPromptsPerNotification ?? AI_DEFAULTS.maxPromptsPerNotification;
+    const leaseSeconds =
+      Math.ceil(((aiConfig.timeoutMs ?? AI_DEFAULTS.timeoutMs) * maxPrompts) / 1000) + 30;
     const lease = this.idempotency.acquireLease
-      ? await this.idempotency.acquireLease(idempotencyKey, 30)
-      : (await this.idempotency.checkAndMark(idempotencyKey, 30))
+      ? await this.idempotency.acquireLease(idempotencyKey, leaseSeconds)
+      : (await this.idempotency.checkAndMark(idempotencyKey, leaseSeconds))
         ? "acquired"
         : "locked";
 
@@ -155,8 +166,6 @@ export class AiWorker extends BaseWorker {
       // Execute AI prompts. Each key is a separate billed model call, so the
       // count is capped rather than being driven by whatever the caller sent.
       const promptEntries = Object.entries(pending.aiPrompts);
-      const maxPrompts =
-        getAiConfig().maxPromptsPerNotification ?? AI_DEFAULTS.maxPromptsPerNotification;
       if (promptEntries.length > maxPrompts) {
         this.logger.warn(
           { messageId: message.id, requested: promptEntries.length, maxPrompts },
@@ -178,9 +187,9 @@ export class AiWorker extends BaseWorker {
 
       const rendered = renderWithTemplate(dbTemplate, finalVars);
 
-      const taskId = randomUUID();
       const destination =
-        pending.channel === "email"
+        pending.destination ??
+        (pending.channel === "email"
           ? pending.recipient.email
           : pending.channel === "sms" || pending.channel === "whatsapp"
             ? pending.recipient.phone
@@ -194,7 +203,7 @@ export class AiWorker extends BaseWorker {
                     ? pending.recipient.slack
                     : pending.channel === "push"
                       ? (pending.recipient.pushTokens?.[0] ?? pending.recipient.pushToken)
-                      : undefined;
+                      : undefined);
       const resolvedDestination =
         destination ?? (pending.channel === "push" ? undefined : pending.recipientId);
 
@@ -214,8 +223,11 @@ export class AiWorker extends BaseWorker {
         deliveryOptions: {
           maxAttempts: 3,
           timeoutMs: 10_000,
+          ...(pending.deliveryHeaders ? { headers: pending.deliveryHeaders } : {}),
         },
         fallbackChain: pending.fallbackChain,
+        campaignId: pending.campaignId,
+        workflowInstanceId: pending.workflowInstanceId,
       };
 
       const envelope = buildStreamEvent(
@@ -230,12 +242,15 @@ export class AiWorker extends BaseWorker {
       const scheduledAt = pending.scheduledAt ? new Date(pending.scheduledAt).getTime() : now;
 
       if (scheduledAt > now) {
-        await this.db.insert(scheduledPayloads).values({
-          taskId,
-          // The send time rides in the payload so the scheduler can re-queue
-          // this from Postgres if Redis loses its data.
-          payload: { ...taskPayload, scheduledAt: pending.scheduledAt },
-        });
+        await this.db
+          .insert(scheduledPayloads)
+          .values({
+            taskId,
+            // The send time rides in the payload so the scheduler can re-queue
+            // this from Postgres if Redis loses its data.
+            payload: { ...taskPayload, scheduledAt: pending.scheduledAt },
+          })
+          .onConflictDoNothing();
 
         const scheduledEnvelope = buildStreamEvent(
           "notification.scheduled",
@@ -305,7 +320,7 @@ export async function startAiWorker() {
     {
       stream: STREAMS.AI_PENDING,
       group: CONSUMER_GROUPS.AI,
-      consumer: `ai-${process.pid}`,
+      consumer: uniqueConsumerId("ai"),
       batchSize: config.WORKER_CONCURRENCY,
     },
   ));

src/services/api/handlers.ts

@@ -43,7 +43,7 @@ import { readJsonBody, sendJson, sendNoContent, sendValidationError } from "./ht
 import type { RouteContext } from "./router.js";
 import { globalEmitter, getPriorityBucket, normaliseTarget } from "@/shared/index.js";
 import { metrics } from "@/metrics/index.js";
-import { eq, desc, and, lt, lte, gte, sql, like, or, isNotNull, inArray } from "drizzle-orm";
+import { eq, desc, and, lte, gte, sql, like, or, isNotNull, inArray } from "drizzle-orm";
 import {
   messageLogs,
   workflowDefinitions,
@@ -177,6 +177,106 @@ async function persistUsers(deps: Deps, users: InlineUserLike[], projectId: stri
   });
 }
 
+/**
+ * A dead-letter entry, read the way StreamConsumer writes it: the original
+ * event JSON under `data`, carrying a `dlq` block that records which stream it
+ * failed on. Flat `payload`/`eventType` fields are accepted for entries written
+ * by hand or by older versions.
+ */
+interface DlqEntry {
+  id: string;
+  eventType: string;
+  payload: any;
+  projectId?: string;
+  error: string;
+  timestamp: string;
+  /** The event as it was published, without its `dlq` block. */
+  event: Record<string, any> | null;
+  originalStream?: string;
+  fields: Record<string, string>;
+}
+
+function parseDlqEntry(id: string, rawFields: string[]): DlqEntry {
+  const fields: Record<string, string> = {};
+  for (let i = 0; i < rawFields.length; i += 2) {
+    fields[rawFields[i]!] = rawFields[i + 1]!;
+  }
+
+  let parsed: Record<string, any> | null = null;
+  if (fields.data) {
+    try {
+      parsed = JSON.parse(fields.data);
+    } catch {
+      parsed = null;
+    }
+  }
+  const { dlq, ...event } = parsed ?? {};
+
+  let payload: any = fields;
+  if (parsed) {
+    payload = event.payload;
+  } else if (fields.payload) {
+    try {
+      payload = JSON.parse(fields.payload);
+    } catch {
+      payload = fields.payload;
+    }
+  }
+
+  const ms = Number(id.split("-")[0]);
+  return {
+    id,
+    eventType: event.type ?? fields.eventType ?? fields.event_type ?? "unknown",
+    payload,
+    projectId: payload?.projectId ?? payload?.project_id ?? fields.projectId,
+    error: dlq?.reason ?? fields.error ?? fields.reason ?? "Dead letter payload",
+    timestamp:
+      dlq?.ackedAt ??
+      fields.timestamp ??
+      (Number.isFinite(ms) ? new Date(ms).toISOString() : new Date().toISOString()),
+    event: parsed ? event : null,
+    originalStream: dlq?.originalStream,
+    fields,
+  };
+}
+
+/**
+ * Ids that live in uuid columns. Anything else can match no row, and passing it
+ * to Postgres fails the query outright — a 500 for what is simply not found.
+ */
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const isUuid = (v: string | undefined): v is string => typeof v === "string" && UUID_RE.test(v);
+
+/** Opaque position in the delivery log: a row's full-precision timestamp and id. */
+function encodeLogCursor(ts: string, id: string): string {
+  return Buffer.from(JSON.stringify([ts, id])).toString("base64url");
+}
+
+function decodeLogCursor(cursor: string | null | undefined): { ts: string; id: string } | null {
+  if (!cursor) return null;
+  // Cursors handed out before this format were a millisecond timestamp.
+  if (/^d+$/.test(cursor)) {
+    return {
+      ts: new Date(Number(cursor)).toISOString(),
+      id: "ffffffff-ffff-ffff-ffff-ffffffffffff",
+    };
+  }
+  try {
+    const [ts, id] = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
+    if (
+      typeof ts === "string" &&
+      typeof id === "string" &&
+      !Number.isNaN(Date.parse(ts)) &&
+      /^[0-9a-f-]{36}$/i.test(id)
+    ) {
+      return { ts, id };
+    }
+  } catch {
+    // fall through
+  }
+  return null;
+}
+
 function getQueryParam(ctx: RouteContext, key: string): string | undefined {
   if (!ctx.query) return undefined;
   if (typeof ctx.query.get === "function") {
@@ -662,34 +762,82 @@ export function createHandlers(deps: Deps) {
       );
     }
 
-    if (cursor) {
-      const cursorDate = new Date(parseInt(cursor, 10));
-      conditions.push(lt(messageLogs.timestamp, cursorDate));
-    }
+    // One entry per task, newest activity first. Rows are scanned in
+    // (timestamp, id) order — a batch of deliveries shares one timestamp, so
+    // the timestamp alone cannot mark a place — and a task is listed at its
+    // newest row. The cursor is that row for the first task of the next page,
+    // and any task with a row newer than the cursor was listed on an earlier
+    // page. Paging on a millisecond timestamp over `limit * 2` rows skipped
+    // tasks that shared a timestamp and dropped any that did not fit the slice.
+    const start = decodeLogCursor(cursor);
+    const tsText = sql<string>`${messageLogs.timestamp}::text`;
+    const position = (ts: string, id: string) => sql`(${ts}::timestamptz, ${id}::uuid)`;
+    const rowKey = sql`(${messageLogs.timestamp}, ${messageLogs.id})`;
 
-    const logs = await deps.db
-      .select()
-      .from(messageLogs)
-      .where(and(...conditions))
-      .orderBy(desc(messageLogs.timestamp))
-      .limit(limit * 2);
+    const order: string[] = [];
+    const rowsByTask = new Map<string, (typeof messageLogs.$inferSelect)[]>();
+    let nextCursor: string | null = null;
+    let scanFrom = start;
+    let inclusive = true;
+    const batchSize = Math.max(limit * 4, 50);
 
-    // Deduplicate by taskId preserving the latest status per taskId
-    const deduplicatedMap = new Map<string, (typeof logs)[0]>();
-    for (const log of logs) {
-      if (!deduplicatedMap.has(log.taskId)) {
-        deduplicatedMap.set(log.taskId, log);
-      } else {
-        const existing = deduplicatedMap.get(log.taskId)!;
-        if (existing.status === "dispatched" && log.status !== "dispatched") {
-          deduplicatedMap.set(log.taskId, log);
-        }
+    for (let round = 0; round < 50 && nextCursor === null; round++) {
+      const batch = await deps.db
+        .select({ row: messageLogs, ts: tsText })
+        .from(messageLogs)
+        .where(
+          and(
+            ...conditions,
+            scanFrom
+              ? sql`${rowKey} ${inclusive ? sql`<=` : sql`<`} ${position(scanFrom.ts, scanFrom.id)}`
+              : undefined,
+          ),
+        )
+        .orderBy(desc(messageLogs.timestamp), desc(messageLogs.id))
+        .limit(batchSize);
+
+      const fresh = [...new Set(batch.map((b) => b.row.taskId))].filter((t) => !rowsByTask.has(t));
+      const listedEarlier = new Set<string>();
+      if (start && fresh.length > 0) {
+        const newer = await deps.db
+          .selectDistinct({ taskId: messageLogs.taskId })
+          .from(messageLogs)
+          .where(
+            and(
+              ...conditions,
+              inArray(messageLogs.taskId, fresh),
+              sql`${rowKey} > ${position(start.ts, start.id)}`,
+            ),
+          );
+        for (const r of newer) listedEarlier.add(r.taskId);
       }
-    }
-    const deduplicatedLogs = Array.from(deduplicatedMap.values()).slice(0, limit);
 
-    const nextCursor =
-      logs.length === limit * 2 ? logs[logs.length - 1]!.timestamp.getTime().toString() : null;
+      for (const { row, ts } of batch) {
+        if (listedEarlier.has(row.taskId)) continue;
+        let rows = rowsByTask.get(row.taskId);
+        if (!rows) {
+          if (order.length === limit) {
+            nextCursor = encodeLogCursor(ts, row.id);
+            break;
+          }
+          rows = [];
+          order.push(row.taskId);
+          rowsByTask.set(row.taskId, rows);
+        }
+        rows.push(row);
+      }
+
+      if (batch.length < batchSize) break;
+      const last = batch[batch.length - 1]!;
+      scanFrom = { ts: last.ts, id: last.row.id };
+      inclusive = false;
+    }
+
+    // The latest outcome, rather than the dispatch that preceded it.
+    const deduplicatedLogs = order.map((taskId) => {
+      const rows = rowsByTask.get(taskId)!;
+      return rows.find((r) => r.status !== "dispatched") ?? rows[0]!;
+    });
 
     sendJson(res, 200, { logs: deduplicatedLogs, nextCursor });
   }
@@ -875,7 +1023,9 @@ export function createHandlers(deps: Deps) {
     ctx: RouteContext,
   ): Promise<void> {
     const instanceId = ctx.params.id!;
-    const workflow = await deps.workflowRepo.getInstance(ctx.projectId!, instanceId);
+    const workflow = isUuid(instanceId)
+      ? await deps.workflowRepo.getInstance(ctx.projectId!, instanceId)
+      : null;
     if (!workflow) return sendJson(res, 404, { error: "workflow_not_found", id: instanceId });
 
     sendJson(res, 200, workflow);
@@ -888,7 +1038,8 @@ export function createHandlers(deps: Deps) {
     ctx: RouteContext,
   ): Promise<void> {
     const instanceId = ctx.params.id!;
-    const canceled = await deps.workflowRepo.cancelInstance(ctx.projectId!, instanceId);
+    const canceled =
+      isUuid(instanceId) && (await deps.workflowRepo.cancelInstance(ctx.projectId!, instanceId));
     if (!canceled) return sendJson(res, 400, { error: "workflow_not_cancelable", id: instanceId });
 
     logger.info({ instanceId }, "workflow canceled");
@@ -956,7 +1107,7 @@ export function createHandlers(deps: Deps) {
     ctx: RouteContext,
   ): Promise<void> {
     const id = ctx.params.id!;
-    const deleted = await deps.projectRepo.delete(id);
+    const deleted = isUuid(id) && (await deps.projectRepo.delete(id));
     if (!deleted) return sendJson(res, 404, { error: "project_not_found" });
 
     sendNoContent(res);
@@ -979,7 +1130,7 @@ export function createHandlers(deps: Deps) {
 
     if (!parsed.success) return sendValidationError(res, parsed.error);
 
-    const updated = await deps.projectRepo.updateSettings(id, parsed.data);
+    const updated = isUuid(id) && (await deps.projectRepo.updateSettings(id, parsed.data));
     if (!updated) return sendJson(res, 404, { error: "project_not_found" });
 
     // Drop the engine's cached throttle overrides so the change applies to the
@@ -1009,6 +1160,11 @@ export function createHandlers(deps: Deps) {
       .safeParse(await readJsonBody(req));
     const role = parsed.success ? parsed.data.role : "admin";
 
+    // A key for a project that does not exist would fail the foreign key.
+    if (!isUuid(id) || !(await deps.projectRepo.findThrottleSettings(id))) {
+      return sendJson(res, 404, { error: "project_not_found" });
+    }
+
     const apiKey = `nk_live_${randomBytes(32).toString("hex")}`;
     const apiKeyHash = createHash("sha256").update(apiKey).digest("hex");
 
@@ -1023,6 +1179,7 @@ export function createHandlers(deps: Deps) {
     ctx: RouteContext,
   ): Promise<void> {
     const id = ctx.params.id!;
+    if (!isUuid(id)) return sendJson(res, 404, { error: "project_not_found" });
     const keys = await deps.projectRepo.listApiKeys(id);
     sendJson(res, 200, { keys });
   }
@@ -1034,7 +1191,7 @@ export function createHandlers(deps: Deps) {
     ctx: RouteContext,
   ): Promise<void> {
     const { id, keyId } = ctx.params;
-    const deleted = await deps.projectRepo.deleteApiKey(id!, keyId!);
+    const deleted = isUuid(id) && isUuid(keyId) && (await deps.projectRepo.deleteApiKey(id, keyId));
     if (!deleted) return sendJson(res, 404, { error: "key_not_found" });
 
     await deps.redis.native.publish("apikey.invalidated", "*");
@@ -1234,41 +1391,40 @@ export function createHandlers(deps: Deps) {
         "COUNT",
         "100",
       );
-      const allMessages = (rawEntries || []).map(([id, fields]: [string, string[]]) => {
-        const fieldMap: Record<string, string> = {};
-        for (let i = 0; i < fields.length; i += 2) {
-          fieldMap[fields[i]!] = fields[i + 1]!;
-        }
-        let payload: any = fieldMap;
-        if (fieldMap.payload) {
-          try {
-            payload = JSON.parse(fieldMap.payload);
-          } catch {
-            payload = fieldMap.payload;
-          }
-        }
-        return {
-          id,
-          eventType: fieldMap.eventType || fieldMap.event_type || "unknown",
-          payload,
-          error: fieldMap.error || fieldMap.reason || "Dead letter payload",
-          timestamp: fieldMap.timestamp || new Date().toISOString(),
-        };
-      });
+      const allMessages = (rawEntries || []).map(([id, fields]: [string, string[]]) =>
+        parseDlqEntry(id, fields),
+      );
 
       const messages = ctx.projectId
-        ? allMessages.filter((m) => {
-            const pId = m.payload?.projectId ?? m.payload?.project_id ?? (m as any).projectId;
-            return pId === ctx.projectId;
-          })
+        ? allMessages.filter((m) => m.projectId === ctx.projectId)
         : allMessages;
 
-      sendJson(res, 200, { messages: messages.slice(0, 50) });
+      sendJson(res, 200, {
+        messages: messages.slice(0, 50).map((m) => ({
+          id: m.id,
+          eventType: m.eventType,
+          payload: m.payload,
+          error: m.error,
+          timestamp: m.timestamp,
+          originalStream: m.originalStream,
+        })),
+      });
     } catch {
       sendJson(res, 200, { messages: [] });
     }
   }
 
+  /** The entry, or null when it does not exist or belongs to another project. */
+  async function findDlqEntry(ctx: RouteContext, messageId: string): Promise<DlqEntry | null> {
+    // Redis rejects a malformed stream id rather than matching nothing.
+    if (!/^\d+-\d+$/.test(messageId)) return null;
+    const rawEntries = await deps.redis.native.xrange(STREAMS.DEAD_LETTER, messageId, messageId);
+    if (!rawEntries || rawEntries.length === 0 || !rawEntries[0]) return null;
+    const entry = parseDlqEntry(rawEntries[0][0], rawEntries[0][1]);
+    if (ctx.projectId && entry.projectId !== ctx.projectId) return null;
+    return entry;
+  }
+
   // ── POST /v1/dlq/replay — replayDLQMessage ────────────────────────────────
   async function replayDLQMessage(
     req: IncomingMessage,
@@ -1284,46 +1440,35 @@ export function createHandlers(deps: Deps) {
     }
 
     try {
-      const rawEntries = await deps.redis.native.xrange(STREAMS.DEAD_LETTER, messageId, messageId);
-      if (!rawEntries || rawEntries.length === 0 || !rawEntries[0]) {
+      const entry = await findDlqEntry(ctx, messageId);
+      if (!entry) {
         sendJson(res, 404, { error: "dlq_message_not_found" });
         return;
       }
 
-      const fields = rawEntries[0][1];
-      const fieldMap: Record<string, string> = {};
-      for (let i = 0; i < fields.length; i += 2) {
-        fieldMap[fields[i]!] = fields[i + 1]!;
-      }
-
-      let payload: any = null;
-      if (fieldMap.payload) {
-        try {
-          payload = JSON.parse(fieldMap.payload);
-        } catch {}
-      }
-
-      if (ctx.projectId) {
-        const pId = payload?.projectId ?? payload?.project_id ?? (fieldMap as any).projectId;
-        if (pId !== ctx.projectId) {
-          sendJson(res, 404, { error: "dlq_message_not_found" });
-          return;
-        }
-      }
-
-      const priority = fieldMap.priority || "normal";
-      const p = getPriorityBucket(priority);
+      // Back onto the stream it failed on: a failed send belongs with delivery,
+      // not at the start of the pipeline, where its payload does not parse.
+      const replayable = new Set<string>(Object.values(STREAMS));
+      replayable.delete(STREAMS.DEAD_LETTER);
+      const inbound =
+        STREAMS[
+          `INBOUND_${getPriorityBucket(entry.payload?.priority ?? entry.fields.priority).toUpperCase()}` as keyof typeof STREAMS
+        ] ?? STREAMS.INBOUND_NORMAL;
       const targetStream =
-        STREAMS[`INBOUND_${p.toUpperCase()}` as keyof typeof STREAMS] || STREAMS.INBOUND_NORMAL;
+        entry.originalStream && replayable.has(entry.originalStream)
+          ? entry.originalStream
+          : inbound;
 
       const xaddArgs: string[] = [];
-      for (const [k, v] of Object.entries(fieldMap)) {
-        xaddArgs.push(k, v);
+      if (entry.event) {
+        xaddArgs.push("data", JSON.stringify(entry.event));
+      } else {
+        for (const [k, v] of Object.entries(entry.fields)) xaddArgs.push(k, v);
       }
       await deps.redis.native.xadd(targetStream, "*", ...xaddArgs);
       await deps.redis.native.xdel(STREAMS.DEAD_LETTER, messageId);
 
-      sendJson(res, 200, { success: true, replayedId: messageId });
+      sendJson(res, 200, { success: true, replayedId: messageId, stream: targetStream });
     } catch (err: any) {
       sendJson(res, 500, { error: "replay_failed", message: err.message });
     }
@@ -1339,34 +1484,10 @@ export function createHandlers(deps: Deps) {
     if (!messageId) return sendJson(res, 400, { error: "missing_id" });
 
     try {
-      if (ctx.projectId) {
-        const rawEntries = await deps.redis.native.xrange(
-          STREAMS.DEAD_LETTER,
-          messageId,
-          messageId,
-        );
-        if (!rawEntries || rawEntries.length === 0 || !rawEntries[0]) {
-          sendJson(res, 404, { error: "dlq_message_not_found" });
-          return;
-        }
-        const fields = rawEntries[0][1];
-        const fieldMap: Record<string, string> = {};
-        for (let i = 0; i < fields.length; i += 2) {
-          fieldMap[fields[i]!] = fields[i + 1]!;
-        }
-        let payload: any = null;
-        if (fieldMap.payload) {
-          try {
-            payload = JSON.parse(fieldMap.payload);
-          } catch {}
-        }
-        const pId = payload?.projectId ?? payload?.project_id ?? (fieldMap as any).projectId;
-        if (pId !== ctx.projectId) {
-          sendJson(res, 404, { error: "dlq_message_not_found" });
-          return;
-        }
+      if (!(await findDlqEntry(ctx, messageId))) {
+        sendJson(res, 404, { error: "dlq_message_not_found" });
+        return;
       }
-
       await deps.redis.native.xdel(STREAMS.DEAD_LETTER, messageId);
       sendJson(res, 200, { success: true });
     } catch (err: any) {
@@ -1766,10 +1887,13 @@ code{background:#f4f4f5;padding:.1rem .35rem;border-radius:4px}</style>
 
       if (ENGAGEMENT.includes(row.kind as Engagement)) {
         bucket[row.kind as Engagement] += n;
-      } else {
-        // A delivery row. `sent` is every task that reached a provider at all,
-        // which is the denominator a bounce rate is measured against.
+      } else if (row.kind === "dispatched") {
+        // `sent` is every task that reached a provider at all, which is the
+        // denominator a bounce rate is measured against. Each attempt logs a
+        // dispatched row besides its outcome row, and the two are grouped
+        // separately, so counting both counted every message twice.
         bucket.sent += n;
+      } else {
         if (row.status === "delivered") bucket.delivered += n;
         if (row.status === "failed") bucket.failed += n;
       }

src/services/delivery/main.ts

@@ -15,6 +15,7 @@ import {
   registry,
   buildStreamEvent,
   type NotificationDispatchedPayload,
+  type DispatchedTaskPayload,
   type RenderedContent,
   PUBSUB_CHANNELS,
 } from "@/index.js";
@@ -73,6 +74,37 @@ let subscriber: Redis | null = null;
 /** How long a send may run before its dispatched log entry is published alone. */
 const DISPATCH_LOG_DELAY_MS = 1_000;
 
+/**
+ * Wait before re-trying a send that failed at the provider, by prior failures.
+ * The first step matches the circuit breaker's reset timeout, so a retry of a
+ * task the open breaker turned away reaches the provider's recovery probe
+ * rather than the same closed door.
+ */
+const RETRY_BACKOFF_MS = [30_000, 120_000, 600_000];
+
+/**
+ * Names the provider at `transports[i]` for its rate limit and circuit breaker.
+ * The rate limit's budget lives in Redis and is shared by every delivery
+ * process, so the name must come out the same everywhere: the class name,
+ * numbered when a channel has two providers of one class.
+ */
+function providerKey(channel: string, transports: any[], i: number): string {
+  const name = transports[i].constructor?.name ?? "provider";
+  const nth = transports.slice(0, i).filter((t) => t.constructor?.name === name).length;
+  return nth === 0 ? `${channel}:${name}` : `${channel}:${name}#${nth + 1}`;
+}
+
+/** A provider's refusal, keeping whether it said another attempt could work. */
+function providerFailure(res: { error?: string; retryable?: boolean }): Error {
+  const err = new Error(res.error ?? "Transport failed");
+  (err as any).retryable = res.retryable;
+  return err;
+}
+
+export function retryDelayMs(priorFailures: number): number {
+  return RETRY_BACKOFF_MS[Math.min(priorFailures, RETRY_BACKOFF_MS.length - 1)]!;
+}
+
 export interface DeliveryWorkerOptions extends WorkerOptions {
   transportRegistry: any;
   idempotency: any;
@@ -196,6 +228,51 @@ export class DeliveryWorker extends BaseWorker {
     return renderWithTemplate(template, task.templateVariables);
   }
 
+  /**
+   * Parks a task in Postgres and the scheduler until `sendAt`. The payload
+   * carries the send time, so the scheduler can rebuild it if Redis loses its
+   * data.
+   */
+  private async reschedule(
+    task: DispatchedTaskPayload,
+    sendAt: Date,
+    reason: string,
+  ): Promise<void> {
+    const scheduledAt = sendAt.toISOString();
+    await this.db
+      .insert(scheduledPayloads)
+      .values({ taskId: task.taskId, payload: { ...task, scheduledAt } })
+      .onConflictDoUpdate({
+        target: scheduledPayloads.taskId,
+        set: { payload: drizzleSql`EXCLUDED.payload` },
+      });
+
+    await this.scheduledProducer.publish(
+      buildStreamEvent(
+        "notification.scheduled",
+        {
+          projectId: task.projectId,
+          enrichedEventId: task.enrichedEventId,
+          taskId: task.taskId,
+          scheduledAt,
+        },
+        "delivery",
+        `${task.taskId}:${reason}:${Date.now()}`,
+      ),
+    );
+  }
+
+  /** Takes a slot from the provider's rate limit, if it has one. */
+  private async providerGate(
+    channel: string,
+    transports: any[],
+    i: number,
+  ): Promise<{ allowed: boolean; retryAfterMs: number }> {
+    const limits = transports[i].limits;
+    if (!limits) return { allowed: true, retryAfterMs: 0 };
+    return this.providerThrottle.check(providerKey(channel, transports, i), limits);
+  }
+
   private getBreaker(name: string): CircuitBreaker {
     let breaker = this.breakers.get(name);
     if (!breaker) {
@@ -219,6 +296,11 @@ export class DeliveryWorker extends BaseWorker {
     }
 
     const task = payloadResult.data as NotificationDispatchedPayload;
+    // Attempts made on earlier trips through the scheduler, plus this one.
+    const attemptNo = (task.deliveryAttemptCount ?? 0) + attempt;
+    // What gets parked for a retry: the task as it arrived, before rendering
+    // and the transport's abort signal were attached to it.
+    const original: DispatchedTaskPayload = { ...task };
 
     const fallbackToNextChannel = async (reason: string) => {
       if (task.fallbackChain && task.fallbackChain.length > 0 && task.recipient) {
@@ -238,6 +320,9 @@ export class DeliveryWorker extends BaseWorker {
           recipient: task.recipient,
           scheduledAt: undefined,
           fallbackChain: remainingChain.length > 0 ? remainingChain : undefined,
+          campaignId: task.campaignId,
+          workflowInstanceId: task.workflowInstanceId,
+          throttleCounted: true,
         };
 
         const p = getPriorityBucket(task.priority);
@@ -288,10 +373,9 @@ export class DeliveryWorker extends BaseWorker {
               failureReason: "no transport registered for channel",
               failureCode: "no_transport",
               retryable: false,
-              attempt,
+              attempt: attemptNo,
               templateId: task.templateId,
-              workflowInstanceId:
-                event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
+              workflowInstanceId: task.workflowInstanceId,
               campaignId: task.campaignId,
             },
             "delivery",
@@ -302,87 +386,67 @@ export class DeliveryWorker extends BaseWorker {
       return;
     }
 
-    const limitConfig = transports[0].limits;
-    if (limitConfig) {
-      const { allowed, retryAfterMs } = await this.providerThrottle.check(
-        task.channel,
-        limitConfig,
-      );
-      if (!allowed) {
-        task.throttleAttemptCount = (task.throttleAttemptCount ?? 0) + 1;
-        const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3;
+    // Each provider has its own budget: start at the first with room left. The
+    // channel is out of capacity only once every provider for it is.
+    let startAt = -1;
+    let retryAfterMs = Infinity;
+    for (let i = 0; i < transports.length; i++) {
+      const gate = await this.providerGate(task.channel, transports, i);
+      if (gate.allowed) {
+        startAt = i;
+        break;
+      }
+      retryAfterMs = Math.min(retryAfterMs, gate.retryAfterMs);
+    }
+    if (startAt === -1) {
+      task.throttleAttemptCount = (task.throttleAttemptCount ?? 0) + 1;
+      const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3;
 
-        if (task.throttleAttemptCount > maxAttempts) {
-          this.logger.warn(
-            { messageId: message.id, taskId: task.taskId, attempts: task.throttleAttemptCount },
-            "provider rate limit max attempts exceeded",
-          );
-          const fallbackTriggered = await fallbackToNextChannel("provider_throttle_exceeded");
-          if (!fallbackTriggered) {
-            this.globalEmitter.emit(
-              "delivery:failed",
-              task.taskId,
-              "provider throttle exceeded",
-              task.channel,
-              task.projectId,
-            );
-            await this.eventProcessor.add(
-              buildStreamEvent(
-                "notification.failed",
-                {
-                  projectId: task.projectId,
-                  taskId: task.taskId,
-                  enrichedEventId: task.enrichedEventId,
-                  recipientId: task.recipientId,
-                  channel: task.channel,
-                  failureReason: "provider rate limit max attempts exceeded",
-                  failureCode: "provider_throttle_exceeded",
-                  retryable: false,
-                  attempt,
-                  templateId: task.templateId,
-                  workflowInstanceId:
-                    event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
-                  campaignId: task.campaignId,
-                },
-                "delivery",
-                event.metadata.traceId,
-              ),
-            );
-          }
-          return;
-        }
-
-        // The retry time rides in the payload so the scheduler can re-queue
-        // this from Postgres if Redis loses its data.
-        const retryAt = new Date(Date.now() + retryAfterMs).toISOString();
-        await this.db
-          .insert(scheduledPayloads)
-          .values({
-            taskId: task.taskId,
-            payload: { ...task, scheduledAt: retryAt },
-          })
-          .onConflictDoUpdate({
-            target: scheduledPayloads.taskId,
-            set: { payload: drizzleSql`EXCLUDED.payload` },
-          });
-
-        await this.scheduledProducer.publish(
-          buildStreamEvent(
-            "notification.scheduled",
-            {
-              projectId: task.projectId,
-              enrichedEventId: task.enrichedEventId,
-              taskId: task.taskId,
-              scheduledAt: retryAt,
-              throttleAttemptCount: task.throttleAttemptCount,
-            },
-            "delivery",
-            `${task.taskId}:throttle:${Date.now()}`,
-          ),
+      if (task.throttleAttemptCount > maxAttempts) {
+        this.logger.warn(
+          { messageId: message.id, taskId: task.taskId, attempts: task.throttleAttemptCount },
+          "provider rate limit max attempts exceeded",
         );
-
+        const fallbackTriggered = await fallbackToNextChannel("provider_throttle_exceeded");
+        if (!fallbackTriggered) {
+          this.globalEmitter.emit(
+            "delivery:failed",
+            task.taskId,
+            "provider throttle exceeded",
+            task.channel,
+            task.projectId,
+          );
+          await this.eventProcessor.add(
+            buildStreamEvent(
+              "notification.failed",
+              {
+                projectId: task.projectId,
+                taskId: task.taskId,
+                enrichedEventId: task.enrichedEventId,
+                recipientId: task.recipientId,
+                channel: task.channel,
+                failureReason: "provider rate limit max attempts exceeded",
+                failureCode: "provider_throttle_exceeded",
+                retryable: false,
+                attempt: attemptNo,
+                templateId: task.templateId,
+                workflowInstanceId: task.workflowInstanceId,
+                campaignId: task.campaignId,
+              },
+              "delivery",
+              event.metadata.traceId,
+            ),
+          );
+        }
         return;
       }
+
+      await this.reschedule(
+        { ...original, throttleAttemptCount: task.throttleAttemptCount },
+        new Date(Date.now() + retryAfterMs),
+        "throttle",
+      );
+      return;
     }
 
     const idempotencyKey = task.taskId;
@@ -433,9 +497,8 @@ export class DeliveryWorker extends BaseWorker {
             recipientId: task.recipientId,
             channel: task.channel,
             templateId: task.templateId,
-            attempt,
-            workflowInstanceId:
-              event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
+            attempt: attemptNo,
+            workflowInstanceId: task.workflowInstanceId,
             campaignId: task.campaignId,
           },
           "delivery",
@@ -450,7 +513,40 @@ export class DeliveryWorker extends BaseWorker {
       clearTimeout(dispatchLogTimer);
       if (dispatchLogged) return payload;
       dispatchLogged = true;
-      return { ...payload, attempt, dispatchedAt };
+      return { ...payload, attempt: attemptNo, dispatchedAt };
+    };
+
+    let retryScheduled = false;
+    /**
+     * Re-queues a send the provider turned away, unless it said the failure is
+     * permanent or the task is out of attempts. An outage, a timeout or an open
+     * circuit is exactly what a later attempt can get past; failing the task
+     * outright turned every blip into lost mail. The attempt stays in the log as
+     * dispatched, and the task's outcome is whatever its last attempt comes to.
+     */
+    const retryLater = async (failure: { retryable?: boolean; error?: string }) => {
+      const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3;
+      if (failure.retryable === false || attemptNo >= maxAttempts) return false;
+      const delayMs = retryDelayMs(task.deliveryAttemptCount ?? 0);
+      await this.reschedule(
+        { ...original, deliveryAttemptCount: attemptNo },
+        new Date(Date.now() + delayMs),
+        "retry",
+      );
+      retryScheduled = true;
+      metrics.deliveryFailed.inc({ channel: task.channel, reason: "retry_scheduled" });
+      this.logger.warn(
+        {
+          taskId: task.taskId,
+          channel: task.channel,
+          attempt: attemptNo,
+          maxAttempts,
+          delayMs,
+          error: failure.error,
+        },
+        "provider send failed — retrying later",
+      );
+      return true;
     };
 
     try {
@@ -469,7 +565,13 @@ export class DeliveryWorker extends BaseWorker {
       if (task.channel === "push") {
         let lastResult: any = { success: false, error: "No transports" };
 
-        for (const transport of transports) {
+        for (let i = startAt; i < transports.length; i++) {
+          const transport = transports[i];
+          // The starting provider's slot was taken above. Failing over to a
+          // later one spends that provider's own budget, or skips it if spent.
+          if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) {
+            continue;
+          }
           const elapsedMs = Date.now() - tDispatchStart;
           const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
           if (remainingBudgetMs <= 0) {
@@ -484,7 +586,7 @@ export class DeliveryWorker extends BaseWorker {
 
           try {
             const tProv = Date.now();
-            const breaker = this.getBreaker(`${task.channel}:${transport.constructor.name}`);
+            const breaker = this.getBreaker(providerKey(task.channel, transports, i));
 
             lastResult = await breaker.execute(async () => {
               const controller = new AbortController();
@@ -504,7 +606,7 @@ export class DeliveryWorker extends BaseWorker {
                   }),
                 ]);
                 if (!res.success && !res.invalidToken) {
-                  throw new Error(res.error ?? "Transport failed");
+                  throw providerFailure(res);
                 }
                 return res;
               } finally {
@@ -515,7 +617,7 @@ export class DeliveryWorker extends BaseWorker {
             (global as any)._telemetry.provider += Date.now() - tProv;
             if (lastResult.success || lastResult.invalidToken) break;
           } catch (err: any) {
-            lastResult = { success: false, error: err.message };
+            lastResult = { success: false, error: err.message, retryable: err.retryable };
           }
         }
 
@@ -560,10 +662,9 @@ export class DeliveryWorker extends BaseWorker {
                   failureReason: "invalid_token",
                   failureCode: "push_failure",
                   retryable: false,
-                  attempt,
+                  attempt: attemptNo,
                   templateId: task.templateId,
-                  workflowInstanceId:
-                    event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
+                  workflowInstanceId: task.workflowInstanceId,
                   campaignId: task.campaignId,
                 }),
                 "delivery",
@@ -614,8 +715,7 @@ export class DeliveryWorker extends BaseWorker {
                   deliveredAt: new Date().toISOString(),
                   providerMessageId,
                   templateId: task.templateId,
-                  workflowInstanceId:
-                    event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
+                  workflowInstanceId: task.workflowInstanceId,
                   campaignId: task.campaignId,
                 }),
                 "delivery",
@@ -623,7 +723,7 @@ export class DeliveryWorker extends BaseWorker {
               ),
             ),
           );
-        } else {
+        } else if (!(await retryLater(lastResult))) {
           this.logger.warn(
             { taskId: task.taskId, error: lastResult.error },
             "push delivery failed",
@@ -659,10 +759,9 @@ export class DeliveryWorker extends BaseWorker {
                   failureReason: lastResult.error ?? "push delivery failed",
                   failureCode: "push_failure",
                   retryable: false,
-                  attempt,
+                  attempt: attemptNo,
                   templateId: task.templateId,
-                  workflowInstanceId:
-                    event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
+                  workflowInstanceId: task.workflowInstanceId,
                   campaignId: task.campaignId,
                 }),
                 "delivery",
@@ -679,7 +778,13 @@ export class DeliveryWorker extends BaseWorker {
       } else {
         // Other channels (email, sms, webhook, whatsapp) use the pre-resolved destination
         let result: any = { success: false, error: "No transports" };
-        for (const transport of transports) {
+        for (let i = startAt; i < transports.length; i++) {
+          const transport = transports[i];
+          // The starting provider's slot was taken above. Failing over to a
+          // later one spends that provider's own budget, or skips it if spent.
+          if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) {
+            continue;
+          }
           const elapsedMs = Date.now() - tDispatchStart;
           const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
           if (remainingBudgetMs <= 0) {
@@ -694,7 +799,7 @@ export class DeliveryWorker extends BaseWorker {
 
           try {
             const tProv = Date.now();
-            const breaker = this.getBreaker(`${task.channel}:${transport.constructor.name}`);
+            const breaker = this.getBreaker(providerKey(task.channel, transports, i));
 
             result = await breaker.execute(async () => {
               const controller = new AbortController();
@@ -714,7 +819,7 @@ export class DeliveryWorker extends BaseWorker {
                   }),
                 ]);
                 if (!res.success) {
-                  throw new Error(res.error ?? "Transport failed");
+                  throw providerFailure(res);
                 }
                 return res;
               } finally {
@@ -725,7 +830,7 @@ export class DeliveryWorker extends BaseWorker {
             (global as any)._telemetry.provider += Date.now() - tProv;
             if (result.success) break;
           } catch (err: any) {
-            result = { success: false, error: err.message };
+            result = { success: false, error: err.message, retryable: err.retryable };
           }
         }
 
@@ -763,8 +868,7 @@ export class DeliveryWorker extends BaseWorker {
                   deliveredAt: new Date().toISOString(),
                   providerMessageId,
                   templateId: task.templateId,
-                  workflowInstanceId:
-                    event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
+                  workflowInstanceId: task.workflowInstanceId,
                   campaignId: task.campaignId,
                 }),
                 "delivery",
@@ -772,7 +876,7 @@ export class DeliveryWorker extends BaseWorker {
               ),
             ),
           );
-        } else {
+        } else if (!(await retryLater(result))) {
           publishPromises.push(
             this.outboxUpdateProcessor.add({
               taskId: task.taskId,
@@ -794,10 +898,9 @@ export class DeliveryWorker extends BaseWorker {
                   failureReason: result.error ?? "delivery failed completely",
                   failureCode: "provider_error",
                   retryable: false,
-                  attempt,
+                  attempt: attemptNo,
                   templateId: task.templateId,
-                  workflowInstanceId:
-                    event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
+                  workflowInstanceId: task.workflowInstanceId,
                   campaignId: task.campaignId,
                 }),
                 "delivery",
@@ -807,7 +910,7 @@ export class DeliveryWorker extends BaseWorker {
           );
         }
 
-        if (!result.success) {
+        if (!result.success && !retryScheduled) {
           this.logger.warn(
             { taskId: task.taskId, channel: task.channel, error: result.error },
             "delivery failed completely across providers",
@@ -825,7 +928,7 @@ export class DeliveryWorker extends BaseWorker {
           if (!fallbackTriggered) {
             throw new NonRetryableError(result.error ?? "delivery failed");
           }
-        } else {
+        } else if (result.success) {
           this.logger.debug(
             { taskId: task.taskId, channel: task.channel, messageId: result.providerMessageId },
             "notification delivered",
@@ -845,9 +948,14 @@ export class DeliveryWorker extends BaseWorker {
       });
       const flushTime = Date.now() - tFlushStart;
 
-      void this.idempotency.markProcessed(idempotencyKey).catch((err: any) => {
-        this.logger.warn({ err, taskId: task.taskId }, "failed to upgrade idempotency TTL");
-      });
+      if (retryScheduled) {
+        // Released so the re-queued attempt can take the lease.
+        await this.idempotency.unmark(idempotencyKey).catch(() => {});
+      } else {
+        void this.idempotency.markProcessed(idempotencyKey).catch((err: any) => {
+          this.logger.warn({ err, taskId: task.taskId }, "failed to upgrade idempotency TTL");
+        });
+      }
 
       const t = ((global as any)._telemetry = (global as any)._telemetry || {
         count: 0,

src/services/engine/main.ts

@@ -414,7 +414,13 @@ export class EngineWorker extends BaseWorker {
       // Quiet hours check
       const qh = enriched.recipient.preferences.quietHours;
       if (qh && qh.length > 0 && enriched.priority !== "critical") {
-        const qhResult = isInQuietHours(enriched.recipient.timezone, qh);
+        // Judged at the moment the message would go out, not the moment the
+        // engine sees it: a send booked for next week must not be pulled
+        // forward to tonight's quiet-hours end, and one booked for 3am must
+        // wait until the quiet hours are over.
+        const requestedAt = enriched.scheduledAt ? new Date(enriched.scheduledAt) : new Date();
+        const sendAt = requestedAt.getTime() > Date.now() ? requestedAt : new Date();
+        const qhResult = isInQuietHours(enriched.recipient.timezone, qh, sendAt);
         if (qhResult.inQuietHours && qhResult.nextActiveTime) {
           this.logger.info(
             {
@@ -444,18 +450,16 @@ export class EngineWorker extends BaseWorker {
         );
       }
 
-      const throttleResult = await this.throttle.check(
-        enriched.projectId,
-        enriched.recipientId,
-        enriched.priority,
-        {
-          limit: projectThrottle.throttleLimit,
-          windowHours: projectThrottle.throttleWindowHours,
-          scheduledAt: enriched.scheduledAt,
-          // A replayed message must not be counted against the user twice.
-          messageId: idempotencyKey,
-        },
-      );
+      // A channel fallback was counted when the notification was first sent.
+      const throttleResult = enriched.throttleCounted
+        ? { allowed: true, count: 0, limit: 0 }
+        : await this.throttle.check(enriched.projectId, enriched.recipientId, enriched.priority, {
+            limit: projectThrottle.throttleLimit,
+            windowHours: projectThrottle.throttleWindowHours,
+            scheduledAt: enriched.scheduledAt,
+            // A replayed message must not be counted against the user twice.
+            messageId: idempotencyKey,
+          });
       if (!throttleResult.allowed) {
         this.logger.info(
           {
@@ -476,8 +480,7 @@ export class EngineWorker extends BaseWorker {
         return;
       }
 
-      // Gather AI Prompts
-      // Gather AI Prompts
+      // Template-level AI prompts, overridden by any on the request.
       let dbTemplate = null;
       if (enriched.templateId) {
         dbTemplate = await this.templateCache.getCachedTemplate(
@@ -508,41 +511,10 @@ export class EngineWorker extends BaseWorker {
       // resulting opt-out applies to.
       const templateTopics: string[] = dbTemplate?.topics ?? [];
 
-      if (Object.keys(aiPrompts).length > 0) {
-        const aiPendingPayload = {
-          projectId: enriched.projectId,
-          enrichedEventId: event.id,
-          recipientId: enriched.recipientId,
-          channel: enriched.channel,
-          priority: enriched.priority,
-          templateId: enriched.templateId,
-          templateVariables: enriched.templateVariables,
-          recipient: enriched.recipient,
-          aiPrompts,
-          scheduledAt: enriched.scheduledAt,
-          fallbackChain: enriched.fallbackChain,
-        };
-
-        const aiPendingEnvelope = buildStreamEvent(
-          "notification.ai_pending",
-          aiPendingPayload as Record<string, unknown>,
-          "engine",
-          event.metadata.traceId,
-        );
-
-        await this.aiPendingProducer.publish(aiPendingEnvelope);
-        this.logger.info(
-          {
-            messageId: message.id,
-            eventId: event.id,
-            recipientId: enriched.recipientId,
-            traceId: event.metadata.traceId,
-          },
-          "task routed to AI worker",
-        );
-        await idempotency.markProcessed(idempotencyKey, customTtl);
-        return;
-      }
+      // AI-generated messages take the same gates as any other — contacts,
+      // topic opt-outs and suppressions below — and are handed to the AI worker
+      // per contact once they have passed them.
+      const useAi = Object.keys(aiPrompts).length > 0;
 
       // Rendering happens in delivery, just before the send, so the rendered
       // body does not travel through the outbound stream.
@@ -588,7 +560,8 @@ export class EngineWorker extends BaseWorker {
       });
 
       for (const contact of activeContacts) {
-        if (contact.preferences?.optedOut) {
+        // This address opted out of one of the template's topics.
+        if (templateTopics.some((t) => contact.preferences?.topics?.[t] === false)) {
           continue;
         }
 
@@ -670,8 +643,42 @@ export class EngineWorker extends BaseWorker {
           },
           fallbackChain: enriched.fallbackChain,
           campaignId: enriched.campaignId,
+          workflowInstanceId: enriched.workflowInstanceId,
         };
 
+        if (useAi) {
+          await this.aiPendingProducer.publish(
+            buildStreamEvent(
+              "notification.ai_pending",
+              {
+                projectId: enriched.projectId,
+                enrichedEventId: event.id,
+                recipientId: enriched.recipientId,
+                channel: enriched.channel,
+                priority: enriched.priority,
+                templateId: enriched.templateId,
+                templateVariables: enriched.templateVariables,
+                recipient: enriched.recipient,
+                aiPrompts,
+                scheduledAt: enriched.scheduledAt,
+                fallbackChain: enriched.fallbackChain,
+                taskId,
+                destination: resolvedDestination,
+                deliveryHeaders: unsubscribeHeaders,
+                campaignId: enriched.campaignId,
+                workflowInstanceId: enriched.workflowInstanceId,
+              },
+              "engine",
+              event.metadata.traceId,
+            ),
+          );
+          this.logger.debug(
+            { messageId: message.id, taskId, traceId: event.metadata.traceId },
+            "task routed to AI worker",
+          );
+          continue;
+        }
+
         const envelope = buildStreamEvent(
           "notification.dispatched",
           taskPayload as Record<string, unknown>,

src/services/enricher/main.ts

@@ -11,6 +11,7 @@ import {
   buildStreamEvent,
   type NotificationCreatedPayload,
   type NotificationEnrichedPayload,
+  type NotificationChannel,
   type StreamEvent,
 } from "@/index.js";
 import { IdempotencyGuard } from "@/index.js";
@@ -21,7 +22,7 @@ import {
   ContactRepository,
 } from "@/index.js";
 import { TemplateCache } from "@/templates/index.js";
-import { getPriorityBucket, type WorkerOptions } from "@/shared/index.js";
+import { getPriorityBucket, globalEmitter, type WorkerOptions } from "@/shared/index.js";
 import { startHealthReporter } from "@/workers/index.js";
 import {
   createPriorityProducers,
@@ -256,10 +257,13 @@ export class EnricherWorker extends BaseWorker {
         const profile = await this.userRepo.findRecordById(raw.projectId, raw.recipientId);
         if (!profile) return;
 
-        const prefs = await this.prefRepo.findByUserId(raw.projectId, raw.recipientId);
-        const optedOutTypes = new Set(
-          prefs.filter((p: any) => !p.optedIn).map((p: any) => p.eventType),
-        );
+        // Opt-outs are keyed on the template's topics, as on the requested
+        // path. `event.type` is always "notification.created", which no user
+        // ever opts out of, so checking it here let every opt-out through.
+        const template = raw.templateId
+          ? await this.templateCache.getCachedTemplate(raw.projectId, raw.templateId)
+          : null;
+        const topics: string[] = template?.topics ?? [];
 
         const enrichedPayload: NotificationEnrichedPayload = {
           projectId: raw.projectId,
@@ -275,8 +279,7 @@ export class EnricherWorker extends BaseWorker {
             locale: profile.language ?? "en",
             timezone: profile.timezone ?? "UTC",
             preferences: {
-              optedOut:
-                optedOutTypes.has(event.type) || profile.preferences.topics?.[event.type] === false,
+              optedOut: topics.some((t) => profile.preferences.topics?.[t] === false),
               channels: Object.entries(profile.preferences.channels ?? {})
                 .filter(([_, enabled]) => !enabled)
                 .map(([channel]) => channel as any),
@@ -311,6 +314,7 @@ export class EnricherWorker extends BaseWorker {
           "event enriched",
         );
       } catch (err) {
+        await this.idempotency.unmark(dedupeId).catch(() => {});
         throw err;
       }
       await Promise.all(publishPromises).catch(async (err) => {
@@ -345,24 +349,16 @@ export class EnricherWorker extends BaseWorker {
 
       const maxUsers = readBaseConfig().SEGMENT_MAX_USERS;
       if (userIds.length > maxUsers) {
+        const reason = `Segment fan-out of ${userIds.length} exceeds limit of ${maxUsers}`;
         this.logger.error(
           { count: userIds.length, max: maxUsers, projectId: raw.projectId, eventId: event.id },
           "Segment fan-out exceeds maximum allowed limit",
         );
-        const p = getPriorityBucket(raw.priority ?? "normal");
-        const producer = this.producers[p] ?? this.producers.normal;
-        await producer.publish(
-          buildStreamEvent(
-            "notification.failed",
-            {
-              projectId: raw.projectId,
-              rawEventId: event.id,
-              error: `Segment fan-out of ${userIds.length} exceeds limit of ${maxUsers}`,
-            },
-            "enricher",
-            event.metadata.traceId,
-          ),
-        );
+        // Reported to the caller, not published: a `notification.failed` on
+        // the enriched stream is not a payload the engine can read, so the
+        // refusal used to vanish there without a trace.
+        globalEmitter.emit("notification:failed", event.id, reason, event.type);
+        await this.idempotency.markProcessed(dedupeId);
         return;
       }
 
@@ -374,15 +370,10 @@ export class EnricherWorker extends BaseWorker {
         : null;
       const topics: string[] = template?.topics ?? [];
 
-      const channels =
-        raw.channels && raw.channels.length > 0 ? raw.channels : (["email"] as any[]);
+      const channels: NotificationChannel[] =
+        raw.channels && raw.channels.length > 0 ? raw.channels : ["email"];
       const isFallback = (raw as any).fallback === true;
 
-      // If fallback is true, we only emit the first channel, and pass the rest in fallbackChain.
-      // If fallback is false, we emit all channels concurrently.
-      const channelsToProcess = isFallback ? [channels[0]] : channels;
-      const fallbackChain = isFallback ? channels.slice(1) : undefined;
-
       const chunkArray = <T>(arr: T[], size: number) =>
         Array.from({ length: Math.ceil(arr.length / size) }, (v, i) =>
           arr.slice(i * size, i * size + size),
@@ -410,9 +401,36 @@ export class EnricherWorker extends BaseWorker {
         };
 
         for (const profile of profiles) {
-          for (const channel of channelsToProcess) {
-            const contacts = contactsByUser.get(profile.userId) ?? [];
-            const channelContacts = contacts.filter((contact: any) => contact.channel === channel);
+          const contacts = contactsByUser.get(profile.userId) ?? [];
+          // This user's addresses on a channel, less any that opted out of one
+          // of this template's topics.
+          const usable = (channel: string) =>
+            contacts.filter(
+              (contact: any) =>
+                contact.channel === channel &&
+                !topics.some((t) => contact.preferences?.topics?.[t] === false),
+            );
+
+          // Without fallback every requested channel is sent at once. With it,
+          // start at the first channel this user can be reached on and chain the
+          // rest behind it — a user with no email address still gets the SMS.
+          let userChannels: NotificationChannel[] = channels;
+          let fallbackChain: NotificationChannel[] | undefined;
+          if (isFallback) {
+            const first = channels.findIndex((c: NotificationChannel) => usable(c).length > 0);
+            if (first === -1) {
+              this.logger.info(
+                { recipientId: profile.userId, channels },
+                "no active contact on any fallback channel",
+              );
+              continue;
+            }
+            userChannels = [channels[first]!];
+            fallbackChain = channels.slice(first + 1);
+          }
+
+          for (const channel of userChannels) {
+            const channelContacts = usable(channel);
             // Push resolves its active tokens at send time so token invalidation
             // remains current. Other channels need one task per address.
             const resolved: ({ id: string; target: string } | undefined)[] =
@@ -430,7 +448,7 @@ export class EnricherWorker extends BaseWorker {
                 projectId: raw.projectId,
                 rawEventId: event.id,
                 recipientId: profile.userId,
-                channel: channel,
+                channel,
                 priority: "normal",
                 templateId: raw.templateId,
                 templateVariables: raw.data,
@@ -464,6 +482,7 @@ export class EnricherWorker extends BaseWorker {
                 contactId: contact?.id,
                 destination,
                 campaignId: raw.campaignId,
+                workflowInstanceId: raw.workflowInstanceId,
               };
 
               const msgPriority = raw.priority ?? "normal";
@@ -520,6 +539,9 @@ export class EnricherWorker extends BaseWorker {
         "event enriched",
       );
     } catch (err) {
+      // Release the marker, or a retry landing inside its TTL is taken for a
+      // duplicate and acked without ever being processed.
+      await this.idempotency.unmark(dedupeId).catch(() => {});
       throw err;
     }
 

src/services/events/main.ts

@@ -14,7 +14,12 @@ import { workflowWaiters, workflowSteps, workflowInstances } from "@/db/schema.j
 import { eq, and, or, isNull, gt } from "drizzle-orm";
 import { type WorkerOptions } from "@/shared/index.js";
 import { startHealthReporter } from "@/workers/index.js";
-import { createStreamConsumer, createWorkerRuntime, shutdownWorker } from "@/workers/bootstrap.js";
+import {
+  createStreamConsumer,
+  createWorkerRuntime,
+  shutdownWorker,
+  uniqueConsumerId,
+} from "@/workers/bootstrap.js";
 
 loadEnv();
 const config = readBaseConfig();
@@ -376,7 +381,7 @@ export async function startEventWorker() {
     {
       stream: STREAMS.EVENTS_INBOUND,
       group: CONSUMER_GROUPS.EVENTS,
-      consumer: `events-${process.pid}`,
+      consumer: uniqueConsumerId("events"),
       batchSize: eventsConcurrency,
       bufferAcks: true,
       // Log rows are not latency-sensitive; let reads fill up.

src/services/scheduler/main.ts

@@ -11,16 +11,18 @@ import {
   buildStreamEvent,
   type NotificationScheduledPayload,
 } from "@/index.js";
-import { getPriorityBucket, type WorkerOptions } from "@/shared/index.js";
+import { getPriorityBucket, globalEmitter, type WorkerOptions } from "@/shared/index.js";
 import { scheduledPayloads } from "@/db/schema.js";
-import { asc, gt, inArray } from "drizzle-orm";
+import { and, asc, eq, gt, inArray, or, sql as drizzleSql } from "drizzle-orm";
 import { startHealthReporter } from "@/workers/index.js";
 import {
   createPriorityProducers,
   createStreamConsumer,
   createWorkerRuntime,
   shutdownWorker,
+  uniqueConsumerId,
 } from "@/workers/bootstrap.js";
+import { findBlockedAtRelease } from "./send-gate.js";
 // ─── Bootstrap ─────────────────────────────────────────────────────────────
 
 loadEnv();
@@ -137,6 +139,11 @@ export async function executeSchedulerPoll(
       .where(inArray(scheduledPayloads.taskId, taskIds));
     const payloadMap = new Map(dbPayloads.map((row: any) => [row.taskId, row.payload]));
 
+    // What the person did while this sat parked still applies. A failed
+    // lookup throws to the catch below: the claimed tasks reappear after the
+    // visibility timeout instead of going out unchecked.
+    const blocked = await findBlockedAtRelease(db, [...payloadMap.values()] as any[]);
+
     const batchedEvents: Record<"critical" | "normal" | "low", Omit<any, "id" | "timestamp">[]> = {
       critical: [],
       normal: [],
@@ -170,6 +177,23 @@ export async function executeSchedulerPoll(
       }
 
       const dispatchPayload = payload as any;
+
+      const blockReason = blocked.get(taskId);
+      if (blockReason) {
+        logger.info({ taskId, reason: blockReason }, "parked send no longer allowed — dropping");
+        globalEmitter.emit("notification:skipped", {
+          projectId: dispatchPayload.projectId,
+          eventId: dispatchPayload.enrichedEventId,
+          recipientId: dispatchPayload.recipientId,
+          reason: blockReason,
+        });
+        released.add(taskId);
+        dbCleanupIds.push(taskId);
+        const shard = parseInt(taskId.slice(-1), 16) || 0;
+        cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
+        continue;
+      }
+
       const p = getPriorityBucket(dispatchPayload.priority);
 
       batchedEvents[p].push(
@@ -194,7 +218,24 @@ export async function executeSchedulerPoll(
 
     if (dbCleanupIds.length > 0) {
       try {
-        await db.delete(scheduledPayloads).where(inArray(scheduledPayloads.taskId, dbCleanupIds));
+        // Only the payload that was released. By now delivery may have
+        // throttled the task and parked it again under the same id with a
+        // later send time; deleting by id alone removed that newer payload,
+        // and its ZSET entry was then dropped as canceled — the send was lost.
+        const sendAt = drizzleSql`${scheduledPayloads.payload}->>'scheduledAt'`;
+        await db.delete(scheduledPayloads).where(
+          or(
+            ...dbCleanupIds.map((taskId) => {
+              const released = (payloadMap.get(taskId) as any)?.scheduledAt;
+              return and(
+                eq(scheduledPayloads.taskId, taskId),
+                typeof released === "string"
+                  ? drizzleSql`${sendAt} = ${released}`
+                  : drizzleSql`${sendAt} IS NULL`,
+              );
+            }),
+          ),
+        );
       } catch (err) {
         logger.error(
           { err },
@@ -284,7 +325,7 @@ export async function startSchedulerWorker() {
     {
       stream: STREAMS.SCHEDULED,
       group: CONSUMER_GROUPS.SCHEDULER,
-      consumer: `scheduler-${process.pid}`,
+      consumer: uniqueConsumerId("scheduler"),
       batchSize: config.WORKER_CONCURRENCY,
     },
   ));

src/services/scheduler/send-gate.ts

@@ -0,0 +1,117 @@
+import { and, eq, inArray } from "drizzle-orm";
+import { suppressions } from "@/db/schema.js";
+import { ContactRepository, TemplateRepository, UserRepository } from "@/repositories/index.js";
+import { normaliseTarget } from "@/shared/index.js";
+
+export type BlockReason =
+  "suppressed" | "recipient_not_found" | "user_opted_out" | "channel_disabled" | "contact_inactive";
+
+/** The fields of a parked `notification.dispatched` payload the gate reads. */
+interface ParkedTask {
+  taskId: string;
+  projectId: string;
+  recipientId: string;
+  channel: string;
+  templateId?: string;
+  destination?: string;
+}
+
+/**
+ * Re-checks parked sends at the moment they are released.
+ *
+ * The engine clears a message against suppressions, opt-outs and contacts when
+ * it first sees it. A send parked for later — a `sendAt`, a quiet-hours
+ * deferral, a retry — can sit for hours or days, and whatever the person did in
+ * between (unsubscribed, complained, hard-bounced, removed the address) must
+ * still stop it. Without this the scheduler sent to them anyway.
+ *
+ * Reads Postgres directly rather than through the repositories' caches: this
+ * runs once per poll, not once per message, and a stale "still subscribed" is
+ * the very thing it exists to prevent.
+ *
+ * Returns the blocked tasks with the reason. Throws if a lookup fails, so the
+ * caller holds the batch rather than sending it unchecked.
+ */
+export async function findBlockedAtRelease(
+  db: any,
+  tasks: ParkedTask[],
+): Promise<Map<string, BlockReason>> {
+  const blocked = new Map<string, BlockReason>();
+
+  const byProject = new Map<string, ParkedTask[]>();
+  for (const t of tasks) {
+    if (!t?.projectId || !t.recipientId) continue;
+    if (!byProject.has(t.projectId)) byProject.set(t.projectId, []);
+    byProject.get(t.projectId)!.push(t);
+  }
+
+  for (const [projectId, group] of byProject) {
+    const recipientIds = [...new Set(group.map((t) => t.recipientId))];
+    const templateIds = [...new Set(group.map((t) => t.templateId).filter(Boolean))] as string[];
+    const targets = [
+      ...new Set(group.map((t) => t.destination && normaliseTarget(t.destination)).filter(Boolean)),
+    ] as string[];
+
+    const templateRepo = new TemplateRepository(db);
+    const [suppressedRows, users, contactsByUser, templates] = await Promise.all([
+      targets.length > 0
+        ? db
+            .select({ channel: suppressions.channel, target: suppressions.target })
+            .from(suppressions)
+            .where(
+              and(eq(suppressions.projectId, projectId), inArray(suppressions.target, targets)),
+            )
+        : [],
+      new UserRepository(db).findRecordsByIds(projectId, recipientIds),
+      new ContactRepository(db).findActiveByUserIds(projectId, recipientIds),
+      Promise.all(templateIds.map((id) => templateRepo.findById(projectId, id))),
+    ]);
+
+    const suppressed = new Set(
+      (suppressedRows as { channel: string; target: string }[]).map(
+        (r) => `${r.channel}:${normaliseTarget(r.target)}`,
+      ),
+    );
+    const userById = new Map(users.map((u) => [u.userId, u]));
+    const topicsByTemplate = new Map<string, string[]>();
+    templateIds.forEach((id, i) => topicsByTemplate.set(id, templates[i]?.topics ?? []));
+
+    for (const t of group) {
+      const topics = t.templateId ? (topicsByTemplate.get(t.templateId) ?? []) : [];
+      const reason = decide(t, topics, suppressed, userById.get(t.recipientId), contactsByUser);
+      if (reason) blocked.set(t.taskId, reason);
+    }
+  }
+
+  return blocked;
+}
+
+function decide(
+  t: ParkedTask,
+  topics: string[],
+  suppressed: Set<string>,
+  user:
+    | { preferences: { topics?: Record<string, boolean>; channels?: Record<string, boolean> } }
+    | undefined,
+  contactsByUser: Map<string, { channel: string; target: string; preferences?: any }[]>,
+): BlockReason | undefined {
+  // Checked first and for every priority, as the engine does: a suppression
+  // records that the address asked us to stop or is dead.
+  if (t.destination && suppressed.has(`${t.channel}:${normaliseTarget(t.destination)}`)) {
+    return "suppressed";
+  }
+  if (!user) return "recipient_not_found";
+  if (topics.some((topic) => user.preferences.topics?.[topic] === false)) return "user_opted_out";
+  if (user.preferences.channels?.[t.channel] === false) return "channel_disabled";
+
+  if (t.destination) {
+    const contact = (contactsByUser.get(t.recipientId) ?? []).find(
+      (c) => c.channel === t.channel && c.target === t.destination,
+    );
+    if (!contact) return "contact_inactive";
+    if (topics.some((topic) => contact.preferences?.topics?.[topic] === false)) {
+      return "user_opted_out";
+    }
+  }
+  return undefined;
+}

src/services/workflow/main.ts

@@ -26,7 +26,12 @@ import {
 } from "@/workflows/index.js";
 import { type WorkerOptions } from "@/shared/index.js";
 import { startHealthReporter } from "@/workers/index.js";
-import { createStreamConsumer, createWorkerRuntime, shutdownWorker } from "@/workers/bootstrap.js";
+import {
+  createStreamConsumer,
+  createWorkerRuntime,
+  shutdownWorker,
+  uniqueConsumerId,
+} from "@/workers/bootstrap.js";
 
 /** How long a workflow instance lock is held before it self-expires. */
 const WORKFLOW_LOCK_TTL_SECONDS = 60;
@@ -34,6 +39,11 @@ const WORKFLOW_LOCK_TTL_SECONDS = 60;
 const WORKFLOW_LOCK_RENEW_MS = (WORKFLOW_LOCK_TTL_SECONDS / 3) * 1000;
 /** How long a claimed workflow timer stays invisible to other pollers. */
 const WORKFLOW_TIMER_VISIBILITY_MS = 60_000;
+/** Wake-ups for sleeping instances: waits, event timeouts, deferred resumes. */
+export const WORKFLOW_TIMERS_KEY = "notif:workflow:timers";
+/** How soon a resume that found its instance busy is tried again. */
+const WORKFLOW_RESUME_RETRY_MS = 2_000;
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 
 loadEnv();
 const config = readBaseConfig();
@@ -119,6 +129,25 @@ export class WorkflowWorker extends BaseWorker {
     }
   }
 
+  /**
+   * Re-delivers a resume through the timer ZSET a few seconds from now. The
+   * poll loop publishes it once the instance is pending again, and drops it if
+   * the instance has finished. The member carries no timestamp, so repeated
+   * deferrals of one instance collapse into a single entry.
+   */
+  private async deferResume(payload: any): Promise<void> {
+    await this.redisCli.zadd(
+      WORKFLOW_TIMERS_KEY,
+      Date.now() + WORKFLOW_RESUME_RETRY_MS,
+      JSON.stringify({
+        instanceId: payload.instanceId,
+        name: payload.name,
+        projectId: payload.projectId,
+        input: payload.input ?? {},
+      }),
+    );
+  }
+
   async process(message: StreamMessage): Promise<void> {
     const { event } = message;
     const publishPromises: Promise<void>[] = [];
@@ -189,7 +218,12 @@ export class WorkflowWorker extends BaseWorker {
       "NX",
     );
     if (!acquired) {
-      this.logger.info({ instanceId }, "Workflow is locked by another process, skipping");
+      // Another process is mid-run on this instance — typically still
+      // persisting the very suspension this resume answers, when the awaited
+      // event arrived quickly. Acking here dropped the resume, and with the
+      // waiter already consumed nothing would ever wake the instance again.
+      this.logger.info({ instanceId }, "Workflow is locked by another process, deferring resume");
+      await this.deferResume(payload);
       return;
     }
 
@@ -225,6 +259,13 @@ export class WorkflowWorker extends BaseWorker {
         instance = rows[0]!;
       }
 
+      if (instance.status === "running") {
+        // Its last holder died mid-run (the reaper will set it back to
+        // pending) or has not finished suspending. Either way this resume
+        // is still owed.
+        await this.deferResume(payload);
+        return;
+      }
       if (instance.status !== "pending") {
         this.logger.info({ instanceId }, "Workflow is not pending, skipping");
         return;
@@ -259,6 +300,9 @@ export class WorkflowWorker extends BaseWorker {
             instance!.input,
             payload.projectId,
             `wf-${instanceId}-${stepId}`,
+            // The contract wants a uuid, which the API always issues; anything
+            // else would fail validation and drop the notification.
+            UUID_RE.test(instanceId) ? instanceId : undefined,
           );
 
           const result = await new Promise<{ messageId: string; notificationId: string }>(
@@ -295,7 +339,15 @@ export class WorkflowWorker extends BaseWorker {
         },
         wait: async (duration) => {
           const stepId = String(currentStepIndex++);
-          if (stepOutputMap.has(stepId)) return;
+          if (stepOutputMap.has(stepId)) {
+            // A resume that arrives early — a duplicate trigger, a replayed
+            // message — must not cut the sleep short. The timer is still set.
+            const until = Number(stepOutputMap.get(stepId)?.scheduledAt);
+            if (Number.isFinite(until) && until > Date.now()) {
+              throw new SuspendExecutionError("wait", { duration });
+            }
+            return;
+          }
 
           // Simple duration parse (e.g. '2h' -> ms)
           let ms = 0;
@@ -311,7 +363,7 @@ export class WorkflowWorker extends BaseWorker {
           // If the process dies after this point the source event is retried;
           // if it dies after the database write, the timer is already durable.
           await this.redisCli.zadd(
-            "notif:workflow:timers",
+            WORKFLOW_TIMERS_KEY,
             resumeAt,
             JSON.stringify({
               instanceId: instance!.id,
@@ -335,7 +387,12 @@ export class WorkflowWorker extends BaseWorker {
           const stepId = String(currentStepIndex++);
           if (stepOutputMap.has(stepId)) {
             const out = stepOutputMap.get(stepId);
-            if (out && typeof out === "object" && (out as any).timedOut === true) {
+            // Still waiting: neither the event nor the timeout has filled the
+            // step in, so this resume is spurious.
+            if (out === null || out === undefined) {
+              throw new SuspendExecutionError("waitForEvent", { eventName });
+            }
+            if (typeof out === "object" && (out as any).timedOut === true) {
               return null;
             }
             return out;
@@ -355,7 +412,7 @@ export class WorkflowWorker extends BaseWorker {
           const resumeAt = Date.now() + ms;
 
           await this.redisCli.zadd(
-            "notif:workflow:timers",
+            WORKFLOW_TIMERS_KEY,
             resumeAt,
             JSON.stringify({
               instanceId: instance!.id,
@@ -368,16 +425,8 @@ export class WorkflowWorker extends BaseWorker {
             }),
           );
 
-          // Register waiter
-          await this.dbConn.insert(workflowWaiters).values({
-            instanceId: instance!.id,
-            projectId: payload.projectId,
-            eventName: eventName,
-            matchCriteria: options.match,
-            expiresAt: new Date(resumeAt),
-          });
-
-          // Register step as pending event
+          // The pending step before the waiter: an event that matches the
+          // waiter the moment it exists must find a step to record itself in.
           await this.dbConn.insert(workflowSteps).values({
             instanceId: instance!.id,
             projectId: payload.projectId,
@@ -386,6 +435,14 @@ export class WorkflowWorker extends BaseWorker {
             output: null, // this will be updated by event worker or timeout
           });
 
+          await this.dbConn.insert(workflowWaiters).values({
+            instanceId: instance!.id,
+            projectId: payload.projectId,
+            eventName: eventName,
+            matchCriteria: options.match,
+            expiresAt: new Date(resumeAt),
+          });
+
           throw new SuspendExecutionError("waitForEvent", { eventName });
         },
         run: async (stepName, fn) => {
@@ -466,7 +523,7 @@ export async function startWorkflowWorker() {
     {
       stream: STREAMS.WORKFLOW_INBOUND,
       group: CONSUMER_GROUPS.WORKFLOW,
-      consumer: `workflow-${process.pid}`,
+      consumer: uniqueConsumerId("workflow"),
       batchSize: config.WORKER_CONCURRENCY,
     },
   ));
@@ -488,7 +545,7 @@ export async function startWorkflowWorker() {
       try {
         const now = Date.now();
         const tasks = await redis.native.schedulerPoll(
-          "notif:workflow:timers",
+          WORKFLOW_TIMERS_KEY,
           now,
           100,
           WORKFLOW_TIMER_VISIBILITY_MS,
@@ -505,8 +562,13 @@ export async function startWorkflowWorker() {
               .where(eq(workflowInstances.id, task.instanceId))
               .limit(1)
           )[0];
-          if (!inst || inst.status === "completed" || inst.status === "failed") {
-            await redis.native.zrem("notif:workflow:timers", taskStr);
+          if (
+            !inst ||
+            inst.status === "completed" ||
+            inst.status === "failed" ||
+            inst.status === "canceled"
+          ) {
+            await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
             continue;
           }
           // A timer can be observed while the worker is still persisting the
@@ -530,7 +592,7 @@ export async function startWorkflowWorker() {
             // If the waiter was already deleted (by EventWorker when event arrived before timeout),
             // this timer is stale. Clean up from Redis and skip re-resuming the workflow.
             if (deleted.length === 0) {
-              await redis.native.zrem("notif:workflow:timers", taskStr);
+              await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
               continue;
             }
 
@@ -556,7 +618,7 @@ export async function startWorkflowWorker() {
           await workflowProducer.publish(
             buildStreamEvent("workflow.resumed", task, "scheduler", undefined),
           );
-          await redis.native.zrem("notif:workflow:timers", taskStr);
+          await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
           logger.info({ instanceId: task.instanceId }, "Workflow resumed from timer");
         }
       } catch (err) {

src/transport/index.ts

@@ -10,6 +10,12 @@ export interface DeliveryResult {
   providerMessageId?: string;
   invalidToken?: boolean;
   error?: string;
+  /**
+   * `false` marks a failure another attempt cannot fix — a malformed address,
+   * a rejected sender. Anything else (an outage, a timeout, a 5xx) is retried
+   * up to the task's `maxAttempts`.
+   */
+  retryable?: boolean;
 }
 
 export interface WebhookEvent {

src/workers/index.ts

@@ -220,7 +220,7 @@ export abstract class BaseWorker {
           { err, eventId: event.id },
           "non-retryable error on inline message — moving to dead-letter queue",
         );
-        await this.consumer.deadLetter(event);
+        await this.consumer.deadLetter(event, (err as Error).message);
         globalEmitter.emit("notification:failed", event.id, (err as Error).message, event.type);
         return;
       }
@@ -239,7 +239,12 @@ export abstract class BaseWorker {
           { messageId: message.id, retryCount },
           "max retries exceeded, moving to dead-letter queue",
         );
-        await this.consumer.nack(message.id, message.event, stream);
+        await this.consumer.nack(
+          message.id,
+          message.event,
+          stream,
+          `max retries exceeded (${retryCount - 1} attempts)`,
+        );
         globalEmitter.emit(
           "notification:failed",
           message.id,
@@ -285,7 +290,7 @@ export abstract class BaseWorker {
           { err, messageId: message.id },
           "non-retryable error encountered, immediately moving to dead-letter queue without retry loop",
         );
-        await this.consumer.nack(message.id, message.event, stream);
+        await this.consumer.nack(message.id, message.event, stream, (err as Error).message);
         globalEmitter.emit(
           "notification:failed",
           message.id,

src/workflows/sdk.ts

@@ -90,6 +90,7 @@ export function buildStepNotifyPayload(
   instanceInput: unknown,
   projectId: string,
   idempotencyKey?: string,
+  workflowInstanceId?: string,
 ): NotificationRequestedPayload {
   return {
     projectId,
@@ -102,5 +103,6 @@ export function buildStepNotifyPayload(
     fallback: args.fallback ?? false,
     scheduledAt: args.sendAt,
     idempotencyKey,
+    ...(workflowInstanceId ? { workflowInstanceId } : {}),
   };
 }

tests/api-admin.test.ts

@@ -427,7 +427,11 @@ describe("API operational handlers", () => {
       );
 
       expect(res.statusCode).toBe(200);
-      expect(parse(res)).toEqual({ success: true, replayedId: "1-0" });
+      expect(parse(res)).toEqual({
+        success: true,
+        replayedId: "1-0",
+        stream: STREAMS.INBOUND_CRITICAL,
+      });
       expect(deps.redis.native.xadd).toHaveBeenCalledWith(
         STREAMS.INBOUND_CRITICAL,
         "*",
@@ -534,10 +538,11 @@ describe("API operational handlers", () => {
 
     it("cancelWorkflow returns 204 once the instance is canceled", async () => {
       const res = createMockRes();
-      await handlers.cancelWorkflow(createMockReq(), res, ctx({ params: { id: "wf_1" } }));
+      const id = "33333333-3333-4333-8333-333333333333";
+      await handlers.cancelWorkflow(createMockReq(), res, ctx({ params: { id } }));
 
       expect(res.statusCode).toBe(204);
-      expect(deps.workflowRepo.cancelInstance).toHaveBeenCalledWith("proj_1", "wf_1");
+      expect(deps.workflowRepo.cancelInstance).toHaveBeenCalledWith("proj_1", id);
     });
 
     it("cancelWorkflow 400s when the instance cannot be canceled", async () => {

tests/api-suppressions.test.ts

@@ -380,6 +380,19 @@ describe("API compliance handlers", () => {
   describe("getCampaignStats", () => {
     const statsCtx = (campaign = "spring-sale") => ctx({ params: { campaign } });
 
+    /**
+     * The grouped rows the stats query returns for `sent` tasks on a channel,
+     * shaped like the real delivery log: every attempt writes a `dispatched`
+     * row and then an `attempt` row carrying its outcome.
+     */
+    const sends = (channel: string, delivered: number, failed = 0) => [
+      { channel, kind: "dispatched", status: "dispatched", tasks: String(delivered + failed) },
+      ...(delivered
+        ? [{ channel, kind: "attempt", status: "delivered", tasks: String(delivered) }]
+        : []),
+      ...(failed ? [{ channel, kind: "attempt", status: "failed", tasks: String(failed) }] : []),
+    ];
+
     it("400s when no campaign is named", async () => {
       const res = createMockRes();
       await handlers.getCampaignStats(createMockReq(), res, ctx({ params: {} }));
@@ -399,10 +412,7 @@ describe("API compliance handlers", () => {
     });
 
     it("splits delivery rows into sent, delivered and failed", async () => {
-      mockDb.queueSelect([
-        { channel: "email", kind: "delivery", status: "delivered", tasks: "90" },
-        { channel: "email", kind: "delivery", status: "failed", tasks: "10" },
-      ]);
+      mockDb.queueSelect(sends("email", 90, 10));
       const res = createMockRes();
 
       await handlers.getCampaignStats(createMockReq(), res, statsCtx());
@@ -416,7 +426,7 @@ describe("API compliance handlers", () => {
 
     it("counts engagement kinds separately from the send total", async () => {
       mockDb.queueSelect([
-        { channel: "email", kind: "delivery", status: "delivered", tasks: "100" },
+        ...sends("email", 100),
         { channel: "email", kind: "opened", status: "opened", tasks: "40" },
         { channel: "email", kind: "clicked", status: "clicked", tasks: "10" },
       ]);
@@ -434,7 +444,7 @@ describe("API compliance handlers", () => {
 
     it("rounds a rate to two decimals", async () => {
       mockDb.queueSelect([
-        { channel: "email", kind: "delivery", status: "delivered", tasks: "3" },
+        ...sends("email", 3),
         { channel: "email", kind: "opened", status: "opened", tasks: "1" },
       ]);
       const res = createMockRes();
@@ -445,7 +455,7 @@ describe("API compliance handlers", () => {
     });
 
     it("reports a rate as null, not zero, when its denominator is zero", async () => {
-      mockDb.queueSelect([{ channel: "email", kind: "delivery", status: "failed", tasks: "5" }]);
+      mockDb.queueSelect(sends("email", 0, 5));
       const res = createMockRes();
 
       await handlers.getCampaignStats(createMockReq(), res, statsCtx());
@@ -458,11 +468,7 @@ describe("API compliance handlers", () => {
     });
 
     it("breaks the funnel down per channel and totals across them", async () => {
-      mockDb.queueSelect([
-        { channel: "email", kind: "delivery", status: "delivered", tasks: "50" },
-        { channel: "sms", kind: "delivery", status: "delivered", tasks: "30" },
-        { channel: "sms", kind: "delivery", status: "failed", tasks: "20" },
-      ]);
+      mockDb.queueSelect([...sends("email", 50), ...sends("sms", 30, 20)]);
       const res = createMockRes();
 
       await handlers.getCampaignStats(createMockReq(), res, statsCtx());
@@ -476,7 +482,7 @@ describe("API compliance handlers", () => {
     });
 
     it("warns that opens are untracked rather than zero on a channel that cannot report them", async () => {
-      mockDb.queueSelect([{ channel: "sms", kind: "delivery", status: "delivered", tasks: "10" }]);
+      mockDb.queueSelect(sends("sms", 10));
       const res = createMockRes();
 
       await handlers.getCampaignStats(createMockReq(), res, statsCtx());
@@ -487,9 +493,7 @@ describe("API compliance handlers", () => {
     });
 
     it("warns when email was delivered but no engagement ever arrived", async () => {
-      mockDb.queueSelect([
-        { channel: "email", kind: "delivery", status: "delivered", tasks: "10" },
-      ]);
+      mockDb.queueSelect(sends("email", 10));
       const res = createMockRes();
 
       await handlers.getCampaignStats(createMockReq(), res, statsCtx());
@@ -501,7 +505,7 @@ describe("API compliance handlers", () => {
 
     it("stays quiet when engagement is both possible and present", async () => {
       mockDb.queueSelect([
-        { channel: "email", kind: "delivery", status: "delivered", tasks: "10" },
+        ...sends("email", 10),
         { channel: "email", kind: "opened", status: "opened", tasks: "4" },
       ]);
       const res = createMockRes();

tests/api.test.ts

@@ -173,6 +173,9 @@ describe("API Handlers", () => {
         listApiKeys: vi.fn().mockResolvedValue([]),
         deleteApiKey: vi.fn().mockResolvedValue(true),
         updateApiKeyHash: vi.fn().mockResolvedValue(true),
+        findThrottleSettings: vi
+          .fn()
+          .mockResolvedValue({ throttleLimit: null, throttleWindowHours: null }),
       } as any,
       workflowRepo: {
         listDefinitions: vi.fn().mockResolvedValue([]),
@@ -509,6 +512,10 @@ describe("API Handlers", () => {
   });
 
   describe("New CRUD Endpoints", () => {
+    const PROJECT_UUID = "11111111-1111-4111-8111-111111111111";
+    const KEY_UUID = "22222222-2222-4222-8222-222222222222";
+    const WF_ID = "33333333-3333-4333-8333-333333333333";
+
     it("listWorkflows", async () => {
       const res = createMockRes();
       await handlers.listWorkflows({} as any, res, {
@@ -525,11 +532,11 @@ describe("API Handlers", () => {
       const res = createMockRes();
       await handlers.getWorkflow({} as any, res, {
         projectId: "proj_1",
-        params: { id: "wf_1" },
+        params: { id: WF_ID },
         query: new URLSearchParams(),
       } as any);
       expect(res.statusCode).toBe(200);
-      expect(deps.workflowRepo.getInstance).toHaveBeenCalledWith("proj_1", "wf_1");
+      expect(deps.workflowRepo.getInstance).toHaveBeenCalledWith("proj_1", WF_ID);
     });
 
     it("listUsers", async () => {
@@ -590,11 +597,11 @@ describe("API Handlers", () => {
       const res = createMockRes();
       await handlers.updateProject(req, res, {
         projectId: "proj_1",
-        params: { id: "proj_1" },
+        params: { id: PROJECT_UUID },
         query: new URLSearchParams(),
       } as any);
       expect(res.statusCode).toBe(200);
-      expect(deps.projectRepo.updateSettings).toHaveBeenCalledWith("proj_1", {
+      expect(deps.projectRepo.updateSettings).toHaveBeenCalledWith(PROJECT_UUID, {
         rateLimitRpm: 1000,
       });
     });
@@ -603,11 +610,11 @@ describe("API Handlers", () => {
       const res = createMockRes();
       await handlers.deleteProject({} as any, res, {
         projectId: "proj_1",
-        params: { id: "proj_1" },
+        params: { id: PROJECT_UUID },
         query: new URLSearchParams(),
       } as any);
       expect(res.statusCode).toBe(204);
-      expect(deps.projectRepo.delete).toHaveBeenCalledWith("proj_1");
+      expect(deps.projectRepo.delete).toHaveBeenCalledWith(PROJECT_UUID);
     });
 
     it("createProjectKey", async () => {
@@ -616,7 +623,7 @@ describe("API Handlers", () => {
       (deps.projectRepo.createApiKey as any).mockResolvedValue({ id: "key_1" });
       await handlers.createProjectKey(req, res, {
         projectId: "proj_1",
-        params: { id: "proj_1" },
+        params: { id: PROJECT_UUID },
         query: new URLSearchParams(),
       } as any);
       expect(res.statusCode).toBe(201);
@@ -630,11 +637,11 @@ describe("API Handlers", () => {
       (deps.projectRepo.listApiKeys as any).mockResolvedValue([{ id: "key_1", role: "admin" }]);
       await handlers.listProjectKeys({ method: "GET" } as any, res, {
         projectId: "proj_1",
-        params: { id: "proj_1" },
+        params: { id: PROJECT_UUID },
         query: new URLSearchParams(),
       } as any);
       expect(res.statusCode).toBe(200);
-      expect(deps.projectRepo.listApiKeys).toHaveBeenCalledWith("proj_1");
+      expect(deps.projectRepo.listApiKeys).toHaveBeenCalledWith(PROJECT_UUID);
     });
 
     it("deleteProjectKey", async () => {
@@ -642,11 +649,11 @@ describe("API Handlers", () => {
       (deps.projectRepo.deleteApiKey as any).mockResolvedValue(true);
       await handlers.deleteProjectKey({ method: "DELETE" } as any, res, {
         projectId: "proj_1",
-        params: { id: "proj_1", keyId: "key_1" },
+        params: { id: PROJECT_UUID, keyId: KEY_UUID },
         query: new URLSearchParams(),
       } as any);
       expect(res.statusCode).toBe(204);
-      expect(deps.projectRepo.deleteApiKey).toHaveBeenCalledWith("proj_1", "key_1");
+      expect(deps.projectRepo.deleteApiKey).toHaveBeenCalledWith(PROJECT_UUID, KEY_UUID);
     });
 
     it("deleteProjectKey returns 404 if key does not exist", async () => {

tests/chaos/crash.test.ts

@@ -26,23 +26,21 @@ describe("Crash Testing (Chaos Monkey)", () => {
 
   const spawnWorker = (type: string, id: string) => {
     logger.info(`Spawning worker: ${type} (${id})`);
-    const isWin = process.platform === "win32";
-    const cp = spawn(
-      isWin ? "cmd.exe" : "npx",
-      isWin
-        ? ["/c", "npx", "--no-install", "tsx", "tests/chaos/worker-runner.ts"]
-        : ["--no-install", "tsx", "tests/chaos/worker-runner.ts"],
-      {
-        env: {
-          ...process.env,
-          WORKER_TYPE: type,
-          DATABASE_URL: dbUrl,
-          REDIS_URL: redisUrl,
-          LOG_LEVEL: "silent",
-        },
-        stdio: "ignore", // ignore output to keep logs clean
+    // The worker is this node binary's own child, with tsx loaded as an import
+    // hook. Launched through `cmd /c npx tsx` (or `npx tsx`) instead, a kill
+    // reached only the wrapper: on Windows the worker itself survived every
+    // "crash", so no crash was ever tested, and each respawn leaked another
+    // worker until the machine ran out of memory.
+    const cp = spawn(process.execPath, ["--import", "tsx", "tests/chaos/worker-runner.ts"], {
+      env: {
+        ...process.env,
+        WORKER_TYPE: type,
+        DATABASE_URL: dbUrl,
+        REDIS_URL: redisUrl,
+        LOG_LEVEL: "silent",
       },
-    );
+      stdio: "ignore", // ignore output to keep logs clean
+    });
 
     cp.on("exit", (code, signal) => {
       logger.info(`Worker ${type} (${id}) exited with signal ${signal} (code ${code})`);

tests/client.test.ts

@@ -2,16 +2,13 @@ import { describe, it, expect, vi, afterEach } from "vitest";
 import { NotifkitClient } from "@/client.js";
 
 /**
- * Stubs `fetch` with a minimal Response double. `status` drives `ok` unless it
- * is given explicitly, which lets the error-path tests cover a 4xx that still
- * carries a JSON body.
+ * Stubs `fetch` with a real `Response` carrying `body` as JSON, so the client
+ * reads it the way it reads a server's. The request/response round trip itself
+ * is covered against a running server in tests/integration/api.test.ts; these
+ * pin down what the client puts on the wire.
  */
 function stubFetch(body: unknown, status = 200) {
-  const fn = vi.fn().mockResolvedValue({
-    status,
-    ok: status >= 200 && status < 300,
-    json: async () => body,
-  });
+  const fn = vi.fn().mockImplementation(async () => new Response(JSON.stringify(body), { status }));
   vi.stubGlobal("fetch", fn);
   return fn;
 }
@@ -76,18 +73,20 @@ describe("NotifkitClient", () => {
     });
 
     it("returns undefined on 204 without parsing a body", async () => {
-      const fetchMock = vi.fn().mockResolvedValue({
-        status: 204,
-        ok: true,
-        json: async () => {
-          throw new Error("204 responses have no body to parse");
-        },
-      });
-      vi.stubGlobal("fetch", fetchMock);
-
+      vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(null, { status: 204 })));
       await expect(client().deleteUser("usr_1")).resolves.toBeUndefined();
     });
 
+    it("reports the status when an error response is not JSON", async () => {
+      vi.stubGlobal(
+        "fetch",
+        vi.fn().mockResolvedValue(new Response("<html>Bad Gateway</html>", { status: 502 })),
+      );
+      await expect(client().addUser({ id: "usr_1" })).rejects.toThrow(
+        "Request failed with status 502",
+      );
+    });
+
     it("sends no body when none is supplied", async () => {
       const fetchMock = stubFetch({ workflows: [] });
       await client().listWorkflows();

tests/integration/ai.test.ts

@@ -0,0 +1,209 @@
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { randomUUID } from "node:crypto";
+import { MockLanguageModelV4 } from "ai/test";
+import { APICallError } from "ai";
+import { STREAMS } from "@/contracts/index.js";
+import { useInfra, waitFor, settle } from "./support/infra.js";
+import { RecordingTransport, startNotifkit, type Running, type Api } from "./support/pipeline.js";
+
+/**
+ * AI-generated content, end to end. The language model is the one stand-in:
+ * it is an external paid API, so a local model records every prompt it is
+ * sent and answers deterministically. Everything else — engine, AI worker,
+ * delivery, Redis, Postgres — is real.
+ */
+
+const infra = useInfra();
+let app: Running;
+const email = new RecordingTransport("email");
+
+const prompts: string[] = [];
+let failNext: Error | null = null;
+
+const model = new MockLanguageModelV4({
+  doGenerate: async (opts: any) => {
+    const prompt = opts.prompt.flatMap((m: any) => m.content.map((c: any) => c.text)).join("");
+    prompts.push(prompt);
+    if (failNext) {
+      const err = failNext;
+      failNext = null;
+      throw err;
+    }
+    return {
+      content: [{ type: "text", text: `AI(${prompt})` }],
+      finishReason: { unified: "stop", raw: "stop" },
+      usage: {
+        inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 },
+        outputTokens: { total: 1, text: 1, reasoning: 0 },
+      },
+      warnings: [],
+    } as any;
+  },
+});
+
+beforeAll(async () => {
+  app = await startNotifkit(infra, {
+    services: ["api", "enricher", "engine", "ai", "scheduler", "delivery", "events"],
+    providers: [email],
+    aiModel: model as any,
+    env: { RATE_LIMIT_PER_HOUR: "100000" },
+  });
+}, 120_000);
+
+afterAll(async () => {
+  await app?.stop();
+}, 60_000);
+
+async function project() {
+  const p = await app.project();
+  await p.api("PUT", "/v1/templates", {
+    templates: [
+      { id: "digest", channel: "email", content: { subject: "Digest", text: "{{summary}}" } },
+      {
+        id: "smart",
+        channel: "email",
+        content: { subject: "{{headline}}" },
+        aiPrompts: { headline: "Write a headline about {{topic}}" },
+      },
+    ],
+  });
+  return p;
+}
+
+async function notify(api: Api, body: Record<string, unknown>) {
+  const res = await api("POST", "/v1/notify", body);
+  expect(res.status).toBe(202);
+}
+
+describe("AI-generated notifications", () => {
+  it("generates each prompt with the request's data and sends the result", async () => {
+    const p = await project();
+    await p.api("POST", "/v1/users", { id: "u", email: "u@x.com" });
+    const marker = randomUUID().slice(0, 8);
+
+    await notify(p.api, {
+      user: "u",
+      template: "digest",
+      data: { name: marker },
+      aiPrompts: { summary: "Summarise the week for {{name}}" },
+    });
+
+    await waitFor("delivered", () => email.for(p.id).length === 1);
+    expect(prompts).toContain(`Summarise the week for ${marker}`);
+    expect(email.for(p.id)[0]!.renderedContent.content).toEqual({
+      subject: "Digest",
+      text: `AI(Summarise the week for ${marker})`,
+    });
+    expect(email.for(p.id)[0]!.destination).toBe("u@x.com");
+  });
+
+  it("uses the template's own prompts", async () => {
+    const p = await project();
+    await p.api("POST", "/v1/users", { id: "u", email: "u@x.com" });
+    const topic = randomUUID().slice(0, 8);
+    await notify(p.api, { user: "u", template: "smart", data: { topic } });
+    await waitFor("delivered", () => email.for(p.id).length === 1);
+    expect(email.for(p.id)[0]!.renderedContent.content).toEqual({
+      subject: `AI(Write a headline about ${topic})`,
+    });
+  });
+
+  it("generates once per address, sending each address its own copy", async () => {
+    const p = await project();
+    await p.api("POST", "/v1/users", { id: "u", email: ["a@x.com", "b@x.com"] });
+    await notify(p.api, { user: "u", template: "digest", aiPrompts: { summary: "hi" } });
+    await waitFor("both delivered", () => email.for(p.id).length === 2);
+    expect(
+      email
+        .for(p.id)
+        .map((t) => t.destination)
+        .sort(),
+    ).toEqual(["a@x.com", "b@x.com"]);
+  });
+
+  it("never sends AI mail to a suppressed address", async () => {
+    const p = await project();
+    await p.api("POST", "/v1/suppressions", { channel: "email", target: "gone@x.com" });
+    await p.api("POST", "/v1/users", { id: "u", email: "gone@x.com" });
+    await p.api("POST", "/v1/users", { id: "control", email: "control@x.com" });
+
+    await notify(p.api, { user: "u", template: "digest", aiPrompts: { summary: "hi" } });
+    await notify(p.api, { user: "control", template: "digest", aiPrompts: { summary: "hi" } });
+
+    await waitFor("control delivered", () => email.for(p.id).length === 1);
+    await settle(1_000);
+    expect(email.for(p.id).map((t) => t.destination)).toEqual(["control@x.com"]);
+  });
+
+  it("sends once when an AI task is replayed after a crash lost its completion marker", async () => {
+    const p = await project();
+    await p.api("POST", "/v1/users", { id: "u", email: "u@x.com" });
+    await notify(p.api, { user: "u", template: "digest", aiPrompts: { summary: "once" } });
+    await waitFor("delivered", () => email.for(p.id).length === 1);
+
+    // A crash between dispatching and recording completion: the marker never
+    // landed, and the pending entry is replayed.
+    const [entry] = (await infra.redis.xrange(STREAMS.AI_PENDING, "-", "+")).filter(([, f]) =>
+      f[1]!.includes(p.id),
+    );
+    for (const key of await infra.redis.keys("notif:processed:ai:*")) await infra.redis.del(key);
+    await infra.redis.xadd(STREAMS.AI_PENDING, "*", "data", entry![1][1]!);
+
+    await settle(3_000);
+    expect(email.for(p.id)).toHaveLength(1);
+  });
+
+  it("fails a notification once, without retrying, when the model rejects the request", async () => {
+    const p = await project();
+    await p.api("POST", "/v1/users", { id: "u", email: "u@x.com" });
+    const failures: unknown[][] = [];
+    const on = (...a: unknown[]) => failures.push(a);
+    app.server.on("notification:failed", on);
+    try {
+      const marker = `reject-${randomUUID().slice(0, 8)}`;
+      failNext = new APICallError({
+        message: "invalid prompt",
+        url: "https://model",
+        requestBodyValues: {},
+        statusCode: 400,
+      });
+      await notify(p.api, { user: "u", template: "digest", aiPrompts: { summary: marker } });
+
+      await waitFor("failure reported", () => failures.length > 0);
+      await settle(1_000);
+      expect(prompts.filter((pr) => pr === marker)).toHaveLength(1);
+      expect(email.for(p.id)).toEqual([]);
+    } finally {
+      app.server.off("notification:failed", on);
+    }
+  });
+
+  it("caps the number of prompts run for one notification", async () => {
+    const p = await project();
+    await p.api("POST", "/v1/users", { id: "u", email: "u@x.com" });
+    const tag = randomUUID().slice(0, 8);
+    const aiPrompts = Object.fromEntries(
+      Array.from({ length: 8 }, (_, i) => [`k${i}`, `${tag}-${i}`]),
+    );
+    await notify(p.api, { user: "u", template: "digest", aiPrompts });
+    await waitFor("delivered", () => email.for(p.id).length === 1);
+    expect(prompts.filter((pr) => pr.startsWith(tag))).toHaveLength(5);
+  });
+
+  it("holds a scheduled AI notification until its time", async () => {
+    const p = await project();
+    await p.api("POST", "/v1/users", { id: "u", email: "u@x.com" });
+    await notify(p.api, {
+      user: "u",
+      template: "digest",
+      aiPrompts: { summary: "later" },
+      sendAt: new Date(Date.now() + 3_000).toISOString(),
+    });
+    await waitFor(
+      "parked",
+      async () => (await p.api("GET", "/v1/notifications/scheduled")).body.scheduled?.length === 1,
+    );
+    expect(email.for(p.id)).toEqual([]);
+    await waitFor("sent", () => email.for(p.id).length === 1, 20_000);
+  }, 30_000);
+});

tests/integration/delivery-worker.test.ts

@@ -0,0 +1,462 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { randomUUID } from "node:crypto";
+import { DeliveryWorker } from "@/services/delivery/main.js";
+import { StreamConsumer, PendingMessageScanner, StreamProducer } from "@/queue/index.js";
+import { IdempotencyGuard } from "@/idempotency/index.js";
+import { ContactRepository, TemplateRepository, UserRepository } from "@/repositories/index.js";
+import { TemplateCache } from "@/templates/index.js";
+import { transportRegistry, type Transport, type DeliveryResult } from "@/transport/index.js";
+import { STREAMS, CONSUMER_GROUPS, OUTBOUND_STREAMS, buildStreamEvent } from "@/contracts/index.js";
+import { createPriorityProducers } from "@/workers/bootstrap.js";
+import { globalEmitter } from "@/shared/index.js";
+import { createLogger } from "@/logger/index.js";
+import { useInfra, waitFor, settle } from "./support/infra.js";
+
+/**
+ * The delivery worker on its own: real streams, real idempotency leases,
+ * real Postgres for the outbox and contacts, and transports that behave like
+ * providers do when they are slow, down, or rejecting.
+ */
+
+const infra = useInfra();
+const logger = createLogger({ name: "test", level: "silent" });
+let project: string;
+const workers: DeliveryWorker[] = [];
+
+beforeEach(async () => {
+  await infra.reset();
+  project = randomUUID();
+  await infra.sql`INSERT INTO projects (id, name) VALUES (${project}, 'p')`;
+});
+
+afterEach(async () => {
+  await Promise.all(workers.splice(0).map((w) => w.stop()));
+});
+
+class FakeProvider implements Transport {
+  calls = 0;
+  constructor(
+    readonly channel: any,
+    private readonly behaviour: (n: number) => Promise<DeliveryResult> | DeliveryResult,
+    readonly limits?: { limit: number; windowSeconds: number },
+  ) {}
+  async send(): Promise<DeliveryResult> {
+    return this.behaviour(++this.calls);
+  }
+}
+
+function registry(...transports: Transport[]) {
+  const r = new (transportRegistry.constructor as any)();
+  transports.forEach((t, i) => r.register(t, transports.length - i));
+  return r;
+}
+
+function worker(reg: any, name = "d1") {
+  const consumer = new StreamConsumer({
+    redis: infra.redis,
+    stream: [...OUTBOUND_STREAMS],
+    group: CONSUMER_GROUPS.DELIVERY,
+    consumer: name,
+    dlqStream: STREAMS.DEAD_LETTER,
+    blockMs: 100,
+    logger,
+  });
+  const pendingScanner = new PendingMessageScanner({
+    redis: infra.redis,
+    stream: [...OUTBOUND_STREAMS],
+    group: CONSUMER_GROUPS.DELIVERY,
+    consumer: name,
+    logger,
+  });
+  const w = new DeliveryWorker({
+    consumer,
+    pendingScanner,
+    logger,
+    concurrency: 10,
+    transportRegistry: reg,
+    idempotency: new IdempotencyGuard({
+      redis: infra.redis,
+      keyPrefix: "notif:processed:delivery",
+    }),
+    redis: infra.redis,
+    scheduledProducer: new StreamProducer({
+      redis: infra.redis,
+      stream: STREAMS.SCHEDULED,
+      logger,
+      maxLen: 1000,
+    }),
+    enrichedProducers: createPriorityProducers(infra.redis, logger, "ENRICHED"),
+    contactRepo: new ContactRepository(infra.db),
+    eventsProducer: new StreamProducer({
+      redis: infra.redis,
+      stream: STREAMS.EVENTS_INBOUND,
+      logger,
+      maxLen: 1000,
+    }),
+    globalEmitter,
+    db: infra.db,
+    templateCache: new TemplateCache(new TemplateRepository(infra.db)),
+  });
+  workers.push(w);
+  return w;
+}
+
+function dispatched(overrides: Record<string, unknown> = {}) {
+  return {
+    ...buildStreamEvent(
+      "notification.dispatched",
+      {
+        projectId: project,
+        taskId: `task-${randomUUID()}`,
+        enrichedEventId: randomUUID(),
+        recipientId: "u",
+        channel: "email",
+        priority: "normal",
+        templateVariables: {},
+        renderedContent: { content: { subject: "s" } },
+        destination: "u@x.com",
+        recipient: {
+          id: "u",
+          email: "u@x.com",
+          locale: "en",
+          timezone: "UTC",
+          preferences: { optedOut: false, channels: [] },
+        },
+        deliveryOptions: { maxAttempts: 3, timeoutMs: 500 },
+        ...overrides,
+      },
+      "engine",
+    ),
+    id: randomUUID(),
+    timestamp: new Date().toISOString(),
+  };
+}
+
+async function publish(ev: any) {
+  await infra.redis.xadd(STREAMS.OUTBOUND_NORMAL, "*", "data", JSON.stringify(ev));
+}
+
+const events = async () =>
+  (await infra.redis.xrange(STREAMS.EVENTS_INBOUND, "-", "+")).map(([, f]) => JSON.parse(f[1]!));
+const parked = async () =>
+  (await infra.sql`SELECT task_id, payload FROM scheduled_payloads ORDER BY task_id`).map(
+    (r) => r.payload as any,
+  );
+const dlq = async () =>
+  (await infra.redis.xrange(STREAMS.DEAD_LETTER, "-", "+")).map(([, f]) => JSON.parse(f[1]!));
+
+describe("DeliveryWorker", () => {
+  it("parks a send the provider timed out on for a later attempt, instead of failing it", async () => {
+    const hang = new FakeProvider("email", () => new Promise(() => {}));
+    const w = worker(registry(hang));
+    const ev = dispatched();
+    await w.processInline(ev as any);
+    await w.stop();
+
+    expect(await dlq()).toEqual([]);
+    expect((await events()).some((e) => e.type === "notification.failed")).toBe(false);
+    const [retry] = await parked();
+    expect(retry).toMatchObject({ taskId: ev.payload.taskId, deliveryAttemptCount: 1 });
+    // Backed off by at least the circuit breaker's reset time.
+    expect(Date.parse(retry.scheduledAt)).toBeGreaterThanOrEqual(Date.now() + 25_000);
+    // Parked as it arrived: no rendered body or abort signal carried along.
+    expect(retry.signal).toBeUndefined();
+    expect(await infra.redis.xlen(STREAMS.SCHEDULED)).toBe(1);
+    // The attempt is still in the log, as dispatched.
+    expect((await events()).filter((e) => e.type === "notification.dispatched")).toHaveLength(1);
+    // Released, so the re-queued attempt can take the lease.
+    expect(await infra.redis.exists(`notif:processed:delivery:${ev.payload.taskId}`)).toBe(0);
+  });
+
+  it("delivers on a later attempt once the provider is back", async () => {
+    const flaky = new FakeProvider("email", (n) =>
+      n === 1
+        ? { success: false, error: "503 Service Unavailable" }
+        : { success: true, providerMessageId: "late-1" },
+    );
+    const w = worker(registry(flaky));
+    await w.processInline(dispatched() as any);
+
+    // What the scheduler would release when the retry falls due.
+    const [retry] = await parked();
+    await w.processInline({
+      ...buildStreamEvent("notification.dispatched", retry, "scheduler"),
+      id: randomUUID(),
+      timestamp: new Date().toISOString(),
+    } as any);
+    await w.stop();
+
+    expect(flaky.calls).toBe(2);
+    const delivered = (await events()).find((e) => e.type === "notification.delivered");
+    expect(delivered.payload).toMatchObject({ providerMessageId: "late-1", attempt: 2 });
+    expect(await dlq()).toEqual([]);
+  });
+
+  it("fails a provider that is still down on the last attempt, and dead-letters the task", async () => {
+    const hang = new FakeProvider("email", () => new Promise(() => {}));
+    const w = worker(registry(hang));
+    await w.start();
+    const ev = dispatched({ deliveryAttemptCount: 2 });
+    await publish(ev);
+
+    await waitFor("dead-lettered", async () => (await dlq()).length === 1, 5_000);
+    expect((await dlq())[0].dlq.reason).toMatch(/timeout/i);
+    await waitFor("failure event", async () =>
+      (await events()).some((e) => e.type === "notification.failed"),
+    );
+    const failed = (await events()).find((e) => e.type === "notification.failed");
+    expect(failed.payload).toMatchObject({
+      taskId: ev.payload.taskId,
+      failureCode: "provider_error",
+      attempt: 3,
+    });
+    expect(await parked()).toEqual([]);
+    // Released, so a replay from the DLQ can try again.
+    expect(await infra.redis.exists(`notif:processed:delivery:${ev.payload.taskId}`)).toBe(0);
+  });
+
+  it("fails over to the next provider for the channel, in priority order", async () => {
+    const primary = new FakeProvider("email", () => ({
+      success: false,
+      error: "503 from primary",
+    }));
+    const backup = new FakeProvider("email", () => ({
+      success: true,
+      providerMessageId: "backup-1",
+    }));
+    const w = worker(registry(primary, backup));
+    await w.start();
+    const ev = dispatched();
+    await publish(ev);
+
+    await waitFor("delivered", async () =>
+      (await events()).some((e) => e.type === "notification.delivered"),
+    );
+    expect(primary.calls).toBe(1);
+    expect(backup.calls).toBe(1);
+    const [outbox] =
+      await infra.sql`SELECT provider_message_id FROM delivery_outbox WHERE task_id = ${ev.payload.taskId as string}`;
+    await waitFor(
+      "outbox",
+      async () =>
+        (await infra.sql`SELECT 1 FROM delivery_outbox WHERE provider_message_id = 'backup-1'`)
+          .length === 1,
+    );
+    void outbox;
+  });
+
+  it("does not retry a failure the provider says is permanent", async () => {
+    const reject = new FakeProvider("email", () => ({
+      success: false,
+      error: "invalid recipient",
+      retryable: false,
+    }));
+    await worker(registry(reject)).processInline(dispatched() as any);
+    expect(reject.calls).toBe(1);
+    expect(await parked()).toEqual([]);
+    expect((await dlq())[0].dlq.reason).toBe("invalid recipient");
+  });
+
+  it("stops calling a provider that keeps failing once its circuit opens, and parks what it turned away", async () => {
+    const down = new FakeProvider("email", () => ({ success: false, error: "down" }));
+    const w = worker(registry(down));
+    // One at a time: sends already in flight when the circuit opens still go out.
+    for (let i = 0; i < 8; i++) await w.processInline(dispatched() as any);
+
+    expect(down.calls).toBe(5);
+    // An open circuit is an outage, not a verdict on the message: nothing is
+    // dead-lettered, and every task waits for another attempt.
+    expect(await dlq()).toEqual([]);
+    const retries = await parked();
+    expect(retries).toHaveLength(8);
+    expect(retries.every((r) => r.deliveryAttemptCount === 1)).toBe(true);
+  });
+
+  it("keeps a circuit per provider, even for two providers of the same class", async () => {
+    const down = new FakeProvider("email", () => ({ success: false, error: "down" }));
+    const backup = new FakeProvider("email", () => ({ success: true, providerMessageId: "b" }));
+    const w = worker(registry(down, backup));
+    for (let i = 0; i < 8; i++) await w.processInline(dispatched() as any);
+
+    // The backup's successes are not the primary's: its circuit opens after
+    // five failures, and every task still goes out through the backup.
+    expect(down.calls).toBe(5);
+    expect(backup.calls).toBe(8);
+    expect(await dlq()).toEqual([]);
+  });
+
+  it("sends a task once when two workers race for it", async () => {
+    let sends = 0;
+    const slow = new FakeProvider("email", async () => {
+      sends++;
+      await settle(200);
+      return { success: true, providerMessageId: "p" };
+    });
+    const reg = registry(slow);
+    const ev = dispatched();
+    await Promise.all([
+      worker(reg, "a").processInline(ev as any),
+      worker(reg, "b").processInline(ev as any),
+    ]);
+    await worker(reg, "c").processInline(ev as any);
+    expect(sends).toBe(1);
+  });
+
+  it("defers sends over the provider's limit, and fails them once the retries run out", async () => {
+    const limited = new FakeProvider("sms", () => ({ success: true, providerMessageId: "x" }), {
+      limit: 1,
+      windowSeconds: 60,
+    });
+    const w = worker(registry(limited));
+    await w.start();
+    const first = dispatched({ channel: "sms", destination: "+1" });
+    const second = dispatched({ channel: "sms", destination: "+2" });
+    const exhausted = dispatched({ channel: "sms", destination: "+3", throttleAttemptCount: 3 });
+    for (const ev of [first, second, exhausted]) await publish(ev);
+
+    await waitFor("one sent", () => limited.calls === 1);
+    await waitFor("deferred", async () => (await infra.redis.xlen(STREAMS.SCHEDULED)) === 1);
+    const [row] = await infra.sql`SELECT payload FROM scheduled_payloads`;
+    expect(row!.payload.throttleAttemptCount).toBe(1);
+    expect(Date.parse(row!.payload.scheduledAt)).toBeGreaterThan(Date.now());
+
+    await waitFor("exhausted one failed", async () =>
+      (await events()).some(
+        (e) =>
+          e.type === "notification.failed" &&
+          e.payload.failureCode === "provider_throttle_exceeded",
+      ),
+    );
+    expect(limited.calls).toBe(1);
+  });
+
+  describe("provider rate limits are per provider, not per channel", () => {
+    const ok = () => ({ success: true, providerMessageId: "x" });
+
+    it("sends through the backup once the primary's limit is spent", async () => {
+      const primary = new FakeProvider("email", ok, { limit: 1, windowSeconds: 60 });
+      const backup = new FakeProvider("email", ok, { limit: 5, windowSeconds: 60 });
+      const w = worker(registry(primary, backup));
+      for (let i = 0; i < 3; i++) await w.processInline(dispatched() as any);
+
+      expect(primary.calls).toBe(1);
+      expect(backup.calls).toBe(2);
+      expect(await parked()).toEqual([]);
+    });
+
+    it("uses an unlimited backup behind a limited primary", async () => {
+      const primary = new FakeProvider("email", ok, { limit: 1, windowSeconds: 60 });
+      const backup = new FakeProvider("email", ok);
+      const w = worker(registry(primary, backup));
+      for (let i = 0; i < 3; i++) await w.processInline(dispatched() as any);
+
+      expect([primary.calls, backup.calls]).toEqual([1, 2]);
+    });
+
+    it("defers a send only once every provider for the channel is at its limit", async () => {
+      const primary = new FakeProvider("email", ok, { limit: 1, windowSeconds: 60 });
+      const backup = new FakeProvider("email", ok, { limit: 1, windowSeconds: 60 });
+      const w = worker(registry(primary, backup));
+      for (let i = 0; i < 3; i++) await w.processInline(dispatched() as any);
+
+      expect([primary.calls, backup.calls]).toEqual([1, 1]);
+      const deferred = await parked();
+      expect(deferred).toHaveLength(1);
+      expect(deferred[0].throttleAttemptCount).toBe(1);
+    });
+  });
+
+  it("dead-letters a task whose template was deleted after it was dispatched", async () => {
+    const ok = new FakeProvider("email", () => ({ success: true, providerMessageId: "x" }));
+    const w = worker(registry(ok));
+    await w.start();
+    await publish(dispatched({ renderedContent: undefined, templateId: "gone" }));
+    await waitFor("dead-lettered", async () => (await dlq()).length === 1);
+    expect((await dlq())[0].dlq.reason).toMatch(/template "gone" not found/);
+    expect(ok.calls).toBe(0);
+  });
+
+  it("renders an unrendered task from its template at send time", async () => {
+    let seen: any;
+    const capture = new FakeProvider("email", () => ({ success: true, providerMessageId: "x" }));
+    (capture as any).send = async (task: any) => {
+      seen = task.renderedContent;
+      return { success: true, providerMessageId: "x" };
+    };
+    await new TemplateRepository(infra.db).upsertMany(project, [
+      { id: "t", channel: "email", content: { subject: "Hi {{n}}" } },
+    ]);
+    await worker(registry(capture)).processInline(
+      dispatched({
+        renderedContent: undefined,
+        templateId: "t",
+        templateVariables: { n: "Bo" },
+      }) as any,
+    );
+    expect(seen).toEqual({ content: { subject: "Hi Bo" } });
+  });
+
+  it("with no provider for the channel, falls back along the chain or records the failure", async () => {
+    const w = worker(registry());
+    await w.processInline(dispatched({ channel: "sms", fallbackChain: ["email"] }) as any);
+    const enriched = await infra.redis.xrange(STREAMS.ENRICHED_NORMAL, "-", "+");
+    const next = JSON.parse(enriched[0]![1][1]!).payload;
+    expect(next.channel).toBe("email");
+    expect(next.fallbackChain).toBeUndefined();
+
+    await w.processInline(dispatched({ channel: "sms" }) as any);
+    await w.stop();
+    const failed = (await events()).find((e) => e.type === "notification.failed");
+    expect(failed.payload.failureCode).toBe("no_transport");
+  });
+
+  describe("push", () => {
+    async function pushUser(token: string) {
+      await new UserRepository(infra.db).upsertFull(project, {
+        userId: "u",
+        segments: [],
+        preferences: {},
+      });
+      await new ContactRepository(infra.db).upsert(project, "u", "push", token);
+    }
+
+    it("deactivates a dead token and falls back to the next channel", async () => {
+      await pushUser("dead");
+      const push = new FakeProvider("push", () => ({ success: false, invalidToken: true }));
+      await worker(registry(push)).processInline(
+        dispatched({ channel: "push", destination: "dead", fallbackChain: ["email"] }) as any,
+      );
+
+      const [contact] = await new ContactRepository(infra.db).findByUserId(project, "u");
+      expect(contact!.active).toBe(false);
+      expect(await infra.redis.xrange(STREAMS.ENRICHED_NORMAL, "-", "+")).toHaveLength(1);
+    });
+
+    it("retries a push that failed, and dead-letters it once out of attempts", async () => {
+      await pushUser("tok");
+      const push = new FakeProvider("push", () => ({ success: false, error: "FCM unavailable" }));
+      const w = worker(registry(push));
+      await w.processInline(dispatched({ channel: "push", destination: "tok" }) as any);
+      expect(await dlq()).toEqual([]);
+      expect(await parked()).toHaveLength(1);
+
+      await w.processInline(
+        dispatched({ channel: "push", destination: "tok", deliveryAttemptCount: 2 }) as any,
+      );
+      expect((await dlq())[0].dlq.reason).toBe("FCM unavailable");
+      const [contact] = await new ContactRepository(infra.db).findByUserId(project, "u");
+      expect(contact!.active).toBe(true);
+    });
+
+    it("delivers a push and records the provider's id", async () => {
+      const push = new FakeProvider("push", () => ({ success: true, providerMessageId: "fcm-1" }));
+      const w = worker(registry(push));
+      await w.processInline(dispatched({ channel: "push", destination: "tok" }) as any);
+      await w.stop();
+      const delivered = (await events()).find((e) => e.type === "notification.delivered");
+      expect(delivered.payload).toMatchObject({ providerMessageId: "fcm-1", channel: "push" });
+      // The dispatch entry rides on the outcome rather than being a second event.
+      expect(delivered.payload.dispatchedAt).toBeDefined();
+    });
+  });
+});

tests/integration/event-worker.test.ts

@@ -0,0 +1,305 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { randomUUID } from "node:crypto";
+import { EventWorker } from "@/services/events/main.js";
+import { StreamConsumer, PendingMessageScanner, StreamProducer } from "@/queue/index.js";
+import { STREAMS, CONSUMER_GROUPS, buildStreamEvent } from "@/contracts/index.js";
+import { insertMessageLogs, upsertOutboxProviderIds } from "@/db/bulk.js";
+import { createLogger } from "@/logger/index.js";
+import { useInfra, waitFor, settle } from "./support/infra.js";
+
+/**
+ * The events worker against real streams and a real delivery log: outcome
+ * events become log rows, and inbound events wake the workflows waiting on
+ * them.
+ */
+
+const infra = useInfra();
+const logger = createLogger({ name: "test", level: "silent" });
+let project: string;
+let worker: EventWorker | undefined;
+
+beforeEach(async () => {
+  await infra.reset();
+  project = randomUUID();
+});
+
+afterEach(async () => {
+  await worker?.stop();
+  worker = undefined;
+});
+
+/**
+ * The worker is slow to make its first read, as it is on a loaded machine
+ * running the whole suite. A wait that passes before the worker has read
+ * anything then fails here every time, rather than once in a while.
+ */
+const FIRST_READ_DELAY_MS = 300;
+
+async function start({ concurrency = 10 } = {}) {
+  const consumer = new StreamConsumer({
+    redis: infra.redis,
+    stream: STREAMS.EVENTS_INBOUND,
+    group: CONSUMER_GROUPS.EVENTS,
+    consumer: "e1",
+    blockMs: 100,
+    logger,
+  });
+  const read = consumer.readBatch.bind(consumer);
+  const stop = consumer.stop.bind(consumer);
+  let stopped = false;
+  consumer.stop = async () => {
+    stopped = true;
+    await stop();
+  };
+  consumer.readBatch = async function* () {
+    await settle(FIRST_READ_DELAY_MS);
+    if (!stopped) yield* read();
+  };
+  const pendingScanner = new PendingMessageScanner({
+    redis: infra.redis,
+    stream: STREAMS.EVENTS_INBOUND,
+    group: CONSUMER_GROUPS.EVENTS,
+    consumer: "e1",
+    logger,
+  });
+  worker = new EventWorker({
+    consumer,
+    pendingScanner,
+    logger,
+    concurrency,
+    db: infra.db,
+    workflowProducer: new StreamProducer({
+      redis: infra.redis,
+      stream: STREAMS.WORKFLOW_INBOUND,
+      logger,
+      maxLen: 1000,
+    }),
+  });
+  await worker.start();
+  return worker;
+}
+
+async function emit(type: string, payload: Record<string, unknown>) {
+  const ev = {
+    ...buildStreamEvent(type, payload, "test"),
+    id: randomUUID(),
+    timestamp: new Date().toISOString(),
+  };
+  await infra.redis.xadd(STREAMS.EVENTS_INBOUND, "*", "data", JSON.stringify(ev));
+  return ev;
+}
+
+/** The events group's XINFO entry: `lag` is entries not yet read, `pending` read but not acked. */
+async function group(): Promise<Record<string, unknown> | undefined> {
+  const groups = (await infra.redis.xinfo("GROUPS", STREAMS.EVENTS_INBOUND)) as unknown[][];
+  return groups
+    .map((flat) =>
+      Object.fromEntries(flat.flatMap((v, i) => (i % 2 ? [] : [[String(v), flat[i + 1]]]))),
+    )
+    .find((g) => g.name === CONSUMER_GROUPS.EVENTS);
+}
+
+/**
+ * Every entry on the stream has been read by the group and acked. Pending
+ * alone is not enough: it is also 0 before the worker has read anything.
+ */
+const drained = async () => {
+  const g = await group();
+  return g?.lag === 0 && g.pending === 0;
+};
+const resumes = async () =>
+  (await infra.redis.xrange(STREAMS.WORKFLOW_INBOUND, "-", "+")).map(([, f]) => JSON.parse(f[1]!));
+
+describe("EventWorker: delivery log", () => {
+  it("writes a dispatched row and an outcome row, in that order, once each", async () => {
+    await start();
+    const dispatchedAt = new Date(Date.now() - 5_000).toISOString();
+    const outcome = {
+      projectId: project,
+      taskId: "t1",
+      channel: "email",
+      attempt: 1,
+      providerMessageId: "pm-1",
+      dispatchedAt,
+      templateId: "tpl",
+      campaignId: "c",
+    };
+    await emit("notification.delivered", outcome);
+    await emit("notification.delivered", outcome); // a replay
+    await emit("notification.failed", {
+      projectId: project,
+      taskId: "t2",
+      channel: "sms",
+      attempt: 2,
+    });
+    await emit("notification.dispatched", { projectId: project, taskId: "t3", channel: "push" });
+
+    await waitFor("read and acked", drained);
+    const rows =
+      await infra.sql`SELECT task_id, kind, status, attempt, provider_message_id, template_id, campaign_id, "timestamp" FROM message_logs ORDER BY task_id, "timestamp"`;
+    expect(rows.map((r) => [r.task_id, r.kind, r.status, r.attempt])).toEqual([
+      ["t1", "dispatched", "dispatched", 1],
+      ["t1", "attempt", "delivered", 1],
+      ["t2", "attempt", "failed", 2],
+      ["t3", "dispatched", "dispatched", 1],
+    ]);
+    expect(new Date(rows[0]!.timestamp).toISOString()).toBe(dispatchedAt);
+    expect(rows[1]).toMatchObject({
+      provider_message_id: "pm-1",
+      template_id: "tpl",
+      campaign_id: "c",
+    });
+  });
+
+  it("flushes everything buffered when it stops", async () => {
+    // Hold rows in the buffer: a flush threshold out of reach (half of 100)
+    // and no interval flush, so only stopping can write them.
+    const w = await start({ concurrency: 100 });
+    clearInterval((w as any).flushInterval);
+    for (let i = 0; i < 5; i++)
+      await emit("notification.delivered", {
+        projectId: project,
+        taskId: `s${i}`,
+        channel: "email",
+      });
+
+    await waitFor("all five buffered", () => (w as any).messageLogBuffer.length === 5);
+    expect(await infra.sql`SELECT 1 FROM message_logs`).toEqual([]);
+
+    await w.stop();
+    worker = undefined;
+    const [{ n }] = (await infra.sql`SELECT count(*)::int AS n FROM message_logs`) as any;
+    expect(n).toBe(5);
+  });
+
+  it("ignores event types it has no use for", async () => {
+    await start();
+    await emit("something.else", { projectId: project });
+    await waitFor("read and acked", drained);
+    expect(await infra.sql`SELECT 1 FROM message_logs`).toEqual([]);
+  });
+});
+
+describe("EventWorker: waking workflows", () => {
+  async function waitingInstance(
+    eventName: string,
+    match: Record<string, unknown>,
+    expiresInMs = 3600_000,
+  ) {
+    const [inst] = await infra.sql`
+      INSERT INTO workflow_instances (project_id, name, status, input) VALUES (${project}, 'wf', 'pending', ${JSON.stringify({ user: { id: "u" } })}::jsonb) RETURNING id`;
+    const id = inst!.id as string;
+    await infra.sql`INSERT INTO workflow_steps (project_id, instance_id, step_index, action) VALUES (${project}, ${id}, '0', 'waitForEvent')`;
+    await infra.sql`INSERT INTO workflow_waiters (project_id, instance_id, event_name, match_criteria, expires_at)
+      VALUES (${project}, ${id}, ${eventName}, ${JSON.stringify(match)}::jsonb, ${new Date(Date.now() + expiresInMs).toISOString()})`;
+    return id;
+  }
+
+  it("resumes the instance whose criteria match, recording the event on its step", async () => {
+    await start();
+    const alice = await waitingInstance("order.paid", { "customer.id": "alice" });
+    const bob = await waitingInstance("order.paid", { "customer.id": "bob" });
+
+    await emit("event.received", {
+      projectId: project,
+      eventName: "order.paid",
+      payload: { customer: { id: "alice" }, total: 5 },
+    });
+    await waitFor("resumed", async () => (await resumes()).length === 1);
+
+    const [resume] = await resumes();
+    expect(resume.type).toBe("workflow.resumed");
+    expect(resume.payload).toMatchObject({
+      projectId: project,
+      instanceId: alice,
+      name: "wf",
+      input: { user: { id: "u" } },
+      reason: "event_matched",
+    });
+    const [step] = await infra.sql`SELECT output FROM workflow_steps WHERE instance_id = ${alice}`;
+    expect(step!.output).toEqual({ customer: { id: "alice" }, total: 5 });
+    const waiters = await infra.sql`SELECT instance_id FROM workflow_waiters`;
+    expect(waiters.map((w) => w.instance_id)).toEqual([bob]);
+  });
+
+  it("does not wake anything for an event in another project, of another name, or after the waiter expired", async () => {
+    await start();
+    await waitingInstance("order.paid", {});
+    await waitingInstance("expired", {}, -1_000);
+    await emit("event.received", { projectId: randomUUID(), eventName: "order.paid", payload: {} });
+    await emit("event.received", { projectId: project, eventName: "order.refunded", payload: {} });
+    await emit("event.received", { projectId: project, eventName: "expired", payload: {} });
+    await emit("event.received", { eventName: "order.paid", payload: {} });
+
+    await waitFor("read and acked", drained);
+    await settle(200);
+    expect(await resumes()).toEqual([]);
+  });
+
+  it("drops a waiter whose instance no longer exists", async () => {
+    await start();
+    const id = await waitingInstance("ping", {});
+    // Orphan the waiter as a crash between deletes would.
+    await infra.sql`ALTER TABLE workflow_waiters DISABLE TRIGGER ALL`;
+    await infra.sql`DELETE FROM workflow_instances WHERE id = ${id}`.catch(() => {});
+    await infra.sql`ALTER TABLE workflow_waiters ENABLE TRIGGER ALL`;
+    const orphaned = (await infra.sql`SELECT 1 FROM workflow_waiters`).length;
+
+    await emit("event.received", {
+      projectId: project,
+      eventName: "ping",
+      payload: "not an object",
+    });
+    await waitFor("read and acked", drained);
+    expect(await resumes()).toEqual([]);
+    if (orphaned) expect(await infra.sql`SELECT 1 FROM workflow_waiters`).toEqual([]);
+  });
+});
+
+describe("bulk log writers", () => {
+  it("insertMessageLogs writes typed rows and skips ones already recorded", async () => {
+    const row = {
+      projectId: project,
+      taskId: "b1",
+      providerMessageId: null,
+      channel: "email",
+      attempt: 1,
+      kind: "attempt",
+      status: "delivered",
+      templateId: null,
+      workflowInstanceId: null,
+      campaignId: null,
+    };
+    await insertMessageLogs(infra.db, [
+      row,
+      {
+        ...row,
+        taskId: "b2",
+        workflowInstanceId: randomUUID(),
+        timestamp: "2020-01-01T00:00:00.000Z",
+      },
+    ]);
+    await insertMessageLogs(infra.db, [row]);
+    await insertMessageLogs(infra.db, []);
+    const rows = await infra.sql`SELECT task_id, "timestamp" FROM message_logs ORDER BY task_id`;
+    expect(rows.map((r) => r.task_id)).toEqual(["b1", "b2"]);
+    expect(new Date(rows[1]!.timestamp).toISOString()).toBe("2020-01-01T00:00:00.000Z");
+  });
+
+  it("upsertOutboxProviderIds records and updates provider ids, even with a task twice in one batch", async () => {
+    const base = { taskId: "o1", channel: "email", destination: "a@x.com" };
+    await upsertOutboxProviderIds(infra.db, [{ ...base, providerMessageId: null }]);
+    await upsertOutboxProviderIds(infra.db, [
+      { ...base, providerMessageId: "first" },
+      { ...base, providerMessageId: "retry" },
+      { taskId: "o2", channel: "sms", destination: "+1", providerMessageId: "s" },
+    ]);
+    await upsertOutboxProviderIds(infra.db, []);
+    const rows =
+      await infra.sql`SELECT task_id, provider_message_id FROM delivery_outbox ORDER BY task_id`;
+    expect(rows.map((r) => [r.task_id, r.provider_message_id])).toEqual([
+      ["o1", "retry"],
+      ["o2", "s"],
+    ]);
+  });
+});

tests/integration/mcp.test.ts

@@ -0,0 +1,440 @@
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { randomUUID } from "node:crypto";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { createServer } from "../../packages/mcp/src/index.js";
+import { NotifkitApi, NotifkitApiError } from "../../packages/mcp/src/client.js";
+import { STREAMS } from "@/contracts/index.js";
+import { useInfra, waitFor } from "./support/infra.js";
+import { ADMIN_KEY, RecordingTransport, startNotifkit, type Running } from "./support/pipeline.js";
+
+/**
+ * The MCP server as an agent uses it: a real MCP client, connected over the
+ * SDK's in-memory transport, calls tools; the tools call a running notifkit.
+ * Every tool is exercised and judged by its effect, not by the URL it built.
+ */
+
+const infra = useInfra();
+let app: Running;
+const email = new RecordingTransport("email");
+const telegram = new RecordingTransport("telegram");
+const slack = new RecordingTransport("slack");
+
+beforeAll(async () => {
+  app = await startNotifkit(infra, {
+    services: ["api", "enricher", "engine", "delivery", "scheduler", "events", "workflow"],
+    providers: [email, telegram, slack],
+    env: { RATE_LIMIT_PER_HOUR: "100000" },
+  });
+}, 120_000);
+
+afterAll(async () => {
+  await app?.stop();
+}, 60_000);
+
+async function agent(apiKey: string, projectId?: string) {
+  const server = createServer({ baseUrl: app.baseUrl, apiKey, projectId });
+  const client = new Client({ name: "test-agent", version: "1.0.0" });
+  const [a, b] = InMemoryTransport.createLinkedPair();
+  await Promise.all([server.connect(a), client.connect(b)]);
+
+  async function call(name: string, args: Record<string, unknown> = {}) {
+    const res: any = await client.callTool({ name, arguments: args });
+    const text = res.content[0].text as string;
+    let body: any = text;
+    try {
+      body = JSON.parse(text);
+    } catch {
+      // plain text result
+    }
+    return { isError: Boolean(res.isError), text, body };
+  }
+  return { client, call, close: () => client.close() };
+}
+
+async function projectAgent() {
+  const p = await app.project();
+  return { ...p, ...(await agent(p.apiKey)) };
+}
+
+describe("MCP server", () => {
+  it("lists all of its tools with input schemas", async () => {
+    const a = await projectAgent();
+    const { tools } = await a.client.listTools();
+    expect(tools).toHaveLength(46);
+    for (const t of tools) expect(t.inputSchema.type).toBe("object");
+    await a.close();
+  });
+
+  it("sends to existing users and reports the campaign", async () => {
+    const a = await projectAgent();
+    expect(
+      (
+        await a.call("upsert_template", {
+          id: "hi",
+          channel: "email",
+          content: { subject: "Hi {{name}}" },
+        })
+      ).isError,
+    ).toBe(false);
+    expect(
+      (await a.call("upsert_user", { id: "u1", email: "u1@x.com", segments: ["beta"] })).isError,
+    ).toBe(false);
+
+    const sent = await a.call("send_notification", {
+      template: "hi",
+      user: "u1",
+      data: { name: "Ann" },
+      campaign: "c-1",
+    });
+    expect(sent.isError).toBe(false);
+    await waitFor("delivered", () => email.for(a.id).length === 1);
+    expect(email.for(a.id)[0]!.renderedContent.content).toEqual({ subject: "Hi Ann" });
+
+    await waitFor(
+      "stats",
+      async () =>
+        (await a.call("get_campaign_stats", { campaign: "c-1" })).body?.totals?.delivered === 1,
+    );
+    const campaigns = await a.call("list_campaigns", {});
+    expect(campaigns.body.campaigns.map((c: any) => c.campaign)).toEqual(["c-1"]);
+    await a.close();
+  });
+
+  it("send_campaign reaches every distinct address once, on email and on chat channels", async () => {
+    const a = await projectAgent();
+    await a.call("upsert_template", {
+      id: "promo",
+      channel: "email",
+      content: { subject: "Sale" },
+    });
+    await a.call("upsert_template", { id: "ping", channel: "telegram", content: { text: "Ping" } });
+    await a.call("upsert_template", { id: "note", channel: "slack", content: { text: "Note" } });
+
+    const res = await a.call("send_campaign", {
+      campaign: "spring",
+      template: "promo",
+      emails: ["A@x.com", "a@x.com", "b@x.com"],
+    });
+    expect(res.body).toMatchObject({ queued: 2, duplicatesRemoved: 1 });
+    await waitFor("emails", () => email.for(a.id).length === 2);
+    expect(
+      email
+        .for(a.id)
+        .map((t) => t.destination)
+        .sort(),
+    ).toEqual(["a@x.com", "b@x.com"]);
+
+    await a.call("send_campaign", {
+      campaign: "chat",
+      template: "ping",
+      channel: "telegram",
+      recipients: ["-1001", "-1002"],
+    });
+    await waitFor("telegram", () => telegram.for(a.id).length === 2, 10_000);
+    expect(
+      telegram
+        .for(a.id)
+        .map((t) => t.destination)
+        .sort(),
+    ).toEqual(["-1001", "-1002"]);
+
+    await a.call("send_campaign", {
+      campaign: "room",
+      template: "note",
+      channel: "slack",
+      recipients: ["C123"],
+    });
+    await waitFor("slack", () => slack.for(a.id).length === 1, 10_000);
+
+    // A user given the same address twice, once as `email` and once as a contact.
+    const both = await a.call("upsert_user", {
+      id: "twice",
+      email: "t@x.com",
+      contacts: [{ channel: "email", target: "t@x.com" }],
+    });
+    expect(both.isError).toBe(false);
+
+    expect((await a.call("send_campaign", { campaign: "x", template: "promo" })).isError).toBe(
+      true,
+    );
+    await a.close();
+  });
+
+  it("previews a template exactly as the pipeline renders it", async () => {
+    const a = await projectAgent();
+    const content = {
+      subject: "Re: {{topic}}",
+      html: "<p>{{body}}</p>",
+      text: "{{body}} / {{{body}}}",
+    };
+    await a.call("upsert_template", { id: "rich", channel: "email", content });
+    await a.call("upsert_user", { id: "u", email: "u@x.com" });
+    const data = { topic: "a\r\nBcc: evil", body: "<b>bold</b>" };
+
+    const preview = await a.call("preview_template", { id: "rich", data: { ...data, extra: 1 } });
+    await a.call("send_notification", { template: "rich", user: "u", data });
+    await waitFor("sent", () => email.for(a.id).length === 1);
+
+    expect(preview.body.rendered).toEqual(email.for(a.id)[0]!.renderedContent.content);
+    expect(preview.body).toMatchObject({ templateId: "rich", channel: "email" });
+    expect(preview.body.resolvedVariables.sort()).toEqual(["body", "topic"]);
+
+    const raw = await a.call("render_template", { content: { text: "{{missing}}!" } });
+    expect(raw.body).toMatchObject({ rendered: { text: "!" }, unresolvedVariables: ["missing"] });
+    expect((await a.call("preview_template", {})).isError).toBe(true);
+    await a.close();
+  });
+
+  it("manages templates", async () => {
+    const a = await projectAgent();
+    await a.call("upsert_template", {
+      id: "t1",
+      channel: "email",
+      topic: "news",
+      content: { subject: "s" },
+    });
+    await a.call("upsert_template", { id: "t2", channel: "sms", content: { text: "x" } });
+    expect(
+      (await a.call("list_templates", { channel: "sms" })).body.templates.map((t: any) => t.id),
+    ).toEqual(["t2"]);
+    expect(
+      (await a.call("list_templates", { topic: "news" })).body.templates.map((t: any) => t.id),
+    ).toEqual(["t1"]);
+    expect((await a.call("get_template", { id: "t1" })).body.topics).toEqual(["news"]);
+    expect((await a.call("delete_template", { id: "t1" })).isError).toBe(false);
+    const gone = await a.call("get_template", { id: "t1" });
+    expect(gone.isError).toBe(true);
+    expect(gone.text).toContain("HTTP 404");
+    await a.close();
+  });
+
+  it("manages users, contacts and preferences", async () => {
+    const a = await projectAgent();
+    await a.call("upsert_user", {
+      id: "u",
+      email: "u@x.com",
+      language: "de",
+      segments: ["vip"],
+      preferences: { topics: { news: true } },
+    });
+    await a.call("update_user", { id: "u", timezone: "Asia/Tokyo" });
+    await a.call("add_user_contact", { userId: "u", channel: "sms", target: "+1555" });
+
+    const user = (await a.call("get_user", { id: "u" })).body;
+    expect(user).toMatchObject({
+      userId: "u",
+      language: "de",
+      timezone: "Asia/Tokyo",
+      segments: ["vip"],
+    });
+    expect(
+      (await a.call("get_user_contacts", { userId: "u" })).body.contacts
+        .map((c: any) => c.target)
+        .sort(),
+    ).toEqual(["+1555", "u@x.com"]);
+    expect(
+      (await a.call("list_users", { segment: "vip" })).body.users.map((u: any) => u.userId),
+    ).toEqual(["u"]);
+    expect((await a.call("list_segments")).body.segments).toEqual(["vip"]);
+
+    await a.call("update_user_preferences", {
+      userId: "u",
+      topics: { news: false },
+      quietHours: [{ start: "22:00", end: "06:00" }],
+    });
+    expect((await a.call("get_user_preferences", { userId: "u" })).body).toMatchObject({
+      topics: { news: false },
+      quietHours: [{ start: "22:00", end: "06:00" }],
+    });
+
+    await a.call("delete_user_contact", { userId: "u", channel: "sms", target: "+1555" });
+    expect((await a.call("get_user_contacts", { userId: "u" })).body.contacts).toHaveLength(1);
+    await a.call("delete_user", { id: "u" });
+    expect((await a.call("get_user", { id: "u" })).isError).toBe(true);
+    await a.close();
+  });
+
+  it("manages suppressions", async () => {
+    const a = await projectAgent();
+    await a.call("suppress_address", {
+      channel: "email",
+      target: "Stop@X.com",
+      reason: "complained",
+    });
+    expect(
+      (await a.call("list_suppressions", { channel: "email" })).body.suppressions.map(
+        (s: any) => s.target,
+      ),
+    ).toEqual(["stop@x.com"]);
+    await a.call("unsuppress_address", { channel: "email", target: "stop@x.com" });
+    expect((await a.call("list_suppressions", {})).body.suppressions).toEqual([]);
+    await a.close();
+  });
+
+  it("lists, inspects and cancels scheduled sends, and reads delivery logs", async () => {
+    const a = await projectAgent();
+    await a.call("upsert_template", { id: "t", channel: "email", content: { subject: "s" } });
+    await a.call("upsert_user", { id: "u", email: "u@x.com" });
+    await a.call("send_notification", {
+      template: "t",
+      user: "u",
+      sendAt: new Date(Date.now() + 3600_000).toISOString(),
+    });
+    await a.call("send_notification", { template: "t", user: "u" });
+
+    let scheduled: any[] = [];
+    await waitFor("parked", async () => {
+      scheduled = (await a.call("list_scheduled", {})).body.scheduled ?? [];
+      return scheduled.length === 1;
+    });
+    expect((await a.call("cancel_notification", { taskId: scheduled[0].taskId })).isError).toBe(
+      false,
+    );
+    expect((await a.call("list_scheduled", {})).body.scheduled).toEqual([]);
+
+    await waitFor(
+      "logged",
+      async () =>
+        (await a.call("get_delivery_logs", { status: "delivered" })).body.logs?.length === 1,
+    );
+    const log = (await a.call("get_delivery_logs", { status: "delivered" })).body.logs[0];
+    expect((await a.call("get_notification", { taskId: log.taskId })).body.status).toBe(
+      "delivered",
+    );
+    await a.close();
+  });
+
+  it("defines, runs, inspects and cancels workflows, and ingests events", async () => {
+    const a = await projectAgent();
+    await a.call("upsert_template", { id: "t", channel: "email", content: { subject: "s" } });
+    await a.call("upsert_user", { id: "u", email: "u@x.com" });
+    await a.call("create_workflow", {
+      name: "drip",
+      steps: [
+        { action: "notify", payload: { template: "t" } },
+        { action: "waitForEvent", event: "clicked", options: { timeout: "1h" } },
+      ],
+    });
+    expect(
+      (await a.call("list_workflows", { search: "dri" })).body.workflows.map((w: any) => w.name),
+    ).toEqual(["drip"]);
+
+    const run = (await a.call("trigger_workflow", { name: "drip", user: "u" })).body;
+    await waitFor(
+      "waiting",
+      async () =>
+        (await a.call("get_workflow_run", { instanceId: run.instanceId })).body.status ===
+        "pending",
+    );
+    await waitFor("first step sent", () => email.for(a.id).length === 1);
+
+    expect((await a.call("ingest_event", { name: "unrelated", properties: {} })).isError).toBe(
+      false,
+    );
+    expect((await a.call("cancel_workflow_run", { instanceId: run.instanceId })).isError).toBe(
+      false,
+    );
+    expect((await a.call("get_workflow_run", { instanceId: run.instanceId })).body.status).toBe(
+      "canceled",
+    );
+    await a.close();
+  });
+
+  it("reports system health, metrics and dead letters", async () => {
+    const a = await projectAgent();
+    expect((await a.call("get_system_health")).body.status).toBe("healthy");
+    expect((await a.call("get_system_metrics")).body.deliveryStats).toBeDefined();
+
+    const ev = {
+      id: randomUUID(),
+      type: "notification.dispatched",
+      timestamp: new Date().toISOString(),
+      payload: { projectId: a.id },
+      metadata: { traceId: "t", source: "s", retryCount: 0 },
+      dlq: {
+        originalStream: STREAMS.OUTBOUND_NORMAL,
+        ackedAt: new Date().toISOString(),
+        reason: "x",
+      },
+    };
+    const id1 = await infra.redis.xadd(STREAMS.DEAD_LETTER, "*", "data", JSON.stringify(ev));
+    const id2 = await infra.redis.xadd(
+      STREAMS.DEAD_LETTER,
+      "*",
+      "data",
+      JSON.stringify({ ...ev, id: randomUUID() }),
+    );
+    const dead = (await a.call("get_dead_letters")).body.messages.map((m: any) => m.id);
+    expect(dead).toEqual(expect.arrayContaining([id1, id2]));
+
+    expect((await a.call("delete_dead_letter", { id: id1 })).isError).toBe(false);
+    expect((await a.call("replay_dead_letter", { id: id2 })).body).toMatchObject({
+      success: true,
+      stream: STREAMS.OUTBOUND_NORMAL,
+    });
+    await a.close();
+  });
+
+  it("administers projects and keys with the admin key", async () => {
+    const admin = await agent(ADMIN_KEY);
+    const created = (await admin.call("create_project", { name: "Via MCP" })).body;
+    expect(created.apiKey).toMatch(/^nk_live_/);
+    expect((await admin.call("list_projects")).body.projects.map((p: any) => p.id)).toContain(
+      created.id,
+    );
+    await admin.call("update_project", { id: created.id, throttleLimit: 3 });
+
+    const key = (
+      await admin.call("create_project_key", { projectId: created.id, role: "read_only" })
+    ).body;
+    expect(
+      (await admin.call("list_project_keys", { projectId: created.id })).body.keys,
+    ).toHaveLength(2);
+    expect(
+      (await admin.call("delete_project_key", { projectId: created.id, keyId: key.id })).isError,
+    ).toBe(false);
+    expect((await admin.call("delete_project", { id: created.id })).isError).toBe(false);
+    expect((await admin.call("list_projects")).body.projects.map((p: any) => p.id)).not.toContain(
+      created.id,
+    );
+    await admin.close();
+  });
+
+  it("reports an API failure to the model instead of crashing", async () => {
+    const bad = await agent("nk_live_not_a_key");
+    const res = await bad.call("list_templates", {});
+    expect(res.isError).toBe(true);
+    expect(res.text).toContain("HTTP 401");
+    await bad.close();
+  });
+});
+
+describe("NotifkitApi (the MCP server's HTTP client)", () => {
+  it("talks to a real server: query strings, bodies, 204s and errors", async () => {
+    const p = await app.project();
+    const api = new NotifkitApi({ baseUrl: `${app.baseUrl}/`, apiKey: p.apiKey });
+    await api.put("/v1/templates", {
+      templates: [
+        { id: "a", channel: "email", content: {} },
+        { id: "b", channel: "sms", content: {} },
+      ],
+    });
+    expect(
+      (
+        await api.get<any>("/v1/templates", { channel: "sms", limit: 5, topic: undefined })
+      ).templates.map((t: any) => t.id),
+    ).toEqual(["b"]);
+    await api.post("/v1/users", { id: "u" });
+    await api.patch("/v1/users/u", { language: "fr" });
+    expect(await api.delete("/v1/templates/a")).toBeUndefined();
+
+    const err: any = await api.get("/v1/templates/a").catch((e) => e);
+    expect(err).toBeInstanceOf(NotifkitApiError);
+    expect(err.status).toBe(404);
+    expect(err.body).toMatchObject({ error: "template_not_found" });
+
+    const admin = new NotifkitApi({ baseUrl: app.baseUrl, apiKey: ADMIN_KEY, projectId: p.id });
+    expect((await admin.get<any>("/v1/users/u")).language).toBe("fr");
+  });
+});

tests/integration/pipeline-fused.test.ts

@@ -0,0 +1,3 @@
+import { definePipelineSuite } from "./support/pipeline-cases.js";
+
+definePipelineSuite({ fused: true });

tests/integration/pipeline-streamed.test.ts

@@ -0,0 +1,3 @@
+import { definePipelineSuite } from "./support/pipeline-cases.js";
+
+definePipelineSuite({ fused: false });

tests/integration/providers.test.ts

@@ -0,0 +1,646 @@
+import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest";
+import { createHmac, randomBytes, randomUUID } from "node:crypto";
+import { TelegramTransport } from "../../packages/provider-telegram/src/index.js";
+import { DiscordTransport } from "../../packages/provider-discord/src/index.js";
+import { SlackTransport } from "../../packages/provider-slack/src/index.js";
+import { WhatsAppTransport } from "../../packages/provider-whatsapp/src/index.js";
+import { ResendTransport } from "../../packages/provider-resend/src/index.js";
+import { TwilioTransport } from "../../packages/provider-twilio/src/index.js";
+import { ConsoleTransport } from "../../packages/provider-console/src/index.js";
+import type { NotificationDispatchedPayload } from "@/contracts/index.js";
+import { startFakeInternet, type FakeInternet } from "./support/fake-internet.js";
+import { useInfra, waitFor } from "./support/infra.js";
+import { startNotifkit, apiClient, type Running } from "./support/pipeline.js";
+
+/**
+ * Each provider sends through its real code — real fetch, real request
+ * bodies, real status handling — against a local stand-in for the vendor's
+ * API. Webhook signatures are computed the way each vendor documents them,
+ * not by the code under test.
+ */
+
+let net: FakeInternet;
+beforeAll(async () => {
+  net = await startFakeInternet();
+});
+afterAll(async () => {
+  await net?.close();
+});
+beforeEach(() => {
+  net.requests.length = 0;
+});
+
+function task(
+  overrides: Partial<NotificationDispatchedPayload> & { content?: Record<string, unknown> } = {},
+): NotificationDispatchedPayload {
+  const { content, ...rest } = overrides;
+  return {
+    projectId: randomUUID(),
+    taskId: `task-${randomUUID()}`,
+    enrichedEventId: randomUUID(),
+    recipientId: "u1",
+    channel: "email",
+    priority: "normal",
+    templateVariables: {},
+    deliveryOptions: { maxAttempts: 3, timeoutMs: 2_000 },
+    destination: "dest",
+    renderedContent: { content: content ?? { subject: "Hello", text: "Body" } },
+    ...rest,
+  } as NotificationDispatchedPayload;
+}
+
+// ─── Telegram ───────────────────────────────────────────────────────────────
+
+describe("TelegramTransport", () => {
+  const t = new TelegramTransport({ botToken: "123:abc", parseMode: "HTML" });
+
+  it("posts the message to the bot API and returns its message id", async () => {
+    net.on("api.telegram.org", () => ({ json: { ok: true, result: { message_id: 77 } } }));
+    const r = await t.send(
+      task({
+        destination: "-100200",
+        content: { subject: "Alert", text: "Disk full", disableNotification: true },
+      }),
+    );
+
+    expect(r).toEqual({ success: true, providerMessageId: "77" });
+    const [req] = net.requests;
+    expect(req!.method).toBe("POST");
+    expect(req!.path).toBe("/bot123:abc/sendMessage");
+    expect(req!.json()).toEqual({
+      chat_id: "-100200",
+      text: "Alert\n\nDisk full",
+      parse_mode: "HTML",
+      disable_notification: true,
+    });
+  });
+
+  it("sends a subject-only template as just the subject", async () => {
+    net.on("api.telegram.org", () => ({ json: { ok: true, result: { message_id: 1 } } }));
+    await t.send(task({ content: { subject: "Only a subject" } }));
+    expect(net.requests[0]!.json().text).toBe("Only a subject");
+  });
+
+  it("marks a blocked bot or vanished chat as a dead destination", async () => {
+    net.on("api.telegram.org", () => ({
+      status: 403,
+      json: { ok: false, error_code: 403, description: "Forbidden: bot was blocked by the user" },
+    }));
+    expect(await t.send(task())).toMatchObject({
+      success: false,
+      invalidToken: true,
+      error: "Forbidden: bot was blocked by the user",
+    });
+
+    net.on("api.telegram.org", () => ({
+      status: 400,
+      json: { ok: false, error_code: 400, description: "Bad Request: chat not found" },
+    }));
+    expect((await t.send(task())).invalidToken).toBe(true);
+  });
+
+  it("treats a rate limit as retryable, not as a dead destination", async () => {
+    net.on("api.telegram.org", () => ({
+      status: 429,
+      json: { ok: false, error_code: 429, description: "Too Many Requests: retry after 5" },
+    }));
+    expect(await t.send(task())).toMatchObject({ success: false, invalidToken: false });
+  });
+
+  it("reports a network failure or a non-JSON gateway page as a failed send", async () => {
+    net.on("api.telegram.org", () => "hang-up");
+    expect((await t.send(task())).success).toBe(false);
+    net.on("api.telegram.org", () => ({ status: 502, text: "<html>Bad Gateway</html>" }));
+    expect((await t.send(task())).success).toBe(false);
+  });
+
+  it("refuses a task with no chat id without calling the API", async () => {
+    expect((await t.send(task({ destination: undefined }))).success).toBe(false);
+    expect(net.requests).toEqual([]);
+  });
+});
+
+// ─── Discord ────────────────────────────────────────────────────────────────
+
+describe("DiscordTransport", () => {
+  const d = new DiscordTransport({ username: "Alerts" });
+  const hook = "https://discord.com/api/webhooks/1/abc";
+
+  it("posts to the webhook, waiting for the created message's id", async () => {
+    net.on("discord.com", () => ({ json: { id: "msg-9" } }));
+    const r = await d.send(
+      task({
+        destination: hook,
+        content: { subject: "Deploy", text: "done", avatarUrl: "https://x/a.png" },
+      }),
+    );
+    expect(r).toEqual({ success: true, providerMessageId: "msg-9" });
+    expect(net.requests[0]!.path).toBe("/api/webhooks/1/abc?wait=true");
+    expect(net.requests[0]!.json()).toEqual({
+      content: "**Deploy**\ndone",
+      username: "Alerts",
+      avatar_url: "https://x/a.png",
+    });
+  });
+
+  it("sends a subject-only template as just the subject", async () => {
+    net.on("discord.com", () => ({ json: { id: "m" } }));
+    await d.send(task({ destination: hook, content: { subject: "Heads up" } }));
+    expect(net.requests[0]!.json().content).toBe("**Heads up**");
+  });
+
+  it("refuses to post anywhere but Discord", async () => {
+    for (const bad of [
+      "http://discord.com/api/webhooks/1/a",
+      "https://evil.com/api/webhooks/1/a",
+      "https://discord.com/other/1",
+    ]) {
+      expect(await d.send(task({ destination: bad }))).toMatchObject({
+        success: false,
+        invalidToken: true,
+      });
+    }
+    expect(net.requests).toEqual([]);
+  });
+
+  it("marks a deleted webhook dead and a server error retryable", async () => {
+    net.on("discord.com", () => ({
+      status: 404,
+      json: { message: "Unknown Webhook", code: 10015 },
+    }));
+    expect(await d.send(task({ destination: hook }))).toMatchObject({
+      success: false,
+      invalidToken: true,
+    });
+    net.on("discord.com", () => ({ status: 500, text: "oops" }));
+    expect(await d.send(task({ destination: hook }))).toMatchObject({
+      success: false,
+      invalidToken: false,
+    });
+  });
+});
+
+// ─── Slack ──────────────────────────────────────────────────────────────────
+
+describe("SlackTransport", () => {
+  it("posts to chat.postMessage with the bot token", async () => {
+    net.on("slack.com", () => ({ json: { ok: true, channel: "C1", ts: "1700.1" } }));
+    const s = new SlackTransport({ botToken: "xoxb-1" });
+    const r = await s.send(
+      task({ destination: "C1", content: { text: "hi", blocks: [{ type: "divider" }] } }),
+    );
+    expect(r).toEqual({ success: true, providerMessageId: "C1:1700.1" });
+    const req = net.requests[0]!;
+    expect(req.path).toBe("/api/chat.postMessage");
+    expect(req.headers.authorization).toBe("Bearer xoxb-1");
+    expect(req.json()).toEqual({ channel: "C1", text: "hi", blocks: [{ type: "divider" }] });
+  });
+
+  it("uses a template's own channel over the recipient's", async () => {
+    net.on("slack.com", () => ({ json: { ok: true, ts: "1" } }));
+    await new SlackTransport({ token: "xoxb" }).send(
+      task({ destination: "D-user", content: { channel: "C-alerts", subject: "S" } }),
+    );
+    expect(net.requests[0]!.json()).toMatchObject({ channel: "C-alerts", text: "S" });
+  });
+
+  it("marks an archived channel dead, but not a rate limit", async () => {
+    const s = new SlackTransport({ botToken: "xoxb" });
+    net.on("slack.com", () => ({ json: { ok: false, error: "is_archived" } }));
+    expect(await s.send(task({ destination: "C" }))).toMatchObject({
+      success: false,
+      invalidToken: true,
+    });
+    net.on("slack.com", () => ({ json: { ok: false, error: "ratelimited" } }));
+    expect(await s.send(task({ destination: "C" }))).toMatchObject({
+      success: false,
+      invalidToken: false,
+    });
+  });
+
+  it("posts to an incoming webhook, and reads its plain-text errors", async () => {
+    const s = new SlackTransport({ webhookUrl: "https://hooks.slack.com/services/T/B/x" });
+    net.on("hooks.slack.com", () => ({ text: "ok" }));
+    expect(
+      await s.send(task({ destination: undefined, content: { text: "hello" } })),
+    ).toMatchObject({ success: true });
+    expect(net.requests[0]!.json()).toEqual({ text: "hello" });
+
+    net.on("hooks.slack.com", () => ({ status: 404, text: "channel_not_found" }));
+    expect(await s.send(task({ destination: undefined }))).toMatchObject({
+      success: false,
+      invalidToken: true,
+    });
+    net.on("hooks.slack.com", () => "hang-up");
+    expect((await s.send(task({ destination: undefined }))).success).toBe(false);
+  });
+
+  it("explains, rather than calls Slack, when it has no way to send", async () => {
+    expect((await new SlackTransport().send(task({ destination: undefined }))).error).toMatch(
+      /No destination/,
+    );
+    expect((await new SlackTransport().send(task({ destination: "C1" }))).error).toMatch(
+      /no botToken/,
+    );
+    expect(net.requests).toEqual([]);
+  });
+});
+
+// ─── WhatsApp ───────────────────────────────────────────────────────────────
+
+describe("WhatsAppTransport", () => {
+  const w = new WhatsAppTransport({
+    phoneNumberId: "PN1",
+    accessToken: "tok",
+    verifyToken: "vt",
+    appSecret: "shh",
+  });
+
+  it("sends a text message to the digits of the number", async () => {
+    net.on("graph.facebook.com", () => ({ json: { messages: [{ id: "wamid.1" }] } }));
+    const r = await w.send(
+      task({ destination: "+1 (555) 010-0200", content: { text: "Your code is 1" } }),
+    );
+    expect(r).toEqual({ success: true, providerMessageId: "wamid.1" });
+    const req = net.requests[0]!;
+    expect(req.path).toBe("/v21.0/PN1/messages");
+    expect(req.headers.authorization).toBe("Bearer tok");
+    expect(req.json()).toEqual({
+      messaging_product: "whatsapp",
+      to: "15550100200",
+      type: "text",
+      text: { body: "Your code is 1" },
+    });
+  });
+
+  it("flags an expired access token, and gives up on a hung API within the task's timeout", async () => {
+    net.on("graph.facebook.com", () => ({
+      status: 401,
+      json: { error: { code: 190, message: "Session expired" } },
+    }));
+    expect(await w.send(task({ destination: "+1555", content: { text: "x" } }))).toMatchObject({
+      success: false,
+      invalidToken: true,
+      error: "Session expired",
+    });
+
+    net.on("graph.facebook.com", () => ({ delayMs: 3_000, json: {} }));
+    const started = Date.now();
+    const r = await w.send(
+      task({
+        destination: "+1555",
+        content: { text: "x" },
+        deliveryOptions: { maxAttempts: 3, timeoutMs: 300 },
+      }),
+    );
+    expect(r.success).toBe(false);
+    expect(Date.now() - started).toBeLessThan(2_000);
+  });
+
+  it("answers Meta's verification handshake only with the right token", () => {
+    expect(
+      w.verifyWebhookChallenge(
+        new URLSearchParams("hub.mode=subscribe&hub.verify_token=vt&hub.challenge=42"),
+      ),
+    ).toBe("42");
+    expect(
+      w.verifyWebhookChallenge(
+        new URLSearchParams("hub.mode=subscribe&hub.verify_token=no&hub.challenge=42"),
+      ),
+    ).toBeUndefined();
+  });
+
+  it("accepts only webhooks signed with the app secret", () => {
+    const raw = JSON.stringify({ entry: [] });
+    const sig = `sha256=${createHmac("sha256", "shh").update(raw).digest("hex")}`;
+    expect(w.verifyWebhook(raw, { "x-hub-signature-256": sig })).toBe(true);
+    expect(w.verifyWebhook(raw + " ", { "x-hub-signature-256": sig })).toBe(false);
+    expect(w.verifyWebhook(raw, {})).toBe(false);
+    expect(
+      new WhatsAppTransport({ phoneNumberId: "p", accessToken: "t" }).verifyWebhook(raw, {
+        "x-hub-signature-256": sig,
+      }),
+    ).toBe(false);
+  });
+
+  it("maps read and failed statuses, ignoring the rest", async () => {
+    const events = await w.parseWebhook({
+      entry: [
+        {
+          changes: [
+            {
+              value: {
+                statuses: [
+                  { id: "a", status: "read", recipient_id: "1555", timestamp: "1700000000" },
+                  { id: "b", status: "failed", recipient_id: "1666", errors: [{ code: 131026 }] },
+                  { id: "c", status: "delivered" },
+                ],
+              },
+            },
+          ],
+        },
+      ],
+    });
+    expect(events.map((e) => [e.providerMessageId, e.status, e.recipient, e.bounceType])).toEqual([
+      ["a", "opened", "1555", undefined],
+      ["b", "bounced", "1666", undefined],
+    ]);
+  });
+});
+
+// ─── Resend ─────────────────────────────────────────────────────────────────
+
+/** Signs a payload the way Resend (via Svix) does. */
+function svixSign(secret: string, id: string, timestamp: number, payload: string) {
+  const key = Buffer.from(secret.replace(/^whsec_/, ""), "base64");
+  const sig = createHmac("sha256", key).update(`${id}.${timestamp}.${payload}`).digest("base64");
+  return { "svix-id": id, "svix-timestamp": String(timestamp), "svix-signature": `v1,${sig}` };
+}
+
+describe("ResendTransport", () => {
+  const secret = `whsec_${randomBytes(24).toString("base64")}`;
+  const r = new ResendTransport({
+    apiKey: "re_test",
+    from: "Acme <no-reply@acme.test>",
+    webhookSecret: secret,
+  });
+
+  it("sends the rendered email with its headers and returns Resend's id", async () => {
+    net.on("api.resend.com", () => ({ json: { id: "re-1" } }));
+    const res = await r.send(
+      task({
+        destination: "u@x.com",
+        content: {
+          subject: "Hi",
+          html: "<b>Hi</b>",
+          text: "Hi",
+          from: "Sales <sales@acme.test>",
+          replyTo: "help@acme.test",
+        },
+        deliveryOptions: {
+          maxAttempts: 3,
+          timeoutMs: 1000,
+          headers: { "List-Unsubscribe": "<https://u>" },
+        },
+      }),
+    );
+    expect(res).toEqual({ success: true, providerMessageId: "re-1" });
+    const req = net.requests[0]!;
+    expect(req.path).toBe("/emails");
+    expect(req.headers.authorization).toBe("Bearer re_test");
+    expect(req.json()).toMatchObject({
+      from: "Sales <sales@acme.test>",
+      to: "u@x.com",
+      subject: "Hi",
+      html: "<b>Hi</b>",
+      text: "Hi",
+      reply_to: "help@acme.test",
+      headers: { "List-Unsubscribe": "<https://u>" },
+    });
+  });
+
+  it("escapes a text-only body when it builds the HTML part", async () => {
+    net.on("api.resend.com", () => ({ json: { id: "re-2" } }));
+    await r.send(
+      task({
+        destination: "u@x.com",
+        content: { subject: "S", text: 'Hi <img src=x onerror="alert(1)">' },
+      }),
+    );
+    const html = net.requests[0]!.json().html as string;
+    expect(html).not.toContain("<img");
+    expect(html).toContain("&lt;img");
+  });
+
+  it("reports Resend's rejection", async () => {
+    net.on("api.resend.com", () => ({
+      status: 422,
+      json: { statusCode: 422, name: "validation_error", message: "Invalid `to` field" },
+    }));
+    expect(await r.send(task({ destination: "nope" }))).toMatchObject({
+      success: false,
+      error: "Invalid `to` field",
+    });
+  });
+
+  it("verifies Svix-signed webhooks and maps bounces, complaints, opens and clicks", async () => {
+    const now = Math.floor(Date.now() / 1000);
+    const payload = JSON.stringify({
+      type: "email.bounced",
+      created_at: new Date().toISOString(),
+      data: {
+        email_id: "re-1",
+        to: ["u@x.com"],
+        bounce: { type: "Permanent", message: "no such user" },
+      },
+    });
+    const headers = svixSign(secret, "msg_1", now, payload);
+
+    expect(r.verifyWebhook(payload, headers)).toBe(true);
+    const [bounce] = await r.parseWebhook(JSON.parse(payload), payload, headers);
+    expect(bounce).toMatchObject({
+      providerMessageId: "re-1",
+      status: "bounced",
+      bounceType: "hard",
+      recipient: "u@x.com",
+    });
+
+    const click = JSON.stringify({
+      type: "email.clicked",
+      data: { email_id: "re-2", to: ["u@x.com"], click: { link: "https://acme.test" } },
+    });
+    const [c] = await r.parseWebhook(null, click, svixSign(secret, "msg_2", now, click));
+    expect(c).toMatchObject({ status: "clicked", metadata: { url: "https://acme.test" } });
+
+    const delivered = JSON.stringify({ type: "email.delivered", data: { email_id: "re-3" } });
+    expect(
+      await r.parseWebhook(null, delivered, svixSign(secret, "msg_3", now, delivered)),
+    ).toEqual([]);
+  });
+
+  it("rejects a forged, replayed-from-long-ago or unsigned webhook", async () => {
+    const payload = JSON.stringify({
+      type: "email.complained",
+      data: { email_id: "re-1", to: ["u@x.com"] },
+    });
+    const forged = svixSign(
+      `whsec_${randomBytes(24).toString("base64")}`,
+      "m",
+      Math.floor(Date.now() / 1000),
+      payload,
+    );
+    expect(r.verifyWebhook(payload, forged)).toBe(false);
+    expect(
+      r.verifyWebhook(
+        payload,
+        svixSign(secret, "m", Math.floor(Date.now() / 1000) - 3600, payload),
+      ),
+    ).toBe(false);
+    expect(r.verifyWebhook(payload, {})).toBe(false);
+    expect(await r.parseWebhook(JSON.parse(payload), payload, forged)).toEqual([]);
+    expect(
+      new ResendTransport({ apiKey: "k", from: "f" }).verifyWebhook(
+        payload,
+        svixSign(secret, "m", Math.floor(Date.now() / 1000), payload),
+      ),
+    ).toBe(false);
+  });
+});
+
+// ─── Twilio (webhook side; sending goes through Twilio's own HTTPS client) ──
+
+/** Twilio's documented request signature: HMAC-SHA1 over URL + sorted params. */
+function twilioSign(authToken: string, url: string, params: Record<string, string>) {
+  const data =
+    url +
+    Object.keys(params)
+      .sort()
+      .map((k) => k + params[k])
+      .join("");
+  return createHmac("sha1", authToken).update(data).digest("base64");
+}
+
+describe("TwilioTransport webhooks", () => {
+  const url = "https://api.acme.test/webhooks/twilio";
+  const t = new TwilioTransport({
+    accountSid: "AC1",
+    authToken: "secret",
+    from: "+1000",
+    statusCallbackUrl: url,
+  });
+
+  it("mounts at the callback URL's path", () => {
+    expect(t.webhookPath).toBe("/webhooks/twilio");
+    expect(
+      () =>
+        new TwilioTransport({
+          accountSid: "a",
+          authToken: "b",
+          from: "c",
+          statusCallbackUrl: "not a url",
+        }),
+    ).toThrow();
+  });
+
+  it("verifies Twilio's signature over every posted field", async () => {
+    const params = {
+      MessageSid: "SM1",
+      MessageStatus: "undelivered",
+      To: "+1555",
+      ErrorCode: "30005",
+      NewField: "x",
+    };
+    const raw = new URLSearchParams(params).toString();
+    const sig = twilioSign("secret", url, params);
+
+    expect(await t.verifyWebhook(raw, { "x-twilio-signature": sig })).toBe(true);
+    expect(
+      await t.verifyWebhook(raw.replace("30005", "30006"), { "x-twilio-signature": sig }),
+    ).toBe(false);
+    expect(await t.verifyWebhook(raw, {})).toBe(false);
+
+    const [e] = await t.parseWebhook({}, raw, { "x-twilio-signature": sig });
+    expect(e).toMatchObject({
+      providerMessageId: "SM1",
+      status: "bounced",
+      bounceType: "hard",
+      recipient: "+1555",
+    });
+  });
+
+  it("treats a STOP reply as an unsubscribe and ignores progress reports", async () => {
+    const stop = { MessageSid: "SM2", MessageStatus: "failed", To: "+1555", ErrorCode: "21610" };
+    const raw = new URLSearchParams(stop).toString();
+    const [e] = await t.parseWebhook({}, raw, {
+      "x-twilio-signature": twilioSign("secret", url, stop),
+    });
+    expect(e!.status).toBe("unsubscribed");
+
+    const sent = { MessageSid: "SM3", MessageStatus: "sent" };
+    const raw2 = new URLSearchParams(sent).toString();
+    expect(
+      await t.parseWebhook({}, raw2, { "x-twilio-signature": twilioSign("secret", url, sent) }),
+    ).toEqual([]);
+  });
+
+  it("refuses to send without a destination or a body", async () => {
+    expect((await t.send(task({ destination: undefined }))).success).toBe(false);
+    expect((await t.send(task({ destination: "+1", content: { subject: "only" } }))).success).toBe(
+      false,
+    );
+  });
+});
+
+describe("ConsoleTransport", () => {
+  it("delivers after the configured latency and returns a message id", async () => {
+    const c = new ConsoleTransport({ channel: "sms", latencyMs: "50-60" });
+    const started = Date.now();
+    const r = await c.send(task());
+    expect(Date.now() - started).toBeGreaterThanOrEqual(45);
+    expect(r.success).toBe(true);
+    expect(r.providerMessageId).toMatch(/^console-/);
+    expect(new ConsoleTransport().channel).toBe("push");
+  });
+});
+
+// ─── A provider end to end ─────────────────────────────────────────────────
+
+describe("Resend inside notifkit", () => {
+  const infra = useInfra();
+  let app: Running;
+  const secret = `whsec_${randomBytes(24).toString("base64")}`;
+
+  beforeAll(async () => {
+    app = await startNotifkit(infra, {
+      services: ["api", "enricher", "engine", "delivery", "events"],
+      providers: [
+        new ResendTransport({
+          apiKey: "re_live",
+          from: "no-reply@acme.test",
+          webhookSecret: secret,
+        }),
+      ],
+      env: { RATE_LIMIT_PER_HOUR: "100000" },
+    });
+  }, 120_000);
+  afterAll(async () => {
+    await app?.stop();
+  }, 60_000);
+
+  it("delivers through Resend's API, then suppresses the address when Resend reports a hard bounce", async () => {
+    let n = 0;
+    net.on("api.resend.com", () => ({ json: { id: `re-${++n}` } }));
+    const p = await app.project();
+    await p.api("PUT", "/v1/templates", {
+      templates: [
+        { id: "t", channel: "email", content: { subject: "Hi {{name}}", text: "Hello" } },
+      ],
+    });
+    await p.api("POST", "/v1/users", { id: "u", email: "gone@x.com" });
+    await p.api("POST", "/v1/notify", { user: "u", template: "t", data: { name: "Al" } });
+
+    await waitFor("sent to Resend", () => net.requests.some((r) => r.host === "api.resend.com"));
+    const sent = net.requests.find((r) => r.host === "api.resend.com")!.json();
+    expect(sent).toMatchObject({ to: "gone@x.com", subject: "Hi Al", from: "no-reply@acme.test" });
+    await waitFor("logged", async () => {
+      const rows =
+        await infra.sql`SELECT 1 FROM message_logs WHERE provider_message_id = 're-1' AND status = 'delivered'`;
+      return rows.length === 1;
+    });
+
+    const payload = JSON.stringify({
+      type: "email.bounced",
+      data: { email_id: "re-1", to: ["gone@x.com"], bounce: { type: "Permanent" } },
+    });
+    const res = await apiClient(app.baseUrl)("POST", "/webhooks/resend", payload, {
+      "content-type": "application/json",
+      ...svixSign(secret, "msg_e2e", Math.floor(Date.now() / 1000), payload),
+    });
+    expect(res.status).toBe(200);
+
+    const suppressions = await p.api("GET", "/v1/suppressions");
+    expect(suppressions.body.suppressions.map((s: any) => [s.target, s.reason])).toEqual([
+      ["gone@x.com", "bounced"],
+    ]);
+  });
+});

tests/integration/queue.test.ts

@@ -0,0 +1,401 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { randomUUID } from "node:crypto";
+import { StreamProducer, StreamConsumer, PendingMessageScanner } from "@/queue/index.js";
+import { STREAMS, CONSUMER_GROUPS, buildStreamEvent, type StreamName } from "@/contracts/index.js";
+import { createLogger } from "@/logger/index.js";
+import { useInfra, waitFor, settle } from "./support/infra.js";
+
+const infra = useInfra();
+const logger = createLogger({ name: "test", level: "silent" });
+
+const A = STREAMS.INBOUND_NORMAL;
+const B = STREAMS.INBOUND_LOW;
+const DLQ = STREAMS.DEAD_LETTER;
+const GROUP = CONSUMER_GROUPS.ENRICHER;
+
+function event(n = 0) {
+  return buildStreamEvent("test.event", { n }, "test");
+}
+
+/** A full envelope, as it would sit in a stream. */
+function envelope(n = 0) {
+  return { ...event(n), id: randomUUID(), timestamp: new Date().toISOString() };
+}
+
+const consumers: StreamConsumer[] = [];
+function consumer(opts: Partial<ConstructorParameters<typeof StreamConsumer>[0]> = {}) {
+  const c = new StreamConsumer({
+    redis: infra.redis,
+    stream: A,
+    group: GROUP,
+    consumer: `c-${randomUUID().slice(0, 6)}`,
+    blockMs: 100,
+    logger,
+    ...opts,
+  });
+  consumers.push(c);
+  return c;
+}
+
+/** Reads until `count` messages arrived or the timeout passes. */
+async function readN(c: StreamConsumer, count: number, timeoutMs = 5_000) {
+  const got: any[] = [];
+  const it = c.readBatch();
+  const deadline = Date.now() + timeoutMs;
+  while (got.length < count && Date.now() < deadline) {
+    const next = await Promise.race([
+      it.next(),
+      new Promise<{ done: true; value: undefined }>((r) =>
+        setTimeout(() => r({ done: true, value: undefined }), deadline - Date.now()),
+      ),
+    ]);
+    if (next.done) break;
+    got.push(...next.value);
+  }
+  await c.stop();
+  return got;
+}
+
+async function pendingCount(stream: string, group = GROUP): Promise<number> {
+  const summary = (await infra.redis.xpending(stream, group)) as any[];
+  return Number(summary[0]);
+}
+
+beforeEach(async () => {
+  await infra.redis.flushdb();
+});
+
+afterEach(async () => {
+  await Promise.all(consumers.splice(0).map((c) => c.stop().catch(() => {})));
+});
+
+describe("StreamProducer", () => {
+  it("writes an envelope with a fresh id and timestamp, and returns the stream entry id", async () => {
+    const producer = new StreamProducer({ redis: infra.redis, stream: A, logger, maxLen: 1000 });
+    const entryId = await producer.publish(event(7));
+
+    const entries = await infra.redis.xrange(A, "-", "+");
+    expect(entries).toHaveLength(1);
+    expect(entries[0]![0]).toBe(entryId);
+    const stored = JSON.parse(entries[0]![1][1]!);
+    expect(stored.payload).toEqual({ n: 7 });
+    expect(stored.id).toMatch(/^[0-9a-f-]{36}$/);
+    expect(Number.isNaN(Date.parse(stored.timestamp))).toBe(false);
+  });
+
+  it("keeps every message and their order when hundreds are published at once", async () => {
+    const producer = new StreamProducer({ redis: infra.redis, stream: A, logger, maxLen: 10_000 });
+    const ids = await Promise.all(
+      Array.from({ length: 350 }, (_, n) => producer.publish(event(n))),
+    );
+
+    expect(new Set(ids).size).toBe(350);
+    const entries = await infra.redis.xrange(A, "-", "+");
+    expect(entries.map((e) => JSON.parse(e[1][1]!).payload.n)).toEqual(
+      Array.from({ length: 350 }, (_, n) => n),
+    );
+  });
+
+  it("trims the stream towards maxLen", async () => {
+    const producer = new StreamProducer({ redis: infra.redis, stream: A, logger, maxLen: 10 });
+    for (let batch = 0; batch < 10; batch++) {
+      await Promise.all(Array.from({ length: 100 }, (_, n) => producer.publish(event(n))));
+    }
+    // "~" trims whole radix-tree nodes (100 entries by default), so the length
+    // lands near, not at, the cap — but nowhere near the 1000 written.
+    expect(await infra.redis.xlen(A)).toBeLessThanOrEqual(200);
+  });
+
+  it("rejects the publish when Redis refuses the write", async () => {
+    await infra.redis.set(A, "not a stream");
+    const producer = new StreamProducer({ redis: infra.redis, stream: A, logger, maxLen: 100 });
+    await expect(producer.publish(event())).rejects.toThrow(/WRONGTYPE/);
+  });
+
+  it("publishBatch returns the ids of the events it actually published", async () => {
+    const producer = new StreamProducer({ redis: infra.redis, stream: A, logger, maxLen: 1000 });
+    const { messageIds, eventIds } = await producer.publishBatch([event(1), event(2), event(3)]);
+
+    const entries = await infra.redis.xrange(A, "-", "+");
+    expect(entries.map((e) => e[0])).toEqual(messageIds);
+    // Callers record eventIds as the notification id (workflow step.notify
+    // does), so each must be the id of the event that entry holds.
+    expect(entries.map((e) => JSON.parse(e[1][1]!).id)).toEqual(eventIds);
+  });
+
+  it("publishBatch of nothing writes nothing", async () => {
+    const producer = new StreamProducer({ redis: infra.redis, stream: A, logger, maxLen: 1000 });
+    expect(await producer.publishBatch([])).toEqual({ messageIds: [], eventIds: [] });
+    expect(await infra.redis.exists(A)).toBe(0);
+  });
+});
+
+describe("StreamConsumer", () => {
+  it("delivers messages published before the group existed", async () => {
+    await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(1)));
+    const c = consumer();
+    await c.ensureGroup();
+
+    const got = await readN(c, 1);
+    expect(got.map((m) => m.event.payload.n)).toEqual([1]);
+  });
+
+  it("ensureGroup is safe to call repeatedly and creates missing streams", async () => {
+    const c = consumer({ stream: [A, B] });
+    await c.ensureGroup();
+    await c.ensureGroup();
+    const groups = (await infra.redis.xinfo("GROUPS", B)) as any[];
+    expect(groups).toHaveLength(1);
+  });
+
+  it("reads from several streams and tags each message with its stream", async () => {
+    const c = consumer({ stream: [A, B] });
+    await c.ensureGroup();
+    await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(1)));
+    await infra.redis.xadd(B, "*", "data", JSON.stringify(envelope(2)));
+
+    const got = await readN(c, 2);
+    const byN = Object.fromEntries(got.map((m) => [m.event.payload.n, m]));
+    expect(byN[1]!.stream).toBe(A);
+    expect(byN[2].stream).toBe(B);
+    expect(byN[1].deliveryCount).toBe(1);
+  });
+
+  it("acks and skips entries it cannot parse instead of leaving them pending", async () => {
+    const c = consumer();
+    await c.ensureGroup();
+    await infra.redis.xadd(A, "*", "other", "x");
+    await infra.redis.xadd(A, "*", "data", "{not json");
+    await infra.redis.xadd(A, "*", "data", JSON.stringify({ hello: "world" }));
+    await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(9)));
+
+    const got = await readN(c, 1);
+    expect(got.map((m) => m.event.payload.n)).toEqual([9]);
+    // Only the good one is still pending; it has not been acked yet.
+    expect(await pendingCount(A)).toBe(1);
+  });
+
+  it("ack removes the message from the pending list", async () => {
+    const c = consumer();
+    await c.ensureGroup();
+    await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope()));
+    const [msg] = await readN(c, 1);
+    expect(await pendingCount(A)).toBe(1);
+
+    await c.ack(msg.id, msg.stream);
+    expect(await pendingCount(A)).toBe(0);
+  });
+
+  it("buffered acks land once the timer fires, and on flushAcks", async () => {
+    const c = consumer({ bufferAcks: true, ackFlushMs: 20, batchSize: 100 });
+    await c.ensureGroup();
+    for (let n = 0; n < 3; n++) await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(n)));
+    const got = await readN(c, 3);
+
+    await c.ack(got[0].id, A);
+    expect(await pendingCount(A)).toBe(3); // still buffered
+    await waitFor("timer flush", async () => (await pendingCount(A)) === 2, 2_000);
+
+    await c.ack([got[1].id, got[2].id], A);
+    await c.flushAcks();
+    expect(await pendingCount(A)).toBe(0);
+  });
+
+  it("buffered acks flush immediately once a batch is full", async () => {
+    const c = consumer({ bufferAcks: true, ackFlushMs: 60_000, batchSize: 2 });
+    await c.ensureGroup();
+    for (let n = 0; n < 2; n++) await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(n)));
+    const got = await readN(c, 2);
+
+    await c.ack(got[0].id, A);
+    await c.ack(got[1].id, A);
+    expect(await pendingCount(A)).toBe(0);
+  });
+
+  it("stop() flushes buffered acks", async () => {
+    const c = consumer({ bufferAcks: true, ackFlushMs: 60_000, batchSize: 100 });
+    await c.ensureGroup();
+    await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope()));
+    const [msg] = await readN(c, 1);
+    await c.ack(msg.id, A);
+    await c.stop();
+    expect(await pendingCount(A)).toBe(0);
+  });
+
+  it("nack moves the message to the dead-letter stream and acks it", async () => {
+    const c = consumer({ dlqStream: DLQ });
+    await c.ensureGroup();
+    const original = envelope(5);
+    await infra.redis.xadd(A, "*", "data", JSON.stringify(original));
+    const [msg] = await readN(c, 1);
+
+    await c.nack(msg.id, msg.event, msg.stream);
+
+    expect(await pendingCount(A)).toBe(0);
+    const dlq = await infra.redis.xrange(DLQ, "-", "+");
+    expect(dlq).toHaveLength(1);
+    const dead = JSON.parse(dlq[0]![1][1]!);
+    expect(dead.id).toBe(original.id);
+    expect(dead.dlq.originalStream).toBe(A);
+  });
+
+  it("nack leaves the message pending when the dead-letter write fails", async () => {
+    await infra.redis.set(DLQ, "wrong type");
+    const c = consumer({ dlqStream: DLQ });
+    await c.ensureGroup();
+    await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope()));
+    const [msg] = await readN(c, 1);
+
+    await expect(c.nack(msg.id, msg.event, msg.stream)).rejects.toThrow();
+    // Never acked against a DLQ write that did not land.
+    expect(await pendingCount(A)).toBe(1);
+  });
+
+  it("nack without a dead-letter stream just acks", async () => {
+    const c = consumer();
+    await c.ensureGroup();
+    await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope()));
+    const [msg] = await readN(c, 1);
+    await c.nack(msg.id, msg.event, msg.stream);
+    expect(await pendingCount(A)).toBe(0);
+    expect(await infra.redis.exists(DLQ)).toBe(0);
+  });
+
+  it("deadLetter records an in-process event against the consumer's stream", async () => {
+    const c = consumer({ dlqStream: DLQ });
+    const ev = envelope(3);
+    await c.deadLetter(ev);
+    const [[, fields]] = (await infra.redis.xrange(DLQ, "-", "+")) as any;
+    const dead = JSON.parse(fields[1]);
+    expect(dead.id).toBe(ev.id);
+    expect(dead.dlq.originalStream).toBe(A);
+  });
+
+  it("recreates its group and keeps reading after Redis loses its data", async () => {
+    const c = consumer();
+    await c.ensureGroup();
+    const it = c.readBatch();
+
+    await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(1)));
+    const first = await it.next();
+    expect(first.value![0]!.event.payload.n).toBe(1);
+
+    await infra.redis.flushdb();
+    // Give the blocked read a moment to hit NOGROUP before anything is written.
+    await settle(300);
+    await infra.redis.xadd(A, "*", "data", JSON.stringify(envelope(2)));
+
+    const second = await Promise.race([
+      it.next(),
+      settle(5_000).then(() => ({ value: [] as any[] })),
+    ]);
+    expect(second.value!.map((m: any) => m.event.payload.n)).toEqual([2]);
+  });
+
+  it("stop() ends the read loop", async () => {
+    const c = consumer();
+    await c.ensureGroup();
+    const it = c.readBatch();
+    const pending = it.next();
+    await settle(50);
+    await c.stop();
+    const result = await Promise.race([pending, settle(3_000).then(() => "hung" as const)]);
+    expect(result).not.toBe("hung");
+  });
+});
+
+describe("PendingMessageScanner", () => {
+  async function deliverUnacked(stream: StreamName, n: number, consumerName = "dead-worker") {
+    // In production the scanner's consumer has already created the group on
+    // every stream it reads.
+    for (const s of [A, B])
+      await infra.redis.xgroup("CREATE", s, GROUP, "0", "MKSTREAM").catch(() => {});
+    for (let i = 0; i < n; i++) {
+      await infra.redis.xadd(stream, "*", "data", JSON.stringify(envelope(i)));
+    }
+    await infra.redis.xreadgroup("GROUP", GROUP, consumerName, "COUNT", n, "STREAMS", stream, ">");
+  }
+
+  function scanner(name = "rescuer", stream: StreamName | StreamName[] = [A, B]) {
+    return new PendingMessageScanner({
+      redis: infra.redis,
+      stream,
+      group: GROUP,
+      consumer: name,
+      logger,
+    });
+  }
+
+  it("counts and lists pending entries across its streams", async () => {
+    await deliverUnacked(A, 2);
+    await deliverUnacked(B, 3);
+    const s = scanner();
+
+    expect(await s.getPendingCount()).toBe(5);
+    const entries = await s.getPendingEntries(100);
+    expect(entries).toHaveLength(5);
+    expect(entries.filter((e) => e.stream === B)).toHaveLength(3);
+    expect(entries.every((e) => e.consumer === "dead-worker" && e.deliveryCount === 1)).toBe(true);
+    expect(await s.getPendingEntries(4)).toHaveLength(4);
+    expect(await s.getPendingEntries(100, A)).toHaveLength(2);
+  });
+
+  it("claims only entries idle past the threshold and bumps their delivery count", async () => {
+    await deliverUnacked(A, 2);
+    const s = scanner();
+
+    expect(await s.autoclaim(60_000)).toEqual([]);
+
+    await settle(150);
+    const claimed = await s.autoclaim(100);
+    expect(claimed).toHaveLength(2);
+    expect(claimed.every((m) => m.stream === A && m.deliveryCount === 2)).toBe(true);
+    const entries = await s.getPendingEntries(10);
+    expect(entries.every((e) => e.consumer === "rescuer")).toBe(true);
+  });
+
+  it("backs off exponentially with each redelivery", async () => {
+    await deliverUnacked(A, 1);
+    const s = scanner();
+    await settle(350);
+    expect(await s.autoclaim(300)).toHaveLength(1); // delivered once: needs > 300ms idle
+
+    // Delivered twice now, so it needs > 600ms idle. Checked straight after the
+    // claim reset its idle time, so a slow machine cannot make this pass or fail.
+    expect(await s.autoclaim(300)).toHaveLength(0);
+    await settle(700);
+    expect(await s.autoclaim(300)).toHaveLength(1);
+  });
+
+  it("never hands one entry to two racing scanners", async () => {
+    await deliverUnacked(A, 20);
+    await settle(150);
+    const [x, y] = await Promise.all([
+      scanner("x").autoclaim(100, 50),
+      scanner("y").autoclaim(100, 50),
+    ]);
+    const ids = [...x, ...y].map((m) => m.id);
+    expect(ids).toHaveLength(20);
+    expect(new Set(ids).size).toBe(20);
+  });
+
+  it("does not claim a message from one stream using an id pending on another", async () => {
+    await infra.redis.xgroup("CREATE", A, GROUP, "0", "MKSTREAM");
+    await infra.redis.xgroup("CREATE", B, GROUP, "0", "MKSTREAM");
+    // Same entry id in both streams; only B's is pending.
+    await infra.redis.xadd(A, "1-1", "data", JSON.stringify(envelope(100)));
+    await infra.redis.xadd(B, "1-1", "data", JSON.stringify(envelope(200)));
+    await infra.redis.xreadgroup("GROUP", GROUP, "dead", "STREAMS", B, ">");
+    await settle(150);
+
+    const claimed = await scanner().autoclaim(100);
+    expect(claimed.map((m) => [m.stream, m.event.payload.n])).toEqual([[B, 200]]);
+    expect(await pendingCount(A)).toBe(0);
+  });
+
+  it("respects the claim limit", async () => {
+    await deliverUnacked(A, 10);
+    await settle(150);
+    expect(await scanner().autoclaim(100, 3)).toHaveLength(3);
+  });
+});

tests/integration/repositories.test.ts

@@ -0,0 +1,692 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { randomUUID } from "node:crypto";
+import {
+  UserRepository,
+  PreferenceRepository,
+  ContactRepository,
+  TemplateRepository,
+  ProjectRepository,
+  WorkflowRepository,
+  SegmentRepository,
+  AdminUserRepository,
+} from "@/repositories/index.js";
+import { useInfra } from "./support/infra.js";
+
+const infra = useInfra();
+
+let P: string; // project under test
+let OTHER: string; // a second tenant, to prove isolation
+
+async function newProject(name = "p") {
+  const id = randomUUID();
+  await infra.sql`INSERT INTO projects (id, name) VALUES (${id}, ${name})`;
+  return id;
+}
+
+beforeEach(async () => {
+  await infra.reset();
+  P = await newProject("main");
+  OTHER = await newProject("other");
+});
+
+function user(userId: string, extra: Record<string, unknown> = {}) {
+  return { userId, segments: [], preferences: {}, ...extra } as any;
+}
+
+// ─── UserRepository ─────────────────────────────────────────────────────────
+
+describe("UserRepository", () => {
+  it("upsertFull stores the profile, segments, preferences and quiet hours", async () => {
+    const repo = new UserRepository(infra.db);
+    await repo.upsertFull(P, {
+      userId: "u1",
+      email: "a@x.com",
+      language: "fr",
+      timezone: "Europe/Paris",
+      segments: ["vip", "beta"],
+      preferences: {
+        topics: { marketing: false, news: true },
+        channels: { sms: false },
+        quietHours: [{ start: "22:00", end: "07:00" }],
+      },
+    });
+
+    const rec = await new UserRepository(infra.db).findRecordById(P, "u1");
+    expect(rec).toMatchObject({
+      userId: "u1",
+      email: "a@x.com",
+      language: "fr",
+      timezone: "Europe/Paris",
+      preferences: {
+        topics: { marketing: false, news: true },
+        channels: { sms: false },
+        quietHours: [{ start: "22:00", end: "07:00" }],
+      },
+    });
+    expect(rec!.segments.sort()).toEqual(["beta", "vip"]);
+    expect(await repo.findById(P, "u1")).toEqual({
+      userId: "u1",
+      email: "a@x.com",
+      language: "fr",
+      timezone: "Europe/Paris",
+    });
+  });
+
+  it("upsertFull on an existing user updates it and replaces quiet hours", async () => {
+    const repo = new UserRepository(infra.db);
+    await repo.upsertFull(
+      P,
+      user("u1", {
+        email: "old@x.com",
+        preferences: { quietHours: [{ start: "01:00", end: "02:00" }] },
+      }),
+    );
+    await repo.upsertFull(
+      P,
+      user("u1", { email: "new@x.com", preferences: { topics: { a: false }, quietHours: [] } }),
+    );
+
+    const rec = await new UserRepository(infra.db).findRecordById(P, "u1");
+    expect(rec!.email).toBe("new@x.com");
+    expect(rec!.preferences.topics).toEqual({ a: false });
+    expect(rec!.preferences.quietHours).toBeUndefined();
+    const { n } = (
+      await infra.sql`SELECT count(*)::int AS n FROM users WHERE project_id = ${P}`
+    )[0]!;
+    expect(n).toBe(1);
+  });
+
+  it("keeps the same external id separate per project", async () => {
+    const repo = new UserRepository(infra.db);
+    await repo.upsertFull(P, user("same", { email: "p@x.com" }));
+    await repo.upsertFull(OTHER, user("same", { email: "o@x.com" }));
+    expect((await repo.findById(P, "same"))!.email).toBe("p@x.com");
+    expect((await repo.findById(OTHER, "same"))!.email).toBe("o@x.com");
+    expect(await repo.findById(P, "nobody")).toBeNull();
+    expect(await repo.findRecordById(P, "nobody")).toBeNull();
+  });
+
+  it("findRecordsByIds returns the users that exist and serves repeats from cache", async () => {
+    const repo = new UserRepository(infra.db);
+    await repo.upsertManyFull(P, [user("a"), user("b"), user("c")]);
+
+    const first = await repo.findRecordsByIds(P, ["a", "b", "missing"]);
+    expect(first.map((r) => r.userId).sort()).toEqual(["a", "b"]);
+
+    // Once cached, a row deleted behind the repository's back is still served.
+    await infra.sql`DELETE FROM users WHERE external_id = 'a'`;
+    const second = await repo.findRecordsByIds(P, ["a", "c"]);
+    expect(second.map((r) => r.userId).sort()).toEqual(["a", "c"]);
+    expect(await repo.findRecordsByIds(P, [])).toEqual([]);
+  });
+
+  it("writes through the repository invalidate its cache", async () => {
+    const repo = new UserRepository(infra.db);
+    await repo.upsertFull(P, user("u", { email: "one@x.com" }));
+    expect((await repo.findRecordById(P, "u"))!.email).toBe("one@x.com");
+
+    await repo.updatePartial(P, "u", { email: "two@x.com" });
+    expect((await repo.findRecordById(P, "u"))!.email).toBe("two@x.com");
+
+    await repo.upsertFull(P, user("u", { email: "three@x.com" }));
+    expect((await repo.findRecordById(P, "u"))!.email).toBe("three@x.com");
+
+    await repo.delete(P, "u");
+    expect(await repo.findRecordById(P, "u")).toBeNull();
+  });
+
+  it("upsertManyFull inserts many users, de-duplicating repeated rows", async () => {
+    const repo = new UserRepository(infra.db);
+    await repo.upsertManyFull(P, [
+      user("a", { segments: ["s", "s"], preferences: { topics: { t: true } } }),
+      user("a", { segments: ["s"], preferences: { topics: { t: false } } }),
+      user("b", {
+        preferences: { channels: { email: false }, quietHours: [{ start: "10:00", end: "11:00" }] },
+      }),
+    ]);
+
+    const [a, b] = await Promise.all([repo.findRecordById(P, "a"), repo.findRecordById(P, "b")]);
+    expect(a!.segments).toEqual(["s"]);
+    expect(a!.preferences.topics).toEqual({ t: false });
+    expect(b!.preferences.channels).toEqual({ email: false });
+    expect(b!.preferences.quietHours).toEqual([{ start: "10:00", end: "11:00" }]);
+    await repo.upsertManyFull(P, []);
+  });
+
+  it("upsertManyFull replaces quiet hours only for users that supplied them", async () => {
+    const repo = new UserRepository(infra.db);
+    await repo.upsertManyFull(P, [
+      user("a", { preferences: { quietHours: [{ start: "01:00", end: "02:00" }] } }),
+      user("b", { preferences: { quietHours: [{ start: "03:00", end: "04:00" }] } }),
+    ]);
+    await repo.upsertManyFull(P, [user("a", { preferences: { quietHours: [] } }), user("b")]);
+
+    const fresh = new UserRepository(infra.db);
+    expect((await fresh.findRecordById(P, "a"))!.preferences.quietHours).toBeUndefined();
+    expect((await fresh.findRecordById(P, "b"))!.preferences.quietHours).toEqual([
+      { start: "03:00", end: "04:00" },
+    ]);
+  });
+
+  it("updatePartial changes only what it is given, and replaces segments", async () => {
+    const repo = new UserRepository(infra.db);
+    await repo.upsertFull(
+      P,
+      user("u", { email: "e@x.com", language: "en", timezone: "UTC", segments: ["a", "b"] }),
+    );
+
+    expect(
+      await repo.updatePartial(P, "u", {
+        language: "de",
+        segments: ["c"],
+        preferences: {
+          topics: { x: false },
+          channels: { push: false },
+          quietHours: [{ start: "05:00", end: "06:00" }],
+        },
+      }),
+    ).toBe(true);
+    const rec = await new UserRepository(infra.db).findRecordById(P, "u");
+    expect(rec).toMatchObject({
+      email: "e@x.com",
+      language: "de",
+      timezone: "UTC",
+      segments: ["c"],
+      preferences: {
+        topics: { x: false },
+        channels: { push: false },
+        quietHours: [{ start: "05:00", end: "06:00" }],
+      },
+    });
+
+    expect(await repo.updatePartial(P, "missing", { language: "x" })).toBe(false);
+    expect(await repo.updatePartial(OTHER, "u", { language: "x" })).toBe(false);
+  });
+
+  it("updatePartial can clear an email with null", async () => {
+    const repo = new UserRepository(infra.db);
+    await repo.upsertFull(P, user("u", { email: "e@x.com" }));
+    await repo.updatePartial(P, "u", { email: null });
+    expect((await repo.findById(P, "u"))!.email ?? null).toBeNull();
+  });
+
+  it("delete removes the user and everything hanging off it, in its own project only", async () => {
+    const repo = new UserRepository(infra.db);
+    const contacts = new ContactRepository(infra.db);
+    await repo.upsertFull(P, user("u", { segments: ["s"] }));
+    await repo.upsertFull(OTHER, user("u"));
+    await contacts.upsert(P, "u", "email", "u@x.com", { topics: { t: false } });
+
+    expect(await repo.delete(P, "u")).toBe(true);
+    expect(await repo.delete(P, "u")).toBe(false);
+    expect(await repo.findById(OTHER, "u")).not.toBeNull();
+    const { n } = (await infra.sql`SELECT count(*)::int AS n FROM user_contacts`)[0]!;
+    expect(n).toBe(0);
+  });
+
+  describe("list", () => {
+    it("filters by search, segment, language, timezone and channel", async () => {
+      const repo = new UserRepository(infra.db);
+      await repo.upsertFull(
+        P,
+        user("alice", {
+          email: "alice@corp.com",
+          language: "en",
+          timezone: "UTC",
+          segments: ["vip"],
+        }),
+      );
+      await repo.upsertFull(
+        P,
+        user("bob", { email: "bob@home.net", language: "fr", timezone: "Europe/Paris" }),
+      );
+      await new ContactRepository(infra.db).upsert(P, "bob", "sms", "+100");
+      await repo.upsertFull(OTHER, user("alice-other"));
+
+      const ids = async (f: any) =>
+        (await repo.list(P, 50, undefined, f)).users.map((u) => u.userId).sort();
+      expect(await ids(undefined)).toEqual(["alice", "bob"]);
+      expect(await ids({ search: "CORP" })).toEqual(["alice"]);
+      expect(await ids({ search: "bo" })).toEqual(["bob"]);
+      expect(await ids({ segment: "vip" })).toEqual(["alice"]);
+      expect(await ids({ language: "fr" })).toEqual(["bob"]);
+      expect(await ids({ timezone: "UTC" })).toEqual(["alice"]);
+      expect(await ids({ channel: "sms" })).toEqual(["bob"]);
+    });
+
+    it("pages through every user, including users imported in one batch", async () => {
+      const repo = new UserRepository(infra.db);
+      // A bulk import writes every row in one transaction, so they all share
+      // one created_at.
+      await repo.upsertManyFull(
+        P,
+        Array.from({ length: 25 }, (_, i) => user(`bulk-${i}`)),
+      );
+
+      const seen: string[] = [];
+      let cursor: string | undefined;
+      for (let page = 0; page < 10; page++) {
+        const { users, nextCursor } = await repo.list(P, 10, cursor);
+        seen.push(...users.map((u) => u.userId));
+        if (!nextCursor) break;
+        cursor = nextCursor;
+      }
+      expect(seen).toHaveLength(25);
+      expect(new Set(seen).size).toBe(25);
+    });
+  });
+
+  it("findUsersBySegment and findUsersByTopic stay inside the project", async () => {
+    const repo = new UserRepository(infra.db);
+    await repo.upsertFull(
+      P,
+      user("a", { segments: ["vip"], preferences: { topics: { news: true } } }),
+    );
+    await repo.upsertFull(
+      P,
+      user("b", { segments: ["vip"], preferences: { topics: { news: false } } }),
+    );
+    await repo.upsertFull(
+      OTHER,
+      user("c", { segments: ["vip"], preferences: { topics: { news: true } } }),
+    );
+
+    expect((await repo.findUsersBySegment(P, "vip")).sort()).toEqual(["a", "b"]);
+    expect(await repo.findUsersBySegment(P, "none")).toEqual([]);
+    expect(await repo.findUsersByTopic(P, "news")).toEqual(["a"]);
+  });
+});
+
+// ─── PreferenceRepository ───────────────────────────────────────────────────
+
+describe("PreferenceRepository", () => {
+  it("reports topic opt-ins, defaulting to opted in", async () => {
+    await new UserRepository(infra.db).upsertFull(
+      P,
+      user("u", { preferences: { topics: { promo: false, news: true } } }),
+    );
+    const prefs = new PreferenceRepository(infra.db);
+
+    expect(await prefs.isOptedIn(P, "u", "promo")).toBe(false);
+    expect(await prefs.isOptedIn(P, "u", "news")).toBe(true);
+    expect(await prefs.isOptedIn(P, "u", "unknown")).toBe(true);
+    expect(
+      (await prefs.findByUserId(P, "u")).sort((a, b) => a.eventType.localeCompare(b.eventType)),
+    ).toEqual([
+      { userId: "u", eventType: "news", optedIn: true },
+      { userId: "u", eventType: "promo", optedIn: false },
+    ]);
+    expect(await prefs.findByUserId(P, "nobody")).toEqual([]);
+    expect(await prefs.findByUserId(OTHER, "u")).toEqual([]);
+  });
+});
+
+// ─── ContactRepository ──────────────────────────────────────────────────────
+
+describe("ContactRepository", () => {
+  beforeEach(async () => {
+    const users = new UserRepository(infra.db);
+    await users.upsertManyFull(P, [user("u"), user("v")]);
+    await users.upsertFull(OTHER, user("u"));
+  });
+
+  it("upsert stores a contact with its topic preferences", async () => {
+    const repo = new ContactRepository(infra.db);
+    await repo.upsert(P, "u", "email", "u@x.com", { topics: { promo: false } });
+    await repo.upsert(P, "u", "email", "u@x.com", { topics: { promo: true, news: false } });
+
+    const contacts = await repo.findByUserId(P, "u");
+    expect(contacts).toHaveLength(1);
+    expect(contacts[0]).toMatchObject({
+      userId: "u",
+      channel: "email",
+      target: "u@x.com",
+      active: true,
+      preferences: { topics: { promo: true, news: false } },
+    });
+    expect(await repo.findByUserId(P, "v")).toEqual([]);
+    expect(await repo.findByUserId(P, "nobody")).toEqual([]);
+  });
+
+  it("upsert for an unknown user is a no-op", async () => {
+    const repo = new ContactRepository(infra.db);
+    await repo.upsert(P, "ghost", "sms", "+1");
+    const { n } = (await infra.sql`SELECT count(*)::int AS n FROM user_contacts`)[0]!;
+    expect(n).toBe(0);
+  });
+
+  it("findActiveByUserIds returns only enabled contacts, grouped by user", async () => {
+    const repo = new ContactRepository(infra.db);
+    await repo.upsert(P, "u", "email", "u@x.com");
+    await repo.upsert(P, "u", "sms", "+1");
+    await repo.upsert(P, "v", "push", "tok");
+    await repo.upsert(OTHER, "u", "email", "other@x.com");
+    await repo.deactivate(P, "u", "sms", "+1");
+
+    const map = await new ContactRepository(infra.db).findActiveByUserIds(P, ["u", "v", "nobody"]);
+    expect(map.get("u")!.map((c) => c.target)).toEqual(["u@x.com"]);
+    expect(map.get("v")!.map((c) => c.channel)).toEqual(["push"]);
+    expect(map.has("nobody")).toBe(false);
+    expect((await repo.findActiveByUserIds(P, [])).size).toBe(0);
+  });
+
+  it("carries each contact's topic opt-outs, so a per-address unsubscribe can be honoured", async () => {
+    const repo = new ContactRepository(infra.db);
+    await repo.upsert(P, "u", "email", "work@x.com", { topics: { promo: false } });
+    const [contact] = (await repo.findActiveByUserIds(P, ["u"])).get("u")!;
+    expect(contact!.preferences.topics).toEqual({ promo: false });
+  });
+
+  it("sees writes made through the repository immediately", async () => {
+    const repo = new ContactRepository(infra.db);
+    await repo.upsert(P, "u", "email", "a@x.com");
+    expect((await repo.findActiveByUserIds(P, ["u"])).get("u")).toHaveLength(1);
+
+    await repo.upsert(P, "u", "sms", "+2");
+    expect((await repo.findActiveByUserIds(P, ["u"])).get("u")).toHaveLength(2);
+
+    await repo.deactivate(P, "u", "sms", "+2");
+    expect((await repo.findActiveByUserIds(P, ["u"])).get("u")).toHaveLength(1);
+
+    await repo.delete(P, "u", "email", "a@x.com");
+    expect((await repo.findActiveByUserIds(P, ["u"])).has("u")).toBe(false);
+
+    await repo.upsertMany(P, [{ userId: "u", channel: "email", target: "b@x.com" }]);
+    expect((await repo.findActiveByUserIds(P, ["u"])).get("u")).toHaveLength(1);
+  });
+
+  it("re-adding a deactivated contact turns it back on", async () => {
+    const repo = new ContactRepository(infra.db);
+    await repo.upsert(P, "u", "push", "tok");
+    expect(await repo.deactivate(P, "u", "push", "tok")).toBe(true);
+    expect((await repo.findByUserId(P, "u"))[0]!.active).toBe(false);
+    await repo.upsert(P, "u", "push", "tok");
+    expect((await repo.findByUserId(P, "u"))[0]!.active).toBe(true);
+  });
+
+  it("deactivate and delete report whether anything matched", async () => {
+    const repo = new ContactRepository(infra.db);
+    await repo.upsert(P, "u", "email", "u@x.com");
+    expect(await repo.deactivate(P, "u", "email", "nope@x.com")).toBe(false);
+    expect(await repo.deactivate(P, "ghost", "email", "u@x.com")).toBe(false);
+    expect(await repo.delete(OTHER, "u", "email", "u@x.com")).toBe(false);
+    expect(await repo.delete(P, "ghost", "email", "u@x.com")).toBe(false);
+    expect(await repo.delete(P, "u", "email", "u@x.com")).toBe(true);
+    expect(await repo.findByUserId(P, "u")).toEqual([]);
+  });
+
+  it("upsertMany adds contacts for known users with their topics, skipping unknown users", async () => {
+    const repo = new ContactRepository(infra.db);
+    await repo.upsertMany(P, [
+      {
+        userId: "u",
+        channel: "email",
+        target: "u@x.com",
+        preferences: { topics: { promo: false } },
+      },
+      { userId: "v", channel: "sms", target: "+9" },
+      { userId: "ghost", channel: "sms", target: "+0" },
+    ]);
+    await repo.upsertMany(P, []);
+
+    expect((await repo.findByUserId(P, "u"))[0]!.preferences.topics).toEqual({ promo: false });
+    expect((await repo.findByUserId(P, "v"))[0]!.target).toBe("+9");
+    const { n } = (await infra.sql`SELECT count(*)::int AS n FROM user_contacts`)[0]!;
+    expect(n).toBe(2);
+    await repo.upsertMany(P, [{ userId: "ghost", channel: "sms", target: "+0" }]);
+  });
+});
+
+// ─── TemplateRepository ─────────────────────────────────────────────────────
+
+describe("TemplateRepository", () => {
+  it("upserts, reads, lists and deletes templates per project", async () => {
+    const repo = new TemplateRepository(infra.db);
+    expect(await repo.upsertMany(P, [])).toBe(0);
+    expect(
+      await repo.upsertMany(P, [
+        {
+          id: "welcome",
+          channel: "email",
+          topics: ["onboarding"],
+          content: { subject: "Hi" },
+          aiPrompts: { intro: "say hi" },
+        },
+        { id: "otp", channel: "sms", content: { text: "{{code}}" } },
+      ]),
+    ).toBe(2);
+    await repo.upsertMany(OTHER, [{ id: "welcome", channel: "push", content: {} }]);
+
+    expect(await repo.findById(P, "welcome")).toEqual({
+      id: "welcome",
+      channel: "email",
+      topics: ["onboarding"],
+      content: { subject: "Hi" },
+      aiPrompts: { intro: "say hi" },
+    });
+    expect((await repo.findById(P, "otp"))!.topics).toEqual([]);
+    expect(await repo.findById(P, "nope")).toBeNull();
+    expect((await repo.list(P)).map((t) => t.id).sort()).toEqual(["otp", "welcome"]);
+
+    await repo.upsertMany(P, [
+      { id: "welcome", channel: "email", topics: [], content: { subject: "Hello" } },
+    ]);
+    const updated = await repo.findById(P, "welcome");
+    expect(updated!.content).toEqual({ subject: "Hello" });
+    expect(updated!.aiPrompts).toBeNull();
+
+    expect(await repo.delete(P, "welcome")).toBe(true);
+    expect(await repo.delete(P, "welcome")).toBe(false);
+    expect(await repo.findById(OTHER, "welcome")).not.toBeNull();
+  });
+});
+
+// ─── ProjectRepository ──────────────────────────────────────────────────────
+
+describe("ProjectRepository", () => {
+  it("lists projects newest first", async () => {
+    const newest = await newProject("newest");
+    const list = await new ProjectRepository(infra.db).list();
+    expect(list[0]!.id).toBe(newest);
+    expect(list.map((p) => p.id)).toEqual(expect.arrayContaining([P, OTHER]));
+  });
+
+  it("reads and updates throttle and rate-limit settings", async () => {
+    const repo = new ProjectRepository(infra.db);
+    expect(await repo.findThrottleSettings(P)).toEqual({
+      throttleLimit: null,
+      throttleWindowHours: null,
+    });
+    expect(
+      await repo.updateSettings(P, { throttleLimit: 5, throttleWindowHours: 2, rateLimitRpm: 100 }),
+    ).toBe(true);
+    expect(await repo.findThrottleSettings(P)).toEqual({
+      throttleLimit: 5,
+      throttleWindowHours: 2,
+    });
+    expect(await repo.updateSettings(P, { throttleLimit: null })).toBe(true);
+    expect((await repo.findThrottleSettings(P))!.throttleLimit).toBeNull();
+    expect(await repo.findThrottleSettings(randomUUID())).toBeNull();
+    expect(await repo.updateSettings(randomUUID(), { throttleLimit: 1 })).toBe(false);
+  });
+
+  it("creates, lists and deletes API keys within a project", async () => {
+    const repo = new ProjectRepository(infra.db);
+    const a = await repo.createApiKey(P, "hash-a");
+    const b = await repo.createApiKey(P, "hash-b", "read_only");
+    const keys = await repo.listApiKeys(P);
+    expect(keys.map((k) => k.id).sort()).toEqual([a.id, b.id].sort());
+    expect(keys.find((k) => k.id === b.id)!.role).toBe("read_only");
+    expect(keys[0]).not.toHaveProperty("keyHash");
+
+    expect(await repo.deleteApiKey(OTHER, a.id)).toBe(false);
+    expect(await repo.deleteApiKey(P, a.id)).toBe(true);
+    expect(await repo.listApiKeys(P)).toHaveLength(1);
+    await expect(repo.createApiKey(OTHER, "hash-b")).rejects.toThrow();
+  });
+
+  it("delete removes the project and all of its data, leaving other projects alone", async () => {
+    const users = new UserRepository(infra.db);
+    const contacts = new ContactRepository(infra.db);
+    const repo = new ProjectRepository(infra.db);
+    for (const pid of [P, OTHER]) {
+      await users.upsertFull(
+        pid,
+        user("u", {
+          segments: ["s"],
+          preferences: {
+            topics: { t: false },
+            channels: { sms: false },
+            quietHours: [{ start: "01:00", end: "02:00" }],
+          },
+        }),
+      );
+      await contacts.upsert(pid, "u", "email", "u@x.com", { topics: { t: false } });
+      await new TemplateRepository(infra.db).upsertMany(pid, [
+        { id: "t", channel: "email", content: {} },
+      ]);
+      await repo.createApiKey(pid, `hash-${pid}`);
+      await infra.sql`INSERT INTO suppressions (project_id, channel, target, reason) VALUES (${pid}, 'email', 'u@x.com', 'manual')`;
+      await infra.sql`INSERT INTO message_logs (project_id, task_id, channel, status) VALUES (${pid}, ${randomUUID()}, 'email', 'delivered')`;
+      await infra.sql`INSERT INTO workflow_definitions (project_id, name, steps) VALUES (${pid}, 'wf', '[]')`;
+      const [inst] =
+        await infra.sql`INSERT INTO workflow_instances (project_id, name) VALUES (${pid}, 'wf') RETURNING id`;
+      await infra.sql`INSERT INTO workflow_steps (project_id, instance_id, step_index, action) VALUES (${pid}, ${inst!.id}, '0', 'run')`;
+    }
+
+    expect(await repo.delete(P)).toBe(true);
+    expect(await repo.delete(P)).toBe(false);
+
+    for (const table of [
+      "users",
+      "templates",
+      "project_api_keys",
+      "suppressions",
+      "message_logs",
+      "workflow_definitions",
+      "workflow_instances",
+      "workflow_steps",
+    ]) {
+      const { mine } = (
+        await infra.sql.unsafe(`SELECT count(*)::int AS mine FROM ${table} WHERE project_id = $1`, [
+          P,
+        ])
+      )[0]!;
+      const { theirs } = (
+        await infra.sql.unsafe(
+          `SELECT count(*)::int AS theirs FROM ${table} WHERE project_id = $1`,
+          [OTHER],
+        )
+      )[0]!;
+      expect({ table, mine }).toEqual({ table, mine: 0 });
+      expect({ table, theirs }).toEqual({ table, theirs: 1 });
+    }
+    const { n } = (await infra.sql`SELECT count(*)::int AS n FROM user_contacts`)[0]!;
+    expect(n).toBe(1);
+  });
+});
+
+// ─── WorkflowRepository ─────────────────────────────────────────────────────
+
+describe("WorkflowRepository", () => {
+  async function instance(pid: string, status = "pending") {
+    const [row] = await infra.sql`
+      INSERT INTO workflow_instances (project_id, name, status) VALUES (${pid}, 'wf', ${status}) RETURNING id`;
+    return row!.id as string;
+  }
+
+  it("lists a project's definitions", async () => {
+    await infra.sql`INSERT INTO workflow_definitions (project_id, name, steps) VALUES (${P}, 'a', '[]'), (${OTHER}, 'b', '[]')`;
+    const defs = await new WorkflowRepository(infra.db).listDefinitions(P);
+    expect(defs.map((d) => d.name)).toEqual(["a"]);
+  });
+
+  it("returns an instance with its steps and waiters, only to its own project", async () => {
+    const id = await instance(P);
+    await infra.sql`INSERT INTO workflow_steps (project_id, instance_id, step_index, action, output) VALUES (${P}, ${id}, '0', 'notify', '{"ok":true}')`;
+    await infra.sql`INSERT INTO workflow_waiters (project_id, instance_id, event_name, match_criteria, expires_at) VALUES (${P}, ${id}, 'paid', '{}', now() + interval '1 hour')`;
+
+    const repo = new WorkflowRepository(infra.db);
+    const got = await repo.getInstance(P, id);
+    expect(got.id).toBe(id);
+    expect(got.steps.map((s: any) => s.action)).toEqual(["notify"]);
+    expect(got.waiters.map((w: any) => w.eventName)).toEqual(["paid"]);
+    expect(await repo.getInstance(OTHER, id)).toBeNull();
+    expect(await repo.getInstance(P, randomUUID())).toBeNull();
+  });
+
+  it("cancels a pending or running instance and drops its waiters", async () => {
+    const repo = new WorkflowRepository(infra.db);
+    const pending = await instance(P, "pending");
+    const running = await instance(P, "running");
+    const done = await instance(P, "completed");
+    await infra.sql`INSERT INTO workflow_waiters (project_id, instance_id, event_name, match_criteria, expires_at) VALUES (${P}, ${pending}, 'e', '{}', now() + interval '1 hour')`;
+
+    expect(await repo.cancelInstance(P, pending)).toBe(true);
+    expect(await repo.cancelInstance(P, running)).toBe(true);
+    expect(await repo.cancelInstance(P, done)).toBe(false);
+    expect(await repo.cancelInstance(OTHER, pending)).toBe(false);
+
+    const { status } = (
+      await infra.sql`SELECT status FROM workflow_instances WHERE id = ${pending}`
+    )[0]!;
+    expect(status).toBe("canceled");
+    const { n } = (await infra.sql`SELECT count(*)::int AS n FROM workflow_waiters`)[0]!;
+    expect(n).toBe(0);
+  });
+});
+
+// ─── SegmentRepository ──────────────────────────────────────────────────────
+
+describe("SegmentRepository", () => {
+  it("lists a project's distinct segments", async () => {
+    const users = new UserRepository(infra.db);
+    await users.upsertManyFull(P, [
+      user("a", { segments: ["vip", "beta"] }),
+      user("b", { segments: ["vip"] }),
+    ]);
+    await users.upsertFull(OTHER, user("c", { segments: ["secret"] }));
+    expect((await new SegmentRepository(infra.db).listSegments(P)).sort()).toEqual(["beta", "vip"]);
+  });
+});
+
+// ─── AdminUserRepository ────────────────────────────────────────────────────
+
+describe("AdminUserRepository", () => {
+  it("creates admins with a normalised email, finds them by email or username, and counts them", async () => {
+    const repo = new AdminUserRepository(infra.db);
+    expect(await repo.count()).toBe(0);
+    const admin = await repo.create({
+      email: "  Boss@Example.COM ",
+      username: " boss ",
+      passwordHash: "h1",
+    });
+    expect(admin.email).toBe("boss@example.com");
+    expect(admin.username).toBe("boss");
+    expect(admin.role).toBe("admin");
+
+    expect((await repo.findByEmailOrUsername("BOSS@example.com"))!.id).toBe(admin.id);
+    expect((await repo.findByEmailOrUsername(" boss "))!.id).toBe(admin.id);
+    expect(await repo.findByEmailOrUsername("nobody")).toBeNull();
+    expect((await repo.findById(admin.id))!.email).toBe("boss@example.com");
+    expect(await repo.findById(randomUUID())).toBeNull();
+    expect(await repo.count()).toBe(1);
+
+    await expect(repo.create({ email: "boss@example.com", passwordHash: "x" })).rejects.toThrow();
+  });
+
+  it("updates a password and never lists hashes", async () => {
+    const repo = new AdminUserRepository(infra.db);
+    const a = await repo.create({ email: "a@x.com", passwordHash: "old" });
+    await repo.create({ email: "b@x.com", passwordHash: "hb", role: "viewer" });
+
+    expect(await repo.updatePassword(a.id, "new")).toBe(true);
+    expect((await repo.findById(a.id))!.passwordHash).toBe("new");
+    expect(await repo.updatePassword(randomUUID(), "x")).toBe(false);
+
+    const list = await repo.list();
+    expect(list.map((u) => u.email)).toEqual(["a@x.com", "b@x.com"]);
+    expect(list.every((u) => !("passwordHash" in u))).toBe(true);
+    expect(list[1]!.role).toBe("viewer");
+  });
+});

tests/integration/runtime.test.ts

@@ -0,0 +1,166 @@
+import { describe, it, expect, afterEach } from "vitest";
+import { spawn } from "node:child_process";
+import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { randomUUID } from "node:crypto";
+import { z } from "zod";
+import {
+  hashPassword,
+  verifyPassword,
+  createAdminSession,
+  getAdminSession,
+  revokeAdminSession,
+  DUMMY_PASSWORD_HASH,
+} from "@/services/auth/index.js";
+import { loadEnv, parseConfig, baseConfigSchema } from "@/config/index.js";
+import { ValidationError } from "@/shared/index.js";
+import { createDatabase, runMigrations } from "@/db/index.js";
+import { useInfra, waitFor, settle } from "./support/infra.js";
+
+/**
+ * Process-level pieces against the real thing: admin sessions in Redis,
+ * configuration read from a real .env file, migrations against Postgres, and
+ * the server shutting down on a real signal.
+ */
+
+const infra = useInfra();
+
+describe("admin sessions", () => {
+  const admin = { id: randomUUID(), email: "a@x.com", username: "a", role: "admin" } as any;
+
+  it("creates a session readable by its token, and revokes it", async () => {
+    const session = await createAdminSession(infra.redis, admin);
+    expect(session.token).toMatch(/^nk_sess_[0-9a-f]{64}$/);
+    expect(await getAdminSession(infra.redis, session.token)).toMatchObject({
+      adminId: admin.id,
+      email: "a@x.com",
+      role: "admin",
+    });
+    expect(await infra.redis.ttl(`notif:session:${session.token}`)).toBeGreaterThan(6 * 24 * 3600);
+
+    await revokeAdminSession(infra.redis, session.token);
+    expect(await getAdminSession(infra.redis, session.token)).toBeNull();
+    await revokeAdminSession(infra.redis, "");
+  });
+
+  it("expires a session at its TTL", async () => {
+    const session = await createAdminSession(infra.redis, admin, 1);
+    await settle(1_100);
+    expect(await getAdminSession(infra.redis, session.token)).toBeNull();
+  });
+
+  it("rejects tokens that are not sessions, and corrupt session data", async () => {
+    expect(await getAdminSession(infra.redis, "nk_live_abc")).toBeNull();
+    expect(await getAdminSession(infra.redis, "")).toBeNull();
+    await infra.redis.set("notif:session:nk_sess_corrupt", "{not json");
+    expect(await getAdminSession(infra.redis, "nk_sess_corrupt")).toBeNull();
+    await infra.redis.set(
+      "notif:session:nk_sess_stale",
+      JSON.stringify({ expiresAt: Date.now() - 1 }),
+    );
+    expect(await getAdminSession(infra.redis, "nk_sess_stale")).toBeNull();
+    expect(await infra.redis.exists("notif:session:nk_sess_stale")).toBe(0);
+  });
+
+  it("hashes passwords with a fresh salt and verifies them", async () => {
+    const [a, b] = await Promise.all([hashPassword("pw"), hashPassword("pw")]);
+    expect(a).not.toBe(b);
+    expect(await verifyPassword("pw", a)).toBe(true);
+    expect(await verifyPassword("PW", a)).toBe(false);
+    expect(await verifyPassword("pw", "garbage")).toBe(false);
+    expect(await verifyPassword("pw", DUMMY_PASSWORD_HASH)).toBe(false);
+  });
+});
+
+describe("configuration", () => {
+  const keys = ["NOTIFKIT_TEST_A", "NOTIFKIT_TEST_B"];
+  afterEach(() => keys.forEach((k) => delete process.env[k]));
+
+  it("loads a .env file without overriding what the environment already set", () => {
+    const dir = mkdtempSync(join(tmpdir(), "nk-env-"));
+    try {
+      writeFileSync(join(dir, ".env"), "NOTIFKIT_TEST_A=from-file\nNOTIFKIT_TEST_B=from-file\n");
+      process.env.NOTIFKIT_TEST_B = "from-env";
+      loadEnv(join(dir, ".env"));
+      expect(process.env.NOTIFKIT_TEST_A).toBe("from-file");
+      expect(process.env.NOTIFKIT_TEST_B).toBe("from-env");
+      loadEnv(join(dir, "missing.env"));
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("applies defaults and rejects invalid values field by field", () => {
+    const cfg = parseConfig(baseConfigSchema, { ADMIN_API_KEY: "  key \n", PIPELINE_FUSED: "1" });
+    expect(cfg).toMatchObject({
+      PORT: 3000,
+      WORKER_CONCURRENCY: 10,
+      ADMIN_API_KEY: "key",
+      PIPELINE_FUSED: true,
+    });
+
+    const err = (() => {
+      try {
+        parseConfig(baseConfigSchema, { PORT: "99999", REDIS_URL: "not a url" });
+      } catch (e) {
+        return e;
+      }
+    })() as ValidationError;
+    expect(err).toBeInstanceOf(ValidationError);
+    expect(err.code).toBe("VALIDATION_ERROR");
+    expect(Object.keys(err.fields!).sort()).toEqual(["PORT", "REDIS_URL"]);
+    expect(parseConfig(z.object({ n: z.coerce.number() }), { n: "4" })).toEqual({ n: 4 });
+  });
+});
+
+describe("database", () => {
+  it("migrations are idempotent on an up-to-date database", async () => {
+    const { db, sql } = createDatabase({ url: infra.dbUrl, maxConnections: 1 });
+    try {
+      await runMigrations(db);
+      const [{ n }] =
+        (await sql`SELECT count(*)::int AS n FROM drizzle.__drizzle_migrations`) as any;
+      expect(n).toBeGreaterThanOrEqual(8);
+      const [{ canceled }] = (await sql`SELECT 'canceled'::workflow_status AS canceled`) as any;
+      expect(canceled).toBe("canceled");
+    } finally {
+      await sql.end();
+    }
+  });
+});
+
+describe("process lifecycle", () => {
+  it.skipIf(process.platform === "win32")(
+    "shuts the server down gracefully on SIGTERM and exits cleanly",
+    async () => {
+      const script = `
+        import { NotifkitServer } from "./src/server.ts";
+        const server = new NotifkitServer({
+          services: ["enricher", "engine", "delivery"],
+          redisUrl: process.env.REDIS_URL, databaseUrl: process.env.DATABASE_URL,
+          logLevel: "silent", autoMigrate: false,
+        });
+        await server.start();
+        console.log("READY");
+      `;
+      const child = spawn("npx", ["--no-install", "tsx", "--eval", script], {
+        env: { ...process.env, REDIS_URL: infra.redisUrl, DATABASE_URL: infra.dbUrl },
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+      let out = "";
+      child.stdout.on("data", (c) => (out += c));
+      const exited = new Promise<number | null>((r) => child.on("exit", (code) => r(code)));
+
+      await waitFor("server ready", () => out.includes("READY"), 60_000);
+      await waitFor(
+        "workers reporting",
+        async () => (await infra.redis.exists("notif:health:engine")) === 1,
+        20_000,
+      );
+      child.kill("SIGTERM");
+      expect(await exited).toBe(0);
+    },
+    90_000,
+  );
+});

tests/integration/scheduler.test.ts

@@ -0,0 +1,410 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { randomUUID } from "node:crypto";
+import {
+  executeSchedulerPoll,
+  rebuildScheduledQueue,
+  SchedulerWorker,
+  SCHEDULER_SENTINEL_KEY,
+} from "@/services/scheduler/main.js";
+import { createPriorityProducers } from "@/workers/bootstrap.js";
+import { StreamConsumer, PendingMessageScanner } from "@/queue/index.js";
+import { STREAMS, CONSUMER_GROUPS, registry, buildStreamEvent } from "@/contracts/index.js";
+import { createLogger } from "@/logger/index.js";
+import { ContactRepository, TemplateRepository, UserRepository } from "@/repositories/index.js";
+import { useInfra, waitFor } from "./support/infra.js";
+
+/**
+ * The scheduler's pieces against real Redis ZSETs and a real
+ * scheduled_payloads table: parking a task, releasing it when due, and
+ * rebuilding the ZSETs from Postgres after Redis loses its data.
+ */
+
+const infra = useInfra();
+const logger = createLogger({ name: "test", level: "silent" });
+
+beforeEach(async () => {
+  await infra.reset();
+});
+
+const shardOf = (taskId: string) => parseInt(taskId.slice(-1), 16) || 0;
+const zkey = (taskId: string) => `notif:scheduled:zset:${shardOf(taskId)}`;
+
+async function park(taskId: string, at: number, payload: Record<string, unknown> = {}) {
+  const full = {
+    projectId: randomUUID(),
+    taskId,
+    channel: "email",
+    priority: "normal",
+    scheduledAt: new Date(at).toISOString(),
+    ...payload,
+  };
+  await infra.sql`INSERT INTO scheduled_payloads (task_id, payload) VALUES (${taskId}, ${JSON.stringify(full)}::jsonb)`;
+  await infra.redis.zadd(
+    zkey(taskId),
+    at,
+    JSON.stringify({ taskId, enrichedEventId: randomUUID(), traceId: `trace-${taskId}` }),
+  );
+  return full;
+}
+
+async function released(stream: string) {
+  return (await infra.redis.xrange(stream, "-", "+")).map(([, f]) => JSON.parse(f[1]!));
+}
+
+async function zsize() {
+  let n = 0;
+  for (let i = 0; i < 16; i++) n += await infra.redis.zcard(`notif:scheduled:zset:${i}`);
+  return n;
+}
+
+describe("executeSchedulerPoll", () => {
+  it("releases due tasks onto the outbound lane for their priority, and only due ones", async () => {
+    const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND");
+    const due = await park("task-a", Date.now() - 1_000, { priority: "critical" });
+    await park("task-b", Date.now() - 1_000, { priority: "low" });
+    await park("task-c", Date.now() + 60_000);
+
+    expect(await executeSchedulerPoll(infra.redis, producers, logger, infra.db)).toBe(false);
+
+    const critical = await released(STREAMS.OUTBOUND_CRITICAL);
+    expect(critical).toHaveLength(1);
+    expect(critical[0]).toMatchObject({
+      type: "notification.dispatched",
+      payload: due,
+      metadata: { traceId: "trace-task-a", source: "scheduler" },
+    });
+    expect(await released(STREAMS.OUTBOUND_LOW)).toHaveLength(1);
+    expect(await released(STREAMS.OUTBOUND_NORMAL)).toHaveLength(0);
+
+    const left = await infra.sql`SELECT task_id FROM scheduled_payloads ORDER BY task_id`;
+    expect(left.map((r) => r.task_id)).toEqual(["task-c"]);
+    expect(await zsize()).toBe(1);
+  });
+
+  it("drops a parked task whose payload was canceled, without sending it", async () => {
+    const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND");
+    await park("task-x", Date.now() - 1_000);
+    await infra.sql`DELETE FROM scheduled_payloads`;
+
+    await executeSchedulerPoll(infra.redis, producers, logger, infra.db);
+    expect(await released(STREAMS.OUTBOUND_NORMAL)).toEqual([]);
+    expect(await zsize()).toBe(0);
+  });
+
+  it("skips a member that is not valid JSON and releases the rest", async () => {
+    const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND");
+    await infra.redis.zadd("notif:scheduled:zset:0", Date.now() - 1, "{not json");
+    await park("task-0", Date.now() - 1_000);
+    await executeSchedulerPoll(infra.redis, producers, logger, infra.db);
+    expect(await released(STREAMS.OUTBOUND_NORMAL)).toHaveLength(1);
+  });
+
+  it("does nothing while another scheduler holds the poll lock", async () => {
+    const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND");
+    await park("task-l", Date.now() - 1_000);
+    await infra.redis.set("notif:lock:scheduler:poll", "someone-else", "EX", 30);
+
+    expect(await executeSchedulerPoll(infra.redis, producers, logger, infra.db)).toBe(false);
+    expect(await released(STREAMS.OUTBOUND_NORMAL)).toEqual([]);
+    // And it must not have released someone else's lock.
+    expect(await infra.redis.get("notif:lock:scheduler:poll")).toBe("someone-else");
+  });
+
+  it("never releases one task twice when schedulers race", async () => {
+    const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND");
+    for (let i = 0; i < 30; i++) await park(`race-${i.toString(16)}`, Date.now() - 1_000);
+    for (let round = 0; round < 10; round++) {
+      await Promise.all(
+        Array.from({ length: 4 }, () =>
+          executeSchedulerPoll(infra.redis, producers, logger, infra.db),
+        ),
+      );
+    }
+    const out = await released(STREAMS.OUTBOUND_NORMAL);
+    expect(out).toHaveLength(30);
+    expect(new Set(out.map((e) => e.payload.taskId)).size).toBe(30);
+  });
+
+  it("asks to be called again while a shard still has a backlog", async () => {
+    const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND");
+    // Every id ends in "0", so all 150 land on one shard (100 per poll).
+    const pipeline = infra.redis.pipeline();
+    const past = Date.now() - 1_000;
+    for (let i = 0; i < 150; i++) {
+      const taskId = `bulk-${i}-0`;
+      pipeline.zadd(zkey(taskId), past, JSON.stringify({ taskId, enrichedEventId: randomUUID() }));
+    }
+    await pipeline.exec();
+    await infra.sql`INSERT INTO scheduled_payloads (task_id, payload)
+      SELECT 'bulk-' || i || '-0', jsonb_build_object('priority', 'normal') FROM generate_series(0, 149) AS i`;
+
+    expect(await executeSchedulerPoll(infra.redis, producers, logger, infra.db)).toBe(true);
+    expect(await executeSchedulerPoll(infra.redis, producers, logger, infra.db)).toBe(false);
+    expect(await released(STREAMS.OUTBOUND_NORMAL)).toHaveLength(150);
+  });
+
+  it("does not delete a task's newer payload when it was re-parked while being released", async () => {
+    // Delivery throttles a released task and parks it again — new send time,
+    // same task id — sometimes before the poll that released it has finished.
+    const base = createPriorityProducers(infra.redis, logger, "OUTBOUND");
+    const retryAt = Date.now() + 5_000;
+    const reparking = {
+      ...base,
+      normal: {
+        publishBatch: async (events: any[]) => {
+          const result = await base.normal.publishBatch(events);
+          for (const e of events) {
+            const payload = {
+              ...e.payload,
+              scheduledAt: new Date(retryAt).toISOString(),
+              throttleAttemptCount: 1,
+            };
+            await infra.sql`
+              INSERT INTO scheduled_payloads (task_id, payload) VALUES (${e.payload.taskId}, ${JSON.stringify(payload)}::jsonb)
+              ON CONFLICT (task_id) DO UPDATE SET payload = EXCLUDED.payload`;
+            await infra.redis.zadd(
+              zkey(e.payload.taskId),
+              retryAt,
+              JSON.stringify({
+                taskId: e.payload.taskId,
+                enrichedEventId: randomUUID(),
+                traceId: "retry",
+              }),
+            );
+          }
+          return result;
+        },
+      },
+    };
+    await park("task-r", Date.now() - 1_000);
+
+    await executeSchedulerPoll(infra.redis, reparking, logger, infra.db);
+
+    const rows = await infra.sql`SELECT payload FROM scheduled_payloads WHERE task_id = 'task-r'`;
+    expect(rows).toHaveLength(1);
+    expect(rows[0]!.payload.throttleAttemptCount).toBe(1);
+  });
+
+  it("keeps releasing the other shards when one cannot be read", async () => {
+    const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND");
+    await infra.redis.set("notif:scheduled:zset:3", "wrong type");
+    await park("task-4", Date.now() - 1_000);
+    await expect(executeSchedulerPoll(infra.redis, producers, logger, infra.db)).resolves.toBe(
+      false,
+    );
+    expect((await released(STREAMS.OUTBOUND_NORMAL)).map((e) => e.payload.taskId)).toEqual([
+      "task-4",
+    ]);
+  });
+});
+
+/**
+ * A parked send was cleared by the engine when it was parked. Whatever the
+ * person did since — hours or days of it, for a `sendAt` or a quiet-hours
+ * deferral — has to stop it at release.
+ */
+describe("executeSchedulerPoll re-checks a parked send when releasing it", () => {
+  let project: string;
+  const users = () => new UserRepository(infra.db);
+  const contacts = () => new ContactRepository(infra.db);
+
+  beforeEach(async () => {
+    project = randomUUID();
+    await infra.sql`INSERT INTO projects (id, name) VALUES (${project}, 'p')`;
+    await new TemplateRepository(infra.db).upsertMany(project, [
+      { id: "promo", channel: "email", content: { subject: "Sale" }, topics: ["marketing"] },
+      { id: "receipt", channel: "email", content: { subject: "Receipt" } },
+    ]);
+    await users().upsertFull(project, { userId: "u", segments: [], preferences: {} });
+    await contacts().upsert(project, "u", "email", "U@X.com");
+  });
+
+  const parkFor = (taskId: string, extra: Record<string, unknown> = {}) =>
+    park(taskId, Date.now() - 1_000, {
+      projectId: project,
+      recipientId: "u",
+      templateId: "promo",
+      destination: "U@X.com",
+      ...extra,
+    });
+
+  async function poll() {
+    await executeSchedulerPoll(
+      infra.redis,
+      createPriorityProducers(infra.redis, logger, "OUTBOUND"),
+      logger,
+      infra.db,
+    );
+    return (await released(STREAMS.OUTBOUND_NORMAL)).map((e) => e.payload.taskId);
+  }
+
+  async function expectDropped(taskId: string) {
+    expect(await poll()).toEqual([]);
+    expect(
+      await infra.sql`SELECT 1 FROM scheduled_payloads WHERE task_id = ${taskId}`,
+    ).toHaveLength(0);
+    expect(await zsize()).toBe(0);
+  }
+
+  it("still releases a send nothing has changed for", async () => {
+    await parkFor("ok-1");
+    expect(await poll()).toEqual(["ok-1"]);
+  });
+
+  it("drops a send to an address suppressed since it was parked", async () => {
+    await parkFor("sup-1");
+    // Stored normalised, while the contact kept its original casing.
+    await infra.sql`INSERT INTO suppressions (project_id, channel, target, reason) VALUES (${project}, 'email', 'u@x.com', 'bounced')`;
+    await expectDropped("sup-1");
+  });
+
+  it("does not let a suppression on another channel or project stop it", async () => {
+    await parkFor("sup-2");
+    await infra.sql`INSERT INTO suppressions (project_id, channel, target, reason) VALUES (${project}, 'sms', 'u@x.com', 'manual')`;
+    await infra.sql`INSERT INTO suppressions (project_id, channel, target, reason) VALUES (${randomUUID()}, 'email', 'u@x.com', 'manual')`;
+    expect(await poll()).toEqual(["sup-2"]);
+  });
+
+  it("drops a send for a topic the user opted out of, and keeps their transactional mail", async () => {
+    await parkFor("topic-1");
+    await parkFor("topic-2", { templateId: "receipt" });
+    await users().upsertFull(project, {
+      userId: "u",
+      segments: [],
+      preferences: { topics: { marketing: false } },
+    });
+    expect(await poll()).toEqual(["topic-2"]);
+  });
+
+  it("drops a send on a channel the user switched off", async () => {
+    await parkFor("chan-1", { templateId: "receipt" });
+    await users().upsertFull(project, {
+      userId: "u",
+      segments: [],
+      preferences: { channels: { email: false } },
+    });
+    await expectDropped("chan-1");
+  });
+
+  it("drops a send to an address that was deactivated, or a user that was deleted", async () => {
+    await parkFor("gone-1");
+    await contacts().deactivate(project, "u", "email", "U@X.com");
+    await expectDropped("gone-1");
+
+    await parkFor("gone-2", { recipientId: "nobody" });
+    await expectDropped("gone-2");
+  });
+});
+
+describe("rebuildScheduledQueue", () => {
+  it("re-queues every parked payload at its send time, page by page, and marks the queue intact", async () => {
+    const at = Date.now() + 3600_000;
+    for (let i = 0; i < 7; i++) await park(`rb-${i}`, at + i);
+    await infra.sql`INSERT INTO scheduled_payloads (task_id, payload) VALUES ('legacy-1', '{"priority":"normal"}')`;
+    await infra.redis.flushdb();
+
+    expect(await rebuildScheduledQueue(infra.redis, infra.db, logger, 3)).toBe(8);
+
+    expect(await zsize()).toBe(8);
+    const score = await infra.redis.zscore(
+      zkey("rb-4"),
+      JSON.stringify({ taskId: "rb-4", enrichedEventId: undefined }),
+    );
+    expect(Number(score)).toBe(at + 4);
+    // A payload with no send time is due now.
+    const legacy = await infra.redis.zrange(zkey("legacy-1"), "0", "-1", "WITHSCORES");
+    expect(Number(legacy[1]!)).toBeLessThanOrEqual(Date.now());
+    expect(await infra.redis.get(SCHEDULER_SENTINEL_KEY)).toBe("1");
+  });
+
+  it("does not duplicate a member that survived", async () => {
+    await park("keep-1", Date.now() + 60_000);
+    await infra.redis.del(zkey("keep-1"));
+    await rebuildScheduledQueue(infra.redis, infra.db, logger);
+    await rebuildScheduledQueue(infra.redis, infra.db, logger);
+    expect(await zsize()).toBe(1);
+  });
+
+  it("marks an empty queue intact too", async () => {
+    expect(await rebuildScheduledQueue(infra.redis, infra.db, logger)).toBe(0);
+    expect(await infra.redis.exists(SCHEDULER_SENTINEL_KEY)).toBe(1);
+  });
+
+  it("lets the poll send a task rebuilt next to a surviving copy only once", async () => {
+    const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND");
+    await park("dup-5", Date.now() - 1_000);
+    await rebuildScheduledQueue(infra.redis, infra.db, logger); // adds a trace-less copy
+    expect(await infra.redis.zcard(zkey("dup-5"))).toBe(2);
+
+    await executeSchedulerPoll(infra.redis, producers, logger, infra.db);
+    expect(await released(STREAMS.OUTBOUND_NORMAL)).toHaveLength(1);
+    expect(await zsize()).toBe(0);
+  });
+});
+
+describe("SchedulerWorker", () => {
+  let worker: SchedulerWorker | undefined;
+  afterEach(async () => {
+    await worker?.stop();
+    worker = undefined;
+  });
+
+  it("parks each scheduled task in its shard at its send time", async () => {
+    const consumer = new StreamConsumer({
+      redis: infra.redis,
+      stream: STREAMS.SCHEDULED,
+      group: CONSUMER_GROUPS.SCHEDULER,
+      consumer: "s1",
+      blockMs: 100,
+      logger,
+    });
+    const pendingScanner = new PendingMessageScanner({
+      redis: infra.redis,
+      stream: STREAMS.SCHEDULED,
+      group: CONSUMER_GROUPS.SCHEDULER,
+      consumer: "s1",
+      logger,
+    });
+    worker = new SchedulerWorker({
+      consumer,
+      pendingScanner,
+      logger,
+      registry,
+      redis: infra.redis,
+    });
+    await worker.start();
+
+    const at = new Date(Date.now() + 60_000).toISOString();
+    const taskId = `${randomUUID()}:abc`;
+    const ev = {
+      ...buildStreamEvent(
+        "notification.scheduled",
+        { projectId: randomUUID(), enrichedEventId: randomUUID(), taskId, scheduledAt: at },
+        "engine",
+        "trace-9",
+      ),
+      id: randomUUID(),
+      timestamp: new Date().toISOString(),
+    };
+    await infra.redis.xadd(STREAMS.SCHEDULED, "*", "data", JSON.stringify(ev));
+    // An invalid payload is skipped, not retried forever.
+    await infra.redis.xadd(
+      STREAMS.SCHEDULED,
+      "*",
+      "data",
+      JSON.stringify({ ...ev, id: randomUUID(), payload: { taskId: 1 } }),
+    );
+
+    await waitFor("parked", async () => (await infra.redis.zcard(zkey(taskId))) === 1);
+    const [member, score] = await infra.redis.zrange(zkey(taskId), "0", "-1", "WITHSCORES");
+    expect(JSON.parse(member!)).toMatchObject({ taskId, traceId: "trace-9" });
+    expect(Number(score)).toBe(Date.parse(at));
+    await waitFor(
+      "both acked",
+      async () =>
+        Number(
+          ((await infra.redis.xpending(STREAMS.SCHEDULED, CONSUMER_GROUPS.SCHEDULER)) as any[])[0],
+        ) === 0,
+    );
+  });
+});

tests/integration/support/fake-internet.ts

@@ -0,0 +1,111 @@
+import http from "node:http";
+import { Agent, getGlobalDispatcher, setGlobalDispatcher, type Dispatcher } from "undici";
+
+/**
+ * Stands in for third-party APIs at the HTTP layer.
+ *
+ * Providers keep calling `fetch("https://api.telegram.org/...")` exactly as in
+ * production; requests to the hosts registered here are routed to a local
+ * server instead, so the real request is serialised, sent, answered with a
+ * real status code and body, and parsed by the provider's own code. Nothing in
+ * the provider is replaced.
+ */
+
+export interface RecordedRequest {
+  method: string;
+  host: string;
+  path: string;
+  headers: http.IncomingHttpHeaders;
+  body: string;
+  json(): any;
+}
+
+export type Reply =
+  | {
+      status?: number;
+      json?: unknown;
+      text?: string;
+      headers?: Record<string, string>;
+      delayMs?: number;
+    }
+  | "hang-up";
+
+export type Handler = (req: RecordedRequest) => Reply | Promise<Reply>;
+
+export interface FakeInternet {
+  requests: RecordedRequest[];
+  /** Answers requests to `host` with `handler`, replacing any previous one. */
+  on(host: string, handler: Handler): void;
+  close(): Promise<void>;
+}
+
+export async function startFakeInternet(): Promise<FakeInternet> {
+  const handlers = new Map<string, Handler>();
+  const requests: RecordedRequest[] = [];
+
+  const server = http.createServer((req, res) => {
+    let body = "";
+    req.setEncoding("utf8");
+    req.on("data", (c) => (body += c));
+    req.on("end", () => void respond());
+    const respond = async () => {
+      // /__host/<host><path>
+      const [, , host, ...rest] = (req.url ?? "/").split("/");
+      const path = `/${rest.join("/")}`;
+      const recorded: RecordedRequest = {
+        method: req.method ?? "GET",
+        host: host ?? "",
+        path,
+        headers: req.headers,
+        body,
+        json: () => JSON.parse(body),
+      };
+      requests.push(recorded);
+      const handler = handlers.get(recorded.host);
+      const reply: Reply = handler
+        ? await handler(recorded)
+        : { status: 404, text: "no fake for host" };
+      if (reply === "hang-up") {
+        req.socket.destroy();
+        return;
+      }
+      if (reply.delayMs) await new Promise((r) => setTimeout(r, reply.delayMs));
+      const headers: Record<string, string> = { ...(reply.headers ?? {}) };
+      let out = reply.text ?? "";
+      if (reply.json !== undefined) {
+        headers["content-type"] ??= "application/json";
+        out = JSON.stringify(reply.json);
+      }
+      res.writeHead(reply.status ?? 200, headers);
+      res.end(out);
+    };
+  });
+  await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
+  const origin = `http://127.0.0.1:${(server.address() as any).port}`;
+
+  const previous = getGlobalDispatcher();
+  const direct = new Agent();
+  class Router extends Agent {
+    override dispatch(
+      opts: Dispatcher.DispatchOptions,
+      handler: Dispatcher.DispatchHandler,
+    ): boolean {
+      const url = new URL(opts.path, opts.origin as string);
+      if (!handlers.has(url.host)) return (previous as any).dispatch(opts, handler);
+      return direct.dispatch({ ...opts, origin, path: `/__host/${url.host}${opts.path}` }, handler);
+    }
+  }
+  setGlobalDispatcher(new Router());
+
+  return {
+    requests,
+    on(host, handler) {
+      handlers.set(host, handler);
+    },
+    async close() {
+      setGlobalDispatcher(previous);
+      await direct.close();
+      await new Promise<void>((r) => server.close(() => r()));
+    },
+  };
+}

tests/integration/support/global-setup.ts

@@ -0,0 +1,76 @@
+import { PostgreSqlContainer, type StartedPostgreSqlContainer } from "@testcontainers/postgresql";
+import { RedisContainer, type StartedRedisContainer } from "@testcontainers/redis";
+import postgres from "postgres";
+import type { GlobalSetupContext } from "vitest/node";
+import { createDatabase, runMigrations } from "../../../src/db/index.js";
+
+/**
+ * One Postgres and one Redis for the whole run.
+ *
+ * Every integration test file gets its own database, cloned from a migrated
+ * template (see `useInfra`), and its own Redis logical database, so files run
+ * in parallel without seeing each other's rows or keys. Starting a container
+ * per file cost several seconds each and was the main reason real-infra tests
+ * were rationed.
+ */
+
+export const TEMPLATE_DB = "notifkit_template";
+
+declare module "vitest" {
+  export interface ProvidedContext {
+    pgAdminUrl: string;
+    redisBaseUrl: string;
+    /** Why the containers could not start, or "" when they did. */
+    infraError: string;
+  }
+}
+
+let pg: StartedPostgreSqlContainer | undefined;
+let redis: StartedRedisContainer | undefined;
+
+export default async function setup({ provide }: GlobalSetupContext) {
+  try {
+    await startContainers(provide);
+  } catch (err) {
+    // Without Docker the unit tests can still run. Each integration file then
+    // fails on its own with this reason, rather than every file in the run
+    // failing here — and rather than integration tests being skipped quietly.
+    if (process.env.CI) throw err;
+    provide("pgAdminUrl", "");
+    provide("redisBaseUrl", "");
+    provide("infraError", err instanceof Error ? err.message : String(err));
+  }
+
+  return async () => {
+    await Promise.all([pg?.stop(), redis?.stop()]);
+  };
+}
+
+async function startContainers(provide: GlobalSetupContext["provide"]) {
+  [pg, redis] = await Promise.all([
+    new PostgreSqlContainer("postgres:16-alpine").start(),
+    new RedisContainer("redis:7-alpine")
+      .withCommand(["redis-server", "--databases", "128"])
+      .start(),
+  ]);
+
+  const adminUrl = pg!.getConnectionUri();
+  const admin = postgres(adminUrl, { max: 1, onnotice: () => {} });
+  await admin.unsafe(`CREATE DATABASE ${TEMPLATE_DB}`);
+  await admin.end();
+
+  const templateUrl = withDatabase(adminUrl, TEMPLATE_DB);
+  const { db, sql } = createDatabase({ url: templateUrl, maxConnections: 1 });
+  await runMigrations(db);
+  await sql.end();
+
+  provide("pgAdminUrl", adminUrl);
+  provide("redisBaseUrl", redis!.getConnectionUrl());
+  provide("infraError", "");
+}
+
+export function withDatabase(url: string, database: string): string {
+  const parsed = new URL(url);
+  parsed.pathname = `/${database}`;
+  return parsed.toString();
+}

tests/integration/support/infra.ts

@@ -0,0 +1,136 @@
+import { afterAll, beforeAll, inject } from "vitest";
+import { randomBytes } from "node:crypto";
+import postgres from "postgres";
+import { Redis } from "ioredis";
+import { drizzle } from "drizzle-orm/postgres-js";
+import type { Db, Sql } from "@/db/index.js";
+import * as schema from "@/db/schema.js";
+import { registerCustomCommands } from "@/redis/index.js";
+import { TEMPLATE_DB, withDatabase } from "./global-setup.js";
+
+export interface Infra {
+  /** Connection string for this file's private database. */
+  dbUrl: string;
+  /** Connection string for this file's private Redis logical database. */
+  redisUrl: string;
+  sql: Sql;
+  db: Db;
+  /** A plain client with the Lua commands registered. */
+  redis: Redis;
+  /** Empties every table and the Redis database. */
+  reset(): Promise<void>;
+}
+
+/**
+ * Registers hooks that give the calling file a real, empty Postgres database
+ * (migrated) and a real, empty Redis database. The returned object is filled
+ * in by `beforeAll`, so read its fields inside hooks and tests, not at import.
+ */
+export function useInfra(): Infra {
+  const infra = {} as Infra;
+  let dbName = "";
+  let adminUrl = "";
+
+  beforeAll(async () => {
+    const infraError = inject("infraError");
+    if (infraError) {
+      throw new Error(
+        `Integration tests need Docker for Postgres and Redis, and it is not available: ${infraError}`,
+      );
+    }
+    adminUrl = inject("pgAdminUrl");
+    dbName = `t_${process.env.VITEST_POOL_ID ?? "0"}_${randomBytes(6).toString("hex")}`;
+    await withAdmin(adminUrl, async (admin) => {
+      // Two files cloning the template at the same moment can collide on its
+      // lock; a short retry absorbs that.
+      for (let attempt = 0; ; attempt++) {
+        try {
+          await admin.unsafe(`CREATE DATABASE ${dbName} TEMPLATE ${TEMPLATE_DB}`);
+          break;
+        } catch (err: any) {
+          if (attempt >= 20 || !String(err?.message).includes("being accessed")) throw err;
+          await new Promise((r) => setTimeout(r, 100 + Math.random() * 200));
+        }
+      }
+    });
+
+    infra.dbUrl = withDatabase(adminUrl, dbName);
+    // The production pool's 10s statement timeout exists to shed load; on a
+    // test machine running the chaos suites alongside, a trivial query can wait
+    // that long for CPU. The code under test still gets its own pools.
+    const sql = postgres(infra.dbUrl, {
+      max: 5,
+      onnotice: () => {},
+      connection: { statement_timeout: 60_000 as any },
+    });
+    const db = drizzle(sql, { schema });
+    infra.sql = sql;
+    infra.db = db;
+
+    const index = (Number(process.env.VITEST_POOL_ID ?? "1") % 127) + 1;
+    infra.redisUrl = `${inject("redisBaseUrl")}/${index}`;
+    infra.redis = new Redis(infra.redisUrl, { maxRetriesPerRequest: null });
+    registerCustomCommands(infra.redis);
+    await infra.redis.flushdb();
+
+    infra.reset = async () => {
+      const tables = await sql<{ tablename: string }[]>`
+        SELECT tablename FROM pg_tables WHERE schemaname = 'public'`;
+      if (tables.length > 0) {
+        await sql.unsafe(
+          `TRUNCATE ${tables.map((t) => `"${t.tablename}"`).join(", ")} RESTART IDENTITY CASCADE`,
+        );
+      }
+      await infra.redis.flushdb();
+    };
+  }, 120_000);
+
+  afterAll(async () => {
+    infra.redis?.disconnect();
+    await infra.sql?.end({ timeout: 5 }).catch(() => {});
+    if (dbName) {
+      await withAdmin(adminUrl, (admin) =>
+        admin.unsafe(`DROP DATABASE IF EXISTS ${dbName} WITH (FORCE)`),
+      ).catch(() => {});
+    }
+  }, 60_000);
+
+  return infra;
+}
+
+async function withAdmin<T>(url: string, fn: (admin: postgres.Sql) => Promise<T>): Promise<T> {
+  const admin = postgres(url, { max: 1, onnotice: () => {} });
+  try {
+    return await fn(admin);
+  } finally {
+    await admin.end({ timeout: 5 });
+  }
+}
+
+/** Polls `condition` until it holds, failing with `what` on timeout. */
+export async function waitFor(
+  what: string,
+  condition: () => Promise<boolean> | boolean,
+  timeoutMs = 20_000,
+  intervalMs = 50,
+): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  let lastErr: unknown;
+  while (Date.now() < deadline) {
+    try {
+      if (await condition()) return;
+    } catch (err) {
+      lastErr = err;
+    }
+    await new Promise((r) => setTimeout(r, intervalMs));
+  }
+  throw new Error(
+    `timed out after ${timeoutMs}ms waiting for: ${what}` +
+      (lastErr ? ` (last error: ${(lastErr as Error).message})` : ""),
+  );
+}
+
+/** Waits `ms`, for asserting that something does *not* happen. */
+export function settle(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}

tests/integration/support/pipeline-cases.ts

@@ -0,0 +1,794 @@
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { randomUUID } from "node:crypto";
+import { STREAMS, buildStreamEvent } from "@/contracts/index.js";
+import { globalEmitter } from "@/shared/index.js";
+import { useInfra, waitFor, settle } from "./infra.js";
+import {
+  RecordingTransport,
+  startNotifkit,
+  apiClient,
+  type Running,
+  type Api,
+} from "./pipeline.js";
+
+/**
+ * The delivery guarantees notifkit makes, checked end to end: a request goes
+ * in through the HTTP API, and the assertion is on what the provider was
+ * actually asked to send — never on which internal function was called.
+ *
+ * Run twice: once with every stage hopping through Redis streams, and once
+ * with the fused in-process pipeline, since the two take different code paths.
+ */
+export function definePipelineSuite(mode: { fused: boolean }) {
+  const infra = useInfra();
+  let app: Running;
+
+  // "bounce…" addresses hard-bounce: permanent, so delivery fails them at once
+  // rather than retrying.
+  const email = new RecordingTransport("email", {
+    behaviour: (task) =>
+      task.destination?.startsWith("bounce")
+        ? { success: false, error: "mailbox unavailable", retryable: false }
+        : { success: true, providerMessageId: `pm-${task.taskId}` },
+  });
+  const sms = new RecordingTransport("sms");
+  const push = new RecordingTransport("push", {
+    behaviour: (task) =>
+      task.destination === "dead-token"
+        ? { success: false, invalidToken: true }
+        : { success: true, providerMessageId: `push-${task.taskId}` },
+  });
+  // A provider that only takes one message every two seconds.
+  const whatsapp = new RecordingTransport("whatsapp", { limits: { limit: 1, windowSeconds: 2 } });
+
+  beforeAll(async () => {
+    app = await startNotifkit(infra, {
+      fusedPipeline: mode.fused,
+      providers: [email, sms, push, whatsapp],
+      env: {
+        RATE_LIMIT_PER_HOUR: "100000",
+        SEGMENT_MAX_USERS: "5",
+        UNSUBSCRIBE_SECRET: "an-unsubscribe-secret-for-tests",
+        PUBLIC_URL: "https://notify.example.com",
+      },
+    });
+  }, 120_000);
+
+  afterAll(async () => {
+    await app?.stop();
+  }, 60_000);
+
+  /** A fresh project with the usual templates, so tests never see each other's data. */
+  async function project() {
+    const p = await app.project();
+    const res = await p.api("PUT", "/v1/templates", {
+      templates: [
+        {
+          id: "receipt",
+          channel: "email",
+          content: {
+            subject: "Receipt for {{name}}",
+            html: "<p>Hi {{name}}</p>",
+            text: "Hi {{name}}, raw: {{{name}}}",
+          },
+        },
+        {
+          id: "news",
+          channel: "email",
+          topic: "newsletter",
+          content: { subject: "News", text: "Hello" },
+        },
+        { id: "otp", channel: "sms", content: { text: "Code {{code}}" } },
+      ],
+    });
+    expect(res.status).toBe(200);
+    return p;
+  }
+
+  async function addUser(api: Api, body: Record<string, unknown>) {
+    const res = await api("POST", "/v1/users", body);
+    expect(res.status).toBe(201);
+  }
+
+  async function notify(api: Api, body: Record<string, unknown>, headers?: Record<string, string>) {
+    const res = await api("POST", "/v1/notify", body, headers);
+    expect(res.status).toBe(202);
+    return res.body;
+  }
+
+  const allSent = (projectId: string) => [
+    ...email.for(projectId),
+    ...sms.for(projectId),
+    ...push.for(projectId),
+    ...whatsapp.for(projectId),
+  ];
+
+  /**
+   * Nothing delivered after the pipeline has had time to act. A marker
+   * notification sent afterwards to a control user proves the pipeline is
+   * running, so an empty result is not just a slow one.
+   */
+  async function expectNothingSentFor(
+    p: { id: string; api: Api },
+    filter: (t: any) => boolean = () => true,
+  ) {
+    const control = `control-${randomUUID().slice(0, 6)}`;
+    await addUser(p.api, { id: control, email: `${control}@x.com` });
+    await notify(p.api, { user: control, template: "receipt", data: { name: "c" } });
+    await waitFor("control delivered", () =>
+      email.for(p.id).some((t) => t.recipientId === control),
+    );
+    await settle(300);
+    expect(
+      allSent(p.id)
+        .filter((t) => t.recipientId !== control)
+        .filter(filter),
+    ).toEqual([]);
+  }
+
+  describe(`pipeline (${mode.fused ? "fused" : "streamed"})`, () => {
+    it("delivers one rendered email per notify, and records it", async () => {
+      const p = await project();
+      await addUser(p.api, { id: "u1", email: "u1@x.com" });
+
+      await notify(p.api, { user: "u1", template: "receipt", data: { name: "<Al>\r\nBcc: x" } });
+
+      await waitFor("delivered", () => email.for(p.id).length === 1);
+      const [task] = email.for(p.id);
+      expect(task!.destination).toBe("u1@x.com");
+      expect(task!.renderedContent.content).toEqual({
+        subject: "Receipt for <Al> Bcc: x",
+        html: "<p>Hi &lt;Al&gt;\r\nBcc: x</p>",
+        text: "Hi <Al>\r\nBcc: x, raw: <Al>\r\nBcc: x",
+      });
+
+      await waitFor("logged", async () => {
+        const rows =
+          await infra.sql`SELECT kind, status FROM message_logs WHERE task_id = ${task!.taskId} ORDER BY kind`;
+        return rows.length === 2;
+      });
+      const rows =
+        await infra.sql`SELECT kind, status, provider_message_id FROM message_logs WHERE task_id = ${task!.taskId} ORDER BY kind`;
+      expect(rows.map((r) => [r.kind, r.status])).toEqual([
+        ["attempt", "delivered"],
+        ["dispatched", "dispatched"],
+      ]);
+      expect(rows[0]!.provider_message_id).toBe(`pm-${task!.taskId}`);
+
+      const status = await p.api("GET", `/v1/notifications/${encodeURIComponent(task!.taskId)}`);
+      expect(status.status).toBe(200);
+      expect(status.body.status).toBe("delivered");
+
+      const [outbox] =
+        await infra.sql`SELECT provider_message_id FROM delivery_outbox WHERE task_id = ${task!.taskId}`;
+      expect(outbox!.provider_message_id).toBe(`pm-${task!.taskId}`);
+
+      await settle(300);
+      expect(email.for(p.id)).toHaveLength(1);
+    });
+
+    it("sends once per requested channel and once per address", async () => {
+      const p = await project();
+      await addUser(p.api, { id: "u", email: ["a@x.com", "b@x.com"], phone: "+15550001" });
+
+      await notify(p.api, { user: "u", template: "receipt", channels: ["email", "sms"] });
+
+      await waitFor("three sends", () => allSent(p.id).length === 3);
+      expect(
+        email
+          .for(p.id)
+          .map((t) => t.destination)
+          .sort(),
+      ).toEqual(["a@x.com", "b@x.com"]);
+      expect(sms.for(p.id).map((t) => t.destination)).toEqual(["+15550001"]);
+      await settle(300);
+      expect(allSent(p.id)).toHaveLength(3);
+    });
+
+    it("creates inline users on the fly", async () => {
+      const p = await project();
+      await notify(p.api, { user: { id: "inline", email: "inline@x.com" }, template: "receipt" });
+      await waitFor("delivered", () => email.for(p.id).length === 1);
+      expect(email.for(p.id)[0]!.destination).toBe("inline@x.com");
+    });
+
+    it("reaches every member of a segment exactly once, and only in this project", async () => {
+      const p = await project();
+      const other = await project();
+      for (const id of ["s1", "s2", "s3"])
+        await addUser(p.api, { id, email: `${id}@x.com`, segments: ["vip"] });
+      await addUser(p.api, { id: "outsider", email: "out@x.com" });
+      await addUser(other.api, { id: "foreign", email: "foreign@x.com", segments: ["vip"] });
+
+      await notify(p.api, { segment: "vip", template: "receipt" });
+
+      await waitFor("3 delivered", () => email.for(p.id).length === 3);
+      await settle(500);
+      expect(
+        email
+          .for(p.id)
+          .map((t) => t.recipientId)
+          .sort(),
+      ).toEqual(["s1", "s2", "s3"]);
+      expect(email.for(other.id)).toEqual([]);
+    });
+
+    it("reaches users subscribed to a topic", async () => {
+      const p = await project();
+      await addUser(p.api, {
+        id: "fan",
+        email: "fan@x.com",
+        preferences: { topics: { launches: true } },
+      });
+      await addUser(p.api, {
+        id: "meh",
+        email: "meh@x.com",
+        preferences: { topics: { launches: false } },
+      });
+
+      await notify(p.api, { topic: "launches", template: "receipt" });
+
+      await waitFor("delivered", () => email.for(p.id).length === 1);
+      expect(email.for(p.id)[0]!.recipientId).toBe("fan");
+    });
+
+    it("refuses a segment larger than the fan-out limit, and says so", async () => {
+      const p = await project();
+      for (let i = 0; i < 6; i++)
+        await addUser(p.api, { id: `big${i}`, email: `big${i}@x.com`, segments: ["huge"] });
+
+      const failures: unknown[][] = [];
+      const onFailed = (...args: unknown[]) => failures.push(args);
+      app.server.on("notification:failed", onFailed);
+      try {
+        await notify(p.api, { segment: "huge", template: "receipt" });
+        await waitFor("failure reported", () => failures.length > 0);
+        expect(String(failures[0]![1])).toMatch(/exceeds limit/);
+        await expectNothingSentFor(p);
+      } finally {
+        app.server.off("notification:failed", onFailed);
+      }
+    });
+
+    it("sends nothing to an unknown user", async () => {
+      const p = await project();
+      await notify(p.api, { user: "ghost", template: "receipt" });
+      await expectNothingSentFor(p);
+    });
+
+    it("sends nothing for a template that does not exist", async () => {
+      const p = await project();
+      await addUser(p.api, { id: "u", email: "u@x.com" });
+      await notify(p.api, { user: "u", template: "missing" });
+      await expectNothingSentFor(p);
+    });
+
+    it("sends once for repeated requests carrying the same idempotency key", async () => {
+      const p = await project();
+      await addUser(p.api, { id: "u", email: "u@x.com" });
+      const key = `order-${randomUUID()}`;
+      await Promise.all([
+        notify(p.api, { user: "u", template: "receipt" }, { "x-idempotency-key": key }),
+        notify(p.api, { user: "u", template: "receipt" }, { "x-idempotency-key": key }),
+        notify(p.api, { user: "u", template: "receipt" }, { "x-idempotency-key": key }),
+      ]);
+      await waitFor("delivered", () => email.for(p.id).length >= 1);
+      await settle(1_000);
+      expect(email.for(p.id)).toHaveLength(1);
+    });
+
+    it("sends once when the same inbound event is read twice", async () => {
+      const p = await project();
+      await addUser(p.api, { id: "u", email: "u@x.com" });
+      const event = {
+        ...buildStreamEvent(
+          "notification.requested",
+          { projectId: p.id, target: { type: "user", userId: "u" }, templateId: "receipt" },
+          "test",
+        ),
+        id: randomUUID(),
+        timestamp: new Date().toISOString(),
+      };
+      await infra.redis.xadd(STREAMS.INBOUND_NORMAL, "*", "data", JSON.stringify(event));
+      await infra.redis.xadd(STREAMS.INBOUND_NORMAL, "*", "data", JSON.stringify(event));
+
+      await waitFor("delivered", () => email.for(p.id).length >= 1);
+      await settle(1_000);
+      expect(email.for(p.id)).toHaveLength(1);
+    });
+
+    describe("legacy notification.created events", () => {
+      async function legacy(projectId: string, recipientId: string, templateId: string) {
+        const event = {
+          ...buildStreamEvent(
+            "notification.created",
+            { projectId, recipientId, channel: "email", templateId, payload: { name: "Legacy" } },
+            "test",
+          ),
+          id: randomUUID(),
+          timestamp: new Date().toISOString(),
+        };
+        await infra.redis.xadd(STREAMS.INBOUND_NORMAL, "*", "data", JSON.stringify(event));
+      }
+
+      it("are still delivered", async () => {
+        const p = await project();
+        await addUser(p.api, { id: "old", email: "old@x.com" });
+        await legacy(p.id, "old", "receipt");
+        await waitFor("delivered", () => email.for(p.id).length === 1);
+        expect(email.for(p.id)[0]!.renderedContent.content).toMatchObject({
+          subject: "Receipt for Legacy",
+        });
+      });
+
+      it("respect the user's topic opt-outs", async () => {
+        const p = await project();
+        await addUser(p.api, {
+          id: "old",
+          email: "old@x.com",
+          preferences: { topics: { newsletter: false } },
+        });
+        await legacy(p.id, "old", "news");
+        await expectNothingSentFor(p);
+      });
+    });
+
+    describe("consent", () => {
+      it("does not send a topic's mail to a user who opted out of that topic", async () => {
+        const p = await project();
+        await addUser(p.api, {
+          id: "u",
+          email: "u@x.com",
+          preferences: { topics: { newsletter: false } },
+        });
+        await notify(p.api, { user: "u", template: "news" });
+        await expectNothingSentFor(p);
+      });
+
+      it("still sends transactional mail to that user", async () => {
+        const p = await project();
+        await addUser(p.api, {
+          id: "u",
+          email: "u@x.com",
+          preferences: { topics: { newsletter: false } },
+        });
+        await notify(p.api, { user: "u", template: "receipt" });
+        await waitFor("delivered", () => email.for(p.id).length === 1);
+      });
+
+      it("skips only the address that opted out of the topic", async () => {
+        const p = await project();
+        await addUser(p.api, {
+          id: "u",
+          contacts: [
+            {
+              channel: "email",
+              target: "work@x.com",
+              preferences: { topics: { newsletter: false } },
+            },
+            { channel: "email", target: "home@x.com" },
+          ],
+        });
+        await notify(p.api, { user: "u", template: "news" });
+        await waitFor("delivered", () => email.for(p.id).length >= 1);
+        await settle(500);
+        expect(email.for(p.id).map((t) => t.destination)).toEqual(["home@x.com"]);
+      });
+
+      it("does not use a channel the user switched off", async () => {
+        const p = await project();
+        await addUser(p.api, {
+          id: "u",
+          email: "u@x.com",
+          phone: "+1555",
+          preferences: { channels: { sms: false } },
+        });
+        await notify(p.api, { user: "u", template: "receipt", channels: ["email", "sms"] });
+        await waitFor("email delivered", () => email.for(p.id).length === 1);
+        await settle(500);
+        expect(sms.for(p.id)).toEqual([]);
+      });
+
+      it("never sends to a suppressed address, whatever its case, even when critical", async () => {
+        const p = await project();
+        const res = await p.api("POST", "/v1/suppressions", {
+          channel: "email",
+          target: "Blocked@X.com",
+          reason: "complained",
+        });
+        expect(res.status).toBe(201);
+        await addUser(p.api, { id: "u", email: "blocked@x.com" });
+        await notify(p.api, { user: "u", template: "receipt", priority: "critical" });
+        await expectNothingSentFor(p, (t) => t.recipientId === "u");
+      });
+
+      it("puts a working one-click unsubscribe on topic mail, and honours it", async () => {
+        const p = await project();
+        await addUser(p.api, { id: "u", email: "u@x.com" });
+        await notify(p.api, { user: "u", template: "news" });
+        await waitFor("delivered", () => email.for(p.id).length === 1);
+
+        const headers = email.for(p.id)[0]!.deliveryOptions.headers!;
+        expect(headers["List-Unsubscribe-Post"]).toBe("List-Unsubscribe=One-Click");
+        const url = new URL(headers["List-Unsubscribe"]!.slice(1, -1));
+        expect(url.origin).toBe("https://notify.example.com");
+
+        // What a mail client does when the recipient presses the button.
+        const oneClick = await apiClient(app.baseUrl)(
+          "POST",
+          `${url.pathname}${url.search}`,
+          "List-Unsubscribe=One-Click",
+          {
+            "content-type": "application/x-www-form-urlencoded",
+          },
+        );
+        expect(oneClick.status).toBe(200);
+
+        const prefs = await p.api("GET", "/v1/users/u/preferences");
+        expect(prefs.body.topics).toEqual({ newsletter: false });
+      });
+
+      it("puts no unsubscribe link on transactional mail", async () => {
+        const p = await project();
+        await addUser(p.api, { id: "u", email: "u@x.com" });
+        await notify(p.api, { user: "u", template: "receipt" });
+        await waitFor("delivered", () => email.for(p.id).length === 1);
+        expect(email.for(p.id)[0]!.deliveryOptions.headers).toBeUndefined();
+      });
+    });
+
+    describe("throttling", () => {
+      it("caps non-critical sends per user at the project's limit", async () => {
+        const p = await project();
+        const patched = await app.admin("PATCH", `/v1/projects/${p.id}`, { throttleLimit: 2 });
+        expect(patched.status).toBe(200);
+        await addUser(p.api, { id: "u", email: "u@x.com" });
+
+        for (let i = 0; i < 4; i++) await notify(p.api, { user: "u", template: "receipt" });
+
+        await waitFor("two delivered", () => email.for(p.id).length === 2);
+        await settle(1_000);
+        expect(email.for(p.id)).toHaveLength(2);
+      });
+
+      it("lets critical sends through the per-user cap", async () => {
+        const p = await project();
+        await app.admin("PATCH", `/v1/projects/${p.id}`, { throttleLimit: 1 });
+        await addUser(p.api, { id: "u", email: "u@x.com" });
+        for (let i = 0; i < 3; i++)
+          await notify(p.api, { user: "u", template: "receipt", priority: "critical" });
+        await waitFor("three delivered", () => email.for(p.id).length === 3);
+      });
+
+      it("defers sends over a provider's rate limit instead of dropping them", async () => {
+        const p = await project();
+        await p.api("PUT", "/v1/templates", {
+          templates: [{ id: "wa", channel: "whatsapp", content: { text: "hi" } }],
+        });
+        await addUser(p.api, { id: "u", contacts: [{ channel: "whatsapp", target: "+1777" }] });
+
+        for (let i = 0; i < 3; i++)
+          await notify(p.api, { user: "u", template: "wa", channels: ["whatsapp"] });
+
+        await waitFor(
+          "all three eventually delivered",
+          () => whatsapp.for(p.id).length === 3,
+          60_000,
+        );
+      }, 90_000);
+    });
+
+    describe("fallback", () => {
+      it("moves to the next channel when the first fails", async () => {
+        const p = await project();
+        await addUser(p.api, { id: "u", email: "bounce@x.com", phone: "+1555" });
+        await notify(p.api, {
+          user: "u",
+          template: "receipt",
+          channels: ["email", "sms"],
+          fallback: true,
+        });
+        await waitFor("sms fallback delivered", () => sms.for(p.id).length === 1);
+        expect(email.for(p.id)).toEqual([]);
+      });
+
+      it("counts a notification once against the user's cap, however many channels it falls back through", async () => {
+        const p = await project();
+        await app.admin("PATCH", `/v1/projects/${p.id}`, { throttleLimit: 1 });
+        await addUser(p.api, { id: "u", email: "bounce-cap@x.com", phone: "+1555" });
+
+        await notify(p.api, {
+          user: "u",
+          template: "receipt",
+          channels: ["email", "sms"],
+          fallback: true,
+        });
+        await waitFor("sms fallback delivered", () => sms.for(p.id).length === 1);
+
+        // The cap itself still holds: a second notification is over it.
+        await notify(p.api, { user: "u", template: "receipt", channels: ["sms"] });
+        await settle(1_500);
+        expect(sms.for(p.id)).toHaveLength(1);
+      });
+
+      it("moves to the next channel when the user has no address on the first", async () => {
+        const p = await project();
+        await addUser(p.api, { id: "u", phone: "+1555" });
+        await notify(p.api, {
+          user: "u",
+          template: "receipt",
+          channels: ["email", "sms"],
+          fallback: true,
+        });
+        await waitFor("sms delivered", () => sms.for(p.id).length === 1, 10_000);
+      });
+
+      it("does not use the fallback when the first channel works", async () => {
+        const p = await project();
+        await addUser(p.api, { id: "u", email: "ok@x.com", phone: "+1555" });
+        await notify(p.api, {
+          user: "u",
+          template: "receipt",
+          channels: ["email", "sms"],
+          fallback: true,
+        });
+        await waitFor("email delivered", () => email.for(p.id).length === 1);
+        await settle(500);
+        expect(sms.for(p.id)).toEqual([]);
+      });
+    });
+
+    describe("failures", () => {
+      it("records a failed send and dead-letters it where the project can see and replay it", async () => {
+        const p = await project();
+        await addUser(p.api, { id: "u", email: "bounce-dlq@x.com" });
+        await notify(p.api, { user: "u", template: "receipt" });
+
+        await waitFor("failure logged", async () => {
+          const rows =
+            await infra.sql`SELECT status FROM message_logs WHERE project_id = ${p.id} AND status = 'failed'`;
+          return rows.length === 1;
+        });
+
+        let dlq: any;
+        await waitFor("visible in the project's DLQ", async () => {
+          dlq = await p.api("GET", "/v1/dlq");
+          return dlq.body.messages?.length === 1;
+        });
+        const entry = dlq.body.messages[0];
+        expect(entry.eventType).toBe("notification.dispatched");
+        expect(entry.payload.projectId).toBe(p.id);
+
+        // The provider recovers; replaying the entry now delivers it.
+        email.behaviour = (task) => ({ success: true, providerMessageId: `pm-${task.taskId}` });
+        try {
+          const replay = await p.api("POST", "/v1/dlq/replay", { id: entry.id });
+          expect(replay.status).toBe(200);
+          await waitFor("replayed and delivered", () => email.for(p.id).length === 1);
+          expect((await p.api("GET", "/v1/dlq")).body.messages).toEqual([]);
+        } finally {
+          email.behaviour = (task) =>
+            task.destination?.startsWith("bounce")
+              ? { success: false, error: "mailbox unavailable", retryable: false }
+              : { success: true, providerMessageId: `pm-${task.taskId}` };
+        }
+      });
+
+      it("deactivates a push token the provider says is dead", async () => {
+        const p = await project();
+        await addUser(p.api, { id: "u", pushToken: ["dead-token", "live-token"] });
+        await notify(p.api, { user: "u", template: "receipt", channels: ["push"] });
+
+        await waitFor("live token delivered", () => push.for(p.id).length === 1);
+        expect(push.for(p.id)[0]!.destination).toBe("live-token");
+        await waitFor("dead token deactivated", async () => {
+          const contacts = await p.api("GET", "/v1/users/u/contacts");
+          return (
+            contacts.body.contacts.find((c: any) => c.target === "dead-token")?.active === false
+          );
+        });
+      });
+    });
+
+    describe("scheduling", () => {
+      it("holds a scheduled send until its time, then sends it once", async () => {
+        const p = await project();
+        await addUser(p.api, { id: "u", email: "u@x.com" });
+        const sendAt = new Date(Date.now() + 3_000);
+        await notify(p.api, { user: "u", template: "receipt", sendAt: sendAt.toISOString() });
+
+        await waitFor(
+          "parked",
+          async () =>
+            (await p.api("GET", "/v1/notifications/scheduled")).body.scheduled?.length === 1,
+        );
+        expect(email.for(p.id)).toEqual([]);
+
+        await waitFor("sent", () => email.for(p.id).length === 1, 20_000);
+        expect(Date.now()).toBeGreaterThanOrEqual(sendAt.getTime());
+        await waitFor(
+          "unparked",
+          async () =>
+            (await p.api("GET", "/v1/notifications/scheduled")).body.scheduled?.length === 0,
+        );
+        await settle(500);
+        expect(email.for(p.id)).toHaveLength(1);
+      }, 30_000);
+
+      it("does not send a scheduled notification that was canceled", async () => {
+        const p = await project();
+        await addUser(p.api, { id: "u", email: "u@x.com" });
+        await notify(p.api, {
+          user: "u",
+          template: "receipt",
+          sendAt: new Date(Date.now() + 3_000).toISOString(),
+        });
+
+        let scheduled: any[] = [];
+        await waitFor("parked", async () => {
+          scheduled = (await p.api("GET", "/v1/notifications/scheduled")).body.scheduled ?? [];
+          return scheduled.length === 1;
+        });
+        const cancel = await p.api(
+          "DELETE",
+          `/v1/notifications/${encodeURIComponent(scheduled[0].taskId)}`,
+        );
+        expect(cancel.status).toBe(200);
+
+        await settle(8_000);
+        expect(email.for(p.id)).toEqual([]);
+      }, 30_000);
+
+      describe("quiet hours", () => {
+        const hhmm = (d: Date) => d.toISOString().slice(11, 16);
+        const minutes = (n: number) => new Date(Date.now() + n * 60_000);
+
+        async function parkedSendTime(p: { api: Api }): Promise<number> {
+          let scheduled: any[] = [];
+          await waitFor("parked", async () => {
+            scheduled = (await p.api("GET", "/v1/notifications/scheduled")).body.scheduled ?? [];
+            return scheduled.length === 1;
+          });
+          return Date.parse(scheduled[0].payload.scheduledAt);
+        }
+
+        it("defers a send during the user's quiet hours until they end", async () => {
+          const p = await project();
+          const end = minutes(60);
+          await addUser(p.api, {
+            id: "u",
+            email: "u@x.com",
+            timezone: "UTC",
+            preferences: { quietHours: [{ start: hhmm(minutes(-60)), end: hhmm(end) }] },
+          });
+          await notify(p.api, { user: "u", template: "receipt" });
+
+          const at = await parkedSendTime(p);
+          expect(Math.abs(at - end.getTime())).toBeLessThan(61_000);
+          expect(email.for(p.id)).toEqual([]);
+        });
+
+        it("sends critical notifications during quiet hours", async () => {
+          const p = await project();
+          await addUser(p.api, {
+            id: "u",
+            email: "u@x.com",
+            timezone: "UTC",
+            preferences: { quietHours: [{ start: hhmm(minutes(-60)), end: hhmm(minutes(60)) }] },
+          });
+          await notify(p.api, { user: "u", template: "receipt", priority: "critical" });
+          await waitFor("delivered", () => email.for(p.id).length === 1);
+        });
+
+        it("keeps a send scheduled for next week on next week, even if it is quiet hours now", async () => {
+          const p = await project();
+          const nextWeek = new Date(Date.now() + 7 * 24 * 3600_000 + 3 * 3600_000);
+          await addUser(p.api, {
+            id: "u",
+            email: "u@x.com",
+            timezone: "UTC",
+            preferences: { quietHours: [{ start: hhmm(minutes(-60)), end: hhmm(minutes(60)) }] },
+          });
+          await notify(p.api, { user: "u", template: "receipt", sendAt: nextWeek.toISOString() });
+
+          expect(await parkedSendTime(p)).toBe(nextWeek.getTime());
+        });
+
+        it("defers a scheduled send that would land inside quiet hours", async () => {
+          const p = await project();
+          const quietStart = minutes(30);
+          const quietEnd = minutes(120);
+          await addUser(p.api, {
+            id: "u",
+            email: "u@x.com",
+            timezone: "UTC",
+            preferences: { quietHours: [{ start: hhmm(quietStart), end: hhmm(quietEnd) }] },
+          });
+          await notify(p.api, {
+            user: "u",
+            template: "receipt",
+            sendAt: minutes(60).toISOString(),
+          });
+
+          const at = await parkedSendTime(p);
+          expect(Math.abs(at - quietEnd.getTime())).toBeLessThan(61_000);
+        });
+      });
+    });
+
+    describe("reporting", () => {
+      it("attributes every message to its campaign", async () => {
+        const p = await project();
+        for (const id of ["c1", "c2"])
+          await addUser(p.api, { id, email: `${id}@x.com`, segments: ["camp"] });
+        await notify(p.api, { segment: "camp", template: "receipt", campaign: "spring-sale" });
+        await waitFor("two delivered", () => email.for(p.id).length === 2);
+
+        let stats: any;
+        await waitFor("stats", async () => {
+          stats = await p.api("GET", "/v1/campaigns/spring-sale/stats");
+          return stats.status === 200 && stats.body.totals.delivered === 2;
+        });
+        expect(stats.body.totals).toMatchObject({
+          sent: 2,
+          delivered: 2,
+          failed: 0,
+          deliveryRate: 100,
+        });
+
+        const list = await p.api("GET", "/v1/campaigns");
+        expect(list.body.campaigns).toEqual([
+          expect.objectContaining({ campaign: "spring-sale", messages: 2 }),
+        ]);
+      });
+
+      it("pages through the delivery log without skipping or repeating a message", async () => {
+        const p = await project();
+        for (let i = 0; i < 12; i++)
+          await addUser(p.api, { id: `l${i}`, email: `l${i}@x.com`, segments: ["logs"] });
+        // SEGMENT_MAX_USERS is 5 in this file; send in batches of users instead.
+        for (let i = 0; i < 12; i++) await notify(p.api, { user: `l${i}`, template: "receipt" });
+        await waitFor("12 delivered", () => email.for(p.id).length === 12);
+        await waitFor("12 logged", async () => {
+          const { n } = (
+            await infra.sql`SELECT count(*)::int AS n FROM message_logs WHERE project_id = ${p.id} AND status = 'delivered'`
+          )[0]!;
+          return n === 12;
+        });
+
+        const seen: string[] = [];
+        let cursor: string | null = null;
+        for (let page = 0; page < 20; page++) {
+          const res: any = await p.api(
+            "GET",
+            `/v1/notifications/logs?limit=5${cursor ? `&cursor=${cursor}` : ""}`,
+          );
+          seen.push(...res.body.logs.map((l: any) => l.taskId));
+          cursor = res.body.nextCursor;
+          if (!cursor) break;
+        }
+        expect(new Set(seen).size).toBe(12);
+        expect(seen).toHaveLength(12);
+      });
+
+      it("emits delivery events on the server", async () => {
+        const p = await project();
+        await addUser(p.api, { id: "u", email: "u@x.com" });
+        const delivered: unknown[][] = [];
+        const on = (...args: unknown[]) => delivered.push(args);
+        app.server.on("delivery:delivered", on);
+        try {
+          await notify(p.api, { user: "u", template: "receipt" });
+          await waitFor("event", () => delivered.some((a) => a[3] === p.id));
+          const [taskId, providerMessageId, channel] = delivered.find((a) => a[3] === p.id)!;
+          expect(channel).toBe("email");
+          expect(providerMessageId).toBe(`pm-${taskId}`);
+        } finally {
+          app.server.off("delivery:delivered", on);
+        }
+      });
+    });
+  });
+
+  return { infra, app: () => app, globalEmitter };
+}

tests/integration/support/pipeline.ts

@@ -0,0 +1,158 @@
+// The package entry point first, as in a real install. The services import
+// through this barrel while some modules also import each other directly;
+// vitest's module runner snapshots `export *` at evaluation time, so loading a
+// leaf module first (the API loads the queue directly) leaves the barrel
+// missing exports. Node's own ESM loader binds them lazily and is unaffected.
+import "@/index.js";
+import { createServer } from "node:net";
+import { NotifkitServer, type NotifkitOptions } from "@/server.js";
+import type { DeliveryResult, Transport } from "@/transport/index.js";
+import type { NotificationChannel, NotificationDispatchedPayload } from "@/contracts/index.js";
+import type { Infra } from "./infra.js";
+import { waitFor } from "./infra.js";
+
+export const ADMIN_KEY = "integration-admin-key-0123456789";
+
+type Behaviour = (task: NotificationDispatchedPayload) => Promise<DeliveryResult> | DeliveryResult;
+
+/**
+ * A provider that records what it was asked to send. By default every send
+ * succeeds; `behaviour` can fail, time out or reject specific destinations.
+ */
+export class RecordingTransport implements Transport {
+  readonly sent: NotificationDispatchedPayload[] = [];
+  behaviour?: Behaviour;
+
+  constructor(
+    readonly channel: NotificationChannel,
+    options: { behaviour?: Behaviour; limits?: { limit: number; windowSeconds: number } } = {},
+  ) {
+    this.behaviour = options.behaviour;
+    if (options.limits) (this as any).limits = options.limits;
+  }
+
+  async send(task: NotificationDispatchedPayload): Promise<DeliveryResult> {
+    const { signal: _signal, ...copy } = task as any;
+    const result = this.behaviour
+      ? await this.behaviour(task)
+      : { success: true, providerMessageId: `pm-${task.taskId}` };
+    if (result.success) this.sent.push(structuredClone(copy));
+    return result;
+  }
+
+  for(projectId: string): NotificationDispatchedPayload[] {
+    return this.sent.filter((t) => t.projectId === projectId);
+  }
+}
+
+export async function freePort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const srv = createServer();
+    srv.once("error", reject);
+    srv.listen(0, "127.0.0.1", () => {
+      const port = (srv.address() as any).port as number;
+      srv.close(() => resolve(port));
+    });
+  });
+}
+
+export interface Api {
+  (
+    method: string,
+    path: string,
+    body?: unknown,
+    headers?: Record<string, string>,
+  ): Promise<{
+    status: number;
+    body: any;
+    headers: Headers;
+  }>;
+}
+
+export function apiClient(
+  baseUrl: string,
+  token?: string,
+  defaultHeaders: Record<string, string> = {},
+): Api {
+  return async (method, path, body, headers = {}) => {
+    const res = await fetch(baseUrl + path, {
+      method,
+      headers: {
+        ...(token ? { authorization: `Bearer ${token}` } : {}),
+        ...(body !== undefined ? { "content-type": "application/json" } : {}),
+        ...defaultHeaders,
+        ...headers,
+      },
+      body: body === undefined ? undefined : typeof body === "string" ? body : JSON.stringify(body),
+    });
+    const text = await res.text();
+    let parsed: any = text;
+    try {
+      parsed = text ? JSON.parse(text) : undefined;
+    } catch {
+      // not JSON — keep the text
+    }
+    return { status: res.status, body: parsed, headers: res.headers };
+  };
+}
+
+export interface Running {
+  server: NotifkitServer;
+  baseUrl: string;
+  admin: Api;
+  /** Creates a project through the API and returns a client authenticated as it. */
+  project(name?: string): Promise<{ id: string; apiKey: string; api: Api }>;
+  stop(): Promise<void>;
+}
+
+/**
+ * Starts a real NotifkitServer — API and workers in this process — against the
+ * file's private Postgres and Redis. Environment overrides are applied before
+ * start, since the services read their configuration once at boot.
+ */
+export async function startNotifkit(
+  infra: Infra,
+  options: Partial<NotifkitOptions> & { env?: Record<string, string> } = {},
+): Promise<Running> {
+  const port = await freePort();
+  Object.assign(process.env, {
+    ADMIN_API_KEY: ADMIN_KEY,
+    HOST: "127.0.0.1",
+    LOG_FLUSH_INTERVAL_MS: "50",
+    ...options.env,
+  });
+
+  const { env: _env, ...serverOptions } = options;
+  const server = new NotifkitServer({
+    services: ["api", "enricher", "engine", "scheduler", "delivery", "events"],
+    redisUrl: infra.redisUrl,
+    databaseUrl: infra.dbUrl,
+    logLevel: (process.env.TEST_LOG_LEVEL as any) ?? "silent",
+    autoMigrate: false,
+    port,
+    ...serverOptions,
+  });
+  await server.start();
+
+  const baseUrl = `http://127.0.0.1:${port}`;
+  const services = serverOptions.services ?? ["api"];
+  if (services.includes("api") || services.includes("all")) {
+    await waitFor("api listening", async () => (await fetch(`${baseUrl}/live`)).ok, 10_000);
+  }
+  const admin = apiClient(baseUrl, ADMIN_KEY);
+
+  return {
+    server,
+    baseUrl,
+    admin,
+    async project(name = "test") {
+      const res = await admin("POST", "/v1/projects", { name });
+      if (res.status !== 201)
+        throw new Error(`create project failed: ${res.status} ${JSON.stringify(res.body)}`);
+      return { id: res.body.id, apiKey: res.body.apiKey, api: apiClient(baseUrl, res.body.apiKey) };
+    },
+    async stop() {
+      await server.stop();
+    },
+  };
+}

tests/integration/worker-runtime.test.ts

@@ -0,0 +1,666 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { randomUUID } from "node:crypto";
+import { IdempotencyGuard } from "@/idempotency/index.js";
+import {
+  ApiRateLimiter,
+  UserThrottle,
+  ProjectSettingsCache,
+  calculateTokenBatchSize,
+} from "@/rate-limiter/index.js";
+import { ProviderThrottle } from "@/services/delivery/throttle.js";
+import { RedisClient } from "@/redis/index.js";
+import {
+  BaseWorker,
+  NonRetryableError,
+  LockHeldError,
+  startHealthReporter,
+  type ProcessResult,
+} from "@/workers/index.js";
+import {
+  createPriorityProducers,
+  createStreamConsumer,
+  shutdownWorker,
+  uniqueConsumerId,
+} from "@/workers/bootstrap.js";
+import { StreamConsumer, PendingMessageScanner, type StreamMessage } from "@/queue/index.js";
+import { STREAMS, CONSUMER_GROUPS, buildStreamEvent } from "@/contracts/index.js";
+import { ProjectRepository } from "@/repositories/index.js";
+import { createLogger } from "@/logger/index.js";
+import { globalEmitter } from "@/shared/index.js";
+import { useInfra, waitFor, settle } from "./support/infra.js";
+
+const infra = useInfra();
+const logger = createLogger({ name: "test", level: "silent" });
+
+beforeEach(async () => {
+  await infra.reset();
+});
+
+// ─── IdempotencyGuard ────────────────────────────────────────────────────────
+
+describe("IdempotencyGuard", () => {
+  const guard = () =>
+    new IdempotencyGuard({ redis: infra.redis, keyPrefix: "t:idem", ttlSeconds: 60 });
+
+  it("grants a lease once, reports it locked to others, and completed after marking", async () => {
+    const g = guard();
+    expect(await g.acquireLease("m1", 30)).toBe("acquired");
+    expect(await g.acquireLease("m1", 30)).toBe("locked");
+    await g.markProcessed("m1");
+    expect(await g.acquireLease("m1", 30)).toBe("completed");
+    expect(await infra.redis.ttl("t:idem:m1")).toBeGreaterThan(30);
+  });
+
+  it("gives exactly one of many concurrent callers the lease", async () => {
+    const g = guard();
+    const results = await Promise.all(Array.from({ length: 50 }, () => g.acquireLease("race", 30)));
+    expect(results.filter((r) => r === "acquired")).toHaveLength(1);
+    expect(results.filter((r) => r === "locked")).toHaveLength(49);
+  });
+
+  it("lets a new holder in once an abandoned lease expires", async () => {
+    const g = guard();
+    expect(await g.acquireLease("crash", 1)).toBe("acquired");
+    await settle(1_100);
+    expect(await g.acquireLease("crash", 1)).toBe("acquired");
+  });
+
+  it("checkAndMark is true exactly once", async () => {
+    const g = guard();
+    const results = await Promise.all(Array.from({ length: 20 }, () => g.checkAndMark("once")));
+    expect(results.filter(Boolean)).toHaveLength(1);
+    expect(await g.isProcessed("once")).toBe(true);
+    expect(await g.isProcessed("never")).toBe(false);
+  });
+
+  it("unmark clears the marker so the id can be processed again", async () => {
+    const g = guard();
+    await g.acquireLease("again", 30);
+    await g.markProcessed("again");
+    await g.unmark("again");
+    expect(await g.isProcessed("again")).toBe(false);
+    expect(await g.acquireLease("again", 30)).toBe("acquired");
+  });
+
+  it("honours a custom TTL", async () => {
+    const g = guard();
+    await g.checkAndMark("ttl", 5);
+    const ttl = await infra.redis.ttl("t:idem:ttl");
+    expect(ttl).toBeGreaterThan(0);
+    expect(ttl).toBeLessThanOrEqual(5);
+  });
+});
+
+// ─── Rate limiting ──────────────────────────────────────────────────────────
+
+describe("calculateTokenBatchSize", () => {
+  it("scales with the limit and stays within bounds", () => {
+    expect(calculateTokenBatchSize(0)).toBe(0);
+    expect(calculateTokenBatchSize(-5)).toBe(0);
+    expect(calculateTokenBatchSize(10)).toBe(1);
+    expect(calculateTokenBatchSize(60)).toBe(5);
+    expect(calculateTokenBatchSize(61)).toBe(10);
+    expect(calculateTokenBatchSize(6_000)).toBe(500);
+    expect(calculateTokenBatchSize(600_000)).toBe(500);
+    expect(calculateTokenBatchSize(600_000, 50)).toBe(50);
+  });
+});
+
+describe("ApiRateLimiter", () => {
+  it("admits exactly the limit in one window, even when requests arrive all at once", async () => {
+    const limiter = new ApiRateLimiter({ redis: infra.redis, windowMs: 60_000 });
+    const results = await Promise.all(Array.from({ length: 200 }, () => limiter.check("p1", 50)));
+    expect(results.filter(Boolean)).toHaveLength(50);
+    expect(await limiter.check("p1", 50)).toBe(false);
+  });
+
+  it("denies everything for a non-positive limit", async () => {
+    const limiter = new ApiRateLimiter({ redis: infra.redis });
+    expect(await limiter.check("p", 0)).toBe(false);
+    expect(await limiter.check("p", -1)).toBe(false);
+  });
+
+  it("keeps projects independent", async () => {
+    const limiter = new ApiRateLimiter({ redis: infra.redis });
+    for (let i = 0; i < 5; i++) await limiter.check("a", 5);
+    expect(await limiter.check("a", 5)).toBe(false);
+    expect(await limiter.check("b", 5)).toBe(true);
+  });
+
+  it("admits again once the window has fully passed", async () => {
+    const limiter = new ApiRateLimiter({ redis: infra.redis, windowMs: 500 });
+    // Align to the start of a bucket so the test does not straddle a boundary.
+    await settle(500 - (Date.now() % 500) + 5);
+    for (let i = 0; i < 3; i++) expect(await limiter.check("w", 3)).toBe(true);
+    expect(await limiter.check("w", 3)).toBe(false);
+    // Two windows later the previous bucket no longer weighs in.
+    await settle(1_000);
+    expect(await limiter.check("w", 3)).toBe(true);
+  });
+
+  it("fails open when Redis errors", async () => {
+    const broken = new RedisClient({ url: infra.redisUrl });
+    await broken.disconnect();
+    const limiter = new ApiRateLimiter({ redis: broken.native, logger });
+    expect(await limiter.check("p", 10)).toBe(true);
+  });
+
+  it("invalidate and clear drop local leases so the next check re-reads Redis", async () => {
+    const limiter = new ApiRateLimiter({ redis: infra.redis });
+    expect(await limiter.check("p", 600)).toBe(true); // leases a chunk locally
+    await infra.redis.flushdb();
+    limiter.invalidate("p");
+    limiter.clear();
+    expect(await limiter.check("p", 600)).toBe(true);
+  });
+});
+
+describe("ProviderThrottle", () => {
+  it("allows up to the provider's limit per window and says when to retry", async () => {
+    const throttle = new ProviderThrottle(infra.redis, logger);
+    const cfg = { limit: 5, windowSeconds: 60 };
+    const results = await Promise.all(Array.from({ length: 8 }, () => throttle.check("sms", cfg)));
+    const allowed = results.filter((r) => r.allowed);
+    const denied = results.filter((r) => !r.allowed);
+    expect(allowed).toHaveLength(5);
+    expect(denied).toHaveLength(3);
+    for (const d of denied) {
+      expect(d.retryAfterMs).toBeGreaterThan(0);
+      expect(d.retryAfterMs).toBeLessThanOrEqual(60_000);
+    }
+  });
+
+  it("counts channels separately", async () => {
+    const throttle = new ProviderThrottle(infra.redis);
+    const cfg = { limit: 1, windowSeconds: 60 };
+    expect((await throttle.check("sms", cfg)).allowed).toBe(true);
+    expect((await throttle.check("sms", cfg)).allowed).toBe(false);
+    expect((await throttle.check("email", cfg)).allowed).toBe(true);
+  });
+});
+
+describe("UserThrottle", () => {
+  it("never throttles critical sends", async () => {
+    const t = new UserThrottle({ redis: infra.redis, maxPerHour: 1 });
+    for (let i = 0; i < 5; i++) {
+      expect((await t.check("p", "u", "critical")).allowed).toBe(true);
+    }
+    expect(await infra.redis.exists("throttle:p:user:u")).toBe(0);
+  });
+
+  it("a project limit of 0 blocks every non-critical send", async () => {
+    const t = new UserThrottle({ redis: infra.redis, maxPerHour: 100 });
+    expect((await t.check("p", "u", "normal", { limit: 0 })).allowed).toBe(false);
+    expect((await t.check("p", "u", "critical", { limit: 0 })).allowed).toBe(true);
+  });
+
+  it("uses the project's window instead of the default", async () => {
+    const t = new UserThrottle({ redis: infra.redis, maxPerHour: 1, windowHours: 1 });
+    expect((await t.check("p", "u", "normal", { windowHours: 24 })).allowed).toBe(true);
+    const ttl = await infra.redis.ttl("throttle:p:user:u");
+    expect(ttl).toBeGreaterThan(3600);
+  });
+
+  it("ignores nonsensical overrides and falls back to the defaults", async () => {
+    const t = new UserThrottle({ redis: infra.redis, maxPerHour: 2 });
+    const r = await t.check("p", "u", "normal", { limit: -3, windowHours: 0 });
+    expect(r.limit).toBe(2);
+    expect(r.allowed).toBe(true);
+  });
+});
+
+describe("ProjectSettingsCache (backed by the real projects table)", () => {
+  async function project(throttleLimit: number | null) {
+    const id = randomUUID();
+    await infra.sql`INSERT INTO projects (id, name, throttle_limit) VALUES (${id}, 'p', ${throttleLimit})`;
+    return id;
+  }
+
+  it("loads settings, caches them, and picks up a change after invalidate", async () => {
+    const repo = new ProjectRepository(infra.db);
+    let loads = 0;
+    const cache = new ProjectSettingsCache((id) => {
+      loads++;
+      return repo.findThrottleSettings(id);
+    });
+    const id = await project(7);
+
+    expect((await cache.get(id)).throttleLimit).toBe(7);
+    await repo.updateSettings(id, { throttleLimit: 9 });
+    expect((await cache.get(id)).throttleLimit).toBe(7); // cached
+    cache.invalidate(id);
+    expect((await cache.get(id)).throttleLimit).toBe(9);
+    expect(loads).toBe(2);
+  });
+
+  it("runs one query for a burst of lookups on a cold project", async () => {
+    const repo = new ProjectRepository(infra.db);
+    let loads = 0;
+    const cache = new ProjectSettingsCache((id) => {
+      loads++;
+      return repo.findThrottleSettings(id);
+    });
+    const id = await project(3);
+    const results = await Promise.all(Array.from({ length: 25 }, () => cache.get(id)));
+    expect(results.every((r) => r.throttleLimit === 3)).toBe(true);
+    expect(loads).toBe(1);
+  });
+
+  it("treats an unknown project as having no overrides", async () => {
+    const repo = new ProjectRepository(infra.db);
+    const cache = new ProjectSettingsCache((id) => repo.findThrottleSettings(id));
+    expect(await cache.get(randomUUID())).toEqual({
+      throttleLimit: null,
+      throttleWindowHours: null,
+    });
+  });
+
+  it("does not cache a failed lookup", async () => {
+    let fail = true;
+    const cache = new ProjectSettingsCache(async () => {
+      if (fail) throw new Error("db down");
+      return { throttleLimit: 1, throttleWindowHours: null };
+    });
+    await expect(cache.get("p")).rejects.toThrow("db down");
+    fail = false;
+    expect((await cache.get("p")).throttleLimit).toBe(1);
+  });
+
+  it("does not let a lookup that raced an invalidate write stale settings", async () => {
+    let release!: (v: any) => void;
+    let calls = 0;
+    const cache = new ProjectSettingsCache(() => {
+      calls++;
+      if (calls === 1) return new Promise((r) => (release = r));
+      return Promise.resolve({ throttleLimit: 2, throttleWindowHours: null });
+    });
+    const stale = cache.get("p");
+    cache.invalidate("p");
+    release({ throttleLimit: 1, throttleWindowHours: null });
+    expect((await stale).throttleLimit).toBe(1);
+    expect((await cache.get("p")).throttleLimit).toBe(2);
+    cache.clear();
+  });
+});
+
+// ─── RedisClient ────────────────────────────────────────────────────────────
+
+describe("RedisClient", () => {
+  it("reports health and stops reporting it after disconnect", async () => {
+    const client = new RedisClient({ url: infra.redisUrl, logger });
+    expect(await client.healthCheck()).toBe(true);
+    await client.disconnect();
+    await client.disconnect(); // idempotent
+    expect(await client.healthCheck()).toBe(false);
+  });
+
+  it("registers the Lua commands", async () => {
+    const client = new RedisClient({ url: infra.redisUrl });
+    await client.native.set("lock", "me");
+    expect(await client.native.releaseLock("lock", "someone-else")).toBe(0);
+    expect(await client.native.releaseLock("lock", "me")).toBe(1);
+    await client.disconnect();
+  });
+
+  it("shares one connection per URL and closes it with the last holder", async () => {
+    const url = `${infra.redisUrl}?shared=${randomUUID()}`;
+    const a = RedisClient.shared({ url });
+    const b = RedisClient.shared({ url });
+    expect(a.native).toBe(b.native);
+
+    await a.disconnect();
+    await a.disconnect(); // a second release must not steal b's reference
+    expect(await b.healthCheck()).toBe(true);
+
+    await b.disconnect();
+    expect(await b.healthCheck()).toBe(false);
+
+    const c = RedisClient.shared({ url });
+    expect(await c.healthCheck()).toBe(true);
+    expect(c.native).not.toBe(a.native);
+    await c.disconnect();
+  });
+});
+
+// ─── BaseWorker ─────────────────────────────────────────────────────────────
+
+type Behaviour = (msg: StreamMessage, attempt?: number) => Promise<ProcessResult>;
+
+class ScriptedWorker extends BaseWorker {
+  readonly seen: { n: number; attempt: number }[] = [];
+  inFlight = 0;
+  maxInFlight = 0;
+  constructor(
+    opts: ConstructorParameters<typeof BaseWorker>[0],
+    public behaviour: Behaviour,
+  ) {
+    super(opts);
+  }
+  protected async process(msg: StreamMessage, attempt = 1): Promise<ProcessResult> {
+    this.inFlight++;
+    this.maxInFlight = Math.max(this.maxInFlight, this.inFlight);
+    try {
+      this.seen.push({ n: (msg.event.payload as any).n, attempt });
+      return await this.behaviour(msg, attempt);
+    } finally {
+      this.inFlight--;
+    }
+  }
+}
+
+describe("BaseWorker", () => {
+  const STREAM = STREAMS.ENRICHED_NORMAL;
+  const GROUP = CONSUMER_GROUPS.ENGINE;
+  const workers: BaseWorker[] = [];
+
+  function worker(
+    behaviour: Behaviour,
+    opts: { concurrency?: number; recoveryIntervalMs?: number; maxRetriesBeforeDlq?: number } = {},
+  ) {
+    const consumer = new StreamConsumer({
+      redis: infra.redis,
+      stream: STREAM,
+      group: GROUP,
+      consumer: "w1",
+      dlqStream: STREAMS.DEAD_LETTER,
+      blockMs: 100,
+      batchSize: 20,
+      logger,
+    });
+    const pendingScanner = new PendingMessageScanner({
+      redis: infra.redis,
+      stream: STREAM,
+      group: GROUP,
+      consumer: "w1",
+      logger,
+    });
+    const w = new ScriptedWorker(
+      { consumer, pendingScanner, logger, recoveryIntervalMs: 60_000, ...opts },
+      behaviour,
+    );
+    workers.push(w);
+    return w;
+  }
+
+  async function publish(n: number) {
+    const ev = {
+      ...buildStreamEvent("t", { n }, "test"),
+      id: randomUUID(),
+      timestamp: new Date().toISOString(),
+    };
+    await infra.redis.xadd(STREAM, "*", "data", JSON.stringify(ev));
+    return ev;
+  }
+
+  const pending = async () => Number(((await infra.redis.xpending(STREAM, GROUP)) as any[])[0]);
+  const dlq = async () =>
+    (await infra.redis.xrange(STREAMS.DEAD_LETTER, "-", "+")).map((e) => JSON.parse(e[1][1]!));
+
+  afterEach(async () => {
+    await Promise.all(workers.splice(0).map((w) => w.stop()));
+  });
+
+  it("processes and acks every message, and counts them", async () => {
+    const w = worker(async () => {});
+    await w.start();
+    for (let n = 0; n < 10; n++) await publish(n);
+
+    await waitFor("10 processed", () => w.health().processedCount === 10);
+    expect(await pending()).toBe(0);
+    const h = w.health();
+    expect(h.state).toBe("running");
+    expect(h.errorCount).toBe(0);
+    expect(h.lastProcessedAt).not.toBeNull();
+  });
+
+  it("refuses to start twice", async () => {
+    const w = worker(async () => {});
+    await w.start();
+    await expect(w.start()).rejects.toThrow(/cannot start/);
+  });
+
+  it("never runs more than `concurrency` messages at once", async () => {
+    const w = worker(() => settle(30), { concurrency: 3 });
+    await w.start();
+    for (let n = 0; n < 15; n++) await publish(n);
+    await waitFor("15 processed", () => w.health().processedCount === 15);
+    expect(w.maxInFlight).toBeLessThanOrEqual(3);
+    expect(w.maxInFlight).toBeGreaterThan(1);
+  });
+
+  it("leaves a failed message pending and retries it through recovery", async () => {
+    let calls = 0;
+    const w = worker(
+      async () => {
+        if (++calls === 1) throw new Error("transient");
+      },
+      { recoveryIntervalMs: 100 },
+    );
+    await w.start();
+    await publish(1);
+
+    await waitFor("retried and processed", () => w.health().processedCount === 1, 10_000);
+    expect(w.seen.map((s) => s.attempt)).toEqual([1, 2]);
+    expect(w.health().errorCount).toBe(1);
+    expect(await pending()).toBe(0);
+  });
+
+  it("dead-letters a message that keeps failing, once, and reports it", async () => {
+    const failed: string[] = [];
+    const onFailed = (id: string) => failed.push(id);
+    globalEmitter.on("notification:failed", onFailed);
+    try {
+      const w = worker(
+        async () => {
+          throw new Error("always");
+        },
+        { recoveryIntervalMs: 50, maxRetriesBeforeDlq: 2 },
+      );
+      await w.start();
+      const ev = await publish(1);
+
+      await waitFor("dead-lettered", async () => (await dlq()).length === 1, 15_000);
+      await settle(300);
+      expect((await dlq()).map((d) => d.id)).toEqual([ev.id]);
+      expect(await pending()).toBe(0);
+      expect(w.seen.map((s) => s.attempt)).toEqual([1, 2]);
+      expect(failed).toHaveLength(1);
+    } finally {
+      globalEmitter.off("notification:failed", onFailed);
+    }
+  });
+
+  it("dead-letters a NonRetryableError immediately", async () => {
+    const w = worker(async () => {
+      throw new NonRetryableError("bad input");
+    });
+    await w.start();
+    const ev = await publish(1);
+    await waitFor("dead-lettered", async () => (await dlq()).length === 1);
+    expect((await dlq())[0].id).toBe(ev.id);
+    expect(await pending()).toBe(0);
+    expect(w.seen).toHaveLength(1);
+  });
+
+  it("leaves a message pending, without counting an error, when the lock is held", async () => {
+    const w = worker(async () => {
+      throw new LockHeldError();
+    });
+    await w.start();
+    await publish(1);
+    await waitFor("seen", () => w.seen.length === 1);
+    await settle(100);
+    expect(await pending()).toBe(1);
+    expect(w.health().errorCount).toBe(0);
+  });
+
+  it("leaves a message pending when process asks not to ack", async () => {
+    const w = worker(async () => ({ ack: false }));
+    await w.start();
+    await publish(1);
+    await waitFor("seen", () => w.seen.length === 1);
+    await settle(100);
+    expect(await pending()).toBe(1);
+    expect(w.health().processedCount).toBe(0);
+  });
+
+  it("stop() lets in-flight messages finish and ack", async () => {
+    const w = worker(() => settle(300));
+    await w.start();
+    await publish(1);
+    await waitFor("started", () => w.inFlight === 1);
+    await w.stop();
+    expect(w.health().state).toBe("stopped");
+    expect(w.health().processedCount).toBe(1);
+    expect(await pending()).toBe(0);
+  });
+
+  it("recover() reports the pending count through health()", async () => {
+    const w = worker(async () => {});
+    expect(w.health().pendingCount).toBeNull();
+    await infra.redis.xgroup("CREATE", STREAM, GROUP, "0", "MKSTREAM");
+    await w.recover();
+    expect(w.health().pendingCount).toBe(0);
+  });
+
+  describe("processInline", () => {
+    const ev = () => ({
+      ...buildStreamEvent("t", { n: 1 }, "test"),
+      id: randomUUID(),
+      timestamp: new Date().toISOString(),
+    });
+
+    it("processes and counts a handed-over event", async () => {
+      const w = worker(async () => {});
+      await w.processInline(ev());
+      expect(w.health().processedCount).toBe(1);
+      expect(w.seen[0]!.attempt).toBe(1);
+    });
+
+    it("dead-letters a non-retryable failure instead of throwing it upstream", async () => {
+      const w = worker(async () => {
+        throw new NonRetryableError("nope");
+      });
+      const e = ev();
+      await w.processInline(e);
+      expect((await dlq()).map((d) => d.id)).toEqual([e.id]);
+      expect(w.health().errorCount).toBe(1);
+    });
+
+    it("swallows a held lock", async () => {
+      const w = worker(async () => {
+        throw new LockHeldError();
+      });
+      await expect(w.processInline(ev())).resolves.toBeUndefined();
+      expect(w.health().errorCount).toBe(0);
+    });
+
+    it("rethrows a retryable failure so the upstream entry stays pending", async () => {
+      const w = worker(async () => {
+        throw new Error("boom");
+      });
+      await expect(w.processInline(ev())).rejects.toThrow("boom");
+    });
+  });
+});
+
+// ─── Bootstrap helpers ──────────────────────────────────────────────────────
+
+describe("bootstrap helpers", () => {
+  it("uniqueConsumerId never repeats", () => {
+    const ids = new Set(Array.from({ length: 1000 }, () => uniqueConsumerId("x")));
+    expect(ids.size).toBe(1000);
+    for (const id of ids) expect(id.startsWith("x-")).toBe(true);
+  });
+
+  it("createStreamConsumer pairs a consumer and scanner that agree", async () => {
+    const redis = new RedisClient({ url: infra.redisUrl });
+    const { consumer, pendingScanner } = createStreamConsumer(
+      { redis, logger },
+      {
+        stream: [STREAMS.OUTBOUND_NORMAL, STREAMS.OUTBOUND_LOW],
+        group: CONSUMER_GROUPS.DELIVERY,
+        consumer: "d1",
+        batchSize: 5,
+      },
+    );
+    await consumer.ensureGroup();
+    await infra.redis.xadd(
+      STREAMS.OUTBOUND_LOW,
+      "*",
+      "data",
+      JSON.stringify({
+        ...buildStreamEvent("t", {}, "s"),
+        id: randomUUID(),
+        timestamp: new Date().toISOString(),
+      }),
+    );
+    const it = consumer.readBatch();
+    const batch = (await it.next()).value!;
+    expect(batch[0]!.stream).toBe(STREAMS.OUTBOUND_LOW);
+    expect(await pendingScanner.getPendingCount()).toBe(1);
+    await consumer.nack(batch[0]!.id, batch[0]!.event, batch[0]!.stream);
+    expect(await infra.redis.xlen(STREAMS.DEAD_LETTER)).toBe(1);
+    await consumer.stop();
+    await redis.disconnect();
+  });
+
+  it("createPriorityProducers writes each lane to its own stream", async () => {
+    const producers = createPriorityProducers(infra.redis, logger, "OUTBOUND");
+    await producers.critical.publish(buildStreamEvent("t", {}, "s"));
+    await producers.normal.publish(buildStreamEvent("t", {}, "s"));
+    await producers.low.publish(buildStreamEvent("t", {}, "s"));
+    expect(await infra.redis.xlen(STREAMS.OUTBOUND_CRITICAL)).toBe(1);
+    expect(await infra.redis.xlen(STREAMS.OUTBOUND_NORMAL)).toBe(1);
+    expect(await infra.redis.xlen(STREAMS.OUTBOUND_LOW)).toBe(1);
+  });
+
+  it("shutdownWorker stops timers, the worker, and closes both connections", async () => {
+    const redis = new RedisClient({ url: infra.redisUrl });
+    const { createDatabase } = await import("@/db/index.js");
+    const { sql } = createDatabase({ url: infra.dbUrl, maxConnections: 1 });
+    await sql`SELECT 1`;
+    let ticks = 0;
+    const timer = setInterval(() => ticks++, 5);
+    let disconnected = false;
+
+    await shutdownWorker("test", {
+      logger,
+      timers: [timer, null],
+      subscriber: { disconnect: () => (disconnected = true) },
+      sql,
+      redis,
+    });
+
+    const after = ticks;
+    await settle(30);
+    expect(ticks).toBe(after);
+    expect(disconnected).toBe(true);
+    expect(await redis.healthCheck()).toBe(false);
+    await expect(sql`SELECT 1`).rejects.toThrow();
+  });
+});
+
+describe("startHealthReporter", () => {
+  it("publishes the worker's health to Redis with a short TTL", async () => {
+    const redis = new RedisClient({ url: infra.redisUrl });
+    const w = { health: () => ({ state: "running", processedCount: 3 }) } as any;
+    const timer = startHealthReporter("svc", w, redis, logger, 20);
+    try {
+      await waitFor("health key", async () => (await infra.redis.exists("notif:health:svc")) === 1);
+      const body = JSON.parse((await infra.redis.get("notif:health:svc"))!);
+      expect(body).toMatchObject({
+        service: "svc",
+        redis: true,
+        state: "running",
+        processedCount: 3,
+      });
+      const ttl = await infra.redis.ttl("notif:health:svc");
+      expect(ttl).toBeGreaterThan(0);
+      expect(ttl).toBeLessThanOrEqual(15);
+    } finally {
+      clearInterval(timer);
+      await redis.disconnect();
+    }
+  });
+});

tests/integration/workflows.test.ts

@@ -0,0 +1,356 @@
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { randomUUID } from "node:crypto";
+import { workflow } from "@/workflows/index.js";
+import { STREAMS, buildStreamEvent } from "@/contracts/index.js";
+import { useInfra, waitFor, settle } from "./support/infra.js";
+import { RecordingTransport, startNotifkit, type Running, type Api } from "./support/pipeline.js";
+
+/**
+ * Workflows end to end: definitions and triggers go in through the API, steps
+ * run in the workflow worker, events come back through the events worker, and
+ * the assertions are on what was delivered and on the instance's recorded state.
+ */
+
+const infra = useInfra();
+let app: Running;
+const email = new RecordingTransport("email");
+
+// Code-defined workflows, registered before the server starts.
+const runs: Record<string, number> = {};
+workflow("onboarding", async ({ step, event }) => {
+  await step.notify({ template: "hello", data: { name: event.user.id } });
+  const paid = await step.waitForEvent("order.paid", {
+    timeout: "1h",
+    match: { userId: event.user.id },
+  });
+  await step.run("record", () => ({ paidAmount: paid?.amount ?? null }));
+  await step.notify({ template: "thanks" });
+});
+workflow("sleepy", async ({ step, event }) => {
+  await step.run("count", () => {
+    runs[event.tag] = (runs[event.tag] ?? 0) + 1;
+    return runs[event.tag];
+  });
+  await step.wait("2s");
+  await step.notify({ template: "hello" });
+});
+workflow("impatient", async ({ step }) => {
+  const got = await step.waitForEvent("never.happens", { timeout: "1s" });
+  await step.run("outcome", () => ({ timedOut: got === null }));
+});
+workflow("waits-an-hour", async ({ step }) => {
+  await step.wait("1h");
+  await step.notify({ template: "hello" });
+});
+workflow("broken", async ({ step }) => {
+  await step.run("explode", () => {
+    throw new Error("handler bug");
+  });
+});
+workflow("await-go", async ({ step }) => {
+  const r = await step.waitForEvent("go", { timeout: "1h" });
+  await step.run("after", () => ({ got: r }));
+});
+
+beforeAll(async () => {
+  app = await startNotifkit(infra, {
+    services: ["api", "enricher", "engine", "delivery", "scheduler", "events", "workflow"],
+    providers: [email],
+    env: { RATE_LIMIT_PER_HOUR: "100000" },
+  });
+}, 120_000);
+
+afterAll(async () => {
+  await app?.stop();
+}, 60_000);
+
+async function project() {
+  const p = await app.project();
+  await p.api("PUT", "/v1/templates", {
+    templates: [
+      { id: "hello", channel: "email", content: { subject: "Hello {{name}}" } },
+      { id: "thanks", channel: "email", content: { subject: "Thanks" } },
+    ],
+  });
+  for (const id of ["alice", "bob"]) {
+    await p.api("POST", "/v1/users", { id, email: `${id}@x.com` });
+  }
+  return p;
+}
+
+async function trigger(api: Api, body: Record<string, unknown>) {
+  const res = await api("POST", "/v1/workflows/trigger", body);
+  expect(res.status).toBe(202);
+  return res.body.instanceId as string;
+}
+
+async function instance(api: Api, id: string) {
+  const res = await api("GET", `/v1/workflows/instances/${id}`);
+  return res.status === 200 ? res.body : null;
+}
+
+async function waitForStatus(api: Api, id: string, status: string, timeoutMs = 20_000) {
+  await waitFor(
+    `instance ${status}`,
+    async () => (await instance(api, id))?.status === status,
+    timeoutMs,
+  );
+}
+
+async function sendEvent(api: Api, name: string, properties: Record<string, unknown>) {
+  const res = await api("POST", "/v1/events", { name, properties });
+  expect(res.status).toBe(202);
+}
+
+describe("code-defined workflows", () => {
+  it("notifies, sleeps on an event that matches its user, then carries on", async () => {
+    const p = await project();
+    const id = await trigger(p.api, { name: "onboarding", user: "alice" });
+
+    await waitFor("welcome sent", () => email.for(p.id).length === 1);
+    expect(email.for(p.id)[0]!.renderedContent.content).toEqual({ subject: "Hello alice" });
+    await waitForStatus(p.api, id, "pending");
+
+    // Someone else's payment must not wake alice's workflow.
+    await sendEvent(p.api, "order.paid", { userId: "bob", amount: 1 });
+    await settle(1_500);
+    expect(email.for(p.id)).toHaveLength(1);
+
+    await sendEvent(p.api, "order.paid", { userId: "alice", amount: 42 });
+    await waitForStatus(p.api, id, "completed");
+    await waitFor("thanks sent", () => email.for(p.id).length === 2);
+
+    const inst = await instance(p.api, id);
+    expect(inst.steps.map((s: any) => s.action)).toEqual([
+      "notify",
+      "waitForEvent",
+      "run",
+      "notify",
+    ]);
+    expect(inst.steps[2].output).toEqual({ paidAmount: 42 });
+    expect(inst.waiters).toEqual([]);
+  });
+
+  it("records a notify step's ids as the notification that was actually sent", async () => {
+    const p = await project();
+    const id = await trigger(p.api, { name: "onboarding", user: "alice" });
+    await waitFor("sent", () => email.for(p.id).length === 1);
+    await waitFor("step recorded", async () => (await instance(p.api, id))?.steps.length >= 1);
+
+    const notifyStep = (await instance(p.api, id)).steps[0];
+    const task = email.for(p.id)[0]!;
+    // The engine derives task ids from the id of the request event.
+    expect(task.taskId.startsWith(`${notifyStep.output.notificationId}:`)).toBe(true);
+  });
+
+  it("files a step's delivery under its workflow instance, so the log can be filtered by it", async () => {
+    const p = await project();
+    const id = await trigger(p.api, { name: "onboarding", user: "alice" });
+    await waitFor("sent", () => email.for(p.id).length === 1);
+
+    const byInstance = async () =>
+      (await p.api("GET", `/v1/notifications/logs?workflowInstanceId=${id}`)).body.logs as any[];
+    await waitFor("logged under the instance", async () => (await byInstance()).length === 1);
+    expect((await byInstance())[0]).toMatchObject({
+      taskId: email.for(p.id)[0]!.taskId,
+      status: "delivered",
+    });
+
+    // Another instance's filter must not pick it up.
+    const other = (await p.api("GET", `/v1/notifications/logs?workflowInstanceId=${randomUUID()}`))
+      .body.logs;
+    expect(other).toEqual([]);
+  });
+
+  it("wakes after a wait, without re-running completed steps", async () => {
+    const p = await project();
+    const tag = randomUUID();
+    const id = await trigger(p.api, { name: "sleepy", user: "alice", input: { tag } });
+
+    await waitForStatus(p.api, id, "pending");
+    expect(email.for(p.id)).toEqual([]);
+    await waitForStatus(p.api, id, "completed", 30_000);
+    await waitFor("sent after waking", () => email.for(p.id).length === 1);
+    expect(runs[tag]).toBe(1);
+  }, 40_000);
+
+  it("resumes with null when an awaited event times out", async () => {
+    const p = await project();
+    const id = await trigger(p.api, { name: "impatient", user: "alice" });
+    await waitForStatus(p.api, id, "completed", 30_000);
+    const inst = await instance(p.api, id);
+    expect(inst.steps.find((s: any) => s.action === "run").output).toEqual({ timedOut: true });
+  }, 40_000);
+
+  it("does not cut a wait short when a stray resume arrives", async () => {
+    const p = await project();
+    const id = await trigger(p.api, { name: "waits-an-hour", user: "alice" });
+    await waitForStatus(p.api, id, "pending");
+
+    await infra.redis.xadd(
+      STREAMS.WORKFLOW_INBOUND,
+      "*",
+      "data",
+      JSON.stringify({
+        ...buildStreamEvent(
+          "workflow.resumed",
+          {
+            projectId: p.id,
+            instanceId: id,
+            name: "waits-an-hour",
+            input: { user: { id: "alice" } },
+          },
+          "test",
+        ),
+        id: randomUUID(),
+        timestamp: new Date().toISOString(),
+      }),
+    );
+    await settle(2_000);
+
+    expect((await instance(p.api, id)).status).toBe("pending");
+    expect(email.for(p.id)).toEqual([]);
+  });
+
+  it("marks an instance failed when its handler throws", async () => {
+    const p = await project();
+    const id = await trigger(p.api, { name: "broken", user: "alice" });
+    await waitForStatus(p.api, id, "failed");
+  });
+
+  it("does not lose a resume that arrives while the instance is still suspending", async () => {
+    const p = await project();
+    // The interleaving under test: another process holds the instance (it is
+    // mid-way through suspending on waitForEvent) when the events worker has
+    // already matched the event, filled the step in, and published the resume.
+    const [row] = await infra.sql`
+      INSERT INTO workflow_instances (project_id, name, status, input)
+      VALUES (${p.id}, 'await-go', 'running', ${JSON.stringify({ user: { id: "alice" } })}::jsonb) RETURNING id`;
+    const id = row!.id as string;
+    await infra.sql`
+      INSERT INTO workflow_steps (project_id, instance_id, step_index, action, output)
+      VALUES (${p.id}, ${id}, '0', 'waitForEvent', ${JSON.stringify({ orderId: "o1" })}::jsonb)`;
+    await infra.redis.set(`lock:workflow:${id}`, "other-process", "EX", 60);
+
+    await infra.redis.xadd(
+      STREAMS.WORKFLOW_INBOUND,
+      "*",
+      "data",
+      JSON.stringify({
+        ...buildStreamEvent(
+          "workflow.resumed",
+          { projectId: p.id, instanceId: id, name: "await-go", input: { user: { id: "alice" } } },
+          "events",
+        ),
+        id: randomUUID(),
+        timestamp: new Date().toISOString(),
+      }),
+    );
+    await settle(500);
+
+    // The other process finishes suspending and lets go.
+    await infra.sql`UPDATE workflow_instances SET status = 'pending' WHERE id = ${id}`;
+    await infra.redis.del(`lock:workflow:${id}`);
+
+    await waitForStatus(p.api, id, "completed", 20_000);
+    const inst = await instance(p.api, id);
+    expect(inst.steps.find((s: any) => s.action === "run").output).toEqual({
+      got: { orderId: "o1" },
+    });
+  }, 30_000);
+
+  it("can be canceled while it sleeps, and then never resumes", async () => {
+    const p = await project();
+    const id = await trigger(p.api, {
+      name: "sleepy",
+      user: "alice",
+      input: { tag: randomUUID() },
+    });
+    await waitForStatus(p.api, id, "pending");
+
+    const res = await p.api("DELETE", `/v1/workflows/instances/${id}`);
+    expect(res.status).toBe(204);
+    expect((await p.api("DELETE", `/v1/workflows/instances/${id}`)).status).toBe(400);
+
+    await settle(8_000);
+    expect((await instance(p.api, id)).status).toBe("canceled");
+    expect(email.for(p.id)).toEqual([]);
+  }, 20_000);
+
+  it("does nothing for a workflow nobody defined", async () => {
+    const p = await project();
+    const id = await trigger(p.api, { name: "no-such-workflow", user: "alice" });
+    await settle(1_000);
+    expect(await instance(p.api, id)).toBeNull();
+  });
+
+  it("keeps instances private to their project", async () => {
+    const p = await project();
+    const other = await project();
+    const id = await trigger(p.api, { name: "broken", user: "alice" });
+    await waitForStatus(p.api, id, "failed");
+    expect((await other.api("GET", `/v1/workflows/instances/${id}`)).status).toBe(404);
+    expect((await other.api("DELETE", `/v1/workflows/instances/${id}`)).status).toBe(400);
+  });
+});
+
+describe("JSON-defined workflows", () => {
+  it("runs notify, wait and waitForEvent steps defined through the API", async () => {
+    const p = await project();
+    const created = await p.api("POST", "/v1/workflows", {
+      name: "json-flow",
+      steps: [
+        { action: "notify", payload: { template: "hello", data: { name: "json" } } },
+        {
+          action: "waitForEvent",
+          event: "signup.done",
+          options: { timeout: "1h", match: { userId: "alice" } },
+        },
+        { action: "notify", payload: { template: "thanks", user: "bob" } },
+      ],
+    });
+    expect(created.status).toBe(201);
+    expect((await p.api("GET", "/v1/workflows")).body.workflows.map((w: any) => w.name)).toEqual([
+      "json-flow",
+    ]);
+
+    const id = await trigger(p.api, { name: "json-flow", user: "alice" });
+    await waitFor("first notify", () => email.for(p.id).length === 1);
+    expect(email.for(p.id)[0]!.recipientId).toBe("alice");
+    await waitForStatus(p.api, id, "pending");
+
+    await sendEvent(p.api, "signup.done", { userId: "someone-else" });
+    await settle(1_500);
+    expect((await instance(p.api, id)).status).toBe("pending");
+
+    await sendEvent(p.api, "signup.done", { userId: "alice" });
+    await waitForStatus(p.api, id, "completed");
+    await waitFor("second notify, to the step's own target", () => email.for(p.id).length === 2);
+    expect(email.for(p.id)[1]!.recipientId).toBe("bob");
+  });
+
+  it("redefining a workflow replaces its steps", async () => {
+    const p = await project();
+    await p.api("POST", "/v1/workflows", {
+      name: "v",
+      steps: [{ action: "notify", payload: { template: "hello" } }],
+    });
+    await p.api("POST", "/v1/workflows", {
+      name: "v",
+      steps: [{ action: "notify", payload: { template: "thanks" } }],
+    });
+    const id = await trigger(p.api, { name: "v", user: "alice" });
+    await waitForStatus(p.api, id, "completed");
+    await waitFor("sent", () => email.for(p.id).length === 1);
+    expect(email.for(p.id)[0]!.templateId).toBe("thanks");
+  });
+
+  it("rejects an invalid definition", async () => {
+    const p = await project();
+    const res = await p.api("POST", "/v1/workflows", {
+      name: "bad",
+      steps: [{ action: "teleport" }],
+    });
+    expect(res.status).toBe(400);
+  });
+});

tests/phase3.test.ts

@@ -1,4 +1,5 @@
 import { describe, it, expect, vi } from "vitest";
+import { globalEmitter } from "@/shared/index.js";
 import { EnricherWorker } from "@/services/enricher/main.js";
 import { CircuitBreaker } from "@/shared/index.js";
 import { StreamConsumer } from "@/queue/index.js";
@@ -51,17 +52,21 @@ describe("Phase 3 Reliability Features", () => {
         },
       };
 
-      await worker.process(message as any);
+      const failures: unknown[][] = [];
+      const onFailed = (...args: unknown[]) => failures.push(args);
+      globalEmitter.on("notification:failed", onFailed);
+      try {
+        await worker.process(message as any);
+      } finally {
+        globalEmitter.off("notification:failed", onFailed);
+      }
 
-      // Verify it emitted a failure event instead of processing
-      expect(mockProducer.publish).toHaveBeenCalledWith(
-        expect.objectContaining({
-          type: "notification.failed",
-          payload: expect.objectContaining({
-            error: expect.stringContaining("exceeds limit"),
-          }),
-        }),
-      );
+      // Reported as a failure, and nothing fanned out. (A `notification.failed`
+      // on the enriched stream is not something the engine can read.)
+      expect(failures).toHaveLength(1);
+      expect(String(failures[0]![1])).toContain("exceeds limit");
+      expect(mockProducer.publish).not.toHaveBeenCalled();
+      expect(mockProducer.publishBatch).not.toHaveBeenCalled();
     });
   });
 

tests/repositories.units.test.ts

@@ -295,6 +295,7 @@ describe("Repository mapping and guards", () => {
       mockDb.queueDelete([]); // suppressions
       mockDb.queueDelete([]); // messageLogs
       mockDb.queueDelete([]); // workflowInstances
+      mockDb.queueDelete([]); // workflowDefinitions
       mockDb.queueDelete([{ id: "proj-1" }]); // projects
       await expect(repo.delete("proj-1")).resolves.toBe(true);
 
@@ -302,6 +303,7 @@ describe("Repository mapping and guards", () => {
       mockDb.queueDelete([]); // suppressions
       mockDb.queueDelete([]); // messageLogs
       mockDb.queueDelete([]); // workflowInstances
+      mockDb.queueDelete([]); // workflowDefinitions
       mockDb.queueDelete([]); // projects
       await expect(repo.delete("ghost")).resolves.toBe(false);
     });
@@ -313,8 +315,8 @@ describe("Repository mapping and guards", () => {
       mockDb.queueSelect([{ id: "u-1" }, { id: "u-2" }]);
       // 2. Child user tables deletes (6 deletes)
       for (let i = 0; i < 6; i++) mockDb.queueDelete([]);
-      // 3. Project child tables deletes (3 deletes)
-      for (let i = 0; i < 3; i++) mockDb.queueDelete([]);
+      // 3. Project child tables deletes (4 deletes)
+      for (let i = 0; i < 4; i++) mockDb.queueDelete([]);
       // 4. Final delete of project returns deleted row
       mockDb.queueDelete([{ id: "proj-1" }]);
 

tests/worker-consumer-ids.test.ts

@@ -0,0 +1,51 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import type * as Bootstrap from "@/workers/bootstrap.js";
+
+/**
+ * Every replica of a worker must join its consumer group under its own name.
+ * In containers each replica is PID 1, so a name built from the PID alone makes
+ * every replica the same group member. Starting a worker twice in one process
+ * reproduces that exactly: same PID, two starts.
+ *
+ * Startup is cut off right after the consumer is named, so nothing connects.
+ */
+
+const seen = vi.hoisted(() => [] as string[]);
+
+vi.mock("@/workers/bootstrap.js", async (importOriginal) => {
+  const actual = await importOriginal<typeof Bootstrap>();
+  const logger = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} };
+  return {
+    ...actual,
+    createWorkerRuntime: () => ({ logger, redis: { native: {} }, sql: {}, db: {} }),
+    createStreamConsumer: (_runtime: unknown, options: { consumer: string }) => {
+      seen.push(options.consumer);
+      throw new Error("stop after naming the consumer");
+    },
+  };
+});
+
+import { startAiWorker } from "@/services/ai/main.js";
+import { startEventWorker } from "@/services/events/main.js";
+import { startSchedulerWorker } from "@/services/scheduler/main.js";
+import { startWorkflowWorker } from "@/services/workflow/main.js";
+
+beforeEach(() => {
+  seen.length = 0;
+});
+
+describe.each([
+  ["ai", startAiWorker],
+  ["events", startEventWorker],
+  ["scheduler", startSchedulerWorker],
+  ["workflow", startWorkflowWorker],
+] as const)("%s worker", (name, start) => {
+  it("joins its consumer group under a name no other replica shares", async () => {
+    await expect(start()).rejects.toThrow("stop after naming the consumer");
+    await expect(start()).rejects.toThrow("stop after naming the consumer");
+
+    expect(seen).toHaveLength(2);
+    expect(seen[0]).not.toBe(seen[1]);
+    for (const id of seen) expect(id.startsWith(`${name}-`)).toBe(true);
+  });
+});

tests/workers-ai.test.ts

@@ -28,7 +28,11 @@ describe("AiWorker Edge Cases", () => {
     };
     mockGenerateAiContent = vi.fn().mockResolvedValue("generated_text");
     mockTemplateCache = { getCachedTemplate: vi.fn().mockResolvedValue(null) };
-    mockDb = { insert: vi.fn().mockReturnThis(), values: vi.fn().mockReturnThis() };
+    mockDb = {
+      insert: vi.fn().mockReturnThis(),
+      values: vi.fn().mockReturnThis(),
+      onConflictDoNothing: vi.fn().mockResolvedValue(undefined),
+    };
 
     worker = new AiWorker({
       consumer: { ack: vi.fn(), nack: vi.fn() } as any,

tests/workers.test.ts

@@ -179,7 +179,12 @@ describe("BaseWorker Retry and DLQ Logic", () => {
 
     await (worker as any).processWithTracking(msg);
 
-    expect(mockConsumer.nack).toHaveBeenCalledWith("msg-1", msg.event, undefined);
+    expect(mockConsumer.nack).toHaveBeenCalledWith(
+      "msg-1",
+      msg.event,
+      undefined,
+      expect.stringContaining("max retries exceeded"),
+    );
   });
 
   it("immediately moves to DLQ on attempt 1 when process throws NonRetryableError", async () => {
@@ -211,7 +216,12 @@ describe("BaseWorker Retry and DLQ Logic", () => {
 
     await (worker as any).processWithTracking(msg);
 
-    expect(mockConsumer.nack).toHaveBeenCalledWith("msg-perm-1", msg.event, undefined);
+    expect(mockConsumer.nack).toHaveBeenCalledWith(
+      "msg-perm-1",
+      msg.event,
+      undefined,
+      "fatal: recipient account terminated",
+    );
   });
 
   it("processes successfully without touching Redis retry keys on happy path", async () => {
@@ -307,7 +317,7 @@ describe("BaseWorker Retry and DLQ Logic", () => {
       });
 
       await expect(worker.processInline(inlineEvent)).resolves.toBeUndefined();
-      expect(mockConsumer.deadLetter).toHaveBeenCalledWith(inlineEvent);
+      expect(mockConsumer.deadLetter).toHaveBeenCalledWith(inlineEvent, expect.any(String));
       expect(mockConsumer.nack).not.toHaveBeenCalled();
     });
 
@@ -1132,7 +1142,7 @@ describe("DeliveryWorker", () => {
     );
   });
 
-  it("handles channel fallback by publishing to the next channel in the fallbackChain if all providers fail", async () => {
+  it("handles channel fallback by publishing to the next channel in the fallbackChain once its attempts are spent", async () => {
     const failingTransport = { send: vi.fn().mockRejectedValue(new Error("Fatal Error")) };
     mockTransportRegistry.getAll.mockReturnValue([failingTransport]);
 
@@ -1151,6 +1161,7 @@ describe("DeliveryWorker", () => {
           channel: "email",
           priority: "normal",
           deliveryOptions: {},
+          deliveryAttemptCount: 2, // the last of the default 3
           fallbackChain: ["sms", "push"], // Fallback chain provided
           recipient: {
             id: "usr-2",
@@ -1378,7 +1389,9 @@ describe("DeliveryWorker", () => {
       { send: vi.fn().mockResolvedValue({ success: false, error: "mailbox full" }) },
     ]);
 
-    await expect(worker.process(dispatched() as any)).rejects.toThrow("mailbox full");
+    await expect(worker.process(dispatched({ deliveryAttemptCount: 2 }) as any)).rejects.toThrow(
+      "mailbox full",
+    );
     expect(mockGlobalEmitter.emit).toHaveBeenCalledWith(
       "delivery:failed",
       "123e4567-e89b-12d3-a456-426614174001",
@@ -1393,7 +1406,9 @@ describe("DeliveryWorker", () => {
       { send: vi.fn().mockRejectedValue(new Error("smtp down")) },
     ]);
 
-    await expect(worker.process(dispatched() as any)).rejects.toThrow("smtp down");
+    await expect(worker.process(dispatched({ deliveryAttemptCount: 2 }) as any)).rejects.toThrow(
+      "smtp down",
+    );
     expect(worker["idempotency"].unmark).toHaveBeenCalledWith(
       "123e4567-e89b-12d3-a456-426614174001",
     );
@@ -1487,6 +1502,9 @@ describe("DeliveryWorker", () => {
 
   describe("provider rate limiting", () => {
     const limitedTransport = (send: any) => ({ send, limits: { limit: 5, windowSeconds: 60 } });
+    /** The payload parked in Postgres, which is what the scheduler releases. */
+    const parkedPayload = () =>
+      (worker as any).db.insert.mock.results[0].value.values.mock.calls[0][0].payload;
 
     it("reschedules the task instead of sending when the provider limit is hit", async () => {
       const send = vi.fn();
@@ -1501,7 +1519,7 @@ describe("DeliveryWorker", () => {
       const envelope = mockScheduledProducer.publish.mock.calls[0]![0];
       expect(envelope.type).toBe("notification.scheduled");
       expect(new Date(envelope.payload.scheduledAt).getTime()).toBeGreaterThan(Date.now());
-      expect(envelope.payload.throttleAttemptCount).toBe(1);
+      expect(parkedPayload().throttleAttemptCount).toBe(1);
     });
 
     it("counts each throttled attempt so the retries cannot loop forever", async () => {
@@ -1510,7 +1528,7 @@ describe("DeliveryWorker", () => {
 
       await worker.process(dispatched({ throttleAttemptCount: 2 }) as any);
 
-      expect(mockScheduledProducer.publish.mock.calls[0]![0].payload.throttleAttemptCount).toBe(3);
+      expect(parkedPayload().throttleAttemptCount).toBe(3);
     });
 
     it("gives up once the throttled retries exceed maxAttempts", async () => {
@@ -1629,9 +1647,9 @@ describe("DeliveryWorker", () => {
   // ── Circuit breaker ───────────────────────────────────────────────────────
 
   /**
-   * The worker keys one breaker per `${channel}:${transport.constructor.name}`,
-   * so these transports are real classes. Two object literals would both report
-   * the name "Object" and quietly share a single breaker.
+   * The worker keys one breaker per provider: its channel and class name,
+   * numbered when a channel has two of one class. These transports are real
+   * classes so the keys read the way production ones do.
    */
   describe("circuit breaker", () => {
     class FlakyTransport {
@@ -1663,16 +1681,36 @@ describe("DeliveryWorker", () => {
       expect(flaky.send).toHaveBeenCalledTimes(5);
     });
 
-    it("reports an open breaker as a delivery failure instead of letting it escape", async () => {
+    it("parks a task an open breaker turned away for a later attempt", async () => {
+      const flaky = new FlakyTransport();
+      mockTransportRegistry.getAll.mockReturnValue([flaky]);
+
+      for (let i = 0; i < 5; i++) await attempt(dispatched());
+      mockScheduledProducer.publish.mockClear();
+      const err = await attempt(dispatched());
+
+      // An open circuit says the provider is down, not that the message is bad.
+      expect(err).toBeUndefined();
+      expect(flaky.send).toHaveBeenCalledTimes(5);
+      expect(mockScheduledProducer.publish).toHaveBeenCalledTimes(1);
+      expect(mockGlobalEmitter.emit).not.toHaveBeenCalledWith(
+        "delivery:failed",
+        expect.anything(),
+        "Circuit breaker is OPEN",
+        expect.anything(),
+        expect.anything(),
+      );
+    });
+
+    it("reports an open breaker on a task's last attempt as a delivery failure", async () => {
       mockTransportRegistry.getAll.mockReturnValue([new FlakyTransport()]);
 
       for (let i = 0; i < 5; i++) await attempt(dispatched());
-      const err = await attempt(dispatched());
+      const err = await attempt(dispatched({ deliveryAttemptCount: 2 }));
 
-      // A short-circuit is still a delivery failure: it must reach the emitter
-      // and come back as NonRetryableError, or the task retries forever against
-      // a provider the worker has already given up on.
-      expect(err).toBeInstanceOf(Error);
+      // Out of attempts, a short-circuit is a delivery failure like any other:
+      // it must reach the emitter and come back as NonRetryableError.
+      expect(err).toBeInstanceOf(NonRetryableError);
       expect((err as Error).message).toBe("Circuit breaker is OPEN");
       expect(mockGlobalEmitter.emit).toHaveBeenLastCalledWith(
         "delivery:failed",
@@ -1766,6 +1804,9 @@ describe("DeliveryWorker", () => {
    * a retryable throw would send an already-exhausted task around again.
    */
   describe("fallback chain exhaustion", () => {
+    /** A task on the last of its attempts, so a failure is final. */
+    const lastAttempt = (payload: Record<string, unknown> = {}) =>
+      dispatched({ deliveryAttemptCount: 2, ...payload });
     const allProvidersFail = () =>
       mockTransportRegistry.getAll.mockReturnValue([
         { send: vi.fn().mockRejectedValue(new Error("Provider down")) },
@@ -1774,7 +1815,7 @@ describe("DeliveryWorker", () => {
     it("ends the task when the only channel fails with nothing to fall back to", async () => {
       allProvidersFail();
 
-      await expect(worker.process(dispatched() as any)).rejects.toBeInstanceOf(NonRetryableError);
+      await expect(worker.process(lastAttempt() as any)).rejects.toBeInstanceOf(NonRetryableError);
 
       expect(mockEnrichedProducers.normal.publish).not.toHaveBeenCalled();
       expect(mockGlobalEmitter.emit).toHaveBeenCalledWith(
@@ -1790,7 +1831,7 @@ describe("DeliveryWorker", () => {
       allProvidersFail();
 
       await expect(
-        worker.process(dispatched({ fallbackChain: [], recipient }) as any),
+        worker.process(lastAttempt({ fallbackChain: [], recipient }) as any),
       ).rejects.toBeInstanceOf(NonRetryableError);
 
       expect(mockEnrichedProducers.normal.publish).not.toHaveBeenCalled();
@@ -1803,7 +1844,7 @@ describe("DeliveryWorker", () => {
       // Without one the chain is inert, so the task has to fail terminally
       // rather than look like it was handed on.
       await expect(
-        worker.process(dispatched({ fallbackChain: ["sms", "push"] }) as any),
+        worker.process(lastAttempt({ fallbackChain: ["sms", "push"] }) as any),
       ).rejects.toBeInstanceOf(NonRetryableError);
 
       expect(mockEnrichedProducers.normal.publish).not.toHaveBeenCalled();
@@ -1812,7 +1853,7 @@ describe("DeliveryWorker", () => {
     it("clears the chain on the final hop instead of passing an empty array on", async () => {
       allProvidersFail();
 
-      await worker.process(dispatched({ fallbackChain: ["sms"], recipient }) as any);
+      await worker.process(lastAttempt({ fallbackChain: ["sms"], recipient }) as any);
 
       const published = mockEnrichedProducers.normal.publish.mock.calls[0]![0];
       expect(published.payload.channel).toBe("sms");
@@ -1824,7 +1865,7 @@ describe("DeliveryWorker", () => {
       allProvidersFail();
 
       // Hop one: email fails and hands the task to sms with the chain spent.
-      await worker.process(dispatched({ fallbackChain: ["sms"], recipient }) as any);
+      await worker.process(lastAttempt({ fallbackChain: ["sms"], recipient }) as any);
       const next = mockEnrichedProducers.normal.publish.mock.calls[0]![0].payload;
       expect(next.channel).toBe("sms");
 
@@ -1832,7 +1873,7 @@ describe("DeliveryWorker", () => {
       mockEnrichedProducers.normal.publish.mockClear();
       await expect(
         worker.process(
-          dispatched({
+          lastAttempt({
             channel: next.channel,
             fallbackChain: next.fallbackChain,
             destination: "+15551234567",
@@ -3265,6 +3306,7 @@ describe("DeliveryWorker", () => {
           channel: "email",
           destination: "test@example.com",
           priority: "normal",
+          deliveryAttemptCount: 2,
           fallbackChain: ["sms"],
           recipient: { phone: "+1234567890" },
         },
@@ -3304,8 +3346,10 @@ describe("DeliveryWorker", () => {
       },
     };
 
-    // process should throw the error so the worker retry logic handles it
-    await expect(worker.process(msg as any)).rejects.toThrow("Circuit breaker is OPEN");
+    // Parked through the scheduler for a later attempt, not failed.
+    await expect(worker.process(msg as any)).resolves.toBeUndefined();
+    expect(mockScheduledProducer.publish).toHaveBeenCalledTimes(1);
+    expect(mockIdempotency.markProcessed).not.toHaveBeenCalled();
     // Ensure it was NOT fallback-published
     expect(mockEnrichedProducers.normal.publish).not.toHaveBeenCalled();
   });
@@ -3332,6 +3376,7 @@ describe("DeliveryWorker", () => {
           channel: "email",
           destination: "test@example.com",
           priority: "normal",
+          deliveryAttemptCount: 2,
           fallbackChain: ["sms"],
           recipient: { phone: "+15551234567" },
         },
@@ -3436,6 +3481,7 @@ describe("DeliveryWorker", () => {
       event: {
         payload: {
           taskId: "task-all-fail",
+          deliveryAttemptCount: 2,
           projectId: "proj-1",
           enrichedEventId: "evt-1",
           recipientId: "usr-1",

tests/workflows.test.ts

@@ -296,8 +296,10 @@ describe("Workflow Engine Edge Cases", () => {
     expect(mockDb.set).toHaveBeenLastCalledWith({ status: "completed" });
   });
 
-  it("handles step.waitForEvent timeout (returns null)", async () => {
-    // 3rd run: waking up from waitForEvent timeout
+  it("goes back to sleep when resumed before the awaited event or its timeout", async () => {
+    // A resume with the step still unfilled — no event recorded, no timeout —
+    // is spurious (a duplicate trigger, a replayed message). A real timeout
+    // writes { timedOut: true }; see the next test.
     const msg: StreamMessage = {
       id: "5-0",
       event: {
@@ -332,13 +334,9 @@ describe("Workflow Engine Edge Cases", () => {
 
     await worker.process(msg);
 
-    // Should NOT have published the success notification
     expect(mockNotificationProducer.publish).not.toHaveBeenCalled();
-    // But should have completed
-    expect(mockDb.update).toHaveBeenCalled();
-    // Wait, mockDb.set could be called multiple times now (first running, then completed)
-    // We expect the LAST call to be completed
-    expect(mockDb.set).toHaveBeenLastCalledWith({ status: "completed" });
+    // Suspended again rather than carrying on as if the wait had timed out.
+    expect(mockDb.set).toHaveBeenLastCalledWith({ status: "pending" });
   });
 
   it("handles step.waitForEvent when persisted step output is { timedOut: true }", async () => {

vitest.config.ts

@@ -11,6 +11,8 @@ export default defineConfig({
     environment: "node",
     include: ["tests/**/*.test.ts", "packages/**/*.test.ts"],
     exclude: ["**/node_modules/**", "**/dist/**", "profiling/**", ".agents/**"],
+    // Starts the Postgres and Redis every file under tests/integration shares.
+    globalSetup: ["tests/integration/support/global-setup.ts"],
     testTimeout: 1200_000,
     hookTimeout: 1200_000,
     coverage: {