feat(modality)!: SnapOtter 2.0 phase 3 modality framework: media/doc engines, pool routing, display modes (#218)

2026-08-03 07:46:42 +02:00 · 2026-06-13 10:18:39 +08:00
parent c451b939c7
commit d647d8ed19
99 changed files with 4380 additions and 1813 deletions
@@ -1,7 +1,8 @@
-import { CATEGORIES, TOOLS } from "@snapotter/shared";
+import { CATEGORIES, MODALITIES, TOOLS } from "@snapotter/shared";
 import { useEffect, useMemo, useState } from "react";
 import { useTranslation } from "@/contexts/i18n-context";
-import { getCategoryName } from "@/lib/tool-i18n";
+import { ICON_MAP } from "@/lib/icon-map";
+import { getCategoryName, getModalityName } from "@/lib/tool-i18n";
 import { useFeaturesStore } from "@/stores/features-store";
 import { useSettingsStore } from "@/stores/settings-store";
 import { SearchBar } from "../common/search-bar";
@@ -38,14 +39,17 @@ export function ToolPanel() {
    );
  }, [search, visibleTools]);

-  const groupedTools = useMemo(() => {
-    const groups = new Map<string, typeof TOOLS>();
+  const groupedByModality = useMemo(() => {
+    const byModality = new Map<string, Map<string, typeof TOOLS>>();
    for (const tool of filteredTools) {
-      const list = groups.get(tool.category) || [];
+      const key = tool.modality === "file" ? "document" : tool.modality;
+      const cats = byModality.get(key) ?? new Map<string, typeof TOOLS>();
+      const list = cats.get(tool.category) ?? [];
      list.push(tool);
-      groups.set(tool.category, list);
+      cats.set(tool.category, list);
+      byModality.set(key, cats);
    }
-    return groups;
+    return byModality;
  }, [filteredTools]);

  return (
@@ -54,18 +58,41 @@ export function ToolPanel() {
        <SearchBar value={search} onChange={setSearch} />
      </div>
      <div className="px-3 pb-4 flex-1">
-        {CATEGORIES.filter((cat) => groupedTools.has(cat.id)).map((category) => (
-          <div key={category.id} className="mb-4">
-            <h3 className="text-xs font-semibold uppercase text-muted-foreground tracking-wider mb-2">
-              {getCategoryName(t, category.id, category.name)}
-            </h3>
-            <div className="space-y-0.5">
-              {groupedTools.get(category.id)?.map((tool) => (
-                <ToolCard key={tool.id} tool={tool} />
-              ))}
-            </div>
-          </div>
-        ))}
+        {MODALITIES.filter((m) => m.id !== "file" && groupedByModality.has(m.id)).map(
+          (modality) => {
+            const ModalityIcon = ICON_MAP[modality.icon] as React.ComponentType<{
+              className?: string;
+            }>;
+            const categoryMap = groupedByModality.get(modality.id);
+            if (!categoryMap) return null;
+            return (
+              <div key={modality.id} className="mb-5">
+                <div className="flex items-center gap-1.5 mb-2">
+                  {ModalityIcon && <ModalityIcon className="h-4 w-4 text-foreground/70 shrink-0" />}
+                  <h2 className="text-xs font-bold uppercase text-foreground/70 tracking-wider">
+                    {getModalityName(
+                      t,
+                      modality.id,
+                      modality.id === "document" ? "Documents & Files" : modality.name,
+                    )}
+                  </h2>
+                </div>
+                {CATEGORIES.filter((cat) => categoryMap.has(cat.id)).map((category) => (
+                  <div key={category.id} className="mb-4">
+                    <h3 className="text-xs font-semibold uppercase text-muted-foreground tracking-wider mb-2">
+                      {getCategoryName(t, category.id, category.name)}
+                    </h3>
+                    <div className="space-y-0.5">
+                      {categoryMap.get(category.id)?.map((tool) => (
+                        <ToolCard key={tool.id} tool={tool} />
+                      ))}
+                    </div>
+                  </div>
+                ))}
+              </div>
+            );
+          },
+        )}
        {filteredTools.length === 0 && (
          <p className="text-sm text-muted-foreground text-center py-8">No tools found</p>
        )}
@@ -0,0 +1,100 @@
+import * as pdfjs from "pdfjs-dist";
+import { useEffect, useRef, useState } from "react";
+import { useTranslation } from "@/contexts/i18n-context";
+import { useFileStore } from "@/stores/file-store";
+
+pdfjs.GlobalWorkerOptions.workerSrc = new URL(
+  "pdfjs-dist/build/pdf.worker.min.mjs",
+  import.meta.url,
+).href;
+
+/** pdf.js canvas viewer for the document display mode (spec 4.6). */
+export function DocumentView() {
+  const { t } = useTranslation();
+  const entry = useFileStore((s) => s.entries[s.selectedIndex]);
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const [page, setPage] = useState(1);
+  const [pageCount, setPageCount] = useState(0);
+  const [error, setError] = useState<string | null>(null);
+  const src = entry?.processedUrl ?? entry?.blobUrl;
+
+  /* A+B: reset pagination and clear stale errors when the document changes.
+     src is intentionally a trigger-only dep (not read inside the callback). */
+  // biome-ignore lint/correctness/useExhaustiveDependencies: src is the trigger
+  useEffect(() => {
+    setPage(1);
+    setPageCount(0);
+    setError(null);
+  }, [src]);
+
+  /* C+D: cancel in-flight renders and destroy the doc proxy on cleanup. */
+  useEffect(() => {
+    if (!src || !canvasRef.current) return;
+    let cancelled = false;
+    let doc: pdfjs.PDFDocumentProxy | undefined;
+    let renderTask: pdfjs.RenderTask | undefined;
+
+    (async () => {
+      try {
+        doc = await pdfjs.getDocument({ url: src }).promise;
+        if (cancelled) return;
+        setPageCount(doc.numPages);
+        const pdfPage = await doc.getPage(Math.min(page, doc.numPages));
+        if (cancelled) return;
+        const viewport = pdfPage.getViewport({ scale: 1.2 });
+        const canvas = canvasRef.current;
+        if (!canvas) return;
+        canvas.width = viewport.width;
+        canvas.height = viewport.height;
+        renderTask = pdfPage.render({ canvas, viewport });
+        await renderTask.promise;
+      } catch (err) {
+        if (!cancelled) {
+          setError(err instanceof Error ? err.message : String(err));
+        }
+      }
+    })();
+
+    return () => {
+      cancelled = true;
+      renderTask?.cancel();
+      doc?.loadingTask.destroy();
+    };
+  }, [src, page]);
+
+  if (!entry) return null;
+
+  return (
+    <div className="flex h-full w-full flex-col items-center gap-2 overflow-auto p-4">
+      {error && <p className="p-4 text-sm text-destructive">{t.tools.documentView.loadFailed}</p>}
+      <canvas
+        ref={canvasRef}
+        className={`max-w-full rounded border${error ? " hidden" : ""}`}
+        data-testid="document-canvas"
+      />
+      {!error && pageCount > 1 && (
+        <div className="flex items-center gap-3 text-sm">
+          <button
+            type="button"
+            disabled={page <= 1}
+            onClick={() => setPage((p) => p - 1)}
+            className="disabled:opacity-50"
+          >
+            {t.tools.documentView.previousPage}
+          </button>
+          <span>
+            {page} / {pageCount}
+          </span>
+          <button
+            type="button"
+            disabled={page >= pageCount}
+            onClick={() => setPage((p) => p + 1)}
+            className="disabled:opacity-50"
+          >
+            {t.tools.documentView.nextPage}
+          </button>
+        </div>
+      )}
+    </div>
+  );
+}
@@ -0,0 +1,33 @@
+import { useTranslation } from "@/contexts/i18n-context";
+import { useFileStore } from "@/stores/file-store";
+
+/**
+ * Native <video>/<audio> playback over the Range-capable download endpoint
+ * (spec 4.6). Shows the processed result when present, else the source file.
+ */
+export function MediaPlayerView() {
+  const { t } = useTranslation();
+  const entry = useFileStore((s) => s.entries[s.selectedIndex]);
+  if (!entry) return null;
+  const src = entry.processedUrl ?? entry.blobUrl;
+  const isAudio = entry.modality === "audio";
+  return (
+    <div className="flex h-full w-full items-center justify-center p-4">
+      {isAudio ? (
+        <audio controls src={src} className="w-full max-w-xl" data-testid="media-player-audio">
+          <track kind="captions" />
+        </audio>
+      ) : (
+        <video
+          controls
+          src={src}
+          className="max-h-full max-w-full rounded-lg"
+          data-testid="media-player-video"
+        >
+          <track kind="captions" />
+          {t.tools.mediaPlayer.unsupported}
+        </video>
+      )}
+    </div>
+  );
+}
@@ -14,6 +14,7 @@ import {
  Expand,
  Eye,
  EyeOff,
+  FileArchive,
  FileImage,
  FileOutput,
  FilePen,
@@ -53,6 +54,7 @@ import {
  Type,
  Undo2,
  UserCheck,
+  Video,
  Wand,
  Wrench,
  Zap,
@@ -76,6 +78,7 @@ export const ICON_MAP: Record<string, LucideIcon> = {
  Expand,
  Eye,
  EyeOff,
+  FileArchive,
  FilePen,
  FileImage,
  FileOutput,
@@ -115,6 +118,7 @@ export const ICON_MAP: Record<string, LucideIcon> = {
  Type,
  Undo2,
  UserCheck,
+  Video,
  Wand,
  Wrench,
  Zap,
@@ -15,7 +15,9 @@ export type DisplayMode =
  | "interactive-eraser"
  | "interactive-split"
  | "no-dropzone"
-  | "custom-results";
+  | "custom-results"
+  | "media-player"
+  | "document";

 export const TOOL_DISPLAY_MODES: Record<string, DisplayMode> = {
  // Essentials
@@ -13,3 +13,13 @@ export function getToolDescription(t: TranslationKeys, toolId: string, fallback:
 export function getCategoryName(t: TranslationKeys, categoryId: string, fallback: string): string {
  return (t.categories as Record<string, string>)[categoryId] ?? fallback;
 }
+
+/**
+ * Returns the i18n display name for a modality group header.
+ * The "document" key maps to the merged "documentsAndFiles" i18n entry;
+ * "file" is never rendered as a top-level header (merged into document).
+ */
+export function getModalityName(t: TranslationKeys, modalityId: string, fallback: string): string {
+  const key = modalityId === "document" ? "documentsAndFiles" : modalityId;
+  return (t.modalities as Record<string, string>)[key] ?? fallback;
+}
@@ -1,4 +1,10 @@
-import { CATEGORIES, PYTHON_SIDECAR_TOOLS, TOOL_BUNDLE_MAP, TOOLS } from "@snapotter/shared";
+import {
+  CATEGORIES,
+  MODALITIES,
+  PYTHON_SIDECAR_TOOLS,
+  TOOL_BUNDLE_MAP,
+  TOOLS,
+} from "@snapotter/shared";
 import { Clock, Download, Loader2 } from "lucide-react";
 import { useCallback, useEffect, useMemo } from "react";
 import { useLocation, useNavigate } from "react-router-dom";
@@ -8,7 +14,7 @@ import { AppLayout } from "@/components/layout/app-layout";
 import { useTranslation } from "@/contexts/i18n-context";
 import { useMobile } from "@/hooks/use-mobile";
 import { ICON_MAP } from "@/lib/icon-map";
-import { getCategoryName, getToolName } from "@/lib/tool-i18n";
+import { getCategoryName, getModalityName, getToolName } from "@/lib/tool-i18n";
 import { useFeaturesStore } from "@/stores/features-store";
 import { useFileStore } from "@/stores/file-store";
 import { useSettingsStore } from "@/stores/settings-store";
@@ -281,68 +287,103 @@ export function HomePage() {
            </div>
          </div>

-          {/* All tools by category */}
+          {/* All tools by modality and category */}
          <div className="p-4">
            <h3 className="text-xs font-semibold uppercase text-muted-foreground tracking-wider mb-3">
              {t.homePage.allTools}
            </h3>
-            {CATEGORIES.map((category) => {
-              const categoryTools = TOOLS.filter((t) => t.category === category.id);
-              if (categoryTools.length === 0) return null;
+            {MODALITIES.filter((m) => {
+              if (m.id === "file") return false;
+              const key = m.id;
+              return TOOLS.some(
+                (tool) => tool.modality === key || (key === "document" && tool.modality === "file"),
+              );
+            }).map((modality) => {
+              const ModalityIcon = ICON_MAP[modality.icon] as React.ComponentType<{
+                className?: string;
+              }>;
+              const modalityTools = TOOLS.filter(
+                (tool) =>
+                  tool.modality === modality.id ||
+                  (modality.id === "document" && tool.modality === "file"),
+              );
+              const categoryMap = new Map<string, typeof TOOLS>();
+              for (const tool of modalityTools) {
+                const list = categoryMap.get(tool.category) ?? [];
+                list.push(tool);
+                categoryMap.set(tool.category, list);
+              }
              return (
-                <div key={category.id} className="mb-4">
-                  <p
-                    className="text-xs font-medium text-muted-foreground mb-1.5"
-                    style={{ color: category.color }}
-                  >
-                    {getCategoryName(t, category.id, category.name)}
-                  </p>
-                  <div className="space-y-0.5">
-                    {categoryTools.map((tool) => {
-                      const Icon =
-                        (ICON_MAP[tool.icon] as React.ComponentType<{ className?: string }>) ??
-                        ICON_MAP.FileImage;
-                      const status = getToolStatus(tool.id);
-                      return (
-                        <button
-                          key={tool.id}
-                          type="button"
-                          onClick={() => navigate(tool.route)}
-                          className="flex items-center gap-2.5 w-full py-1.5 px-2 rounded-lg text-start transition-colors hover:bg-muted text-foreground"
-                        >
-                          <Icon className="h-4 w-4 text-muted-foreground shrink-0" />
-                          <span className="text-sm">{getToolName(t, tool.id, tool.name)}</span>
-                          {status === "not_installed" && (
-                            <>
-                              <Download
-                                className="h-3.5 w-3.5 text-muted-foreground ms-auto"
-                                aria-hidden="true"
-                              />
-                              <span className="sr-only">{t.a11y.notInstalled}</span>
-                            </>
-                          )}
-                          {status === "queued" && (
-                            <>
-                              <Clock
-                                className="h-3.5 w-3.5 text-muted-foreground ms-auto"
-                                aria-hidden="true"
-                              />
-                              <span className="sr-only">{t.a11y.queued}</span>
-                            </>
-                          )}
-                          {status === "installing" && (
-                            <>
-                              <Loader2
-                                className="h-3.5 w-3.5 text-muted-foreground ms-auto animate-spin"
-                                aria-hidden="true"
-                              />
-                              <span className="sr-only">{t.a11y.installing}</span>
-                            </>
-                          )}
-                        </button>
-                      );
-                    })}
+                <div key={modality.id} className="mb-5">
+                  <div className="flex items-center gap-1.5 mb-2">
+                    {ModalityIcon && (
+                      <ModalityIcon className="h-4 w-4 text-foreground/70 shrink-0" />
+                    )}
+                    <p className="text-xs font-bold uppercase text-foreground/70 tracking-wider">
+                      {getModalityName(
+                        t,
+                        modality.id,
+                        modality.id === "document" ? "Documents & Files" : modality.name,
+                      )}
+                    </p>
                  </div>
+                  {CATEGORIES.filter((cat) => categoryMap.has(cat.id)).map((category) => (
+                    <div key={category.id} className="mb-4">
+                      <p
+                        className="text-xs font-medium text-muted-foreground mb-1.5"
+                        style={{ color: category.color }}
+                      >
+                        {getCategoryName(t, category.id, category.name)}
+                      </p>
+                      <div className="space-y-0.5">
+                        {categoryMap.get(category.id)?.map((tool) => {
+                          const Icon =
+                            (ICON_MAP[tool.icon] as React.ComponentType<{
+                              className?: string;
+                            }>) ?? ICON_MAP.FileImage;
+                          const status = getToolStatus(tool.id);
+                          return (
+                            <button
+                              key={tool.id}
+                              type="button"
+                              onClick={() => navigate(tool.route)}
+                              className="flex items-center gap-2.5 w-full py-1.5 px-2 rounded-lg text-start transition-colors hover:bg-muted text-foreground"
+                            >
+                              <Icon className="h-4 w-4 text-muted-foreground shrink-0" />
+                              <span className="text-sm">{getToolName(t, tool.id, tool.name)}</span>
+                              {status === "not_installed" && (
+                                <>
+                                  <Download
+                                    className="h-3.5 w-3.5 text-muted-foreground ms-auto"
+                                    aria-hidden="true"
+                                  />
+                                  <span className="sr-only">{t.a11y.notInstalled}</span>
+                                </>
+                              )}
+                              {status === "queued" && (
+                                <>
+                                  <Clock
+                                    className="h-3.5 w-3.5 text-muted-foreground ms-auto"
+                                    aria-hidden="true"
+                                  />
+                                  <span className="sr-only">{t.a11y.queued}</span>
+                                </>
+                              )}
+                              {status === "installing" && (
+                                <>
+                                  <Loader2
+                                    className="h-3.5 w-3.5 text-muted-foreground ms-auto animate-spin"
+                                    aria-hidden="true"
+                                  />
+                                  <span className="sr-only">{t.a11y.installing}</span>
+                                </>
+                              )}
+                            </button>
+                          );
+                        })}
+                      </div>
+                    </div>
+                  ))}
                </div>
              );
            })}
@@ -7,7 +7,7 @@ import {
  FileImage,
  Loader2,
 } from "lucide-react";
-import { Suspense, useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState } from "react";
 import type { Crop } from "react-image-crop";
 import { Link, useParams } from "react-router-dom";
 import { BeforeAfterSlider } from "@/components/common/before-after-slider";
@@ -41,6 +41,13 @@ import { usePdfToImageStore } from "@/stores/pdf-to-image-store";
 import { useQrStore } from "@/stores/qr-store";
 import { useSplitStore } from "@/stores/split-store";

+const MediaPlayerView = lazy(() =>
+  import("@/components/tools/media-player-view").then((m) => ({ default: m.MediaPlayerView })),
+);
+const DocumentView = lazy(() =>
+  import("@/components/tools/document-view").then((m) => ({ default: m.DocumentView })),
+);
+
 /** Formats that browsers can render in <img> tags. */
 const BROWSER_PREVIEWABLE_EXTS = new Set([
  "jpg",
@@ -469,6 +476,24 @@ export function ToolPage() {
      );
    }

+    // Media player: native <video>/<audio> element
+    if (displayMode === "media-player" && hasFile) {
+      return (
+        <Suspense fallback={<div className="text-sm text-muted-foreground">Loading...</div>}>
+          <MediaPlayerView />
+        </Suspense>
+      );
+    }
+
+    // Document viewer: pdf.js canvas with pagination
+    if (displayMode === "document" && hasFile) {
+      return (
+        <Suspense fallback={<div className="text-sm text-muted-foreground">Loading...</div>}>
+          <DocumentView />
+        </Suspense>
+      );
+    }
+
    // Show error state for failed batch files (before interactive canvas blocks,
    // which also match !hasProcessed and would show the canvas instead of the error)
    if (hasFile && !hasProcessed && currentEntry?.status === "failed") {
@@ -1,6 +1,24 @@
+import { detectModalityFromMime, type Modality } from "@snapotter/shared";
 import { create } from "zustand";
 import { fetchDecodedPreview, needsServerPreview } from "@/lib/image-preview";

+export type PreviewKind = "image" | "video" | "audio" | "document" | "none";
+
+export function previewKindFor(modality: Modality): PreviewKind {
+  switch (modality) {
+    case "image":
+      return "image";
+    case "video":
+      return "video";
+    case "audio":
+      return "audio";
+    case "document":
+      return "document";
+    default:
+      return "none";
+  }
+}
+
 export interface FileEntry {
  file: File;
  blobUrl: string;
@@ -15,6 +33,8 @@ export interface FileEntry {
  status: "pending" | "processing" | "completed" | "failed";
  error: string | null;
  serverFileId?: string;
+  modality: Modality;
+  previewKind: PreviewKind;
 }

 // ---------------------------------------------------------------------------
@@ -22,6 +42,7 @@ export interface FileEntry {
 // ---------------------------------------------------------------------------

 function createEntry(file: File): FileEntry {
+  const modality = detectModalityFromMime(file.type);
  return {
    file,
    blobUrl: URL.createObjectURL(file),
@@ -36,6 +57,8 @@ function createEntry(file: File): FileEntry {
    status: "pending",
    error: null,
    serverFileId: undefined,
+    modality,
+    previewKind: previewKindFor(modality),
  };
 }