feat(automate): make the pipeline builder fully multi-modal (#335)

* feat(shared): add outputModality to Tool metadata for crossing tools

* feat(shared): add modalityForExtension, toolInputModality, toolOutputModality

* fix(pipeline): route finalize and parent jobs to the pipeline's modality pool

* feat(automate): add ConvertAudioControls pipeline step (exemplar)

* feat(automate): add video tool settings controls to pipelines

* feat(automate): add audio tool settings controls to pipelines

* feat(automate): add document tool settings controls to pipelines

* feat(automate): add chart-maker settings control to pipelines

* feat(automate): warn on modality-incompatible pipeline steps

* feat(automate): add single-file download button for pipeline results

* refactor(automate): modality-aware icons, nav handler rename, mobile size bar

* test(pipeline): cover audio, document, file, and cross-modality chains

* i18n(automate): translate the modality-warning tooltip

* test(pipeline): gate media-pool routing assertion on ffmpeg availability

packages/shared/src/constants.ts

@@ -146,6 +146,7 @@ export const TOOLS: Tool[] = [
     route: "/image-to-pdf",
     modality: "image",
     acceptedInputs: IMAGE_INPUTS,
+    outputModality: "document",
     executionHint: "fast",
   },
   {
@@ -247,6 +248,7 @@ export const TOOLS: Tool[] = [
     route: "/ocr",
     modality: "image",
     acceptedInputs: IMAGE_INPUTS,
+    outputModality: "file",
     executionHint: "long",
   },
   {
@@ -258,6 +260,7 @@ export const TOOLS: Tool[] = [
     route: "/ocr-pdf",
     modality: "document",
     acceptedInputs: [".pdf"],
+    outputModality: "file",
     executionHint: "long",
   },
   {
@@ -390,6 +393,7 @@ export const TOOLS: Tool[] = [
     route: "/transcribe-audio",
     modality: "audio",
     acceptedInputs: AUDIO_INPUTS,
+    outputModality: "file",
     executionHint: "long",
   },
   {
@@ -401,6 +405,7 @@ export const TOOLS: Tool[] = [
     route: "/auto-subtitles",
     modality: "video",
     acceptedInputs: VIDEO_INPUTS,
+    outputModality: "file",
     executionHint: "long",
   },
   {
@@ -791,6 +796,7 @@ export const TOOLS: Tool[] = [
     route: "/pdf-to-image",
     modality: "document",
     acceptedInputs: [".pdf"],
+    outputModality: "image",
     executionHint: "fast",
   },
   // Video
@@ -847,6 +853,7 @@ export const TOOLS: Tool[] = [
     route: "/video-to-gif",
     modality: "video",
     acceptedInputs: VIDEO_INPUTS,
+    outputModality: "image",
     executionHint: "long",
   },
   {
@@ -1001,6 +1008,7 @@ export const TOOLS: Tool[] = [
     route: "/video-to-webp",
     modality: "video",
     acceptedInputs: VIDEO_INPUTS,
+    outputModality: "image",
     executionHint: "fast",
   },
   {
@@ -1012,6 +1020,7 @@ export const TOOLS: Tool[] = [
     route: "/video-to-frames",
     modality: "video",
     acceptedInputs: VIDEO_INPUTS,
+    outputModality: "file",
     executionHint: "fast",
   },
   {
@@ -1067,6 +1076,7 @@ export const TOOLS: Tool[] = [
     route: "/extract-subtitles",
     modality: "video",
     acceptedInputs: VIDEO_INPUTS,
+    outputModality: "file",
     executionHint: "fast",
   },
   {
@@ -1123,6 +1133,7 @@ export const TOOLS: Tool[] = [
     route: "/extract-audio",
     modality: "video",
     acceptedInputs: VIDEO_INPUTS,
+    outputModality: "audio",
     executionHint: "fast",
   },
   {
@@ -1266,6 +1277,7 @@ export const TOOLS: Tool[] = [
     route: "/waveform-image",
     modality: "audio",
     acceptedInputs: AUDIO_INPUTS,
+    outputModality: "image",
     executionHint: "fast",
   },
   {
@@ -1565,6 +1577,7 @@ export const TOOLS: Tool[] = [
     route: "/pdf-to-text",
     modality: "document",
     acceptedInputs: [".pdf"],
+    outputModality: "file",
     executionHint: "fast",
   },
   {
@@ -1679,6 +1692,7 @@ export const TOOLS: Tool[] = [
     route: "/chart-maker",
     modality: "file",
     acceptedInputs: [".csv", ".json"],
+    outputModality: "image",
     executionHint: "fast",
   },
   {

packages/shared/src/i18n/ar.ts

@@ -3199,6 +3199,9 @@ export const ar: TranslationKeys = {
     noStepsHeading: "لا توجد خطوات بعد",
     searchToolsPlaceholder: "بحث في الأدوات...",
     step: "خطوة",
+    modalityWarning: "عدم توافق النوع",
+    modalityWarningTooltip:
+      "تتوقع هذه الخطوة {expected} لكنها تتلقى {received}؛ قد تفشل عند تشغيل المسار.",
   },
   nav: {
     tools: "الأدوات",

packages/shared/src/i18n/de.ts

@@ -3238,6 +3238,9 @@ export const de: TranslationKeys = {
     noStepsHeading: "Noch keine Schritte",
     searchToolsPlaceholder: "Werkzeuge suchen...",
     step: "Schritt",
+    modalityWarning: "Modalitätskonflikt",
+    modalityWarningTooltip:
+      "Dieser Schritt erwartet {expected}, erhält aber {received}; er kann beim Ausführen der Pipeline fehlschlagen.",
   },
   nav: {
     tools: "Werkzeuge",

packages/shared/src/i18n/en.ts

@@ -3168,6 +3168,9 @@ export const en = {
     noStepsHeading: "No steps yet",
     searchToolsPlaceholder: "Search tools...",
     step: "Step",
+    modalityWarning: "Modality mismatch",
+    modalityWarningTooltip:
+      "This step expects {expected} but receives {received}; it may fail when you run the pipeline.",
   },
   nav: {
     tools: "Tools",

packages/shared/src/i18n/es.ts

@@ -3220,6 +3220,9 @@ export const es: TranslationKeys = {
     noStepsHeading: "Aún no hay pasos",
     searchToolsPlaceholder: "Buscar herramientas...",
     step: "Paso",
+    modalityWarning: "Incompatibilidad de modalidad",
+    modalityWarningTooltip:
+      "Este paso espera {expected} pero recibe {received}; puede fallar al ejecutar la canalización.",
   },
   nav: {
     tools: "Herramientas",

packages/shared/src/i18n/fr.ts

@@ -3243,6 +3243,9 @@ export const fr: TranslationKeys = {
     noStepsHeading: "Aucune étape pour le moment",
     searchToolsPlaceholder: "Rechercher des outils...",
     step: "Étape",
+    modalityWarning: "Incompatibilité de modalité",
+    modalityWarningTooltip:
+      "Cette étape attend {expected} mais reçoit {received} ; elle peut échouer lors de l'exécution du pipeline.",
   },
   nav: {
     tools: "Outils",

packages/shared/src/i18n/hi.ts

@@ -3194,6 +3194,9 @@ export const hi: TranslationKeys = {
     noStepsHeading: "अभी तक कोई चरण नहीं",
     searchToolsPlaceholder: "टूल खोजें...",
     step: "स्टेप",
+    modalityWarning: "मोडैलिटी मेल नहीं खाती",
+    modalityWarningTooltip:
+      "यह चरण {expected} की अपेक्षा करता है लेकिन {received} प्राप्त करता है; पाइपलाइन चलाने पर यह विफल हो सकता है।",
   },
   nav: {
     tools: "टूल्स",

packages/shared/src/i18n/id.ts

@@ -3220,6 +3220,9 @@ export const id: TranslationKeys = {
     noStepsHeading: "Belum ada langkah",
     searchToolsPlaceholder: "Cari alat...",
     step: "Langkah",
+    modalityWarning: "Ketidakcocokan modalitas",
+    modalityWarningTooltip:
+      "Langkah ini mengharapkan {expected} tetapi menerima {received}; mungkin gagal saat Anda menjalankan pipeline.",
   },
   nav: {
     tools: "Alat",

packages/shared/src/i18n/it.ts

@@ -3232,6 +3232,9 @@ export const it: TranslationKeys = {
     noStepsHeading: "Nessun passaggio",
     searchToolsPlaceholder: "Cerca strumenti...",
     step: "Passaggio",
+    modalityWarning: "Incompatibilità di modalità",
+    modalityWarningTooltip:
+      "Questo passaggio prevede {expected} ma riceve {received}; potrebbe non riuscire durante l'esecuzione della pipeline.",
   },
   nav: {
     tools: "Strumenti",

packages/shared/src/i18n/ja.ts

@@ -3171,6 +3171,9 @@ export const ja: TranslationKeys = {
     noStepsHeading: "ステップがありません",
     searchToolsPlaceholder: "ツールを検索...",
     step: "ステップ",
+    modalityWarning: "モダリティ不一致",
+    modalityWarningTooltip:
+      "このステップは {expected} を想定していますが {received} を受け取ります。パイプラインの実行時に失敗する可能性があります。",
   },
   nav: {
     tools: "ツール",

packages/shared/src/i18n/ko.ts

@@ -3154,6 +3154,9 @@ export const ko: TranslationKeys = {
     noStepsHeading: "단계가 없습니다",
     searchToolsPlaceholder: "도구 검색...",
     step: "단계",
+    modalityWarning: "모달리티 불일치",
+    modalityWarningTooltip:
+      "이 단계는 {expected}을(를) 기대하지만 {received}을(를) 받습니다. 파이프라인을 실행하면 실패할 수 있습니다.",
   },
   nav: {
     tools: "도구",

packages/shared/src/i18n/nl.ts

@@ -3228,6 +3228,9 @@ export const nl: TranslationKeys = {
     noStepsHeading: "Nog geen stappen",
     searchToolsPlaceholder: "Tools zoeken...",
     step: "Stap",
+    modalityWarning: "Modaliteitsmismatch",
+    modalityWarningTooltip:
+      "Deze stap verwacht {expected} maar ontvangt {received}; mogelijk mislukt deze bij het uitvoeren van de pipeline.",
   },
   nav: {
     tools: "Tools",

packages/shared/src/i18n/pl.ts

@@ -3235,6 +3235,9 @@ export const pl: TranslationKeys = {
     noStepsHeading: "Brak kroków",
     searchToolsPlaceholder: "Szukaj narzędzi...",
     step: "Krok",
+    modalityWarning: "Niezgodność modalności",
+    modalityWarningTooltip:
+      "Ten krok oczekuje {expected}, ale otrzymuje {received}; może się nie powieść podczas uruchamiania potoku.",
   },
   nav: {
     tools: "Narzędzia",

packages/shared/src/i18n/pt-BR.ts

@@ -3227,6 +3227,9 @@ export const ptBR: TranslationKeys = {
     noStepsHeading: "Nenhuma etapa ainda",
     searchToolsPlaceholder: "Buscar ferramentas...",
     step: "Passo",
+    modalityWarning: "Incompatibilidade de modalidade",
+    modalityWarningTooltip:
+      "Esta etapa espera {expected}, mas recebe {received}; pode falhar ao executar o pipeline.",
   },
   nav: {
     tools: "Ferramentas",

packages/shared/src/i18n/ru.ts

@@ -3223,6 +3223,9 @@ export const ru: TranslationKeys = {
     noStepsHeading: "Шагов пока нет",
     searchToolsPlaceholder: "Поиск инструментов...",
     step: "Шаг",
+    modalityWarning: "Несоответствие модальности",
+    modalityWarningTooltip:
+      "Этот шаг ожидает {expected}, но получает {received}; он может завершиться ошибкой при запуске конвейера.",
   },
   nav: {
     tools: "Инструменты",

packages/shared/src/i18n/sv.ts

@@ -3216,6 +3216,9 @@ export const sv: TranslationKeys = {
     noStepsHeading: "Inga steg ännu",
     searchToolsPlaceholder: "Sök verktyg...",
     step: "Steg",
+    modalityWarning: "Modalitetsfel",
+    modalityWarningTooltip:
+      "Det här steget förväntar sig {expected} men tar emot {received}; det kan misslyckas när du kör pipelinen.",
   },
   nav: {
     tools: "Verktyg",

packages/shared/src/i18n/th.ts

@@ -3180,6 +3180,8 @@ export const th: TranslationKeys = {
     noStepsHeading: "ยังไม่มีขั้นตอน",
     searchToolsPlaceholder: "ค้นหาเครื่องมือ...",
     step: "ขั้นตอน",
+    modalityWarning: "ประเภทไม่ตรงกัน",
+    modalityWarningTooltip: "ขั้นตอนนี้คาดหวัง {expected} แต่ได้รับ {received} อาจล้มเหลวเมื่อคุณเรียกใช้ไปป์ไลน์",
   },
   nav: {
     tools: "เครื่องมือ",

packages/shared/src/i18n/tr.ts

@@ -3229,6 +3229,9 @@ export const tr: TranslationKeys = {
     noStepsHeading: "Henüz adım yok",
     searchToolsPlaceholder: "Araç ara...",
     step: "Adım",
+    modalityWarning: "Modalite uyumsuzluğu",
+    modalityWarningTooltip:
+      "Bu adım {expected} bekliyor ancak {received} alıyor; işlem hattını çalıştırdığınızda başarısız olabilir.",
   },
   nav: {
     tools: "Araçlar",

packages/shared/src/i18n/uk.ts

@@ -3225,6 +3225,9 @@ export const uk: TranslationKeys = {
     noStepsHeading: "Кроків поки немає",
     searchToolsPlaceholder: "Пошук інструментів...",
     step: "Крок",
+    modalityWarning: "Невідповідність модальності",
+    modalityWarningTooltip:
+      "Цей крок очікує {expected}, але отримує {received}; він може завершитися помилкою під час запуску конвеєра.",
   },
   nav: {
     tools: "Інструменти",

packages/shared/src/i18n/vi.ts

@@ -3217,6 +3217,9 @@ export const vi: TranslationKeys = {
     noStepsHeading: "Chưa có bước nào",
     searchToolsPlaceholder: "Tìm công cụ...",
     step: "Bước",
+    modalityWarning: "Không tương thích loại",
+    modalityWarningTooltip:
+      "Bước này cần {expected} nhưng nhận {received}; có thể không thành công khi bạn chạy quy trình.",
   },
   nav: {
     tools: "Công cụ",

packages/shared/src/i18n/zh-CN.ts

@@ -3130,6 +3130,8 @@ export const zhCN: TranslationKeys = {
     noStepsHeading: "暂无步骤",
     searchToolsPlaceholder: "搜索工具...",
     step: "步骤",
+    modalityWarning: "模态不匹配",
+    modalityWarningTooltip: "此步骤需要 {expected}，但接收到 {received}；运行管道时可能会失败。",
   },
   nav: {
     tools: "工具",

packages/shared/src/i18n/zh-TW.ts

@@ -3129,6 +3129,8 @@ export const zhTW: TranslationKeys = {
     noStepsHeading: "尚無步驟",
     searchToolsPlaceholder: "搜尋工具...",
     step: "步驟",
+    modalityWarning: "模態不相符",
+    modalityWarningTooltip: "此步驟需要 {expected}，但收到 {received}；執行管線時可能會失敗。",
   },
   nav: {
     tools: "工具",

packages/shared/src/modality.ts

@@ -134,3 +134,42 @@ export function detectModalityFromMime(mime: string): Modality {
   }
   return "file";
 }
+
+const MODALITY_EXTENSION_SETS: [Modality, readonly string[]][] = [
+  ["image", IMAGE_INPUTS],
+  ["video", VIDEO_INPUTS],
+  ["audio", AUDIO_INPUTS],
+  ["document", DOCUMENT_INPUTS],
+  ["file", FILE_INPUTS],
+];
+
+/** Owning modality for a file extension (with or without leading dot), or null. */
+export function modalityForExtension(ext: string): Modality | null {
+  const norm = (ext.startsWith(".") ? ext : `.${ext}`).toLowerCase();
+  for (const [modality, set] of MODALITY_EXTENSION_SETS) {
+    if (set.includes(norm)) return modality;
+  }
+  return null;
+}
+
+/** The modality a tool consumes. Derived from acceptedInputs so tools whose `modality`
+ *  differs from their real input (gif-to-video, images-to-video) are correct. */
+export function toolInputModality(tool: {
+  modality: Modality;
+  acceptedInputs: string[];
+}): Modality {
+  const first = tool.acceptedInputs[0];
+  if (first) {
+    const m = modalityForExtension(first);
+    if (m) return m;
+  }
+  return tool.modality;
+}
+
+/** The modality a tool produces. Uses the explicit override, else the tool modality. */
+export function toolOutputModality(tool: {
+  modality: Modality;
+  outputModality?: Modality;
+}): Modality {
+  return tool.outputModality ?? tool.modality;
+}

packages/shared/src/types.ts

@@ -9,6 +9,8 @@ export interface Tool {
   route: string;
   modality: Modality;
   acceptedInputs: string[];
+  /** Modality of this tool's output, when it differs from `modality`. Defaults to `modality`. */
+  outputModality?: Modality;
   executionHint: "fast" | "long";
   maxInputSizeMB?: number;
   shortcut?: string;