feat(automate): make the pipeline builder fully multi-modal (#335)

* feat(shared): add outputModality to Tool metadata for crossing tools

* feat(shared): add modalityForExtension, toolInputModality, toolOutputModality

* fix(pipeline): route finalize and parent jobs to the pipeline's modality pool

* feat(automate): add ConvertAudioControls pipeline step (exemplar)

* feat(automate): add video tool settings controls to pipelines

* feat(automate): add audio tool settings controls to pipelines

* feat(automate): add document tool settings controls to pipelines

* feat(automate): add chart-maker settings control to pipelines

* feat(automate): warn on modality-incompatible pipeline steps

* feat(automate): add single-file download button for pipeline results

* refactor(automate): modality-aware icons, nav handler rename, mobile size bar

* test(pipeline): cover audio, document, file, and cross-modality chains

* i18n(automate): translate the modality-warning tooltip

* test(pipeline): gate media-pool routing assertion on ffmpeg availability
This commit is contained in:
SnapOtter
2026-06-24 00:21:41 +08:00
committed by GitHub
parent 35e18d8b79
commit a53038ed96
69 changed files with 3258 additions and 55 deletions
+63 -5
View File
@@ -1,3 +1,4 @@
import { modalityForExtension } from "@snapotter/shared";
import {
CheckCircle2,
ChevronDown,
@@ -5,8 +6,12 @@ import {
ChevronRight,
ChevronUp,
Download,
FileImage,
FileArchive,
FileAudio,
FileText,
FileVideo,
FolderOpen,
Image as ImageIcon,
Layers,
Play,
Plus,
@@ -45,6 +50,21 @@ const WaveformPlayer = lazy(() =>
import("@/components/common/waveform-player").then((m) => ({ default: m.WaveformPlayer })),
);
function previewIcon(kind: string) {
switch (kind) {
case "image":
return ImageIcon;
case "video":
return FileVideo;
case "audio":
return FileAudio;
case "document":
return FileText;
default:
return FileArchive;
}
}
export function AutomatePage() {
const { t } = useTranslation();
usePageTitle(t.sidebar.automate);
@@ -104,6 +124,12 @@ export function AutomatePage() {
const hasPrev = selectedIndex > 0;
const hasNext = selectedIndex < entries.length - 1;
const firstEntry = entries[0];
const uploadedModality = firstEntry
? (modalityForExtension(firstEntry.file.name.slice(firstEntry.file.name.lastIndexOf("."))) ??
firstEntry.modality)
: null;
useEffect(() => {
(async () => {
try {
@@ -347,7 +373,15 @@ export function AutomatePage() {
URL.revokeObjectURL(url);
}, [batchZipBlob, batchZipFilename]);
const handleImageKeyDown = useCallback(
const handleDownloadSingle = useCallback(() => {
if (!processedUrl) return;
const a = document.createElement("a");
a.href = processedUrl;
a.download = currentEntry?.processedFilename ?? "result";
a.click();
}, [processedUrl, currentEntry]);
const handleNavKeyDown = useCallback(
(e: React.KeyboardEvent) => {
if (e.key === "ArrowLeft") {
e.preventDefault();
@@ -453,7 +487,10 @@ export function AutomatePage() {
<div className="flex items-center justify-center h-full">
<div className="text-center p-6 max-w-xs">
<div className="mx-auto w-14 h-14 rounded-2xl bg-muted flex items-center justify-center mb-3">
<FileImage className="h-7 w-7 text-muted-foreground" />
{(() => {
const Icon = previewIcon(kind);
return <Icon className="h-7 w-7 text-muted-foreground" />;
})()}
</div>
<p className="font-medium text-foreground mb-1">{fname}</p>
<p className="text-xs text-muted-foreground">
@@ -537,7 +574,7 @@ export function AutomatePage() {
{hasFile && hasProcessed && originalBlobUrl && (
<div className="mb-3 rounded-lg border border-border overflow-hidden">
<div className="relative h-48">{renderPipelinePreview("result")}</div>
{processedSize != null && currentEntry?.previewKind === "image" && (
{processedSize != null && (
<div className="flex items-center justify-between px-3 py-1.5 border-t border-border text-xs text-muted-foreground">
<span className="truncate">{selectedFileName ?? files[0].name}</span>
<span>
@@ -575,6 +612,7 @@ export function AutomatePage() {
<PipelineBuilder
steps={steps}
expandedStepId={expandedStepId}
uploadedModality={uploadedModality}
onRemoveStep={removeStep}
onReorderSteps={reorderSteps}
onUpdateSettings={updateStepSettings}
@@ -626,6 +664,15 @@ export function AutomatePage() {
<Download className="h-4 w-4" />
</button>
)}
{hasProcessed && !batchZipBlob && processedUrl && (
<button
type="button"
onClick={handleDownloadSingle}
className="px-4 py-2.5 rounded-lg border border-primary text-primary"
>
<Download className="h-4 w-4" />
</button>
)}
</div>
{/* Mobile FAB for tool palette */}
@@ -877,6 +924,7 @@ export function AutomatePage() {
<PipelineBuilder
steps={steps}
expandedStepId={expandedStepId}
uploadedModality={uploadedModality}
onRemoveStep={removeStep}
onReorderSteps={reorderSteps}
onUpdateSettings={updateStepSettings}
@@ -925,6 +973,16 @@ export function AutomatePage() {
{t.automate.downloadZip}
</button>
)}
{hasProcessed && !batchZipBlob && processedUrl && (
<button
type="button"
onClick={handleDownloadSingle}
className="px-4 py-2.5 rounded-lg border border-primary text-primary font-medium flex items-center gap-2 hover:bg-primary/5"
>
<Download className="h-4 w-4" />
{t.common.download}
</button>
)}
<span className="flex-1" />
@@ -1011,7 +1069,7 @@ export function AutomatePage() {
<section
aria-label={t.a11y.imageArea}
className="flex-1 flex flex-col overflow-hidden min-h-0"
onKeyDown={hasMultiple ? handleImageKeyDown : undefined}
onKeyDown={hasMultiple ? handleNavKeyDown : undefined}
tabIndex={hasMultiple ? 0 : undefined}
>
<div className="flex-1 relative flex items-center justify-center px-6 py-2 min-h-0">