feat: improve remove background with edge smoothing, color decontamination, output formats

- Expose birefnet-hr-matting in UI (People/Ultra) and fix model defaults (People/Max now uses birefnet-matting for true alpha matting) - Add output format selector (PNG/WebP/AVIF) with lossless alpha support - Add edge smoothing post-processing (Off/Light/Medium/Strong) via morphological mask refinement to reduce gray halo artifacts - Add color decontamination to remove background color spill from semi-transparent edge pixels - Thread new settings through full stack: frontend -> API schema -> Python sidecar -> Sharp effects pipeline - Add i18n keys for all 21 locales - Add unit tests for new option serialization (3 tests) - Add integration tests for new settings validation (4 tests)
2026-08-03 07:46:42 +02:00 · 2026-06-05 23:05:25 +08:00
parent f7282afe92
commit 80957f6e10
28 changed files with 485 additions and 15 deletions
@@ -1,5 +1,26 @@
 import sharp from "sharp";

+export type BgOutputFormat = "png" | "webp" | "avif";
+
+export const BG_OUTPUT_FORMATS: BgOutputFormat[] = ["png", "webp", "avif"];
+
+export const BG_FORMAT_CONTENT_TYPES: Record<BgOutputFormat, string> = {
+  png: "image/png",
+  webp: "image/webp",
+  avif: "image/avif",
+};
+
+function toOutputFormat(pipeline: sharp.Sharp, format: BgOutputFormat): Promise<Buffer> {
+  switch (format) {
+    case "webp":
+      return pipeline.webp({ lossless: true }).toBuffer();
+    case "avif":
+      return pipeline.avif({ lossless: true }).toBuffer();
+    default:
+      return pipeline.png().toBuffer();
+  }
+}
+
 /**
 * Background removal post-processing effects.
 * All effects use Sharp (libvips) for fast server-side image manipulation.
@@ -182,6 +203,7 @@ export async function applyEffects(
    blurIntensity?: number;
    shadowEnabled?: boolean;
    shadowOpacity?: number;
+    outputFormat?: BgOutputFormat;
  },
 ): Promise<Buffer> {
  const meta = await sharp(subjectBuffer).metadata();
@@ -238,13 +260,12 @@ export async function applyEffects(
  }
  // else: transparent - no background layer

+  const fmt = settings.outputFormat ?? "png";
+
  // Step 3: Composite subject onto background
  if (background) {
-    return sharp(background)
-      .composite([{ input: subject, blend: "over" }])
-      .png()
-      .toBuffer();
+    return toOutputFormat(sharp(background).composite([{ input: subject, blend: "over" }]), fmt);
  }

-  return subject;
+  return toOutputFormat(sharp(subject), fmt);
 }
@@ -6,7 +6,11 @@ import { getBundleForTool, TOOL_BUNDLE_MAP } from "@snapotter/shared";
 import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
 import { z } from "zod";
 import { autoOrient } from "../../lib/auto-orient.js";
-import { applyEffects } from "../../lib/bg-effects.js";
+import {
+  applyEffects,
+  BG_FORMAT_CONTENT_TYPES,
+  type BgOutputFormat,
+} from "../../lib/bg-effects.js";
 import { formatZodErrors } from "../../lib/errors.js";
 import { isToolInstalled } from "../../lib/feature-status.js";
 import { validateImageBuffer } from "../../lib/file-validation.js";
@@ -28,6 +32,9 @@ const settingsSchema = z.object({
  blurIntensity: z.number().min(0).max(100).optional(),
  shadowEnabled: z.boolean().optional(),
  shadowOpacity: z.number().min(0).max(100).optional(),
+  outputFormat: z.enum(["png", "webp", "avif"]).optional(),
+  edgeRefine: z.number().int().min(0).max(3).optional(),
+  decontaminate: z.boolean().optional(),
 });

 /**
@@ -176,7 +183,11 @@ export function registerRemoveBackground(app: FastifyInstance) {
        const transparentResult = await removeBackground(
          fileBuffer,
          join(workspacePath, "output"),
-          { model: settings.model },
+          {
+            model: settings.model,
+            edgeRefine: settings.edgeRefine,
+            decontaminate: settings.decontaminate,
+          },
          onProgress,
        );

@@ -265,6 +276,7 @@ export function registerRemoveBackground(app: FastifyInstance) {
        blurIntensity: z.number().min(0).max(100).optional(),
        shadowEnabled: z.boolean().optional(),
        shadowOpacity: z.number().min(0).max(100).optional(),
+        outputFormat: z.enum(["png", "webp", "avif"]).optional(),
      });

      try {
@@ -308,6 +320,7 @@ export function registerRemoveBackground(app: FastifyInstance) {
        }

        // Apply effects using cached mask + original
+        const fmt = (settings.outputFormat ?? "png") as BgOutputFormat;
        const resultBuffer = await applyEffects(maskBuffer, originalBuffer, {
          backgroundType: settings.backgroundType,
          backgroundColor: settings.backgroundColor,
@@ -319,10 +332,11 @@ export function registerRemoveBackground(app: FastifyInstance) {
          blurIntensity: settings.blurIntensity,
          shadowEnabled: settings.shadowEnabled,
          shadowOpacity: settings.shadowOpacity,
+          outputFormat: fmt,
        });

        // Save the final output
-        const outputFilename = `${baseName}_nobg.png`;
+        const outputFilename = `${baseName}_nobg.${fmt}`;
        const outputPath = join(workspacePath, "output", outputFilename);
        await writeFile(outputPath, resultBuffer);

@@ -354,9 +368,10 @@ export function registerRemoveBackground(app: FastifyInstance) {
      const transparentResult = await removeBackground(
        orientedBuffer,
        join(workspacePath, "output"),
-        { model: s.model },
+        { model: s.model, edgeRefine: s.edgeRefine, decontaminate: s.decontaminate },
      );

+      const fmt = (s.outputFormat ?? "png") as BgOutputFormat;
      const resultBuffer = await applyEffects(transparentResult, orientedBuffer, {
        backgroundType: s.backgroundType,
        backgroundColor: s.backgroundColor,
@@ -367,10 +382,15 @@ export function registerRemoveBackground(app: FastifyInstance) {
        blurIntensity: s.blurIntensity,
        shadowEnabled: s.shadowEnabled,
        shadowOpacity: s.shadowOpacity,
+        outputFormat: fmt,
      });

-      const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_nobg.png`;
-      return { buffer: resultBuffer, filename: outputFilename, contentType: "image/png" };
+      const outputFilename = `${filename.replace(/\.[^.]+$/, "")}_nobg.${fmt}`;
+      return {
+        buffer: resultBuffer,
+        filename: outputFilename,
+        contentType: BG_FORMAT_CONTENT_TYPES[fmt],
+      };
    },
  });
 }
@@ -22,6 +22,7 @@ type BackgroundType = "transparent" | "color" | "gradient" | "image";
 type BgModel =
  | "birefnet-general"
  | "birefnet-general-lite"
+  | "birefnet-hr-matting"
  | "birefnet-matting"
  | "birefnet-portrait"
  | "bria-rmbg"
@@ -31,8 +32,8 @@ const MODEL_MAP: Record<SubjectType, Partial<Record<Quality, BgModel>>> = {
  people: {
    fast: "u2net",
    balanced: "birefnet-portrait",
-    best: "birefnet-portrait",
-    ultra: "birefnet-matting",
+    best: "birefnet-matting",
+    ultra: "birefnet-hr-matting",
  },
  products: { fast: "u2net", balanced: "bria-rmbg", best: "birefnet-general" },
  general: { fast: "u2net", balanced: "birefnet-general-lite", best: "birefnet-general" },
@@ -105,6 +106,13 @@ export function RemoveBgControls({ settings: _settings, onChange }: RemoveBgCont
  const [shadowEnabled, setShadowEnabled] = useState(false);
  const [shadowOpacity, setShadowOpacity] = useState(35);

+  // Post-processing
+  const [edgeRefine, setEdgeRefine] = useState(0);
+  const [decontaminate, setDecontaminate] = useState(false);
+
+  // Output
+  const [outputFormat, setOutputFormat] = useState<"png" | "webp" | "avif">("png");
+
  // Expandable sections
  const [effectsOpen, setEffectsOpen] = useState(false);

@@ -152,6 +160,10 @@ export function RemoveBgControls({ settings: _settings, onChange }: RemoveBgCont
      next._bgImageFile = bgImageFile;
    }

+    if (edgeRefine > 0) next.edgeRefine = edgeRefine;
+    if (decontaminate) next.decontaminate = true;
+    if (outputFormat !== "png") next.outputFormat = outputFormat;
+
    onChangeRef.current(next);
  }, [
    model,
@@ -165,6 +177,9 @@ export function RemoveBgControls({ settings: _settings, onChange }: RemoveBgCont
    blurIntensity,
    shadowEnabled,
    shadowOpacity,
+    edgeRefine,
+    decontaminate,
+    outputFormat,
  ]);

  return (
@@ -387,6 +402,25 @@ export function RemoveBgControls({ settings: _settings, onChange }: RemoveBgCont
        )}
      </div>

+      {/* Output format */}
+      <SectionLabel>{t.toolSettings["remove-background"].outputFormat}</SectionLabel>
+      <div className="grid grid-cols-3 gap-1.5">
+        {(["png", "webp", "avif"] as const).map((fmt) => (
+          <button
+            key={fmt}
+            type="button"
+            onClick={() => setOutputFormat(fmt)}
+            className={`py-2 px-2 rounded-lg border text-xs font-medium uppercase transition-colors ${
+              outputFormat === fmt
+                ? "border-primary bg-primary/10 text-primary"
+                : "border-border text-muted-foreground hover:border-primary/50"
+            }`}
+          >
+            {fmt}
+          </button>
+        ))}
+      </div>
+
      {/* Effects */}
      <button
        type="button"
@@ -395,7 +429,7 @@ export function RemoveBgControls({ settings: _settings, onChange }: RemoveBgCont
      >
        {effectsOpen ? <ChevronDown className="h-3 w-3" /> : <ChevronRight className="h-3 w-3" />}
        Effects
-        {(blurEnabled || shadowEnabled) && (
+        {(blurEnabled || shadowEnabled || edgeRefine > 0 || decontaminate) && (
          <span className="ms-auto text-primary text-[10px] normal-case font-normal">active</span>
        )}
      </button>
@@ -467,6 +501,48 @@ export function RemoveBgControls({ settings: _settings, onChange }: RemoveBgCont
              </div>
            )}
          </div>
+
+          {/* Edge smoothing */}
+          <div>
+            <div className="flex justify-between items-center">
+              <span className="text-xs text-muted-foreground">
+                {t.toolSettings["remove-background"].edgeSmoothing}
+              </span>
+              <span className="text-xs font-mono text-foreground tabular-nums">
+                {edgeRefine === 0
+                  ? t.toolSettings["remove-background"].edgeSmoothingOff
+                  : edgeRefine === 1
+                    ? t.toolSettings["remove-background"].edgeSmoothingLight
+                    : edgeRefine === 2
+                      ? t.toolSettings["remove-background"].edgeSmoothingMedium
+                      : t.toolSettings["remove-background"].edgeSmoothingStrong}
+              </span>
+            </div>
+            <input
+              type="range"
+              min={0}
+              max={3}
+              step={1}
+              value={edgeRefine}
+              onChange={(e) => setEdgeRefine(Number(e.target.value))}
+              className="w-full mt-0.5"
+            />
+          </div>
+
+          {/* Color decontamination */}
+          <div>
+            <label className="flex items-center gap-2 cursor-pointer">
+              <input
+                type="checkbox"
+                checked={decontaminate}
+                onChange={(e) => setDecontaminate(e.target.checked)}
+                className="rounded border-border accent-primary"
+              />
+              <span className="text-xs text-muted-foreground">
+                {t.toolSettings["remove-background"].colorDecontamination}
+              </span>
+            </label>
+          </div>
        </div>
      )}
    </div>
@@ -742,6 +818,7 @@ export function RemoveBgSettings({ onBgPreview }: RemoveBgSettingsProps = {}) {
        blurIntensity: settings.blurIntensity,
        shadowEnabled: settings.shadowEnabled,
        shadowOpacity: settings.shadowOpacity,
+        outputFormat: settings.outputFormat,
      };
      formData.append("settings", JSON.stringify(effectSettings));