Merge pull request #1986 from heygen-com/feat/studio-media-processing-routes

feat(studio-server): add media processing routes

packages/cli/src/server/studioServer.ts

@@ -18,14 +18,17 @@ import {
 import { VERSION as version } from "../version.js";
 import { buildStudioHeadScripts, resolveCliTelemetryDistinctId } from "./telemetryIdentity.js";
 import { emitStudioRenderComplete, emitStudioRenderError } from "./studioRenderTelemetry.js";
+import { isDevMode } from "../utils/env.js";
 import {
   createStudioManualEditsRenderBodyScript,
   createStudioApi,
   createProjectSignature,
+  createBackgroundRemovalJob,
   getMimeType,
   type StudioApiAdapter,
   type ResolvedProject,
   type RenderJobState,
+  type BackgroundRemovalRender,
 } from "@hyperframes/studio-server";
 import { getElementScreenshotClip } from "@hyperframes/studio-server/screenshot-clip";
 import type { ScreenshotClip } from "@hyperframes/studio-server/screenshot-clip";
@@ -35,6 +38,12 @@ const STUDIO_MANUAL_EDITS_PATH = ".hyperframes/studio-manual-edits.json";
 const REMOTE_GIF_IMG_SRC_RE =
   /<img\b[^>]*?\bsrc\s*=\s*["'](https?:\/\/[^"']+\.gif(?:[?#][^"']*)?)["'][^>]*>/gi;
 
+async function loadStudioProducer() {
+  return isDevMode()
+    ? await import("../../../producer/src/index.js")
+    : await import("@hyperframes/producer");
+}
+
 // ── Path resolution ─────────────────────────────────────────────────────────
 
 function resolveDistDir(): string {
@@ -302,7 +311,10 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
         // we can point it at our hot-reloadable local runtime endpoint. Inlining
         // ~150 KB of runtime body on every preview render would defeat browser
         // caching across composition edits.
-        let html = await bundleToSingleHtml(dir, { runtime: "placeholder" });
+        let html = await bundleToSingleHtml(dir, {
+          runtime: "placeholder",
+          inlineColorGradingLuts: false,
+        });
         html = html.replace(
           'data-hyperframes-preview-runtime="1" src=""',
           'data-hyperframes-preview-runtime="1" src="/api/runtime.js"',
@@ -361,7 +373,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
       (async () => {
         let renderJob: RenderJob | undefined;
         try {
-          const { createRenderJob, executeRenderJob } = await import("@hyperframes/producer");
+          const { createRenderJob, executeRenderJob } = await loadStudioProducer();
           const { ensureBrowser } = await import("../browser/manager.js");
 
           try {
@@ -416,6 +428,16 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
       return state;
     },
 
+    startBackgroundRemoval(opts) {
+      return createBackgroundRemovalJob(opts, async (renderOpts) => {
+        const sourcePipelinePath = "../background-removal/pipeline.ts";
+        const pipeline = (await import("../background-removal/pipeline.js").catch(
+          () => import(sourcePipelinePath),
+        )) as { render: BackgroundRemovalRender };
+        return pipeline.render(renderOpts);
+      });
+    },
+
     async generateThumbnail(opts): Promise<Buffer | null> {
       const browser = await getThumbnailBrowser();
       if (!browser) {
@@ -438,6 +460,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
             { timeout: 5000 },
           )
           .catch(() => {});
+        // fallow-ignore-next-line code-duplication
         await page.evaluate((t: number) => {
           const w = window as Window & {
             __player?: { seek?: (time: number) => void };

packages/studio-server/src/createStudioApi.ts

@@ -11,6 +11,7 @@ import { registerWaveformRoutes } from "./routes/waveform.js";
 import { registerFontRoutes } from "./routes/fonts.js";
 import { registerRegistryRoutes } from "./routes/registry.js";
 import { registerSelectionRoutes } from "./routes/selection.js";
+import { registerMediaRoutes } from "./routes/media.js";
 
 /**
  * Create a Hono sub-app with all studio API routes.
@@ -29,6 +30,7 @@ export function createStudioApi(adapter: StudioApiAdapter): Hono {
   registerRenderRoutes(api, adapter);
   registerThumbnailRoutes(api, adapter);
   registerSelectionRoutes(api, adapter);
+  registerMediaRoutes(api, adapter);
   registerWaveformRoutes(api, adapter);
   registerFontRoutes(api);
   registerRegistryRoutes(api, adapter);

packages/studio-server/src/helpers/backgroundRemovalJob.ts

@@ -0,0 +1,90 @@
+import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js";
+
+export type BackgroundRemovalJobOptions = Parameters<
+  NonNullable<StudioApiAdapter["startBackgroundRemoval"]>
+>[0];
+
+export type BackgroundRemovalProgressEvent =
+  | { kind: "info"; message: string }
+  | { kind: "metadata"; width: number; height: number; fps: number; frameCount: number }
+  | { kind: "frame"; index: number; total: number; avgMsPerFrame: number };
+
+export type BackgroundRemovalRender = (options: {
+  inputPath: string;
+  outputPath: string;
+  backgroundOutputPath?: string;
+  device?: BackgroundRemovalJobOptions["device"];
+  quality?: BackgroundRemovalJobOptions["quality"];
+  onProgress?: (event: BackgroundRemovalProgressEvent) => void;
+}) => Promise<{
+  provider: string;
+  framesProcessed: number;
+  durationSeconds: number;
+  avgMsPerFrame: number;
+}>;
+
+export function createBackgroundRemovalJob(
+  opts: BackgroundRemovalJobOptions,
+  render: BackgroundRemovalRender,
+): MediaProcessingJobState {
+  const state: MediaProcessingJobState = {
+    id: opts.jobId,
+    status: "processing",
+    progress: 0,
+    stage: "Preparing background removal",
+    inputAssetPath: opts.inputAssetPath,
+    outputAssetPath: opts.outputAssetPath,
+    outputPath: opts.outputPath,
+    ...(opts.backgroundOutputPath ? { backgroundOutputPath: opts.backgroundOutputPath } : {}),
+    ...(opts.backgroundOutputAssetPath
+      ? { backgroundOutputAssetPath: opts.backgroundOutputAssetPath }
+      : {}),
+  };
+
+  void (async () => {
+    try {
+      const result = await render({
+        inputPath: opts.inputPath,
+        outputPath: opts.outputPath,
+        backgroundOutputPath: opts.backgroundOutputPath,
+        device: opts.device,
+        quality: opts.quality,
+        onProgress: (event) => updateBackgroundRemovalProgress(state, event),
+      });
+      state.status = "complete";
+      state.progress = 100;
+      state.stage = "Complete";
+      state.provider = result.provider;
+      state.framesProcessed = result.framesProcessed;
+      state.durationSeconds = result.durationSeconds;
+      state.avgMsPerFrame = result.avgMsPerFrame;
+    } catch (err) {
+      state.status = "failed";
+      state.error = err instanceof Error ? err.message : String(err);
+      state.stage = "Failed";
+    }
+  })();
+
+  return state;
+}
+
+function updateBackgroundRemovalProgress(
+  state: MediaProcessingJobState,
+  event: BackgroundRemovalProgressEvent,
+): void {
+  if (event.kind === "info") {
+    state.stage = event.message;
+    return;
+  }
+  if (event.kind === "metadata") {
+    state.stage = `Source ${event.width}×${event.height}`;
+    state.progress = 2;
+    return;
+  }
+  state.progress = event.total ? Math.min(99, Math.floor((event.index / event.total) * 100)) : 0;
+  state.stage = event.total
+    ? `Removing background ${event.index}/${event.total}`
+    : `Removing background frame ${event.index}`;
+  state.framesProcessed = event.index;
+  state.avgMsPerFrame = event.avgMsPerFrame;
+}

packages/studio-server/src/helpers/mediaMetadata.test.ts

@@ -0,0 +1,97 @@
+import { describe, expect, it } from "vitest";
+import { classifyMediaColor, probeMediaMetadata } from "./mediaMetadata.js";
+
+describe("classifyMediaColor", () => {
+  it("detects HDR PQ from BT.2020 + smpte2084 metadata", () => {
+    expect(
+      classifyMediaColor({
+        codec_type: "video",
+        codec_name: "hevc",
+        profile: "Main 10",
+        pix_fmt: "yuv420p10le",
+        color_space: "bt2020nc",
+        color_transfer: "smpte2084",
+        color_primaries: "bt2020",
+      }),
+    ).toMatchObject({
+      dynamicRange: "hdr",
+      hdrTransfer: "pq",
+      label: "HDR PQ",
+      isHdr: true,
+    });
+  });
+
+  it("detects HDR HLG from arib-std-b67 metadata", () => {
+    expect(
+      classifyMediaColor({
+        codec_type: "video",
+        color_space: "bt2020nc",
+        color_transfer: "arib-std-b67",
+        color_primaries: "bt2020",
+      }),
+    ).toMatchObject({
+      dynamicRange: "hdr",
+      hdrTransfer: "hlg",
+      label: "HDR HLG",
+      isHdr: true,
+    });
+  });
+
+  it("labels BT.709 media as SDR Rec.709", () => {
+    expect(
+      classifyMediaColor({
+        codec_type: "video",
+        color_space: "bt709",
+        color_transfer: "bt709",
+        color_primaries: "bt709",
+      }),
+    ).toMatchObject({
+      dynamicRange: "sdr",
+      hdrTransfer: null,
+      label: "SDR Rec.709",
+      isHdr: false,
+    });
+  });
+});
+
+describe("probeMediaMetadata", () => {
+  it("reads the first video stream from ffprobe JSON", () => {
+    const metadata = probeMediaMetadata("/tmp/clip.mp4", () => ({
+      status: 0,
+      stdout: JSON.stringify({
+        streams: [
+          { codec_type: "audio", codec_name: "aac" },
+          {
+            codec_type: "video",
+            codec_name: "hevc",
+            pix_fmt: "yuv420p10le",
+            color_space: "bt2020nc",
+            color_transfer: "smpte2084",
+            color_primaries: "bt2020",
+          },
+        ],
+      }),
+      stderr: "",
+    }));
+
+    expect(metadata).toMatchObject({
+      kind: "video",
+      color: { isHdr: true, label: "HDR PQ" },
+    });
+  });
+
+  it("returns unknown metadata when ffprobe is unavailable", () => {
+    expect(
+      probeMediaMetadata("/tmp/clip.mp4", () => ({
+        status: null,
+        stdout: "",
+        stderr: "",
+        error: { code: "ENOENT" } as NodeJS.ErrnoException,
+      })),
+    ).toMatchObject({
+      kind: "video",
+      color: { dynamicRange: "unknown", isHdr: false },
+      probeError: "ffprobe unavailable",
+    });
+  });
+});

packages/studio-server/src/helpers/mediaMetadata.ts

@@ -0,0 +1,160 @@
+import { spawnSync, type SpawnSyncOptions } from "node:child_process";
+import { extname } from "node:path";
+
+type FfprobeRunner = (
+  command: string,
+  args: string[],
+  options?: SpawnSyncOptions,
+) => {
+  status: number | null;
+  stdout: string | Buffer;
+  stderr: string | Buffer;
+  error?: NodeJS.ErrnoException;
+};
+
+export type MediaDynamicRange = "hdr" | "sdr" | "unknown";
+export type MediaHdrTransfer = "pq" | "hlg" | "unknown";
+
+export interface MediaColorMetadata {
+  dynamicRange: MediaDynamicRange;
+  hdrTransfer: MediaHdrTransfer | null;
+  label: string;
+  isHdr: boolean;
+  codecName?: string;
+  profile?: string;
+  pixelFormat?: string;
+  colorSpace?: string;
+  colorTransfer?: string;
+  colorPrimaries?: string;
+  bitsPerRawSample?: string;
+}
+
+export interface MediaMetadata {
+  kind: "video" | "image" | "audio" | "unknown";
+  color: MediaColorMetadata;
+  probeError?: string;
+}
+
+interface FfprobeStream {
+  codec_type?: string;
+  codec_name?: string;
+  profile?: string;
+  pix_fmt?: string;
+  color_space?: string;
+  color_transfer?: string;
+  color_primaries?: string;
+  bits_per_raw_sample?: string;
+}
+
+const VIDEO_EXT = new Set([".mp4", ".mov", ".webm", ".mkv", ".avi", ".m4v"]);
+const IMAGE_EXT = new Set([".jpg", ".jpeg", ".png", ".webp", ".avif"]);
+const AUDIO_EXT = new Set([".mp3", ".wav", ".ogg", ".m4a", ".aac"]);
+
+function lower(value: string | undefined): string {
+  return value?.toLowerCase() ?? "";
+}
+
+function inferKindFromPath(path: string): MediaMetadata["kind"] {
+  const ext = extname(path).toLowerCase();
+  if (VIDEO_EXT.has(ext)) return "video";
+  if (IMAGE_EXT.has(ext)) return "image";
+  if (AUDIO_EXT.has(ext)) return "audio";
+  return "unknown";
+}
+
+function colorLabel(input: {
+  isHdr: boolean;
+  hdrTransfer: MediaHdrTransfer | null;
+  colorPrimaries: string;
+  colorSpace: string;
+  colorTransfer: string;
+}): string {
+  if (input.isHdr) {
+    if (input.hdrTransfer === "pq") return "HDR PQ";
+    if (input.hdrTransfer === "hlg") return "HDR HLG";
+    return "HDR";
+  }
+  if (
+    input.colorPrimaries.includes("bt709") ||
+    input.colorSpace.includes("bt709") ||
+    input.colorTransfer.includes("bt709")
+  ) {
+    return "SDR Rec.709";
+  }
+  return "SDR/unknown";
+}
+
+export function classifyMediaColor(stream: FfprobeStream | null | undefined): MediaColorMetadata {
+  const colorPrimaries = lower(stream?.color_primaries);
+  const colorSpace = lower(stream?.color_space);
+  const colorTransfer = lower(stream?.color_transfer);
+  const isHdr =
+    colorPrimaries.includes("bt2020") ||
+    colorSpace.includes("bt2020") ||
+    colorTransfer === "smpte2084" ||
+    colorTransfer === "arib-std-b67";
+  const hdrTransfer: MediaHdrTransfer | null = isHdr
+    ? colorTransfer === "smpte2084"
+      ? "pq"
+      : colorTransfer === "arib-std-b67"
+        ? "hlg"
+        : "unknown"
+    : null;
+
+  return {
+    dynamicRange: stream ? (isHdr ? "hdr" : "sdr") : "unknown",
+    hdrTransfer,
+    label: stream
+      ? colorLabel({ isHdr, hdrTransfer, colorPrimaries, colorSpace, colorTransfer })
+      : "Unknown",
+    isHdr,
+    codecName: stream?.codec_name,
+    profile: stream?.profile,
+    pixelFormat: stream?.pix_fmt,
+    colorSpace: stream?.color_space,
+    colorTransfer: stream?.color_transfer,
+    colorPrimaries: stream?.color_primaries,
+    bitsPerRawSample: stream?.bits_per_raw_sample,
+  };
+}
+
+export function probeMediaMetadata(
+  filePath: string,
+  runner: FfprobeRunner = spawnSync as unknown as FfprobeRunner,
+): MediaMetadata {
+  const kind = inferKindFromPath(filePath);
+  if (kind === "audio" || kind === "unknown") {
+    return { kind, color: classifyMediaColor(null) };
+  }
+
+  const result = runner(
+    "ffprobe",
+    [
+      "-v",
+      "error",
+      "-show_entries",
+      "stream=codec_type,codec_name,profile,pix_fmt,color_space,color_transfer,color_primaries,bits_per_raw_sample",
+      "-of",
+      "json",
+      filePath,
+    ],
+    { timeout: 15_000, maxBuffer: 1024 * 1024 },
+  );
+
+  if (result.error?.code === "ENOENT") {
+    return { kind, color: classifyMediaColor(null), probeError: "ffprobe unavailable" };
+  }
+  if (result.status !== 0) {
+    return { kind, color: classifyMediaColor(null), probeError: "ffprobe failed" };
+  }
+
+  try {
+    const parsed = JSON.parse(String(result.stdout || "{}")) as { streams?: FfprobeStream[] };
+    const stream = parsed.streams?.find((item) =>
+      kind === "image" ? item.codec_type === "video" : item.codec_type === kind,
+    );
+    return { kind, color: classifyMediaColor(stream) };
+  } catch {
+    return { kind, color: classifyMediaColor(null), probeError: "ffprobe returned invalid json" };
+  }
+}

packages/studio-server/src/index.ts

@@ -4,6 +4,7 @@ export type {
   StudioApiAdapter,
   ResolvedProject,
   RenderJobState,
+  MediaProcessingJobState,
   LintResult,
   StudioSelectionResponse,
   StudioSelectionSnapshot,
@@ -13,6 +14,10 @@ export { isSafePath, walkDir } from "./helpers/safePath.js";
 export { getMimeType, MIME_TYPES } from "./helpers/mime.js";
 export { buildSubCompositionHtml } from "./helpers/subComposition.js";
 export { getElementScreenshotClip, type ScreenshotClip } from "./helpers/screenshotClip.js";
+export {
+  createBackgroundRemovalJob,
+  type BackgroundRemovalRender,
+} from "./helpers/backgroundRemovalJob.js";
 export {
   STUDIO_MANUAL_EDITS_PATH,
   createStudioManualEditsRenderBodyScript,

packages/studio-server/src/routes/media.test.ts

@@ -0,0 +1,231 @@
+// fallow-ignore-file code-duplication
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { Hono } from "hono";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { registerMediaRoutes } from "./media";
+import type { MediaProcessingJobState, StudioApiAdapter } from "../types";
+
+const tempProjectDirs: string[] = [];
+
+afterEach(() => {
+  for (const dir of tempProjectDirs.splice(0)) {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+function createAdapter(
+  startBackgroundRemoval?: StudioApiAdapter["startBackgroundRemoval"],
+  probeMediaMetadata?: NonNullable<Parameters<typeof registerMediaRoutes>[2]>["probeMediaMetadata"],
+): {
+  app: Hono;
+  projectDir: string;
+  startBackgroundRemoval: ReturnType<typeof vi.fn>;
+} {
+  const projectDir = mkdtempSync(join(tmpdir(), "hf-media-route-test-"));
+  tempProjectDirs.push(projectDir);
+
+  mkdirSync(join(projectDir, "assets"), { recursive: true });
+  writeFileSync(join(projectDir, "assets", "clip.mp4"), "video");
+  writeFileSync(join(projectDir, "assets", "photo.jpg"), "image");
+
+  const spy = vi.fn(startBackgroundRemoval);
+  const adapter: StudioApiAdapter = {
+    listProjects: () => [],
+    resolveProject: async (id: string) => ({ id, dir: projectDir }),
+    bundle: async () => null,
+    lint: async () => ({ findings: [] }),
+    runtimeUrl: "/api/runtime.js",
+    rendersDir: () => "/tmp/renders",
+    startRender: () => ({
+      id: "job-1",
+      status: "rendering",
+      progress: 0,
+      outputPath: "/tmp/out.mp4",
+    }),
+    ...(startBackgroundRemoval ? { startBackgroundRemoval: spy } : {}),
+  };
+  const app = new Hono();
+  registerMediaRoutes(app, adapter, probeMediaMetadata ? { probeMediaMetadata } : undefined);
+  return { app, projectDir, startBackgroundRemoval: spy };
+}
+
+function completeJob(opts: Parameters<NonNullable<StudioApiAdapter["startBackgroundRemoval"]>>[0]) {
+  return {
+    id: opts.jobId,
+    status: "complete",
+    progress: 100,
+    inputAssetPath: opts.inputAssetPath,
+    outputAssetPath: opts.outputAssetPath,
+    outputPath: opts.outputPath,
+  } satisfies MediaProcessingJobState;
+}
+
+describe("registerMediaRoutes", () => {
+  it("returns metadata for a project-local media asset", async () => {
+    const probe = vi.fn(() => ({
+      kind: "video" as const,
+      color: {
+        dynamicRange: "hdr" as const,
+        hdrTransfer: "hlg" as const,
+        label: "HDR HLG",
+        isHdr: true,
+      },
+    }));
+    const { app, projectDir } = createAdapter(undefined, probe);
+
+    const response = await app.request(
+      "http://localhost/projects/demo/media/metadata?path=assets%2Fclip.mp4",
+    );
+    const data = (await response.json()) as { metadata: { color: { label: string } } };
+
+    expect(response.status).toBe(200);
+    expect(data.metadata.color.label).toBe("HDR HLG");
+    expect(probe).toHaveBeenCalledWith(join(projectDir, "assets", "clip.mp4"));
+  });
+
+  it("rejects media metadata paths outside the project", async () => {
+    const { app } = createAdapter();
+
+    const response = await app.request(
+      "http://localhost/projects/demo/media/metadata?path=..%2Fsecret.mp4",
+    );
+
+    expect(response.status).toBe(403);
+  });
+
+  it("rejects null bytes in media metadata paths", async () => {
+    const { app } = createAdapter();
+
+    const response = await app.request(
+      "http://localhost/projects/demo/media/metadata?path=assets%00clip.mp4",
+    );
+
+    expect(response.status).toBe(403);
+  });
+
+  it("returns 501 when background removal is not available", async () => {
+    const { app } = createAdapter();
+
+    const response = await app.request("http://localhost/projects/demo/media/remove-background", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ inputPath: "assets/clip.mp4" }),
+    });
+
+    expect(response.status).toBe(501);
+  });
+
+  it("rejects remote input paths", async () => {
+    const { app, startBackgroundRemoval } = createAdapter(completeJob);
+
+    const response = await app.request("http://localhost/projects/demo/media/remove-background", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ inputPath: "https://example.com/clip.mp4" }),
+    });
+
+    expect(response.status).toBe(400);
+    expect(startBackgroundRemoval).not.toHaveBeenCalled();
+  });
+
+  it("rejects null bytes in background-removal paths", async () => {
+    const { app, startBackgroundRemoval } = createAdapter(completeJob);
+
+    const inputResponse = await app.request(
+      "http://localhost/projects/demo/media/remove-background",
+      {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ inputPath: "assets/clip\0.mp4" }),
+      },
+    );
+    const outputResponse = await app.request(
+      "http://localhost/projects/demo/media/remove-background",
+      {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ inputPath: "assets/clip.mp4", outputPath: "assets/out\0.webm" }),
+      },
+    );
+
+    expect(inputResponse.status).toBe(403);
+    expect(outputResponse.status).toBe(403);
+    expect(startBackgroundRemoval).not.toHaveBeenCalled();
+  });
+
+  it("starts a video cutout job with safe default output paths", async () => {
+    const { app, projectDir, startBackgroundRemoval } = createAdapter(completeJob);
+
+    const response = await app.request("http://localhost/projects/demo/media/remove-background", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ inputPath: "assets/clip.mp4", createBackgroundPlate: true }),
+    });
+    const data = (await response.json()) as {
+      jobId: string;
+      outputPath: string;
+      backgroundOutputPath: string;
+    };
+
+    expect(response.status).toBe(200);
+    expect(data.outputPath).toBe("assets/cutouts/clip-cutout.webm");
+    expect(data.backgroundOutputPath).toBe("assets/cutouts/clip-plate.webm");
+    expect(startBackgroundRemoval).toHaveBeenCalledWith(
+      expect.objectContaining({
+        project: { id: "demo", dir: projectDir },
+        inputAssetPath: "assets/clip.mp4",
+        outputAssetPath: "assets/cutouts/clip-cutout.webm",
+        backgroundOutputAssetPath: "assets/cutouts/clip-plate.webm",
+        quality: "balanced",
+        device: "auto",
+        jobId: data.jobId,
+      }),
+    );
+  });
+
+  it("normalizes query strings from local media paths", async () => {
+    const { app, startBackgroundRemoval } = createAdapter(completeJob);
+
+    const response = await app.request("http://localhost/projects/demo/media/remove-background", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ inputPath: "./assets/clip.mp4?v=123#frame" }),
+    });
+
+    expect(response.status).toBe(200);
+    expect(startBackgroundRemoval).toHaveBeenCalledWith(
+      expect.objectContaining({
+        inputAssetPath: "assets/clip.mp4",
+        outputAssetPath: "assets/cutouts/clip-cutout.webm",
+      }),
+    );
+  });
+
+  it("requires png output for image cutouts", async () => {
+    const { app, startBackgroundRemoval } = createAdapter(completeJob);
+
+    const response = await app.request("http://localhost/projects/demo/media/remove-background", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ inputPath: "assets/photo.jpg", outputPath: "assets/photo.webm" }),
+    });
+
+    expect(response.status).toBe(400);
+    expect(startBackgroundRemoval).not.toHaveBeenCalled();
+  });
+
+  it("keeps output paths inside the project", async () => {
+    const { app, startBackgroundRemoval } = createAdapter(completeJob);
+
+    const response = await app.request("http://localhost/projects/demo/media/remove-background", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ inputPath: "assets/clip.mp4", outputPath: "../escape.webm" }),
+    });
+
+    expect(response.status).toBe(403);
+    expect(startBackgroundRemoval).not.toHaveBeenCalled();
+  });
+});

packages/studio-server/src/routes/media.ts

@@ -0,0 +1,255 @@
+import type { Hono } from "hono";
+import { streamSSE } from "hono/streaming";
+import { existsSync, mkdirSync } from "node:fs";
+import { basename, dirname, extname, join } from "node:path";
+import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js";
+import { resolveWithinProject } from "../helpers/safePath.js";
+import { probeMediaMetadata } from "../helpers/mediaMetadata.js";
+
+const VIDEO_EXTENSIONS = new Set([".mp4", ".mov", ".webm", ".mkv", ".avi"]);
+const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".webp"]);
+const VIDEO_OUTPUT_EXTENSIONS = new Set([".webm", ".mov"]);
+const QUALITIES = new Set(["fast", "balanced", "best"]);
+const DEVICES = new Set(["auto", "cpu", "coreml", "cuda"]);
+
+type BackgroundRemovalQuality = "fast" | "balanced" | "best";
+type BackgroundRemovalDevice = "auto" | "cpu" | "coreml" | "cuda";
+
+interface BackgroundRemovalBody {
+  inputPath?: string;
+  outputPath?: string;
+  createBackgroundPlate?: boolean;
+  quality?: string;
+  device?: string;
+}
+
+type JobWithCreatedAt = MediaProcessingJobState & { createdAt: number };
+type ProbeMediaMetadata = typeof probeMediaMetadata;
+
+function isVideoPath(path: string): boolean {
+  return VIDEO_EXTENSIONS.has(extname(path).toLowerCase());
+}
+
+function isImagePath(path: string): boolean {
+  return IMAGE_EXTENSIONS.has(extname(path).toLowerCase());
+}
+
+function normalizeProjectAssetPath(path: string): string {
+  return path
+    .trim()
+    .replace(/^[.]\//, "")
+    .replace(/[?#].*$/, "");
+}
+
+function containsNullByte(path: string): boolean {
+  return path.includes("\0");
+}
+
+function slugFileBase(path: string): string {
+  const name = basename(path, extname(path))
+    .replace(/[^a-zA-Z0-9._-]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  return name || "media";
+}
+
+function uniqueAssetPath(projectDir: string, assetPath: string): string {
+  const ext = extname(assetPath);
+  const withoutExt = assetPath.slice(0, -ext.length);
+  let candidate = assetPath;
+  for (let index = 2; existsSync(join(projectDir, candidate)); index++) {
+    candidate = `${withoutExt}-${index}${ext}`;
+  }
+  return candidate;
+}
+
+function defaultOutputPath(projectDir: string, inputPath: string): string {
+  const ext = isImagePath(inputPath) ? ".png" : ".webm";
+  return uniqueAssetPath(projectDir, `assets/cutouts/${slugFileBase(inputPath)}-cutout${ext}`);
+}
+
+function defaultPlatePath(projectDir: string, inputPath: string): string {
+  return uniqueAssetPath(projectDir, `assets/cutouts/${slugFileBase(inputPath)}-plate.webm`);
+}
+
+function makeJobId(projectId: string, mediaJobs: Map<string, JobWithCreatedAt>): string {
+  const stamp = new Date()
+    .toISOString()
+    .replace(/[-:.TZ]/g, "")
+    .slice(0, 14);
+  const safeProject = projectId.replace(/[^a-zA-Z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
+  const base = `${safeProject || "project"}_remove-bg_${stamp}`;
+  if (!mediaJobs.has(base)) return base;
+  for (let index = 2; ; index++) {
+    const candidate = `${base}-${index}`;
+    if (!mediaJobs.has(candidate)) return candidate;
+  }
+}
+
+function normalizeQuality(value: string | undefined): BackgroundRemovalQuality {
+  return QUALITIES.has(value ?? "") ? (value as BackgroundRemovalQuality) : "balanced";
+}
+
+function normalizeDevice(value: string | undefined): BackgroundRemovalDevice {
+  return DEVICES.has(value ?? "") ? (value as BackgroundRemovalDevice) : "auto";
+}
+
+export function registerMediaRoutes(
+  api: Hono,
+  adapter: StudioApiAdapter,
+  options: { probeMediaMetadata?: ProbeMediaMetadata } = {},
+): void {
+  const mediaJobs = new Map<string, JobWithCreatedAt>();
+  const TTL_MS = 300_000;
+  const readMediaMetadata = options.probeMediaMetadata ?? probeMediaMetadata;
+
+  function cleanupFinishedJobs(): void {
+    const now = Date.now();
+    for (const [id, job] of mediaJobs) {
+      if ((job.status === "complete" || job.status === "failed") && now - job.createdAt > TTL_MS) {
+        mediaJobs.delete(id);
+      }
+    }
+  }
+
+  api.get("/projects/:id/media/metadata", async (c) => {
+    const project = await adapter.resolveProject(c.req.param("id"));
+    if (!project) return c.json({ error: "not found" }, 404);
+
+    const assetPath = normalizeProjectAssetPath(c.req.query("path") ?? "");
+    if (!assetPath) return c.json({ error: "path required" }, 400);
+    if (containsNullByte(assetPath)) return c.json({ error: "forbidden" }, 403);
+    if (/^(?:https?:|data:|blob:)/i.test(assetPath)) {
+      return c.json({ error: "media metadata requires a project-local asset" }, 400);
+    }
+
+    const filePath = resolveWithinProject(project.dir, assetPath);
+    if (!filePath) return c.json({ error: "forbidden" }, 403);
+    if (!existsSync(filePath)) return c.json({ error: "media not found" }, 404);
+
+    return c.json({ path: assetPath, metadata: readMediaMetadata(filePath) });
+  });
+
+  api.post(
+    "/projects/:id/media/remove-background",
+    // fallow-ignore-next-line complexity
+    async (c) => {
+      cleanupFinishedJobs();
+      if (!adapter.startBackgroundRemoval) {
+        return c.json({ error: "background removal is not available in this Studio server" }, 501);
+      }
+
+      // fallow-ignore-next-line code-duplication
+      const project = await adapter.resolveProject(c.req.param("id"));
+      if (!project) return c.json({ error: "not found" }, 404);
+
+      const body = (await c.req.json().catch(() => ({}))) as BackgroundRemovalBody;
+      const inputAssetPath = body.inputPath ? normalizeProjectAssetPath(body.inputPath) : "";
+      if (!inputAssetPath) return c.json({ error: "inputPath required" }, 400);
+      if (containsNullByte(inputAssetPath)) return c.json({ error: "forbidden" }, 403);
+      if (/^(?:https?:|data:|blob:)/i.test(inputAssetPath)) {
+        return c.json({ error: "background removal requires a project-local media asset" }, 400);
+      }
+
+      const inputPath = resolveWithinProject(project.dir, inputAssetPath);
+      if (!inputPath) return c.json({ error: "forbidden" }, 403);
+      if (!existsSync(inputPath)) return c.json({ error: "input media not found" }, 404);
+
+      const inputIsVideo = isVideoPath(inputAssetPath);
+      const inputIsImage = isImagePath(inputAssetPath);
+      if (!inputIsVideo && !inputIsImage) {
+        return c.json({ error: "background removal supports video or image assets only" }, 400);
+      }
+
+      const requestedOutput = body.outputPath ? normalizeProjectAssetPath(body.outputPath) : "";
+      if (requestedOutput && containsNullByte(requestedOutput)) {
+        return c.json({ error: "forbidden" }, 403);
+      }
+      if (requestedOutput && !resolveWithinProject(project.dir, requestedOutput)) {
+        return c.json({ error: "forbidden" }, 403);
+      }
+      const outputAssetPath = requestedOutput
+        ? uniqueAssetPath(project.dir, requestedOutput)
+        : defaultOutputPath(project.dir, inputAssetPath);
+      const outputPath = resolveWithinProject(project.dir, outputAssetPath);
+      if (!outputPath) return c.json({ error: "forbidden" }, 403);
+      if (inputIsVideo && !VIDEO_OUTPUT_EXTENSIONS.has(extname(outputAssetPath).toLowerCase())) {
+        return c.json({ error: "video background removal output must be .webm or .mov" }, 400);
+      }
+      if (inputIsImage && extname(outputAssetPath).toLowerCase() !== ".png") {
+        return c.json({ error: "image background removal output must be .png" }, 400);
+      }
+
+      let backgroundOutputAssetPath: string | undefined;
+      let backgroundOutputPath: string | undefined;
+      if (body.createBackgroundPlate) {
+        if (!inputIsVideo) {
+          return c.json({ error: "background plates are only supported for video inputs" }, 400);
+        }
+        backgroundOutputAssetPath = defaultPlatePath(project.dir, inputAssetPath);
+        backgroundOutputPath =
+          resolveWithinProject(project.dir, backgroundOutputAssetPath) ?? undefined;
+        if (!backgroundOutputPath) {
+          return c.json({ error: "forbidden" }, 403);
+        }
+      }
+
+      mkdirSync(dirname(outputPath), { recursive: true });
+      if (backgroundOutputPath) mkdirSync(dirname(backgroundOutputPath), { recursive: true });
+
+      const jobId = makeJobId(project.id, mediaJobs);
+      const state = adapter.startBackgroundRemoval({
+        project,
+        inputPath,
+        inputAssetPath,
+        outputPath,
+        outputAssetPath,
+        backgroundOutputPath,
+        backgroundOutputAssetPath,
+        quality: normalizeQuality(body.quality),
+        device: normalizeDevice(body.device),
+        jobId,
+      }) as JobWithCreatedAt;
+      state.createdAt = Date.now();
+      mediaJobs.set(jobId, state);
+
+      return c.json({
+        jobId,
+        status: state.status,
+        outputPath: outputAssetPath,
+        backgroundOutputPath: backgroundOutputAssetPath,
+      });
+    },
+  );
+
+  api.get("/media-jobs/:jobId/progress", (c) => {
+    cleanupFinishedJobs();
+    const { jobId } = c.req.param();
+    const job = mediaJobs.get(jobId);
+    if (!job) return c.json({ error: "not found" }, 404);
+
+    return streamSSE(c, async (stream) => {
+      while (true) {
+        const current = mediaJobs.get(jobId);
+        if (!current) break;
+        await stream.writeSSE({
+          event: "progress",
+          data: JSON.stringify({
+            id: current.id,
+            status: current.status,
+            progress: current.progress,
+            stage: current.stage,
+            outputPath: current.outputAssetPath,
+            backgroundOutputPath: current.backgroundOutputAssetPath,
+            error: current.error,
+            provider: current.provider,
+            framesProcessed: current.framesProcessed,
+            durationSeconds: current.durationSeconds,
+            avgMsPerFrame: current.avgMsPerFrame,
+          }),
+        });
+        if (current.status === "complete" || current.status === "failed") break;
+        await stream.sleep(500);
+      }
+    });
+  });
+}

packages/studio-server/src/types.ts

@@ -19,6 +19,23 @@ export interface RenderJobState {
   error?: string;
 }
 
+export interface MediaProcessingJobState {
+  id: string;
+  status: "processing" | "complete" | "failed";
+  progress: number;
+  stage?: string;
+  inputAssetPath: string;
+  outputAssetPath: string;
+  outputPath: string;
+  backgroundOutputAssetPath?: string;
+  backgroundOutputPath?: string;
+  error?: string;
+  provider?: string;
+  framesProcessed?: number;
+  durationSeconds?: number;
+  avgMsPerFrame?: number;
+}
+
 /** Lint result from the core linter. */
 export interface LintResult {
   findings: Array<{
@@ -137,6 +154,19 @@ export interface StudioApiAdapter {
     distinctId?: string;
   }): RenderJobState;
 
+  startBackgroundRemoval?: (opts: {
+    project: ResolvedProject;
+    inputPath: string;
+    inputAssetPath: string;
+    outputPath: string;
+    outputAssetPath: string;
+    backgroundOutputPath?: string;
+    backgroundOutputAssetPath?: string;
+    quality: "fast" | "balanced" | "best";
+    device?: "auto" | "cpu" | "coreml" | "cuda";
+    jobId: string;
+  }) => MediaProcessingJobState;
+
   /** Optional: generate a JPEG thumbnail via Puppeteer or similar. */
   generateThumbnail?: (opts: {
     project: ResolvedProject;

packages/studio/vite.adapter.ts

@@ -15,6 +15,8 @@ import {
   type ResolvedProject,
   type RenderJobState,
   type StudioApiAdapter,
+  type BackgroundRemovalRender,
+  createBackgroundRemovalJob,
   createProjectSignature,
 } from "@hyperframes/studio-server";
 import type { RegistryItem } from "@hyperframes/core/registry";
@@ -32,7 +34,10 @@ export function isPathWithin(parentDir: string, childPath: string): boolean {
 
 export function createViteAdapter(dataDir: string, server: ViteDevServer): StudioApiAdapter {
   let _bundler:
-    | ((dir: string, options?: { runtime?: "inline" | "placeholder" }) => Promise<string>)
+    | ((
+        dir: string,
+        options?: { runtime?: "inline" | "placeholder"; inlineColorGradingLuts?: boolean },
+      ) => Promise<string>)
     | null = null;
   let _producerModuleLoader:
     | (() => Promise<{
@@ -162,7 +167,7 @@ export function createViteAdapter(dataDir: string, server: ViteDevServer): Studi
     async bundle(dir: string) {
       const bundler = await getBundler();
       if (!bundler) return null;
-      let html = await bundler(dir, { runtime: "placeholder" });
+      let html = await bundler(dir, { runtime: "placeholder", inlineColorGradingLuts: false });
       html = html.replace(
         'data-hyperframes-preview-runtime="1" src=""',
         `data-hyperframes-preview-runtime="1" src="${this.runtimeUrl}"`,
@@ -251,6 +256,16 @@ export function createViteAdapter(dataDir: string, server: ViteDevServer): Studi
       return state;
     },
 
+    startBackgroundRemoval(opts) {
+      return createBackgroundRemovalJob(opts, async (renderOpts) => {
+        const mod = await server.ssrLoadModule(
+          resolve(__dirname, "../cli/src/background-removal/pipeline.ts"),
+        );
+        const render = mod.render as BackgroundRemovalRender;
+        return render(renderOpts);
+      });
+    },
+
     async generateThumbnail(opts) {
       return generateThumbnail(opts);
     },