fix(studio-server): share background removal job runner

This commit is contained in:
ukimsanov
2026-07-06 13:27:45 -07:00
parent a3bf7eb995
commit 413ee07da5
6 changed files with 200 additions and 226 deletions
+10 -64
View File
@@ -23,11 +23,12 @@ import {
createStudioManualEditsRenderBodyScript,
createStudioApi,
createProjectSignature,
createBackgroundRemovalJob,
getMimeType,
type StudioApiAdapter,
type ResolvedProject,
type RenderJobState,
type MediaProcessingJobState,
type BackgroundRemovalRender,
} from "@hyperframes/studio-server";
import { getElementScreenshotClip } from "@hyperframes/studio-server/screenshot-clip";
import type { ScreenshotClip } from "@hyperframes/studio-server/screenshot-clip";
@@ -428,69 +429,13 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
},
startBackgroundRemoval(opts) {
const state: MediaProcessingJobState = {
id: opts.jobId,
status: "processing",
progress: 0,
stage: "Preparing background removal",
inputAssetPath: opts.inputAssetPath,
outputAssetPath: opts.outputAssetPath,
outputPath: opts.outputPath,
...(opts.backgroundOutputPath ? { backgroundOutputPath: opts.backgroundOutputPath } : {}),
...(opts.backgroundOutputAssetPath
? { backgroundOutputAssetPath: opts.backgroundOutputAssetPath }
: {}),
};
(async () => {
try {
const sourcePipelinePath = "../background-removal/pipeline.ts";
const pipeline = (await import("../background-removal/pipeline.js").catch(
() => import(sourcePipelinePath),
)) as typeof import("../background-removal/pipeline.js");
const { render } = pipeline;
const result = await render({
inputPath: opts.inputPath,
outputPath: opts.outputPath,
backgroundOutputPath: opts.backgroundOutputPath,
device: opts.device,
quality: opts.quality,
onProgress: (event) => {
if (event.kind === "info") {
state.stage = event.message;
return;
}
if (event.kind === "metadata") {
state.stage = `Source ${event.width}×${event.height}`;
state.progress = 2;
return;
}
const pct = event.total
? Math.min(99, Math.floor((event.index / event.total) * 100))
: 0;
state.progress = pct;
state.stage = event.total
? `Removing background ${event.index}/${event.total}`
: `Removing background frame ${event.index}`;
state.framesProcessed = event.index;
state.avgMsPerFrame = event.avgMsPerFrame;
},
});
state.status = "complete";
state.progress = 100;
state.stage = "Complete";
state.provider = result.provider;
state.framesProcessed = result.framesProcessed;
state.durationSeconds = result.durationSeconds;
state.avgMsPerFrame = result.avgMsPerFrame;
} catch (err) {
state.status = "failed";
state.error = err instanceof Error ? err.message : String(err);
state.stage = "Failed";
}
})();
return state;
return createBackgroundRemovalJob(opts, async (renderOpts) => {
const sourcePipelinePath = "../background-removal/pipeline.ts";
const pipeline = (await import("../background-removal/pipeline.js").catch(
() => import(sourcePipelinePath),
)) as { render: BackgroundRemovalRender };
return pipeline.render(renderOpts);
});
},
async generateThumbnail(opts): Promise<Buffer | null> {
@@ -515,6 +460,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
{ timeout: 5000 },
)
.catch(() => {});
// fallow-ignore-next-line code-duplication
await page.evaluate((t: number) => {
const w = window as Window & {
__player?: { seek?: (time: number) => void };
@@ -0,0 +1,90 @@
import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js";
export type BackgroundRemovalJobOptions = Parameters<
NonNullable<StudioApiAdapter["startBackgroundRemoval"]>
>[0];
export type BackgroundRemovalProgressEvent =
| { kind: "info"; message: string }
| { kind: "metadata"; width: number; height: number; fps: number; frameCount: number }
| { kind: "frame"; index: number; total: number; avgMsPerFrame: number };
export type BackgroundRemovalRender = (options: {
inputPath: string;
outputPath: string;
backgroundOutputPath?: string;
device?: BackgroundRemovalJobOptions["device"];
quality?: BackgroundRemovalJobOptions["quality"];
onProgress?: (event: BackgroundRemovalProgressEvent) => void;
}) => Promise<{
provider: string;
framesProcessed: number;
durationSeconds: number;
avgMsPerFrame: number;
}>;
export function createBackgroundRemovalJob(
opts: BackgroundRemovalJobOptions,
render: BackgroundRemovalRender,
): MediaProcessingJobState {
const state: MediaProcessingJobState = {
id: opts.jobId,
status: "processing",
progress: 0,
stage: "Preparing background removal",
inputAssetPath: opts.inputAssetPath,
outputAssetPath: opts.outputAssetPath,
outputPath: opts.outputPath,
...(opts.backgroundOutputPath ? { backgroundOutputPath: opts.backgroundOutputPath } : {}),
...(opts.backgroundOutputAssetPath
? { backgroundOutputAssetPath: opts.backgroundOutputAssetPath }
: {}),
};
void (async () => {
try {
const result = await render({
inputPath: opts.inputPath,
outputPath: opts.outputPath,
backgroundOutputPath: opts.backgroundOutputPath,
device: opts.device,
quality: opts.quality,
onProgress: (event) => updateBackgroundRemovalProgress(state, event),
});
state.status = "complete";
state.progress = 100;
state.stage = "Complete";
state.provider = result.provider;
state.framesProcessed = result.framesProcessed;
state.durationSeconds = result.durationSeconds;
state.avgMsPerFrame = result.avgMsPerFrame;
} catch (err) {
state.status = "failed";
state.error = err instanceof Error ? err.message : String(err);
state.stage = "Failed";
}
})();
return state;
}
function updateBackgroundRemovalProgress(
state: MediaProcessingJobState,
event: BackgroundRemovalProgressEvent,
): void {
if (event.kind === "info") {
state.stage = event.message;
return;
}
if (event.kind === "metadata") {
state.stage = `Source ${event.width}×${event.height}`;
state.progress = 2;
return;
}
state.progress = event.total ? Math.min(99, Math.floor((event.index / event.total) * 100)) : 0;
state.stage = event.total
? `Removing background ${event.index}/${event.total}`
: `Removing background frame ${event.index}`;
state.framesProcessed = event.index;
state.avgMsPerFrame = event.avgMsPerFrame;
}
+4
View File
@@ -14,6 +14,10 @@ export { isSafePath, walkDir } from "./helpers/safePath.js";
export { getMimeType, MIME_TYPES } from "./helpers/mime.js";
export { buildSubCompositionHtml } from "./helpers/subComposition.js";
export { getElementScreenshotClip, type ScreenshotClip } from "./helpers/screenshotClip.js";
export {
createBackgroundRemovalJob,
type BackgroundRemovalRender,
} from "./helpers/backgroundRemovalJob.js";
export {
STUDIO_MANUAL_EDITS_PATH,
createStudioManualEditsRenderBodyScript,
@@ -1,3 +1,4 @@
// fallow-ignore-file code-duplication
import { afterEach, describe, expect, it, vi } from "vitest";
import { Hono } from "hono";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+85 -80
View File
@@ -129,92 +129,97 @@ export function registerMediaRoutes(
return c.json({ path: assetPath, metadata: readMediaMetadata(filePath) });
});
api.post("/projects/:id/media/remove-background", async (c) => {
cleanupFinishedJobs();
if (!adapter.startBackgroundRemoval) {
return c.json({ error: "background removal is not available in this Studio server" }, 501);
}
const project = await adapter.resolveProject(c.req.param("id"));
if (!project) return c.json({ error: "not found" }, 404);
const body = (await c.req.json().catch(() => ({}))) as BackgroundRemovalBody;
const inputAssetPath = body.inputPath ? normalizeProjectAssetPath(body.inputPath) : "";
if (!inputAssetPath) return c.json({ error: "inputPath required" }, 400);
if (containsNullByte(inputAssetPath)) return c.json({ error: "forbidden" }, 403);
if (/^(?:https?:|data:|blob:)/i.test(inputAssetPath)) {
return c.json({ error: "background removal requires a project-local media asset" }, 400);
}
const inputPath = resolveWithinProject(project.dir, inputAssetPath);
if (!inputPath) return c.json({ error: "forbidden" }, 403);
if (!existsSync(inputPath)) return c.json({ error: "input media not found" }, 404);
const inputIsVideo = isVideoPath(inputAssetPath);
const inputIsImage = isImagePath(inputAssetPath);
if (!inputIsVideo && !inputIsImage) {
return c.json({ error: "background removal supports video or image assets only" }, 400);
}
const requestedOutput = body.outputPath ? normalizeProjectAssetPath(body.outputPath) : "";
if (requestedOutput && containsNullByte(requestedOutput)) {
return c.json({ error: "forbidden" }, 403);
}
if (requestedOutput && !resolveWithinProject(project.dir, requestedOutput)) {
return c.json({ error: "forbidden" }, 403);
}
const outputAssetPath = requestedOutput
? uniqueAssetPath(project.dir, requestedOutput)
: defaultOutputPath(project.dir, inputAssetPath);
const outputPath = resolveWithinProject(project.dir, outputAssetPath);
if (!outputPath) return c.json({ error: "forbidden" }, 403);
if (inputIsVideo && !VIDEO_OUTPUT_EXTENSIONS.has(extname(outputAssetPath).toLowerCase())) {
return c.json({ error: "video background removal output must be .webm or .mov" }, 400);
}
if (inputIsImage && extname(outputAssetPath).toLowerCase() !== ".png") {
return c.json({ error: "image background removal output must be .png" }, 400);
}
let backgroundOutputAssetPath: string | undefined;
let backgroundOutputPath: string | undefined;
if (body.createBackgroundPlate) {
if (!inputIsVideo) {
return c.json({ error: "background plates are only supported for video inputs" }, 400);
api.post(
"/projects/:id/media/remove-background",
// fallow-ignore-next-line complexity
async (c) => {
cleanupFinishedJobs();
if (!adapter.startBackgroundRemoval) {
return c.json({ error: "background removal is not available in this Studio server" }, 501);
}
backgroundOutputAssetPath = defaultPlatePath(project.dir, inputAssetPath);
backgroundOutputPath =
resolveWithinProject(project.dir, backgroundOutputAssetPath) ?? undefined;
if (!backgroundOutputPath) {
// fallow-ignore-next-line code-duplication
const project = await adapter.resolveProject(c.req.param("id"));
if (!project) return c.json({ error: "not found" }, 404);
const body = (await c.req.json().catch(() => ({}))) as BackgroundRemovalBody;
const inputAssetPath = body.inputPath ? normalizeProjectAssetPath(body.inputPath) : "";
if (!inputAssetPath) return c.json({ error: "inputPath required" }, 400);
if (containsNullByte(inputAssetPath)) return c.json({ error: "forbidden" }, 403);
if (/^(?:https?:|data:|blob:)/i.test(inputAssetPath)) {
return c.json({ error: "background removal requires a project-local media asset" }, 400);
}
const inputPath = resolveWithinProject(project.dir, inputAssetPath);
if (!inputPath) return c.json({ error: "forbidden" }, 403);
if (!existsSync(inputPath)) return c.json({ error: "input media not found" }, 404);
const inputIsVideo = isVideoPath(inputAssetPath);
const inputIsImage = isImagePath(inputAssetPath);
if (!inputIsVideo && !inputIsImage) {
return c.json({ error: "background removal supports video or image assets only" }, 400);
}
const requestedOutput = body.outputPath ? normalizeProjectAssetPath(body.outputPath) : "";
if (requestedOutput && containsNullByte(requestedOutput)) {
return c.json({ error: "forbidden" }, 403);
}
}
if (requestedOutput && !resolveWithinProject(project.dir, requestedOutput)) {
return c.json({ error: "forbidden" }, 403);
}
const outputAssetPath = requestedOutput
? uniqueAssetPath(project.dir, requestedOutput)
: defaultOutputPath(project.dir, inputAssetPath);
const outputPath = resolveWithinProject(project.dir, outputAssetPath);
if (!outputPath) return c.json({ error: "forbidden" }, 403);
if (inputIsVideo && !VIDEO_OUTPUT_EXTENSIONS.has(extname(outputAssetPath).toLowerCase())) {
return c.json({ error: "video background removal output must be .webm or .mov" }, 400);
}
if (inputIsImage && extname(outputAssetPath).toLowerCase() !== ".png") {
return c.json({ error: "image background removal output must be .png" }, 400);
}
mkdirSync(dirname(outputPath), { recursive: true });
if (backgroundOutputPath) mkdirSync(dirname(backgroundOutputPath), { recursive: true });
let backgroundOutputAssetPath: string | undefined;
let backgroundOutputPath: string | undefined;
if (body.createBackgroundPlate) {
if (!inputIsVideo) {
return c.json({ error: "background plates are only supported for video inputs" }, 400);
}
backgroundOutputAssetPath = defaultPlatePath(project.dir, inputAssetPath);
backgroundOutputPath =
resolveWithinProject(project.dir, backgroundOutputAssetPath) ?? undefined;
if (!backgroundOutputPath) {
return c.json({ error: "forbidden" }, 403);
}
}
const jobId = makeJobId(project.id, mediaJobs);
const state = adapter.startBackgroundRemoval({
project,
inputPath,
inputAssetPath,
outputPath,
outputAssetPath,
backgroundOutputPath,
backgroundOutputAssetPath,
quality: normalizeQuality(body.quality),
device: normalizeDevice(body.device),
jobId,
}) as JobWithCreatedAt;
state.createdAt = Date.now();
mediaJobs.set(jobId, state);
mkdirSync(dirname(outputPath), { recursive: true });
if (backgroundOutputPath) mkdirSync(dirname(backgroundOutputPath), { recursive: true });
return c.json({
jobId,
status: state.status,
outputPath: outputAssetPath,
backgroundOutputPath: backgroundOutputAssetPath,
});
});
const jobId = makeJobId(project.id, mediaJobs);
const state = adapter.startBackgroundRemoval({
project,
inputPath,
inputAssetPath,
outputPath,
outputAssetPath,
backgroundOutputPath,
backgroundOutputAssetPath,
quality: normalizeQuality(body.quality),
device: normalizeDevice(body.device),
jobId,
}) as JobWithCreatedAt;
state.createdAt = Date.now();
mediaJobs.set(jobId, state);
return c.json({
jobId,
status: state.status,
outputPath: outputAssetPath,
backgroundOutputPath: backgroundOutputAssetPath,
});
},
);
api.get("/media-jobs/:jobId/progress", (c) => {
cleanupFinishedJobs();
+10 -82
View File
@@ -14,8 +14,9 @@ import type { ViteDevServer } from "vite";
import {
type ResolvedProject,
type RenderJobState,
type MediaProcessingJobState,
type StudioApiAdapter,
type BackgroundRemovalRender,
createBackgroundRemovalJob,
createProjectSignature,
} from "@hyperframes/studio-server";
import type { RegistryItem } from "@hyperframes/core/registry";
@@ -23,25 +24,6 @@ import { createRetryingModuleLoader, ensureProducerDist } from "./vite.producer"
import { createStudioDevRenderBodyScripts } from "./vite.studioMotion";
import { generateThumbnail, findSystemChrome } from "./vite.browser";
type BackgroundRemovalRender = (options: {
inputPath: string;
outputPath: string;
backgroundOutputPath?: string;
device?: "auto" | "cpu" | "coreml" | "cuda";
quality?: "fast" | "balanced" | "best";
onProgress?: (
event:
| { kind: "info"; message: string }
| { kind: "metadata"; width: number; height: number; fps: number; frameCount: number }
| { kind: "frame"; index: number; total: number; avgMsPerFrame: number },
) => void;
}) => Promise<{
provider: string;
framesProcessed: number;
durationSeconds: number;
avgMsPerFrame: number;
}>;
export function isPathWithin(parentDir: string, childPath: string): boolean {
const childRelativePath = relative(resolve(parentDir), resolve(childPath));
return (
@@ -274,68 +256,14 @@ export function createViteAdapter(dataDir: string, server: ViteDevServer): Studi
return state;
},
startBackgroundRemoval(opts): MediaProcessingJobState {
const state: MediaProcessingJobState = {
id: opts.jobId,
status: "processing",
progress: 0,
stage: "Preparing background removal",
inputAssetPath: opts.inputAssetPath,
outputAssetPath: opts.outputAssetPath,
outputPath: opts.outputPath,
...(opts.backgroundOutputPath ? { backgroundOutputPath: opts.backgroundOutputPath } : {}),
...(opts.backgroundOutputAssetPath
? { backgroundOutputAssetPath: opts.backgroundOutputAssetPath }
: {}),
};
(async () => {
try {
const mod = await server.ssrLoadModule(
resolve(__dirname, "../cli/src/background-removal/pipeline.ts"),
);
const render = mod.render as BackgroundRemovalRender;
const result = await render({
inputPath: opts.inputPath,
outputPath: opts.outputPath,
backgroundOutputPath: opts.backgroundOutputPath,
device: opts.device,
quality: opts.quality,
onProgress: (event) => {
if (event.kind === "info") {
state.stage = event.message;
return;
}
if (event.kind === "metadata") {
state.stage = `Source ${event.width}×${event.height}`;
state.progress = 2;
return;
}
state.progress = event.total
? Math.min(99, Math.floor((event.index / event.total) * 100))
: 0;
state.stage = event.total
? `Removing background ${event.index}/${event.total}`
: `Removing background frame ${event.index}`;
state.framesProcessed = event.index;
state.avgMsPerFrame = event.avgMsPerFrame;
},
});
state.status = "complete";
state.progress = 100;
state.stage = "Complete";
state.provider = result.provider;
state.framesProcessed = result.framesProcessed;
state.durationSeconds = result.durationSeconds;
state.avgMsPerFrame = result.avgMsPerFrame;
} catch (err) {
state.status = "failed";
state.error = err instanceof Error ? err.message : String(err);
state.stage = "Failed";
}
})();
return state;
startBackgroundRemoval(opts) {
return createBackgroundRemovalJob(opts, async (renderOpts) => {
const mod = await server.ssrLoadModule(
resolve(__dirname, "../cli/src/background-removal/pipeline.ts"),
);
const render = mod.render as BackgroundRemovalRender;
return render(renderOpts);
});
},
async generateThumbnail(opts) {