mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-17 15:24:19 +00:00
fix(studio-server): share background removal job runner
This commit is contained in:
@@ -23,11 +23,12 @@ import {
|
||||
createStudioManualEditsRenderBodyScript,
|
||||
createStudioApi,
|
||||
createProjectSignature,
|
||||
createBackgroundRemovalJob,
|
||||
getMimeType,
|
||||
type StudioApiAdapter,
|
||||
type ResolvedProject,
|
||||
type RenderJobState,
|
||||
type MediaProcessingJobState,
|
||||
type BackgroundRemovalRender,
|
||||
} from "@hyperframes/studio-server";
|
||||
import { getElementScreenshotClip } from "@hyperframes/studio-server/screenshot-clip";
|
||||
import type { ScreenshotClip } from "@hyperframes/studio-server/screenshot-clip";
|
||||
@@ -428,69 +429,13 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
||||
},
|
||||
|
||||
startBackgroundRemoval(opts) {
|
||||
const state: MediaProcessingJobState = {
|
||||
id: opts.jobId,
|
||||
status: "processing",
|
||||
progress: 0,
|
||||
stage: "Preparing background removal",
|
||||
inputAssetPath: opts.inputAssetPath,
|
||||
outputAssetPath: opts.outputAssetPath,
|
||||
outputPath: opts.outputPath,
|
||||
...(opts.backgroundOutputPath ? { backgroundOutputPath: opts.backgroundOutputPath } : {}),
|
||||
...(opts.backgroundOutputAssetPath
|
||||
? { backgroundOutputAssetPath: opts.backgroundOutputAssetPath }
|
||||
: {}),
|
||||
};
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const sourcePipelinePath = "../background-removal/pipeline.ts";
|
||||
const pipeline = (await import("../background-removal/pipeline.js").catch(
|
||||
() => import(sourcePipelinePath),
|
||||
)) as typeof import("../background-removal/pipeline.js");
|
||||
const { render } = pipeline;
|
||||
const result = await render({
|
||||
inputPath: opts.inputPath,
|
||||
outputPath: opts.outputPath,
|
||||
backgroundOutputPath: opts.backgroundOutputPath,
|
||||
device: opts.device,
|
||||
quality: opts.quality,
|
||||
onProgress: (event) => {
|
||||
if (event.kind === "info") {
|
||||
state.stage = event.message;
|
||||
return;
|
||||
}
|
||||
if (event.kind === "metadata") {
|
||||
state.stage = `Source ${event.width}×${event.height}`;
|
||||
state.progress = 2;
|
||||
return;
|
||||
}
|
||||
const pct = event.total
|
||||
? Math.min(99, Math.floor((event.index / event.total) * 100))
|
||||
: 0;
|
||||
state.progress = pct;
|
||||
state.stage = event.total
|
||||
? `Removing background ${event.index}/${event.total}`
|
||||
: `Removing background frame ${event.index}`;
|
||||
state.framesProcessed = event.index;
|
||||
state.avgMsPerFrame = event.avgMsPerFrame;
|
||||
},
|
||||
});
|
||||
state.status = "complete";
|
||||
state.progress = 100;
|
||||
state.stage = "Complete";
|
||||
state.provider = result.provider;
|
||||
state.framesProcessed = result.framesProcessed;
|
||||
state.durationSeconds = result.durationSeconds;
|
||||
state.avgMsPerFrame = result.avgMsPerFrame;
|
||||
} catch (err) {
|
||||
state.status = "failed";
|
||||
state.error = err instanceof Error ? err.message : String(err);
|
||||
state.stage = "Failed";
|
||||
}
|
||||
})();
|
||||
|
||||
return state;
|
||||
return createBackgroundRemovalJob(opts, async (renderOpts) => {
|
||||
const sourcePipelinePath = "../background-removal/pipeline.ts";
|
||||
const pipeline = (await import("../background-removal/pipeline.js").catch(
|
||||
() => import(sourcePipelinePath),
|
||||
)) as { render: BackgroundRemovalRender };
|
||||
return pipeline.render(renderOpts);
|
||||
});
|
||||
},
|
||||
|
||||
async generateThumbnail(opts): Promise<Buffer | null> {
|
||||
@@ -515,6 +460,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
||||
{ timeout: 5000 },
|
||||
)
|
||||
.catch(() => {});
|
||||
// fallow-ignore-next-line code-duplication
|
||||
await page.evaluate((t: number) => {
|
||||
const w = window as Window & {
|
||||
__player?: { seek?: (time: number) => void };
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js";
|
||||
|
||||
export type BackgroundRemovalJobOptions = Parameters<
|
||||
NonNullable<StudioApiAdapter["startBackgroundRemoval"]>
|
||||
>[0];
|
||||
|
||||
export type BackgroundRemovalProgressEvent =
|
||||
| { kind: "info"; message: string }
|
||||
| { kind: "metadata"; width: number; height: number; fps: number; frameCount: number }
|
||||
| { kind: "frame"; index: number; total: number; avgMsPerFrame: number };
|
||||
|
||||
export type BackgroundRemovalRender = (options: {
|
||||
inputPath: string;
|
||||
outputPath: string;
|
||||
backgroundOutputPath?: string;
|
||||
device?: BackgroundRemovalJobOptions["device"];
|
||||
quality?: BackgroundRemovalJobOptions["quality"];
|
||||
onProgress?: (event: BackgroundRemovalProgressEvent) => void;
|
||||
}) => Promise<{
|
||||
provider: string;
|
||||
framesProcessed: number;
|
||||
durationSeconds: number;
|
||||
avgMsPerFrame: number;
|
||||
}>;
|
||||
|
||||
export function createBackgroundRemovalJob(
|
||||
opts: BackgroundRemovalJobOptions,
|
||||
render: BackgroundRemovalRender,
|
||||
): MediaProcessingJobState {
|
||||
const state: MediaProcessingJobState = {
|
||||
id: opts.jobId,
|
||||
status: "processing",
|
||||
progress: 0,
|
||||
stage: "Preparing background removal",
|
||||
inputAssetPath: opts.inputAssetPath,
|
||||
outputAssetPath: opts.outputAssetPath,
|
||||
outputPath: opts.outputPath,
|
||||
...(opts.backgroundOutputPath ? { backgroundOutputPath: opts.backgroundOutputPath } : {}),
|
||||
...(opts.backgroundOutputAssetPath
|
||||
? { backgroundOutputAssetPath: opts.backgroundOutputAssetPath }
|
||||
: {}),
|
||||
};
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const result = await render({
|
||||
inputPath: opts.inputPath,
|
||||
outputPath: opts.outputPath,
|
||||
backgroundOutputPath: opts.backgroundOutputPath,
|
||||
device: opts.device,
|
||||
quality: opts.quality,
|
||||
onProgress: (event) => updateBackgroundRemovalProgress(state, event),
|
||||
});
|
||||
state.status = "complete";
|
||||
state.progress = 100;
|
||||
state.stage = "Complete";
|
||||
state.provider = result.provider;
|
||||
state.framesProcessed = result.framesProcessed;
|
||||
state.durationSeconds = result.durationSeconds;
|
||||
state.avgMsPerFrame = result.avgMsPerFrame;
|
||||
} catch (err) {
|
||||
state.status = "failed";
|
||||
state.error = err instanceof Error ? err.message : String(err);
|
||||
state.stage = "Failed";
|
||||
}
|
||||
})();
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
function updateBackgroundRemovalProgress(
|
||||
state: MediaProcessingJobState,
|
||||
event: BackgroundRemovalProgressEvent,
|
||||
): void {
|
||||
if (event.kind === "info") {
|
||||
state.stage = event.message;
|
||||
return;
|
||||
}
|
||||
if (event.kind === "metadata") {
|
||||
state.stage = `Source ${event.width}×${event.height}`;
|
||||
state.progress = 2;
|
||||
return;
|
||||
}
|
||||
state.progress = event.total ? Math.min(99, Math.floor((event.index / event.total) * 100)) : 0;
|
||||
state.stage = event.total
|
||||
? `Removing background ${event.index}/${event.total}`
|
||||
: `Removing background frame ${event.index}`;
|
||||
state.framesProcessed = event.index;
|
||||
state.avgMsPerFrame = event.avgMsPerFrame;
|
||||
}
|
||||
@@ -14,6 +14,10 @@ export { isSafePath, walkDir } from "./helpers/safePath.js";
|
||||
export { getMimeType, MIME_TYPES } from "./helpers/mime.js";
|
||||
export { buildSubCompositionHtml } from "./helpers/subComposition.js";
|
||||
export { getElementScreenshotClip, type ScreenshotClip } from "./helpers/screenshotClip.js";
|
||||
export {
|
||||
createBackgroundRemovalJob,
|
||||
type BackgroundRemovalRender,
|
||||
} from "./helpers/backgroundRemovalJob.js";
|
||||
export {
|
||||
STUDIO_MANUAL_EDITS_PATH,
|
||||
createStudioManualEditsRenderBodyScript,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// fallow-ignore-file code-duplication
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { Hono } from "hono";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
|
||||
@@ -129,92 +129,97 @@ export function registerMediaRoutes(
|
||||
return c.json({ path: assetPath, metadata: readMediaMetadata(filePath) });
|
||||
});
|
||||
|
||||
api.post("/projects/:id/media/remove-background", async (c) => {
|
||||
cleanupFinishedJobs();
|
||||
if (!adapter.startBackgroundRemoval) {
|
||||
return c.json({ error: "background removal is not available in this Studio server" }, 501);
|
||||
}
|
||||
|
||||
const project = await adapter.resolveProject(c.req.param("id"));
|
||||
if (!project) return c.json({ error: "not found" }, 404);
|
||||
|
||||
const body = (await c.req.json().catch(() => ({}))) as BackgroundRemovalBody;
|
||||
const inputAssetPath = body.inputPath ? normalizeProjectAssetPath(body.inputPath) : "";
|
||||
if (!inputAssetPath) return c.json({ error: "inputPath required" }, 400);
|
||||
if (containsNullByte(inputAssetPath)) return c.json({ error: "forbidden" }, 403);
|
||||
if (/^(?:https?:|data:|blob:)/i.test(inputAssetPath)) {
|
||||
return c.json({ error: "background removal requires a project-local media asset" }, 400);
|
||||
}
|
||||
|
||||
const inputPath = resolveWithinProject(project.dir, inputAssetPath);
|
||||
if (!inputPath) return c.json({ error: "forbidden" }, 403);
|
||||
if (!existsSync(inputPath)) return c.json({ error: "input media not found" }, 404);
|
||||
|
||||
const inputIsVideo = isVideoPath(inputAssetPath);
|
||||
const inputIsImage = isImagePath(inputAssetPath);
|
||||
if (!inputIsVideo && !inputIsImage) {
|
||||
return c.json({ error: "background removal supports video or image assets only" }, 400);
|
||||
}
|
||||
|
||||
const requestedOutput = body.outputPath ? normalizeProjectAssetPath(body.outputPath) : "";
|
||||
if (requestedOutput && containsNullByte(requestedOutput)) {
|
||||
return c.json({ error: "forbidden" }, 403);
|
||||
}
|
||||
if (requestedOutput && !resolveWithinProject(project.dir, requestedOutput)) {
|
||||
return c.json({ error: "forbidden" }, 403);
|
||||
}
|
||||
const outputAssetPath = requestedOutput
|
||||
? uniqueAssetPath(project.dir, requestedOutput)
|
||||
: defaultOutputPath(project.dir, inputAssetPath);
|
||||
const outputPath = resolveWithinProject(project.dir, outputAssetPath);
|
||||
if (!outputPath) return c.json({ error: "forbidden" }, 403);
|
||||
if (inputIsVideo && !VIDEO_OUTPUT_EXTENSIONS.has(extname(outputAssetPath).toLowerCase())) {
|
||||
return c.json({ error: "video background removal output must be .webm or .mov" }, 400);
|
||||
}
|
||||
if (inputIsImage && extname(outputAssetPath).toLowerCase() !== ".png") {
|
||||
return c.json({ error: "image background removal output must be .png" }, 400);
|
||||
}
|
||||
|
||||
let backgroundOutputAssetPath: string | undefined;
|
||||
let backgroundOutputPath: string | undefined;
|
||||
if (body.createBackgroundPlate) {
|
||||
if (!inputIsVideo) {
|
||||
return c.json({ error: "background plates are only supported for video inputs" }, 400);
|
||||
api.post(
|
||||
"/projects/:id/media/remove-background",
|
||||
// fallow-ignore-next-line complexity
|
||||
async (c) => {
|
||||
cleanupFinishedJobs();
|
||||
if (!adapter.startBackgroundRemoval) {
|
||||
return c.json({ error: "background removal is not available in this Studio server" }, 501);
|
||||
}
|
||||
backgroundOutputAssetPath = defaultPlatePath(project.dir, inputAssetPath);
|
||||
backgroundOutputPath =
|
||||
resolveWithinProject(project.dir, backgroundOutputAssetPath) ?? undefined;
|
||||
if (!backgroundOutputPath) {
|
||||
|
||||
// fallow-ignore-next-line code-duplication
|
||||
const project = await adapter.resolveProject(c.req.param("id"));
|
||||
if (!project) return c.json({ error: "not found" }, 404);
|
||||
|
||||
const body = (await c.req.json().catch(() => ({}))) as BackgroundRemovalBody;
|
||||
const inputAssetPath = body.inputPath ? normalizeProjectAssetPath(body.inputPath) : "";
|
||||
if (!inputAssetPath) return c.json({ error: "inputPath required" }, 400);
|
||||
if (containsNullByte(inputAssetPath)) return c.json({ error: "forbidden" }, 403);
|
||||
if (/^(?:https?:|data:|blob:)/i.test(inputAssetPath)) {
|
||||
return c.json({ error: "background removal requires a project-local media asset" }, 400);
|
||||
}
|
||||
|
||||
const inputPath = resolveWithinProject(project.dir, inputAssetPath);
|
||||
if (!inputPath) return c.json({ error: "forbidden" }, 403);
|
||||
if (!existsSync(inputPath)) return c.json({ error: "input media not found" }, 404);
|
||||
|
||||
const inputIsVideo = isVideoPath(inputAssetPath);
|
||||
const inputIsImage = isImagePath(inputAssetPath);
|
||||
if (!inputIsVideo && !inputIsImage) {
|
||||
return c.json({ error: "background removal supports video or image assets only" }, 400);
|
||||
}
|
||||
|
||||
const requestedOutput = body.outputPath ? normalizeProjectAssetPath(body.outputPath) : "";
|
||||
if (requestedOutput && containsNullByte(requestedOutput)) {
|
||||
return c.json({ error: "forbidden" }, 403);
|
||||
}
|
||||
}
|
||||
if (requestedOutput && !resolveWithinProject(project.dir, requestedOutput)) {
|
||||
return c.json({ error: "forbidden" }, 403);
|
||||
}
|
||||
const outputAssetPath = requestedOutput
|
||||
? uniqueAssetPath(project.dir, requestedOutput)
|
||||
: defaultOutputPath(project.dir, inputAssetPath);
|
||||
const outputPath = resolveWithinProject(project.dir, outputAssetPath);
|
||||
if (!outputPath) return c.json({ error: "forbidden" }, 403);
|
||||
if (inputIsVideo && !VIDEO_OUTPUT_EXTENSIONS.has(extname(outputAssetPath).toLowerCase())) {
|
||||
return c.json({ error: "video background removal output must be .webm or .mov" }, 400);
|
||||
}
|
||||
if (inputIsImage && extname(outputAssetPath).toLowerCase() !== ".png") {
|
||||
return c.json({ error: "image background removal output must be .png" }, 400);
|
||||
}
|
||||
|
||||
mkdirSync(dirname(outputPath), { recursive: true });
|
||||
if (backgroundOutputPath) mkdirSync(dirname(backgroundOutputPath), { recursive: true });
|
||||
let backgroundOutputAssetPath: string | undefined;
|
||||
let backgroundOutputPath: string | undefined;
|
||||
if (body.createBackgroundPlate) {
|
||||
if (!inputIsVideo) {
|
||||
return c.json({ error: "background plates are only supported for video inputs" }, 400);
|
||||
}
|
||||
backgroundOutputAssetPath = defaultPlatePath(project.dir, inputAssetPath);
|
||||
backgroundOutputPath =
|
||||
resolveWithinProject(project.dir, backgroundOutputAssetPath) ?? undefined;
|
||||
if (!backgroundOutputPath) {
|
||||
return c.json({ error: "forbidden" }, 403);
|
||||
}
|
||||
}
|
||||
|
||||
const jobId = makeJobId(project.id, mediaJobs);
|
||||
const state = adapter.startBackgroundRemoval({
|
||||
project,
|
||||
inputPath,
|
||||
inputAssetPath,
|
||||
outputPath,
|
||||
outputAssetPath,
|
||||
backgroundOutputPath,
|
||||
backgroundOutputAssetPath,
|
||||
quality: normalizeQuality(body.quality),
|
||||
device: normalizeDevice(body.device),
|
||||
jobId,
|
||||
}) as JobWithCreatedAt;
|
||||
state.createdAt = Date.now();
|
||||
mediaJobs.set(jobId, state);
|
||||
mkdirSync(dirname(outputPath), { recursive: true });
|
||||
if (backgroundOutputPath) mkdirSync(dirname(backgroundOutputPath), { recursive: true });
|
||||
|
||||
return c.json({
|
||||
jobId,
|
||||
status: state.status,
|
||||
outputPath: outputAssetPath,
|
||||
backgroundOutputPath: backgroundOutputAssetPath,
|
||||
});
|
||||
});
|
||||
const jobId = makeJobId(project.id, mediaJobs);
|
||||
const state = adapter.startBackgroundRemoval({
|
||||
project,
|
||||
inputPath,
|
||||
inputAssetPath,
|
||||
outputPath,
|
||||
outputAssetPath,
|
||||
backgroundOutputPath,
|
||||
backgroundOutputAssetPath,
|
||||
quality: normalizeQuality(body.quality),
|
||||
device: normalizeDevice(body.device),
|
||||
jobId,
|
||||
}) as JobWithCreatedAt;
|
||||
state.createdAt = Date.now();
|
||||
mediaJobs.set(jobId, state);
|
||||
|
||||
return c.json({
|
||||
jobId,
|
||||
status: state.status,
|
||||
outputPath: outputAssetPath,
|
||||
backgroundOutputPath: backgroundOutputAssetPath,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
api.get("/media-jobs/:jobId/progress", (c) => {
|
||||
cleanupFinishedJobs();
|
||||
|
||||
@@ -14,8 +14,9 @@ import type { ViteDevServer } from "vite";
|
||||
import {
|
||||
type ResolvedProject,
|
||||
type RenderJobState,
|
||||
type MediaProcessingJobState,
|
||||
type StudioApiAdapter,
|
||||
type BackgroundRemovalRender,
|
||||
createBackgroundRemovalJob,
|
||||
createProjectSignature,
|
||||
} from "@hyperframes/studio-server";
|
||||
import type { RegistryItem } from "@hyperframes/core/registry";
|
||||
@@ -23,25 +24,6 @@ import { createRetryingModuleLoader, ensureProducerDist } from "./vite.producer"
|
||||
import { createStudioDevRenderBodyScripts } from "./vite.studioMotion";
|
||||
import { generateThumbnail, findSystemChrome } from "./vite.browser";
|
||||
|
||||
type BackgroundRemovalRender = (options: {
|
||||
inputPath: string;
|
||||
outputPath: string;
|
||||
backgroundOutputPath?: string;
|
||||
device?: "auto" | "cpu" | "coreml" | "cuda";
|
||||
quality?: "fast" | "balanced" | "best";
|
||||
onProgress?: (
|
||||
event:
|
||||
| { kind: "info"; message: string }
|
||||
| { kind: "metadata"; width: number; height: number; fps: number; frameCount: number }
|
||||
| { kind: "frame"; index: number; total: number; avgMsPerFrame: number },
|
||||
) => void;
|
||||
}) => Promise<{
|
||||
provider: string;
|
||||
framesProcessed: number;
|
||||
durationSeconds: number;
|
||||
avgMsPerFrame: number;
|
||||
}>;
|
||||
|
||||
export function isPathWithin(parentDir: string, childPath: string): boolean {
|
||||
const childRelativePath = relative(resolve(parentDir), resolve(childPath));
|
||||
return (
|
||||
@@ -274,68 +256,14 @@ export function createViteAdapter(dataDir: string, server: ViteDevServer): Studi
|
||||
return state;
|
||||
},
|
||||
|
||||
startBackgroundRemoval(opts): MediaProcessingJobState {
|
||||
const state: MediaProcessingJobState = {
|
||||
id: opts.jobId,
|
||||
status: "processing",
|
||||
progress: 0,
|
||||
stage: "Preparing background removal",
|
||||
inputAssetPath: opts.inputAssetPath,
|
||||
outputAssetPath: opts.outputAssetPath,
|
||||
outputPath: opts.outputPath,
|
||||
...(opts.backgroundOutputPath ? { backgroundOutputPath: opts.backgroundOutputPath } : {}),
|
||||
...(opts.backgroundOutputAssetPath
|
||||
? { backgroundOutputAssetPath: opts.backgroundOutputAssetPath }
|
||||
: {}),
|
||||
};
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const mod = await server.ssrLoadModule(
|
||||
resolve(__dirname, "../cli/src/background-removal/pipeline.ts"),
|
||||
);
|
||||
const render = mod.render as BackgroundRemovalRender;
|
||||
const result = await render({
|
||||
inputPath: opts.inputPath,
|
||||
outputPath: opts.outputPath,
|
||||
backgroundOutputPath: opts.backgroundOutputPath,
|
||||
device: opts.device,
|
||||
quality: opts.quality,
|
||||
onProgress: (event) => {
|
||||
if (event.kind === "info") {
|
||||
state.stage = event.message;
|
||||
return;
|
||||
}
|
||||
if (event.kind === "metadata") {
|
||||
state.stage = `Source ${event.width}×${event.height}`;
|
||||
state.progress = 2;
|
||||
return;
|
||||
}
|
||||
state.progress = event.total
|
||||
? Math.min(99, Math.floor((event.index / event.total) * 100))
|
||||
: 0;
|
||||
state.stage = event.total
|
||||
? `Removing background ${event.index}/${event.total}`
|
||||
: `Removing background frame ${event.index}`;
|
||||
state.framesProcessed = event.index;
|
||||
state.avgMsPerFrame = event.avgMsPerFrame;
|
||||
},
|
||||
});
|
||||
state.status = "complete";
|
||||
state.progress = 100;
|
||||
state.stage = "Complete";
|
||||
state.provider = result.provider;
|
||||
state.framesProcessed = result.framesProcessed;
|
||||
state.durationSeconds = result.durationSeconds;
|
||||
state.avgMsPerFrame = result.avgMsPerFrame;
|
||||
} catch (err) {
|
||||
state.status = "failed";
|
||||
state.error = err instanceof Error ? err.message : String(err);
|
||||
state.stage = "Failed";
|
||||
}
|
||||
})();
|
||||
|
||||
return state;
|
||||
startBackgroundRemoval(opts) {
|
||||
return createBackgroundRemovalJob(opts, async (renderOpts) => {
|
||||
const mod = await server.ssrLoadModule(
|
||||
resolve(__dirname, "../cli/src/background-removal/pipeline.ts"),
|
||||
);
|
||||
const render = mod.render as BackgroundRemovalRender;
|
||||
return render(renderOpts);
|
||||
});
|
||||
},
|
||||
|
||||
async generateThumbnail(opts) {
|
||||
|
||||
Reference in New Issue
Block a user