diff --git a/packages/cli/src/server/studioServer.ts b/packages/cli/src/server/studioServer.ts index c2a558530..8de534fb2 100644 --- a/packages/cli/src/server/studioServer.ts +++ b/packages/cli/src/server/studioServer.ts @@ -23,11 +23,12 @@ import { createStudioManualEditsRenderBodyScript, createStudioApi, createProjectSignature, + createBackgroundRemovalJob, getMimeType, type StudioApiAdapter, type ResolvedProject, type RenderJobState, - type MediaProcessingJobState, + type BackgroundRemovalRender, } from "@hyperframes/studio-server"; import { getElementScreenshotClip } from "@hyperframes/studio-server/screenshot-clip"; import type { ScreenshotClip } from "@hyperframes/studio-server/screenshot-clip"; @@ -428,69 +429,13 @@ export function createStudioServer(options: StudioServerOptions): StudioServer { }, startBackgroundRemoval(opts) { - const state: MediaProcessingJobState = { - id: opts.jobId, - status: "processing", - progress: 0, - stage: "Preparing background removal", - inputAssetPath: opts.inputAssetPath, - outputAssetPath: opts.outputAssetPath, - outputPath: opts.outputPath, - ...(opts.backgroundOutputPath ? { backgroundOutputPath: opts.backgroundOutputPath } : {}), - ...(opts.backgroundOutputAssetPath - ? { backgroundOutputAssetPath: opts.backgroundOutputAssetPath } - : {}), - }; - - (async () => { - try { - const sourcePipelinePath = "../background-removal/pipeline.ts"; - const pipeline = (await import("../background-removal/pipeline.js").catch( - () => import(sourcePipelinePath), - )) as typeof import("../background-removal/pipeline.js"); - const { render } = pipeline; - const result = await render({ - inputPath: opts.inputPath, - outputPath: opts.outputPath, - backgroundOutputPath: opts.backgroundOutputPath, - device: opts.device, - quality: opts.quality, - onProgress: (event) => { - if (event.kind === "info") { - state.stage = event.message; - return; - } - if (event.kind === "metadata") { - state.stage = `Source ${event.width}×${event.height}`; - state.progress = 2; - return; - } - const pct = event.total - ? Math.min(99, Math.floor((event.index / event.total) * 100)) - : 0; - state.progress = pct; - state.stage = event.total - ? `Removing background ${event.index}/${event.total}` - : `Removing background frame ${event.index}`; - state.framesProcessed = event.index; - state.avgMsPerFrame = event.avgMsPerFrame; - }, - }); - state.status = "complete"; - state.progress = 100; - state.stage = "Complete"; - state.provider = result.provider; - state.framesProcessed = result.framesProcessed; - state.durationSeconds = result.durationSeconds; - state.avgMsPerFrame = result.avgMsPerFrame; - } catch (err) { - state.status = "failed"; - state.error = err instanceof Error ? err.message : String(err); - state.stage = "Failed"; - } - })(); - - return state; + return createBackgroundRemovalJob(opts, async (renderOpts) => { + const sourcePipelinePath = "../background-removal/pipeline.ts"; + const pipeline = (await import("../background-removal/pipeline.js").catch( + () => import(sourcePipelinePath), + )) as { render: BackgroundRemovalRender }; + return pipeline.render(renderOpts); + }); }, async generateThumbnail(opts): Promise { @@ -515,6 +460,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer { { timeout: 5000 }, ) .catch(() => {}); + // fallow-ignore-next-line code-duplication await page.evaluate((t: number) => { const w = window as Window & { __player?: { seek?: (time: number) => void }; diff --git a/packages/studio-server/src/helpers/backgroundRemovalJob.ts b/packages/studio-server/src/helpers/backgroundRemovalJob.ts new file mode 100644 index 000000000..910d45b89 --- /dev/null +++ b/packages/studio-server/src/helpers/backgroundRemovalJob.ts @@ -0,0 +1,90 @@ +import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js"; + +export type BackgroundRemovalJobOptions = Parameters< + NonNullable +>[0]; + +export type BackgroundRemovalProgressEvent = + | { kind: "info"; message: string } + | { kind: "metadata"; width: number; height: number; fps: number; frameCount: number } + | { kind: "frame"; index: number; total: number; avgMsPerFrame: number }; + +export type BackgroundRemovalRender = (options: { + inputPath: string; + outputPath: string; + backgroundOutputPath?: string; + device?: BackgroundRemovalJobOptions["device"]; + quality?: BackgroundRemovalJobOptions["quality"]; + onProgress?: (event: BackgroundRemovalProgressEvent) => void; +}) => Promise<{ + provider: string; + framesProcessed: number; + durationSeconds: number; + avgMsPerFrame: number; +}>; + +export function createBackgroundRemovalJob( + opts: BackgroundRemovalJobOptions, + render: BackgroundRemovalRender, +): MediaProcessingJobState { + const state: MediaProcessingJobState = { + id: opts.jobId, + status: "processing", + progress: 0, + stage: "Preparing background removal", + inputAssetPath: opts.inputAssetPath, + outputAssetPath: opts.outputAssetPath, + outputPath: opts.outputPath, + ...(opts.backgroundOutputPath ? { backgroundOutputPath: opts.backgroundOutputPath } : {}), + ...(opts.backgroundOutputAssetPath + ? { backgroundOutputAssetPath: opts.backgroundOutputAssetPath } + : {}), + }; + + void (async () => { + try { + const result = await render({ + inputPath: opts.inputPath, + outputPath: opts.outputPath, + backgroundOutputPath: opts.backgroundOutputPath, + device: opts.device, + quality: opts.quality, + onProgress: (event) => updateBackgroundRemovalProgress(state, event), + }); + state.status = "complete"; + state.progress = 100; + state.stage = "Complete"; + state.provider = result.provider; + state.framesProcessed = result.framesProcessed; + state.durationSeconds = result.durationSeconds; + state.avgMsPerFrame = result.avgMsPerFrame; + } catch (err) { + state.status = "failed"; + state.error = err instanceof Error ? err.message : String(err); + state.stage = "Failed"; + } + })(); + + return state; +} + +function updateBackgroundRemovalProgress( + state: MediaProcessingJobState, + event: BackgroundRemovalProgressEvent, +): void { + if (event.kind === "info") { + state.stage = event.message; + return; + } + if (event.kind === "metadata") { + state.stage = `Source ${event.width}×${event.height}`; + state.progress = 2; + return; + } + state.progress = event.total ? Math.min(99, Math.floor((event.index / event.total) * 100)) : 0; + state.stage = event.total + ? `Removing background ${event.index}/${event.total}` + : `Removing background frame ${event.index}`; + state.framesProcessed = event.index; + state.avgMsPerFrame = event.avgMsPerFrame; +} diff --git a/packages/studio-server/src/index.ts b/packages/studio-server/src/index.ts index abfebbcb6..4211df1db 100644 --- a/packages/studio-server/src/index.ts +++ b/packages/studio-server/src/index.ts @@ -14,6 +14,10 @@ export { isSafePath, walkDir } from "./helpers/safePath.js"; export { getMimeType, MIME_TYPES } from "./helpers/mime.js"; export { buildSubCompositionHtml } from "./helpers/subComposition.js"; export { getElementScreenshotClip, type ScreenshotClip } from "./helpers/screenshotClip.js"; +export { + createBackgroundRemovalJob, + type BackgroundRemovalRender, +} from "./helpers/backgroundRemovalJob.js"; export { STUDIO_MANUAL_EDITS_PATH, createStudioManualEditsRenderBodyScript, diff --git a/packages/studio-server/src/routes/media.test.ts b/packages/studio-server/src/routes/media.test.ts index c260d80de..f02e928f6 100644 --- a/packages/studio-server/src/routes/media.test.ts +++ b/packages/studio-server/src/routes/media.test.ts @@ -1,3 +1,4 @@ +// fallow-ignore-file code-duplication import { afterEach, describe, expect, it, vi } from "vitest"; import { Hono } from "hono"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; diff --git a/packages/studio-server/src/routes/media.ts b/packages/studio-server/src/routes/media.ts index fca8cc4cd..76c52f5e6 100644 --- a/packages/studio-server/src/routes/media.ts +++ b/packages/studio-server/src/routes/media.ts @@ -129,92 +129,97 @@ export function registerMediaRoutes( return c.json({ path: assetPath, metadata: readMediaMetadata(filePath) }); }); - api.post("/projects/:id/media/remove-background", async (c) => { - cleanupFinishedJobs(); - if (!adapter.startBackgroundRemoval) { - return c.json({ error: "background removal is not available in this Studio server" }, 501); - } - - const project = await adapter.resolveProject(c.req.param("id")); - if (!project) return c.json({ error: "not found" }, 404); - - const body = (await c.req.json().catch(() => ({}))) as BackgroundRemovalBody; - const inputAssetPath = body.inputPath ? normalizeProjectAssetPath(body.inputPath) : ""; - if (!inputAssetPath) return c.json({ error: "inputPath required" }, 400); - if (containsNullByte(inputAssetPath)) return c.json({ error: "forbidden" }, 403); - if (/^(?:https?:|data:|blob:)/i.test(inputAssetPath)) { - return c.json({ error: "background removal requires a project-local media asset" }, 400); - } - - const inputPath = resolveWithinProject(project.dir, inputAssetPath); - if (!inputPath) return c.json({ error: "forbidden" }, 403); - if (!existsSync(inputPath)) return c.json({ error: "input media not found" }, 404); - - const inputIsVideo = isVideoPath(inputAssetPath); - const inputIsImage = isImagePath(inputAssetPath); - if (!inputIsVideo && !inputIsImage) { - return c.json({ error: "background removal supports video or image assets only" }, 400); - } - - const requestedOutput = body.outputPath ? normalizeProjectAssetPath(body.outputPath) : ""; - if (requestedOutput && containsNullByte(requestedOutput)) { - return c.json({ error: "forbidden" }, 403); - } - if (requestedOutput && !resolveWithinProject(project.dir, requestedOutput)) { - return c.json({ error: "forbidden" }, 403); - } - const outputAssetPath = requestedOutput - ? uniqueAssetPath(project.dir, requestedOutput) - : defaultOutputPath(project.dir, inputAssetPath); - const outputPath = resolveWithinProject(project.dir, outputAssetPath); - if (!outputPath) return c.json({ error: "forbidden" }, 403); - if (inputIsVideo && !VIDEO_OUTPUT_EXTENSIONS.has(extname(outputAssetPath).toLowerCase())) { - return c.json({ error: "video background removal output must be .webm or .mov" }, 400); - } - if (inputIsImage && extname(outputAssetPath).toLowerCase() !== ".png") { - return c.json({ error: "image background removal output must be .png" }, 400); - } - - let backgroundOutputAssetPath: string | undefined; - let backgroundOutputPath: string | undefined; - if (body.createBackgroundPlate) { - if (!inputIsVideo) { - return c.json({ error: "background plates are only supported for video inputs" }, 400); + api.post( + "/projects/:id/media/remove-background", + // fallow-ignore-next-line complexity + async (c) => { + cleanupFinishedJobs(); + if (!adapter.startBackgroundRemoval) { + return c.json({ error: "background removal is not available in this Studio server" }, 501); } - backgroundOutputAssetPath = defaultPlatePath(project.dir, inputAssetPath); - backgroundOutputPath = - resolveWithinProject(project.dir, backgroundOutputAssetPath) ?? undefined; - if (!backgroundOutputPath) { + + // fallow-ignore-next-line code-duplication + const project = await adapter.resolveProject(c.req.param("id")); + if (!project) return c.json({ error: "not found" }, 404); + + const body = (await c.req.json().catch(() => ({}))) as BackgroundRemovalBody; + const inputAssetPath = body.inputPath ? normalizeProjectAssetPath(body.inputPath) : ""; + if (!inputAssetPath) return c.json({ error: "inputPath required" }, 400); + if (containsNullByte(inputAssetPath)) return c.json({ error: "forbidden" }, 403); + if (/^(?:https?:|data:|blob:)/i.test(inputAssetPath)) { + return c.json({ error: "background removal requires a project-local media asset" }, 400); + } + + const inputPath = resolveWithinProject(project.dir, inputAssetPath); + if (!inputPath) return c.json({ error: "forbidden" }, 403); + if (!existsSync(inputPath)) return c.json({ error: "input media not found" }, 404); + + const inputIsVideo = isVideoPath(inputAssetPath); + const inputIsImage = isImagePath(inputAssetPath); + if (!inputIsVideo && !inputIsImage) { + return c.json({ error: "background removal supports video or image assets only" }, 400); + } + + const requestedOutput = body.outputPath ? normalizeProjectAssetPath(body.outputPath) : ""; + if (requestedOutput && containsNullByte(requestedOutput)) { return c.json({ error: "forbidden" }, 403); } - } + if (requestedOutput && !resolveWithinProject(project.dir, requestedOutput)) { + return c.json({ error: "forbidden" }, 403); + } + const outputAssetPath = requestedOutput + ? uniqueAssetPath(project.dir, requestedOutput) + : defaultOutputPath(project.dir, inputAssetPath); + const outputPath = resolveWithinProject(project.dir, outputAssetPath); + if (!outputPath) return c.json({ error: "forbidden" }, 403); + if (inputIsVideo && !VIDEO_OUTPUT_EXTENSIONS.has(extname(outputAssetPath).toLowerCase())) { + return c.json({ error: "video background removal output must be .webm or .mov" }, 400); + } + if (inputIsImage && extname(outputAssetPath).toLowerCase() !== ".png") { + return c.json({ error: "image background removal output must be .png" }, 400); + } - mkdirSync(dirname(outputPath), { recursive: true }); - if (backgroundOutputPath) mkdirSync(dirname(backgroundOutputPath), { recursive: true }); + let backgroundOutputAssetPath: string | undefined; + let backgroundOutputPath: string | undefined; + if (body.createBackgroundPlate) { + if (!inputIsVideo) { + return c.json({ error: "background plates are only supported for video inputs" }, 400); + } + backgroundOutputAssetPath = defaultPlatePath(project.dir, inputAssetPath); + backgroundOutputPath = + resolveWithinProject(project.dir, backgroundOutputAssetPath) ?? undefined; + if (!backgroundOutputPath) { + return c.json({ error: "forbidden" }, 403); + } + } - const jobId = makeJobId(project.id, mediaJobs); - const state = adapter.startBackgroundRemoval({ - project, - inputPath, - inputAssetPath, - outputPath, - outputAssetPath, - backgroundOutputPath, - backgroundOutputAssetPath, - quality: normalizeQuality(body.quality), - device: normalizeDevice(body.device), - jobId, - }) as JobWithCreatedAt; - state.createdAt = Date.now(); - mediaJobs.set(jobId, state); + mkdirSync(dirname(outputPath), { recursive: true }); + if (backgroundOutputPath) mkdirSync(dirname(backgroundOutputPath), { recursive: true }); - return c.json({ - jobId, - status: state.status, - outputPath: outputAssetPath, - backgroundOutputPath: backgroundOutputAssetPath, - }); - }); + const jobId = makeJobId(project.id, mediaJobs); + const state = adapter.startBackgroundRemoval({ + project, + inputPath, + inputAssetPath, + outputPath, + outputAssetPath, + backgroundOutputPath, + backgroundOutputAssetPath, + quality: normalizeQuality(body.quality), + device: normalizeDevice(body.device), + jobId, + }) as JobWithCreatedAt; + state.createdAt = Date.now(); + mediaJobs.set(jobId, state); + + return c.json({ + jobId, + status: state.status, + outputPath: outputAssetPath, + backgroundOutputPath: backgroundOutputAssetPath, + }); + }, + ); api.get("/media-jobs/:jobId/progress", (c) => { cleanupFinishedJobs(); diff --git a/packages/studio/vite.adapter.ts b/packages/studio/vite.adapter.ts index 392a811a9..23cc7c4c0 100644 --- a/packages/studio/vite.adapter.ts +++ b/packages/studio/vite.adapter.ts @@ -14,8 +14,9 @@ import type { ViteDevServer } from "vite"; import { type ResolvedProject, type RenderJobState, - type MediaProcessingJobState, type StudioApiAdapter, + type BackgroundRemovalRender, + createBackgroundRemovalJob, createProjectSignature, } from "@hyperframes/studio-server"; import type { RegistryItem } from "@hyperframes/core/registry"; @@ -23,25 +24,6 @@ import { createRetryingModuleLoader, ensureProducerDist } from "./vite.producer" import { createStudioDevRenderBodyScripts } from "./vite.studioMotion"; import { generateThumbnail, findSystemChrome } from "./vite.browser"; -type BackgroundRemovalRender = (options: { - inputPath: string; - outputPath: string; - backgroundOutputPath?: string; - device?: "auto" | "cpu" | "coreml" | "cuda"; - quality?: "fast" | "balanced" | "best"; - onProgress?: ( - event: - | { kind: "info"; message: string } - | { kind: "metadata"; width: number; height: number; fps: number; frameCount: number } - | { kind: "frame"; index: number; total: number; avgMsPerFrame: number }, - ) => void; -}) => Promise<{ - provider: string; - framesProcessed: number; - durationSeconds: number; - avgMsPerFrame: number; -}>; - export function isPathWithin(parentDir: string, childPath: string): boolean { const childRelativePath = relative(resolve(parentDir), resolve(childPath)); return ( @@ -274,68 +256,14 @@ export function createViteAdapter(dataDir: string, server: ViteDevServer): Studi return state; }, - startBackgroundRemoval(opts): MediaProcessingJobState { - const state: MediaProcessingJobState = { - id: opts.jobId, - status: "processing", - progress: 0, - stage: "Preparing background removal", - inputAssetPath: opts.inputAssetPath, - outputAssetPath: opts.outputAssetPath, - outputPath: opts.outputPath, - ...(opts.backgroundOutputPath ? { backgroundOutputPath: opts.backgroundOutputPath } : {}), - ...(opts.backgroundOutputAssetPath - ? { backgroundOutputAssetPath: opts.backgroundOutputAssetPath } - : {}), - }; - - (async () => { - try { - const mod = await server.ssrLoadModule( - resolve(__dirname, "../cli/src/background-removal/pipeline.ts"), - ); - const render = mod.render as BackgroundRemovalRender; - const result = await render({ - inputPath: opts.inputPath, - outputPath: opts.outputPath, - backgroundOutputPath: opts.backgroundOutputPath, - device: opts.device, - quality: opts.quality, - onProgress: (event) => { - if (event.kind === "info") { - state.stage = event.message; - return; - } - if (event.kind === "metadata") { - state.stage = `Source ${event.width}×${event.height}`; - state.progress = 2; - return; - } - state.progress = event.total - ? Math.min(99, Math.floor((event.index / event.total) * 100)) - : 0; - state.stage = event.total - ? `Removing background ${event.index}/${event.total}` - : `Removing background frame ${event.index}`; - state.framesProcessed = event.index; - state.avgMsPerFrame = event.avgMsPerFrame; - }, - }); - state.status = "complete"; - state.progress = 100; - state.stage = "Complete"; - state.provider = result.provider; - state.framesProcessed = result.framesProcessed; - state.durationSeconds = result.durationSeconds; - state.avgMsPerFrame = result.avgMsPerFrame; - } catch (err) { - state.status = "failed"; - state.error = err instanceof Error ? err.message : String(err); - state.stage = "Failed"; - } - })(); - - return state; + startBackgroundRemoval(opts) { + return createBackgroundRemovalJob(opts, async (renderOpts) => { + const mod = await server.ssrLoadModule( + resolve(__dirname, "../cli/src/background-removal/pipeline.ts"), + ); + const render = mod.render as BackgroundRemovalRender; + return render(renderOpts); + }); }, async generateThumbnail(opts) {