mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-02 12:08:50 +00:00
Merge pull request #1986 from heygen-com/feat/studio-media-processing-routes
feat(studio-server): add media processing routes
This commit is contained in:
Ular Kimsanov
@@ -18,14 +18,17 @@ import {
|
||||
import { VERSION as version } from "../version.js";
|
||||
import { buildStudioHeadScripts, resolveCliTelemetryDistinctId } from "./telemetryIdentity.js";
|
||||
import { emitStudioRenderComplete, emitStudioRenderError } from "./studioRenderTelemetry.js";
|
||||
import { isDevMode } from "../utils/env.js";
|
||||
import {
|
||||
createStudioManualEditsRenderBodyScript,
|
||||
createStudioApi,
|
||||
createProjectSignature,
|
||||
createBackgroundRemovalJob,
|
||||
getMimeType,
|
||||
type StudioApiAdapter,
|
||||
type ResolvedProject,
|
||||
type RenderJobState,
|
||||
type BackgroundRemovalRender,
|
||||
} from "@hyperframes/studio-server";
|
||||
import { getElementScreenshotClip } from "@hyperframes/studio-server/screenshot-clip";
|
||||
import type { ScreenshotClip } from "@hyperframes/studio-server/screenshot-clip";
|
||||
@@ -35,6 +38,12 @@ const STUDIO_MANUAL_EDITS_PATH = ".hyperframes/studio-manual-edits.json";
|
||||
const REMOTE_GIF_IMG_SRC_RE =
|
||||
/<img\b[^>]*?\bsrc\s*=\s*["'](https?:\/\/[^"']+\.gif(?:[?#][^"']*)?)["'][^>]*>/gi;
|
||||
|
||||
async function loadStudioProducer() {
|
||||
return isDevMode()
|
||||
? await import("../../../producer/src/index.js")
|
||||
: await import("@hyperframes/producer");
|
||||
}
|
||||
|
||||
// ── Path resolution ─────────────────────────────────────────────────────────
|
||||
|
||||
function resolveDistDir(): string {
|
||||
@@ -302,7 +311,10 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
||||
// we can point it at our hot-reloadable local runtime endpoint. Inlining
|
||||
// ~150 KB of runtime body on every preview render would defeat browser
|
||||
// caching across composition edits.
|
||||
let html = await bundleToSingleHtml(dir, { runtime: "placeholder" });
|
||||
let html = await bundleToSingleHtml(dir, {
|
||||
runtime: "placeholder",
|
||||
inlineColorGradingLuts: false,
|
||||
});
|
||||
html = html.replace(
|
||||
'data-hyperframes-preview-runtime="1" src=""',
|
||||
'data-hyperframes-preview-runtime="1" src="/api/runtime.js"',
|
||||
@@ -361,7 +373,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
||||
(async () => {
|
||||
let renderJob: RenderJob | undefined;
|
||||
try {
|
||||
const { createRenderJob, executeRenderJob } = await import("@hyperframes/producer");
|
||||
const { createRenderJob, executeRenderJob } = await loadStudioProducer();
|
||||
const { ensureBrowser } = await import("../browser/manager.js");
|
||||
|
||||
try {
|
||||
@@ -416,6 +428,16 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
||||
return state;
|
||||
},
|
||||
|
||||
startBackgroundRemoval(opts) {
|
||||
return createBackgroundRemovalJob(opts, async (renderOpts) => {
|
||||
const sourcePipelinePath = "../background-removal/pipeline.ts";
|
||||
const pipeline = (await import("../background-removal/pipeline.js").catch(
|
||||
() => import(sourcePipelinePath),
|
||||
)) as { render: BackgroundRemovalRender };
|
||||
return pipeline.render(renderOpts);
|
||||
});
|
||||
},
|
||||
|
||||
async generateThumbnail(opts): Promise<Buffer | null> {
|
||||
const browser = await getThumbnailBrowser();
|
||||
if (!browser) {
|
||||
@@ -438,6 +460,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
|
||||
{ timeout: 5000 },
|
||||
)
|
||||
.catch(() => {});
|
||||
// fallow-ignore-next-line code-duplication
|
||||
await page.evaluate((t: number) => {
|
||||
const w = window as Window & {
|
||||
__player?: { seek?: (time: number) => void };
|
||||
|
||||
@@ -11,6 +11,7 @@ import { registerWaveformRoutes } from "./routes/waveform.js";
|
||||
import { registerFontRoutes } from "./routes/fonts.js";
|
||||
import { registerRegistryRoutes } from "./routes/registry.js";
|
||||
import { registerSelectionRoutes } from "./routes/selection.js";
|
||||
import { registerMediaRoutes } from "./routes/media.js";
|
||||
|
||||
/**
|
||||
* Create a Hono sub-app with all studio API routes.
|
||||
@@ -29,6 +30,7 @@ export function createStudioApi(adapter: StudioApiAdapter): Hono {
|
||||
registerRenderRoutes(api, adapter);
|
||||
registerThumbnailRoutes(api, adapter);
|
||||
registerSelectionRoutes(api, adapter);
|
||||
registerMediaRoutes(api, adapter);
|
||||
registerWaveformRoutes(api, adapter);
|
||||
registerFontRoutes(api);
|
||||
registerRegistryRoutes(api, adapter);
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js";
|
||||
|
||||
export type BackgroundRemovalJobOptions = Parameters<
|
||||
NonNullable<StudioApiAdapter["startBackgroundRemoval"]>
|
||||
>[0];
|
||||
|
||||
export type BackgroundRemovalProgressEvent =
|
||||
| { kind: "info"; message: string }
|
||||
| { kind: "metadata"; width: number; height: number; fps: number; frameCount: number }
|
||||
| { kind: "frame"; index: number; total: number; avgMsPerFrame: number };
|
||||
|
||||
export type BackgroundRemovalRender = (options: {
|
||||
inputPath: string;
|
||||
outputPath: string;
|
||||
backgroundOutputPath?: string;
|
||||
device?: BackgroundRemovalJobOptions["device"];
|
||||
quality?: BackgroundRemovalJobOptions["quality"];
|
||||
onProgress?: (event: BackgroundRemovalProgressEvent) => void;
|
||||
}) => Promise<{
|
||||
provider: string;
|
||||
framesProcessed: number;
|
||||
durationSeconds: number;
|
||||
avgMsPerFrame: number;
|
||||
}>;
|
||||
|
||||
export function createBackgroundRemovalJob(
|
||||
opts: BackgroundRemovalJobOptions,
|
||||
render: BackgroundRemovalRender,
|
||||
): MediaProcessingJobState {
|
||||
const state: MediaProcessingJobState = {
|
||||
id: opts.jobId,
|
||||
status: "processing",
|
||||
progress: 0,
|
||||
stage: "Preparing background removal",
|
||||
inputAssetPath: opts.inputAssetPath,
|
||||
outputAssetPath: opts.outputAssetPath,
|
||||
outputPath: opts.outputPath,
|
||||
...(opts.backgroundOutputPath ? { backgroundOutputPath: opts.backgroundOutputPath } : {}),
|
||||
...(opts.backgroundOutputAssetPath
|
||||
? { backgroundOutputAssetPath: opts.backgroundOutputAssetPath }
|
||||
: {}),
|
||||
};
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const result = await render({
|
||||
inputPath: opts.inputPath,
|
||||
outputPath: opts.outputPath,
|
||||
backgroundOutputPath: opts.backgroundOutputPath,
|
||||
device: opts.device,
|
||||
quality: opts.quality,
|
||||
onProgress: (event) => updateBackgroundRemovalProgress(state, event),
|
||||
});
|
||||
state.status = "complete";
|
||||
state.progress = 100;
|
||||
state.stage = "Complete";
|
||||
state.provider = result.provider;
|
||||
state.framesProcessed = result.framesProcessed;
|
||||
state.durationSeconds = result.durationSeconds;
|
||||
state.avgMsPerFrame = result.avgMsPerFrame;
|
||||
} catch (err) {
|
||||
state.status = "failed";
|
||||
state.error = err instanceof Error ? err.message : String(err);
|
||||
state.stage = "Failed";
|
||||
}
|
||||
})();
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
function updateBackgroundRemovalProgress(
|
||||
state: MediaProcessingJobState,
|
||||
event: BackgroundRemovalProgressEvent,
|
||||
): void {
|
||||
if (event.kind === "info") {
|
||||
state.stage = event.message;
|
||||
return;
|
||||
}
|
||||
if (event.kind === "metadata") {
|
||||
state.stage = `Source ${event.width}×${event.height}`;
|
||||
state.progress = 2;
|
||||
return;
|
||||
}
|
||||
state.progress = event.total ? Math.min(99, Math.floor((event.index / event.total) * 100)) : 0;
|
||||
state.stage = event.total
|
||||
? `Removing background ${event.index}/${event.total}`
|
||||
: `Removing background frame ${event.index}`;
|
||||
state.framesProcessed = event.index;
|
||||
state.avgMsPerFrame = event.avgMsPerFrame;
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { classifyMediaColor, probeMediaMetadata } from "./mediaMetadata.js";
|
||||
|
||||
describe("classifyMediaColor", () => {
|
||||
it("detects HDR PQ from BT.2020 + smpte2084 metadata", () => {
|
||||
expect(
|
||||
classifyMediaColor({
|
||||
codec_type: "video",
|
||||
codec_name: "hevc",
|
||||
profile: "Main 10",
|
||||
pix_fmt: "yuv420p10le",
|
||||
color_space: "bt2020nc",
|
||||
color_transfer: "smpte2084",
|
||||
color_primaries: "bt2020",
|
||||
}),
|
||||
).toMatchObject({
|
||||
dynamicRange: "hdr",
|
||||
hdrTransfer: "pq",
|
||||
label: "HDR PQ",
|
||||
isHdr: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("detects HDR HLG from arib-std-b67 metadata", () => {
|
||||
expect(
|
||||
classifyMediaColor({
|
||||
codec_type: "video",
|
||||
color_space: "bt2020nc",
|
||||
color_transfer: "arib-std-b67",
|
||||
color_primaries: "bt2020",
|
||||
}),
|
||||
).toMatchObject({
|
||||
dynamicRange: "hdr",
|
||||
hdrTransfer: "hlg",
|
||||
label: "HDR HLG",
|
||||
isHdr: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("labels BT.709 media as SDR Rec.709", () => {
|
||||
expect(
|
||||
classifyMediaColor({
|
||||
codec_type: "video",
|
||||
color_space: "bt709",
|
||||
color_transfer: "bt709",
|
||||
color_primaries: "bt709",
|
||||
}),
|
||||
).toMatchObject({
|
||||
dynamicRange: "sdr",
|
||||
hdrTransfer: null,
|
||||
label: "SDR Rec.709",
|
||||
isHdr: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("probeMediaMetadata", () => {
|
||||
it("reads the first video stream from ffprobe JSON", () => {
|
||||
const metadata = probeMediaMetadata("/tmp/clip.mp4", () => ({
|
||||
status: 0,
|
||||
stdout: JSON.stringify({
|
||||
streams: [
|
||||
{ codec_type: "audio", codec_name: "aac" },
|
||||
{
|
||||
codec_type: "video",
|
||||
codec_name: "hevc",
|
||||
pix_fmt: "yuv420p10le",
|
||||
color_space: "bt2020nc",
|
||||
color_transfer: "smpte2084",
|
||||
color_primaries: "bt2020",
|
||||
},
|
||||
],
|
||||
}),
|
||||
stderr: "",
|
||||
}));
|
||||
|
||||
expect(metadata).toMatchObject({
|
||||
kind: "video",
|
||||
color: { isHdr: true, label: "HDR PQ" },
|
||||
});
|
||||
});
|
||||
|
||||
it("returns unknown metadata when ffprobe is unavailable", () => {
|
||||
expect(
|
||||
probeMediaMetadata("/tmp/clip.mp4", () => ({
|
||||
status: null,
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
error: { code: "ENOENT" } as NodeJS.ErrnoException,
|
||||
})),
|
||||
).toMatchObject({
|
||||
kind: "video",
|
||||
color: { dynamicRange: "unknown", isHdr: false },
|
||||
probeError: "ffprobe unavailable",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,160 @@
|
||||
import { spawnSync, type SpawnSyncOptions } from "node:child_process";
|
||||
import { extname } from "node:path";
|
||||
|
||||
type FfprobeRunner = (
|
||||
command: string,
|
||||
args: string[],
|
||||
options?: SpawnSyncOptions,
|
||||
) => {
|
||||
status: number | null;
|
||||
stdout: string | Buffer;
|
||||
stderr: string | Buffer;
|
||||
error?: NodeJS.ErrnoException;
|
||||
};
|
||||
|
||||
export type MediaDynamicRange = "hdr" | "sdr" | "unknown";
|
||||
export type MediaHdrTransfer = "pq" | "hlg" | "unknown";
|
||||
|
||||
export interface MediaColorMetadata {
|
||||
dynamicRange: MediaDynamicRange;
|
||||
hdrTransfer: MediaHdrTransfer | null;
|
||||
label: string;
|
||||
isHdr: boolean;
|
||||
codecName?: string;
|
||||
profile?: string;
|
||||
pixelFormat?: string;
|
||||
colorSpace?: string;
|
||||
colorTransfer?: string;
|
||||
colorPrimaries?: string;
|
||||
bitsPerRawSample?: string;
|
||||
}
|
||||
|
||||
export interface MediaMetadata {
|
||||
kind: "video" | "image" | "audio" | "unknown";
|
||||
color: MediaColorMetadata;
|
||||
probeError?: string;
|
||||
}
|
||||
|
||||
interface FfprobeStream {
|
||||
codec_type?: string;
|
||||
codec_name?: string;
|
||||
profile?: string;
|
||||
pix_fmt?: string;
|
||||
color_space?: string;
|
||||
color_transfer?: string;
|
||||
color_primaries?: string;
|
||||
bits_per_raw_sample?: string;
|
||||
}
|
||||
|
||||
const VIDEO_EXT = new Set([".mp4", ".mov", ".webm", ".mkv", ".avi", ".m4v"]);
|
||||
const IMAGE_EXT = new Set([".jpg", ".jpeg", ".png", ".webp", ".avif"]);
|
||||
const AUDIO_EXT = new Set([".mp3", ".wav", ".ogg", ".m4a", ".aac"]);
|
||||
|
||||
function lower(value: string | undefined): string {
|
||||
return value?.toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
function inferKindFromPath(path: string): MediaMetadata["kind"] {
|
||||
const ext = extname(path).toLowerCase();
|
||||
if (VIDEO_EXT.has(ext)) return "video";
|
||||
if (IMAGE_EXT.has(ext)) return "image";
|
||||
if (AUDIO_EXT.has(ext)) return "audio";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function colorLabel(input: {
|
||||
isHdr: boolean;
|
||||
hdrTransfer: MediaHdrTransfer | null;
|
||||
colorPrimaries: string;
|
||||
colorSpace: string;
|
||||
colorTransfer: string;
|
||||
}): string {
|
||||
if (input.isHdr) {
|
||||
if (input.hdrTransfer === "pq") return "HDR PQ";
|
||||
if (input.hdrTransfer === "hlg") return "HDR HLG";
|
||||
return "HDR";
|
||||
}
|
||||
if (
|
||||
input.colorPrimaries.includes("bt709") ||
|
||||
input.colorSpace.includes("bt709") ||
|
||||
input.colorTransfer.includes("bt709")
|
||||
) {
|
||||
return "SDR Rec.709";
|
||||
}
|
||||
return "SDR/unknown";
|
||||
}
|
||||
|
||||
export function classifyMediaColor(stream: FfprobeStream | null | undefined): MediaColorMetadata {
|
||||
const colorPrimaries = lower(stream?.color_primaries);
|
||||
const colorSpace = lower(stream?.color_space);
|
||||
const colorTransfer = lower(stream?.color_transfer);
|
||||
const isHdr =
|
||||
colorPrimaries.includes("bt2020") ||
|
||||
colorSpace.includes("bt2020") ||
|
||||
colorTransfer === "smpte2084" ||
|
||||
colorTransfer === "arib-std-b67";
|
||||
const hdrTransfer: MediaHdrTransfer | null = isHdr
|
||||
? colorTransfer === "smpte2084"
|
||||
? "pq"
|
||||
: colorTransfer === "arib-std-b67"
|
||||
? "hlg"
|
||||
: "unknown"
|
||||
: null;
|
||||
|
||||
return {
|
||||
dynamicRange: stream ? (isHdr ? "hdr" : "sdr") : "unknown",
|
||||
hdrTransfer,
|
||||
label: stream
|
||||
? colorLabel({ isHdr, hdrTransfer, colorPrimaries, colorSpace, colorTransfer })
|
||||
: "Unknown",
|
||||
isHdr,
|
||||
codecName: stream?.codec_name,
|
||||
profile: stream?.profile,
|
||||
pixelFormat: stream?.pix_fmt,
|
||||
colorSpace: stream?.color_space,
|
||||
colorTransfer: stream?.color_transfer,
|
||||
colorPrimaries: stream?.color_primaries,
|
||||
bitsPerRawSample: stream?.bits_per_raw_sample,
|
||||
};
|
||||
}
|
||||
|
||||
export function probeMediaMetadata(
|
||||
filePath: string,
|
||||
runner: FfprobeRunner = spawnSync as unknown as FfprobeRunner,
|
||||
): MediaMetadata {
|
||||
const kind = inferKindFromPath(filePath);
|
||||
if (kind === "audio" || kind === "unknown") {
|
||||
return { kind, color: classifyMediaColor(null) };
|
||||
}
|
||||
|
||||
const result = runner(
|
||||
"ffprobe",
|
||||
[
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"stream=codec_type,codec_name,profile,pix_fmt,color_space,color_transfer,color_primaries,bits_per_raw_sample",
|
||||
"-of",
|
||||
"json",
|
||||
filePath,
|
||||
],
|
||||
{ timeout: 15_000, maxBuffer: 1024 * 1024 },
|
||||
);
|
||||
|
||||
if (result.error?.code === "ENOENT") {
|
||||
return { kind, color: classifyMediaColor(null), probeError: "ffprobe unavailable" };
|
||||
}
|
||||
if (result.status !== 0) {
|
||||
return { kind, color: classifyMediaColor(null), probeError: "ffprobe failed" };
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(String(result.stdout || "{}")) as { streams?: FfprobeStream[] };
|
||||
const stream = parsed.streams?.find((item) =>
|
||||
kind === "image" ? item.codec_type === "video" : item.codec_type === kind,
|
||||
);
|
||||
return { kind, color: classifyMediaColor(stream) };
|
||||
} catch {
|
||||
return { kind, color: classifyMediaColor(null), probeError: "ffprobe returned invalid json" };
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ export type {
|
||||
StudioApiAdapter,
|
||||
ResolvedProject,
|
||||
RenderJobState,
|
||||
MediaProcessingJobState,
|
||||
LintResult,
|
||||
StudioSelectionResponse,
|
||||
StudioSelectionSnapshot,
|
||||
@@ -13,6 +14,10 @@ export { isSafePath, walkDir } from "./helpers/safePath.js";
|
||||
export { getMimeType, MIME_TYPES } from "./helpers/mime.js";
|
||||
export { buildSubCompositionHtml } from "./helpers/subComposition.js";
|
||||
export { getElementScreenshotClip, type ScreenshotClip } from "./helpers/screenshotClip.js";
|
||||
export {
|
||||
createBackgroundRemovalJob,
|
||||
type BackgroundRemovalRender,
|
||||
} from "./helpers/backgroundRemovalJob.js";
|
||||
export {
|
||||
STUDIO_MANUAL_EDITS_PATH,
|
||||
createStudioManualEditsRenderBodyScript,
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
// fallow-ignore-file code-duplication
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { Hono } from "hono";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { registerMediaRoutes } from "./media";
|
||||
import type { MediaProcessingJobState, StudioApiAdapter } from "../types";
|
||||
|
||||
const tempProjectDirs: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempProjectDirs.splice(0)) {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function createAdapter(
|
||||
startBackgroundRemoval?: StudioApiAdapter["startBackgroundRemoval"],
|
||||
probeMediaMetadata?: NonNullable<Parameters<typeof registerMediaRoutes>[2]>["probeMediaMetadata"],
|
||||
): {
|
||||
app: Hono;
|
||||
projectDir: string;
|
||||
startBackgroundRemoval: ReturnType<typeof vi.fn>;
|
||||
} {
|
||||
const projectDir = mkdtempSync(join(tmpdir(), "hf-media-route-test-"));
|
||||
tempProjectDirs.push(projectDir);
|
||||
|
||||
mkdirSync(join(projectDir, "assets"), { recursive: true });
|
||||
writeFileSync(join(projectDir, "assets", "clip.mp4"), "video");
|
||||
writeFileSync(join(projectDir, "assets", "photo.jpg"), "image");
|
||||
|
||||
const spy = vi.fn(startBackgroundRemoval);
|
||||
const adapter: StudioApiAdapter = {
|
||||
listProjects: () => [],
|
||||
resolveProject: async (id: string) => ({ id, dir: projectDir }),
|
||||
bundle: async () => null,
|
||||
lint: async () => ({ findings: [] }),
|
||||
runtimeUrl: "/api/runtime.js",
|
||||
rendersDir: () => "/tmp/renders",
|
||||
startRender: () => ({
|
||||
id: "job-1",
|
||||
status: "rendering",
|
||||
progress: 0,
|
||||
outputPath: "/tmp/out.mp4",
|
||||
}),
|
||||
...(startBackgroundRemoval ? { startBackgroundRemoval: spy } : {}),
|
||||
};
|
||||
const app = new Hono();
|
||||
registerMediaRoutes(app, adapter, probeMediaMetadata ? { probeMediaMetadata } : undefined);
|
||||
return { app, projectDir, startBackgroundRemoval: spy };
|
||||
}
|
||||
|
||||
function completeJob(opts: Parameters<NonNullable<StudioApiAdapter["startBackgroundRemoval"]>>[0]) {
|
||||
return {
|
||||
id: opts.jobId,
|
||||
status: "complete",
|
||||
progress: 100,
|
||||
inputAssetPath: opts.inputAssetPath,
|
||||
outputAssetPath: opts.outputAssetPath,
|
||||
outputPath: opts.outputPath,
|
||||
} satisfies MediaProcessingJobState;
|
||||
}
|
||||
|
||||
describe("registerMediaRoutes", () => {
|
||||
it("returns metadata for a project-local media asset", async () => {
|
||||
const probe = vi.fn(() => ({
|
||||
kind: "video" as const,
|
||||
color: {
|
||||
dynamicRange: "hdr" as const,
|
||||
hdrTransfer: "hlg" as const,
|
||||
label: "HDR HLG",
|
||||
isHdr: true,
|
||||
},
|
||||
}));
|
||||
const { app, projectDir } = createAdapter(undefined, probe);
|
||||
|
||||
const response = await app.request(
|
||||
"http://localhost/projects/demo/media/metadata?path=assets%2Fclip.mp4",
|
||||
);
|
||||
const data = (await response.json()) as { metadata: { color: { label: string } } };
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(data.metadata.color.label).toBe("HDR HLG");
|
||||
expect(probe).toHaveBeenCalledWith(join(projectDir, "assets", "clip.mp4"));
|
||||
});
|
||||
|
||||
it("rejects media metadata paths outside the project", async () => {
|
||||
const { app } = createAdapter();
|
||||
|
||||
const response = await app.request(
|
||||
"http://localhost/projects/demo/media/metadata?path=..%2Fsecret.mp4",
|
||||
);
|
||||
|
||||
expect(response.status).toBe(403);
|
||||
});
|
||||
|
||||
it("rejects null bytes in media metadata paths", async () => {
|
||||
const { app } = createAdapter();
|
||||
|
||||
const response = await app.request(
|
||||
"http://localhost/projects/demo/media/metadata?path=assets%00clip.mp4",
|
||||
);
|
||||
|
||||
expect(response.status).toBe(403);
|
||||
});
|
||||
|
||||
it("returns 501 when background removal is not available", async () => {
|
||||
const { app } = createAdapter();
|
||||
|
||||
const response = await app.request("http://localhost/projects/demo/media/remove-background", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ inputPath: "assets/clip.mp4" }),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(501);
|
||||
});
|
||||
|
||||
it("rejects remote input paths", async () => {
|
||||
const { app, startBackgroundRemoval } = createAdapter(completeJob);
|
||||
|
||||
const response = await app.request("http://localhost/projects/demo/media/remove-background", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ inputPath: "https://example.com/clip.mp4" }),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(startBackgroundRemoval).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects null bytes in background-removal paths", async () => {
|
||||
const { app, startBackgroundRemoval } = createAdapter(completeJob);
|
||||
|
||||
const inputResponse = await app.request(
|
||||
"http://localhost/projects/demo/media/remove-background",
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ inputPath: "assets/clip\0.mp4" }),
|
||||
},
|
||||
);
|
||||
const outputResponse = await app.request(
|
||||
"http://localhost/projects/demo/media/remove-background",
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ inputPath: "assets/clip.mp4", outputPath: "assets/out\0.webm" }),
|
||||
},
|
||||
);
|
||||
|
||||
expect(inputResponse.status).toBe(403);
|
||||
expect(outputResponse.status).toBe(403);
|
||||
expect(startBackgroundRemoval).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("starts a video cutout job with safe default output paths", async () => {
|
||||
const { app, projectDir, startBackgroundRemoval } = createAdapter(completeJob);
|
||||
|
||||
const response = await app.request("http://localhost/projects/demo/media/remove-background", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ inputPath: "assets/clip.mp4", createBackgroundPlate: true }),
|
||||
});
|
||||
const data = (await response.json()) as {
|
||||
jobId: string;
|
||||
outputPath: string;
|
||||
backgroundOutputPath: string;
|
||||
};
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(data.outputPath).toBe("assets/cutouts/clip-cutout.webm");
|
||||
expect(data.backgroundOutputPath).toBe("assets/cutouts/clip-plate.webm");
|
||||
expect(startBackgroundRemoval).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
project: { id: "demo", dir: projectDir },
|
||||
inputAssetPath: "assets/clip.mp4",
|
||||
outputAssetPath: "assets/cutouts/clip-cutout.webm",
|
||||
backgroundOutputAssetPath: "assets/cutouts/clip-plate.webm",
|
||||
quality: "balanced",
|
||||
device: "auto",
|
||||
jobId: data.jobId,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("normalizes query strings from local media paths", async () => {
|
||||
const { app, startBackgroundRemoval } = createAdapter(completeJob);
|
||||
|
||||
const response = await app.request("http://localhost/projects/demo/media/remove-background", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ inputPath: "./assets/clip.mp4?v=123#frame" }),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(startBackgroundRemoval).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
inputAssetPath: "assets/clip.mp4",
|
||||
outputAssetPath: "assets/cutouts/clip-cutout.webm",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("requires png output for image cutouts", async () => {
|
||||
const { app, startBackgroundRemoval } = createAdapter(completeJob);
|
||||
|
||||
const response = await app.request("http://localhost/projects/demo/media/remove-background", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ inputPath: "assets/photo.jpg", outputPath: "assets/photo.webm" }),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(startBackgroundRemoval).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps output paths inside the project", async () => {
|
||||
const { app, startBackgroundRemoval } = createAdapter(completeJob);
|
||||
|
||||
const response = await app.request("http://localhost/projects/demo/media/remove-background", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ inputPath: "assets/clip.mp4", outputPath: "../escape.webm" }),
|
||||
});
|
||||
|
||||
expect(response.status).toBe(403);
|
||||
expect(startBackgroundRemoval).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,255 @@
|
||||
import type { Hono } from "hono";
|
||||
import { streamSSE } from "hono/streaming";
|
||||
import { existsSync, mkdirSync } from "node:fs";
|
||||
import { basename, dirname, extname, join } from "node:path";
|
||||
import type { MediaProcessingJobState, StudioApiAdapter } from "../types.js";
|
||||
import { resolveWithinProject } from "../helpers/safePath.js";
|
||||
import { probeMediaMetadata } from "../helpers/mediaMetadata.js";
|
||||
|
||||
const VIDEO_EXTENSIONS = new Set([".mp4", ".mov", ".webm", ".mkv", ".avi"]);
|
||||
const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".webp"]);
|
||||
const VIDEO_OUTPUT_EXTENSIONS = new Set([".webm", ".mov"]);
|
||||
const QUALITIES = new Set(["fast", "balanced", "best"]);
|
||||
const DEVICES = new Set(["auto", "cpu", "coreml", "cuda"]);
|
||||
|
||||
type BackgroundRemovalQuality = "fast" | "balanced" | "best";
|
||||
type BackgroundRemovalDevice = "auto" | "cpu" | "coreml" | "cuda";
|
||||
|
||||
interface BackgroundRemovalBody {
|
||||
inputPath?: string;
|
||||
outputPath?: string;
|
||||
createBackgroundPlate?: boolean;
|
||||
quality?: string;
|
||||
device?: string;
|
||||
}
|
||||
|
||||
type JobWithCreatedAt = MediaProcessingJobState & { createdAt: number };
|
||||
type ProbeMediaMetadata = typeof probeMediaMetadata;
|
||||
|
||||
function isVideoPath(path: string): boolean {
|
||||
return VIDEO_EXTENSIONS.has(extname(path).toLowerCase());
|
||||
}
|
||||
|
||||
function isImagePath(path: string): boolean {
|
||||
return IMAGE_EXTENSIONS.has(extname(path).toLowerCase());
|
||||
}
|
||||
|
||||
function normalizeProjectAssetPath(path: string): string {
|
||||
return path
|
||||
.trim()
|
||||
.replace(/^[.]\//, "")
|
||||
.replace(/[?#].*$/, "");
|
||||
}
|
||||
|
||||
function containsNullByte(path: string): boolean {
|
||||
return path.includes("\0");
|
||||
}
|
||||
|
||||
function slugFileBase(path: string): string {
|
||||
const name = basename(path, extname(path))
|
||||
.replace(/[^a-zA-Z0-9._-]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "");
|
||||
return name || "media";
|
||||
}
|
||||
|
||||
function uniqueAssetPath(projectDir: string, assetPath: string): string {
|
||||
const ext = extname(assetPath);
|
||||
const withoutExt = assetPath.slice(0, -ext.length);
|
||||
let candidate = assetPath;
|
||||
for (let index = 2; existsSync(join(projectDir, candidate)); index++) {
|
||||
candidate = `${withoutExt}-${index}${ext}`;
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function defaultOutputPath(projectDir: string, inputPath: string): string {
|
||||
const ext = isImagePath(inputPath) ? ".png" : ".webm";
|
||||
return uniqueAssetPath(projectDir, `assets/cutouts/${slugFileBase(inputPath)}-cutout${ext}`);
|
||||
}
|
||||
|
||||
function defaultPlatePath(projectDir: string, inputPath: string): string {
|
||||
return uniqueAssetPath(projectDir, `assets/cutouts/${slugFileBase(inputPath)}-plate.webm`);
|
||||
}
|
||||
|
||||
function makeJobId(projectId: string, mediaJobs: Map<string, JobWithCreatedAt>): string {
|
||||
const stamp = new Date()
|
||||
.toISOString()
|
||||
.replace(/[-:.TZ]/g, "")
|
||||
.slice(0, 14);
|
||||
const safeProject = projectId.replace(/[^a-zA-Z0-9_-]+/g, "-").replace(/^-+|-+$/g, "");
|
||||
const base = `${safeProject || "project"}_remove-bg_${stamp}`;
|
||||
if (!mediaJobs.has(base)) return base;
|
||||
for (let index = 2; ; index++) {
|
||||
const candidate = `${base}-${index}`;
|
||||
if (!mediaJobs.has(candidate)) return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeQuality(value: string | undefined): BackgroundRemovalQuality {
|
||||
return QUALITIES.has(value ?? "") ? (value as BackgroundRemovalQuality) : "balanced";
|
||||
}
|
||||
|
||||
function normalizeDevice(value: string | undefined): BackgroundRemovalDevice {
|
||||
return DEVICES.has(value ?? "") ? (value as BackgroundRemovalDevice) : "auto";
|
||||
}
|
||||
|
||||
export function registerMediaRoutes(
|
||||
api: Hono,
|
||||
adapter: StudioApiAdapter,
|
||||
options: { probeMediaMetadata?: ProbeMediaMetadata } = {},
|
||||
): void {
|
||||
const mediaJobs = new Map<string, JobWithCreatedAt>();
|
||||
const TTL_MS = 300_000;
|
||||
const readMediaMetadata = options.probeMediaMetadata ?? probeMediaMetadata;
|
||||
|
||||
function cleanupFinishedJobs(): void {
|
||||
const now = Date.now();
|
||||
for (const [id, job] of mediaJobs) {
|
||||
if ((job.status === "complete" || job.status === "failed") && now - job.createdAt > TTL_MS) {
|
||||
mediaJobs.delete(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
api.get("/projects/:id/media/metadata", async (c) => {
|
||||
const project = await adapter.resolveProject(c.req.param("id"));
|
||||
if (!project) return c.json({ error: "not found" }, 404);
|
||||
|
||||
const assetPath = normalizeProjectAssetPath(c.req.query("path") ?? "");
|
||||
if (!assetPath) return c.json({ error: "path required" }, 400);
|
||||
if (containsNullByte(assetPath)) return c.json({ error: "forbidden" }, 403);
|
||||
if (/^(?:https?:|data:|blob:)/i.test(assetPath)) {
|
||||
return c.json({ error: "media metadata requires a project-local asset" }, 400);
|
||||
}
|
||||
|
||||
const filePath = resolveWithinProject(project.dir, assetPath);
|
||||
if (!filePath) return c.json({ error: "forbidden" }, 403);
|
||||
if (!existsSync(filePath)) return c.json({ error: "media not found" }, 404);
|
||||
|
||||
return c.json({ path: assetPath, metadata: readMediaMetadata(filePath) });
|
||||
});
|
||||
|
||||
api.post(
|
||||
"/projects/:id/media/remove-background",
|
||||
// fallow-ignore-next-line complexity
|
||||
async (c) => {
|
||||
cleanupFinishedJobs();
|
||||
if (!adapter.startBackgroundRemoval) {
|
||||
return c.json({ error: "background removal is not available in this Studio server" }, 501);
|
||||
}
|
||||
|
||||
// fallow-ignore-next-line code-duplication
|
||||
const project = await adapter.resolveProject(c.req.param("id"));
|
||||
if (!project) return c.json({ error: "not found" }, 404);
|
||||
|
||||
const body = (await c.req.json().catch(() => ({}))) as BackgroundRemovalBody;
|
||||
const inputAssetPath = body.inputPath ? normalizeProjectAssetPath(body.inputPath) : "";
|
||||
if (!inputAssetPath) return c.json({ error: "inputPath required" }, 400);
|
||||
if (containsNullByte(inputAssetPath)) return c.json({ error: "forbidden" }, 403);
|
||||
if (/^(?:https?:|data:|blob:)/i.test(inputAssetPath)) {
|
||||
return c.json({ error: "background removal requires a project-local media asset" }, 400);
|
||||
}
|
||||
|
||||
const inputPath = resolveWithinProject(project.dir, inputAssetPath);
|
||||
if (!inputPath) return c.json({ error: "forbidden" }, 403);
|
||||
if (!existsSync(inputPath)) return c.json({ error: "input media not found" }, 404);
|
||||
|
||||
const inputIsVideo = isVideoPath(inputAssetPath);
|
||||
const inputIsImage = isImagePath(inputAssetPath);
|
||||
if (!inputIsVideo && !inputIsImage) {
|
||||
return c.json({ error: "background removal supports video or image assets only" }, 400);
|
||||
}
|
||||
|
||||
const requestedOutput = body.outputPath ? normalizeProjectAssetPath(body.outputPath) : "";
|
||||
if (requestedOutput && containsNullByte(requestedOutput)) {
|
||||
return c.json({ error: "forbidden" }, 403);
|
||||
}
|
||||
if (requestedOutput && !resolveWithinProject(project.dir, requestedOutput)) {
|
||||
return c.json({ error: "forbidden" }, 403);
|
||||
}
|
||||
const outputAssetPath = requestedOutput
|
||||
? uniqueAssetPath(project.dir, requestedOutput)
|
||||
: defaultOutputPath(project.dir, inputAssetPath);
|
||||
const outputPath = resolveWithinProject(project.dir, outputAssetPath);
|
||||
if (!outputPath) return c.json({ error: "forbidden" }, 403);
|
||||
if (inputIsVideo && !VIDEO_OUTPUT_EXTENSIONS.has(extname(outputAssetPath).toLowerCase())) {
|
||||
return c.json({ error: "video background removal output must be .webm or .mov" }, 400);
|
||||
}
|
||||
if (inputIsImage && extname(outputAssetPath).toLowerCase() !== ".png") {
|
||||
return c.json({ error: "image background removal output must be .png" }, 400);
|
||||
}
|
||||
|
||||
let backgroundOutputAssetPath: string | undefined;
|
||||
let backgroundOutputPath: string | undefined;
|
||||
if (body.createBackgroundPlate) {
|
||||
if (!inputIsVideo) {
|
||||
return c.json({ error: "background plates are only supported for video inputs" }, 400);
|
||||
}
|
||||
backgroundOutputAssetPath = defaultPlatePath(project.dir, inputAssetPath);
|
||||
backgroundOutputPath =
|
||||
resolveWithinProject(project.dir, backgroundOutputAssetPath) ?? undefined;
|
||||
if (!backgroundOutputPath) {
|
||||
return c.json({ error: "forbidden" }, 403);
|
||||
}
|
||||
}
|
||||
|
||||
mkdirSync(dirname(outputPath), { recursive: true });
|
||||
if (backgroundOutputPath) mkdirSync(dirname(backgroundOutputPath), { recursive: true });
|
||||
|
||||
const jobId = makeJobId(project.id, mediaJobs);
|
||||
const state = adapter.startBackgroundRemoval({
|
||||
project,
|
||||
inputPath,
|
||||
inputAssetPath,
|
||||
outputPath,
|
||||
outputAssetPath,
|
||||
backgroundOutputPath,
|
||||
backgroundOutputAssetPath,
|
||||
quality: normalizeQuality(body.quality),
|
||||
device: normalizeDevice(body.device),
|
||||
jobId,
|
||||
}) as JobWithCreatedAt;
|
||||
state.createdAt = Date.now();
|
||||
mediaJobs.set(jobId, state);
|
||||
|
||||
return c.json({
|
||||
jobId,
|
||||
status: state.status,
|
||||
outputPath: outputAssetPath,
|
||||
backgroundOutputPath: backgroundOutputAssetPath,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
api.get("/media-jobs/:jobId/progress", (c) => {
|
||||
cleanupFinishedJobs();
|
||||
const { jobId } = c.req.param();
|
||||
const job = mediaJobs.get(jobId);
|
||||
if (!job) return c.json({ error: "not found" }, 404);
|
||||
|
||||
return streamSSE(c, async (stream) => {
|
||||
while (true) {
|
||||
const current = mediaJobs.get(jobId);
|
||||
if (!current) break;
|
||||
await stream.writeSSE({
|
||||
event: "progress",
|
||||
data: JSON.stringify({
|
||||
id: current.id,
|
||||
status: current.status,
|
||||
progress: current.progress,
|
||||
stage: current.stage,
|
||||
outputPath: current.outputAssetPath,
|
||||
backgroundOutputPath: current.backgroundOutputAssetPath,
|
||||
error: current.error,
|
||||
provider: current.provider,
|
||||
framesProcessed: current.framesProcessed,
|
||||
durationSeconds: current.durationSeconds,
|
||||
avgMsPerFrame: current.avgMsPerFrame,
|
||||
}),
|
||||
});
|
||||
if (current.status === "complete" || current.status === "failed") break;
|
||||
await stream.sleep(500);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -19,6 +19,23 @@ export interface RenderJobState {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface MediaProcessingJobState {
|
||||
id: string;
|
||||
status: "processing" | "complete" | "failed";
|
||||
progress: number;
|
||||
stage?: string;
|
||||
inputAssetPath: string;
|
||||
outputAssetPath: string;
|
||||
outputPath: string;
|
||||
backgroundOutputAssetPath?: string;
|
||||
backgroundOutputPath?: string;
|
||||
error?: string;
|
||||
provider?: string;
|
||||
framesProcessed?: number;
|
||||
durationSeconds?: number;
|
||||
avgMsPerFrame?: number;
|
||||
}
|
||||
|
||||
/** Lint result from the core linter. */
|
||||
export interface LintResult {
|
||||
findings: Array<{
|
||||
@@ -137,6 +154,19 @@ export interface StudioApiAdapter {
|
||||
distinctId?: string;
|
||||
}): RenderJobState;
|
||||
|
||||
startBackgroundRemoval?: (opts: {
|
||||
project: ResolvedProject;
|
||||
inputPath: string;
|
||||
inputAssetPath: string;
|
||||
outputPath: string;
|
||||
outputAssetPath: string;
|
||||
backgroundOutputPath?: string;
|
||||
backgroundOutputAssetPath?: string;
|
||||
quality: "fast" | "balanced" | "best";
|
||||
device?: "auto" | "cpu" | "coreml" | "cuda";
|
||||
jobId: string;
|
||||
}) => MediaProcessingJobState;
|
||||
|
||||
/** Optional: generate a JPEG thumbnail via Puppeteer or similar. */
|
||||
generateThumbnail?: (opts: {
|
||||
project: ResolvedProject;
|
||||
|
||||
@@ -15,6 +15,8 @@ import {
|
||||
type ResolvedProject,
|
||||
type RenderJobState,
|
||||
type StudioApiAdapter,
|
||||
type BackgroundRemovalRender,
|
||||
createBackgroundRemovalJob,
|
||||
createProjectSignature,
|
||||
} from "@hyperframes/studio-server";
|
||||
import type { RegistryItem } from "@hyperframes/core/registry";
|
||||
@@ -32,7 +34,10 @@ export function isPathWithin(parentDir: string, childPath: string): boolean {
|
||||
|
||||
export function createViteAdapter(dataDir: string, server: ViteDevServer): StudioApiAdapter {
|
||||
let _bundler:
|
||||
| ((dir: string, options?: { runtime?: "inline" | "placeholder" }) => Promise<string>)
|
||||
| ((
|
||||
dir: string,
|
||||
options?: { runtime?: "inline" | "placeholder"; inlineColorGradingLuts?: boolean },
|
||||
) => Promise<string>)
|
||||
| null = null;
|
||||
let _producerModuleLoader:
|
||||
| (() => Promise<{
|
||||
@@ -162,7 +167,7 @@ export function createViteAdapter(dataDir: string, server: ViteDevServer): Studi
|
||||
async bundle(dir: string) {
|
||||
const bundler = await getBundler();
|
||||
if (!bundler) return null;
|
||||
let html = await bundler(dir, { runtime: "placeholder" });
|
||||
let html = await bundler(dir, { runtime: "placeholder", inlineColorGradingLuts: false });
|
||||
html = html.replace(
|
||||
'data-hyperframes-preview-runtime="1" src=""',
|
||||
`data-hyperframes-preview-runtime="1" src="${this.runtimeUrl}"`,
|
||||
@@ -251,6 +256,16 @@ export function createViteAdapter(dataDir: string, server: ViteDevServer): Studi
|
||||
return state;
|
||||
},
|
||||
|
||||
startBackgroundRemoval(opts) {
|
||||
return createBackgroundRemovalJob(opts, async (renderOpts) => {
|
||||
const mod = await server.ssrLoadModule(
|
||||
resolve(__dirname, "../cli/src/background-removal/pipeline.ts"),
|
||||
);
|
||||
const render = mod.render as BackgroundRemovalRender;
|
||||
return render(renderOpts);
|
||||
});
|
||||
},
|
||||
|
||||
async generateThumbnail(opts) {
|
||||
return generateThumbnail(opts);
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user