From e8371a7accfb1ccd88c792898b65910aec60b0bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 17 Jul 2026 03:26:55 -0400 Subject: [PATCH] feat(media): alpha-capable authoring proxies (#2598) * feat(media): alpha-capable authoring proxies Alpha sources were refused a proxy before the codec map ever asked whether the browser could decode them, so a ProRes 4444 alpha file (which no browser previews at all) rendered black forever, while an alpha WebM (which previews fine) was already covered by the browser-safe check on the next line. The alpha veto earned nothing and cost the one case that needed help. Alpha is now a target-codec choice rather than a veto: alpha sources transcode to VP9 + yuva420p in WebM, everything else keeps the existing H.264/MP4 path byte for byte. Only files no browser can preview are proxied, which is the rule the runtime already followed everywhere else. WebM cannot carry AAC, so the VP9 path uses Opus and drops the MP4-only faststart flag. PROXY_PARAMS_VERSION moves to v3 so clients stop serving the previously cached proxies. Safari does not decode VP9 alpha and still shows black for alpha sources, as it does today: this is better on Chromium and Firefox and no worse anywhere. * fix(media): infer proxy variant for rescue * fix(media): preserve alpha proxy hardening after restack --- packages/cli/src/commands/play.test.ts | 43 +++++-- packages/cli/src/commands/play.ts | 30 +++-- packages/cli/src/commands/preview.ts | 2 +- packages/cli/src/utils/checkBrowser.test.ts | 15 ++- packages/cli/src/utils/checkBrowser.ts | 17 ++- packages/cli/src/utils/projectConfig.ts | 2 +- .../cli/src/utils/publishProxyBake.test.ts | 31 +++-- packages/cli/src/utils/publishProxyBake.ts | 28 ++--- .../cli/src/utils/staticProjectServer.test.ts | 51 +++++--- packages/cli/src/utils/staticProjectServer.ts | 77 ++++++++---- packages/core/src/runtime/mediaProxy.test.ts | 31 ++--- packages/core/src/runtime/mediaProxy.ts | 45 ++----- .../src/helpers/mediaCodecMap.test.ts | 57 ++++++++- .../src/helpers/mediaCodecMap.ts | 45 ++++++- .../src/helpers/mediaProxyPreview.ts | 20 +-- .../src/helpers/proxyCache.test.ts | 20 +++ .../studio-server/src/helpers/proxyCache.ts | 8 +- .../src/helpers/proxyTranscoder.test.ts | 53 ++++++++ .../src/helpers/proxyTranscoder.ts | 119 +++++++++++++----- .../studio-server/src/routes/preview.test.ts | 115 +++++++++++++---- packages/studio-server/src/routes/preview.ts | 32 +++-- 21 files changed, 601 insertions(+), 240 deletions(-) diff --git a/packages/cli/src/commands/play.test.ts b/packages/cli/src/commands/play.test.ts index 95d493ac4..68f1d9675 100644 --- a/packages/cli/src/commands/play.test.ts +++ b/packages/cli/src/commands/play.test.ts @@ -59,10 +59,20 @@ const mediaMocks = vi.hoisted(() => ({ representativeMime: "video/mp4", hasAlpha: false, })), - decideMediaProxyEligibility: vi.fn( - (facts: { hasAlpha: boolean; browserHostile: boolean } | null) => - facts?.hasAlpha ? { eligible: false, reason: "alpha_source" } : { eligible: true }, + decideMediaProxyEligibility: vi.fn((facts: { browserHostile: boolean } | null) => + facts?.browserHostile ? { eligible: true } : { eligible: false, reason: "browser_safe_codec" }, ), + isProxyVariant: (value: string) => value === "h264" || value === "vp9", + isProxyVariantRequest: (value: string) => value === "auto" || value === "h264" || value === "vp9", + proxyVariantFor: (facts: { hasAlpha?: boolean }) => (facts.hasAlpha ? "vp9" : "h264"), + resolveProxyVariantRequest: (request: "auto" | "h264" | "vp9", facts: { hasAlpha?: boolean }) => { + const expected = facts.hasAlpha ? "vp9" : "h264"; + return request === "auto" || request === expected ? expected : null; + }, + PROXY_VARIANT_CONFIG: { + h264: { extension: ".mp4", contentType: "video/mp4" }, + vp9: { extension: ".webm", contentType: "video/webm" }, + }, })); vi.mock("@hyperframes/studio-server/proxy-transcoder", () => ({ @@ -109,7 +119,7 @@ afterEach(() => { dir = undefined; }); -it("rejects direct proxy requests for alpha sources", async () => { +it("serves direct VP9 proxy requests for alpha sources", async () => { const project = tmpProject(); writeFileSync(join(project.dir, "clip.mov"), "alpha-prores"); mediaMocks.probeAssetCodec.mockResolvedValueOnce({ @@ -118,17 +128,20 @@ it("rejects direct proxy requests for alpha sources", async () => { representativeMime: "video/quicktime", hasAlpha: true, }); - mediaMocks.decideMediaProxyEligibility.mockReturnValueOnce({ - eligible: false, - reason: "alpha_source", - }); + const proxyPath = join(project.dir, "proxy.webm"); + writeFileSync(proxyPath, "vp9-alpha-proxy"); + mocks.resolveProxy.mockResolvedValueOnce(proxyPath); const app = await buildApp(project, true); - const res = await app.request("/composition/clip.mov?hf-proxy=h264"); + const res = await app.request("/composition/clip.mov?hf-proxy=auto"); - expect(res.status).toBe(422); - expect(await res.text()).toContain("alpha_source"); - expect(mocks.resolveProxy).not.toHaveBeenCalled(); + expect(res.status).toBe(200); + expect(res.headers.get("content-type")).toBe("video/webm"); + expect(mocks.resolveProxy).toHaveBeenCalledWith( + project.dir, + join(project.dir, "clip.mov"), + "vp9", + ); }); async function buildApp(project: ProjectDir, autoProxy: boolean): Promise { @@ -163,7 +176,11 @@ describe("registerCompositionRoute", () => { expect(res.status).toBe(200); expect(await res.text()).toBe("transcoded-h264-bytes"); - expect(mocks.resolveProxy).toHaveBeenCalledWith(project.dir, join(project.dir, "clip.mp4")); + expect(mocks.resolveProxy).toHaveBeenCalledWith( + project.dir, + join(project.dir, "clip.mp4"), + "h264", + ); }); it("serves ?hf-proxy=h264 for a .mov hostile asset as Content-Type video/mp4 (the proxy IS mp4)", async () => { diff --git a/packages/cli/src/commands/play.ts b/packages/cli/src/commands/play.ts index c2b2af88f..c7171aaa3 100644 --- a/packages/cli/src/commands/play.ts +++ b/packages/cli/src/commands/play.ts @@ -44,7 +44,10 @@ import { } from "@hyperframes/studio-server/proxy-transcoder"; import { decideMediaProxyEligibility, + isProxyVariantRequest, probeAssetCodec, + resolveProxyVariantRequest, + PROXY_VARIANT_CONFIG, } from "@hyperframes/studio-server/media-codec-map"; export default defineCommand({ @@ -72,7 +75,7 @@ export default defineCommand({ proxy: { type: "boolean", description: - "Auto-transcode browser-hostile video codecs (HEVC, ProRes, AV1) to a cached H.264 proxy for preview (default: on; overrides hyperframes.json's media.autoProxy)", + "Auto-transcode browser-hostile video codecs (HEVC, ProRes, AV1) to a cached authoring proxy for preview (default: on; overrides hyperframes.json's media.autoProxy)", negativeDescription: "Disable auto-proxying of browser-hostile video codecs", }, }, @@ -192,8 +195,8 @@ export default defineCommand({ * Registers the `/composition/*` route: serves composition HTML (runtime + * `__HF_MEDIA_CODEC_MAP__` injected) and asset files, with byte-Range support * (`play` previously did a whole-file `readFileSync`, so seeking/duration - * probing on media elements never worked) and a `?hf-proxy=h264` branch that - * serves the cached H.264 authoring proxy for a browser-hostile video asset + * probing on media elements never worked) and a `?hf-proxy=` branch that + * serves the alpha-aware authoring proxy for a browser-hostile video asset * (per docs/plans/2026-07-14-002-feat-transparent-media-proxies-plan.md, * unit U4). Exported standalone (rather than inlined in `run()`) so tests can * exercise it via `app.request(...)` without booting a real HTTP listener, @@ -228,19 +231,28 @@ export async function registerCompositionRoute( } const contentType = assetContentType(filePath); - if (ctx.req.query("hf-proxy") === "h264") { + const proxyParam = ctx.req.query("hf-proxy"); + if (proxyParam !== undefined && isProxyVariantRequest(proxyParam)) { // Opt-out (or a non-video asset) 404s the param without attempting a // transcode; a missing asset already 404'd above. if (!autoProxy || !contentType.startsWith("video/")) return ctx.text("Not found", 404); try { - const eligibility = decideMediaProxyEligibility(await probeAssetCodec(filePath)); + const facts = await probeAssetCodec(filePath); + const eligibility = decideMediaProxyEligibility(facts); if (!eligibility.eligible) { return ctx.text(`Media proxy unavailable: ${eligibility.reason}`, 422); } - const proxyPath = await resolveProxy(project.dir, filePath); - // The proxy IS an mp4 regardless of the source's extension (.mov, - // .mkv, ...) — serve its real type, matching the preview route. - return buildRangeResponse(proxyPath, "video/mp4", ctx.req.header("Range")); + if (!facts) return ctx.text("Media proxy unavailable: unknown_codec", 422); + const proxyVariant = resolveProxyVariantRequest(proxyParam, facts); + if (!proxyVariant) { + return ctx.text("Media proxy variant does not match asset", 422); + } + const proxyPath = await resolveProxy(project.dir, filePath, proxyVariant); + return buildRangeResponse( + proxyPath, + PROXY_VARIANT_CONFIG[proxyVariant].contentType, + ctx.req.header("Range"), + ); } catch (err) { if (err instanceof ProxyCapacityError) { return ctx.text(`Proxy transcode deferred: ${err.message}`, 503, { diff --git a/packages/cli/src/commands/preview.ts b/packages/cli/src/commands/preview.ts index 43140cdc0..bcfc7fab8 100644 --- a/packages/cli/src/commands/preview.ts +++ b/packages/cli/src/commands/preview.ts @@ -192,7 +192,7 @@ export default defineCommand({ proxy: { type: "boolean", description: - "Auto-transcode browser-hostile video codecs (HEVC, ProRes, AV1) to a cached H.264 proxy for preview (default: on; overrides hyperframes.json's media.autoProxy)", + "Auto-transcode browser-hostile video codecs (HEVC, ProRes, AV1) to a cached authoring proxy for preview (default: on; overrides hyperframes.json's media.autoProxy)", negativeDescription: "Disable auto-proxying of browser-hostile video codecs", }, }, diff --git a/packages/cli/src/utils/checkBrowser.test.ts b/packages/cli/src/utils/checkBrowser.test.ts index 17babf7cb..25127376e 100644 --- a/packages/cli/src/utils/checkBrowser.test.ts +++ b/packages/cli/src/utils/checkBrowser.test.ts @@ -69,6 +69,7 @@ vi.mock("./staticProjectServer.js", () => ({ vi.mock("@hyperframes/studio-server/media-codec-map", async (importOriginal) => ({ ...(await importOriginal()), scanProjectMediaCodecMap: mocks.scanProjectMediaCodecMap, + proxyVariantFor: (facts: { hasAlpha?: boolean }) => (facts.hasAlpha ? "vp9" : "h264"), })); vi.mock("@hyperframes/studio-server/proxy-transcoder", () => ({ resolveProxy: mocks.resolveProxy, @@ -341,7 +342,7 @@ it("surfaces the runtime's media-proxy-fallback console.info line as an info fin const fallbackMessage = fakeConsoleMessage( "info", '[hyperframes] runtime_media_proxy_fallback: "assets/clip.mp4" uses a codec (hevc) this browser can\'t decode; ' + - "auto-swapped to an H.264 proxy for this preview only. Render output is unaffected.", + "auto-swapped to an authoring proxy for this preview only. Render output is unaffected.", ); const unrelatedInfo = fakeConsoleMessage("info", "[hyperframes] render runtime fps 30"); const authorInfo = fakeConsoleMessage("info", "debug runtime_media_proxy_probe"); @@ -455,7 +456,11 @@ describe("preResolveHostileMediaProxies", () => { await Promise.resolve(); await Promise.resolve(); expect(mocks.resolveProxy).toHaveBeenCalledTimes(1); - expect(mocks.resolveProxy).toHaveBeenCalledWith(projectDir, join(projectDir, "clip.mp4")); + expect(mocks.resolveProxy).toHaveBeenCalledWith( + projectDir, + join(projectDir, "clip.mp4"), + "h264", + ); expect(settled).toBe(false); // still waiting on the hostile entry's transcode resolveTranscode?.(); @@ -500,10 +505,10 @@ describe("preResolveHostileMediaProxies", () => { it("does not pre-resolve a hostile asset rejected by the shared proxy policy", async () => { const projectDir = mkProjectDir(); mocks.scanProjectMediaCodecMap.mockResolvedValue({ - "/alpha.mov": { - codecName: "prores", + "/alpha.webm": { + codecName: "vp9", browserHostile: true, - representativeMime: null, + representativeMime: 'video/webm; codecs="vp09.00.10.08"', hasAlpha: true, }, }); diff --git a/packages/cli/src/utils/checkBrowser.ts b/packages/cli/src/utils/checkBrowser.ts index c161fe711..427719e23 100644 --- a/packages/cli/src/utils/checkBrowser.ts +++ b/packages/cli/src/utils/checkBrowser.ts @@ -21,6 +21,7 @@ import { serveStaticProjectHtml } from "./staticProjectServer.js"; import { resolveAutoProxy } from "./projectConfig.js"; import { decideMediaProxyEligibility, + proxyVariantFor, scanProjectMediaCodecMap, } from "@hyperframes/studio-server/media-codec-map"; import { resolveProxy } from "@hyperframes/studio-server/proxy-transcoder"; @@ -115,15 +116,19 @@ export async function preResolveHostileMediaProxies( ); return; } - const hostilePathnames = Object.entries(codecMap) - .filter(([, facts]) => decideMediaProxyEligibility(facts).eligible) - .map(([pathname]) => pathname); - if (hostilePathnames.length === 0) return; + const hostileEntries = Object.entries(codecMap).filter( + ([, facts]) => decideMediaProxyEligibility(facts).eligible, + ); + if (hostileEntries.length === 0) return; const startedAt = Date.now(); const results = await Promise.allSettled( - hostilePathnames.map((pathname) => - resolveProxy(projectDir, resolve(projectDir, pathname.replace(/^\/+/, ""))), + hostileEntries.map(([pathname, facts]) => + resolveProxy( + projectDir, + resolve(projectDir, pathname.replace(/^\/+/, "")), + proxyVariantFor(facts), + ), ), ); const failed = results.filter((result) => result.status === "rejected").length; diff --git a/packages/cli/src/utils/projectConfig.ts b/packages/cli/src/utils/projectConfig.ts index d9dc78759..e16e72848 100644 --- a/packages/cli/src/utils/projectConfig.ts +++ b/packages/cli/src/utils/projectConfig.ts @@ -26,7 +26,7 @@ export interface ProjectConfigPaths { export interface ProjectConfigMedia { /** * Auto-transcode browser-hostile video codecs (e.g. HEVC) to a cached - * H.264 proxy for supported preview surfaces. Render always uses the + * alpha-aware authoring proxy for supported preview surfaces. Render always uses the * original file regardless of this setting. Default true. */ autoProxy?: boolean; diff --git a/packages/cli/src/utils/publishProxyBake.test.ts b/packages/cli/src/utils/publishProxyBake.test.ts index 7e35c5d29..db0c93395 100644 --- a/packages/cli/src/utils/publishProxyBake.test.ts +++ b/packages/cli/src/utils/publishProxyBake.test.ts @@ -49,6 +49,7 @@ vi.mock("@hyperframes/studio-server/proxy-transcoder", () => ({ vi.mock("@hyperframes/studio-server/media-codec-map", () => ({ scanProjectMediaCodecMap: mocks.scanProjectMediaCodecMap, + proxyVariantFor: (facts: { hasAlpha?: boolean }) => (facts.hasAlpha ? "vp9" : "h264"), })); const { bakeMediaProxies, PROXY_ARCHIVE_PREFIX } = await import("./publishProxyBake.js"); @@ -60,10 +61,10 @@ const { bakeMediaProxies, PROXY_ARCHIVE_PREFIX } = await import("./publishProxyB const PROJECT_DIR = resolve("/project"); const tempDirs: string[] = []; -function tmpProxyFile(content: string): string { +function tmpProxyFile(content: string, extension = ".mp4"): string { const dir = mkdtempSync(join(tmpdir(), "hf-publish-proxy-bake-")); tempDirs.push(dir); - const path = join(dir, "proxy.mp4"); + const path = join(dir, `proxy${extension}`); writeFileSync(path, content, "utf-8"); return path; } @@ -108,7 +109,11 @@ describe("bakeMediaProxies", () => { expect(html).toContain(proxyEntries[0]!); expect(html).not.toContain('src="clip.mp4"'); - expect(mocks.resolveProxy).toHaveBeenCalledWith(PROJECT_DIR, join(PROJECT_DIR, "clip.mp4")); + expect(mocks.resolveProxy).toHaveBeenCalledWith( + PROJECT_DIR, + join(PROJECT_DIR, "clip.mp4"), + "h264", + ); expect(mocks.waitForProxy).toHaveBeenCalledWith(expect.any(Promise), 15 * 60 * 1000); expect(manifest).toEqual({ proxied: ["/clip.mp4"], skippedAlpha: [], failed: [] }); }); @@ -191,7 +196,7 @@ describe("bakeMediaProxies", () => { expect(html).not.toContain("assets/my%20clip.mp4"); }); - it("reports an alpha-bearing hostile asset as skipped while keeping HTML on the original", async () => { + it("bakes an alpha-bearing hostile asset as a VP9 WebM proxy", async () => { mocks.scanProjectMediaCodecMap.mockResolvedValue({ "/clip.mov": { codecName: "prores", @@ -200,6 +205,8 @@ describe("bakeMediaProxies", () => { hasAlpha: true, }, }); + const proxyPath = tmpProxyFile("PROXY_VP9_ALPHA_BYTES", ".webm"); + mocks.resolveProxy.mockResolvedValue(proxyPath); const fileContents = new Map([ ["index.html", indexHtml(``)], ["clip.mov", Buffer.from("ORIGINAL_PRORES_4444_BYTES", "utf-8")], @@ -207,12 +214,18 @@ describe("bakeMediaProxies", () => { const manifest = await bakeMediaProxies(PROJECT_DIR, fileContents); - expect(mocks.resolveProxy).not.toHaveBeenCalled(); - expect([...fileContents.keys()].some((k) => k.startsWith(`${PROXY_ARCHIVE_PREFIX}/`))).toBe( - false, + expect(mocks.resolveProxy).toHaveBeenCalledWith( + PROJECT_DIR, + join(PROJECT_DIR, "clip.mov"), + "vp9", ); - expect(fileContents.get("index.html")?.toString("utf-8")).toContain('src="clip.mov"'); - expect(manifest).toEqual({ proxied: [], skippedAlpha: ["/clip.mov"], failed: [] }); + const proxyEntries = [...fileContents.keys()].filter((key) => + key.startsWith(`${PROXY_ARCHIVE_PREFIX}/`), + ); + expect(proxyEntries).toEqual([`${PROXY_ARCHIVE_PREFIX}/proxy.webm`]); + expect(fileContents.get(proxyEntries[0]!)?.toString("utf-8")).toBe("PROXY_VP9_ALPHA_BYTES"); + expect(fileContents.get("index.html")?.toString("utf-8")).toContain(proxyEntries[0]!); + expect(manifest).toEqual({ proxied: ["/clip.mov"], skippedAlpha: [], failed: [] }); }); it("returns deterministic manifest ordering across concurrent transcodes", async () => { diff --git a/packages/cli/src/utils/publishProxyBake.ts b/packages/cli/src/utils/publishProxyBake.ts index c4b331be4..9827d56b6 100644 --- a/packages/cli/src/utils/publishProxyBake.ts +++ b/packages/cli/src/utils/publishProxyBake.ts @@ -2,7 +2,7 @@ * Publish-time proxy baking (U6 of * docs/plans/2026-07-14-002-feat-transparent-media-proxies-plan.md). * - * Published pages are static (no server), so the on-demand `?hf-proxy=h264` + * Published pages are static (no server), so the on-demand `?hf-proxy=` * negotiation the preview/play surfaces use (U3/U4) isn't possible there. * Instead this scans the archive's HTML entries for local `