From 0561adc11c7bd6a375244170893f530cc51a9587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Fri, 17 Jul 2026 02:05:25 -0400 Subject: [PATCH] feat(cli): resolve proxies before check's timed browser phase (#2594) * feat(studio-server): serve H.264 proxies from the preview route Wires the codec manifest and the transcoder into the preview surface: the route negotiates a proxy via a query param and serves it through the existing range and ETag machinery, composition HTML carries a codec map for the runtime, and hostile assets pre-warm so a first play does not wait on a cold transcode. Exposes the three subpath exports the CLI surfaces consume upstack. Drops the TEMP fallow entry added with the transcoder: it has real importers now. * fix(studio-server): publish media proxy exports * fix(parsers): scan HTML comments linearly * feat(cli): let projects opt out of automatic proxying Adds media.autoProxy to hyperframes.json plus --proxy/--no-proxy flags, and forwards the resolved value into the studio and preview servers and the vite adapter. Lands before the runtime slice that turns auto-proxying on, so the switch exists before there is any behavior to switch off. * fix(cli): align media config schema * feat(core): swap undecodable video to its proxy at runtime Adds the browser-side half: before first load the runtime consults the injected codec map and swaps a hostile source to its proxy, and if a video still reports zero decodable width it rescues it reactively. An HEVC file carrying AAC fires no error event, so zero videoWidth, not the error event, is the reliable signal. Audio elements and alpha sources are never proxied, render mode never proxies, and each swap evicts the element's stale sync state and reports once. This completes the loop: auto-proxying is live for preview and studio from here. The opt-out (media.autoProxy, --no-proxy) shipped in the previous slice. * feat(cli): serve proxies from play, present, and the static project server Adds proxy negotiation to the CLI-side servers and gives play byte-range serving it never had, so a swapped video can seek. The static project server behind check, snapshot, compare and friends injects the codec map once, so all of its callers inherit the behavior; snapshot forwards its own proxy flag. * fix(cli): serve proxies for camera formats * feat(cli): resolve proxies before check's timed browser phase check pre-resolves hostile assets so a cold transcode cannot exhaust the render-ready budget, and surfaces the runtime's proxy diagnostics as findings so a swap is visible rather than silent. * fix(cli): harden proxy pre-resolution --- packages/cli/src/commands/check.test.ts | 15 ++ packages/cli/src/commands/check.ts | 7 + packages/cli/src/utils/checkBrowser.test.ts | 237 +++++++++++++++++++- packages/cli/src/utils/checkBrowser.ts | 90 +++++++- packages/cli/src/utils/checkTypes.ts | 2 + 5 files changed, 347 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/commands/check.test.ts b/packages/cli/src/commands/check.test.ts index 06af01eb1..2b49f1a94 100644 --- a/packages/cli/src/commands/check.test.ts +++ b/packages/cli/src/commands/check.test.ts @@ -354,6 +354,21 @@ it("parses the caption-zone grammar and enables the frame gate", async () => { ); }); +it("threads --no-proxy into the browser check options", async () => { + const { report } = await runScenario(fakeDriver()); + const runPipeline = vi.fn(async (_project: ProjectDir, _options: CheckOptions) => report); + vi.spyOn(console, "log").mockImplementation(() => undefined); + const command = createCheckCommand({ + resolveProject: () => PROJECT, + runPipeline, + withMeta: (value) => value, + }); + + await runCommand(command, { rawArgs: ["--json", "--no-proxy"] }); + + expect(runPipeline).toHaveBeenCalledWith(PROJECT, expect.objectContaining({ autoProxy: false })); +}); + it("rejects malformed caption-zone specs instead of silently disabling the gate", async () => { const { report } = await runScenario(fakeDriver()); const runPipeline = vi.fn(async () => report); diff --git a/packages/cli/src/commands/check.ts b/packages/cli/src/commands/check.ts index a0adf3767..d480fde36 100644 --- a/packages/cli/src/commands/check.ts +++ b/packages/cli/src/commands/check.ts @@ -98,6 +98,12 @@ export function createCheckCommand( description: "Exit non-zero on warnings too", default: false, }, + proxy: { + type: "boolean", + description: + "Auto-transcode browser-hostile video codecs (default: hyperframes.json media.autoProxy, which defaults on)", + default: undefined, + }, snapshots: { type: "boolean", description: "Save the five contrast-pass PNGs under snapshots/", @@ -161,6 +167,7 @@ function parseCheckOptions(args: Record): CheckOptions { snapshots: args.snapshots === true, captionZone: parseCaptionZone(args["caption-zone"]), frameCheck: parseFrameCheck(args["frame-check"]), + autoProxy: args.proxy as boolean | undefined, }; } diff --git a/packages/cli/src/utils/checkBrowser.test.ts b/packages/cli/src/utils/checkBrowser.test.ts index 333cdad75..17babf7cb 100644 --- a/packages/cli/src/utils/checkBrowser.test.ts +++ b/packages/cli/src/utils/checkBrowser.test.ts @@ -1,16 +1,37 @@ // @vitest-environment happy-dom import { afterEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { openSettledCompositionPage, type OpenSettledCompositionPageOptions, } from "../capture/captureCompositionFrame.js"; import { DEFAULT_CHECK_OPTIONS, runAuditGrid } from "./checkPipeline.js"; -import { captureOverviewShot, runBrowserCheck } from "./checkBrowser.js"; +import { + captureOverviewShot, + preResolveHostileMediaProxies, + runBrowserCheck, +} from "./checkBrowser.js"; import type { ProjectDir } from "./project.js"; const mocks = vi.hoisted(() => ({ bundleWithLocalizedFonts: vi.fn(async () => ""), serverClose: vi.fn(async () => undefined), + resolveProxy: vi.fn<(projectDir: string, absoluteSourcePath: string) => Promise>(), + scanProjectMediaCodecMap: vi.fn< + (...args: unknown[]) => Promise< + Record< + string, + { + codecName: string; + browserHostile: boolean; + representativeMime: string | null; + hasAlpha: boolean; + } + > + > + >(async () => ({})), })); vi.mock("./bundleWithLocalizedFonts.js", () => ({ @@ -44,6 +65,15 @@ vi.mock("./staticProjectServer.js", () => ({ })), })); +// `preResolveHostileMediaProxies` reaches these two studio-server helpers via +vi.mock("@hyperframes/studio-server/media-codec-map", async (importOriginal) => ({ + ...(await importOriginal()), + scanProjectMediaCodecMap: mocks.scanProjectMediaCodecMap, +})); +vi.mock("@hyperframes/studio-server/proxy-transcoder", () => ({ + resolveProxy: mocks.resolveProxy, +})); + const PROJECT: ProjectDir = { dir: "/project", name: "project", @@ -61,6 +91,7 @@ afterEach(() => { Reflect.deleteProperty(window, "__contrastAuditFinish"); Reflect.deleteProperty(window, "__contrastAuditRestores"); Reflect.deleteProperty(window, "__contrastAuditRestoreIfPending"); + mocks.scanProjectMediaCodecMap.mockResolvedValue({}); }); function installSessionMock(page: ReturnType): void { @@ -303,6 +334,202 @@ it("carries validate's clip-duration audit into the runtime findings", async () ]); }); +it("surfaces the runtime's media-proxy-fallback console.info line as an info finding, ignoring unrelated info logs", async () => { + vi.spyOn(Date, "now").mockReturnValue(100); + mountCanvasFixture(); + const page = fakePage(); + const fallbackMessage = fakeConsoleMessage( + "info", + '[hyperframes] runtime_media_proxy_fallback: "assets/clip.mp4" uses a codec (hevc) this browser can\'t decode; ' + + "auto-swapped to an H.264 proxy for this preview only. Render output is unaffected.", + ); + const unrelatedInfo = fakeConsoleMessage("info", "[hyperframes] render runtime fps 30"); + const authorInfo = fakeConsoleMessage("info", "debug runtime_media_proxy_probe"); + page.on = vi.fn( + (event: string, handler: (message: ReturnType) => void) => { + if (event === "console") { + handler(fallbackMessage); + handler(unrelatedInfo); + handler(authorInfo); + } + }, + ); + installSessionMock(page); + + const result = await runBrowserCheck( + PROJECT, + { ...DEFAULT_CHECK_OPTIONS, samples: 1, contrast: false }, + { kind: "none" }, + runAuditGrid, + ); + + expect(result.runtimeFindings).toContainEqual( + expect.objectContaining({ + code: "media_proxy_fallback", + severity: "info", + message: fallbackMessage.text(), + }), + ); + expect(result.runtimeFindings.some((finding) => finding.message === unrelatedInfo.text())).toBe( + false, + ); + expect(result.runtimeFindings.some((finding) => finding.message === authorInfo.text())).toBe( + false, + ); +}); + +it("surfaces the runtime's media-proxy-unavailable console.info line as its own info finding", async () => { + vi.spyOn(Date, "now").mockReturnValue(100); + mountCanvasFixture(); + const page = fakePage(); + const unavailableMessage = fakeConsoleMessage( + "info", + '[hyperframes] runtime_media_proxy_unavailable: "https://cdn.example.com/video.mp4" (cross_origin): ' + + "video reports zero decodable width but its source is cross-origin; no local proxy can be served for it", + ); + page.on = vi.fn( + (event: string, handler: (message: ReturnType) => void) => { + if (event === "console") { + handler(unavailableMessage); + } + }, + ); + installSessionMock(page); + + const result = await runBrowserCheck( + PROJECT, + { ...DEFAULT_CHECK_OPTIONS, samples: 1, contrast: false }, + { kind: "none" }, + runAuditGrid, + ); + + expect(result.runtimeFindings).toContainEqual( + expect.objectContaining({ + code: "media_proxy_unavailable", + severity: "info", + message: unavailableMessage.text(), + }), + ); +}); + +describe("preResolveHostileMediaProxies", () => { + const dirs: string[] = []; + const mkProjectDir = (): string => { + const d = mkdtempSync(join(tmpdir(), "hf-check-preresolve-")); + dirs.push(d); + return d; + }; + afterEach(() => { + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); + }); + + it("awaits resolveProxy for every browser-hostile entry before returning", async () => { + const projectDir = mkProjectDir(); + mocks.scanProjectMediaCodecMap.mockResolvedValue({ + "/clip.mp4": { + codecName: "hevc", + browserHostile: true, + representativeMime: null, + hasAlpha: false, + }, + "/plain.mp4": { + codecName: "h264", + browserHostile: false, + representativeMime: null, + hasAlpha: false, + }, + }); + let resolveTranscode: (() => void) | undefined; + mocks.resolveProxy.mockImplementation( + () => + new Promise((resolveIt) => { + resolveTranscode = () => resolveIt("/cache/clip.mp4"); + }), + ); + + let settled = false; + const pending = preResolveHostileMediaProxies(projectDir, "").then(() => { + settled = true; + }); + + await Promise.resolve(); + await Promise.resolve(); + expect(mocks.resolveProxy).toHaveBeenCalledTimes(1); + expect(mocks.resolveProxy).toHaveBeenCalledWith(projectDir, join(projectDir, "clip.mp4")); + expect(settled).toBe(false); // still waiting on the hostile entry's transcode + + resolveTranscode?.(); + await pending; + expect(settled).toBe(true); + }); + + it("does nothing (no scan, no resolveProxy) when autoProxy is off in hyperframes.json", async () => { + const projectDir = mkProjectDir(); + writeFileSync( + join(projectDir, "hyperframes.json"), + JSON.stringify({ media: { autoProxy: false } }), + ); + + await preResolveHostileMediaProxies(projectDir, ""); + + expect(mocks.scanProjectMediaCodecMap).not.toHaveBeenCalled(); + expect(mocks.resolveProxy).not.toHaveBeenCalled(); + }); + + it("swallows a resolveProxy rejection instead of throwing", async () => { + const projectDir = mkProjectDir(); + mocks.scanProjectMediaCodecMap.mockResolvedValue({ + "/clip.mp4": { + codecName: "hevc", + browserHostile: true, + representativeMime: null, + hasAlpha: false, + }, + }); + mocks.resolveProxy.mockRejectedValue(new Error("ffmpeg exited with code 1")); + const infoSpy = vi.spyOn(console, "info").mockImplementation(() => undefined); + + await expect( + preResolveHostileMediaProxies(projectDir, ""), + ).resolves.toBeUndefined(); + expect(infoSpy).toHaveBeenCalledWith( + expect.stringContaining("media proxy pre-resolve: 0/1 ready, 1 failed"), + ); + }); + + it("does not pre-resolve a hostile asset rejected by the shared proxy policy", async () => { + const projectDir = mkProjectDir(); + mocks.scanProjectMediaCodecMap.mockResolvedValue({ + "/alpha.mov": { + codecName: "prores", + browserHostile: true, + representativeMime: null, + hasAlpha: true, + }, + }); + + await preResolveHostileMediaProxies(projectDir, ""); + + expect(mocks.resolveProxy).not.toHaveBeenCalled(); + }); + + it("is a no-op when the codec map has no hostile entries", async () => { + const projectDir = mkProjectDir(); + mocks.scanProjectMediaCodecMap.mockResolvedValue({ + "/plain.mp4": { + codecName: "h264", + browserHostile: false, + representativeMime: null, + hasAlpha: false, + }, + }); + + await preResolveHostileMediaProxies(projectDir, ""); + + expect(mocks.resolveProxy).not.toHaveBeenCalled(); + }); +}); + describe("captureOverviewShot", () => { it("injects the annotation overlay before the overview shot and removes it right after", async () => { const calls: string[] = []; @@ -349,6 +576,14 @@ function installRects(): void { vi.spyOn(image, "getBoundingClientRect").mockReturnValue(new DOMRect(600, 80, 200, 100)); } +function fakeConsoleMessage(type: string, text: string) { + return { + type: () => type, + text: () => text, + location: () => ({ url: "http://127.0.0.1:3000/index.html", lineNumber: 1 }), + }; +} + function fakePage() { return Object.assign(Object.create(null), { on: vi.fn(), diff --git a/packages/cli/src/utils/checkBrowser.ts b/packages/cli/src/utils/checkBrowser.ts index 272258762..c161fe711 100644 --- a/packages/cli/src/utils/checkBrowser.ts +++ b/packages/cli/src/utils/checkBrowser.ts @@ -1,5 +1,5 @@ import { mkdirSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import type { Page } from "puppeteer-core"; import { AUDIT_SEEK_OPTIONS, @@ -18,6 +18,12 @@ import { normalizeErrorMessage } from "./errorMessage.js"; import { ambiguousIssue, type MotionFrame } from "./motionAudit.js"; import type { LayoutIssue, LayoutIssueCode, LayoutRect } from "./layoutAudit.js"; import { serveStaticProjectHtml } from "./staticProjectServer.js"; +import { resolveAutoProxy } from "./projectConfig.js"; +import { + decideMediaProxyEligibility, + scanProjectMediaCodecMap, +} from "@hyperframes/studio-server/media-codec-map"; +import { resolveProxy } from "@hyperframes/studio-server/proxy-transcoder"; import { rectToBbox } from "./checkTypes.js"; import type { AnchoredLayoutIssue, @@ -82,6 +88,50 @@ interface FinishedContrast { bg: string; } +/** + * Awaits the H.264 authoring proxy for every browser-hostile local video + * asset in `html` BEFORE the timed render-ready wait starts. `check`'s + * render-ready wait defaults to 3000ms (`DEFAULT_CHECK_OPTIONS.timeout`, + * passed through as `renderReadyTimeoutMs`), and a cold `.transcode-cache` + * cannot fit inside that window — without this, a project's first + * hostile-asset check (e.g. a fresh CI checkout) would race the timeout + * instead of paying a bounded one-time transcode cost + * (docs/plans/2026-07-14-002-feat-transparent-media-proxies-plan.md, unit U4). + * Best-effort: a probe or transcode failure does not fail `check`; one summary + * line records the pre-resolve outcome before the runtime attempts playback. + */ +export async function preResolveHostileMediaProxies( + projectDir: string, + html: string, + autoProxyOverride?: boolean, +): Promise { + if (!resolveAutoProxy(projectDir, autoProxyOverride)) return; + let codecMap: Awaited>; + try { + codecMap = await scanProjectMediaCodecMap(projectDir, [{ html }]); + } catch (err) { + console.info( + `[hyperframes] media proxy pre-resolve: scan failed (${normalizeErrorMessage(err)})`, + ); + return; + } + const hostilePathnames = Object.entries(codecMap) + .filter(([, facts]) => decideMediaProxyEligibility(facts).eligible) + .map(([pathname]) => pathname); + if (hostilePathnames.length === 0) return; + + const startedAt = Date.now(); + const results = await Promise.allSettled( + hostilePathnames.map((pathname) => + resolveProxy(projectDir, resolve(projectDir, pathname.replace(/^\/+/, ""))), + ), + ); + const failed = results.filter((result) => result.status === "rejected").length; + console.info( + `[hyperframes] media proxy pre-resolve: ${results.length - failed}/${results.length} ready, ${failed} failed (${Date.now() - startedAt}ms)`, + ); +} + export async function runBrowserCheck( project: ProjectDir, options: CheckOptions, @@ -90,7 +140,14 @@ export async function runBrowserCheck( ): Promise { const { bundleWithLocalizedFonts } = await import("./bundleWithLocalizedFonts.js"); const html = await bundleWithLocalizedFonts(project.dir); - const server = await serveStaticProjectHtml(project.dir, html, "Failed to bind check server"); + await preResolveHostileMediaProxies(project.dir, html, options.autoProxy); + const server = await serveStaticProjectHtml( + project.dir, + html, + "Failed to bind check server", + [], + options.autoProxy, + ); const drafts: RuntimeDraft[] = []; let currentTime = 0; let chromeBrowser: import("puppeteer-core").Browser | undefined; @@ -147,7 +204,13 @@ export async function captureFindingCrops( if (requests.length === 0) return []; const { bundleWithLocalizedFonts } = await import("./bundleWithLocalizedFonts.js"); const html = await bundleWithLocalizedFonts(project.dir); - const server = await serveStaticProjectHtml(project.dir, html, "Failed to bind check server"); + const server = await serveStaticProjectHtml( + project.dir, + html, + "Failed to bind check server", + [], + options.autoProxy, + ); let chromeBrowser: import("puppeteer-core").Browser | undefined; const written: string[] = []; try { @@ -180,6 +243,15 @@ export async function captureFindingCrops( } } +// `swapToProxy` / `emitUnavailableDiagnostic` (packages/core/src/runtime/ +// mediaProxy.ts) embed their stable diagnostic codes in the console.info line +// precisely so this scraper can match a token instead of prose. Matching the +// shared "runtime_media_proxy_" prefix surfaces both codes; only those +// runtime-emitted info lines should ever become findings here — an ordinary +// `console.info` from a composition author's own script must not. +const MEDIA_PROXY_MARKER_PREFIX = "[hyperframes] runtime_media_proxy_"; +const MEDIA_PROXY_UNAVAILABLE_MARKER = "[hyperframes] runtime_media_proxy_unavailable"; + function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: () => number): void { page.on("console", (message) => { const type = message.type(); @@ -204,6 +276,18 @@ function wireRuntimeListeners(page: Page, drafts: RuntimeDraft[], currentTime: ( url: location.url, line: location.lineNumber, }); + } else if (type === "info" && text.startsWith(MEDIA_PROXY_MARKER_PREFIX)) { + const location = message.location(); + drafts.push({ + code: text.includes(MEDIA_PROXY_UNAVAILABLE_MARKER) + ? "media_proxy_unavailable" + : "media_proxy_fallback", + severity: "info", + message: text, + time: currentTime(), + url: location.url, + line: location.lineNumber, + }); } }); page.on("pageerror", (error) => { diff --git a/packages/cli/src/utils/checkTypes.ts b/packages/cli/src/utils/checkTypes.ts index 02996dc9e..59da4c069 100644 --- a/packages/cli/src/utils/checkTypes.ts +++ b/packages/cli/src/utils/checkTypes.ts @@ -18,6 +18,8 @@ export interface CheckOptions { snapshots: boolean; captionZone?: CaptionZoneOptions; frameCheck?: FrameCheckOptions; + /** Explicit --proxy/--no-proxy override; undefined preserves project config. */ + autoProxy?: boolean; } export interface CaptionZoneOptions {