mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-16 06:19:21 +00:00
* fix(studio): text in Studio reads without em or en dashes * fix(studio): the dash check also catches HTML entities, and peak reads Unavailable * test(studio): the dash check test covers every entity form * fix(studio): messages Studio shows from lint, sdk, core and the renderer drop dashes too * fix(producer): the gate check stops early on the engine's new fallback wording
309 lines
14 KiB
TypeScript
309 lines
14 KiB
TypeScript
import { HF_AUDIO_AUTOMATION_ATTR } from "../audioAutomation.js";
|
|
import { HF_AUDIO_FX_ATTR } from "../audioFx.js";
|
|
import { HF_AUDIO_GROUP_ATTR } from "../audioGroups.js";
|
|
import { postRuntimeMessage } from "./bridge";
|
|
import type { RuntimeJson } from "./types";
|
|
|
|
/**
|
|
* Which transport may claim an `<audio>` or audible `<video>` element's output.
|
|
*
|
|
* `createMediaElementSource()` is a one-way door, and over a resource that fails
|
|
* the CORS-cross-origin check it outputs SILENCE without throwing (#3458). So the
|
|
* verdict is reached BEFORE the call by this pure classifier, shared by media
|
|
* discovery (diagnostic) and scheduling (withholding the node). Callers that build
|
|
* their own `AudioContext` (e.g. `AudioRow.tsx`) must route through it too.
|
|
* Known gaps: `srcObject` elements always read `web-audio`, and the RAW url is
|
|
* judged, not a redirect chain's final target.
|
|
*/
|
|
export type WebAudioMediaRoute =
|
|
/** Same-origin, CORS-opted-in, or a scheme the check doesn't apply to. */
|
|
| { kind: "web-audio" }
|
|
/**
|
|
* Cross-origin without a `crossorigin` opt-in. MediaElementSource would be
|
|
* silent, but `fetch` + `decodeAudioData` may still succeed — a CDN that
|
|
* sends `Access-Control-Allow-Origin` while the author simply never wrote the
|
|
* attribute is the common shape of this bug — and that route keeps the whole
|
|
* FX graph. So: withhold the node, still let the decode path try.
|
|
*/
|
|
| { kind: "decode-only"; reason: WebAudioSilenceReason; asset: string };
|
|
|
|
/**
|
|
* Why capture was withheld. The two cases need DIFFERENT remediation, which is
|
|
* the whole reason they are separate values: a foreign asset can be proxied to
|
|
* a same-origin URL, but an opaque document has no same-origin URL to proxy to.
|
|
*/
|
|
export type WebAudioSilenceReason =
|
|
/** The asset's origin differs from ours, and no `crossorigin` opt-in is set. */
|
|
| "cross_origin_no_cors"
|
|
/** OUR document's security origin is opaque, so nothing is same-origin with it. */
|
|
| "opaque_document_origin";
|
|
|
|
/** Fired when the runtime withholds Web Audio capture from a media element. */
|
|
const DIAGNOSTIC_BYPASS_CODE = "runtime_web_audio_bypass";
|
|
|
|
function getAttr(el: HTMLMediaElement, name: string): string | null {
|
|
return typeof el.getAttribute === "function" ? el.getAttribute(name) : null;
|
|
}
|
|
|
|
function hasAttr(el: HTMLMediaElement, name: string): boolean {
|
|
return getAttr(el, name) !== null;
|
|
}
|
|
|
|
/**
|
|
* `crossorigin` is an enumerated attribute whose invalid-value default is
|
|
* `anonymous`, so PRESENCE is the opt-in — `crossorigin=""` and even
|
|
* `crossorigin="garbage"` both make the fetch a CORS request. Comparing the
|
|
* value against `"anonymous"` would wrongly block those.
|
|
*
|
|
* Two independent reads, because a spec-faithful host and a permissive one
|
|
* disagree about where the truth lives:
|
|
* - `getAttribute` is the primary read and covers every real browser: the
|
|
* markup is unambiguous regardless of what the IDL getter does with it.
|
|
* - `el.crossOrigin` is a secondary read for a host that sets the IDL
|
|
* property without reflecting it back to the attribute — some
|
|
* jsdom-style test/preview hosts do this. The check is `!= null`
|
|
* (covers both `null` and `undefined`), not a truthiness check, ON
|
|
* PURPOSE: `crossorigin=""` is a valid, common opt-in (see above), and
|
|
* its IDL fallback value is the empty string — a falsy value that
|
|
* `Boolean(el.crossOrigin)` would silently misread as "not opted in",
|
|
* reintroducing the exact silent-audio bug this module exists to close.
|
|
*/
|
|
function hasCorsOptIn(el: HTMLMediaElement): boolean {
|
|
if (hasAttr(el, "crossorigin")) return true;
|
|
return el.crossOrigin != null;
|
|
}
|
|
|
|
function baseUri(el: HTMLMediaElement): string {
|
|
if (typeof el.baseURI === "string" && el.baseURI) return el.baseURI;
|
|
return typeof document !== "undefined" ? document.baseURI : "";
|
|
}
|
|
|
|
/**
|
|
* The URLs whose origin could decide this element's route.
|
|
*
|
|
* Order matters and mirrors the HTML resource selection algorithm. Once
|
|
* `currentSrc` is set the browser has COMMITTED to that resource, so it is the
|
|
* only candidate that can matter; a `<source>` sibling it passed over must not
|
|
* cost a same-origin element its Web Audio graph. A `src` attribute is equally
|
|
* definitive — the spec has it win outright over `<source>` children. Only
|
|
* before selection settles (no `currentSrc`, no `src`) do the `<source>`
|
|
* candidates matter, and there the conservative read is right: any of them
|
|
* could be the one that gets picked.
|
|
*/
|
|
function routeCandidates(el: HTMLMediaElement): string[] {
|
|
const current = typeof el.currentSrc === "string" ? el.currentSrc : "";
|
|
if (current) return [current];
|
|
const srcAttr = getAttr(el, "src");
|
|
if (srcAttr) return [srcAttr];
|
|
if (typeof el.querySelectorAll !== "function") return [];
|
|
const sources: string[] = [];
|
|
for (const source of Array.from(el.querySelectorAll("source"))) {
|
|
const value = source.getAttribute("src");
|
|
if (value) sources.push(value);
|
|
}
|
|
return sources;
|
|
}
|
|
|
|
/**
|
|
* Whether this URL would make MediaElementSource silent. Only http(s) is
|
|
* judged: `blob:` and `data:` are same-origin by construction, and a `file:`
|
|
* page's opaque origin can't be compared meaningfully — so those keep the
|
|
* pre-existing behaviour rather than losing Web Audio on a guess. The guard
|
|
* changes behaviour ONLY in the case that is already known-broken.
|
|
*/
|
|
/**
|
|
* The candidate as an absolute URL, or null when it is not a fetch this check
|
|
* governs: unparseable, or a scheme (`data:`, `blob:`, `file:`) where origin
|
|
* comparison means nothing.
|
|
*/
|
|
function httpMediaUrl(rawUrl: string, el: HTMLMediaElement): URL | null {
|
|
let url: URL;
|
|
try {
|
|
url = new URL(rawUrl, baseUri(el));
|
|
} catch {
|
|
return null;
|
|
}
|
|
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
|
|
return url;
|
|
}
|
|
|
|
/**
|
|
* Our OWN security origin, which is `"null"` for an opaque document.
|
|
*
|
|
* Not `location.origin`: that is the URL's origin, so a sandboxed iframe
|
|
* without `allow-same-origin` still reports the parent-looking URL there
|
|
* while `window.origin` reads `"null"`. An opaque document cannot prove
|
|
* same-origin with anything, including an asset served from its own host.
|
|
*/
|
|
function documentSecurityOrigin(): string {
|
|
return typeof window.origin === "string" ? window.origin : window.location.origin;
|
|
}
|
|
|
|
function corsSilenceReason(rawUrl: string, el: HTMLMediaElement): WebAudioSilenceReason | null {
|
|
if (typeof window === "undefined") return null;
|
|
const url = httpMediaUrl(rawUrl, el);
|
|
if (!url) return null;
|
|
const selfOrigin = documentSecurityOrigin();
|
|
const opaqueDocument = selfOrigin === "null";
|
|
if (!opaqueDocument && url.origin === selfOrigin) return null;
|
|
if (hasCorsOptIn(el)) return null;
|
|
// Opacity wins when both hold. It is the blocking condition, and it is the
|
|
// one the author cannot fix by moving the asset.
|
|
return opaqueDocument ? "opaque_document_origin" : "cross_origin_no_cors";
|
|
}
|
|
|
|
/**
|
|
* Whether resource selection has settled enough for a verdict to be a FACT
|
|
* rather than a guess. `currentSrc`/`src` are both definitive per the HTML
|
|
* resource-selection algorithm (see `routeCandidates` above); before either
|
|
* is set, a verdict can only be built from `<source>` children, any of which
|
|
* the browser may still pass over before committing.
|
|
*
|
|
* `classifyWebAudioMediaRoute` itself stays unsettled-tolerant on purpose —
|
|
* the schedule path needs *a* verdict even before selection settles, and
|
|
* conservatively withholding the node there costs nothing but a decode-only
|
|
* fallback. This predicate exists for the one caller that must NOT act on a
|
|
* guess: the discovery-time diagnostic, which drops a message in a human's
|
|
* lap and only gets to say it once (see `reportWebAudioMediaRoute`'s latch).
|
|
*/
|
|
export function isRouteSelectionSettled(el: HTMLMediaElement): boolean {
|
|
const current = typeof el.currentSrc === "string" ? el.currentSrc : "";
|
|
if (current) return true;
|
|
return hasAttr(el, "src");
|
|
}
|
|
|
|
/**
|
|
* Pure — no node creation, no diagnostics, no element mutation. Called from
|
|
* both the schedule path (where it withholds the node) and the discovery path
|
|
* (where it only reports), which is the point: `hyperframes check` never calls
|
|
* `play()`, so a verdict reachable only from the transport would be invisible
|
|
* to the very gate meant to surface it.
|
|
*/
|
|
export function classifyWebAudioMediaRoute(el: HTMLMediaElement): WebAudioMediaRoute {
|
|
for (const candidate of routeCandidates(el)) {
|
|
const reason = corsSilenceReason(candidate, el);
|
|
if (reason) {
|
|
return { kind: "decode-only", reason, asset: candidate };
|
|
}
|
|
}
|
|
return { kind: "web-audio" };
|
|
}
|
|
|
|
/**
|
|
* Processing the native HTMLMediaElement fallback cannot reproduce. The track
|
|
* stays AUDIBLE either way — silence is the bug being fixed, so failing closed
|
|
* would just reinstate it — but these authored intentions are quietly dropped,
|
|
* which is worth saying out loud.
|
|
*
|
|
* Wider than the FX/automation pair the schedule path already tests: group
|
|
* membership carries a whole shared bus (chain, fader, its own automation
|
|
* clock), and an above-unity `data-volume` cannot survive a route whose only
|
|
* gain stage is `el.volume`, which the spec pins to [0,1].
|
|
*/
|
|
export function nativeUnexpressibleProcessing(el: HTMLMediaElement): string[] {
|
|
const lost: string[] = [];
|
|
if (hasAttr(el, HF_AUDIO_FX_ATTR)) lost.push("fx-chain");
|
|
if (hasAttr(el, HF_AUDIO_AUTOMATION_ATTR)) lost.push("automation");
|
|
if (hasAttr(el, HF_AUDIO_GROUP_ATTR)) lost.push("audio-group");
|
|
const volume = Number.parseFloat(getAttr(el, "data-volume") ?? "");
|
|
if (Number.isFinite(volume) && volume > 1) lost.push("above-unity-gain");
|
|
return lost;
|
|
}
|
|
|
|
/**
|
|
* Render never plays through Web Audio at all: the producer mixes offline from
|
|
* the source files, and that mix applies the FX chain itself (see
|
|
* `applyAudioFxChain` in `packages/engine/src/services/audioMixer.ts`). So a
|
|
* bypass is not a fact about the render, and `lostProcessing` would be an
|
|
* outright false claim there — the offline mix DOES reproduce the chain the
|
|
* line says native output cannot. The engine forwards every console line to
|
|
* producer stdout, so an ungated report would also land in every render log.
|
|
*
|
|
* Reporting is all that is gated: `classifyWebAudioMediaRoute` stays pure and
|
|
* the routing decision is unchanged, which costs nothing in render because the
|
|
* element is not the audio source there in the first place.
|
|
*
|
|
* Same signal `mediaProxy.ts`'s `isRenderMode` gates on. The `<video>` half of
|
|
* that check (the injected render-frame sibling) is not mirrored: the
|
|
* export-seek config alone gates reporting.
|
|
*/
|
|
function isRenderMode(): boolean {
|
|
// Read through an inline cast rather than the ambient `Window` augmentation
|
|
// in `window.d.ts`: that augmentation is only in scope for programs that
|
|
// include it (core's own tsconfig does), and this module is also exported
|
|
// as `./runtime/web-audio-route` for non-runtime consumers (e.g. the studio
|
|
// asset sidebar's preview player, `AudioRow.tsx`) whose tsconfig doesn't
|
|
// pull it in. This is a plain existence check, so the cast costs nothing.
|
|
return (
|
|
typeof window !== "undefined" &&
|
|
!!(window as unknown as { __HF_EXPORT_RENDER_SEEK_CONFIG?: unknown })
|
|
.__HF_EXPORT_RENDER_SEEK_CONFIG
|
|
);
|
|
}
|
|
|
|
// One diagnostic per element. Latched only when something is actually emitted,
|
|
// so an early "web-audio" verdict taken before the resource selection settled
|
|
// can't suppress the real one at `loadedmetadata`.
|
|
const diagnosedElements = new WeakSet<HTMLMediaElement>();
|
|
|
|
/**
|
|
* Emit the one-time diagnostic for a non-Web-Audio verdict.
|
|
*
|
|
* The bypass always reports: it is an accident by definition, and the whole
|
|
* complaint in #3458 is that nothing was said.
|
|
*/
|
|
/**
|
|
* What was actually wrong, per reason. The opaque case is NOT "cross-origin
|
|
* media": the asset can be same-origin by URL and still unreadable, because it
|
|
* is OUR document that cannot prove an origin.
|
|
*/
|
|
const NOTE_BY_REASON: Record<WebAudioSilenceReason, string> = {
|
|
cross_origin_no_cors:
|
|
"cross-origin media without a `crossorigin` opt-in is silent through createMediaElementSource (Web Audio spec); using native playback instead",
|
|
opaque_document_origin:
|
|
"this document's security origin is opaque (a sandbox without `allow-same-origin`), so no media is same-origin with it and createMediaElementSource would be silent (Web Audio spec); using native playback instead",
|
|
};
|
|
|
|
/**
|
|
* How to get the processing back, per reason. Keeping these apart is the point:
|
|
* telling an opaque document to "proxy the asset to a same-origin URL" names a
|
|
* fix that cannot exist, since nothing is same-origin with an opaque origin.
|
|
*/
|
|
const REMEDY_BY_REASON: Record<WebAudioSilenceReason, string> = {
|
|
cross_origin_no_cors: "proxy or download the asset to a same-origin URL to keep it.",
|
|
opaque_document_origin:
|
|
"add `allow-same-origin` to the sandbox, or serve the asset with a `crossorigin` attribute and an `Access-Control-Allow-Origin` of `null` or `*`, to keep it.",
|
|
};
|
|
|
|
export function reportWebAudioMediaRoute(el: HTMLMediaElement, route: WebAudioMediaRoute): void {
|
|
if (route.kind === "web-audio") return;
|
|
if (isRenderMode()) return;
|
|
if (diagnosedElements.has(el)) return;
|
|
const lost = nativeUnexpressibleProcessing(el);
|
|
diagnosedElements.add(el);
|
|
|
|
const details: Record<string, RuntimeJson> = {
|
|
asset: route.asset,
|
|
reason: route.reason,
|
|
lostProcessing: lost,
|
|
note: NOTE_BY_REASON[route.reason],
|
|
};
|
|
postRuntimeMessage({
|
|
source: "hf-preview",
|
|
type: "diagnostic",
|
|
code: DIAGNOSTIC_BYPASS_CODE,
|
|
details,
|
|
});
|
|
// The stable code lives in the console text so the CLI's scraper
|
|
// (packages/cli/src/utils/checkBrowser.ts) can match a token, not prose —
|
|
// same contract mediaProxy.ts's diagnostics use.
|
|
const lostNote =
|
|
lost.length > 0
|
|
? ` Native playback cannot reproduce: ${lost.join(", ")}. Fix: ${REMEDY_BY_REASON[route.reason]}`
|
|
: "";
|
|
console.info(
|
|
`[hyperframes] ${DIAGNOSTIC_BYPASS_CODE}: "${route.asset}" (${route.reason}): ` +
|
|
`Web Audio capture withheld; the track plays through native HTMLMediaElement output.${lostNote}`,
|
|
);
|
|
}
|