Files
hyperframes/packages/core/src/runtime/webAudioRoute.ts
T
Miguel Ángel 9825614d91 fix(studio): Studio's own text reads without em or en dashes (#5414)
* fix(studio): text in Studio reads without em or en dashes

* fix(studio): the dash check also catches HTML entities, and peak reads Unavailable

* test(studio): the dash check test covers every entity form

* fix(studio): messages Studio shows from lint, sdk, core and the renderer drop dashes too

* fix(producer): the gate check stops early on the engine's new fallback wording
2026-10-10 19:08:30 +00:00

309 lines
14 KiB
TypeScript

import { HF_AUDIO_AUTOMATION_ATTR } from "../audioAutomation.js";
import { HF_AUDIO_FX_ATTR } from "../audioFx.js";
import { HF_AUDIO_GROUP_ATTR } from "../audioGroups.js";
import { postRuntimeMessage } from "./bridge";
import type { RuntimeJson } from "./types";
/**
* Which transport may claim an `<audio>` or audible `<video>` element's output.
*
* `createMediaElementSource()` is a one-way door, and over a resource that fails
* the CORS-cross-origin check it outputs SILENCE without throwing (#3458). So the
* verdict is reached BEFORE the call by this pure classifier, shared by media
* discovery (diagnostic) and scheduling (withholding the node). Callers that build
* their own `AudioContext` (e.g. `AudioRow.tsx`) must route through it too.
* Known gaps: `srcObject` elements always read `web-audio`, and the RAW url is
* judged, not a redirect chain's final target.
*/
export type WebAudioMediaRoute =
/** Same-origin, CORS-opted-in, or a scheme the check doesn't apply to. */
| { kind: "web-audio" }
/**
* Cross-origin without a `crossorigin` opt-in. MediaElementSource would be
* silent, but `fetch` + `decodeAudioData` may still succeed — a CDN that
* sends `Access-Control-Allow-Origin` while the author simply never wrote the
* attribute is the common shape of this bug — and that route keeps the whole
* FX graph. So: withhold the node, still let the decode path try.
*/
| { kind: "decode-only"; reason: WebAudioSilenceReason; asset: string };
/**
* Why capture was withheld. The two cases need DIFFERENT remediation, which is
* the whole reason they are separate values: a foreign asset can be proxied to
* a same-origin URL, but an opaque document has no same-origin URL to proxy to.
*/
export type WebAudioSilenceReason =
/** The asset's origin differs from ours, and no `crossorigin` opt-in is set. */
| "cross_origin_no_cors"
/** OUR document's security origin is opaque, so nothing is same-origin with it. */
| "opaque_document_origin";
/** Fired when the runtime withholds Web Audio capture from a media element. */
const DIAGNOSTIC_BYPASS_CODE = "runtime_web_audio_bypass";
function getAttr(el: HTMLMediaElement, name: string): string | null {
return typeof el.getAttribute === "function" ? el.getAttribute(name) : null;
}
function hasAttr(el: HTMLMediaElement, name: string): boolean {
return getAttr(el, name) !== null;
}
/**
* `crossorigin` is an enumerated attribute whose invalid-value default is
* `anonymous`, so PRESENCE is the opt-in — `crossorigin=""` and even
* `crossorigin="garbage"` both make the fetch a CORS request. Comparing the
* value against `"anonymous"` would wrongly block those.
*
* Two independent reads, because a spec-faithful host and a permissive one
* disagree about where the truth lives:
* - `getAttribute` is the primary read and covers every real browser: the
* markup is unambiguous regardless of what the IDL getter does with it.
* - `el.crossOrigin` is a secondary read for a host that sets the IDL
* property without reflecting it back to the attribute — some
* jsdom-style test/preview hosts do this. The check is `!= null`
* (covers both `null` and `undefined`), not a truthiness check, ON
* PURPOSE: `crossorigin=""` is a valid, common opt-in (see above), and
* its IDL fallback value is the empty string — a falsy value that
* `Boolean(el.crossOrigin)` would silently misread as "not opted in",
* reintroducing the exact silent-audio bug this module exists to close.
*/
function hasCorsOptIn(el: HTMLMediaElement): boolean {
if (hasAttr(el, "crossorigin")) return true;
return el.crossOrigin != null;
}
function baseUri(el: HTMLMediaElement): string {
if (typeof el.baseURI === "string" && el.baseURI) return el.baseURI;
return typeof document !== "undefined" ? document.baseURI : "";
}
/**
* The URLs whose origin could decide this element's route.
*
* Order matters and mirrors the HTML resource selection algorithm. Once
* `currentSrc` is set the browser has COMMITTED to that resource, so it is the
* only candidate that can matter; a `<source>` sibling it passed over must not
* cost a same-origin element its Web Audio graph. A `src` attribute is equally
* definitive — the spec has it win outright over `<source>` children. Only
* before selection settles (no `currentSrc`, no `src`) do the `<source>`
* candidates matter, and there the conservative read is right: any of them
* could be the one that gets picked.
*/
function routeCandidates(el: HTMLMediaElement): string[] {
const current = typeof el.currentSrc === "string" ? el.currentSrc : "";
if (current) return [current];
const srcAttr = getAttr(el, "src");
if (srcAttr) return [srcAttr];
if (typeof el.querySelectorAll !== "function") return [];
const sources: string[] = [];
for (const source of Array.from(el.querySelectorAll("source"))) {
const value = source.getAttribute("src");
if (value) sources.push(value);
}
return sources;
}
/**
* Whether this URL would make MediaElementSource silent. Only http(s) is
* judged: `blob:` and `data:` are same-origin by construction, and a `file:`
* page's opaque origin can't be compared meaningfully — so those keep the
* pre-existing behaviour rather than losing Web Audio on a guess. The guard
* changes behaviour ONLY in the case that is already known-broken.
*/
/**
* The candidate as an absolute URL, or null when it is not a fetch this check
* governs: unparseable, or a scheme (`data:`, `blob:`, `file:`) where origin
* comparison means nothing.
*/
function httpMediaUrl(rawUrl: string, el: HTMLMediaElement): URL | null {
let url: URL;
try {
url = new URL(rawUrl, baseUri(el));
} catch {
return null;
}
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
return url;
}
/**
* Our OWN security origin, which is `"null"` for an opaque document.
*
* Not `location.origin`: that is the URL's origin, so a sandboxed iframe
* without `allow-same-origin` still reports the parent-looking URL there
* while `window.origin` reads `"null"`. An opaque document cannot prove
* same-origin with anything, including an asset served from its own host.
*/
function documentSecurityOrigin(): string {
return typeof window.origin === "string" ? window.origin : window.location.origin;
}
function corsSilenceReason(rawUrl: string, el: HTMLMediaElement): WebAudioSilenceReason | null {
if (typeof window === "undefined") return null;
const url = httpMediaUrl(rawUrl, el);
if (!url) return null;
const selfOrigin = documentSecurityOrigin();
const opaqueDocument = selfOrigin === "null";
if (!opaqueDocument && url.origin === selfOrigin) return null;
if (hasCorsOptIn(el)) return null;
// Opacity wins when both hold. It is the blocking condition, and it is the
// one the author cannot fix by moving the asset.
return opaqueDocument ? "opaque_document_origin" : "cross_origin_no_cors";
}
/**
* Whether resource selection has settled enough for a verdict to be a FACT
* rather than a guess. `currentSrc`/`src` are both definitive per the HTML
* resource-selection algorithm (see `routeCandidates` above); before either
* is set, a verdict can only be built from `<source>` children, any of which
* the browser may still pass over before committing.
*
* `classifyWebAudioMediaRoute` itself stays unsettled-tolerant on purpose —
* the schedule path needs *a* verdict even before selection settles, and
* conservatively withholding the node there costs nothing but a decode-only
* fallback. This predicate exists for the one caller that must NOT act on a
* guess: the discovery-time diagnostic, which drops a message in a human's
* lap and only gets to say it once (see `reportWebAudioMediaRoute`'s latch).
*/
export function isRouteSelectionSettled(el: HTMLMediaElement): boolean {
const current = typeof el.currentSrc === "string" ? el.currentSrc : "";
if (current) return true;
return hasAttr(el, "src");
}
/**
* Pure — no node creation, no diagnostics, no element mutation. Called from
* both the schedule path (where it withholds the node) and the discovery path
* (where it only reports), which is the point: `hyperframes check` never calls
* `play()`, so a verdict reachable only from the transport would be invisible
* to the very gate meant to surface it.
*/
export function classifyWebAudioMediaRoute(el: HTMLMediaElement): WebAudioMediaRoute {
for (const candidate of routeCandidates(el)) {
const reason = corsSilenceReason(candidate, el);
if (reason) {
return { kind: "decode-only", reason, asset: candidate };
}
}
return { kind: "web-audio" };
}
/**
* Processing the native HTMLMediaElement fallback cannot reproduce. The track
* stays AUDIBLE either way — silence is the bug being fixed, so failing closed
* would just reinstate it — but these authored intentions are quietly dropped,
* which is worth saying out loud.
*
* Wider than the FX/automation pair the schedule path already tests: group
* membership carries a whole shared bus (chain, fader, its own automation
* clock), and an above-unity `data-volume` cannot survive a route whose only
* gain stage is `el.volume`, which the spec pins to [0,1].
*/
export function nativeUnexpressibleProcessing(el: HTMLMediaElement): string[] {
const lost: string[] = [];
if (hasAttr(el, HF_AUDIO_FX_ATTR)) lost.push("fx-chain");
if (hasAttr(el, HF_AUDIO_AUTOMATION_ATTR)) lost.push("automation");
if (hasAttr(el, HF_AUDIO_GROUP_ATTR)) lost.push("audio-group");
const volume = Number.parseFloat(getAttr(el, "data-volume") ?? "");
if (Number.isFinite(volume) && volume > 1) lost.push("above-unity-gain");
return lost;
}
/**
* Render never plays through Web Audio at all: the producer mixes offline from
* the source files, and that mix applies the FX chain itself (see
* `applyAudioFxChain` in `packages/engine/src/services/audioMixer.ts`). So a
* bypass is not a fact about the render, and `lostProcessing` would be an
* outright false claim there — the offline mix DOES reproduce the chain the
* line says native output cannot. The engine forwards every console line to
* producer stdout, so an ungated report would also land in every render log.
*
* Reporting is all that is gated: `classifyWebAudioMediaRoute` stays pure and
* the routing decision is unchanged, which costs nothing in render because the
* element is not the audio source there in the first place.
*
* Same signal `mediaProxy.ts`'s `isRenderMode` gates on. The `<video>` half of
* that check (the injected render-frame sibling) is not mirrored: the
* export-seek config alone gates reporting.
*/
function isRenderMode(): boolean {
// Read through an inline cast rather than the ambient `Window` augmentation
// in `window.d.ts`: that augmentation is only in scope for programs that
// include it (core's own tsconfig does), and this module is also exported
// as `./runtime/web-audio-route` for non-runtime consumers (e.g. the studio
// asset sidebar's preview player, `AudioRow.tsx`) whose tsconfig doesn't
// pull it in. This is a plain existence check, so the cast costs nothing.
return (
typeof window !== "undefined" &&
!!(window as unknown as { __HF_EXPORT_RENDER_SEEK_CONFIG?: unknown })
.__HF_EXPORT_RENDER_SEEK_CONFIG
);
}
// One diagnostic per element. Latched only when something is actually emitted,
// so an early "web-audio" verdict taken before the resource selection settled
// can't suppress the real one at `loadedmetadata`.
const diagnosedElements = new WeakSet<HTMLMediaElement>();
/**
* Emit the one-time diagnostic for a non-Web-Audio verdict.
*
* The bypass always reports: it is an accident by definition, and the whole
* complaint in #3458 is that nothing was said.
*/
/**
* What was actually wrong, per reason. The opaque case is NOT "cross-origin
* media": the asset can be same-origin by URL and still unreadable, because it
* is OUR document that cannot prove an origin.
*/
const NOTE_BY_REASON: Record<WebAudioSilenceReason, string> = {
cross_origin_no_cors:
"cross-origin media without a `crossorigin` opt-in is silent through createMediaElementSource (Web Audio spec); using native playback instead",
opaque_document_origin:
"this document's security origin is opaque (a sandbox without `allow-same-origin`), so no media is same-origin with it and createMediaElementSource would be silent (Web Audio spec); using native playback instead",
};
/**
* How to get the processing back, per reason. Keeping these apart is the point:
* telling an opaque document to "proxy the asset to a same-origin URL" names a
* fix that cannot exist, since nothing is same-origin with an opaque origin.
*/
const REMEDY_BY_REASON: Record<WebAudioSilenceReason, string> = {
cross_origin_no_cors: "proxy or download the asset to a same-origin URL to keep it.",
opaque_document_origin:
"add `allow-same-origin` to the sandbox, or serve the asset with a `crossorigin` attribute and an `Access-Control-Allow-Origin` of `null` or `*`, to keep it.",
};
export function reportWebAudioMediaRoute(el: HTMLMediaElement, route: WebAudioMediaRoute): void {
if (route.kind === "web-audio") return;
if (isRenderMode()) return;
if (diagnosedElements.has(el)) return;
const lost = nativeUnexpressibleProcessing(el);
diagnosedElements.add(el);
const details: Record<string, RuntimeJson> = {
asset: route.asset,
reason: route.reason,
lostProcessing: lost,
note: NOTE_BY_REASON[route.reason],
};
postRuntimeMessage({
source: "hf-preview",
type: "diagnostic",
code: DIAGNOSTIC_BYPASS_CODE,
details,
});
// The stable code lives in the console text so the CLI's scraper
// (packages/cli/src/utils/checkBrowser.ts) can match a token, not prose —
// same contract mediaProxy.ts's diagnostics use.
const lostNote =
lost.length > 0
? ` Native playback cannot reproduce: ${lost.join(", ")}. Fix: ${REMEDY_BY_REASON[route.reason]}`
: "";
console.info(
`[hyperframes] ${DIAGNOSTIC_BYPASS_CODE}: "${route.asset}" (${route.reason}): ` +
`Web Audio capture withheld; the track plays through native HTMLMediaElement output.${lostNote}`,
);
}