mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-10-30 09:16:55 +00:00
fix(render): probe runtime media source mutations (#3729)
This commit is contained in:
@@ -379,6 +379,62 @@ describe("media rules", () => {
|
||||
expect(finding).toBeUndefined();
|
||||
});
|
||||
|
||||
it("warns on raw src mutations targeting existing video and audio", async () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<div data-composition-id="c1" data-width="1920" data-height="1080">
|
||||
<video id="clip" src="video-a.mp4"></video>
|
||||
<audio id="voice" src="audio-a.wav"></audio>
|
||||
</div>
|
||||
<script>
|
||||
const clip = document.getElementById("clip");
|
||||
clip.src = "video-b.mp4";
|
||||
document.querySelector("#voice").setAttribute("src", "audio-b.wav");
|
||||
</script>
|
||||
</body></html>`;
|
||||
|
||||
const result = await lintHyperframeHtml(html);
|
||||
const findings = result.findings.filter(
|
||||
(finding) => finding.code === "media_runtime_src_mutation",
|
||||
);
|
||||
expect(findings).toHaveLength(2);
|
||||
expect(findings.map((finding) => finding.elementId).sort()).toEqual(["clip", "voice"]);
|
||||
expect(findings.every((finding) => finding.fixHint?.includes("data-var-src"))).toBe(true);
|
||||
});
|
||||
|
||||
it("does not flag img or script src mutations", async () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<div data-composition-id="c1" data-width="1920" data-height="1080">
|
||||
<img id="poster" src="a.png"><script id="loader"></script>
|
||||
</div>
|
||||
<script>
|
||||
document.getElementById("poster").src = "b.png";
|
||||
document.getElementById("loader").setAttribute("src", "loader-b.js");
|
||||
</script>
|
||||
</body></html>`;
|
||||
|
||||
const result = await lintHyperframeHtml(html);
|
||||
expect(
|
||||
result.findings.find((finding) => finding.code === "media_runtime_src_mutation"),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("warns when a source child of existing media is mutated", async () => {
|
||||
const html = `
|
||||
<html><body>
|
||||
<div data-composition-id="c1" data-width="1920" data-height="1080">
|
||||
<video id="clip"><source id="clip-source" src="video-a.mp4"></video>
|
||||
</div>
|
||||
<script>document.getElementById("clip-source").src = "video-b.mp4";</script>
|
||||
</body></html>`;
|
||||
|
||||
const result = await lintHyperframeHtml(html);
|
||||
expect(
|
||||
result.findings.find((finding) => finding.code === "media_runtime_src_mutation"),
|
||||
).toMatchObject({ severity: "warning", elementId: "clip" });
|
||||
});
|
||||
|
||||
it("does not flag <video> inside a sub-composition (runtime drives nested media)", async () => {
|
||||
// The runtime's global media sweep (querySelectorAll("video, audio")) drives
|
||||
// media at any nesting depth, and startResolver re-bases each nested clip's
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import type { LintContext, HyperframeLintFinding, OpenTag } from "../context";
|
||||
import { readAttr, readDecodedAttr, stripJsComments, truncateSnippet, isMediaTag } from "../utils";
|
||||
import { validateColorGradingContract } from "@hyperframes/parsers/color-grading-contract";
|
||||
import { extractMediaSrcMutations } from "@hyperframes/parsers";
|
||||
import { parseHTML } from "linkedom";
|
||||
|
||||
/**
|
||||
* Does the GSAP call that names `#id` also set `volume` in the same call?
|
||||
@@ -333,6 +335,46 @@ function findImperativeMediaControlFindings(ctx: LintContext): HyperframeLintFin
|
||||
return findings;
|
||||
}
|
||||
|
||||
function findRuntimeMediaSrcMutationFindings(ctx: LintContext): HyperframeLintFinding[] {
|
||||
const { document } = parseHTML(ctx.source);
|
||||
const findings: HyperframeLintFinding[] = [];
|
||||
for (const script of ctx.scripts) {
|
||||
for (const mutation of extractMediaSrcMutations(script.content)) {
|
||||
let targets: Element[];
|
||||
try {
|
||||
const id = /^#[A-Za-z_][\w-]*$/.test(mutation.selector) ? mutation.selector.slice(1) : null;
|
||||
const idTarget = id ? document.getElementById(id) : null;
|
||||
targets = id
|
||||
? idTarget
|
||||
? [idTarget]
|
||||
: []
|
||||
: [...document.querySelectorAll(mutation.selector)];
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const mediaTargets = targets
|
||||
.map((element) => {
|
||||
const name = element.tagName.toLowerCase();
|
||||
if (name === "video" || name === "audio") return element;
|
||||
return name === "source" ? element.closest("video, audio") : null;
|
||||
})
|
||||
.filter((element): element is Element => element !== null);
|
||||
if (mediaTargets.length === 0) continue;
|
||||
findings.push({
|
||||
code: "media_runtime_src_mutation",
|
||||
severity: "warning",
|
||||
message: `Inline script mutates the source of existing managed media via ${mutation.operation === "src_assignment" ? ".src assignment" : "setAttribute('src', ...)"}. Browser probing can reconcile synchronous writes, but external or delayed writes can still diverge between preview and extraction.`,
|
||||
elementId: mediaTargets[0]?.getAttribute("id") || undefined,
|
||||
selector: mutation.selector,
|
||||
fixHint:
|
||||
"Author the final static src, or bind data-var-src to a declared image/string variable so the selected source is applied before media discovery and extraction.",
|
||||
snippet: truncateSnippet(mutation.raw),
|
||||
});
|
||||
}
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
export const mediaRules: Array<(ctx: LintContext) => HyperframeLintFinding[]> = [
|
||||
// duplicate_media_id + duplicate_media_discovery_risk
|
||||
({ tags }) => {
|
||||
@@ -738,6 +780,7 @@ export const mediaRules: Array<(ctx: LintContext) => HyperframeLintFinding[]> =
|
||||
|
||||
// imperative_media_control
|
||||
findImperativeMediaControlFindings,
|
||||
findRuntimeMediaSrcMutationFindings,
|
||||
|
||||
// audio_volume_double_automation
|
||||
findVolumeDoubleAutomationFindings,
|
||||
|
||||
@@ -14,6 +14,7 @@ export * from "./compositionContract.js";
|
||||
// browser-safe.
|
||||
export { decodeUrlPathVariants } from "./utils/urlPath.js";
|
||||
export { scanVariableUsage, type VariableUsageScan } from "./variableUsage.js";
|
||||
export { extractMediaSrcMutations, type MediaSrcMutation } from "./mediaSrcMutation.js";
|
||||
export {
|
||||
FONT_ALIAS_MAP,
|
||||
FONT_ALIAS_KEYS,
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import * as acorn from "acorn";
|
||||
import * as acornWalk from "acorn-walk";
|
||||
|
||||
export interface MediaSrcMutation {
|
||||
selector: string;
|
||||
operation: "src_assignment" | "set_attribute";
|
||||
raw: string;
|
||||
}
|
||||
|
||||
function parseProgram(script: string): any {
|
||||
try {
|
||||
return acorn.parse(script, { ecmaVersion: "latest", sourceType: "script" });
|
||||
} catch {
|
||||
return acorn.parse(script, { ecmaVersion: "latest", sourceType: "module" });
|
||||
}
|
||||
}
|
||||
|
||||
function literalString(node: any): string | undefined {
|
||||
if (node?.type === "Literal" && typeof node.value === "string") return node.value;
|
||||
if (node?.type === "TemplateLiteral" && node.expressions?.length === 0) {
|
||||
return node.quasis?.[0]?.value?.cooked;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function memberName(node: any): string | undefined {
|
||||
if (node?.type !== "MemberExpression") return undefined;
|
||||
if (!node.computed && node.property?.type === "Identifier") return node.property.name;
|
||||
return literalString(node.property);
|
||||
}
|
||||
|
||||
function lookupSelector(node: any, bindings: ReadonlyMap<string, string>): string | undefined {
|
||||
if (node?.type === "Identifier") return bindings.get(node.name);
|
||||
if (node?.type !== "CallExpression" || node.callee?.type !== "MemberExpression") {
|
||||
return undefined;
|
||||
}
|
||||
const method = memberName(node.callee);
|
||||
if (method !== "getElementById" && method !== "querySelector") return undefined;
|
||||
const value = literalString(node.arguments?.[0]);
|
||||
if (!value) return undefined;
|
||||
return method === "getElementById" ? `#${value}` : value;
|
||||
}
|
||||
|
||||
/** Find literal source writes whose target is a statically resolvable DOM lookup. */
|
||||
export function extractMediaSrcMutations(script: string): MediaSrcMutation[] {
|
||||
try {
|
||||
const ast = parseProgram(script);
|
||||
const bindings = new Map<string, string>();
|
||||
acornWalk.simple(ast, {
|
||||
VariableDeclarator(node: any) {
|
||||
if (node.id?.type !== "Identifier") return;
|
||||
const selector = lookupSelector(node.init, bindings);
|
||||
if (selector) bindings.set(node.id.name, selector);
|
||||
},
|
||||
});
|
||||
|
||||
const mutations: MediaSrcMutation[] = [];
|
||||
acornWalk.simple(ast, {
|
||||
AssignmentExpression(node: any) {
|
||||
if (node.operator !== "=" || memberName(node.left) !== "src") return;
|
||||
const selector = lookupSelector(node.left.object, bindings);
|
||||
if (!selector) return;
|
||||
mutations.push({
|
||||
selector,
|
||||
operation: "src_assignment",
|
||||
raw: script.slice(node.start, node.end),
|
||||
});
|
||||
},
|
||||
CallExpression(node: any) {
|
||||
if (memberName(node.callee) !== "setAttribute") return;
|
||||
if (literalString(node.arguments?.[0])?.toLowerCase() !== "src") return;
|
||||
const selector = lookupSelector(node.callee.object, bindings);
|
||||
if (!selector) return;
|
||||
mutations.push({
|
||||
selector,
|
||||
operation: "set_attribute",
|
||||
raw: script.slice(node.start, node.end),
|
||||
});
|
||||
},
|
||||
});
|
||||
return mutations;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -174,7 +174,7 @@ export interface RenderExtractionObservability {
|
||||
* a coarse proxy for the ts=1784144554 field signal shape (147-clip
|
||||
* composition, 130 word-level caption divs authored-clip-count-scaled
|
||||
* failure). Static scan; dynamic script-inserted timed clips land in
|
||||
* the probe-stage's `hasRuntimeInsertedMedia` path (PR #2474).
|
||||
* the probe-stage's `hasRuntimeMediaChanges` path (PR #2474).
|
||||
*/
|
||||
authoredTimedClipCount?: number;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, expect, it, mock } from "bun:test";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
hasAutoStartVideos,
|
||||
hasScriptedAudioVolumeAutomation,
|
||||
@@ -583,6 +584,113 @@ describe("runProbeStage — forceScreenshot threading", () => {
|
||||
expect(capturedCfgs.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("probes and reconciles synchronous src mutations on existing video and audio", async () => {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
const hash = (value: string) => createHash("sha256").update(value).digest("hex");
|
||||
const videoA = `assets/video-a-${hash("sanitized-video-a")}.mp4`;
|
||||
const videoB = `assets/video-b-${hash("sanitized-video-b")}.mp4`;
|
||||
const audioA = `assets/audio-a-${hash("sanitized-audio-a")}.wav`;
|
||||
const audioB = `assets/audio-b-${hash("sanitized-audio-b")}.wav`;
|
||||
expect(hash("sanitized-video-a")).not.toBe(hash("sanitized-video-b"));
|
||||
expect(hash("sanitized-audio-a")).not.toBe(hash("sanitized-audio-b"));
|
||||
browserMediaResults = [
|
||||
{
|
||||
id: "clip",
|
||||
tagName: "video",
|
||||
src: videoB,
|
||||
start: 0,
|
||||
end: 5,
|
||||
duration: 5,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
volume: 1,
|
||||
muted: true,
|
||||
},
|
||||
{
|
||||
id: "voice",
|
||||
tagName: "audio",
|
||||
src: audioB,
|
||||
start: 0,
|
||||
end: 5,
|
||||
duration: 5,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: true,
|
||||
volume: 1,
|
||||
muted: false,
|
||||
},
|
||||
];
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({});
|
||||
input.composition.duration = 5;
|
||||
input.composition.videos.push({
|
||||
id: "clip",
|
||||
src: videoA,
|
||||
start: 0,
|
||||
end: 5,
|
||||
mediaStart: 0,
|
||||
loop: false,
|
||||
hasAudio: false,
|
||||
});
|
||||
input.composition.audios.push({
|
||||
id: "voice",
|
||||
src: audioA,
|
||||
start: 0,
|
||||
end: 5,
|
||||
mediaStart: 0,
|
||||
layer: 0,
|
||||
volume: 1,
|
||||
type: "audio",
|
||||
});
|
||||
input.compiled.html = `<video id="clip" src="${videoA}"></video>
|
||||
<audio id="voice" src="${audioA}"></audio>
|
||||
<script>
|
||||
const clip = document.getElementById("clip");
|
||||
clip.src = ${JSON.stringify(videoB)};
|
||||
document.querySelector("#voice").setAttribute("src", ${JSON.stringify(audioB)});
|
||||
</script>`;
|
||||
|
||||
await runProbeStage(input);
|
||||
|
||||
expect(capturedCfgs.length).toBeGreaterThan(0);
|
||||
expect(input.composition.videos[0]?.src).toBe(videoB);
|
||||
expect(input.composition.audios[0]?.src).toBe(audioB);
|
||||
expect(mediaPreflightComposition).toBe(input.composition);
|
||||
});
|
||||
|
||||
it("does not probe for img or script src mutations", async () => {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({});
|
||||
input.composition.duration = 5;
|
||||
input.compiled.html = `<img id="poster" src="a.png"><script id="loader"></script>
|
||||
<script>
|
||||
document.getElementById("poster").src = "b.png";
|
||||
document.getElementById("loader").setAttribute("src", "loader-b.js");
|
||||
</script>`;
|
||||
|
||||
await runProbeStage(input);
|
||||
|
||||
expect(capturedCfgs).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("probes when a source child of existing media is mutated", async () => {
|
||||
resetRetryMocks();
|
||||
capturedCfgs.length = 0;
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
const input = makeProbeInput({});
|
||||
input.composition.duration = 5;
|
||||
input.compiled.html = `<video id="clip"><source id="clip-source" src="video-a.mp4"></video>
|
||||
<script>document.getElementById("clip-source").src = "video-b.mp4";</script>`;
|
||||
|
||||
await runProbeStage(input);
|
||||
|
||||
expect(capturedCfgs.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("launches a probe when a static-duration composition uses the Audio constructor", async () => {
|
||||
capturedCfgs.length = 0;
|
||||
const { runProbeStage } = await import("./probeStage.js");
|
||||
|
||||
@@ -43,6 +43,7 @@ import {
|
||||
probeBeginFrameLiveness,
|
||||
} from "@hyperframes/engine";
|
||||
import { fpsToNumber } from "@hyperframes/core";
|
||||
import { extractMediaSrcMutations } from "@hyperframes/parsers";
|
||||
import type { CompiledComposition } from "../../htmlCompiler.js";
|
||||
import {
|
||||
discoverMediaFromBrowser,
|
||||
@@ -193,16 +194,41 @@ function reconcileBrowserMediaEnd(
|
||||
* extraction, even when the root duration is already known. External script
|
||||
* sources have no inline text to inspect and remain a known heuristic gap.
|
||||
*/
|
||||
function hasRuntimeInsertedMedia(html: string): boolean {
|
||||
function hasRuntimeMediaChanges(html: string): boolean {
|
||||
const { document } = parseHTML(html);
|
||||
const scriptBodies = [...document.querySelectorAll("script")]
|
||||
.map((script) => script.textContent ?? "")
|
||||
.join("\n");
|
||||
return (
|
||||
/\bcreateElement\s*\(\s*["'`](?:video|audio)["'`]\s*\)/i.test(scriptBodies) ||
|
||||
/\bnew\s+(?:Audio|Video)\s*\(/i.test(scriptBodies) ||
|
||||
/<(?:video|audio)\b[^>]*>/i.test(scriptBodies)
|
||||
const scriptBodies = [...document.querySelectorAll("script")].map(
|
||||
(script) => script.textContent ?? "",
|
||||
);
|
||||
const insertedMedia = scriptBodies.some(
|
||||
(script) =>
|
||||
/\bcreateElement\s*\(\s*["'`](?:video|audio)["'`]\s*\)/i.test(script) ||
|
||||
/\bnew\s+(?:Audio|Video)\s*\(/i.test(script) ||
|
||||
/<(?:video|audio)\b[^>]*>/i.test(script),
|
||||
);
|
||||
if (insertedMedia) return true;
|
||||
|
||||
const isManagedMedia = (element: Element): boolean => {
|
||||
const name = element.tagName.toLowerCase();
|
||||
if (name === "video" || name === "audio") return true;
|
||||
return name === "source" && element.closest("video, audio") !== null;
|
||||
};
|
||||
for (const script of scriptBodies) {
|
||||
for (const mutation of extractMediaSrcMutations(script)) {
|
||||
try {
|
||||
const id = /^#[A-Za-z_][\w-]*$/.test(mutation.selector) ? mutation.selector.slice(1) : null;
|
||||
const idTarget = id ? document.getElementById(id) : null;
|
||||
const targets = id
|
||||
? idTarget
|
||||
? [idTarget]
|
||||
: []
|
||||
: [...document.querySelectorAll(mutation.selector)];
|
||||
if (targets.some(isManagedMedia)) return true;
|
||||
} catch {
|
||||
// Invalid selectors are diagnosed by lint and cannot prove a media target here.
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -214,9 +240,9 @@ function hasRuntimeInsertedMedia(html: string): boolean {
|
||||
* `resolveCompositionElementCount` / `resolveDeShortBand`). Notably NONE of
|
||||
* these conditions fire for a known-duration, media-free composition that
|
||||
* builds thousands of `div`/`span` nodes in its own init script —
|
||||
* `hasRuntimeInsertedMedia` matches only `createElement("video"|"audio")` —
|
||||
* so that shape is measured statically and must never reach the band's
|
||||
* `applied` cohort (review finding, R4).
|
||||
* `hasRuntimeMediaChanges` matches only media creation/source changes — so
|
||||
* that shape is measured statically and must never reach the band's `applied`
|
||||
* cohort (review finding, R4).
|
||||
*/
|
||||
export function probeRequiresBrowser(args: {
|
||||
durationSeconds: number;
|
||||
@@ -269,7 +295,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
composition.audios.length,
|
||||
);
|
||||
const hasVariableMedia = hasVariableBoundMedia(compiled.html, job.config.variables);
|
||||
const hasInsertedMedia = hasRuntimeInsertedMedia(compiled.html);
|
||||
const hasInsertedMedia = hasRuntimeMediaChanges(compiled.html);
|
||||
const needsBrowser = probeRequiresBrowser({
|
||||
durationSeconds: composition.duration,
|
||||
unresolvedCompositionCount: compiled.unresolvedCompositions.length,
|
||||
@@ -286,7 +312,7 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
|
||||
reasons.push(`${compiled.unresolvedCompositions.length} unresolved composition(s)`);
|
||||
if (hasAutoStart) reasons.push("auto-start video(s)");
|
||||
if (hasScriptedAudio) reasons.push("scripted audio volume");
|
||||
if (hasInsertedMedia) reasons.push("runtime-inserted media");
|
||||
if (hasInsertedMedia) reasons.push("runtime-created or source-mutated media");
|
||||
if (hasVariableMedia) reasons.push("variable-bound media source(s)");
|
||||
|
||||
log.info("Launching browser for composition probe...", {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* Per-clip render-time frame-coverage accounting + threshold fail-loud gate.
|
||||
*
|
||||
* Sibling to #2474's `hasRuntimeInsertedMedia` probe: that PR guarantees the
|
||||
* Sibling to #2474's `hasRuntimeMediaChanges` probe: that PR guarantees the
|
||||
* DISCOVERY of runtime-inserted media (so the browser probe launches and
|
||||
* reconciles element identity). This module owns the DELIVERY side —
|
||||
* for each authored/discovered video clip on the timeline, did the
|
||||
@@ -258,7 +258,7 @@ export function assertVideoFrameCoverage(
|
||||
* composition is queryable in telemetry (the ts=1784144554 field signal
|
||||
* shape). Runtime `syncTimedElementVisibility` iterates the same set at
|
||||
* render time; counting statically here is a coarse proxy — dynamic
|
||||
* script-inserted `[data-start]` divs land in `hasRuntimeInsertedMedia`'s
|
||||
* script-inserted `[data-start]` divs land in `hasRuntimeMediaChanges`'s
|
||||
* probe path (PR #2474), not this static scan.
|
||||
*/
|
||||
export function countAuthoredTimedClips(html: string): number {
|
||||
|
||||
Reference in New Issue
Block a user