mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(engine): a render no longer finishes without an image that loaded late (#7304)
* fix(engine): a render no longer finishes without an image that loaded late * style(engine): format the frame gate * fix(engine): an image outside the frame no longer stops a render * fix(engine): a render stops and warns when a frame's image never arrives, with no cloud retries * fix(producer): a cloud chunk keeps the missing image's error name so retries stop
This commit is contained in:
@@ -460,6 +460,7 @@ Resources:
|
||||
- PlanV2IntegrityError
|
||||
- PLAN_ARTIFACT_DIGEST_MISMATCH
|
||||
- ChromeBinaryUnavailableError
|
||||
- ImageMissingFromFrameError
|
||||
MaxAttempts: 0
|
||||
- ErrorEquals: [Lambda.TooManyRequestsException]
|
||||
IntervalSeconds: 5
|
||||
|
||||
@@ -234,6 +234,14 @@ describe("HyperframesRenderStack — snapshot", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("stops a chunk's retries for an image a frame shows but never got, in SAM and CDK", () => {
|
||||
for (const definition of [SYNTHED.definition, readSamDefinition()]) {
|
||||
const errors = new Set<string>();
|
||||
collectNonRetryableErrors(getV2TaskStates(definition).RenderChunkV2, errors);
|
||||
expect(errors.has("ImageMissingFromFrameError")).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("lists the error the SDK checks for before rendering in every SAM and CDK plan task", () => {
|
||||
for (const definition of [SYNTHED.definition, readSamDefinition()]) {
|
||||
const errors = new Set<string>();
|
||||
|
||||
@@ -250,6 +250,7 @@ export class HyperframesRenderStack extends Construct {
|
||||
"PlanV2IntegrityError",
|
||||
"PLAN_ARTIFACT_DIGEST_MISMATCH",
|
||||
"ChromeBinaryUnavailableError",
|
||||
"ImageMissingFromFrameError",
|
||||
];
|
||||
const NON_RETRYABLE_ASSEMBLE = [
|
||||
"FFMPEG_VERSION_MISMATCH",
|
||||
|
||||
@@ -133,6 +133,13 @@ describe("render diagnostic wire", () => {
|
||||
}),
|
||||
"COMPOSITION_SETUP_TIMEOUT",
|
||||
],
|
||||
[
|
||||
new CaptureFailure({
|
||||
kind: "authoring",
|
||||
message: `Frame 0 shows ${PRIVATE}, which did not load in time, so the render stopped rather than make a video missing it.`,
|
||||
}),
|
||||
"IMAGE_MISSING_FROM_FRAME",
|
||||
],
|
||||
[
|
||||
new CaptureFailure({ kind: "authoring", message: PRIVATE, pageErrorLines: [PRIVATE] }),
|
||||
"PAGE_ERROR",
|
||||
|
||||
@@ -42,6 +42,8 @@ function producerCodeOf(error: unknown): string | undefined {
|
||||
function captureCodeOf(error: unknown, message: string): string | undefined {
|
||||
if (/Composition has zero duration/.test(message)) return "COMPOSITION_ZERO_DURATION";
|
||||
if (/Composition setup did not finish/.test(message)) return "COMPOSITION_SETUP_TIMEOUT";
|
||||
if (/so the render stopped rather than make a video missing it/.test(message))
|
||||
return "IMAGE_MISSING_FROM_FRAME";
|
||||
if (!(error instanceof CaptureFailure)) return undefined;
|
||||
return error.pageErrorLines.length > 0 ? "PAGE_ERROR" : `CAPTURE_${error.kind.toUpperCase()}`;
|
||||
}
|
||||
|
||||
@@ -155,6 +155,7 @@ export {
|
||||
type DiscardWarmupInnerCapture,
|
||||
type StaticVerificationOutcome,
|
||||
VfxFailureError,
|
||||
ImageMissingFromFrameError,
|
||||
waitForImagesReady,
|
||||
evaluateHfDiagnostic,
|
||||
buildZeroDurationDiagnostic,
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { Page } from "puppeteer-core";
|
||||
import {
|
||||
ImageMissingFromFrameError,
|
||||
waitForShownLateImages,
|
||||
type CaptureSession,
|
||||
} from "./frameCapture.js";
|
||||
|
||||
type Reading = { arrived: string[]; waiting: string[]; broken: string[] };
|
||||
|
||||
function sessionReading(readings: Reading[], late: string[], playerReadyTimeout = 60_000) {
|
||||
const evaluate = vi.fn(async () => readings.shift() ?? { arrived: [], waiting: [], broken: [] });
|
||||
const session = {
|
||||
page: { evaluate } as unknown as Page,
|
||||
lateImages: new Set(late),
|
||||
warnings: [],
|
||||
config: { playerReadyTimeout },
|
||||
} as unknown as CaptureSession;
|
||||
return { session, evaluate };
|
||||
}
|
||||
|
||||
const PHOTO = "http://localhost:1/assets/photo.png";
|
||||
|
||||
describe("waitForShownLateImages", () => {
|
||||
it("never touches the page when no image was late", async () => {
|
||||
const { session, evaluate } = sessionReading([], []);
|
||||
await waitForShownLateImages(session, 0);
|
||||
expect(evaluate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("waits for a late image the frame shows, then stops asking about it", async () => {
|
||||
const { session, evaluate } = sessionReading(
|
||||
[
|
||||
{ arrived: [], waiting: [PHOTO], broken: [] },
|
||||
{ arrived: [PHOTO], waiting: [], broken: [] },
|
||||
],
|
||||
[PHOTO],
|
||||
);
|
||||
await waitForShownLateImages(session, 12);
|
||||
expect(evaluate).toHaveBeenCalledTimes(2);
|
||||
expect(session.lateImages.size).toBe(0);
|
||||
await waitForShownLateImages(session, 13);
|
||||
expect(evaluate).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("captures on when the late image is not in this frame, and keeps it listed", async () => {
|
||||
const { session } = sessionReading([{ arrived: [], waiting: [], broken: [] }], [PHOTO]);
|
||||
await waitForShownLateImages(session, 0);
|
||||
expect([...session.lateImages]).toEqual([PHOTO]);
|
||||
});
|
||||
|
||||
// fails without the fix: the frame was captured with the image missing and the render reported success.
|
||||
it("stops the render when a shown image has not arrived by the limit, naming frame and path", async () => {
|
||||
const { session } = sessionReading([{ arrived: [], waiting: [PHOTO], broken: [] }], [PHOTO], 0);
|
||||
const stopped = await waitForShownLateImages(session, 360).catch((error: unknown) => error);
|
||||
expect(stopped).toBeInstanceOf(ImageMissingFromFrameError);
|
||||
expect(stopped).toMatchObject({ frameIndex: 360, sources: [PHOTO], failedToLoad: false });
|
||||
expect((stopped as Error).message).toContain("did not load in time");
|
||||
expect(session.warnings).toEqual([
|
||||
expect.objectContaining({
|
||||
code: "image_missing_from_frame",
|
||||
message: (stopped as Error).message,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("stops at once for a shown image that failed to load, since waiting cannot help", async () => {
|
||||
const { session, evaluate } = sessionReading(
|
||||
[{ arrived: [], waiting: [], broken: [PHOTO] }],
|
||||
[PHOTO],
|
||||
);
|
||||
const stopped = await waitForShownLateImages(session, 5).catch((error: unknown) => error);
|
||||
expect(stopped).toBeInstanceOf(ImageMissingFromFrameError);
|
||||
expect((stopped as Error).message).toContain("failed to load");
|
||||
expect(evaluate).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -153,7 +153,8 @@ type CaptureStep =
|
||||
| "waiting for page seek handlers"
|
||||
| "loading color-grading LUTs"
|
||||
| "compositing the page"
|
||||
| "capturing the frame";
|
||||
| "capturing the frame"
|
||||
| "waiting for a late image";
|
||||
|
||||
export interface CaptureSession {
|
||||
browser: Browser;
|
||||
@@ -246,6 +247,8 @@ export interface CaptureSession {
|
||||
subTimelineWaitMemo: SubTimelineWaitMemo;
|
||||
/** Structured readiness warnings surfaced to the producer's render policy. */
|
||||
warnings: CaptureWarning[];
|
||||
/** Images still loading when capture began: a frame that shows one waits for it (waitForShownLateImages). */
|
||||
lateImages?: Set<string>;
|
||||
initTelemetry?: {
|
||||
initDurationMs: number;
|
||||
tweenCount: number;
|
||||
@@ -2519,6 +2522,99 @@ function recordCaptureWarnings(session: CaptureSession, warnings: readonly Captu
|
||||
}
|
||||
}
|
||||
|
||||
function recordReadiness(session: CaptureSession, warnings: readonly CaptureWarning[]): void {
|
||||
recordCaptureWarnings(session, warnings);
|
||||
for (const warning of warnings) {
|
||||
if (warning.code !== "media_readiness_timeout" || warning.details?.mediaType !== "image")
|
||||
continue;
|
||||
session.lateImages ??= new Set();
|
||||
for (const source of warning.details.sources ?? []) session.lateImages.add(source);
|
||||
}
|
||||
}
|
||||
|
||||
/** Thrown when a frame would show an image that never arrived: the video would silently lack it. */
|
||||
export class ImageMissingFromFrameError extends Error {
|
||||
constructor(
|
||||
readonly frameIndex: number,
|
||||
readonly sources: readonly string[],
|
||||
readonly failedToLoad: boolean,
|
||||
) {
|
||||
super(
|
||||
`Frame ${frameIndex} shows ${sources.join(", ")}, which ${failedToLoad ? "failed to load" : "did not load in time"}, ` +
|
||||
`so the render stopped rather than make a video missing it. ` +
|
||||
(failedToLoad
|
||||
? `Check that the image exists and can be opened.`
|
||||
: `Try fewer workers (--workers 1) or a longer --player-ready-timeout.`),
|
||||
);
|
||||
this.name = "ImageMissingFromFrameError";
|
||||
}
|
||||
}
|
||||
|
||||
const LATE_IMAGE_POLL_MS = 100;
|
||||
|
||||
/** Before a frame: wait for any late image it shows, for as long as the page was given to get ready, else stop. */
|
||||
export async function waitForShownLateImages(
|
||||
session: CaptureSession,
|
||||
frameIndex: number,
|
||||
): Promise<void> {
|
||||
const lateImages = session.lateImages;
|
||||
if (!lateImages?.size) return;
|
||||
session.captureStep = "waiting for a late image";
|
||||
const timeoutMs = session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout;
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
for (;;) {
|
||||
const { arrived, waiting, broken } = await session.page.evaluate(async (urls: string[]) => {
|
||||
const listed = new Set(urls);
|
||||
const found = Array.from(document.querySelectorAll("img")).filter((img) =>
|
||||
listed.has(img.src),
|
||||
);
|
||||
const loaded = found.filter((img) => img.complete && img.naturalWidth > 0);
|
||||
await Promise.allSettled(loaded.map((img) => img.decode()));
|
||||
const inViewport = (img: HTMLImageElement) => {
|
||||
const box = img.getBoundingClientRect();
|
||||
return (
|
||||
box.width > 0 &&
|
||||
box.height > 0 &&
|
||||
box.right > 0 &&
|
||||
box.bottom > 0 &&
|
||||
box.left < window.innerWidth &&
|
||||
box.top < window.innerHeight
|
||||
);
|
||||
};
|
||||
const shown = found.filter(
|
||||
(img) =>
|
||||
!loaded.includes(img) &&
|
||||
img.checkVisibility({ opacityProperty: true, visibilityProperty: true }) &&
|
||||
inViewport(img),
|
||||
);
|
||||
return {
|
||||
arrived: loaded.map((img) => img.src),
|
||||
waiting: shown.filter((img) => !img.complete).map((img) => img.src),
|
||||
broken: shown.filter((img) => img.complete).map((img) => img.src),
|
||||
};
|
||||
}, Array.from(lateImages));
|
||||
for (const source of arrived) lateImages.delete(source);
|
||||
const missing = Date.now() >= deadline ? [...waiting, ...broken] : broken;
|
||||
if (missing.length > 0) {
|
||||
const stopped = new ImageMissingFromFrameError(
|
||||
frameIndex,
|
||||
[...new Set(missing.map(sanitizeDiagnosticUrl))].sort(),
|
||||
broken.length > 0,
|
||||
);
|
||||
recordCaptureWarnings(session, [
|
||||
{
|
||||
code: "image_missing_from_frame",
|
||||
message: stopped.message,
|
||||
details: { mediaType: "image", sources: [...stopped.sources] },
|
||||
},
|
||||
]);
|
||||
throw stopped;
|
||||
}
|
||||
if (waiting.length === 0) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, LATE_IMAGE_POLL_MS));
|
||||
}
|
||||
}
|
||||
|
||||
/** Report background urls that timed out decoding; the shown url drops credentials and query. */
|
||||
export function recordBackgroundDecodeTimeouts(
|
||||
session: CaptureSession,
|
||||
@@ -3109,7 +3205,7 @@ export async function initializeSession(session: CaptureSession): Promise<void>
|
||||
`Continuing render. Affected videos will appear as blank/black frames.`,
|
||||
);
|
||||
}
|
||||
recordCaptureWarnings(
|
||||
recordReadiness(
|
||||
session,
|
||||
await collectMediaReadinessWarnings(page, skipVideoIds, pageReadyTimeout),
|
||||
);
|
||||
@@ -3251,7 +3347,7 @@ export async function initializeSession(session: CaptureSession): Promise<void>
|
||||
`Continuing render. Affected videos will appear as blank/black frames.`,
|
||||
);
|
||||
}
|
||||
recordCaptureWarnings(
|
||||
recordReadiness(
|
||||
session,
|
||||
await collectMediaReadinessWarnings(page, bfSkipVideoIds, pageReadyTimeout),
|
||||
);
|
||||
@@ -3426,6 +3522,7 @@ async function prepareFrameForCapture(
|
||||
const seekStart = Date.now();
|
||||
session.captureStep = "seeking the timeline";
|
||||
const hasPendingComposite = await seekPageTimeline(page, quantizedTime, seekOptions);
|
||||
await waitForShownLateImages(session, frameIndex);
|
||||
|
||||
session.captureStep = "decoding CSS background images";
|
||||
// Mocked pages answer evaluate() with undefined, so tolerate a missing list.
|
||||
@@ -4860,6 +4957,7 @@ async function captureAccumulatedFrame(
|
||||
const eventfulSeekStart = Date.now();
|
||||
session.captureStep = "seeking the timeline";
|
||||
await seekPageTimeline(session.page, frameTime, undefined);
|
||||
await waitForShownLateImages(session, frameIndex);
|
||||
const totals = { seekMs: Date.now() - eventfulSeekStart, beforeCaptureMs: 0, screenshotMs: 0 };
|
||||
if (session.onBeforeCapture) {
|
||||
// Sub-samples leave a video's style alone: inject at frameTime first so the held frame
|
||||
@@ -5283,6 +5381,19 @@ export async function captureFramesBatchPipelined(
|
||||
}
|
||||
assertVfxIntact(session);
|
||||
for (const frameIndex of frameIndices) assertNotRiskFrame(session, frameIndex);
|
||||
// The fused batch seeks inside the page, with no stop between frames to wait for a late image.
|
||||
if (session.lateImages?.size) {
|
||||
const oneByOne: Array<{ frameIndex: number; encodeResult: Promise<Buffer> }> = [];
|
||||
for (const [i, frameIndex] of frameIndices.entries()) {
|
||||
const { encodeResult } = await captureFrameToBufferPipelined(
|
||||
session,
|
||||
frameIndex,
|
||||
times[i] ?? 0,
|
||||
);
|
||||
oneByOne.push({ frameIndex, encodeResult });
|
||||
}
|
||||
return oneByOne;
|
||||
}
|
||||
const startTime = Date.now();
|
||||
const fps = fpsToNumber(options.fps);
|
||||
const quantized = times.map((t) => quantizeTimeToFrame(t, fps));
|
||||
|
||||
@@ -41,6 +41,7 @@ export interface SubTimelineWaitMemo {
|
||||
export type CaptureWarningCode =
|
||||
| "media_readiness_timeout"
|
||||
| "media_load_failed"
|
||||
| "image_missing_from_frame"
|
||||
| "audio_processing_failed"
|
||||
| "sub_timeline_readiness_timeout"
|
||||
| "sub_timeline_script_failure"
|
||||
|
||||
@@ -429,6 +429,41 @@ describe("createApp HTTP mapping", () => {
|
||||
expect(await res.json()).toMatchObject({ error: "PLAN_PROTOCOL_UNSUPPORTED" });
|
||||
});
|
||||
|
||||
it("does not retry a chunk whose frame shows an image that never arrived", async () => {
|
||||
const gcs = new FakeGcs();
|
||||
await seedProjectTar(gcs, planBody.ProjectGcsUri);
|
||||
const root = mkTmp("hf-image-missing-");
|
||||
const deps = depsWith(gcs, {
|
||||
planV2WithPublisher: async (_projectDir, _config, publisher) => {
|
||||
const v1Dir = join(root, "v1");
|
||||
makeMinimalV1PlanDir(v1Dir, false);
|
||||
return publishPlanV2FromExecutionPlan(v1Dir, publisher);
|
||||
},
|
||||
renderChunk: async () => {
|
||||
throw Object.assign(new Error("test"), { name: "ImageMissingFromFrameError" });
|
||||
},
|
||||
});
|
||||
const planned = await dispatch(planBody, deps);
|
||||
if (!("PlanProtocol" in planned) || planned.PlanProtocol !== "v2") {
|
||||
throw new Error("expected v2 plan result");
|
||||
}
|
||||
const res = await createApp(deps).request("/", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
Action: "renderChunk",
|
||||
PlanV2ManifestGcsUri: planned.PlanV2ManifestGcsUri,
|
||||
PlanV2ArtifactGcsPrefix: planned.PlanV2ArtifactGcsPrefix,
|
||||
PlanHash: planned.PlanHash,
|
||||
ChunkIndex: 0,
|
||||
ChunkOutputGcsPrefix: "gs://b/renders/image-missing/",
|
||||
Format: "mp4",
|
||||
}),
|
||||
});
|
||||
expect(res.status).toBe(400);
|
||||
expect(await res.json()).toMatchObject({ error: "ImageMissingFromFrameError" });
|
||||
});
|
||||
|
||||
it.each([
|
||||
["INVALID_VIDEO_METADATA", 400],
|
||||
[PLAN_V2_INTEGRITY_UNRECOVERABLE, 400],
|
||||
|
||||
@@ -683,6 +683,7 @@ const NON_RETRYABLE_ERROR_NAMES = new Set([
|
||||
"BROWSER_GPU_NOT_SOFTWARE",
|
||||
"FONT_FETCH_FAILED",
|
||||
"ChromeBinaryUnavailableError",
|
||||
"ImageMissingFromFrameError",
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -383,14 +383,26 @@ export async function runCaptureWithScreenshotFallback<T>(input: {
|
||||
try {
|
||||
return await input.run(input.forceScreenshot);
|
||||
} catch (error) {
|
||||
if (input.forceScreenshot || (await input.canRetry?.()) === false) throw error;
|
||||
if (!shouldRetryChunkCaptureWithScreenshot(error)) throw error;
|
||||
if (input.forceScreenshot || (await input.canRetry?.()) === false)
|
||||
throw imageMissingIn(error) ?? error;
|
||||
if (!shouldRetryChunkCaptureWithScreenshot(error)) throw imageMissingIn(error) ?? error;
|
||||
input.onFallback?.(error);
|
||||
await input.resetForScreenshotRetry();
|
||||
return await input.run(true);
|
||||
return await input.run(true).catch((retryError: unknown) => {
|
||||
throw imageMissingIn(retryError) ?? retryError;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function imageMissingIn(error: unknown): unknown {
|
||||
let e = error;
|
||||
for (let depth = 0; depth < 5 && typeof e === "object" && e !== null; depth++) {
|
||||
if ((e as { name?: unknown }).name === "ImageMissingFromFrameError") return e;
|
||||
e = (e as { cause?: unknown }).cause;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe an initialized distributed session at a monotonic tick between warmup
|
||||
* and frame zero. `true` means the entire chunk should use screenshot mode.
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import {
|
||||
CaptureFailure,
|
||||
type CaptureSession,
|
||||
ImageMissingFromFrameError,
|
||||
createFrameLookupTable,
|
||||
readWebGlVendorInfoFromCanvas,
|
||||
resolveConfig,
|
||||
@@ -15,6 +17,7 @@ import {
|
||||
runCaptureWithScreenshotFallback,
|
||||
shouldRetryChunkCaptureWithScreenshot,
|
||||
} from "./renderChunk.js";
|
||||
import { CaptureStageError } from "../render/captureStageError.js";
|
||||
|
||||
const originalProbeTimeout = process.env.PRODUCER_BEGINFRAME_PROBE_TIMEOUT_MS;
|
||||
|
||||
@@ -306,3 +309,48 @@ describe("a chunk whose workers launched in different capture modes", () => {
|
||||
expect(() => observedChunkCaptureMode([{ captureMode: "gpu" }])).toThrow(/invalid mode/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a chunk whose frame shows an image that never arrived", () => {
|
||||
const stopped = () => new ImageMissingFromFrameError(0, ["photo.png"], false);
|
||||
const throughMultiWorkerCapture = () =>
|
||||
new CaptureStageError({
|
||||
cause: new CaptureFailure({
|
||||
kind: "authoring",
|
||||
message: "[Parallel] Capture failed: Worker 0",
|
||||
cause: new CaptureFailure({ kind: "authoring", message: "worker", cause: stopped() }),
|
||||
}),
|
||||
browserConsole: [],
|
||||
});
|
||||
const run = (first: unknown, second?: unknown) =>
|
||||
runCaptureWithScreenshotFallback({
|
||||
forceScreenshot: false,
|
||||
run: async (forceScreenshot) => {
|
||||
throw forceScreenshot ? second : first;
|
||||
},
|
||||
resetForScreenshotRetry: () => {},
|
||||
});
|
||||
|
||||
it("throws the image's own error from a multi-worker capture, so cloud retry lists see its name", async () => {
|
||||
await expect(run(throughMultiWorkerCapture())).rejects.toMatchObject({
|
||||
name: "ImageMissingFromFrameError",
|
||||
sources: ["photo.png"],
|
||||
});
|
||||
});
|
||||
|
||||
it("throws it from a single-worker capture too", async () => {
|
||||
const wrapped = new CaptureStageError({ cause: stopped(), browserConsole: [] });
|
||||
await expect(run(wrapped)).rejects.toMatchObject({ name: "ImageMissingFromFrameError" });
|
||||
});
|
||||
|
||||
it("throws it when it comes from the screenshot retry", async () => {
|
||||
const beginFrameFailed = new Error("HeadlessExperimental.beginFrame timed out");
|
||||
await expect(run(beginFrameFailed, throughMultiWorkerCapture())).rejects.toMatchObject({
|
||||
name: "ImageMissingFromFrameError",
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves any other capture failure as it was", async () => {
|
||||
const other = new CaptureStageError({ cause: new Error("boom"), browserConsole: [] });
|
||||
await expect(run(other)).rejects.toBe(other);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user