fix(engine): hold finite video slots past EOF (#3962)

Allow explicit finite non-looping video slots to use the existing held-final-frame extraction path when data-media-start is at or beyond source duration. Warn during project lint for local static slots that will use this behavior.

Co-authored-by: heygengenesis[bot] <262951085+heygengenesis[bot]@users.noreply.github.com>
Co-authored-by: miguel.sierra <229591595+miguel-heygen@users.noreply.github.com>

packages/engine/src/services/videoFrameExtractor.test.ts

@@ -334,6 +334,34 @@ describe("resolveVideoExtractionDuration", () => {
     ).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
   });
 
+  it("plans a one-frame held tail when an explicit non-looping slot starts past EOF", () => {
+    expect(resolveVideoExtractionWindow(video({ end: 6, mediaStart: 5 }), metadata(2))).toEqual({
+      compositionStart: 0,
+      mediaStart: 1.999999,
+      durationSeconds: 0.000001,
+      preserveTimelineEnd: true,
+      ensureFinalFrame: true,
+    });
+  });
+
+  it("keeps the playable suffix when an explicit non-looping slot starts just inside EOF", () => {
+    expect(
+      resolveVideoExtractionWindow(video({ end: 6, mediaStart: 1.9 }), metadata(2), 6),
+    ).toEqual({
+      compositionStart: 0,
+      mediaStart: 1.9,
+      durationSeconds: 0.10000000000000009,
+      preserveTimelineEnd: true,
+      ensureFinalFrame: true,
+    });
+  });
+
+  it("rejects a looping explicit slot that starts at source EOF", () => {
+    expect(() =>
+      resolveVideoExtractionWindow(video({ end: 6, mediaStart: 2, loop: true }), metadata(2), 6),
+    ).toThrowError(expect.objectContaining({ kind: "media_start_out_of_range", retryable: false }));
+  });
+
   it("rebases a loop phase when the visible window stays within one cycle", () => {
     expect(
       resolveVideoExtractionWindow(
@@ -1548,6 +1576,48 @@ describe.skipIf(!HAS_FFMPEG)("held tails on sparse-timestamp sources", () => {
     },
     30_000,
   );
+
+  it("renders the same final decoded frame just inside and past EOF", async () => {
+    const metadata = await extractVideoMetadata(cfrFixture);
+    const sourceDuration = metadata.videoStreamDurationSeconds;
+    const outputDir = mkdtempSync(join(fixtureDir, "eof-out-"));
+    const videos: VideoElement[] = [
+      {
+        id: "just-inside-eof",
+        src: cfrFixture,
+        start: 0,
+        end: 5,
+        mediaStart: sourceDuration - 0.001,
+        loop: false,
+        hasAudio: false,
+      },
+      {
+        id: "past-eof",
+        src: cfrFixture,
+        start: 0,
+        end: 5,
+        mediaStart: sourceDuration + 1,
+        loop: false,
+        hasAudio: false,
+      },
+    ];
+
+    const result = await extractAllVideoFrames(videos, fixtureDir, {
+      fps: 30,
+      format: "png",
+      outputDir,
+      timelineEnd: 5,
+    });
+
+    expect(result.errors).toEqual([]);
+    expect(result.extracted).toHaveLength(2);
+    const insideFrame = result.extracted[0]?.framePaths.get(0);
+    const pastFrame = result.extracted[1]?.framePaths.get(0);
+    expect(insideFrame).toBeDefined();
+    expect(pastFrame).toBeDefined();
+    if (!insideFrame || !pastFrame) throw new Error("expected both final-frame outputs");
+    expect(readFileSync(pastFrame)).toEqual(readFileSync(insideFrame));
+  }, 30_000);
 });
 
 // Regression test for the VFR (variable frame rate) freeze bug.

packages/engine/src/services/videoFrameExtractor.ts

@@ -1015,6 +1015,11 @@ type TimelineWindowVideo = Pick<VideoElement, "start" | "end" | "mediaStart"> &
   Partial<Pick<VideoElement, "playbackRate">> &
   Partial<Pick<VideoElement, "loop">>;
 
+function canHoldFinalFramePastEof(video: TimelineWindowVideo): boolean {
+  const timelineDuration = video.end - video.start;
+  return !video.loop && Number.isFinite(timelineDuration) && timelineDuration > 0;
+}
+
 // Logical duration assigned to a one-frame held-tail representation. This is
 // deliberately below any supported output frame interval: coverage expects
 // one frame, while FFmpeg seeks to the separately probed real frame timestamp.
@@ -1118,6 +1123,18 @@ export function resolveTimelineExtractionWindow(
         },
         visibleDuration,
       );
+    } else if (canHoldFinalFramePastEof(video)) {
+      const logicalDuration = Math.min(sourceDuration, FINAL_FRAME_LOGICAL_DURATION_SECONDS);
+      return withTimelineDuration(
+        {
+          compositionStart: video.start,
+          mediaStart: sourceDuration - logicalDuration,
+          durationSeconds: logicalDuration,
+          preserveTimelineEnd: true,
+          ensureFinalFrame: true,
+        },
+        visibleDuration,
+      );
     }
   }
   return withTimelineDuration(
@@ -1155,7 +1172,10 @@ export async function resolveFinalFrameExtractionWindow(
   if (window.mediaStart < finalFrameTimestamp - 1e-9) return window;
 
   const sourceRemaining = playableDuration - video.mediaStart;
-  const logicalDuration = Math.min(sourceRemaining, FINAL_FRAME_LOGICAL_DURATION_SECONDS);
+  const logicalDuration = Math.min(
+    Math.max(sourceRemaining, window.durationSeconds),
+    FINAL_FRAME_LOGICAL_DURATION_SECONDS,
+  );
   return {
     compositionStart: Math.max(0, video.start),
     mediaStart: playableDuration - logicalDuration,
@@ -1184,7 +1204,9 @@ export function resolveVideoExtractionWindow(
       `Playable video stream duration is ${playableDuration}s`,
     );
   }
-  if (video.mediaStart >= playableDuration) {
+  const requestedTimelineDuration = video.end - video.start;
+  const heldPastEof = video.mediaStart >= playableDuration && canHoldFinalFramePastEof(video);
+  if (video.mediaStart >= playableDuration && !heldPastEof) {
     throw new VideoSourceExtractionError(
       "media_start_out_of_range",
       false,
@@ -1193,13 +1215,17 @@ export function resolveVideoExtractionWindow(
     );
   }
   const playbackRate = normalizePlaybackRate(video.playbackRate ?? 1);
-  const requestedTimelineDuration = video.end - video.start;
   const resolvedDuration =
     Number.isFinite(requestedTimelineDuration) && requestedTimelineDuration > 0
       ? requestedTimelineDuration
       : resolveSegmentDuration(requestedTimelineDuration, video.mediaStart, playableDuration) /
         playbackRate;
-  return resolveTimelineExtractionWindow(video, resolvedDuration, timelineEnd, playableDuration);
+  return resolveTimelineExtractionWindow(
+    video,
+    resolvedDuration,
+    timelineEnd ?? (heldPastEof ? video.end : undefined),
+    playableDuration,
+  );
 }
 
 export function resolveVideoExtractionDuration(
@@ -1723,11 +1749,10 @@ export async function extractAllVideoFrames(
         const metadata = videoMetadata[i];
         if (!entry || !metadata) continue;
 
-        // Guard against mediaStart past EOF — FFmpeg's `-ss` silently produces
-        // a 0-byte file when seeking beyond the source duration, and the
-        // downstream extractor then points at a broken input.
+        // Guard past-EOF windows that cannot use the non-looping held-tail plan.
+        // FFmpeg's `-ss` otherwise silently produces a 0-byte intermediate.
         const playableDuration = resolvePlayableVideoDuration(metadata);
-        if (entry.video.mediaStart >= playableDuration) {
+        if (entry.video.mediaStart >= playableDuration && !canHoldFinalFramePastEof(entry.video)) {
           errors.push({
             videoId: entry.video.id,
             kind: "media_start_out_of_range",

packages/lint/src/hevcPreviewLint.ts

@@ -2,6 +2,7 @@ import { execFile } from "node:child_process";
 import { existsSync } from "node:fs";
 import { join } from "node:path";
 import { rewriteAssetPath } from "@hyperframes/parsers/asset-paths";
+import { parseNumeric } from "@hyperframes/parsers/composition-contract";
 import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
 import {
   cleanAssetUrl,
@@ -10,6 +11,7 @@ import {
   maskNonScannableRanges,
   resolveExistingLocalAsset,
 } from "@hyperframes/parsers/asset-resolution";
+import { parseHTML } from "linkedom";
 import type { HyperframeLintFinding } from "./types.js";
 import { mediaSrcTagRe } from "./utils";
 
@@ -34,6 +36,40 @@ function execFileAsync(file: string, args: string[]): Promise<string> {
   });
 }
 
+async function mapWithProbeConcurrency<T, R>(
+  entries: T[],
+  probe: (entry: T) => Promise<R>,
+): Promise<R[]> {
+  const results = new Array<R>(entries.length);
+  let nextIndex = 0;
+  const workerCount = Math.min(PROBE_CONCURRENCY, entries.length);
+  const runWorker = async (): Promise<void> => {
+    for (;;) {
+      const index = nextIndex++;
+      if (index >= entries.length) return;
+      const entry = entries[index];
+      if (entry !== undefined) results[index] = await probe(entry);
+    }
+  };
+  await Promise.all(Array.from({ length: workerCount }, runWorker));
+  return results;
+}
+
+function resolveLocalVideoReference(
+  projectDir: string,
+  rawSrc: string,
+  compSrcPath?: string,
+): { resolved: string; src: string } | null {
+  if (isUnresolvedAssetPlaceholder(rawSrc)) return null;
+  const src = cleanAssetUrl(rawSrc);
+  if (!src || isRemoteOrInlineUrl(src)) return null;
+  const rootRelative = compSrcPath
+    ? rewriteAssetPath(compSrcPath, src, (path) => existsSync(join(projectDir, path)))
+    : src;
+  const asset = resolveExistingLocalAsset(projectDir, rootRelative);
+  return asset ? { resolved: asset.resolved, src } : null;
+}
+
 function hasHevcStream(json: unknown): boolean {
   if (typeof json !== "object" || json === null) return false;
   const streams = Reflect.get(json, "streams");
@@ -91,23 +127,170 @@ export function collectLocalVideoCandidates(
     let match: RegExpExecArray | null;
     while ((match = re.exec(scannable)) !== null) {
       const rawSrc = match[2] ?? "";
-      // Placeholder check runs on the RAW value: cleanAssetUrl() splits on ?/# and would chop inside a ${...} token.
-      if (isUnresolvedAssetPlaceholder(rawSrc)) continue;
-      const src = cleanAssetUrl(rawSrc);
-      if (!src) continue;
-      if (isRemoteOrInlineUrl(src)) continue;
-      const rootRelative = compSrcPath
-        ? rewriteAssetPath(compSrcPath, src, (path) => existsSync(join(projectDir, path)))
-        : src;
-      const resolvedAsset = resolveExistingLocalAsset(projectDir, rootRelative);
-      if (!resolvedAsset) continue;
-      if (!candidates.has(resolvedAsset.resolved)) candidates.set(resolvedAsset.resolved, src);
+      const reference = resolveLocalVideoReference(projectDir, rawSrc, compSrcPath);
+      if (!reference || candidates.has(reference.resolved)) continue;
+      candidates.set(reference.resolved, reference.src);
     }
   }
 
   return candidates;
 }
 
+interface LocalFiniteVideoSlot {
+  src: string;
+  file: string;
+  elementId?: string;
+  mediaStart: number;
+}
+
+interface UnresolvedFiniteVideoSlot extends Omit<LocalFiniteVideoSlot, "src"> {
+  rawSrc: string;
+}
+
+function collectVideoElements(html: string): Element[] {
+  const { document } = parseHTML(html);
+  const roots: ParentNode[] = [document];
+  const videos: Element[] = [];
+  for (let index = 0; index < roots.length; index++) {
+    const root = roots[index];
+    if (!root) continue;
+    videos.push(...root.querySelectorAll("video"));
+    for (const template of root.querySelectorAll("template")) {
+      roots.push((template as HTMLTemplateElement).content);
+    }
+  }
+  return videos;
+}
+
+function readPositiveNumber(raw: string | null): number | null {
+  const value = parseNumeric(raw);
+  return value !== null && value > 0 ? value : null;
+}
+
+function readNonNegativeNumber(raw: string | null): number | null {
+  const value = parseNumeric(raw);
+  return value !== null && value >= 0 ? value : null;
+}
+
+function readFiniteVideoSlot(video: Element, file: string): UnresolvedFiniteVideoSlot | null {
+  if (video.hasAttribute("loop")) return null;
+  if (video.hasAttribute("data-var-src")) return null;
+  if (video.hasAttribute("data-playback-start")) return null;
+  if (readPositiveNumber(video.getAttribute("data-duration")) === null) return null;
+  const mediaStart = readNonNegativeNumber(video.getAttribute("data-media-start"));
+  if (mediaStart === null) return null;
+  return {
+    rawSrc: video.getAttribute("src") ?? "",
+    file,
+    ...(video.id ? { elementId: video.id } : {}),
+    mediaStart,
+  };
+}
+
+function collectLocalFiniteVideoSlots(
+  projectDir: string,
+  htmlSources: HtmlSourceLike[],
+): Map<string, LocalFiniteVideoSlot[]> {
+  const slotsByPath = new Map<string, LocalFiniteVideoSlot[]>();
+  for (const { html, compSrcPath } of htmlSources) {
+    for (const video of collectVideoElements(html)) {
+      const slot = readFiniteVideoSlot(video, compSrcPath ?? "index.html");
+      if (!slot) continue;
+      const reference = resolveLocalVideoReference(projectDir, slot.rawSrc, compSrcPath);
+      if (!reference) continue;
+      const slots = slotsByPath.get(reference.resolved) ?? [];
+      slots.push({
+        src: reference.src,
+        file: slot.file,
+        ...(slot.elementId ? { elementId: slot.elementId } : {}),
+        mediaStart: slot.mediaStart,
+      });
+      slotsByPath.set(reference.resolved, slots);
+    }
+  }
+  return slotsByPath;
+}
+
+function parsePositiveDuration(value: unknown): number | null {
+  const duration = typeof value === "number" || typeof value === "string" ? Number(value) : NaN;
+  return Number.isFinite(duration) && duration > 0 ? duration : null;
+}
+
+function readPlayableVideoDuration(stdout: string): number | null {
+  try {
+    const metadata: unknown = JSON.parse(stdout);
+    if (typeof metadata !== "object" || metadata === null) return null;
+    const streams = Reflect.get(metadata, "streams");
+    const stream = Array.isArray(streams)
+      ? streams.find((candidate) => typeof candidate === "object" && candidate !== null)
+      : null;
+    const streamDuration = stream ? parsePositiveDuration(Reflect.get(stream, "duration")) : null;
+    if (streamDuration !== null) return streamDuration;
+    const format = Reflect.get(metadata, "format");
+    return typeof format === "object" && format !== null
+      ? parsePositiveDuration(Reflect.get(format, "duration"))
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+async function probePlayableVideoDuration(
+  ffprobePath: string,
+  filePath: string,
+): Promise<number | null> {
+  try {
+    return readPlayableVideoDuration(
+      await execFileAsync(ffprobePath, [
+        "-v",
+        "error",
+        "-select_streams",
+        "v:0",
+        "-show_entries",
+        "stream=duration:format=duration",
+        "-of",
+        "json",
+        "--",
+        filePath,
+      ]),
+    );
+  } catch {
+    return null;
+  }
+}
+
+export async function lintVideoMediaStartPastEof(
+  projectDir: string,
+  htmlSources: HtmlSourceLike[],
+): Promise<HyperframeLintFinding[]> {
+  const slotsByPath = collectLocalFiniteVideoSlots(projectDir, htmlSources);
+  if (slotsByPath.size === 0) return [];
+  const ffprobePath = findFfBinary("ffprobe", { configuredMustExist: true });
+  if (!ffprobePath) return [];
+
+  const entries = [...slotsByPath.entries()];
+  const durations = await mapWithProbeConcurrency(entries, (entry) =>
+    probePlayableVideoDuration(ffprobePath, entry[0]),
+  );
+  const findings: HyperframeLintFinding[] = [];
+  for (const [index, [, slots]] of entries.entries()) {
+    const sourceDuration = durations[index];
+    if (sourceDuration == null) continue;
+    for (const slot of slots) {
+      if (slot.mediaStart < sourceDuration) continue;
+      findings.push({
+        code: "video_media_start_at_or_past_eof",
+        severity: "warning",
+        message: `Video "${slot.src}" starts at ${slot.mediaStart}s, at or past its ${sourceDuration}s playable source duration. The video will hold its final frame for the explicit slot.`,
+        file: slot.file,
+        ...(slot.elementId ? { elementId: slot.elementId } : {}),
+        fixHint: `Trim data-media-start below ${sourceDuration}s if this final-frame hold is unintended.`,
+      });
+    }
+  }
+  return findings;
+}
+
 /**
  * INFO-only finding: a locally referenced `<video>` file is encoded as
  * HEVC/H.265. The render pipeline pre-decodes video with FFmpeg (never the
@@ -129,18 +312,8 @@ export async function lintHevcPreviewCodec(
   if (!ffprobePath) return [];
 
   const entries = [...candidates.entries()];
-  const isHevc = new Array<boolean>(entries.length).fill(false);
-  let nextIndex = 0;
-  const workerCount = Math.min(PROBE_CONCURRENCY, entries.length);
-  await Promise.all(
-    Array.from({ length: workerCount }, async () => {
-      while (nextIndex < entries.length) {
-        const index = nextIndex++;
-        const entry = entries[index];
-        if (!entry) break;
-        isHevc[index] = await probeIsHevc(ffprobePath, entry[0]);
-      }
-    }),
+  const isHevc = await mapWithProbeConcurrency(entries, (entry) =>
+    probeIsHevc(ffprobePath, entry[0]),
   );
 
   const hevcSrcs = entries.filter((_, i) => isHevc[i]).map(([, src]) => src);

packages/lint/src/project.test.ts

@@ -560,6 +560,110 @@ describe("hevc_preview_codec", () => {
   });
 });
 
+describe("video_media_start_at_or_past_eof", () => {
+  const mockExecFile = vi.mocked(execFile);
+  const FAKE_FFPROBE_PATH = process.execPath;
+
+  function videoProject(videoTags: string): { project: string; videoAbsPath: string } {
+    const project = makeProject(`<html><body>
+  <div data-composition-id="main" data-width="1920" data-height="1080" data-start="0" data-duration="10">
+    ${videoTags}
+  </div>
+  <script src="https://cdn.jsdelivr.net/npm/gsap@3/dist/gsap.min.js"></script>
+  <script>window.__timelines = window.__timelines || {}; window.__timelines["main"] = gsap.timeline({ paused: true });</script>
+</body></html>`);
+    const videoAbsPath = join(project, "clip.mp4");
+    writeFileSync(videoAbsPath, "fake video bytes");
+    return { project, videoAbsPath };
+  }
+
+  function mockDurationProbe(streamDuration: number, containerDuration = streamDuration): void {
+    mockExecFile.mockImplementation((_file, _args, _options, callback) => {
+      callback(
+        null,
+        Buffer.from(
+          JSON.stringify({
+            streams: [{ codec_name: "h264", duration: String(streamDuration) }],
+            format: { duration: String(containerDuration) },
+          }),
+        ),
+        Buffer.alloc(0),
+      );
+      return new ChildProcess();
+    });
+  }
+
+  async function mediaStartFindings(project: string): Promise<HyperframeLintFinding[]> {
+    const { results } = await lintProject(project);
+    return results
+      .flatMap((entry) => entry.result.findings)
+      .filter((finding) => finding.code === "video_media_start_at_or_past_eof");
+  }
+
+  beforeEach(() => {
+    process.env.HYPERFRAMES_FFPROBE_PATH = FAKE_FFPROBE_PATH;
+    mockExecFile.mockReset();
+  });
+
+  afterEach(() => {
+    delete process.env.HYPERFRAMES_FFPROBE_PATH;
+    mockExecFile.mockReset();
+  });
+
+  it.each([2, 5])(
+    "warns that a finite non-looping local video starting at/past stream EOF (%ss) holds its final frame",
+    async (mediaStart) => {
+      const { project } = videoProject(
+        `<video id="clip" src="clip.mp4" data-start="0" data-duration="6" data-media-start="${mediaStart}" muted></video>`,
+      );
+      mockDurationProbe(2, 60);
+
+      const result = await lintProject(project);
+      const findings = result.results
+        .flatMap((entry) => entry.result.findings)
+        .filter((finding) => finding.code === "video_media_start_at_or_past_eof");
+
+      expect(findings).toHaveLength(1);
+      expect(findings[0]).toMatchObject({
+        severity: "warning",
+        file: "index.html",
+        elementId: "clip",
+      });
+      expect(findings[0]?.message).toContain("will hold its final frame");
+      expect(findings[0]?.fixHint).toContain("Trim data-media-start");
+      expect(result.totalWarnings).toBeGreaterThanOrEqual(1);
+      expect(result.totalErrors).toBe(0);
+      expect(result.results[0]?.result.ok).toBe(true);
+    },
+  );
+
+  it("does not warn when the media start is just inside EOF", async () => {
+    const { project } = videoProject(
+      '<video id="clip" src="clip.mp4" data-start="0" data-duration="6" data-media-start="1.999" muted></video>',
+    );
+    mockDurationProbe(2);
+
+    expect(await mediaStartFindings(project)).toEqual([]);
+  });
+
+  it("silently skips open, looping, remote, variable, missing, and unprobeable slots", async () => {
+    const { project } = videoProject(`
+      <video src="clip.mp4" data-start="0" data-duration="6" data-media-start="1.9" muted></video>
+      <video src="clip.mp4" data-start="0" data-media-start="5" muted></video>
+      <video src="clip.mp4" data-start="0" data-duration="6" data-media-start="5" loop muted></video>
+      <video src="clip.mp4" data-var-src="selectedVideo" data-start="0" data-duration="6" data-media-start="5" muted></video>
+      <video src="https://cdn.example.com/clip.mp4" data-start="0" data-duration="6" data-media-start="5" muted></video>
+      <video src="missing.mp4" data-start="0" data-duration="6" data-media-start="5" muted></video>
+    `);
+    mockExecFile.mockImplementation((_file, _args, _options, callback) => {
+      callback(new Error("ffprobe failed"), Buffer.alloc(0), Buffer.alloc(0));
+      return new ChildProcess();
+    });
+
+    expect(await mediaStartFindings(project)).toEqual([]);
+  });
+});
+
 describe("audio_src_not_found with templating tokens", () => {
   // A src carrying an unresolved templating placeholder is late-bound before render,
   // so the static linter cannot resolve it to a file and must not report it missing.

packages/lint/src/project.ts

@@ -15,7 +15,11 @@ import {
   resolveExistingLocalAsset,
   resolveLocalAssetCandidates,
 } from "@hyperframes/parsers/asset-resolution";
-import { collectLocalVideoCandidates, lintHevcPreviewCodec } from "./hevcPreviewLint.js";
+import {
+  collectLocalVideoCandidates,
+  lintHevcPreviewCodec,
+  lintVideoMediaStartPastEof,
+} from "./hevcPreviewLint.js";
 import { lintHyperframeHtml } from "./hyperframeLinter.js";
 import type { HyperframeLintFinding, HyperframeLintResult } from "./types.js";
 import type { ParsableDocumentLike } from "@hyperframes/parsers/sub-composition-validity";
@@ -235,6 +239,7 @@ export async function lintProject(
     ...(!entryFile ? lintBlankRootWithStandaloneComposition(rootHtml, allHtmlSources) : []),
     ...lintDuplicateAudioTracks(allHtmlSources),
     ...lintMissingOrEmptySubComposition(projectDir, rootHtml),
+    ...(await lintVideoMediaStartPastEof(projectDir, allHtmlSources)),
     ...(await lintHevcPreviewCodec(collectLocalVideoCandidates(projectDir, allHtmlSources))),
   ];
   if (projectFindings.length > 0) {