fix(engine): a video on a non-square pixel grid renders at its display shape (#6772)

* fix(engine): a video on a non-square pixel grid renders at its display shape

* chore(engine): drop comments the names already carry

* fix(engine): a rotated clip with a non-square pixel grid keeps its display shape

* test(engine): the frame probes end their options with -- like every ffprobe call

packages/engine/src/services/videoFrameExtractor.test.ts

@@ -4417,3 +4417,104 @@ describe("parseVideoElements hidden flag", () => {
     ]);
   });
 });
+
+describe.skipIf(!HAS_FFMPEG)("a clip stored on a non-square pixel grid", () => {
+  const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-sar-"));
+  const SOURCE = join(FIXTURE_DIR, "anamorphic.mp4");
+
+  beforeAll(async () => {
+    const result = await runFfmpeg([
+      "-y",
+      "-v",
+      "error",
+      "-f",
+      "lavfi",
+      "-i",
+      "color=c=red:s=64x64:d=1:r=10",
+      "-vf",
+      "setsar=1/2",
+      "-c:v",
+      "libx264",
+      "-pix_fmt",
+      "yuv420p",
+      SOURCE,
+    ]);
+    if (!result.success) throw new Error(`fixture synthesis failed: ${result.stderr.slice(-400)}`);
+  }, 30_000);
+
+  afterAll(() => rmSync(FIXTURE_DIR, { recursive: true, force: true }), 30_000);
+
+  it("keeps the display shape of a rotated clip, whose autorotate swaps the pixel aspect", async () => {
+    const rotated = join(FIXTURE_DIR, "rotated.mp4");
+    const synth = await runFfmpeg([
+      "-y",
+      "-v",
+      "error",
+      "-display_rotation:v:0",
+      "90",
+      "-i",
+      SOURCE,
+      "-c",
+      "copy",
+      rotated,
+    ]);
+    expect(synth.success).toBe(true);
+    const outDir = join(FIXTURE_DIR, "out-rot");
+    mkdirSync(outDir, { recursive: true });
+    const video: VideoElement = {
+      id: "r",
+      src: rotated,
+      start: 0,
+      end: 1,
+      mediaStart: 0,
+      loop: false,
+      hasAudio: false,
+    };
+    const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
+      fps: 10,
+      outputDir: outDir,
+    });
+    expect(result.errors).toEqual([]);
+    const probe = spawnSync("ffprobe", [
+      "-v",
+      "error",
+      "-show_entries",
+      "stream=width,height",
+      "-of",
+      "csv=p=0",
+      "--",
+      result.extracted[0]!.framePaths.get(0)!,
+    ]);
+    expect(probe.stdout.toString().trim()).toBe("128,64");
+  });
+
+  it("extracts frames at the shape Chrome displays, not the stored grid", async () => {
+    const outDir = join(FIXTURE_DIR, "out");
+    mkdirSync(outDir, { recursive: true });
+    const video: VideoElement = {
+      id: "v",
+      src: SOURCE,
+      start: 0,
+      end: 1,
+      mediaStart: 0,
+      loop: false,
+      hasAudio: false,
+    };
+    const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
+      fps: 10,
+      outputDir: outDir,
+    });
+    expect(result.errors).toEqual([]);
+    const probe = spawnSync("ffprobe", [
+      "-v",
+      "error",
+      "-show_entries",
+      "stream=width,height",
+      "-of",
+      "csv=p=0",
+      "--",
+      result.extracted[0]!.framePaths.get(0)!,
+    ]);
+    expect(probe.stdout.toString().trim()).toBe("32,64");
+  });
+});

packages/engine/src/services/videoFrameExtractor.ts

@@ -1188,6 +1188,9 @@ async function extractVideoFramesRangeUnredacted(
     vfFilters.push("format=nv12");
   }
   const samplingAt = vfFilters.length;
+  if (metadata.sampleAspectRatio) {
+    vfFilters.push("scale=trunc(iw*sar/2)*2:ih", "setsar=1");
+  }
   // Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer
   // first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).
   const samplingFilters = (seek: number): string[] =>

packages/engine/src/utils/ffprobe.ts

@@ -224,6 +224,7 @@ export interface VideoMetadata {
   colorSpace: VideoColorSpace | null;
   colorRange?: string;
   pixelFormat?: string;
+  sampleAspectRatio?: number;
   /** Decoded frame count from the video stream's `nb_frames`. Omitted when the
    * container does not surface a reliable count (still images, malformed
    * streams, or muxes that require `-count_packets` to populate). Callers
@@ -263,6 +264,7 @@ interface FFProbeStream {
   nb_frames?: string;
   nb_read_packets?: string;
   pix_fmt?: string;
+  sample_aspect_ratio?: string;
   r_frame_rate?: string;
   avg_frame_rate?: string;
   sample_rate?: string;
@@ -843,6 +845,13 @@ export function parseFrameRate(frameRateStr: string | undefined): number {
   return rounded > 0 ? rounded : 0.01;
 }
 
+export function parseSampleAspectRatio(value: string | undefined): number | undefined {
+  const [num, den] = (value ?? "").split(":").map(Number);
+  if (!num || !den || !Number.isFinite(num) || !Number.isFinite(den)) return undefined;
+  const ratio = num / den;
+  return Math.abs(ratio - 1) < 1e-3 ? undefined : ratio;
+}
+
 /**
  * Whether the first frame decoded with `decoder` (alpha-aware) has an alpha pixel format; undefined on any
  * failure. ffprobe cannot pick a decoder, so ffmpeg writes the frame as rawvideo, which keeps its pix_fmt.
@@ -977,6 +986,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
     const colorSpace =
       merged.colorTransfer || merged.colorPrimaries || merged.colorSpace ? merged : null;
     const pixelFormat = videoStream.pix_fmt || "";
+    const sampleAspectRatio = parseSampleAspectRatio(videoStream.sample_aspect_ratio);
     const hasAlpha = videoStreamHasAlpha(videoStream);
     const alphaDecoder = hasAlpha ? null : alphaDecoderFor(videoStream.codec_name);
     const alphaPlaneDecoded = alphaDecoder
@@ -1031,6 +1041,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
       colorSpace,
       colorRange: videoStream.color_range,
       pixelFormat,
+      ...(sampleAspectRatio === undefined ? {} : { sampleAspectRatio }),
       frames,
     };
   })();