mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-18 07:50:13 +00:00
fix(cli): check times a line held across a split clip as one hold (#7733)
* fix(cli): check times a line held across a split clip as one hold A clip split in two hands its line to a second element at the same place. line_unread timed each element alone, so the first half warned while the same text stayed on screen. * refactor(cli): name the split-clip line hand-off instead of commenting it
This commit is contained in:
@@ -26,6 +26,8 @@ interface Shown {
|
||||
block?: number;
|
||||
/** The block's rendered text; defaults to `text`. */
|
||||
blockText?: string;
|
||||
/** Top edge in px; defaults to a row per key. */
|
||||
top?: number;
|
||||
}
|
||||
|
||||
function part(shown: Shown): CraftTextPart {
|
||||
@@ -40,7 +42,7 @@ function part(shown: Shown): CraftTextPart {
|
||||
blockText: shown.blockText ?? shown.text,
|
||||
opacity: 1,
|
||||
fontPx,
|
||||
rect: { left: 100, top: 100 + shown.key * 10, width: 800, height: fontPx * 1.2 },
|
||||
rect: { left: 100, top: shown.top ?? 100 + shown.key * 10, width: 800, height: fontPx * 1.2 },
|
||||
captions: shown.captions ?? false,
|
||||
};
|
||||
}
|
||||
@@ -218,6 +220,25 @@ describe("line_unread", () => {
|
||||
expect(await detectLineUnread(film(12, shown), 12, CANVAS, probeFor(shown))).toEqual([]);
|
||||
});
|
||||
|
||||
// fails without the fix: each instance of a split clip was timed alone, so its 0.5 s first half warned.
|
||||
it("times a line held across a split clip's two instances as one hold", async () => {
|
||||
const line = "Your reminder card holds here";
|
||||
const shown: Shown[] = [
|
||||
{ key: 1, text: line, from: 3, to: 3.5, top: 800 },
|
||||
{ key: 2, text: line, from: 3.5, to: 8, top: 800 },
|
||||
];
|
||||
expect(await detectLineUnread(film(8, shown), 8, CANVAS, probeFor(shown))).toEqual([]);
|
||||
});
|
||||
|
||||
it("still times a different line that takes over the same place on its own", async () => {
|
||||
const shown: Shown[] = [
|
||||
{ key: 1, text: "Every team gets one shared calendar now", from: 3, to: 3.5, top: 800 },
|
||||
{ key: 2, text: "Start your free trial today", from: 3.5, to: 8, top: 800 },
|
||||
];
|
||||
const findings = await detectLineUnread(film(8, shown), 8, CANVAS, probeFor(shown));
|
||||
expect(findings.map((finding) => finding.selector)).toEqual(["#t1"]);
|
||||
});
|
||||
|
||||
it("skips text too small to be a reading line", async () => {
|
||||
const shown: Shown[] = [
|
||||
{ key: 1, text: "Legal fine print that flashes by", from: 1, to: 1.3, fontPx: 18 },
|
||||
|
||||
@@ -211,7 +211,7 @@ interface CraftRect {
|
||||
}
|
||||
|
||||
interface BlockTrack {
|
||||
key: number;
|
||||
blockKeys: Set<number>;
|
||||
selector: string;
|
||||
sourceFile: string;
|
||||
/** Frame index → what the block showed there. */
|
||||
@@ -258,20 +258,47 @@ interface Anchor {
|
||||
}
|
||||
|
||||
function trackBlocks(frames: CraftFrame[], readHeight: number): BlockTrack[] {
|
||||
const tracks = new Map<number, BlockTrack>();
|
||||
const tracks: BlockTrack[] = [];
|
||||
const byKey = new Map<number, BlockTrack>();
|
||||
frames.forEach((frame, index) => {
|
||||
for (const [key, block] of readableBlocks(frame, readHeight)) {
|
||||
const track = tracks.get(key) ?? {
|
||||
key,
|
||||
selector: block.selector,
|
||||
sourceFile: block.sourceFile,
|
||||
seen: new Map<number, BlockFrame>(),
|
||||
};
|
||||
let track = byKey.get(key) ?? trackContinuedBy(tracks, index, block);
|
||||
if (!track) {
|
||||
track = {
|
||||
blockKeys: new Set(),
|
||||
selector: block.selector,
|
||||
sourceFile: block.sourceFile,
|
||||
seen: new Map(),
|
||||
};
|
||||
tracks.push(track);
|
||||
}
|
||||
track.blockKeys.add(key);
|
||||
byKey.set(key, track);
|
||||
track.seen.set(index, { text: block.text, rect: block.rect });
|
||||
tracks.set(key, track);
|
||||
}
|
||||
});
|
||||
return [...tracks.values()];
|
||||
return tracks;
|
||||
}
|
||||
|
||||
function trackContinuedBy(
|
||||
tracks: BlockTrack[],
|
||||
index: number,
|
||||
block: BlockFrame,
|
||||
): BlockTrack | undefined {
|
||||
return tracks.find((track) => {
|
||||
const last = track.seen.get(index) ?? track.seen.get(index - 1);
|
||||
return !!last && last.text === block.text && sameBox(last.rect, block.rect);
|
||||
});
|
||||
}
|
||||
|
||||
const SAME_BOX_OVERLAP = 0.9;
|
||||
|
||||
function sameBox(a: CraftRect, b: CraftRect): boolean {
|
||||
const width = Math.min(a.right, b.right) - Math.max(a.left, b.left);
|
||||
const height = Math.min(a.bottom, b.bottom) - Math.max(a.top, b.top);
|
||||
if (width <= 0 || height <= 0) return false;
|
||||
const area = (rect: CraftRect) => (rect.right - rect.left) * (rect.bottom - rect.top);
|
||||
return (width * height) / Math.max(area(a), area(b)) >= SAME_BOX_OVERLAP;
|
||||
}
|
||||
|
||||
/** Maximal runs of consecutive frame indices. */
|
||||
@@ -374,8 +401,10 @@ async function refineWindow(
|
||||
canvas: Canvas,
|
||||
): Promise<LineWindow> {
|
||||
const readHeight = T.readHeightAt1080 * (Math.min(canvas.width, canvas.height) / 1080);
|
||||
const visibleAt = async (time: number) =>
|
||||
readableBlocks(await probe(time), readHeight).has(candidate.track.key);
|
||||
const visibleAt = async (time: number) => {
|
||||
const blocks = readableBlocks(await probe(time), readHeight);
|
||||
return [...candidate.track.blockKeys].some((key) => blocks.has(key));
|
||||
};
|
||||
const window = { ...candidate.window };
|
||||
for (let step = 0; step < T.refineSteps; step += 1) {
|
||||
if (window.startIn > window.startOut) {
|
||||
@@ -445,7 +474,7 @@ function lineUnreadFinding(
|
||||
words,
|
||||
seconds: round(held),
|
||||
floorSeconds: round(floor),
|
||||
measured: measuredAround(frames, track.key, candidate.firstIndex, candidate.lastIndex),
|
||||
measured: measuredAround(frames, track.blockKeys, candidate.firstIndex, candidate.lastIndex),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -454,7 +483,7 @@ const MEASURED_SPAN = 3;
|
||||
/** The samples at both ends of the line's run, as the sampler measured them. */
|
||||
function measuredAround(
|
||||
frames: CraftFrame[],
|
||||
key: number,
|
||||
keys: Set<number>,
|
||||
first: number,
|
||||
last: number,
|
||||
): NonNullable<CraftFinding["measured"]> {
|
||||
@@ -466,7 +495,7 @@ function measuredAround(
|
||||
for (const index of [...indices].sort((a, b) => a - b)) {
|
||||
const frame = frames[index];
|
||||
if (!frame) continue;
|
||||
const parts = frame.parts.filter((part) => part.block === key && !part.captions);
|
||||
const parts = frame.parts.filter((part) => keys.has(part.block) && !part.captions);
|
||||
if (parts.length === 0) {
|
||||
out.push({ time: round(frame.time), opacity: null, heightPx: null });
|
||||
continue;
|
||||
|
||||
Reference in New Issue
Block a user