From b7ac593df600449ce9c5481ae6ac0e2379a0e439 Mon Sep 17 00:00:00 2001 From: ukimsanov Date: Wed, 8 Jul 2026 13:08:33 -0700 Subject: [PATCH] feat(studio): enforced CapCut track zones (overlay/main/audio normalization) --- .../player/components/timelineZones.test.ts | 71 +++++++++++++++++++ .../src/player/components/timelineZones.ts | 68 ++++++++++++++++++ .../src/player/hooks/useTimelinePlayer.ts | 18 +++-- .../studio/src/utils/timelineInspector.ts | 2 +- 4 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 packages/studio/src/player/components/timelineZones.test.ts create mode 100644 packages/studio/src/player/components/timelineZones.ts diff --git a/packages/studio/src/player/components/timelineZones.test.ts b/packages/studio/src/player/components/timelineZones.test.ts new file mode 100644 index 0000000000..5b746299e5 --- /dev/null +++ b/packages/studio/src/player/components/timelineZones.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import type { TimelineElement } from "../store/playerStore"; +import { classifyZone, normalizeToZones, resolveMainOriginTrack } from "./timelineZones"; + +function el(id: string, tag: string, track: number): TimelineElement { + return { id, tag, start: 0, duration: 2, track }; +} + +function trackOf(els: TimelineElement[], id: string): number { + return els.find((e) => e.id === id)!.track; +} + +describe("resolveMainOriginTrack", () => { + it("is the lowest track carrying a video clip", () => { + expect( + resolveMainOriginTrack([el("a", "img", 0), el("v", "video", 2), el("v2", "video", 4)]), + ).toBe(2); + }); + it("is null when there is no video", () => { + expect(resolveMainOriginTrack([el("a", "img", 0), el("m", "audio", 1)])).toBe(null); + }); +}); + +describe("classifyZone", () => { + it("audio → audio, main-track video → main, others → overlay", () => { + expect(classifyZone(el("m", "audio", 3), 1)).toBe("audio"); + expect(classifyZone(el("v", "video", 1), 1)).toBe("main"); + expect(classifyZone(el("i", "img", 0), 1)).toBe("overlay"); + expect(classifyZone(el("v2", "video", 4), 1)).toBe("overlay"); // video not on the main track + expect(classifyZone(el("i", "img", 1), 1)).toBe("overlay"); // non-video ON the main track → overlay + }); +}); + +describe("normalizeToZones", () => { + it("orders overlay → main → audio by ascending index", () => { + const out = normalizeToZones([ + el("img", "img", 0), + el("vid", "video", 2), + el("mus", "audio", 5), + ]); + expect(trackOf(out, "img")).toBe(0); // overlay (top) + expect(trackOf(out, "vid")).toBe(1); // main (middle) + expect(trackOf(out, "mus")).toBe(2); // audio (bottom) + }); + + it("demotes non-main video tracks into the overlay zone above main", () => { + const out = normalizeToZones([el("v0", "video", 0), el("i1", "img", 1), el("v3", "video", 3)]); + // main = track 0 (lowest video); overlay tracks (1,3) sorted → i1=0, v3=1; main → 2 + expect(trackOf(out, "i1")).toBe(0); + expect(trackOf(out, "v3")).toBe(1); + expect(trackOf(out, "v0")).toBe(2); // main sits below the overlays (no audio present) + }); + + it("splits a mixed video+audio track: audio drops below the main video", () => { + const out = normalizeToZones([el("v", "video", 0), el("a", "audio", 0)]); + expect(trackOf(out, "v")).toBe(0); // main + expect(trackOf(out, "a")).toBe(1); // audio zone, below main + }); + + it("returns the same array (identity) when already zoned", () => { + const input = [el("i", "img", 0), el("v", "video", 1), el("a", "audio", 2)]; + expect(normalizeToZones(input)).toBe(input); + }); + + it("groups multiple audio tracks at the bottom preserving relative order", () => { + const out = normalizeToZones([el("v", "video", 0), el("a1", "audio", 1), el("a2", "audio", 4)]); + expect(trackOf(out, "v")).toBe(0); // main (no overlays) + expect(trackOf(out, "a1")).toBe(1); + expect(trackOf(out, "a2")).toBe(2); + }); +}); diff --git a/packages/studio/src/player/components/timelineZones.ts b/packages/studio/src/player/components/timelineZones.ts new file mode 100644 index 0000000000..8514a649c2 --- /dev/null +++ b/packages/studio/src/player/components/timelineZones.ts @@ -0,0 +1,68 @@ +import type { TimelineElement } from "../store/playerStore"; +import { isAudioTimelineElement } from "../../utils/timelineInspector"; + +/** CapCut-style vertical zones, top → bottom: overlay, main, audio. */ +export type TrackZone = "overlay" | "main" | "audio"; + +function sortedDistinct(values: number[]): number[] { + return [...new Set(values)].sort((a, b) => a - b); +} + +/** + * The authored track index treated as the single "main" video track: the lowest + * track that carries a `video` clip. null when the timeline has no video. + */ +export function resolveMainOriginTrack(elements: TimelineElement[]): number | null { + const videoTracks = elements + .filter((el) => el.tag === "video" && !isAudioTimelineElement(el)) + .map((el) => el.track); + return videoTracks.length > 0 ? Math.min(...videoTracks) : null; +} + +/** Which zone a clip belongs to under the enforced model. The main lane holds + * only video on the main track; non-video on that track goes to overlay. */ +export function classifyZone(el: TimelineElement, mainOriginTrack: number | null): TrackZone { + if (isAudioTimelineElement(el)) return "audio"; + if (mainOriginTrack !== null && el.track === mainOriginTrack && el.tag === "video") return "main"; + return "overlay"; +} + +/** + * Enforced CapCut zoning: remap every clip's track so ascending index runs + * overlay → main → audio (overlays on top, the single main video track in the + * middle, audio at the bottom). Relative order within the overlay and audio + * zones is preserved from the authored indices. Pure — returns a new array; + * clips whose track is unchanged keep their identity. + */ +export function normalizeToZones(elements: TimelineElement[]): TimelineElement[] { + if (elements.length === 0) return elements; + const mainOrigin = resolveMainOriginTrack(elements); + const zoneOf = (el: TimelineElement) => classifyZone(el, mainOrigin); + + const overlayTracks = sortedDistinct( + elements.filter((el) => zoneOf(el) === "overlay").map((el) => el.track), + ); + const audioTracks = sortedDistinct( + elements.filter((el) => zoneOf(el) === "audio").map((el) => el.track), + ); + + // Assign contiguous new indices in visual order: overlays, then main, then audio. + const newIndexByZoneTrack = new Map(); + let next = 0; + for (const t of overlayTracks) newIndexByZoneTrack.set(`overlay:${t}`, next++); + const mainIndex = mainOrigin !== null ? next++ : null; + for (const t of audioTracks) newIndexByZoneTrack.set(`audio:${t}`, next++); + + let changed = false; + const remapped = elements.map((el) => { + const zone = zoneOf(el); + const nextTrack = + zone === "main" && mainIndex !== null + ? mainIndex + : (newIndexByZoneTrack.get(`${zone}:${el.track}`) ?? el.track); + if (nextTrack === el.track) return el; + changed = true; + return { ...el, track: nextTrack }; + }); + return changed ? remapped : elements; +} diff --git a/packages/studio/src/player/hooks/useTimelinePlayer.ts b/packages/studio/src/player/hooks/useTimelinePlayer.ts index 70414c5771..d9109e037d 100644 --- a/packages/studio/src/player/hooks/useTimelinePlayer.ts +++ b/packages/studio/src/player/hooks/useTimelinePlayer.ts @@ -36,6 +36,7 @@ import { mergeTimelineElementsPreservingDowngrades, parseTimelineFromDOM, } from "../lib/timelineDOM"; +import { normalizeToZones } from "../components/timelineZones"; import { setPreviewMediaMuted, setPreviewPlaybackRate, @@ -90,12 +91,17 @@ export function useTimelinePlayer() { // applyCachedSourceDurations re-applies the cached probe duration: re-derived // elements (e.g. after a clip move) can arrive without sourceDuration, which // otherwise makes trimmed waveforms lose their window. - const mergedElements = applyCachedSourceDurations( - mergeTimelineElementsPreservingDowngrades( - state.elements, - elements, - state.duration, - resolvedDuration, + // Enforced CapCut zoning (overlay → main → audio): normalize track indices + // on every discovery. Idempotent — already-zoned input is returned as-is, so + // drops persist zoned indices and reloads re-zone to the same (no drift). + const mergedElements = normalizeToZones( + applyCachedSourceDurations( + mergeTimelineElementsPreservingDowngrades( + state.elements, + elements, + state.duration, + resolvedDuration, + ), ), ); diff --git a/packages/studio/src/utils/timelineInspector.ts b/packages/studio/src/utils/timelineInspector.ts index b1297e1900..705fa314f0 100644 --- a/packages/studio/src/utils/timelineInspector.ts +++ b/packages/studio/src/utils/timelineInspector.ts @@ -4,7 +4,7 @@ const AUDIO_TIMELINE_TAGS = new Set(["audio", "music", "sfx", "sound", "narratio const AUDIO_SOURCE_EXT_RE = /\.(aac|flac|m4a|mp3|ogg|opus|wav)(?:[?#].*)?$/i; const MUSIC_ID_RE = /\b(music|bgm|soundtrack|background[-_]?music)\b/i; -function isAudioTimelineElement( +export function isAudioTimelineElement( element: Pick | null | undefined, ): boolean { if (!element) return false;