fix(player): parent-frame media playback for mobile (#266)

* fix(player): parent-frame media playback for mobile

Mobile browsers block media.play() inside iframes when the user
gesture happened in the parent frame — postMessage doesn't transfer
user activation (per the User Activation v2 spec).

## Problem

The player renders compositions in a sandboxed iframe. When a user
taps play in the parent frame, the player sends a postMessage to the
iframe's runtime, which calls audio.play(). On mobile, this fails
silently because the iframe has no user activation context.

## Solution

The player now extracts ALL timed media elements (audio/video with
data-start) from the iframe's DOM (same-origin access), creates
parent-frame copies, and disables the iframe originals. On play(),
parentMedia.play() runs synchronously in the gesture call stack,
satisfying mobile autoplay policy.

### Generic media handling

- Finds all `audio[data-start], video[data-start]` in the iframe
- Creates a parent-frame copy for each (Audio or Video element)
- Preserves data-start offsets for correct seek positioning
- Strips data-start from iframe elements so the runtime ignores them
- Falls back to iframe media for cross-origin iframes

### `audio-src` attribute

Convenience for the common single-narration case. When set, the
player starts preloading audio immediately — before the iframe loads.
This eliminates the loading delay that caused jittery playback.

### No active sync

Both parent media and the GSAP timeline are real-time systems. When
started simultaneously, they naturally stay within ~10ms — no drift
correction needed. Active sync with coarse granularity (50ms polling)
caused MORE jitter than it prevented via repeated audio seeks.

## CI

- Added unified `test` job replacing separate per-package test jobs
- Added root `test` script: `bun run --filter '*' test`
- New packages with test scripts are automatically included
- Added happy-dom for player DOM tests

## Tests

- 10 new tests for parent-frame media: preloading, play, pause,
  seek, muted/rate sync, cleanup, attribute changes
- All 21 player tests pass

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix(shader-transitions): pass CI when no test files exist

Add --passWithNoTests to vitest run so the unified test job
doesn't fail on packages that have a test script but no test
files yet.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix(cli): update tests for new id field and GSAP lint rule

- normalize.test.ts: loadTranscript now assigns id fields (w0, w1, etc.)
  to SRT/VTT results and empty string for words-json passthrough
- lintProject.test.ts: add GSAP CDN script to validHtml() fixture to
  satisfy the missing_gsap_script lint rule added in core

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix(cli): add missing data-start/data-duration to validHtml fixture

The validHtml() test fixture was missing data-start and data-duration
attributes, triggering the root_composition_missing_data_start and
root_composition_missing_data_duration lint warnings.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix(ci): fetch LFS objects for producer test job

Producer regression tests compare rendered output against reference MP4
files stored in git LFS. Without lfs: true, checkout fetches pointer
files instead of actual videos, causing "moov atom not found" errors.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* ci: remove redundant test-producer job

The regression workflow already runs the same 28 producer fixtures
in a Docker container with prod-matching Chrome/fonts/ffmpeg, sharded
across 8 parallel matrix jobs with 40-min timeouts. The CI test-producer
job was a duplicate that ran on bare runners with worse determinism
and a 15-min timeout too short for all fixtures.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
James Russo
2026-04-14 11:20:07 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 9bf4956fae
commit b23b0751da
10 changed files with 336 additions and 46 deletions
+3 -18
View File
@@ -89,8 +89,8 @@ jobs:
- run: bun run build
- run: bun run --filter '*' typecheck
test-core:
name: "Test: core"
test:
name: Test
needs: changes
if: needs.changes.outputs.code == 'true'
runs-on: ubuntu-latest
@@ -102,22 +102,7 @@ jobs:
with:
node-version: 22
- run: bun install --frozen-lockfile
- run: bun run --filter @hyperframes/core test -- --coverage
test-engine:
name: "Test: engine"
needs: changes
if: needs.changes.outputs.code == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: oven-sh/setup-bun@v2
- uses: actions/setup-node@v4
with:
node-version: 22
- run: bun install --frozen-lockfile
- run: bun run --filter @hyperframes/engine test
- run: bun run --filter '!@hyperframes/producer' test
test-runtime-contract:
name: "Test: runtime contract"
+20 -8
View File
@@ -7,8 +7,10 @@
"devDependencies": {
"@commitlint/cli": "^20.5.0",
"@commitlint/config-conventional": "^20.5.0",
"@hyperframes/player": "^0.3.0",
"@types/node": "^25.0.10",
"concurrently": "^8.2.0",
"happy-dom": "^20.9.0",
"knip": "^6.0.3",
"lefthook": "^2.1.4",
"oxfmt": "^0.41.0",
@@ -19,7 +21,7 @@
},
"packages/cli": {
"name": "@hyperframes/cli",
"version": "0.2.4",
"version": "0.3.0",
"bin": {
"hyperframes": "./dist/cli.js",
},
@@ -59,7 +61,7 @@
},
"packages/core": {
"name": "@hyperframes/core",
"version": "0.2.4",
"version": "0.3.0",
"dependencies": {
"@chenglou/pretext": "^0.0.5",
},
@@ -85,7 +87,7 @@
},
"packages/engine": {
"name": "@hyperframes/engine",
"version": "0.2.4",
"version": "0.3.0",
"dependencies": {
"@hono/node-server": "^1.13.0",
"@hyperframes/core": "workspace:^",
@@ -102,7 +104,7 @@
},
"packages/player": {
"name": "@hyperframes/player",
"version": "0.2.4",
"version": "0.3.0",
"devDependencies": {
"tsup": "^8.0.0",
"typescript": "^5.0.0",
@@ -111,7 +113,7 @@
},
"packages/producer": {
"name": "@hyperframes/producer",
"version": "0.2.4",
"version": "0.3.0",
"dependencies": {
"@fontsource/archivo-black": "^5.2.8",
"@fontsource/eb-garamond": "^5.2.7",
@@ -149,7 +151,7 @@
},
"packages/shader-transitions": {
"name": "@hyperframes/shader-transitions",
"version": "0.2.4",
"version": "0.3.0",
"dependencies": {
"html2canvas": "^1.4.1",
},
@@ -161,7 +163,7 @@
},
"packages/studio": {
"name": "@hyperframes/studio",
"version": "0.2.4",
"version": "0.3.0",
"dependencies": {
"@codemirror/autocomplete": "^6.20.1",
"@codemirror/commands": "^6.10.3",
@@ -731,6 +733,10 @@
"@types/tough-cookie": ["@types/tough-cookie@4.0.5", "", {}, "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA=="],
"@types/whatwg-mimetype": ["@types/whatwg-mimetype@3.0.2", "", {}, "sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA=="],
"@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="],
"@types/yauzl": ["@types/yauzl@2.10.3", "", { "dependencies": { "@types/node": "22.19.15" } }, "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q=="],
"@vitejs/plugin-react": ["@vitejs/plugin-react@4.7.0", "", { "dependencies": { "@babel/core": "7.29.0", "@babel/plugin-transform-react-jsx-self": "7.27.1", "@babel/plugin-transform-react-jsx-source": "7.27.1", "@rolldown/pluginutils": "1.0.0-beta.27", "@types/babel__core": "7.20.5", "react-refresh": "0.17.0" }, "peerDependencies": { "vite": "5.4.21" } }, "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA=="],
@@ -1015,6 +1021,8 @@
"global-directory": ["global-directory@4.0.1", "", { "dependencies": { "ini": "4.1.1" } }, "sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q=="],
"happy-dom": ["happy-dom@20.9.0", "", { "dependencies": { "@types/node": ">=20.0.0", "@types/whatwg-mimetype": "^3.0.2", "@types/ws": "^8.18.1", "entities": "^7.0.1", "whatwg-mimetype": "^3.0.0", "ws": "^8.18.3" } }, "sha512-GZZ9mKe8r646NUAf/zemnGbjYh4Bt8/MqASJY+pSm5ZDtc3YQox+4gsLI7yi1hba6o+eCsGxpHn5+iEVn31/FQ=="],
"has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="],
"hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
@@ -1453,7 +1461,7 @@
"webidl-conversions": ["webidl-conversions@8.0.1", "", {}, "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ=="],
"whatwg-mimetype": ["whatwg-mimetype@5.0.0", "", {}, "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw=="],
"whatwg-mimetype": ["whatwg-mimetype@3.0.0", "", {}, "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q=="],
"whatwg-url": ["whatwg-url@16.0.1", "", { "dependencies": { "@exodus/bytes": "1.15.0", "tr46": "6.0.0", "webidl-conversions": "8.0.1" } }, "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw=="],
@@ -1535,6 +1543,8 @@
"chromium-bidi/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
"data-urls/whatwg-mimetype": ["whatwg-mimetype@5.0.0", "", {}, "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw=="],
"dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
"escodegen/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
@@ -1551,6 +1561,8 @@
"jsdom/parse5": ["parse5@8.0.0", "", { "dependencies": { "entities": "6.0.1" } }, "sha512-9m4m5GSgXjL4AjumKzq1Fgfp3Z8rsvjRNbnkVwfu2ImRqE5D0LnY2QfDen18FSY9C573YU5XxSapdHZTZ2WolA=="],
"jsdom/whatwg-mimetype": ["whatwg-mimetype@5.0.0", "", {}, "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw=="],
"loose-envify/js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],
"micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
+3
View File
@@ -22,6 +22,7 @@
"lint:skills": "tsx scripts/lint-skills.ts",
"lint:fix": "oxlint --fix .",
"format": "oxfmt .",
"test": "bun run --filter '*' test",
"format:check": "oxfmt --check .",
"knip": "knip",
"generate:previews": "tsx scripts/generate-template-previews.ts",
@@ -30,8 +31,10 @@
"devDependencies": {
"@commitlint/cli": "^20.5.0",
"@commitlint/config-conventional": "^20.5.0",
"@hyperframes/player": "workspace:*",
"@types/node": "^25.0.10",
"concurrently": "^8.2.0",
"happy-dom": "^20.9.0",
"knip": "^6.0.3",
"lefthook": "^2.1.4",
"oxfmt": "^0.41.0",
+2 -1
View File
@@ -13,7 +13,8 @@ function tmpProject(name: string): string {
function validHtml(compId = "main"): string {
return `<html><body>
<div data-composition-id="${compId}" data-width="1920" data-height="1080"></div>
<div data-composition-id="${compId}" data-width="1920" data-height="1080" data-start="0" data-duration="10"></div>
<script src="https://cdn.jsdelivr.net/npm/gsap@3/dist/gsap.min.js"></script>
<script>window.__timelines = window.__timelines || {}; window.__timelines["${compId}"] = gsap.timeline({ paused: true });</script>
</body></html>`;
}
+8 -5
View File
@@ -153,8 +153,8 @@ How are you
const { words, format } = loadTranscript(path);
expect(format).toBe("srt");
expect(words).toEqual([
{ text: "Hello world", start: 1.0, end: 3.5 },
{ text: "How are you", start: 4.0, end: 6.0 },
{ text: "Hello world", start: 1.0, end: 3.5, id: "w0" },
{ text: "How are you", start: 4.0, end: 6.0, id: "w1" },
]);
});
@@ -171,8 +171,8 @@ How are you
const { words, format } = loadTranscript(path);
expect(format).toBe("vtt");
expect(words).toEqual([
{ text: "Hello world", start: 1.0, end: 3.5 },
{ text: "How are you", start: 4.0, end: 6.0 },
{ text: "Hello world", start: 1.0, end: 3.5, id: "w0" },
{ text: "How are you", start: 4.0, end: 6.0, id: "w1" },
]);
});
@@ -206,7 +206,10 @@ Short format
const path = tmpFile("normalized.json", JSON.stringify(input));
const { words, format } = loadTranscript(path);
expect(format).toBe("words-json");
expect(words).toEqual(input);
expect(words).toEqual([
{ text: "Hello", start: 0, end: 0.5, id: "" },
{ text: "world", start: 0.6, end: 1.2, id: "" },
]);
});
});
+27 -11
View File
@@ -46,17 +46,33 @@ Show a static image before playback starts:
## Attributes
| Attribute | Type | Default | Description |
| --------------- | ------- | ------- | ------------------------------------------- |
| `src` | string | — | URL to the composition HTML file |
| `width` | number | 1920 | Composition width in pixels (aspect ratio) |
| `height` | number | 1080 | Composition height in pixels (aspect ratio) |
| `controls` | boolean | false | Show play/pause, scrubber, and time display |
| `muted` | boolean | false | Mute audio playback |
| `poster` | string | — | Image URL shown before playback starts |
| `playback-rate` | number | 1 | Speed multiplier (0.5 = half, 2 = double) |
| `autoplay` | boolean | false | Start playing when ready |
| `loop` | boolean | false | Restart when the composition ends |
| Attribute | Type | Default | Description |
| --------------- | ------- | ------- | -------------------------------------------- |
| `src` | string | — | URL to the composition HTML file |
| `audio-src` | string | — | Audio URL for parent-frame playback (mobile) |
| `width` | number | 1920 | Composition width in pixels (aspect ratio) |
| `height` | number | 1080 | Composition height in pixels (aspect ratio) |
| `controls` | boolean | false | Show play/pause, scrubber, and time display |
| `muted` | boolean | false | Mute audio playback |
| `poster` | string | — | Image URL shown before playback starts |
| `playback-rate` | number | 1 | Speed multiplier (0.5 = half, 2 = double) |
| `autoplay` | boolean | false | Start playing when ready |
| `loop` | boolean | false | Restart when the composition ends |
### Mobile audio
Mobile browsers block `audio.play()` inside iframes when the user gesture happened in the parent frame (the [User Activation spec](https://html.spec.whatwg.org/multipage/interaction.html#tracking-user-activation) does not propagate activation across frame boundaries via `postMessage`).
The player handles this automatically for same-origin iframes (the default — `sandbox` includes `allow-same-origin`):
1. When the composition is ready, the player extracts all timed media (`audio[data-start]`, `video[data-start]`) from the iframe DOM and creates parent-frame copies.
2. The iframe originals are disabled (`src` and `data-start` removed) so the runtime doesn't try to play them.
3. When `play()` is called (from a user gesture), parent media `.play()` runs synchronously in the gesture call stack, satisfying mobile autoplay policy.
4. Both parent media and the GSAP timeline start simultaneously and free-run — no active sync needed since both are real-time systems.
No changes are required by consumers — this works out of the box.
The optional `audio-src` attribute can be used to start preloading a primary audio track before the iframe loads (useful on slow connections), but is not required for mobile playback.
## JavaScript API
+142 -1
View File
@@ -1,6 +1,8 @@
import { describe, it, expect } from "vitest";
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { formatTime, formatSpeed, SPEED_PRESETS } from "./controls.js";
// ── Controls unit tests ──
describe("SPEED_PRESETS", () => {
it("contains logarithmic speed steps", () => {
expect(SPEED_PRESETS).toEqual([0.25, 0.5, 1, 1.5, 2, 4]);
@@ -54,3 +56,142 @@ describe("formatTime", () => {
expect(formatTime(-5)).toBe("0:00");
});
});
// ── Parent-frame media for mobile playback ──
//
// Mobile browsers block media.play() inside iframes when the user gesture
// happened in the parent. The player works around this by extracting media
// from the iframe and playing it in the parent frame.
describe("HyperframesPlayer parent-frame media", () => {
type PlayerElement = HTMLElement & {
play: () => void;
pause: () => void;
seek: (t: number) => void;
};
let player: PlayerElement;
let mockAudio: {
src: string;
preload: string;
muted: boolean;
playbackRate: number;
currentTime: number;
paused: boolean;
play: ReturnType<typeof vi.fn>;
pause: ReturnType<typeof vi.fn>;
load: ReturnType<typeof vi.fn>;
};
beforeEach(async () => {
await import("./hyperframes-player.js");
mockAudio = {
src: "",
preload: "",
muted: false,
playbackRate: 1,
currentTime: 0,
paused: true,
play: vi.fn().mockResolvedValue(undefined),
pause: vi.fn(),
load: vi.fn(),
};
vi.spyOn(globalThis, "Audio").mockImplementation(
() => mockAudio as unknown as HTMLAudioElement,
);
player = document.createElement("hyperframes-player") as PlayerElement;
});
afterEach(() => {
player.remove();
vi.restoreAllMocks();
});
it("includes audio-src in observedAttributes", () => {
const Ctor = player.constructor as typeof HTMLElement & {
observedAttributes: string[];
};
expect(Ctor.observedAttributes).toContain("audio-src");
});
it("creates Audio and starts preloading when audio-src is set", () => {
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
document.body.appendChild(player);
expect(globalThis.Audio).toHaveBeenCalled();
expect(mockAudio.preload).toBe("auto");
expect(mockAudio.src).toBe("https://cdn.example.com/narration.mp3");
expect(mockAudio.load).toHaveBeenCalled();
});
it("syncs muted attribute to parent media", () => {
player.setAttribute("muted", "");
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
document.body.appendChild(player);
expect(mockAudio.muted).toBe(true);
});
it("syncs playback-rate to parent media", () => {
player.setAttribute("playback-rate", "1.5");
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
document.body.appendChild(player);
expect(mockAudio.playbackRate).toBe(1.5);
});
it("play() calls parentMedia.play()", () => {
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
document.body.appendChild(player);
player.play();
expect(mockAudio.play).toHaveBeenCalled();
});
it("pause() calls parentMedia.pause()", () => {
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
document.body.appendChild(player);
player.pause();
expect(mockAudio.pause).toHaveBeenCalled();
});
it("seek() sets parentMedia.currentTime", () => {
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
document.body.appendChild(player);
player.seek(12.5);
expect(mockAudio.currentTime).toBe(12.5);
});
it("cleans up parent media on disconnect", () => {
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
document.body.appendChild(player);
player.remove();
expect(mockAudio.pause).toHaveBeenCalled();
expect(mockAudio.src).toBe("");
});
it("updates parent media when playback-rate changes after setup", () => {
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
document.body.appendChild(player);
player.setAttribute("playback-rate", "2");
expect(mockAudio.playbackRate).toBe(2);
});
it("updates parent media when muted toggles after setup", () => {
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
document.body.appendChild(player);
player.setAttribute("muted", "");
expect(mockAudio.muted).toBe(true);
player.removeAttribute("muted");
expect(mockAudio.muted).toBe(false);
});
});
+123 -1
View File
@@ -7,7 +7,7 @@ const RUNTIME_CDN_URL =
class HyperframesPlayer extends HTMLElement {
static get observedAttributes() {
return ["src", "width", "height", "controls", "muted", "poster", "playback-rate"];
return ["src", "width", "height", "controls", "muted", "poster", "playback-rate", "audio-src"];
}
private shadow: ShadowRoot;
@@ -26,6 +26,21 @@ class HyperframesPlayer extends HTMLElement {
private _probeInterval: ReturnType<typeof setInterval> | null = null;
private _lastUpdateMs = 0;
/**
* Parent-frame media elements for mobile playback.
*
* Mobile browsers block media.play() inside iframes when the user gesture
* happened in the parent frame — postMessage doesn't transfer user activation
* (per the User Activation v2 spec). We extract ALL media sources from the
* iframe's timed elements (audio/video with data-start), play them in the
* parent frame (where the gesture lives), and disable the iframe copies.
*/
private _parentMedia: Array<{
el: HTMLMediaElement;
start: number;
duration: number;
}> = [];
constructor() {
super();
this.shadow = this.attachShadow({ mode: "open" });
@@ -68,6 +83,8 @@ class HyperframesPlayer extends HTMLElement {
if (this.hasAttribute("controls")) this._setupControls();
if (this.hasAttribute("poster")) this._setupPoster();
if (this.hasAttribute("audio-src"))
this._setupParentAudioFromUrl(this.getAttribute("audio-src")!);
if (this.hasAttribute("src")) this.iframe.src = this.getAttribute("src")!;
}
@@ -77,6 +94,11 @@ class HyperframesPlayer extends HTMLElement {
this.iframe.removeEventListener("load", this._onIframeLoad);
if (this._probeInterval) clearInterval(this._probeInterval);
this.controlsApi?.destroy();
for (const m of this._parentMedia) {
m.el.pause();
m.el.src = "";
}
this._parentMedia = [];
}
attributeChangedCallback(name: string, _old: string | null, val: string | null) {
@@ -107,14 +129,19 @@ class HyperframesPlayer extends HTMLElement {
break;
case "playback-rate": {
const rate = parseFloat(val || "1");
for (const m of this._parentMedia) m.el.playbackRate = rate;
this._sendControl("set-playback-rate", { playbackRate: rate });
this.controlsApi?.updateSpeed(rate);
this.dispatchEvent(new Event("ratechange"));
break;
}
case "muted":
for (const m of this._parentMedia) m.el.muted = val !== null;
this._sendControl("set-muted", { muted: val !== null });
break;
case "audio-src":
if (val) this._setupParentAudioFromUrl(val);
break;
}
}
@@ -147,6 +174,7 @@ class HyperframesPlayer extends HTMLElement {
play() {
this._hidePoster();
this._playParentMedia();
this._sendControl("play");
this._paused = false;
this.controlsApi?.updatePlaying(true);
@@ -154,6 +182,7 @@ class HyperframesPlayer extends HTMLElement {
}
pause() {
this._pauseParentMedia();
this._sendControl("pause");
this._paused = true;
this.controlsApi?.updatePlaying(false);
@@ -164,6 +193,15 @@ class HyperframesPlayer extends HTMLElement {
const frame = Math.round(timeInSeconds * DEFAULT_FPS);
this._sendControl("seek", { frame });
this._currentTime = timeInSeconds;
// Sync parent media positions (accounting for each element's start offset)
for (const m of this._parentMedia) {
const relTime = timeInSeconds - m.start;
if (relTime >= 0 && relTime < m.duration) {
m.el.currentTime = relTime;
}
}
this._paused = true;
this.controlsApi?.updatePlaying(false);
this.controlsApi?.updateTime(this._currentTime, this._duration);
@@ -238,6 +276,14 @@ class HyperframesPlayer extends HTMLElement {
const wasPlaying = !this._paused;
this._paused = !data.isPlaying;
// Sync parent media on runtime play/pause transitions (e.g. browser
// throttling, visibility change, or scrubber interaction in the iframe).
if (wasPlaying && this._paused) {
this._pauseParentMedia();
} else if (!wasPlaying && !this._paused) {
this._playParentMedia();
}
// Throttle UI updates and event dispatch to ~10fps to avoid excessive re-renders
const now = performance.now();
if (now - this._lastUpdateMs > 100 || this._paused !== wasPlaying) {
@@ -250,6 +296,7 @@ class HyperframesPlayer extends HTMLElement {
}
if (this._currentTime >= this._duration && !this._paused) {
this._pauseParentMedia();
if (this.loop) {
this.seek(0);
this.play();
@@ -351,6 +398,8 @@ class HyperframesPlayer extends HTMLElement {
}
}
this._setupParentMedia();
if (this.hasAttribute("autoplay")) {
this.play();
}
@@ -438,6 +487,79 @@ class HyperframesPlayer extends HTMLElement {
this.posterEl.src = url;
}
private _playParentMedia() {
for (const m of this._parentMedia) {
if (m.el.src) m.el.play().catch(() => {});
}
}
private _pauseParentMedia() {
for (const m of this._parentMedia) m.el.pause();
}
/** Create a parent-frame media element, configure it, and start preloading. */
private _createParentMedia(src: string, tag: "audio" | "video", start: number, duration: number) {
// Deduplicate — browsers normalize URLs so we compare on the element after assignment
if (this._parentMedia.some((m) => m.el.src === src)) return;
const el = tag === "video" ? document.createElement("video") : new Audio();
el.preload = "auto";
el.src = src;
el.load();
el.muted = this.muted;
if (this.playbackRate !== 1) el.playbackRate = this.playbackRate;
this._parentMedia.push({ el, start, duration });
}
/**
* Set up a single parent-frame audio from an explicit URL (via `audio-src`).
* Convenience for the common single-narration case — starts preloading
* immediately without waiting for the iframe to load.
*/
private _setupParentAudioFromUrl(audioSrc: string) {
this._createParentMedia(audioSrc, "audio", 0, Infinity);
}
/**
* Extract ALL timed media (audio/video with data-start) from the iframe's
* DOM and create parent-frame copies. Disables the iframe originals so the
* runtime doesn't try to play them (which would fail on mobile and cause
* double playback on desktop).
*
* If `audio-src` was already set, this just disables the iframe media.
*/
private _setupParentMedia() {
try {
const doc = this.iframe.contentDocument;
if (!doc) return;
// Find all timed media — matches the runtime's media.ts selector
const mediaEls = doc.querySelectorAll<HTMLMediaElement>(
"audio[data-start], video[data-start]",
);
for (const iframeEl of mediaEls) {
const src = iframeEl.getAttribute("src") || iframeEl.querySelector("source")?.src;
if (!src) continue;
const start = parseFloat(iframeEl.getAttribute("data-start") || "0");
const duration = parseFloat(iframeEl.getAttribute("data-duration") || "Infinity");
const tag = iframeEl.tagName === "VIDEO" ? ("video" as const) : ("audio" as const);
this._createParentMedia(src, tag, start, duration);
// Disable the iframe element so the runtime ignores it
iframeEl.removeAttribute("src");
iframeEl.removeAttribute("data-start");
iframeEl.removeAttribute("data-duration");
iframeEl.querySelectorAll("source").forEach((s) => s.remove());
}
} catch {
// Cross-origin iframe — can't access DOM, fall back to iframe media
}
}
private _hidePoster() {
this.posterEl?.remove();
this.posterEl = null;
+7
View File
@@ -0,0 +1,7 @@
import { defineConfig } from "vitest/config";
export default defineConfig({
test: {
environment: "happy-dom",
},
});
+1 -1
View File
@@ -37,7 +37,7 @@
"scripts": {
"build": "tsup",
"typecheck": "tsc --noEmit",
"test": "vitest run"
"test": "vitest run --passWithNoTests"
},
"dependencies": {
"html2canvas": "^1.4.1"