mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-20 12:25:51 +00:00
fix(player): parent-frame media playback for mobile (#266)
* fix(player): parent-frame media playback for mobile Mobile browsers block media.play() inside iframes when the user gesture happened in the parent frame — postMessage doesn't transfer user activation (per the User Activation v2 spec). ## Problem The player renders compositions in a sandboxed iframe. When a user taps play in the parent frame, the player sends a postMessage to the iframe's runtime, which calls audio.play(). On mobile, this fails silently because the iframe has no user activation context. ## Solution The player now extracts ALL timed media elements (audio/video with data-start) from the iframe's DOM (same-origin access), creates parent-frame copies, and disables the iframe originals. On play(), parentMedia.play() runs synchronously in the gesture call stack, satisfying mobile autoplay policy. ### Generic media handling - Finds all `audio[data-start], video[data-start]` in the iframe - Creates a parent-frame copy for each (Audio or Video element) - Preserves data-start offsets for correct seek positioning - Strips data-start from iframe elements so the runtime ignores them - Falls back to iframe media for cross-origin iframes ### `audio-src` attribute Convenience for the common single-narration case. When set, the player starts preloading audio immediately — before the iframe loads. This eliminates the loading delay that caused jittery playback. ### No active sync Both parent media and the GSAP timeline are real-time systems. When started simultaneously, they naturally stay within ~10ms — no drift correction needed. Active sync with coarse granularity (50ms polling) caused MORE jitter than it prevented via repeated audio seeks. ## CI - Added unified `test` job replacing separate per-package test jobs - Added root `test` script: `bun run --filter '*' test` - New packages with test scripts are automatically included - Added happy-dom for player DOM tests ## Tests - 10 new tests for parent-frame media: preloading, play, pause, seek, muted/rate sync, cleanup, attribute changes - All 21 player tests pass Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(shader-transitions): pass CI when no test files exist Add --passWithNoTests to vitest run so the unified test job doesn't fail on packages that have a test script but no test files yet. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(cli): update tests for new id field and GSAP lint rule - normalize.test.ts: loadTranscript now assigns id fields (w0, w1, etc.) to SRT/VTT results and empty string for words-json passthrough - lintProject.test.ts: add GSAP CDN script to validHtml() fixture to satisfy the missing_gsap_script lint rule added in core Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(cli): add missing data-start/data-duration to validHtml fixture The validHtml() test fixture was missing data-start and data-duration attributes, triggering the root_composition_missing_data_start and root_composition_missing_data_duration lint warnings. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(ci): fetch LFS objects for producer test job Producer regression tests compare rendered output against reference MP4 files stored in git LFS. Without lfs: true, checkout fetches pointer files instead of actual videos, causing "moov atom not found" errors. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * ci: remove redundant test-producer job The regression workflow already runs the same 28 producer fixtures in a Docker container with prod-matching Chrome/fonts/ffmpeg, sharded across 8 parallel matrix jobs with 40-min timeouts. The CI test-producer job was a duplicate that ran on bare runners with worse determinism and a 15-min timeout too short for all fixtures. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
9bf4956fae
commit
b23b0751da
@@ -89,8 +89,8 @@ jobs:
|
||||
- run: bun run build
|
||||
- run: bun run --filter '*' typecheck
|
||||
|
||||
test-core:
|
||||
name: "Test: core"
|
||||
test:
|
||||
name: Test
|
||||
needs: changes
|
||||
if: needs.changes.outputs.code == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
@@ -102,22 +102,7 @@ jobs:
|
||||
with:
|
||||
node-version: 22
|
||||
- run: bun install --frozen-lockfile
|
||||
- run: bun run --filter @hyperframes/core test -- --coverage
|
||||
|
||||
test-engine:
|
||||
name: "Test: engine"
|
||||
needs: changes
|
||||
if: needs.changes.outputs.code == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
- run: bun install --frozen-lockfile
|
||||
- run: bun run --filter @hyperframes/engine test
|
||||
- run: bun run --filter '!@hyperframes/producer' test
|
||||
|
||||
test-runtime-contract:
|
||||
name: "Test: runtime contract"
|
||||
|
||||
@@ -7,8 +7,10 @@
|
||||
"devDependencies": {
|
||||
"@commitlint/cli": "^20.5.0",
|
||||
"@commitlint/config-conventional": "^20.5.0",
|
||||
"@hyperframes/player": "^0.3.0",
|
||||
"@types/node": "^25.0.10",
|
||||
"concurrently": "^8.2.0",
|
||||
"happy-dom": "^20.9.0",
|
||||
"knip": "^6.0.3",
|
||||
"lefthook": "^2.1.4",
|
||||
"oxfmt": "^0.41.0",
|
||||
@@ -19,7 +21,7 @@
|
||||
},
|
||||
"packages/cli": {
|
||||
"name": "@hyperframes/cli",
|
||||
"version": "0.2.4",
|
||||
"version": "0.3.0",
|
||||
"bin": {
|
||||
"hyperframes": "./dist/cli.js",
|
||||
},
|
||||
@@ -59,7 +61,7 @@
|
||||
},
|
||||
"packages/core": {
|
||||
"name": "@hyperframes/core",
|
||||
"version": "0.2.4",
|
||||
"version": "0.3.0",
|
||||
"dependencies": {
|
||||
"@chenglou/pretext": "^0.0.5",
|
||||
},
|
||||
@@ -85,7 +87,7 @@
|
||||
},
|
||||
"packages/engine": {
|
||||
"name": "@hyperframes/engine",
|
||||
"version": "0.2.4",
|
||||
"version": "0.3.0",
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.13.0",
|
||||
"@hyperframes/core": "workspace:^",
|
||||
@@ -102,7 +104,7 @@
|
||||
},
|
||||
"packages/player": {
|
||||
"name": "@hyperframes/player",
|
||||
"version": "0.2.4",
|
||||
"version": "0.3.0",
|
||||
"devDependencies": {
|
||||
"tsup": "^8.0.0",
|
||||
"typescript": "^5.0.0",
|
||||
@@ -111,7 +113,7 @@
|
||||
},
|
||||
"packages/producer": {
|
||||
"name": "@hyperframes/producer",
|
||||
"version": "0.2.4",
|
||||
"version": "0.3.0",
|
||||
"dependencies": {
|
||||
"@fontsource/archivo-black": "^5.2.8",
|
||||
"@fontsource/eb-garamond": "^5.2.7",
|
||||
@@ -149,7 +151,7 @@
|
||||
},
|
||||
"packages/shader-transitions": {
|
||||
"name": "@hyperframes/shader-transitions",
|
||||
"version": "0.2.4",
|
||||
"version": "0.3.0",
|
||||
"dependencies": {
|
||||
"html2canvas": "^1.4.1",
|
||||
},
|
||||
@@ -161,7 +163,7 @@
|
||||
},
|
||||
"packages/studio": {
|
||||
"name": "@hyperframes/studio",
|
||||
"version": "0.2.4",
|
||||
"version": "0.3.0",
|
||||
"dependencies": {
|
||||
"@codemirror/autocomplete": "^6.20.1",
|
||||
"@codemirror/commands": "^6.10.3",
|
||||
@@ -731,6 +733,10 @@
|
||||
|
||||
"@types/tough-cookie": ["@types/tough-cookie@4.0.5", "", {}, "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA=="],
|
||||
|
||||
"@types/whatwg-mimetype": ["@types/whatwg-mimetype@3.0.2", "", {}, "sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA=="],
|
||||
|
||||
"@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="],
|
||||
|
||||
"@types/yauzl": ["@types/yauzl@2.10.3", "", { "dependencies": { "@types/node": "22.19.15" } }, "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q=="],
|
||||
|
||||
"@vitejs/plugin-react": ["@vitejs/plugin-react@4.7.0", "", { "dependencies": { "@babel/core": "7.29.0", "@babel/plugin-transform-react-jsx-self": "7.27.1", "@babel/plugin-transform-react-jsx-source": "7.27.1", "@rolldown/pluginutils": "1.0.0-beta.27", "@types/babel__core": "7.20.5", "react-refresh": "0.17.0" }, "peerDependencies": { "vite": "5.4.21" } }, "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA=="],
|
||||
@@ -1015,6 +1021,8 @@
|
||||
|
||||
"global-directory": ["global-directory@4.0.1", "", { "dependencies": { "ini": "4.1.1" } }, "sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q=="],
|
||||
|
||||
"happy-dom": ["happy-dom@20.9.0", "", { "dependencies": { "@types/node": ">=20.0.0", "@types/whatwg-mimetype": "^3.0.2", "@types/ws": "^8.18.1", "entities": "^7.0.1", "whatwg-mimetype": "^3.0.0", "ws": "^8.18.3" } }, "sha512-GZZ9mKe8r646NUAf/zemnGbjYh4Bt8/MqASJY+pSm5ZDtc3YQox+4gsLI7yi1hba6o+eCsGxpHn5+iEVn31/FQ=="],
|
||||
|
||||
"has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="],
|
||||
|
||||
"hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
@@ -1453,7 +1461,7 @@
|
||||
|
||||
"webidl-conversions": ["webidl-conversions@8.0.1", "", {}, "sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ=="],
|
||||
|
||||
"whatwg-mimetype": ["whatwg-mimetype@5.0.0", "", {}, "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw=="],
|
||||
"whatwg-mimetype": ["whatwg-mimetype@3.0.0", "", {}, "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q=="],
|
||||
|
||||
"whatwg-url": ["whatwg-url@16.0.1", "", { "dependencies": { "@exodus/bytes": "1.15.0", "tr46": "6.0.0", "webidl-conversions": "8.0.1" } }, "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw=="],
|
||||
|
||||
@@ -1535,6 +1543,8 @@
|
||||
|
||||
"chromium-bidi/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
|
||||
|
||||
"data-urls/whatwg-mimetype": ["whatwg-mimetype@5.0.0", "", {}, "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw=="],
|
||||
|
||||
"dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
||||
|
||||
"escodegen/source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
|
||||
@@ -1551,6 +1561,8 @@
|
||||
|
||||
"jsdom/parse5": ["parse5@8.0.0", "", { "dependencies": { "entities": "6.0.1" } }, "sha512-9m4m5GSgXjL4AjumKzq1Fgfp3Z8rsvjRNbnkVwfu2ImRqE5D0LnY2QfDen18FSY9C573YU5XxSapdHZTZ2WolA=="],
|
||||
|
||||
"jsdom/whatwg-mimetype": ["whatwg-mimetype@5.0.0", "", {}, "sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw=="],
|
||||
|
||||
"loose-envify/js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],
|
||||
|
||||
"micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
"lint:skills": "tsx scripts/lint-skills.ts",
|
||||
"lint:fix": "oxlint --fix .",
|
||||
"format": "oxfmt .",
|
||||
"test": "bun run --filter '*' test",
|
||||
"format:check": "oxfmt --check .",
|
||||
"knip": "knip",
|
||||
"generate:previews": "tsx scripts/generate-template-previews.ts",
|
||||
@@ -30,8 +31,10 @@
|
||||
"devDependencies": {
|
||||
"@commitlint/cli": "^20.5.0",
|
||||
"@commitlint/config-conventional": "^20.5.0",
|
||||
"@hyperframes/player": "workspace:*",
|
||||
"@types/node": "^25.0.10",
|
||||
"concurrently": "^8.2.0",
|
||||
"happy-dom": "^20.9.0",
|
||||
"knip": "^6.0.3",
|
||||
"lefthook": "^2.1.4",
|
||||
"oxfmt": "^0.41.0",
|
||||
|
||||
@@ -13,7 +13,8 @@ function tmpProject(name: string): string {
|
||||
|
||||
function validHtml(compId = "main"): string {
|
||||
return `<html><body>
|
||||
<div data-composition-id="${compId}" data-width="1920" data-height="1080"></div>
|
||||
<div data-composition-id="${compId}" data-width="1920" data-height="1080" data-start="0" data-duration="10"></div>
|
||||
<script src="https://cdn.jsdelivr.net/npm/gsap@3/dist/gsap.min.js"></script>
|
||||
<script>window.__timelines = window.__timelines || {}; window.__timelines["${compId}"] = gsap.timeline({ paused: true });</script>
|
||||
</body></html>`;
|
||||
}
|
||||
|
||||
@@ -153,8 +153,8 @@ How are you
|
||||
const { words, format } = loadTranscript(path);
|
||||
expect(format).toBe("srt");
|
||||
expect(words).toEqual([
|
||||
{ text: "Hello world", start: 1.0, end: 3.5 },
|
||||
{ text: "How are you", start: 4.0, end: 6.0 },
|
||||
{ text: "Hello world", start: 1.0, end: 3.5, id: "w0" },
|
||||
{ text: "How are you", start: 4.0, end: 6.0, id: "w1" },
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -171,8 +171,8 @@ How are you
|
||||
const { words, format } = loadTranscript(path);
|
||||
expect(format).toBe("vtt");
|
||||
expect(words).toEqual([
|
||||
{ text: "Hello world", start: 1.0, end: 3.5 },
|
||||
{ text: "How are you", start: 4.0, end: 6.0 },
|
||||
{ text: "Hello world", start: 1.0, end: 3.5, id: "w0" },
|
||||
{ text: "How are you", start: 4.0, end: 6.0, id: "w1" },
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -206,7 +206,10 @@ Short format
|
||||
const path = tmpFile("normalized.json", JSON.stringify(input));
|
||||
const { words, format } = loadTranscript(path);
|
||||
expect(format).toBe("words-json");
|
||||
expect(words).toEqual(input);
|
||||
expect(words).toEqual([
|
||||
{ text: "Hello", start: 0, end: 0.5, id: "" },
|
||||
{ text: "world", start: 0.6, end: 1.2, id: "" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+27
-11
@@ -46,17 +46,33 @@ Show a static image before playback starts:
|
||||
|
||||
## Attributes
|
||||
|
||||
| Attribute | Type | Default | Description |
|
||||
| --------------- | ------- | ------- | ------------------------------------------- |
|
||||
| `src` | string | — | URL to the composition HTML file |
|
||||
| `width` | number | 1920 | Composition width in pixels (aspect ratio) |
|
||||
| `height` | number | 1080 | Composition height in pixels (aspect ratio) |
|
||||
| `controls` | boolean | false | Show play/pause, scrubber, and time display |
|
||||
| `muted` | boolean | false | Mute audio playback |
|
||||
| `poster` | string | — | Image URL shown before playback starts |
|
||||
| `playback-rate` | number | 1 | Speed multiplier (0.5 = half, 2 = double) |
|
||||
| `autoplay` | boolean | false | Start playing when ready |
|
||||
| `loop` | boolean | false | Restart when the composition ends |
|
||||
| Attribute | Type | Default | Description |
|
||||
| --------------- | ------- | ------- | -------------------------------------------- |
|
||||
| `src` | string | — | URL to the composition HTML file |
|
||||
| `audio-src` | string | — | Audio URL for parent-frame playback (mobile) |
|
||||
| `width` | number | 1920 | Composition width in pixels (aspect ratio) |
|
||||
| `height` | number | 1080 | Composition height in pixels (aspect ratio) |
|
||||
| `controls` | boolean | false | Show play/pause, scrubber, and time display |
|
||||
| `muted` | boolean | false | Mute audio playback |
|
||||
| `poster` | string | — | Image URL shown before playback starts |
|
||||
| `playback-rate` | number | 1 | Speed multiplier (0.5 = half, 2 = double) |
|
||||
| `autoplay` | boolean | false | Start playing when ready |
|
||||
| `loop` | boolean | false | Restart when the composition ends |
|
||||
|
||||
### Mobile audio
|
||||
|
||||
Mobile browsers block `audio.play()` inside iframes when the user gesture happened in the parent frame (the [User Activation spec](https://html.spec.whatwg.org/multipage/interaction.html#tracking-user-activation) does not propagate activation across frame boundaries via `postMessage`).
|
||||
|
||||
The player handles this automatically for same-origin iframes (the default — `sandbox` includes `allow-same-origin`):
|
||||
|
||||
1. When the composition is ready, the player extracts all timed media (`audio[data-start]`, `video[data-start]`) from the iframe DOM and creates parent-frame copies.
|
||||
2. The iframe originals are disabled (`src` and `data-start` removed) so the runtime doesn't try to play them.
|
||||
3. When `play()` is called (from a user gesture), parent media `.play()` runs synchronously in the gesture call stack, satisfying mobile autoplay policy.
|
||||
4. Both parent media and the GSAP timeline start simultaneously and free-run — no active sync needed since both are real-time systems.
|
||||
|
||||
No changes are required by consumers — this works out of the box.
|
||||
|
||||
The optional `audio-src` attribute can be used to start preloading a primary audio track before the iframe loads (useful on slow connections), but is not required for mobile playback.
|
||||
|
||||
## JavaScript API
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { formatTime, formatSpeed, SPEED_PRESETS } from "./controls.js";
|
||||
|
||||
// ── Controls unit tests ──
|
||||
|
||||
describe("SPEED_PRESETS", () => {
|
||||
it("contains logarithmic speed steps", () => {
|
||||
expect(SPEED_PRESETS).toEqual([0.25, 0.5, 1, 1.5, 2, 4]);
|
||||
@@ -54,3 +56,142 @@ describe("formatTime", () => {
|
||||
expect(formatTime(-5)).toBe("0:00");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Parent-frame media for mobile playback ──
|
||||
//
|
||||
// Mobile browsers block media.play() inside iframes when the user gesture
|
||||
// happened in the parent. The player works around this by extracting media
|
||||
// from the iframe and playing it in the parent frame.
|
||||
|
||||
describe("HyperframesPlayer parent-frame media", () => {
|
||||
type PlayerElement = HTMLElement & {
|
||||
play: () => void;
|
||||
pause: () => void;
|
||||
seek: (t: number) => void;
|
||||
};
|
||||
|
||||
let player: PlayerElement;
|
||||
let mockAudio: {
|
||||
src: string;
|
||||
preload: string;
|
||||
muted: boolean;
|
||||
playbackRate: number;
|
||||
currentTime: number;
|
||||
paused: boolean;
|
||||
play: ReturnType<typeof vi.fn>;
|
||||
pause: ReturnType<typeof vi.fn>;
|
||||
load: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
await import("./hyperframes-player.js");
|
||||
|
||||
mockAudio = {
|
||||
src: "",
|
||||
preload: "",
|
||||
muted: false,
|
||||
playbackRate: 1,
|
||||
currentTime: 0,
|
||||
paused: true,
|
||||
play: vi.fn().mockResolvedValue(undefined),
|
||||
pause: vi.fn(),
|
||||
load: vi.fn(),
|
||||
};
|
||||
|
||||
vi.spyOn(globalThis, "Audio").mockImplementation(
|
||||
() => mockAudio as unknown as HTMLAudioElement,
|
||||
);
|
||||
|
||||
player = document.createElement("hyperframes-player") as PlayerElement;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
player.remove();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("includes audio-src in observedAttributes", () => {
|
||||
const Ctor = player.constructor as typeof HTMLElement & {
|
||||
observedAttributes: string[];
|
||||
};
|
||||
expect(Ctor.observedAttributes).toContain("audio-src");
|
||||
});
|
||||
|
||||
it("creates Audio and starts preloading when audio-src is set", () => {
|
||||
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
|
||||
document.body.appendChild(player);
|
||||
|
||||
expect(globalThis.Audio).toHaveBeenCalled();
|
||||
expect(mockAudio.preload).toBe("auto");
|
||||
expect(mockAudio.src).toBe("https://cdn.example.com/narration.mp3");
|
||||
expect(mockAudio.load).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("syncs muted attribute to parent media", () => {
|
||||
player.setAttribute("muted", "");
|
||||
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
|
||||
document.body.appendChild(player);
|
||||
|
||||
expect(mockAudio.muted).toBe(true);
|
||||
});
|
||||
|
||||
it("syncs playback-rate to parent media", () => {
|
||||
player.setAttribute("playback-rate", "1.5");
|
||||
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
|
||||
document.body.appendChild(player);
|
||||
|
||||
expect(mockAudio.playbackRate).toBe(1.5);
|
||||
});
|
||||
|
||||
it("play() calls parentMedia.play()", () => {
|
||||
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
|
||||
document.body.appendChild(player);
|
||||
|
||||
player.play();
|
||||
expect(mockAudio.play).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("pause() calls parentMedia.pause()", () => {
|
||||
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
|
||||
document.body.appendChild(player);
|
||||
|
||||
player.pause();
|
||||
expect(mockAudio.pause).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("seek() sets parentMedia.currentTime", () => {
|
||||
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
|
||||
document.body.appendChild(player);
|
||||
|
||||
player.seek(12.5);
|
||||
expect(mockAudio.currentTime).toBe(12.5);
|
||||
});
|
||||
|
||||
it("cleans up parent media on disconnect", () => {
|
||||
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
|
||||
document.body.appendChild(player);
|
||||
|
||||
player.remove();
|
||||
expect(mockAudio.pause).toHaveBeenCalled();
|
||||
expect(mockAudio.src).toBe("");
|
||||
});
|
||||
|
||||
it("updates parent media when playback-rate changes after setup", () => {
|
||||
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
|
||||
document.body.appendChild(player);
|
||||
|
||||
player.setAttribute("playback-rate", "2");
|
||||
expect(mockAudio.playbackRate).toBe(2);
|
||||
});
|
||||
|
||||
it("updates parent media when muted toggles after setup", () => {
|
||||
player.setAttribute("audio-src", "https://cdn.example.com/narration.mp3");
|
||||
document.body.appendChild(player);
|
||||
|
||||
player.setAttribute("muted", "");
|
||||
expect(mockAudio.muted).toBe(true);
|
||||
|
||||
player.removeAttribute("muted");
|
||||
expect(mockAudio.muted).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -7,7 +7,7 @@ const RUNTIME_CDN_URL =
|
||||
|
||||
class HyperframesPlayer extends HTMLElement {
|
||||
static get observedAttributes() {
|
||||
return ["src", "width", "height", "controls", "muted", "poster", "playback-rate"];
|
||||
return ["src", "width", "height", "controls", "muted", "poster", "playback-rate", "audio-src"];
|
||||
}
|
||||
|
||||
private shadow: ShadowRoot;
|
||||
@@ -26,6 +26,21 @@ class HyperframesPlayer extends HTMLElement {
|
||||
private _probeInterval: ReturnType<typeof setInterval> | null = null;
|
||||
private _lastUpdateMs = 0;
|
||||
|
||||
/**
|
||||
* Parent-frame media elements for mobile playback.
|
||||
*
|
||||
* Mobile browsers block media.play() inside iframes when the user gesture
|
||||
* happened in the parent frame — postMessage doesn't transfer user activation
|
||||
* (per the User Activation v2 spec). We extract ALL media sources from the
|
||||
* iframe's timed elements (audio/video with data-start), play them in the
|
||||
* parent frame (where the gesture lives), and disable the iframe copies.
|
||||
*/
|
||||
private _parentMedia: Array<{
|
||||
el: HTMLMediaElement;
|
||||
start: number;
|
||||
duration: number;
|
||||
}> = [];
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.shadow = this.attachShadow({ mode: "open" });
|
||||
@@ -68,6 +83,8 @@ class HyperframesPlayer extends HTMLElement {
|
||||
|
||||
if (this.hasAttribute("controls")) this._setupControls();
|
||||
if (this.hasAttribute("poster")) this._setupPoster();
|
||||
if (this.hasAttribute("audio-src"))
|
||||
this._setupParentAudioFromUrl(this.getAttribute("audio-src")!);
|
||||
if (this.hasAttribute("src")) this.iframe.src = this.getAttribute("src")!;
|
||||
}
|
||||
|
||||
@@ -77,6 +94,11 @@ class HyperframesPlayer extends HTMLElement {
|
||||
this.iframe.removeEventListener("load", this._onIframeLoad);
|
||||
if (this._probeInterval) clearInterval(this._probeInterval);
|
||||
this.controlsApi?.destroy();
|
||||
for (const m of this._parentMedia) {
|
||||
m.el.pause();
|
||||
m.el.src = "";
|
||||
}
|
||||
this._parentMedia = [];
|
||||
}
|
||||
|
||||
attributeChangedCallback(name: string, _old: string | null, val: string | null) {
|
||||
@@ -107,14 +129,19 @@ class HyperframesPlayer extends HTMLElement {
|
||||
break;
|
||||
case "playback-rate": {
|
||||
const rate = parseFloat(val || "1");
|
||||
for (const m of this._parentMedia) m.el.playbackRate = rate;
|
||||
this._sendControl("set-playback-rate", { playbackRate: rate });
|
||||
this.controlsApi?.updateSpeed(rate);
|
||||
this.dispatchEvent(new Event("ratechange"));
|
||||
break;
|
||||
}
|
||||
case "muted":
|
||||
for (const m of this._parentMedia) m.el.muted = val !== null;
|
||||
this._sendControl("set-muted", { muted: val !== null });
|
||||
break;
|
||||
case "audio-src":
|
||||
if (val) this._setupParentAudioFromUrl(val);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -147,6 +174,7 @@ class HyperframesPlayer extends HTMLElement {
|
||||
|
||||
play() {
|
||||
this._hidePoster();
|
||||
this._playParentMedia();
|
||||
this._sendControl("play");
|
||||
this._paused = false;
|
||||
this.controlsApi?.updatePlaying(true);
|
||||
@@ -154,6 +182,7 @@ class HyperframesPlayer extends HTMLElement {
|
||||
}
|
||||
|
||||
pause() {
|
||||
this._pauseParentMedia();
|
||||
this._sendControl("pause");
|
||||
this._paused = true;
|
||||
this.controlsApi?.updatePlaying(false);
|
||||
@@ -164,6 +193,15 @@ class HyperframesPlayer extends HTMLElement {
|
||||
const frame = Math.round(timeInSeconds * DEFAULT_FPS);
|
||||
this._sendControl("seek", { frame });
|
||||
this._currentTime = timeInSeconds;
|
||||
|
||||
// Sync parent media positions (accounting for each element's start offset)
|
||||
for (const m of this._parentMedia) {
|
||||
const relTime = timeInSeconds - m.start;
|
||||
if (relTime >= 0 && relTime < m.duration) {
|
||||
m.el.currentTime = relTime;
|
||||
}
|
||||
}
|
||||
|
||||
this._paused = true;
|
||||
this.controlsApi?.updatePlaying(false);
|
||||
this.controlsApi?.updateTime(this._currentTime, this._duration);
|
||||
@@ -238,6 +276,14 @@ class HyperframesPlayer extends HTMLElement {
|
||||
const wasPlaying = !this._paused;
|
||||
this._paused = !data.isPlaying;
|
||||
|
||||
// Sync parent media on runtime play/pause transitions (e.g. browser
|
||||
// throttling, visibility change, or scrubber interaction in the iframe).
|
||||
if (wasPlaying && this._paused) {
|
||||
this._pauseParentMedia();
|
||||
} else if (!wasPlaying && !this._paused) {
|
||||
this._playParentMedia();
|
||||
}
|
||||
|
||||
// Throttle UI updates and event dispatch to ~10fps to avoid excessive re-renders
|
||||
const now = performance.now();
|
||||
if (now - this._lastUpdateMs > 100 || this._paused !== wasPlaying) {
|
||||
@@ -250,6 +296,7 @@ class HyperframesPlayer extends HTMLElement {
|
||||
}
|
||||
|
||||
if (this._currentTime >= this._duration && !this._paused) {
|
||||
this._pauseParentMedia();
|
||||
if (this.loop) {
|
||||
this.seek(0);
|
||||
this.play();
|
||||
@@ -351,6 +398,8 @@ class HyperframesPlayer extends HTMLElement {
|
||||
}
|
||||
}
|
||||
|
||||
this._setupParentMedia();
|
||||
|
||||
if (this.hasAttribute("autoplay")) {
|
||||
this.play();
|
||||
}
|
||||
@@ -438,6 +487,79 @@ class HyperframesPlayer extends HTMLElement {
|
||||
this.posterEl.src = url;
|
||||
}
|
||||
|
||||
private _playParentMedia() {
|
||||
for (const m of this._parentMedia) {
|
||||
if (m.el.src) m.el.play().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
private _pauseParentMedia() {
|
||||
for (const m of this._parentMedia) m.el.pause();
|
||||
}
|
||||
|
||||
/** Create a parent-frame media element, configure it, and start preloading. */
|
||||
private _createParentMedia(src: string, tag: "audio" | "video", start: number, duration: number) {
|
||||
// Deduplicate — browsers normalize URLs so we compare on the element after assignment
|
||||
if (this._parentMedia.some((m) => m.el.src === src)) return;
|
||||
|
||||
const el = tag === "video" ? document.createElement("video") : new Audio();
|
||||
el.preload = "auto";
|
||||
el.src = src;
|
||||
el.load();
|
||||
el.muted = this.muted;
|
||||
if (this.playbackRate !== 1) el.playbackRate = this.playbackRate;
|
||||
|
||||
this._parentMedia.push({ el, start, duration });
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up a single parent-frame audio from an explicit URL (via `audio-src`).
|
||||
* Convenience for the common single-narration case — starts preloading
|
||||
* immediately without waiting for the iframe to load.
|
||||
*/
|
||||
private _setupParentAudioFromUrl(audioSrc: string) {
|
||||
this._createParentMedia(audioSrc, "audio", 0, Infinity);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract ALL timed media (audio/video with data-start) from the iframe's
|
||||
* DOM and create parent-frame copies. Disables the iframe originals so the
|
||||
* runtime doesn't try to play them (which would fail on mobile and cause
|
||||
* double playback on desktop).
|
||||
*
|
||||
* If `audio-src` was already set, this just disables the iframe media.
|
||||
*/
|
||||
private _setupParentMedia() {
|
||||
try {
|
||||
const doc = this.iframe.contentDocument;
|
||||
if (!doc) return;
|
||||
|
||||
// Find all timed media — matches the runtime's media.ts selector
|
||||
const mediaEls = doc.querySelectorAll<HTMLMediaElement>(
|
||||
"audio[data-start], video[data-start]",
|
||||
);
|
||||
|
||||
for (const iframeEl of mediaEls) {
|
||||
const src = iframeEl.getAttribute("src") || iframeEl.querySelector("source")?.src;
|
||||
if (!src) continue;
|
||||
|
||||
const start = parseFloat(iframeEl.getAttribute("data-start") || "0");
|
||||
const duration = parseFloat(iframeEl.getAttribute("data-duration") || "Infinity");
|
||||
const tag = iframeEl.tagName === "VIDEO" ? ("video" as const) : ("audio" as const);
|
||||
|
||||
this._createParentMedia(src, tag, start, duration);
|
||||
|
||||
// Disable the iframe element so the runtime ignores it
|
||||
iframeEl.removeAttribute("src");
|
||||
iframeEl.removeAttribute("data-start");
|
||||
iframeEl.removeAttribute("data-duration");
|
||||
iframeEl.querySelectorAll("source").forEach((s) => s.remove());
|
||||
}
|
||||
} catch {
|
||||
// Cross-origin iframe — can't access DOM, fall back to iframe media
|
||||
}
|
||||
}
|
||||
|
||||
private _hidePoster() {
|
||||
this.posterEl?.remove();
|
||||
this.posterEl = null;
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
environment: "happy-dom",
|
||||
},
|
||||
});
|
||||
@@ -37,7 +37,7 @@
|
||||
"scripts": {
|
||||
"build": "tsup",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "vitest run"
|
||||
"test": "vitest run --passWithNoTests"
|
||||
},
|
||||
"dependencies": {
|
||||
"html2canvas": "^1.4.1"
|
||||
|
||||
Reference in New Issue
Block a user