mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-03 12:54:29 +00:00
feat(core): pitch shift — a granular shifter as the fifth FX worklet
Adds hf-pitchshift alongside the four existing dynamics worklets: a dual-tap granular delay line, 100 ms grain, taps 180° apart so one is always crossfading in as the other resets — hides the splice each tap makes on wrap. Read-tap speed relative to the write head tracks the semitone ratio, so pitch shifts without changing duration. Registered through the same workletBuilder/dispose-message path the other four use (so shapeOf never rebuilds on a param tweak, and a chain drop retires it), wired into the registry with a plain-language copy entry and a ~0.2s chain tail (two grains). One implementation, shared by preview (Web Audio in the page) and render (the same worklet run inside an OfflineAudioContext in the headless browser) — confirmed by a browser-render test that measures the actual output frequency, not just that it differs from input. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
adfdb69a78
commit
fa05d3a7c6
@@ -505,6 +505,7 @@ const BUILDERS: Record<string, Builder> = {
|
||||
"worklet-limiter": workletBuilder("hf-limiter"),
|
||||
"worklet-gate": workletBuilder("hf-gate"),
|
||||
"worklet-bitcrush": workletBuilder("hf-bitcrush"),
|
||||
"worklet-pitchshift": workletBuilder("hf-pitchshift"),
|
||||
waveshaper,
|
||||
"delay-feedback": delayFeedback,
|
||||
"chorus-lfo": chorusLfo,
|
||||
|
||||
@@ -58,23 +58,42 @@ function delayTail(time: number, feedback: number): number {
|
||||
return Math.ceil(Math.log(TAIL_FLOOR) / Math.log(fb)) * gap;
|
||||
}
|
||||
|
||||
// Exactly the generated impulse's length — see synthesizeReverbImpulse, which
|
||||
// is the same expression. A convolution is as long as its impulse.
|
||||
function reverbTail(node: HfAudioFxNode, automation?: HfAutomation): number {
|
||||
return knobMax(node, "wet", automation) > 0
|
||||
? 0.6 + Math.max(0, Math.min(1, knobMax(node, "size", automation))) * 2.6
|
||||
: 0;
|
||||
}
|
||||
|
||||
function delayNodeTail(node: HfAudioFxNode, automation?: HfAutomation): number {
|
||||
return knobMax(node, "mix", automation) > 0
|
||||
? delayTail(knobMax(node, "time", automation), knobMax(node, "feedback", automation))
|
||||
: 0;
|
||||
}
|
||||
|
||||
/** A single delay line, no feedback: it rings for one delay (≤100 ms). */
|
||||
function chorusTail(node: HfAudioFxNode, automation?: HfAutomation): number {
|
||||
return knobMax(node, "mix", automation) > 0 ? knobMax(node, "delay", automation) / 1000 : 0;
|
||||
}
|
||||
|
||||
/** Two 100 ms grains: worst case the tail is still draining the grain that was mid-crossfade when the input stopped. */
|
||||
function pitchshiftTail(node: HfAudioFxNode, automation?: HfAutomation): number {
|
||||
return knobMax(node, "mix", automation) > 0 ? 0.2 : 0;
|
||||
}
|
||||
|
||||
/** One node's tail. Zero when it has none, or when it is mixed out entirely. */
|
||||
function nodeTail(node: HfAudioFxNode, automation?: HfAutomation): number {
|
||||
if (node.enabled === false) return 0;
|
||||
switch (node.type) {
|
||||
case "reverb":
|
||||
// Exactly the generated impulse's length — see synthesizeReverbImpulse,
|
||||
// which is the same expression. A convolution is as long as its impulse.
|
||||
return knobMax(node, "wet", automation) > 0
|
||||
? 0.6 + Math.max(0, Math.min(1, knobMax(node, "size", automation))) * 2.6
|
||||
: 0;
|
||||
return reverbTail(node, automation);
|
||||
case "delay":
|
||||
return knobMax(node, "mix", automation) > 0
|
||||
? delayTail(knobMax(node, "time", automation), knobMax(node, "feedback", automation))
|
||||
: 0;
|
||||
return delayNodeTail(node, automation);
|
||||
case "chorus":
|
||||
// A single delay line, no feedback: it rings for one delay (≤100 ms).
|
||||
return knobMax(node, "mix", automation) > 0 ? knobMax(node, "delay", automation) / 1000 : 0;
|
||||
return chorusTail(node, automation);
|
||||
case "pitchshift":
|
||||
return pitchshiftTail(node, automation);
|
||||
default:
|
||||
// Everything else settles with its input. The phaser is an all-pass chain
|
||||
// with no recirculation (group delay, not a tail); the dynamics nodes have
|
||||
|
||||
@@ -88,6 +88,7 @@ describe("the worklet processors themselves", () => {
|
||||
"hf-limiter",
|
||||
"hf-gate",
|
||||
"hf-bitcrush",
|
||||
"hf-pitchshift",
|
||||
]);
|
||||
|
||||
for (const [name, Cls] of processors) {
|
||||
@@ -100,4 +101,81 @@ describe("the worklet processors themselves", () => {
|
||||
expect(p.process(block(), block()), `${name} came back to life`).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
describe("HfPitchshift", () => {
|
||||
const SR = 48000;
|
||||
const BLOCK = 128;
|
||||
|
||||
/** Run a mono processor over a whole signal, 128 samples at a time. */
|
||||
function run(p: Processor, signal: Float32Array): Float32Array {
|
||||
const out = new Float32Array(signal.length);
|
||||
for (let at = 0; at < signal.length; at += BLOCK) {
|
||||
const inBlock = new Float32Array(BLOCK);
|
||||
inBlock.set(signal.subarray(at, at + BLOCK));
|
||||
const outBlock = new Float32Array(BLOCK);
|
||||
p.process([[inBlock]], [[outBlock]]);
|
||||
out.set(outBlock.subarray(0, Math.min(BLOCK, signal.length - at)), at);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function sine(freq: number, seconds: number): Float32Array {
|
||||
const n = Math.round(SR * seconds);
|
||||
const s = new Float32Array(n);
|
||||
for (let i = 0; i < n; i++) s[i] = Math.sin((2 * Math.PI * freq * i) / SR);
|
||||
return s;
|
||||
}
|
||||
|
||||
/** Rising zero-crossings per second — coarse but enough to catch an octave. */
|
||||
function estimateFreq(s: Float32Array, from: number): number {
|
||||
const start = Math.round(from * SR);
|
||||
let crossings = 0;
|
||||
for (let i = start + 1; i < s.length; i++) {
|
||||
if ((s[i - 1] ?? 0) < 0 && (s[i] ?? 0) >= 0) crossings++;
|
||||
}
|
||||
return crossings / ((s.length - start) / SR);
|
||||
}
|
||||
|
||||
it("at semitones: 0, mix: 1 reproduces the input, delayed by exactly one grain/2", async () => {
|
||||
const HfPitchshift = (await loadProcessors()).get("hf-pitchshift");
|
||||
if (!HfPitchshift) throw new Error("hf-pitchshift not registered");
|
||||
const p = new HfPitchshift({ processorOptions: { semitones: 0, mix: 1 } });
|
||||
const input = sine(440, 0.5);
|
||||
const output = run(p, input);
|
||||
const grain = Math.round(SR * 0.1);
|
||||
// readTap reads from `write - 1`, i.e. one sample behind the one just
|
||||
// written in this same iteration — so the effective delay is one sample
|
||||
// more than the nominal grain/2.
|
||||
const delay = grain / 2 + 1;
|
||||
// Skip the first grain while the ring buffer is still filling.
|
||||
let maxErr = 0;
|
||||
for (let i = grain * 2; i < input.length; i++) {
|
||||
maxErr = Math.max(maxErr, Math.abs((output[i] ?? 0) - (input[i - delay] ?? 0)));
|
||||
}
|
||||
expect(maxErr).toBeLessThan(1e-6);
|
||||
});
|
||||
|
||||
it("at semitones: 12, doubles the fundamental (one octave up)", async () => {
|
||||
const HfPitchshift = (await loadProcessors()).get("hf-pitchshift");
|
||||
if (!HfPitchshift) throw new Error("hf-pitchshift not registered");
|
||||
const p = new HfPitchshift({ processorOptions: { semitones: 12, mix: 1 } });
|
||||
const input = sine(220, 0.5);
|
||||
const output = run(p, input);
|
||||
// Skip the first couple of grains so the ring buffer is warm.
|
||||
const freq = estimateFreq(output, 0.05);
|
||||
expect(freq).toBeGreaterThan(220 * 1.7);
|
||||
expect(freq).toBeLessThan(220 * 2.3);
|
||||
});
|
||||
|
||||
it("at semitones: -12, halves the fundamental (one octave down)", async () => {
|
||||
const HfPitchshift = (await loadProcessors()).get("hf-pitchshift");
|
||||
if (!HfPitchshift) throw new Error("hf-pitchshift not registered");
|
||||
const p = new HfPitchshift({ processorOptions: { semitones: -12, mix: 1 } });
|
||||
const input = sine(440, 0.5);
|
||||
const output = run(p, input);
|
||||
const freq = estimateFreq(output, 0.05);
|
||||
expect(freq).toBeGreaterThan(440 * 0.35);
|
||||
expect(freq).toBeLessThan(440 * 0.65);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -219,6 +219,83 @@ class HfBitcrush extends AudioWorkletProcessor {
|
||||
}
|
||||
}
|
||||
registerProcessor("hf-bitcrush", HfBitcrush);
|
||||
|
||||
/** Linear-interpolated read, \`delaySamples\` behind the write head. */
|
||||
function readTap(ring, write, delaySamples) {
|
||||
const len = ring.length;
|
||||
const pos = (write - 1 - delaySamples + len) % len;
|
||||
const i0 = Math.floor(pos);
|
||||
const frac = pos - i0;
|
||||
const i1 = (i0 + 1) % len;
|
||||
return ring[i0] * (1 - frac) + ring[i1] * frac;
|
||||
}
|
||||
|
||||
/** Equal-power-ish crossfade, zero at a tap's reset point — hides the splice. */
|
||||
function xfade(phase) {
|
||||
return Math.sin(Math.PI * phase);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dual-tap granular delay line: two read taps 180° apart in a 100 ms grain,
|
||||
* each sweeping at a speed relative to the write head that shifts pitch
|
||||
* without changing duration. One tap is always fading in as the other fades
|
||||
* out, which hides the splice each tap makes when it wraps.
|
||||
*
|
||||
* write/phase are block-level state, advanced once per SAMPLE across all
|
||||
* channels together (not once per channel) — advancing them inside the
|
||||
* per-channel loop would move the tap 2x/4x too fast on a stereo/quad input.
|
||||
*/
|
||||
class HfPitchshift extends AudioWorkletProcessor {
|
||||
constructor(o) {
|
||||
super();
|
||||
this.p = o.processorOptions || {};
|
||||
this.grain = Math.round(sampleRate * 0.1);
|
||||
this.buf = [];
|
||||
this.write = 0;
|
||||
this.phase = 0;
|
||||
this.port.onmessage = (e) => {
|
||||
if (e.data && e.data.__hfDispose) { this.dead = true; return; }
|
||||
this.p = { ...this.p, ...e.data };
|
||||
};
|
||||
}
|
||||
process(inputs, outputs) {
|
||||
if (this.dead) return false;
|
||||
const i = inputs[0], o = outputs[0];
|
||||
if (!i || !i.length) return true;
|
||||
const p = this.p;
|
||||
const semitones = Math.max(-12, Math.min(12, p.semitones ?? 0));
|
||||
const mix = Math.max(0, Math.min(1, p.mix ?? 1));
|
||||
const ratio = Math.pow(2, semitones / 12);
|
||||
const grain = this.grain;
|
||||
const ringLen = grain * 2;
|
||||
const inc = (1 - ratio) / grain;
|
||||
const n = i[0] ? i[0].length : 0;
|
||||
for (let ch = 0; ch < i.length; ch++) {
|
||||
if (!this.buf[ch]) this.buf[ch] = new Float32Array(ringLen);
|
||||
}
|
||||
let write = this.write, phase = this.phase;
|
||||
for (let s = 0; s < n; s++) {
|
||||
phase += inc;
|
||||
phase -= Math.floor(phase);
|
||||
const phaseB = (phase + 0.5) % 1;
|
||||
const gA = xfade(phase), gB = xfade(phaseB);
|
||||
for (let ch = 0; ch < i.length; ch++) {
|
||||
const ring = this.buf[ch];
|
||||
const inp = i[ch], out = o[ch];
|
||||
const x = inp[s];
|
||||
ring[write] = x;
|
||||
const wet =
|
||||
readTap(ring, write, phase * grain) * gA + readTap(ring, write, phaseB * grain) * gB;
|
||||
out[s] = x * (1 - mix) + wet * mix;
|
||||
}
|
||||
write = (write + 1) % ringLen;
|
||||
}
|
||||
this.write = write;
|
||||
this.phase = phase;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor("hf-pitchshift", HfPitchshift);
|
||||
`;
|
||||
|
||||
// Registration is per context, not per module: a processor registered on one
|
||||
|
||||
@@ -505,6 +505,35 @@ export const HF_AUDIO_FX: readonly HfAudioFxDef[] = [
|
||||
],
|
||||
web: "worklet-bitcrush",
|
||||
},
|
||||
{
|
||||
id: "pitchshift",
|
||||
label: "Pitch shift",
|
||||
group: "time",
|
||||
description: "Shifts pitch up or down without changing playback speed.",
|
||||
params: [
|
||||
{
|
||||
kind: "number",
|
||||
key: "semitones",
|
||||
label: "Semitones",
|
||||
unit: "st",
|
||||
min: -12,
|
||||
max: 12,
|
||||
step: 1,
|
||||
default: 0,
|
||||
},
|
||||
{
|
||||
kind: "number",
|
||||
key: "mix",
|
||||
label: "Mix",
|
||||
unit: "",
|
||||
min: 0,
|
||||
max: 1,
|
||||
step: 0.01,
|
||||
default: 1,
|
||||
},
|
||||
],
|
||||
web: "worklet-pitchshift",
|
||||
},
|
||||
|
||||
{
|
||||
id: "delay",
|
||||
|
||||
@@ -221,6 +221,17 @@ export const EFFECT_COPY: Record<string, EffectCopy> = {
|
||||
mix: { label: "Blend with the original" },
|
||||
},
|
||||
},
|
||||
pitchshift: {
|
||||
title: "Higher or Lower",
|
||||
does: "Shifts everything up or down without changing its speed.",
|
||||
reachFor: "It should sound squeakier, or deeper.",
|
||||
primary: "semitones",
|
||||
primaryEnds: { low: "Much deeper", high: "Much higher" },
|
||||
params: {
|
||||
semitones: { label: "How far", ends: { low: "Much deeper", high: "Much higher" } },
|
||||
mix: { label: "Blend with the original" },
|
||||
},
|
||||
},
|
||||
delay: {
|
||||
title: "Echo",
|
||||
does: "Repeats the sound after a gap.",
|
||||
@@ -386,6 +397,10 @@ export const SUMMARY: Record<string, (p: P) => string> = {
|
||||
saturate: (p) =>
|
||||
`${strength(Math.min(1, Math.abs(n(p.threshold, -6)) / 30), ["A little", "Some", "Heavy"])} warmth`,
|
||||
bitcrush: (p) => `Crushed to ${n(p.bits, 8)} bits`,
|
||||
pitchshift: (p) =>
|
||||
n(p.semitones, 0) === 0
|
||||
? "Unchanged pitch"
|
||||
: `${n(p.semitones, 0) > 0 ? "Up" : "Down"} ${Math.abs(n(p.semitones, 0))} semitones`,
|
||||
delay: (p) => `Echo every ${n(p.time, 250)} ms`,
|
||||
reverb: (p) =>
|
||||
`${strength(n(p.size, 0.7), ["A small", "A medium", "A large"])} room, ${strength(n(p.wet, 0.35), ["lightly", "moderately", "heavily"])}`,
|
||||
|
||||
@@ -80,6 +80,24 @@ const rms = (s: Float32Array): number =>
|
||||
Math.sqrt(s.reduce((a, x) => a + x * x, 0) / Math.max(1, s.length));
|
||||
const db = (x: number): number => 20 * Math.log10(x + 1e-30);
|
||||
|
||||
/**
|
||||
* Rising zero-crossings per second, over `[from, to)` seconds.
|
||||
*
|
||||
* `to` matters as much as `from`: a chain with a tail (reverb, delay,
|
||||
* pitchshift) renders extra silence/decay past the input's own end, and
|
||||
* averaging crossings over that stretch too dilutes the estimate toward zero
|
||||
* — measure only the steady, driven portion.
|
||||
*/
|
||||
function estimateFreq(s: Float32Array, sampleRate: number, from = 0.05, to?: number): number {
|
||||
const start = Math.round(from * sampleRate);
|
||||
const end = to === undefined ? s.length : Math.min(s.length, Math.round(to * sampleRate));
|
||||
let crossings = 0;
|
||||
for (let i = start + 1; i < end; i++) {
|
||||
if ((s[i - 1] ?? 0) < 0 && (s[i] ?? 0) >= 0) crossings++;
|
||||
}
|
||||
return crossings / ((end - start) / sampleRate);
|
||||
}
|
||||
|
||||
describe("readWav / writeWav", () => {
|
||||
it("round-trips samples as 16-bit PCM, the format the volume bake requires", () => {
|
||||
const p = join(dir, "rt.wav");
|
||||
@@ -244,6 +262,33 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {
|
||||
expect(db(rms(readWav(outPath).samples))).toBeLessThan(db(rms(readWav(input).samples)) - 3);
|
||||
}, 180_000);
|
||||
|
||||
it("shifts pitch up an octave, matching the preview worklet", async () => {
|
||||
const input = join(dir, "in.wav");
|
||||
tone(input, 0.5, 220);
|
||||
const outPath = join(dir, "out.wav");
|
||||
await applyAudioFxChain(
|
||||
input,
|
||||
{
|
||||
version: 1,
|
||||
nodes: [
|
||||
{
|
||||
type: "pitchshift",
|
||||
enabled: true,
|
||||
params: { ...defaultAudioFxParams("pitchshift"), semitones: 12, mix: 1 },
|
||||
},
|
||||
],
|
||||
},
|
||||
outPath,
|
||||
{ trackId: "t" },
|
||||
);
|
||||
// Measure only the driven portion — chainTailSeconds appends ~0.2s of
|
||||
// decaying tail past the clip's own 0.5s, and averaging crossings over
|
||||
// that too dilutes the estimate.
|
||||
const freq = estimateFreq(readWav(outPath).samples, SR, 0.05, 0.45);
|
||||
expect(freq).toBeGreaterThan(220 * 1.7);
|
||||
expect(freq).toBeLessThan(220 * 2.3);
|
||||
}, 180_000);
|
||||
|
||||
it("sweeps a filter across the clip when a lane automates it", async () => {
|
||||
// A 2 kHz tone under a lowpass whose cutoff rises from below it to well
|
||||
// above: the start should be attenuated and the end should not. This is
|
||||
|
||||
Reference in New Issue
Block a user