mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-05 10:14:30 +00:00
feat(cli): one-shot DE parallel-router trial per install for real telemetry
HF_DE_PARALLEL_ROUTER is a producer env var with no self-serve opt-in path for real users, so waiting for someone to manually enable it would never produce the real-traffic telemetry (revert rate, verify-db distribution) the router's soak plan calls for. renderLocal now enables the experiment for free on a fresh install's CLI renders until it actually engages once (routed or reverted — either produces telemetry), then persists that to ~/.hyperframes/config.json and never touches it again for that install. A render whose frame count never crosses the router's own eligibility threshold doesn't consume the trial — it stays available for a later render that does qualify. Never overrides a user's own explicit HF_DE_PARALLEL_ROUTER setting, and only engages when telemetry is enabled (no point risking the experimental path if we can't record the resulting signal). Scoped to the in-process CLI render path only — Docker renders don't thread perfSummary/errorDetails back to the CLI process, so trial consumption can't be detected there. Verified the config round-trip against a real file (fresh install -> undefined -> write true -> persists across reread), not just the mocked unit tests. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
a355fb2f6b
commit
37b6a4e7e5
@@ -4,6 +4,18 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vite
|
||||
const producerState = vi.hoisted(() => ({
|
||||
createdJobs: [] as Array<Record<string, unknown>>,
|
||||
resolveConfigCalls: [] as Array<Record<string, unknown>>,
|
||||
// Overridable per-test hook so the DE-parallel-router-trial tests can
|
||||
// mutate the job (perfSummary/errorDetails) or throw, without perturbing
|
||||
// every other test in this file that expects a plain no-op resolve.
|
||||
executeImpl: async (_job: Record<string, unknown>): Promise<void> => undefined,
|
||||
}));
|
||||
|
||||
const configState = vi.hoisted(() => ({
|
||||
// Defaults to "trial already fired" so the pre-existing renderLocal tests
|
||||
// below (which predate the DE-parallel-router trial and don't expect
|
||||
// HF_DE_PARALLEL_ROUTER to be touched) keep their exact prior behavior.
|
||||
config: { telemetryEnabled: true, deParallelRouterTrialFired: true } as Record<string, unknown>,
|
||||
writeConfigCalls: [] as Array<Record<string, unknown>>,
|
||||
}));
|
||||
|
||||
const preflightState = vi.hoisted(() => ({
|
||||
@@ -41,10 +53,18 @@ vi.mock("../utils/producer.js", () => ({
|
||||
producerState.createdJobs.push(config);
|
||||
return { config, progress: 100 };
|
||||
}),
|
||||
executeRenderJob: vi.fn(async () => undefined),
|
||||
executeRenderJob: vi.fn(async (job: Record<string, unknown>) => producerState.executeImpl(job)),
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("../telemetry/config.js", () => ({
|
||||
readConfig: vi.fn(() => ({ ...configState.config })),
|
||||
writeConfig: vi.fn((config: Record<string, unknown>) => {
|
||||
configState.config = { ...config };
|
||||
configState.writeConfigCalls.push({ ...config });
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock("../telemetry/events.js", () => ({
|
||||
trackRenderComplete: vi.fn(),
|
||||
trackRenderError: vi.fn(),
|
||||
@@ -81,13 +101,18 @@ describe("renderLocal browser GPU config", () => {
|
||||
beforeEach(() => {
|
||||
producerState.createdJobs = [];
|
||||
producerState.resolveConfigCalls = [];
|
||||
producerState.executeImpl = async () => undefined;
|
||||
configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: true };
|
||||
configState.writeConfigCalls = [];
|
||||
savedEnv.clear();
|
||||
savedEnv.set("HYPERFRAMES_FFMPEG_PATH", process.env.HYPERFRAMES_FFMPEG_PATH);
|
||||
savedEnv.set("HYPERFRAMES_FFPROBE_PATH", process.env.HYPERFRAMES_FFPROBE_PATH);
|
||||
savedEnv.set("PRODUCER_HEADLESS_SHELL_PATH", process.env.PRODUCER_HEADLESS_SHELL_PATH);
|
||||
savedEnv.set("HF_DE_PARALLEL_ROUTER", process.env.HF_DE_PARALLEL_ROUTER);
|
||||
delete process.env.HYPERFRAMES_FFMPEG_PATH;
|
||||
delete process.env.HYPERFRAMES_FFPROBE_PATH;
|
||||
delete process.env.PRODUCER_HEADLESS_SHELL_PATH;
|
||||
delete process.env.HF_DE_PARALLEL_ROUTER;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -414,6 +439,110 @@ describe("renderLocal browser GPU config", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderLocal — DE parallel-router CLI trial", () => {
|
||||
let renderLocal: typeof import("./render.js").renderLocal;
|
||||
const savedEnv = new Map<string, string | undefined>();
|
||||
|
||||
beforeAll(async () => {
|
||||
({ renderLocal } = await import("./render.js"));
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
producerState.createdJobs = [];
|
||||
producerState.executeImpl = async () => undefined;
|
||||
configState.writeConfigCalls = [];
|
||||
savedEnv.clear();
|
||||
savedEnv.set("HF_DE_PARALLEL_ROUTER", process.env.HF_DE_PARALLEL_ROUTER);
|
||||
savedEnv.set("HYPERFRAMES_FFMPEG_PATH", process.env.HYPERFRAMES_FFMPEG_PATH);
|
||||
savedEnv.set("HYPERFRAMES_FFPROBE_PATH", process.env.HYPERFRAMES_FFPROBE_PATH);
|
||||
savedEnv.set("PRODUCER_HEADLESS_SHELL_PATH", process.env.PRODUCER_HEADLESS_SHELL_PATH);
|
||||
delete process.env.HF_DE_PARALLEL_ROUTER;
|
||||
delete process.env.HYPERFRAMES_FFMPEG_PATH;
|
||||
delete process.env.HYPERFRAMES_FFPROBE_PATH;
|
||||
delete process.env.PRODUCER_HEADLESS_SHELL_PATH;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const [key, value] of savedEnv) {
|
||||
if (value === undefined) delete process.env[key];
|
||||
else process.env[key] = value;
|
||||
}
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
const baseOptions = {
|
||||
fps: { num: 30, den: 1 },
|
||||
quality: "standard" as const,
|
||||
format: "mp4" as const,
|
||||
gpu: false,
|
||||
browserGpuMode: "software" as const,
|
||||
hdrMode: "auto" as const,
|
||||
quiet: true,
|
||||
};
|
||||
|
||||
it("enables the trial (sets the env var) on a fresh install with telemetry on", async () => {
|
||||
configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: false };
|
||||
await renderLocal("/tmp/project", "/tmp/out.mp4", baseOptions);
|
||||
expect(process.env.HF_DE_PARALLEL_ROUTER).toBe("true");
|
||||
});
|
||||
|
||||
it("does not override an env var the user already set themselves", async () => {
|
||||
configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: false };
|
||||
process.env.HF_DE_PARALLEL_ROUTER = "false";
|
||||
await renderLocal("/tmp/project", "/tmp/out.mp4", baseOptions);
|
||||
expect(process.env.HF_DE_PARALLEL_ROUTER).toBe("false");
|
||||
});
|
||||
|
||||
it("does not enable the trial once it has already fired for this install", async () => {
|
||||
configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: true };
|
||||
await renderLocal("/tmp/project", "/tmp/out.mp4", baseOptions);
|
||||
expect(process.env.HF_DE_PARALLEL_ROUTER).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not enable the trial when telemetry is disabled", async () => {
|
||||
configState.config = { telemetryEnabled: false, deParallelRouterTrialFired: false };
|
||||
await renderLocal("/tmp/project", "/tmp/out.mp4", baseOptions);
|
||||
expect(process.env.HF_DE_PARALLEL_ROUTER).toBeUndefined();
|
||||
});
|
||||
|
||||
it("persists deParallelRouterTrialFired when the router actually engages on a successful render", async () => {
|
||||
configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: false };
|
||||
producerState.executeImpl = async (job) => {
|
||||
job.perfSummary = {
|
||||
resolution: { width: 100, height: 100 },
|
||||
drawElement: { parallelRouter: "routed" },
|
||||
};
|
||||
};
|
||||
await renderLocal("/tmp/project", "/tmp/out.mp4", baseOptions);
|
||||
expect(configState.writeConfigCalls).toContainEqual(
|
||||
expect.objectContaining({ deParallelRouterTrialFired: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not persist the trial as fired when the router never became eligible for this render", async () => {
|
||||
configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: false };
|
||||
producerState.executeImpl = async (job) => {
|
||||
job.perfSummary = { resolution: { width: 100, height: 100 }, drawElement: {} };
|
||||
};
|
||||
await renderLocal("/tmp/project", "/tmp/out.mp4", baseOptions);
|
||||
expect(configState.writeConfigCalls).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("persists the trial as fired from the failure path when the router engaged before a hard crash", async () => {
|
||||
configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: false };
|
||||
producerState.executeImpl = async (job) => {
|
||||
job.errorDetails = { observability: { capture: { deParallelRouter: "routed" } } };
|
||||
throw new Error("worker crashed");
|
||||
};
|
||||
await renderLocal("/tmp/project", "/tmp/out.mp4", { ...baseOptions, throwOnError: true }).catch(
|
||||
() => {},
|
||||
);
|
||||
expect(configState.writeConfigCalls).toContainEqual(
|
||||
expect.objectContaining({ deParallelRouterTrialFired: true }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("checkRenderResolutionPreflight", () => {
|
||||
let checkRenderResolutionPreflight: typeof import("./render.js").checkRenderResolutionPreflight;
|
||||
|
||||
|
||||
@@ -66,6 +66,7 @@ import {
|
||||
trackRenderPreflightRejected,
|
||||
} from "../telemetry/events.js";
|
||||
import { maybePromptRenderFeedback } from "../telemetry/feedback.js";
|
||||
import { readConfig, writeConfig } from "../telemetry/config.js";
|
||||
import { renderJobObservabilityTelemetryPayload } from "../telemetry/renderObservability.js";
|
||||
import { normalizeSkillSlug } from "../telemetry/skill.js";
|
||||
import { bytesToMb } from "../telemetry/system.js";
|
||||
@@ -1420,6 +1421,7 @@ export async function renderLocal(
|
||||
}
|
||||
|
||||
const producer = await loadProducer();
|
||||
const deParallelRouterTrialArmed = maybeEnableDeParallelRouterTrial(options.quiet);
|
||||
|
||||
const startTime = Date.now();
|
||||
const logger = createRenderTelemetryLogger(
|
||||
@@ -1462,6 +1464,7 @@ export async function renderLocal(
|
||||
try {
|
||||
await producer.executeRenderJob(job, projectDir, outputPath, onProgress);
|
||||
} catch (error: unknown) {
|
||||
maybeConsumeDeParallelRouterTrial(deParallelRouterTrialArmed, job);
|
||||
handleRenderError(
|
||||
error,
|
||||
options,
|
||||
@@ -1473,6 +1476,7 @@ export async function renderLocal(
|
||||
);
|
||||
}
|
||||
|
||||
maybeConsumeDeParallelRouterTrial(deParallelRouterTrialArmed, job);
|
||||
const elapsed = Date.now() - startTime;
|
||||
trackRenderMetrics(job, elapsed, options, false);
|
||||
printRenderComplete(
|
||||
@@ -1606,6 +1610,53 @@ function createNoopProducerLogger(): ProducerLogger {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable the DE parallel-router experiment (`HF_DE_PARALLEL_ROUTER`, default
|
||||
* off) for this render, one time per install, so we get real-traffic router
|
||||
* telemetry (revert rate, verify-db distribution) without requiring anyone
|
||||
* to manually set the env var — see `HyperframesConfig.deParallelRouterTrialFired`.
|
||||
* Returns whether this call armed it (so the caller knows to check for
|
||||
* consumption afterward) — false if it's already fired once, or the user
|
||||
* already set the env var themselves (never override an explicit choice),
|
||||
* or telemetry is disabled (no point risking the experimental path if we
|
||||
* can't even record the resulting signal).
|
||||
*/
|
||||
function maybeEnableDeParallelRouterTrial(quiet: boolean): boolean {
|
||||
if (process.env.HF_DE_PARALLEL_ROUTER !== undefined) return false;
|
||||
const config = readConfig();
|
||||
if (config.deParallelRouterTrialFired || !config.telemetryEnabled) return false;
|
||||
process.env.HF_DE_PARALLEL_ROUTER = "true";
|
||||
if (!quiet) {
|
||||
console.log(
|
||||
c.dim(
|
||||
" Trying the experimental parallel drawElement capture path once for this install " +
|
||||
"(opt out: HF_DE_PARALLEL_ROUTER=false)",
|
||||
),
|
||||
);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* After a trial-armed render, persist that the experiment actually engaged
|
||||
* (routed or reverted — either produces telemetry) so it's never enabled
|
||||
* again for this install. Checks both the success path (`perfSummary`) and
|
||||
* the failure path (`errorDetails.observability.capture`, mutated in place
|
||||
* before a hard failure throws) — a crash while routed still counts as a
|
||||
* fired trial. No-ops if the router never actually became eligible for this
|
||||
* render (e.g. too few frames): the trial stays available for a future run.
|
||||
*/
|
||||
function maybeConsumeDeParallelRouterTrial(trialArmed: boolean, job: RenderJob): void {
|
||||
if (!trialArmed) return;
|
||||
const engaged =
|
||||
job.perfSummary?.drawElement?.parallelRouter ??
|
||||
job.errorDetails?.observability?.capture.deParallelRouter;
|
||||
if (engaged === undefined) return;
|
||||
const config = readConfig();
|
||||
config.deParallelRouterTrialFired = true;
|
||||
writeConfig(config);
|
||||
}
|
||||
|
||||
function handleRenderError(
|
||||
error: unknown,
|
||||
options: RenderOptions,
|
||||
|
||||
@@ -63,6 +63,16 @@ export interface HyperframesConfig {
|
||||
skillsOutdatedCount?: number;
|
||||
/** How many skills were missing (not installed) at the last check. */
|
||||
skillsMissingCount?: number;
|
||||
/**
|
||||
* True once the DE parallel-router experiment ("HF_DE_PARALLEL_ROUTER")
|
||||
* has actually engaged (routed or reverted — either produces telemetry)
|
||||
* on a render from this install. The CLI enables the experiment for free
|
||||
* on renders from a fresh install until this fires once, then never
|
||||
* touches it again — a one-shot trial to get real-traffic router
|
||||
* telemetry without requiring anyone to manually opt in via env var.
|
||||
* See `renderLocal`'s `maybeEnableDeParallelRouterTrial`.
|
||||
*/
|
||||
deParallelRouterTrialFired?: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_CONFIG: HyperframesConfig = {
|
||||
@@ -108,6 +118,7 @@ export function readConfig(): HyperframesConfig {
|
||||
skillsUpdateAvailable: parsed.skillsUpdateAvailable,
|
||||
skillsOutdatedCount: parsed.skillsOutdatedCount,
|
||||
skillsMissingCount: parsed.skillsMissingCount,
|
||||
deParallelRouterTrialFired: parsed.deParallelRouterTrialFired,
|
||||
};
|
||||
|
||||
cachedConfig = config;
|
||||
|
||||
Reference in New Issue
Block a user