From 37b6a4e7e557d4ec7008ea64d22743602a0c65fb Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Thu, 9 Jul 2026 21:40:16 -0700 Subject: [PATCH] feat(cli): one-shot DE parallel-router trial per install for real telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HF_DE_PARALLEL_ROUTER is a producer env var with no self-serve opt-in path for real users, so waiting for someone to manually enable it would never produce the real-traffic telemetry (revert rate, verify-db distribution) the router's soak plan calls for. renderLocal now enables the experiment for free on a fresh install's CLI renders until it actually engages once (routed or reverted — either produces telemetry), then persists that to ~/.hyperframes/config.json and never touches it again for that install. A render whose frame count never crosses the router's own eligibility threshold doesn't consume the trial — it stays available for a later render that does qualify. Never overrides a user's own explicit HF_DE_PARALLEL_ROUTER setting, and only engages when telemetry is enabled (no point risking the experimental path if we can't record the resulting signal). Scoped to the in-process CLI render path only — Docker renders don't thread perfSummary/errorDetails back to the CLI process, so trial consumption can't be detected there. Verified the config round-trip against a real file (fresh install -> undefined -> write true -> persists across reread), not just the mocked unit tests. Co-Authored-By: Claude Sonnet 5 --- packages/cli/src/commands/render.test.ts | 131 ++++++++++++++++++++++- packages/cli/src/commands/render.ts | 51 +++++++++ packages/cli/src/telemetry/config.ts | 11 ++ 3 files changed, 192 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/commands/render.test.ts b/packages/cli/src/commands/render.test.ts index 8df4a75e0..551f086e4 100644 --- a/packages/cli/src/commands/render.test.ts +++ b/packages/cli/src/commands/render.test.ts @@ -4,6 +4,18 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vite const producerState = vi.hoisted(() => ({ createdJobs: [] as Array>, resolveConfigCalls: [] as Array>, + // Overridable per-test hook so the DE-parallel-router-trial tests can + // mutate the job (perfSummary/errorDetails) or throw, without perturbing + // every other test in this file that expects a plain no-op resolve. + executeImpl: async (_job: Record): Promise => undefined, +})); + +const configState = vi.hoisted(() => ({ + // Defaults to "trial already fired" so the pre-existing renderLocal tests + // below (which predate the DE-parallel-router trial and don't expect + // HF_DE_PARALLEL_ROUTER to be touched) keep their exact prior behavior. + config: { telemetryEnabled: true, deParallelRouterTrialFired: true } as Record, + writeConfigCalls: [] as Array>, })); const preflightState = vi.hoisted(() => ({ @@ -41,10 +53,18 @@ vi.mock("../utils/producer.js", () => ({ producerState.createdJobs.push(config); return { config, progress: 100 }; }), - executeRenderJob: vi.fn(async () => undefined), + executeRenderJob: vi.fn(async (job: Record) => producerState.executeImpl(job)), })), })); +vi.mock("../telemetry/config.js", () => ({ + readConfig: vi.fn(() => ({ ...configState.config })), + writeConfig: vi.fn((config: Record) => { + configState.config = { ...config }; + configState.writeConfigCalls.push({ ...config }); + }), +})); + vi.mock("../telemetry/events.js", () => ({ trackRenderComplete: vi.fn(), trackRenderError: vi.fn(), @@ -81,13 +101,18 @@ describe("renderLocal browser GPU config", () => { beforeEach(() => { producerState.createdJobs = []; producerState.resolveConfigCalls = []; + producerState.executeImpl = async () => undefined; + configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: true }; + configState.writeConfigCalls = []; savedEnv.clear(); savedEnv.set("HYPERFRAMES_FFMPEG_PATH", process.env.HYPERFRAMES_FFMPEG_PATH); savedEnv.set("HYPERFRAMES_FFPROBE_PATH", process.env.HYPERFRAMES_FFPROBE_PATH); savedEnv.set("PRODUCER_HEADLESS_SHELL_PATH", process.env.PRODUCER_HEADLESS_SHELL_PATH); + savedEnv.set("HF_DE_PARALLEL_ROUTER", process.env.HF_DE_PARALLEL_ROUTER); delete process.env.HYPERFRAMES_FFMPEG_PATH; delete process.env.HYPERFRAMES_FFPROBE_PATH; delete process.env.PRODUCER_HEADLESS_SHELL_PATH; + delete process.env.HF_DE_PARALLEL_ROUTER; }); afterEach(() => { @@ -414,6 +439,110 @@ describe("renderLocal browser GPU config", () => { }); }); +describe("renderLocal — DE parallel-router CLI trial", () => { + let renderLocal: typeof import("./render.js").renderLocal; + const savedEnv = new Map(); + + beforeAll(async () => { + ({ renderLocal } = await import("./render.js")); + }); + + beforeEach(() => { + producerState.createdJobs = []; + producerState.executeImpl = async () => undefined; + configState.writeConfigCalls = []; + savedEnv.clear(); + savedEnv.set("HF_DE_PARALLEL_ROUTER", process.env.HF_DE_PARALLEL_ROUTER); + savedEnv.set("HYPERFRAMES_FFMPEG_PATH", process.env.HYPERFRAMES_FFMPEG_PATH); + savedEnv.set("HYPERFRAMES_FFPROBE_PATH", process.env.HYPERFRAMES_FFPROBE_PATH); + savedEnv.set("PRODUCER_HEADLESS_SHELL_PATH", process.env.PRODUCER_HEADLESS_SHELL_PATH); + delete process.env.HF_DE_PARALLEL_ROUTER; + delete process.env.HYPERFRAMES_FFMPEG_PATH; + delete process.env.HYPERFRAMES_FFPROBE_PATH; + delete process.env.PRODUCER_HEADLESS_SHELL_PATH; + }); + + afterEach(() => { + for (const [key, value] of savedEnv) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + vi.clearAllMocks(); + }); + + const baseOptions = { + fps: { num: 30, den: 1 }, + quality: "standard" as const, + format: "mp4" as const, + gpu: false, + browserGpuMode: "software" as const, + hdrMode: "auto" as const, + quiet: true, + }; + + it("enables the trial (sets the env var) on a fresh install with telemetry on", async () => { + configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: false }; + await renderLocal("/tmp/project", "/tmp/out.mp4", baseOptions); + expect(process.env.HF_DE_PARALLEL_ROUTER).toBe("true"); + }); + + it("does not override an env var the user already set themselves", async () => { + configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: false }; + process.env.HF_DE_PARALLEL_ROUTER = "false"; + await renderLocal("/tmp/project", "/tmp/out.mp4", baseOptions); + expect(process.env.HF_DE_PARALLEL_ROUTER).toBe("false"); + }); + + it("does not enable the trial once it has already fired for this install", async () => { + configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: true }; + await renderLocal("/tmp/project", "/tmp/out.mp4", baseOptions); + expect(process.env.HF_DE_PARALLEL_ROUTER).toBeUndefined(); + }); + + it("does not enable the trial when telemetry is disabled", async () => { + configState.config = { telemetryEnabled: false, deParallelRouterTrialFired: false }; + await renderLocal("/tmp/project", "/tmp/out.mp4", baseOptions); + expect(process.env.HF_DE_PARALLEL_ROUTER).toBeUndefined(); + }); + + it("persists deParallelRouterTrialFired when the router actually engages on a successful render", async () => { + configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: false }; + producerState.executeImpl = async (job) => { + job.perfSummary = { + resolution: { width: 100, height: 100 }, + drawElement: { parallelRouter: "routed" }, + }; + }; + await renderLocal("/tmp/project", "/tmp/out.mp4", baseOptions); + expect(configState.writeConfigCalls).toContainEqual( + expect.objectContaining({ deParallelRouterTrialFired: true }), + ); + }); + + it("does not persist the trial as fired when the router never became eligible for this render", async () => { + configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: false }; + producerState.executeImpl = async (job) => { + job.perfSummary = { resolution: { width: 100, height: 100 }, drawElement: {} }; + }; + await renderLocal("/tmp/project", "/tmp/out.mp4", baseOptions); + expect(configState.writeConfigCalls).toHaveLength(0); + }); + + it("persists the trial as fired from the failure path when the router engaged before a hard crash", async () => { + configState.config = { telemetryEnabled: true, deParallelRouterTrialFired: false }; + producerState.executeImpl = async (job) => { + job.errorDetails = { observability: { capture: { deParallelRouter: "routed" } } }; + throw new Error("worker crashed"); + }; + await renderLocal("/tmp/project", "/tmp/out.mp4", { ...baseOptions, throwOnError: true }).catch( + () => {}, + ); + expect(configState.writeConfigCalls).toContainEqual( + expect.objectContaining({ deParallelRouterTrialFired: true }), + ); + }); +}); + describe("checkRenderResolutionPreflight", () => { let checkRenderResolutionPreflight: typeof import("./render.js").checkRenderResolutionPreflight; diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index e4a7af554..10a9e9f39 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -66,6 +66,7 @@ import { trackRenderPreflightRejected, } from "../telemetry/events.js"; import { maybePromptRenderFeedback } from "../telemetry/feedback.js"; +import { readConfig, writeConfig } from "../telemetry/config.js"; import { renderJobObservabilityTelemetryPayload } from "../telemetry/renderObservability.js"; import { normalizeSkillSlug } from "../telemetry/skill.js"; import { bytesToMb } from "../telemetry/system.js"; @@ -1420,6 +1421,7 @@ export async function renderLocal( } const producer = await loadProducer(); + const deParallelRouterTrialArmed = maybeEnableDeParallelRouterTrial(options.quiet); const startTime = Date.now(); const logger = createRenderTelemetryLogger( @@ -1462,6 +1464,7 @@ export async function renderLocal( try { await producer.executeRenderJob(job, projectDir, outputPath, onProgress); } catch (error: unknown) { + maybeConsumeDeParallelRouterTrial(deParallelRouterTrialArmed, job); handleRenderError( error, options, @@ -1473,6 +1476,7 @@ export async function renderLocal( ); } + maybeConsumeDeParallelRouterTrial(deParallelRouterTrialArmed, job); const elapsed = Date.now() - startTime; trackRenderMetrics(job, elapsed, options, false); printRenderComplete( @@ -1606,6 +1610,53 @@ function createNoopProducerLogger(): ProducerLogger { }; } +/** + * Enable the DE parallel-router experiment (`HF_DE_PARALLEL_ROUTER`, default + * off) for this render, one time per install, so we get real-traffic router + * telemetry (revert rate, verify-db distribution) without requiring anyone + * to manually set the env var — see `HyperframesConfig.deParallelRouterTrialFired`. + * Returns whether this call armed it (so the caller knows to check for + * consumption afterward) — false if it's already fired once, or the user + * already set the env var themselves (never override an explicit choice), + * or telemetry is disabled (no point risking the experimental path if we + * can't even record the resulting signal). + */ +function maybeEnableDeParallelRouterTrial(quiet: boolean): boolean { + if (process.env.HF_DE_PARALLEL_ROUTER !== undefined) return false; + const config = readConfig(); + if (config.deParallelRouterTrialFired || !config.telemetryEnabled) return false; + process.env.HF_DE_PARALLEL_ROUTER = "true"; + if (!quiet) { + console.log( + c.dim( + " Trying the experimental parallel drawElement capture path once for this install " + + "(opt out: HF_DE_PARALLEL_ROUTER=false)", + ), + ); + } + return true; +} + +/** + * After a trial-armed render, persist that the experiment actually engaged + * (routed or reverted — either produces telemetry) so it's never enabled + * again for this install. Checks both the success path (`perfSummary`) and + * the failure path (`errorDetails.observability.capture`, mutated in place + * before a hard failure throws) — a crash while routed still counts as a + * fired trial. No-ops if the router never actually became eligible for this + * render (e.g. too few frames): the trial stays available for a future run. + */ +function maybeConsumeDeParallelRouterTrial(trialArmed: boolean, job: RenderJob): void { + if (!trialArmed) return; + const engaged = + job.perfSummary?.drawElement?.parallelRouter ?? + job.errorDetails?.observability?.capture.deParallelRouter; + if (engaged === undefined) return; + const config = readConfig(); + config.deParallelRouterTrialFired = true; + writeConfig(config); +} + function handleRenderError( error: unknown, options: RenderOptions, diff --git a/packages/cli/src/telemetry/config.ts b/packages/cli/src/telemetry/config.ts index 0a6d3461c..4dcab4dae 100644 --- a/packages/cli/src/telemetry/config.ts +++ b/packages/cli/src/telemetry/config.ts @@ -63,6 +63,16 @@ export interface HyperframesConfig { skillsOutdatedCount?: number; /** How many skills were missing (not installed) at the last check. */ skillsMissingCount?: number; + /** + * True once the DE parallel-router experiment ("HF_DE_PARALLEL_ROUTER") + * has actually engaged (routed or reverted — either produces telemetry) + * on a render from this install. The CLI enables the experiment for free + * on renders from a fresh install until this fires once, then never + * touches it again — a one-shot trial to get real-traffic router + * telemetry without requiring anyone to manually opt in via env var. + * See `renderLocal`'s `maybeEnableDeParallelRouterTrial`. + */ + deParallelRouterTrialFired?: boolean; } const DEFAULT_CONFIG: HyperframesConfig = { @@ -108,6 +118,7 @@ export function readConfig(): HyperframesConfig { skillsUpdateAvailable: parsed.skillsUpdateAvailable, skillsOutdatedCount: parsed.skillsOutdatedCount, skillsMissingCount: parsed.skillsMissingCount, + deParallelRouterTrialFired: parsed.deParallelRouterTrialFired, }; cachedConfig = config;