mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-11-05 05:37:09 +00:00
Merge pull request #4110 from heygen-com/lfr/03-phase-minus1-chrome-memory
feat(engine,producer,cli): Chrome memory telemetry per capture session (Phase -1)
This commit is contained in:
@@ -1751,6 +1751,11 @@ function trackRenderMetrics(
|
||||
catalogUsage: options.catalogUsage,
|
||||
...renderOutputShapeTelemetryPayload(options),
|
||||
...renderEnvironmentTelemetryPayload(options),
|
||||
chromeBrowserRssPeakMb: perf?.chromeMemory?.browserRssPeakMb,
|
||||
chromeRendererRssPeakMb: perf?.chromeMemory?.rendererRssPeakMb,
|
||||
chromeRssLastMb: perf?.chromeMemory?.rssLastMb,
|
||||
chromeGpuProcessSeenLastSample: perf?.chromeMemory?.gpuProcessSeenLastSample,
|
||||
chromeMemorySamples: perf?.chromeMemory?.samples,
|
||||
staticDedupEnabled: perf?.staticDedup?.enabled,
|
||||
staticDedupArmed: perf?.staticDedup?.armed,
|
||||
staticDedupSkipReason: perf?.staticDedup?.skipReason,
|
||||
|
||||
@@ -680,6 +680,89 @@ describe("render telemetry events", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("maps Chrome memory and capture path observability onto render_error", () => {
|
||||
trackRenderError({
|
||||
fps: 30,
|
||||
quality: "draft",
|
||||
docker: false,
|
||||
captureChromeBrowserRssPeakMb: 210,
|
||||
captureChromeRendererRssPeakMb: 1900,
|
||||
captureChromeRssLastMb: 2400,
|
||||
captureChromeGpuProcessSeenLastSample: true,
|
||||
captureChromeMemorySamples: 42,
|
||||
captureCapturePath: "streaming",
|
||||
captureSegmentIndex: 3,
|
||||
captureSegmentRetries: 1,
|
||||
});
|
||||
|
||||
expect(trackEvent).toHaveBeenCalledWith(
|
||||
"render_error",
|
||||
expect.objectContaining({
|
||||
chrome_browser_rss_peak_mb: 210,
|
||||
chrome_renderer_rss_peak_mb: 1900,
|
||||
chrome_rss_last_mb: 2400,
|
||||
gpu_process_seen_last_sample: true,
|
||||
chrome_memory_samples: 42,
|
||||
capture_path: "streaming",
|
||||
segment_index: 3,
|
||||
segment_retries: 1,
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("prefers the aggregate Chrome memory over the live sample on render_complete", () => {
|
||||
// The live observability values are the last session's; the perf summary
|
||||
// aggregates every worker. On success both are present and the aggregate
|
||||
// must win, or a multi-worker render reports one worker's peak as the
|
||||
// fleet's.
|
||||
trackRenderComplete({
|
||||
durationMs: 1000,
|
||||
fps: 30,
|
||||
quality: "draft",
|
||||
docker: false,
|
||||
gpu: false,
|
||||
captureChromeBrowserRssPeakMb: 100,
|
||||
captureChromeRendererRssPeakMb: 800,
|
||||
captureChromeRssLastMb: 900,
|
||||
captureChromeMemorySamples: 5,
|
||||
chromeBrowserRssPeakMb: 210,
|
||||
chromeRendererRssPeakMb: 1900,
|
||||
chromeRssLastMb: 2400,
|
||||
chromeGpuProcessSeenLastSample: true,
|
||||
chromeMemorySamples: 42,
|
||||
});
|
||||
|
||||
const props = trackEvent.mock.calls[0]?.[1] as Record<string, unknown>;
|
||||
expect(props).toMatchObject({
|
||||
chrome_browser_rss_peak_mb: 210,
|
||||
chrome_renderer_rss_peak_mb: 1900,
|
||||
chrome_rss_last_mb: 2400,
|
||||
gpu_process_seen_last_sample: true,
|
||||
chrome_memory_samples: 42,
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to the live Chrome memory sample when no aggregate exists", () => {
|
||||
trackRenderComplete({
|
||||
durationMs: 1000,
|
||||
fps: 30,
|
||||
quality: "draft",
|
||||
docker: false,
|
||||
gpu: false,
|
||||
captureChromeBrowserRssPeakMb: 100,
|
||||
captureChromeMemorySamples: 5,
|
||||
captureCapturePath: "disk",
|
||||
});
|
||||
|
||||
const props = trackEvent.mock.calls[0]?.[1] as Record<string, unknown>;
|
||||
expect(props).toMatchObject({
|
||||
chrome_browser_rss_peak_mb: 100,
|
||||
chrome_memory_samples: 5,
|
||||
capture_path: "disk",
|
||||
});
|
||||
});
|
||||
|
||||
it("carries the DE parallel-router/inversion cohort on render_error (hard failure, not just self-verify revert)", () => {
|
||||
trackRenderError({
|
||||
fps: 30,
|
||||
|
||||
@@ -99,6 +99,15 @@ export interface RenderObservabilityTelemetryPayload {
|
||||
/** Non-DE parallel-streaming router outcome ("screenshot" | "beginframe" —
|
||||
* routed; "eligible_off" — would route but the kill switch is off). */
|
||||
captureParallelStream?: string;
|
||||
/** Chrome memory from the engine sampler (Phase −1, long-form render plan). */
|
||||
captureChromeBrowserRssPeakMb?: number;
|
||||
captureChromeRendererRssPeakMb?: number;
|
||||
captureChromeRssLastMb?: number;
|
||||
captureChromeGpuProcessSeenLastSample?: boolean;
|
||||
captureChromeMemorySamples?: number;
|
||||
captureCapturePath?: string;
|
||||
captureSegmentIndex?: number;
|
||||
captureSegmentRetries?: number;
|
||||
observabilityExtractVideoCount?: number;
|
||||
observabilityExtractedVideoCount?: number;
|
||||
observabilityExtractTotalFrames?: number;
|
||||
@@ -171,6 +180,14 @@ function renderObservabilityEventProperties(props: RenderObservabilityTelemetryP
|
||||
de_fallback_frame_index: props.captureDeFallbackFrameIndex,
|
||||
de_fallback_threshold_db: props.captureDeFallbackThresholdDb,
|
||||
capture_parallel_stream: props.captureParallelStream,
|
||||
chrome_browser_rss_peak_mb: props.captureChromeBrowserRssPeakMb,
|
||||
chrome_renderer_rss_peak_mb: props.captureChromeRendererRssPeakMb,
|
||||
chrome_rss_last_mb: props.captureChromeRssLastMb,
|
||||
gpu_process_seen_last_sample: props.captureChromeGpuProcessSeenLastSample,
|
||||
chrome_memory_samples: props.captureChromeMemorySamples,
|
||||
capture_path: props.captureCapturePath,
|
||||
segment_index: props.captureSegmentIndex,
|
||||
segment_retries: props.captureSegmentRetries,
|
||||
observability_extract_video_count: props.observabilityExtractVideoCount,
|
||||
observability_extracted_video_count: props.observabilityExtractedVideoCount,
|
||||
observability_extract_total_frames: props.observabilityExtractTotalFrames,
|
||||
@@ -401,6 +418,14 @@ export function trackRenderComplete(
|
||||
capturePeakMs?: number;
|
||||
// Resource usage
|
||||
peakMemoryMb?: number;
|
||||
// Aggregate Chrome memory (RenderPerfSummary.chromeMemory); overrides the
|
||||
// live observability values when both are present, because the live ones
|
||||
// are only the last session's and the aggregate covers every worker.
|
||||
chromeBrowserRssPeakMb?: number;
|
||||
chromeRendererRssPeakMb?: number;
|
||||
chromeRssLastMb?: number;
|
||||
chromeGpuProcessSeenLastSample?: boolean;
|
||||
chromeMemorySamples?: number;
|
||||
memoryFreeMb?: number;
|
||||
tmpPeakBytes?: number;
|
||||
// Per-stage timings (subset of RenderPerfSummary.stages)
|
||||
@@ -438,6 +463,14 @@ export function trackRenderComplete(
|
||||
// studioRenderTelemetry.ts never populates drawElement, so without the
|
||||
// fallback the explicit key still wins the spread with an undefined.
|
||||
...renderObservabilityEventProperties(props),
|
||||
chrome_browser_rss_peak_mb:
|
||||
props.chromeBrowserRssPeakMb ?? props.captureChromeBrowserRssPeakMb,
|
||||
chrome_renderer_rss_peak_mb:
|
||||
props.chromeRendererRssPeakMb ?? props.captureChromeRendererRssPeakMb,
|
||||
chrome_rss_last_mb: props.chromeRssLastMb ?? props.captureChromeRssLastMb,
|
||||
gpu_process_seen_last_sample:
|
||||
props.chromeGpuProcessSeenLastSample ?? props.captureChromeGpuProcessSeenLastSample,
|
||||
chrome_memory_samples: props.chromeMemorySamples ?? props.captureChromeMemorySamples,
|
||||
duration_ms: props.durationMs,
|
||||
fps: props.fps,
|
||||
quality: props.quality,
|
||||
|
||||
@@ -66,6 +66,14 @@ export function renderObservabilityTelemetryPayload(
|
||||
captureDeFallbackFrameIndex: capture.deFallbackFrameIndex,
|
||||
captureDeFallbackThresholdDb: capture.deFallbackThresholdDb,
|
||||
captureParallelStream: capture.captureParallelStream,
|
||||
captureChromeBrowserRssPeakMb: capture.chromeBrowserRssPeakMb,
|
||||
captureChromeRendererRssPeakMb: capture.chromeRendererRssPeakMb,
|
||||
captureChromeRssLastMb: capture.chromeRssLastMb,
|
||||
captureChromeGpuProcessSeenLastSample: capture.chromeGpuProcessSeenLastSample,
|
||||
captureChromeMemorySamples: capture.chromeMemorySamples,
|
||||
captureCapturePath: capture.capturePath,
|
||||
captureSegmentIndex: capture.segmentIndex,
|
||||
captureSegmentRetries: capture.segmentRetries,
|
||||
observabilityExtractVideoCount: extraction?.videoCount,
|
||||
observabilityExtractedVideoCount: extraction?.extractedVideoCount,
|
||||
observabilityExtractTotalFrames: extraction?.totalFramesExtracted,
|
||||
|
||||
@@ -139,6 +139,10 @@ export {
|
||||
type CaptureFailureKind,
|
||||
type CaptureWorkerDiagnostic,
|
||||
} from "./services/captureFailure.js";
|
||||
export {
|
||||
createChromeMemorySampler,
|
||||
type ChromeMemoryStats,
|
||||
} from "./services/chromeMemorySampler.js";
|
||||
|
||||
// ── Screenshot (BeginFrame) ─────────────────────────────────────────────────────
|
||||
export {
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
createChromeMemorySampler,
|
||||
mergeSample,
|
||||
type ChromeMemoryStats,
|
||||
type ChromePids,
|
||||
} from "./chromeMemorySampler.js";
|
||||
|
||||
const empty: ChromeMemoryStats = { samples: 0 };
|
||||
|
||||
describe("mergeSample", () => {
|
||||
it("tracks browser and renderer peaks separately and sums the last sample", () => {
|
||||
const pids: ChromePids = { browser: 1, renderers: [2, 3], gpu: [4] };
|
||||
const first = mergeSample(empty, pids, [
|
||||
{ pid: 1, rssMb: 100 },
|
||||
{ pid: 2, rssMb: 300 },
|
||||
{ pid: 3, rssMb: 200 },
|
||||
{ pid: 4, rssMb: 50 },
|
||||
]);
|
||||
expect(first).toEqual({
|
||||
browserRssPeakMb: 100,
|
||||
rendererRssPeakMb: 300,
|
||||
rssLastMb: 650,
|
||||
gpuProcessSeenLastSample: true,
|
||||
samples: 1,
|
||||
});
|
||||
const second = mergeSample(first, { browser: 1, renderers: [2], gpu: [] }, [
|
||||
{ pid: 1, rssMb: 90 },
|
||||
{ pid: 2, rssMb: 250 },
|
||||
]);
|
||||
expect(second).toEqual({
|
||||
browserRssPeakMb: 100,
|
||||
rendererRssPeakMb: 300,
|
||||
rssLastMb: 340,
|
||||
gpuProcessSeenLastSample: false,
|
||||
samples: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not count a sample that returned no rows", () => {
|
||||
expect(mergeSample(empty, { renderers: [], gpu: [] }, [])).toEqual(empty);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createChromeMemorySampler", () => {
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
it("samples on the interval, reports via onSample, and stops cleanly", async () => {
|
||||
vi.useFakeTimers();
|
||||
const seen: ChromeMemoryStats[] = [];
|
||||
const sampler = createChromeMemorySampler({
|
||||
getPids: async () => ({ browser: 1, renderers: [2], gpu: [] }),
|
||||
sampleRss: async () => [
|
||||
{ pid: 1, rssMb: 10 },
|
||||
{ pid: 2, rssMb: 20 },
|
||||
],
|
||||
intervalMs: 1000,
|
||||
onSample: (s) => seen.push(s),
|
||||
});
|
||||
sampler.start();
|
||||
await vi.advanceTimersByTimeAsync(2500);
|
||||
sampler.stop();
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(seen.length).toBe(2);
|
||||
expect(sampler.stats()).toEqual({
|
||||
browserRssPeakMb: 10,
|
||||
rendererRssPeakMb: 20,
|
||||
rssLastMb: 30,
|
||||
gpuProcessSeenLastSample: false,
|
||||
samples: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("never lets a dependency error escape and keeps sampling", async () => {
|
||||
vi.useFakeTimers();
|
||||
let calls = 0;
|
||||
const sampler = createChromeMemorySampler({
|
||||
getPids: async () => {
|
||||
calls += 1;
|
||||
if (calls === 1) throw new Error("cdp gone");
|
||||
return { browser: 1, renderers: [], gpu: [] };
|
||||
},
|
||||
sampleRss: async () => [{ pid: 1, rssMb: 5 }],
|
||||
intervalMs: 100,
|
||||
});
|
||||
sampler.start();
|
||||
await vi.advanceTimersByTimeAsync(250);
|
||||
sampler.stop();
|
||||
expect(sampler.stats().samples).toBe(1);
|
||||
});
|
||||
|
||||
it("resolves sampleOnce() even when a dependency throws", async () => {
|
||||
// Direct pin on the catch: the interval test above still ends at one
|
||||
// sample if the catch is removed (the next tick succeeds), so only this
|
||||
// assertion fails when the guard is gone.
|
||||
const sampler = createChromeMemorySampler({
|
||||
getPids: async () => {
|
||||
throw new Error("cdp gone");
|
||||
},
|
||||
sampleRss: async () => [],
|
||||
intervalMs: 100,
|
||||
});
|
||||
await expect(sampler.sampleOnce()).resolves.toBeUndefined();
|
||||
expect(sampler.stats()).toEqual(empty);
|
||||
});
|
||||
|
||||
it("skips a tick while the previous sample is still in flight", async () => {
|
||||
vi.useFakeTimers();
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
const sampler = createChromeMemorySampler({
|
||||
getPids: async () => {
|
||||
inFlight += 1;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
await new Promise((r) => setTimeout(r, 350));
|
||||
inFlight -= 1;
|
||||
return { browser: 1, renderers: [], gpu: [] };
|
||||
},
|
||||
sampleRss: async () => [{ pid: 1, rssMb: 1 }],
|
||||
intervalMs: 100,
|
||||
});
|
||||
sampler.start();
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
sampler.stop();
|
||||
expect(maxInFlight).toBe(1);
|
||||
});
|
||||
|
||||
it("stop(); await sampleOnce() joins an in-flight tick: stats current at close, no late onSample", async () => {
|
||||
// Mirrors the closeCaptureSession sequence. The tick at 100 ms is still
|
||||
// waiting on getPids (350 ms) when close runs; the close-time sample must
|
||||
// resolve after that tick has merged, not before.
|
||||
vi.useFakeTimers();
|
||||
const onSample = vi.fn();
|
||||
const sampler = createChromeMemorySampler({
|
||||
getPids: async () => {
|
||||
await new Promise((r) => setTimeout(r, 350));
|
||||
return { browser: 1, renderers: [], gpu: [] };
|
||||
},
|
||||
sampleRss: async () => [{ pid: 1, rssMb: 1 }],
|
||||
intervalMs: 100,
|
||||
onSample,
|
||||
});
|
||||
sampler.start();
|
||||
await vi.advanceTimersByTimeAsync(150);
|
||||
sampler.stop();
|
||||
let samplesAtClose = -1;
|
||||
let onSampleCallsAtClose = -1;
|
||||
const closing = sampler.sampleOnce().then(() => {
|
||||
samplesAtClose = sampler.stats().samples;
|
||||
onSampleCallsAtClose = onSample.mock.calls.length;
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
await closing;
|
||||
expect(samplesAtClose).toBe(1);
|
||||
expect(onSampleCallsAtClose).toBe(1);
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(onSample).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not invoke onSample when a sample returned no rows", async () => {
|
||||
const onSample = vi.fn();
|
||||
const sampler = createChromeMemorySampler({
|
||||
getPids: async () => ({ browser: 1, renderers: [], gpu: [] }),
|
||||
sampleRss: async () => [],
|
||||
intervalMs: 100,
|
||||
onSample,
|
||||
});
|
||||
await sampler.sampleOnce();
|
||||
expect(onSample).not.toHaveBeenCalled();
|
||||
expect(sampler.stats()).toEqual(empty);
|
||||
});
|
||||
|
||||
it("unrefs the interval so a forgotten stop() cannot hold the process open", () => {
|
||||
// Real timers: a real NodeJS.Timeout reports hasRef(); the interval is far
|
||||
// enough out that no tick fires during the test.
|
||||
let handle: NodeJS.Timeout | undefined;
|
||||
const sampler = createChromeMemorySampler({
|
||||
getPids: async () => ({ browser: 1, renderers: [], gpu: [] }),
|
||||
sampleRss: async () => [{ pid: 1, rssMb: 1 }],
|
||||
intervalMs: 60_000,
|
||||
setIntervalFn: (callback, ms) => {
|
||||
handle = setInterval(callback, ms);
|
||||
return handle;
|
||||
},
|
||||
});
|
||||
sampler.start();
|
||||
if (handle === undefined) throw new Error("start() did not schedule an interval");
|
||||
const refed = handle.hasRef();
|
||||
sampler.stop();
|
||||
expect(refed).toBe(false);
|
||||
});
|
||||
|
||||
it("start() twice schedules a single interval; stop() is safe before start and twice", async () => {
|
||||
vi.useFakeTimers();
|
||||
const sampler = createChromeMemorySampler({
|
||||
getPids: async () => ({ browser: 1, renderers: [], gpu: [] }),
|
||||
sampleRss: async () => [{ pid: 1, rssMb: 1 }],
|
||||
intervalMs: 100,
|
||||
});
|
||||
sampler.stop();
|
||||
sampler.start();
|
||||
sampler.start();
|
||||
await vi.advanceTimersByTimeAsync(250);
|
||||
sampler.stop();
|
||||
sampler.stop();
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
expect(sampler.stats().samples).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,144 @@
|
||||
import type { ProcessRssSample } from "../utils/processRss.js";
|
||||
|
||||
/**
|
||||
* Chrome process memory observed during one capture session. Peaks are
|
||||
* per-process-type maxima across samples; `rssLastMb` is the whole tree at
|
||||
* the most recent successful sample. Surfaced on render_error /
|
||||
* render_complete so the long-render "Target closed" class (spec §2.5) can be
|
||||
* correlated with memory instead of guessed at.
|
||||
*/
|
||||
export interface ChromeMemoryStats {
|
||||
browserRssPeakMb?: number;
|
||||
rendererRssPeakMb?: number;
|
||||
rssLastMb?: number;
|
||||
gpuProcessSeenLastSample?: boolean;
|
||||
samples: number;
|
||||
}
|
||||
|
||||
export interface ChromePids {
|
||||
browser?: number;
|
||||
renderers: number[];
|
||||
gpu: number[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Timer seams, typed to the two-argument shape the sampler uses rather than
|
||||
* `typeof setInterval`: with the DOM lib in scope that type also carries the
|
||||
* `(handler, timeout) => number` overload, so a test could only satisfy it
|
||||
* with a cast. The globals are assignable to these narrower shapes.
|
||||
*/
|
||||
type SetIntervalFn = (callback: () => void, ms: number) => NodeJS.Timeout;
|
||||
type ClearIntervalFn = (timer: NodeJS.Timeout) => void;
|
||||
|
||||
export interface ChromeMemorySamplerDeps {
|
||||
getPids: () => Promise<ChromePids>;
|
||||
sampleRss: (pids: readonly number[]) => Promise<ProcessRssSample[]>;
|
||||
intervalMs: number;
|
||||
onSample?: (stats: ChromeMemoryStats) => void;
|
||||
/** Test seam; defaults to the global setInterval. */
|
||||
setIntervalFn?: SetIntervalFn;
|
||||
/** Test seam; defaults to the global clearInterval. */
|
||||
clearIntervalFn?: ClearIntervalFn;
|
||||
}
|
||||
|
||||
export interface ChromeMemorySampler {
|
||||
start(): void;
|
||||
stop(): void;
|
||||
stats(): ChromeMemoryStats;
|
||||
/**
|
||||
* One sample, awaited by the caller; used by tests and by close(). If a
|
||||
* sample is already in flight this resolves when that sample finishes
|
||||
* instead of starting another, so `stop(); await sampleOnce()` always sees
|
||||
* the running tick's result and `onSample` cannot fire after it resolves.
|
||||
*/
|
||||
sampleOnce(): Promise<void>;
|
||||
}
|
||||
|
||||
function maxDefined(a: number | undefined, b: number | undefined): number | undefined {
|
||||
if (a === undefined) return b;
|
||||
if (b === undefined) return a;
|
||||
return Math.max(a, b);
|
||||
}
|
||||
|
||||
/** Pure merge of one sample into the running stats. Empty `rss` → unchanged. */
|
||||
export function mergeSample(
|
||||
prev: ChromeMemoryStats,
|
||||
pids: ChromePids,
|
||||
rss: readonly ProcessRssSample[],
|
||||
): ChromeMemoryStats {
|
||||
if (rss.length === 0) return prev;
|
||||
const byPid = new Map(rss.map((s) => [s.pid, s.rssMb]));
|
||||
const browserMb = pids.browser === undefined ? undefined : byPid.get(pids.browser);
|
||||
const rendererMbs = pids.renderers
|
||||
.map((p) => byPid.get(p))
|
||||
.filter((v): v is number => v !== undefined);
|
||||
const gpuMbs = pids.gpu.map((p) => byPid.get(p)).filter((v): v is number => v !== undefined);
|
||||
const rendererMax = rendererMbs.length > 0 ? Math.max(...rendererMbs) : undefined;
|
||||
const total = rss.reduce((sum, s) => sum + s.rssMb, 0);
|
||||
return {
|
||||
browserRssPeakMb: maxDefined(prev.browserRssPeakMb, browserMb),
|
||||
rendererRssPeakMb: maxDefined(prev.rendererRssPeakMb, rendererMax),
|
||||
rssLastMb: total,
|
||||
gpuProcessSeenLastSample: gpuMbs.length > 0,
|
||||
samples: prev.samples + 1,
|
||||
};
|
||||
}
|
||||
|
||||
export function createChromeMemorySampler(deps: ChromeMemorySamplerDeps): ChromeMemorySampler {
|
||||
const setIntervalFn: SetIntervalFn = deps.setIntervalFn ?? setInterval;
|
||||
const clearIntervalFn: ClearIntervalFn = deps.clearIntervalFn ?? clearInterval;
|
||||
let stats: ChromeMemoryStats = { samples: 0 };
|
||||
let timer: NodeJS.Timeout | null = null;
|
||||
let inFlight: Promise<void> | null = null;
|
||||
|
||||
const runSample = async (): Promise<void> => {
|
||||
try {
|
||||
const pids = await deps.getPids();
|
||||
const all = [pids.browser, ...pids.renderers, ...pids.gpu].filter(
|
||||
(p): p is number => typeof p === "number",
|
||||
);
|
||||
const rss = await deps.sampleRss(all);
|
||||
const next = mergeSample(stats, pids, rss);
|
||||
if (next !== stats) {
|
||||
stats = next;
|
||||
deps.onSample?.(stats);
|
||||
}
|
||||
} catch {
|
||||
// Sampling is observability only; a CDP or ps failure must never reach
|
||||
// the capture loop. The next tick retries.
|
||||
}
|
||||
};
|
||||
|
||||
const sampleOnce = (): Promise<void> => {
|
||||
// Reentrancy guard: an interval tick that lands while a sample is still
|
||||
// running joins it rather than stacking a second CDP/ps round-trip, and a
|
||||
// close-time call waits for the running tick instead of skipping it.
|
||||
// runSample never rejects, so the .finally reset always runs; it runs on
|
||||
// a microtask, after the assignment below.
|
||||
if (inFlight !== null) return inFlight;
|
||||
inFlight = runSample().finally(() => {
|
||||
inFlight = null;
|
||||
});
|
||||
return inFlight;
|
||||
};
|
||||
|
||||
return {
|
||||
start() {
|
||||
if (timer !== null) return;
|
||||
// Node's Timeout has unref(); the engine already relies on it
|
||||
// (utils/processTracker.ts). Keeps the interval from holding the
|
||||
// process open if a caller forgets stop().
|
||||
timer = setIntervalFn(() => {
|
||||
void sampleOnce();
|
||||
}, deps.intervalMs);
|
||||
timer.unref();
|
||||
},
|
||||
stop() {
|
||||
if (timer === null) return;
|
||||
clearIntervalFn(timer);
|
||||
timer = null;
|
||||
},
|
||||
stats: () => stats,
|
||||
sampleOnce,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { classifyChromeProcesses, readChromePids } from "./frameCapture.js";
|
||||
|
||||
describe("classifyChromeProcesses", () => {
|
||||
it("splits CDP SystemInfo.getProcessInfo rows by type and keeps the browser pid", () => {
|
||||
const pids = classifyChromeProcesses(100, [
|
||||
{ type: "browser", id: 100, cpuTime: 0 },
|
||||
{ type: "renderer", id: 101, cpuTime: 0 },
|
||||
{ type: "renderer", id: 102, cpuTime: 0 },
|
||||
{ type: "GPU", id: 103, cpuTime: 0 },
|
||||
{ type: "utility", id: 104, cpuTime: 0 },
|
||||
]);
|
||||
expect(pids).toEqual({ browser: 100, renderers: [101, 102], gpu: [103] });
|
||||
});
|
||||
|
||||
it("works without a browser pid", () => {
|
||||
expect(classifyChromeProcesses(undefined, [{ type: "renderer", id: 7, cpuTime: 0 }])).toEqual({
|
||||
renderers: [7],
|
||||
gpu: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the browser pid even when CDP reports no children", () => {
|
||||
expect(classifyChromeProcesses(100, [])).toEqual({ browser: 100, renderers: [], gpu: [] });
|
||||
});
|
||||
|
||||
it("ignores process types that are neither renderer nor GPU", () => {
|
||||
// Guard mutation: dropping the `type` checks would fold utility/browser
|
||||
// rows into `renderers` and inflate the renderer peak.
|
||||
expect(
|
||||
classifyChromeProcesses(1, [
|
||||
{ type: "utility", id: 2, cpuTime: 0 },
|
||||
{ type: "browser", id: 1, cpuTime: 0 },
|
||||
{ type: "gpu", id: 3, cpuTime: 0 },
|
||||
]),
|
||||
).toEqual({ browser: 1, renderers: [], gpu: [] });
|
||||
});
|
||||
});
|
||||
|
||||
describe("readChromePids", () => {
|
||||
// Chrome answers SystemInfo.getProcessInfo ONLY on the browser target; a
|
||||
// page-target session rejects with "is only supported on the browser
|
||||
// target". A page session here yields zero samples for the whole render and
|
||||
// the telemetry is silently always-null, so the fake refuses it the way
|
||||
// Chrome does.
|
||||
function fakeCdp(target: "page" | "browser") {
|
||||
let detached = false;
|
||||
return {
|
||||
session: {
|
||||
send: async (method: "SystemInfo.getProcessInfo") => {
|
||||
if (target === "page") {
|
||||
throw new Error(
|
||||
`Protocol error (${method}): ${method} is only supported on the browser target`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
processInfo: [
|
||||
{ type: "browser", id: 10, cpuTime: 0 },
|
||||
{ type: "renderer", id: 11, cpuTime: 0 },
|
||||
{ type: "GPU", id: 12, cpuTime: 0 },
|
||||
],
|
||||
};
|
||||
},
|
||||
detach: async () => {
|
||||
detached = true;
|
||||
},
|
||||
},
|
||||
wasDetached: () => detached,
|
||||
};
|
||||
}
|
||||
|
||||
it("reads process info from the browser target", async () => {
|
||||
const cdp = fakeCdp("browser");
|
||||
await expect(readChromePids(10, async () => cdp.session)).resolves.toEqual({
|
||||
browser: 10,
|
||||
renderers: [11],
|
||||
gpu: [12],
|
||||
});
|
||||
expect(cdp.wasDetached()).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects when handed a page-target session, instead of silently sampling nothing", async () => {
|
||||
const cdp = fakeCdp("page");
|
||||
await expect(readChromePids(10, async () => cdp.session)).rejects.toThrow(
|
||||
"only supported on the browser target",
|
||||
);
|
||||
// Still detached: a failed probe must not leak a CDP session per tick.
|
||||
expect(cdp.wasDetached()).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -84,6 +84,12 @@ import type {
|
||||
} from "../types.js";
|
||||
import { cloneCaptureWarnings } from "./captureWarning.js";
|
||||
import { installMediaRenderIdBridge } from "./mediaRenderIdBridge.js";
|
||||
import {
|
||||
createChromeMemorySampler,
|
||||
type ChromeMemorySampler,
|
||||
type ChromePids,
|
||||
} from "./chromeMemorySampler.js";
|
||||
import { sampleProcessRss } from "../utils/processRss.js";
|
||||
export { isMemoryExhaustionError, isTransientBrowserError } from "./captureFailure.js";
|
||||
|
||||
export type { CaptureOptions, CaptureResult, CaptureBufferResult, CapturePerfSummary };
|
||||
@@ -116,6 +122,8 @@ export interface CaptureSession {
|
||||
lastFrameAbsoluteIndex?: number;
|
||||
/** Count of frames served from a reused buffer (dedup telemetry). */
|
||||
staticDedupCount?: number;
|
||||
/** Live Chrome memory sampler; started in initializeSession, stopped in closeCaptureSession. */
|
||||
chromeMemory?: ChromeMemorySampler;
|
||||
/**
|
||||
* Resolved sub-frame motion-blur plan, or undefined when off. Set once by
|
||||
* `resolveSessionMotionBlur` at the end of initialization, where the capture mode has
|
||||
@@ -3858,6 +3866,79 @@ async function captureFrameSurface(
|
||||
function finalizeSessionInit(session: CaptureSession): void {
|
||||
session.motionBlur = resolveSessionMotionBlur(session);
|
||||
session.isInitialized = true;
|
||||
startChromeMemorySampler(session);
|
||||
}
|
||||
|
||||
/** Shape of one `SystemInfo.getProcessInfo` row; only the fields we read. */
|
||||
export interface CdpProcessInfoRow {
|
||||
type: string;
|
||||
id: number;
|
||||
cpuTime: number;
|
||||
}
|
||||
|
||||
/** Group Chrome child pids by role. `browser` comes from puppeteer, not CDP. */
|
||||
export function classifyChromeProcesses(
|
||||
browserPid: number | undefined,
|
||||
processInfo: readonly CdpProcessInfoRow[],
|
||||
): ChromePids {
|
||||
const renderers: number[] = [];
|
||||
const gpu: number[] = [];
|
||||
for (const row of processInfo) {
|
||||
if (row.type === "renderer") renderers.push(row.id);
|
||||
else if (row.type === "GPU") gpu.push(row.id);
|
||||
}
|
||||
return browserPid === undefined ? { renderers, gpu } : { browser: browserPid, renderers, gpu };
|
||||
}
|
||||
|
||||
/** The slice of a CDP session `readChromePids` needs. */
|
||||
interface ProcessInfoCdpSession {
|
||||
send(method: "SystemInfo.getProcessInfo"): Promise<{
|
||||
processInfo: readonly CdpProcessInfoRow[];
|
||||
}>;
|
||||
detach(): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chrome pids for one browser. `SystemInfo.getProcessInfo` is served ONLY by
|
||||
* the browser target — a page-target session rejects with "is only supported
|
||||
* on the browser target", which the sampler would swallow into a permanently
|
||||
* empty reading. Hence the explicit browser-session factory.
|
||||
*/
|
||||
export async function readChromePids(
|
||||
browserPid: number | undefined,
|
||||
createBrowserCdpSession: () => Promise<ProcessInfoCdpSession>,
|
||||
): Promise<ChromePids> {
|
||||
const cdp = await createBrowserCdpSession();
|
||||
try {
|
||||
const info = await cdp.send("SystemInfo.getProcessInfo");
|
||||
return classifyChromeProcesses(browserPid, info.processInfo);
|
||||
} finally {
|
||||
await cdp.detach().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
const CHROME_MEMORY_SAMPLE_MS_DEFAULT = 2_000;
|
||||
|
||||
function resolveChromeMemorySampleMs(): number {
|
||||
const raw = process.env.HF_CHROME_MEMORY_SAMPLE_MS;
|
||||
const parsed = raw === undefined ? Number.NaN : Number(raw);
|
||||
return Number.isFinite(parsed) && parsed >= 250 ? parsed : CHROME_MEMORY_SAMPLE_MS_DEFAULT;
|
||||
}
|
||||
|
||||
function startChromeMemorySampler(session: CaptureSession): void {
|
||||
if (session.chromeMemory) return;
|
||||
if (process.env.HF_CHROME_MEMORY_SAMPLER === "false") return;
|
||||
const sampler = createChromeMemorySampler({
|
||||
intervalMs: resolveChromeMemorySampleMs(),
|
||||
onSample: session.options.onMemorySample,
|
||||
sampleRss: (pids) => sampleProcessRss(pids),
|
||||
getPids: () =>
|
||||
readChromePids(session.browser.process()?.pid, () =>
|
||||
session.browser.target().createCDPSession(),
|
||||
),
|
||||
});
|
||||
session.chromeMemory = sampler;
|
||||
sampler.start();
|
||||
}
|
||||
|
||||
/** Choose how many samples this frame gets when the plan is adaptive: the floor with
|
||||
@@ -4456,6 +4537,12 @@ export async function discardWarmupCapture(
|
||||
}
|
||||
|
||||
export async function closeCaptureSession(session: CaptureSession): Promise<void> {
|
||||
if (session.chromeMemory) {
|
||||
session.chromeMemory.stop();
|
||||
// One last sample so the summary reflects the session's end state, not a
|
||||
// point up to `intervalMs` earlier. Best effort; the page may be gone.
|
||||
await session.chromeMemory.sampleOnce();
|
||||
}
|
||||
// Realized static-dedup telemetry: how much the cache actually helped this
|
||||
// render (vs the prediction logged at arm time). Both capture paths
|
||||
// (sequential orchestrator + parallel workers) close their session here, so
|
||||
@@ -4753,6 +4840,11 @@ export function getCapturePerfSummary(session: CaptureSession): CapturePerfSumma
|
||||
beginFrameHasDamage: session.beginFrameHasDamageCount,
|
||||
captureMode: session.captureMode,
|
||||
gpuRenderer: session.gpuRenderer,
|
||||
chromeBrowserRssPeakMb: session.chromeMemory?.stats().browserRssPeakMb,
|
||||
chromeRendererRssPeakMb: session.chromeMemory?.stats().rendererRssPeakMb,
|
||||
chromeRssLastMb: session.chromeMemory?.stats().rssLastMb,
|
||||
chromeGpuProcessSeenLastSample: session.chromeMemory?.stats().gpuProcessSeenLastSample,
|
||||
chromeMemorySamples: session.chromeMemory?.stats().samples,
|
||||
deGateReason: session.deGateReason,
|
||||
deFallbackTrigger: session.deFallbackTrigger,
|
||||
deWorkerEncode: session.workerEncodeEnabled ?? false,
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
* as video must expose `window.__hf` implementing the HfProtocol interface.
|
||||
*/
|
||||
import type { Fps } from "@hyperframes/core";
|
||||
import type { ChromeMemoryStats } from "./services/chromeMemorySampler.js";
|
||||
import type { MotionBlurOptions } from "./services/motionBlur.js";
|
||||
|
||||
/**
|
||||
@@ -136,6 +137,13 @@ export interface CaptureOptions {
|
||||
compositionDurationSeconds?: number;
|
||||
/** The composition declares `data-requires-webgpu`; skips createCaptureSession's own fetch. */
|
||||
requiresWebGpu?: boolean;
|
||||
/**
|
||||
* Live Chrome memory samples during capture (browser/renderer RSS peaks,
|
||||
* last total, GPU process presence). Invoked from an unref'd interval; must
|
||||
* not throw. The producer forwards these to capture observability so a
|
||||
* crash mid-render still reports the last known memory state.
|
||||
*/
|
||||
onMemorySample?: (stats: ChromeMemoryStats) => void;
|
||||
/**
|
||||
* Frame rate as an exact rational. Integer fps is `{ num: 30, den: 1 }`;
|
||||
* NTSC is `{ num: 30000, den: 1001 }`. Captures are scheduled by the
|
||||
@@ -353,6 +361,14 @@ export interface CapturePerfSummary {
|
||||
* see `classifyGpuRenderer`.
|
||||
*/
|
||||
gpuRenderer?: string;
|
||||
// ── Chrome process memory (spec: long-form render capture, Phase −1).
|
||||
// Undefined when the sampler was disabled (HF_CHROME_MEMORY_SAMPLER=false)
|
||||
// or never produced a successful sample. ──
|
||||
chromeBrowserRssPeakMb?: number;
|
||||
chromeRendererRssPeakMb?: number;
|
||||
chromeRssLastMb?: number;
|
||||
chromeGpuProcessSeenLastSample?: boolean;
|
||||
chromeMemorySamples?: number;
|
||||
/**
|
||||
* Low-cardinality init-time gate that routed a drawElement-eligible session
|
||||
* to the baseline: `swiftshader` | `css_effect:<fx>` | `at_risk_timeline` |
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
parseProcStatusRss,
|
||||
parsePsRss,
|
||||
parseTasklistCsv,
|
||||
sampleProcessRss,
|
||||
} from "./processRss.js";
|
||||
|
||||
describe("parsePsRss", () => {
|
||||
it("parses pid and rss kilobytes into megabytes", () => {
|
||||
const out = " 4242 1048576\n 4243 2048\n";
|
||||
expect(parsePsRss(out)).toEqual([
|
||||
{ pid: 4242, rssMb: 1024 },
|
||||
{ pid: 4243, rssMb: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("skips blank and malformed lines", () => {
|
||||
expect(parsePsRss("\n PID RSS\nabc def\n 7 10240\n")).toEqual([{ pid: 7, rssMb: 10 }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseTasklistCsv", () => {
|
||||
it("parses the memory column with thousands separators", () => {
|
||||
const out = '"chrome.exe","4242","Console","1","1,048,576 K"\r\n';
|
||||
expect(parseTasklistCsv(4242, out)).toEqual([{ pid: 4242, rssMb: 1024 }]);
|
||||
});
|
||||
|
||||
it("returns [] for the 'no tasks' message", () => {
|
||||
expect(
|
||||
parseTasklistCsv(4242, "INFO: No tasks are running which match the specified criteria."),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns [] when the memory column is not a positive number", () => {
|
||||
expect(parseTasklistCsv(7, '"chrome.exe","7","Console","1","N/A"\r\n')).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseProcStatusRss", () => {
|
||||
it("parses the VmRSS line of /proc/<pid>/status", () => {
|
||||
const status =
|
||||
"Name:\tchrome\nState:\tS (sleeping)\nVmPeak:\t 9999999 kB\nVmRSS:\t 1048576 kB\nThreads:\t12\n";
|
||||
expect(parseProcStatusRss(4242, status)).toEqual([{ pid: 4242, rssMb: 1024 }]);
|
||||
});
|
||||
|
||||
it("returns [] when VmRSS is absent (zombie) or zero", () => {
|
||||
expect(parseProcStatusRss(1, "Name:\tchrome\nState:\tZ (zombie)\nThreads:\t1\n")).toEqual([]);
|
||||
expect(parseProcStatusRss(1, "VmRSS:\t 0 kB\n")).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sampleProcessRss", () => {
|
||||
it("reads /proc/<pid>/status per pid on linux and never shells out", async () => {
|
||||
let execCalled = false;
|
||||
const exec = async () => {
|
||||
execCalled = true;
|
||||
return { stdout: "" };
|
||||
};
|
||||
const reads: string[] = [];
|
||||
const readStatus = async (path: string) => {
|
||||
reads.push(path);
|
||||
return `Name:\tchrome\nVmRSS:\t ${path.endsWith("/1/status") ? 1024 : 2048} kB\n`;
|
||||
};
|
||||
const result = await sampleProcessRss([1, 2], exec, "linux", readStatus);
|
||||
expect(reads).toEqual(["/proc/1/status", "/proc/2/status"]);
|
||||
expect(execCalled).toBe(false);
|
||||
expect(result).toEqual([
|
||||
{ pid: 1, rssMb: 1 },
|
||||
{ pid: 2, rssMb: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("omits only the pid whose /proc read fails on linux", async () => {
|
||||
const exec = async () => ({ stdout: "" });
|
||||
const readStatus = async (path: string) => {
|
||||
if (path === "/proc/1/status") throw new Error("ENOENT");
|
||||
return "VmRSS:\t 2048 kB\n";
|
||||
};
|
||||
expect(await sampleProcessRss([1, 2], exec, "linux", readStatus)).toEqual([
|
||||
{ pid: 2, rssMb: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("never rejects on linux when every /proc read fails", async () => {
|
||||
const exec = async () => ({ stdout: "" });
|
||||
const readStatus = async () => {
|
||||
throw new Error("ENOENT");
|
||||
};
|
||||
expect(await sampleProcessRss([1, 2], exec, "linux", readStatus)).toEqual([]);
|
||||
});
|
||||
|
||||
it("uses one ps call for all pids on posix", async () => {
|
||||
const calls: Array<{ file: string; args: readonly string[] }> = [];
|
||||
const exec = async (file: string, args: readonly string[]) => {
|
||||
calls.push({ file, args });
|
||||
return { stdout: " 1 1024\n 2 2048\n" };
|
||||
};
|
||||
const result = await sampleProcessRss([1, 2], exec, "darwin");
|
||||
expect(calls).toEqual([{ file: "ps", args: ["-o", "pid=,rss=", "-p", "1,2"] }]);
|
||||
expect(result).toEqual([
|
||||
{ pid: 1, rssMb: 1 },
|
||||
{ pid: 2, rssMb: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses one tasklist call per pid on win32", async () => {
|
||||
const calls: string[][] = [];
|
||||
const exec = async (_file: string, args: readonly string[]) => {
|
||||
calls.push([...args]);
|
||||
return { stdout: '"chrome.exe","9","Console","1","2,048 K"\r\n' };
|
||||
};
|
||||
const result = await sampleProcessRss([9], exec, "win32");
|
||||
expect(calls).toEqual([["/FO", "CSV", "/NH", "/FI", "PID eq 9"]]);
|
||||
expect(result).toEqual([{ pid: 9, rssMb: 2 }]);
|
||||
});
|
||||
|
||||
it("drops invalid pids and duplicates before calling ps", async () => {
|
||||
const calls: Array<readonly string[]> = [];
|
||||
const exec = async (_file: string, args: readonly string[]) => {
|
||||
calls.push(args);
|
||||
return { stdout: " 3 1024\n 4 2048\n" };
|
||||
};
|
||||
// NaN / 0 / -1 / 2.5 must never reach `ps -p`: one bad pid fails the
|
||||
// whole call and every other pid's sample is lost with it.
|
||||
await sampleProcessRss([3, 3, NaN, 0, -1, 2.5, 4], exec, "darwin");
|
||||
expect(calls).toEqual([["-o", "pid=,rss=", "-p", "3,4"]]);
|
||||
});
|
||||
|
||||
it("returns [] on empty input without calling exec", async () => {
|
||||
let called = false;
|
||||
const exec = async () => {
|
||||
called = true;
|
||||
return { stdout: "" };
|
||||
};
|
||||
expect(await sampleProcessRss([], exec, "linux")).toEqual([]);
|
||||
expect(called).toBe(false);
|
||||
});
|
||||
|
||||
it("never rejects when exec fails", async () => {
|
||||
const exec = async () => {
|
||||
throw new Error("ENOENT");
|
||||
};
|
||||
// darwin, not linux: linux no longer reaches exec at all.
|
||||
expect(await sampleProcessRss([1], exec, "darwin")).toEqual([]);
|
||||
expect(await sampleProcessRss([1], exec, "win32")).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,104 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
export interface ProcessRssSample {
|
||||
pid: number;
|
||||
rssMb: number;
|
||||
}
|
||||
|
||||
export type ExecFileLike = (file: string, args: readonly string[]) => Promise<{ stdout: string }>;
|
||||
export type ReadFileLike = (path: string) => Promise<string>;
|
||||
|
||||
const KB_PER_MB = 1024;
|
||||
|
||||
const defaultReadFile: ReadFileLike = (path) => readFile(path, "utf8");
|
||||
|
||||
function defaultExec(): ExecFileLike {
|
||||
// promisify lazily so vitest module mocks of child_process still take effect
|
||||
// (same reason psnr.ts does this).
|
||||
const execFileP = promisify(execFile);
|
||||
return async (file, args) => {
|
||||
const { stdout } = await execFileP(file, [...args], { windowsHide: true, timeout: 5_000 });
|
||||
return { stdout: String(stdout) };
|
||||
};
|
||||
}
|
||||
|
||||
/** `ps -o pid=,rss=` output: one "<pid> <rss-kb>" pair per line. */
|
||||
export function parsePsRss(stdout: string): ProcessRssSample[] {
|
||||
const samples: ProcessRssSample[] = [];
|
||||
for (const line of stdout.split("\n")) {
|
||||
const match = /^\s*(\d+)\s+(\d+)\s*$/.exec(line);
|
||||
if (!match) continue;
|
||||
const pid = Number(match[1]);
|
||||
const rssKb = Number(match[2]);
|
||||
if (!Number.isFinite(pid) || !Number.isFinite(rssKb)) continue;
|
||||
samples.push({ pid, rssMb: Math.round(rssKb / KB_PER_MB) });
|
||||
}
|
||||
return samples;
|
||||
}
|
||||
|
||||
/** `tasklist /FO CSV /NH /FI "PID eq N"`: `"image","pid","session","sess#","12,345 K"`. */
|
||||
export function parseTasklistCsv(pid: number, stdout: string): ProcessRssSample[] {
|
||||
const line = stdout.split(/\r?\n/).find((l) => l.startsWith('"'));
|
||||
if (!line) return [];
|
||||
const cells = line.split('","').map((c) => c.replace(/^"|"$/g, ""));
|
||||
const memCell = cells[4];
|
||||
if (typeof memCell !== "string") return [];
|
||||
const kb = Number(memCell.replace(/[^0-9]/g, ""));
|
||||
if (!Number.isFinite(kb) || kb <= 0) return [];
|
||||
return [{ pid, rssMb: Math.round(kb / KB_PER_MB) }];
|
||||
}
|
||||
|
||||
/** `/proc/<pid>/status` line `VmRSS:\t 12345 kB`; absent for a zombie. */
|
||||
export function parseProcStatusRss(pid: number, status: string): ProcessRssSample[] {
|
||||
const match = /^VmRSS:\s*(\d+)\s*kB/m.exec(status);
|
||||
if (!match) return [];
|
||||
const kb = Number(match[1]);
|
||||
if (!Number.isFinite(kb) || kb <= 0) return [];
|
||||
return [{ pid, rssMb: Math.round(kb / KB_PER_MB) }];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resident set size per pid, in MiB. Zero dependencies: `/proc/<pid>/status`
|
||||
* on Linux (the render images are `node:*-slim` and ship no `ps`; same
|
||||
* precedent as the CLI's orphanCleanup / processTree), `ps` on other POSIX
|
||||
* (one call for all pids), `tasklist` on Windows (one call per pid). Never
|
||||
* rejects; a pid that exited between discovery and sampling is simply absent.
|
||||
*/
|
||||
export async function sampleProcessRss(
|
||||
pids: readonly number[],
|
||||
exec: ExecFileLike = defaultExec(),
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
readStatus: ReadFileLike = defaultReadFile,
|
||||
): Promise<ProcessRssSample[]> {
|
||||
const unique = [...new Set(pids.filter((p) => Number.isInteger(p) && p > 0))];
|
||||
if (unique.length === 0) return [];
|
||||
if (platform === "linux") {
|
||||
// Per-pid catch: one exited pid must not erase the others' samples.
|
||||
const perPid = await Promise.all(
|
||||
unique.map(async (pid) => {
|
||||
try {
|
||||
return parseProcStatusRss(pid, await readStatus(`/proc/${pid}/status`));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}),
|
||||
);
|
||||
return perPid.flat();
|
||||
}
|
||||
try {
|
||||
if (platform === "win32") {
|
||||
const results: ProcessRssSample[] = [];
|
||||
for (const pid of unique) {
|
||||
const { stdout } = await exec("tasklist", ["/FO", "CSV", "/NH", "/FI", `PID eq ${pid}`]);
|
||||
results.push(...parseTasklistCsv(pid, stdout));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
const { stdout } = await exec("ps", ["-o", "pid=,rss=", "-p", unique.join(",")]);
|
||||
return parsePsRss(stdout);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@
|
||||
* transition between variants.
|
||||
*/
|
||||
|
||||
import type { CapturePath } from "./observability.js";
|
||||
|
||||
export type CapturePlanTarget = Readonly<{
|
||||
kind: "sdr_streaming" | "sdr_disk";
|
||||
workerCount: number;
|
||||
@@ -51,6 +53,22 @@ export interface HdrLayeredCapturePlan extends CapturePlanBase {
|
||||
|
||||
export type CapturePlan = SdrStreamingCapturePlan | SdrDiskCapturePlan | HdrLayeredCapturePlan;
|
||||
|
||||
/**
|
||||
* Telemetry name for the stage a plan runs on. Exhaustive over `CapturePlan`,
|
||||
* so a new plan kind fails to compile until it declares its capture path
|
||||
* rather than silently reporting the wrong one.
|
||||
*/
|
||||
export function capturePathForPlanKind(kind: CapturePlan["kind"]): CapturePath {
|
||||
switch (kind) {
|
||||
case "sdr_streaming":
|
||||
return "streaming";
|
||||
case "sdr_disk":
|
||||
return "disk";
|
||||
case "hdr_layered":
|
||||
return "hdr_layered";
|
||||
}
|
||||
}
|
||||
|
||||
export interface CreateCapturePlanInput {
|
||||
workerCount: number;
|
||||
forceScreenshot: boolean;
|
||||
|
||||
@@ -30,6 +30,12 @@ export interface BrowserDiagnosticSummary {
|
||||
consoleWarnings: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Which capture stage produced the frames. Maps 1:1 from `CapturePlan.kind`
|
||||
* (see `capturePathForPlanKind`); named in render telemetry as `capture_path`.
|
||||
*/
|
||||
export type CapturePath = "streaming" | "disk" | "segmented" | "hdr_layered";
|
||||
|
||||
export interface RenderCaptureObservability {
|
||||
forceScreenshot: boolean;
|
||||
captureMode: "screenshot" | "beginframe";
|
||||
@@ -176,6 +182,23 @@ export interface RenderCaptureObservability {
|
||||
*/
|
||||
transientRetries?: number;
|
||||
memoryExhaustionDetected?: boolean;
|
||||
/**
|
||||
* Chrome process memory from the engine's per-session sampler (Phase −1 of
|
||||
* the long-form render plan). Updated live during capture so a
|
||||
* render_error still carries the last known state — the failure path never
|
||||
* builds a RenderPerfSummary, so this is the only channel that survives a
|
||||
* mid-capture target loss.
|
||||
*/
|
||||
chromeBrowserRssPeakMb?: number;
|
||||
chromeRendererRssPeakMb?: number;
|
||||
chromeRssLastMb?: number;
|
||||
chromeGpuProcessSeenLastSample?: boolean;
|
||||
chromeMemorySamples?: number;
|
||||
/** Which capture stage ran. Set once the capture plan resolves. */
|
||||
capturePath?: CapturePath;
|
||||
/** Segmented capture only (Phase 2): current segment and retries so far. */
|
||||
segmentIndex?: number;
|
||||
segmentRetries?: number;
|
||||
}
|
||||
|
||||
export interface RenderExtractionObservability {
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import type { CapturePerfSummary } from "@hyperframes/engine";
|
||||
import { buildRenderPerfSummary } from "./perfSummary.js";
|
||||
import { createRenderJob } from "../renderOrchestrator.js";
|
||||
|
||||
function baseInput(
|
||||
dedupPerfs: CapturePerfSummary[],
|
||||
overrides: Partial<Parameters<typeof buildRenderPerfSummary>[0]> = {},
|
||||
) {
|
||||
return {
|
||||
job: createRenderJob({ fps: { num: 30, den: 1 }, quality: "high" }),
|
||||
workerCount: dedupPerfs.length || 1,
|
||||
enableChunkedEncode: false,
|
||||
chunkedEncodeSize: 0,
|
||||
compositionDurationSeconds: 5,
|
||||
totalFrames: 150,
|
||||
outputWidth: 1920,
|
||||
outputHeight: 1080,
|
||||
videoCount: 0,
|
||||
audioCount: 0,
|
||||
totalElapsedMs: 1000,
|
||||
perfStages: {},
|
||||
videoExtractBreakdown: undefined,
|
||||
tmpPeakBytes: 0,
|
||||
captureAttempts: [],
|
||||
hdrDiagnostics: { videoExtractionFailures: 0, imageDecodeFailures: 0 },
|
||||
peakRssBytes: 0,
|
||||
peakHeapUsedBytes: 0,
|
||||
dedupPerfs,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function perf(overrides: Partial<CapturePerfSummary>): CapturePerfSummary {
|
||||
return {
|
||||
frames: 10,
|
||||
avgTotalMs: 1,
|
||||
avgSeekMs: 1,
|
||||
avgBeforeCaptureMs: 0,
|
||||
avgScreenshotMs: 0,
|
||||
staticDedupReused: 0,
|
||||
staticDedupEnabled: false,
|
||||
staticDedupArmed: false,
|
||||
staticDedupPredicted: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("aggregateChromeMemory", () => {
|
||||
it("takes the max peak across workers and the sum of samples", () => {
|
||||
const summary = buildRenderPerfSummary(
|
||||
baseInput([
|
||||
perf({
|
||||
chromeBrowserRssPeakMb: 100,
|
||||
chromeRendererRssPeakMb: 900,
|
||||
chromeRssLastMb: 1000,
|
||||
chromeGpuProcessSeenLastSample: true,
|
||||
chromeMemorySamples: 5,
|
||||
}),
|
||||
perf({
|
||||
chromeBrowserRssPeakMb: 120,
|
||||
chromeRendererRssPeakMb: 700,
|
||||
chromeRssLastMb: 800,
|
||||
chromeGpuProcessSeenLastSample: false,
|
||||
chromeMemorySamples: 4,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(summary.chromeMemory).toEqual({
|
||||
browserRssPeakMb: 120,
|
||||
rendererRssPeakMb: 900,
|
||||
rssLastMb: 1800,
|
||||
gpuProcessSeenLastSample: true,
|
||||
samples: 9,
|
||||
});
|
||||
});
|
||||
|
||||
it("is undefined when no session sampled", () => {
|
||||
const summary = buildRenderPerfSummary(baseInput([perf({})]));
|
||||
expect(summary.chromeMemory).toBeUndefined();
|
||||
});
|
||||
|
||||
it("aggregates a mixed fleet where only some workers sampled", () => {
|
||||
// A worker whose sampler never produced a reading must not drag the
|
||||
// result: no renderer peak invented, and gpuProcessSeenLastSample stays
|
||||
// false rather than inheriting the sampling worker's `undefined`.
|
||||
const summary = buildRenderPerfSummary(
|
||||
baseInput([
|
||||
perf({ chromeMemorySamples: 0 }),
|
||||
perf({ chromeBrowserRssPeakMb: 42, chromeRssLastMb: 42, chromeMemorySamples: 3 }),
|
||||
]),
|
||||
);
|
||||
expect(summary.chromeMemory).toEqual({
|
||||
browserRssPeakMb: 42,
|
||||
rendererRssPeakMb: undefined,
|
||||
rssLastMb: 42,
|
||||
gpuProcessSeenLastSample: false,
|
||||
samples: 3,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -189,6 +189,30 @@ function aggregateDrawElement(
|
||||
};
|
||||
}
|
||||
|
||||
function maxDefined(values: Array<number | undefined>): number | undefined {
|
||||
const present = values.filter((v): v is number => typeof v === "number");
|
||||
return present.length > 0 ? Math.max(...present) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chrome memory across capture sessions: max of per-session peaks, sum of the
|
||||
* last samples. Sessions that never produced a reading are excluded so
|
||||
* `samples: 0` cannot be published alongside another worker's real peaks.
|
||||
*/
|
||||
function aggregateChromeMemory(perfs: CapturePerfSummary[]): RenderPerfSummary["chromeMemory"] {
|
||||
const sampled = perfs.filter((p) => (p.chromeMemorySamples ?? 0) > 0);
|
||||
if (sampled.length === 0) return undefined;
|
||||
return {
|
||||
browserRssPeakMb: maxDefined(sampled.map((p) => p.chromeBrowserRssPeakMb)),
|
||||
rendererRssPeakMb: maxDefined(sampled.map((p) => p.chromeRendererRssPeakMb)),
|
||||
// Last samples are per session; summing approximates the whole fleet's
|
||||
// footprint at the end of capture (workers run concurrently).
|
||||
rssLastMb: sampled.reduce((sum, p) => sum + (p.chromeRssLastMb ?? 0), 0),
|
||||
gpuProcessSeenLastSample: sampled.some((p) => p.chromeGpuProcessSeenLastSample === true),
|
||||
samples: sampled.reduce((sum, p) => sum + (p.chromeMemorySamples ?? 0), 0),
|
||||
};
|
||||
}
|
||||
|
||||
function aggregateDedup(perfs: CapturePerfSummary[]): RenderPerfSummary["staticDedup"] {
|
||||
if (perfs.length === 0) return undefined;
|
||||
const armed = perfs.some((p) => p.staticDedupArmed);
|
||||
@@ -357,6 +381,7 @@ export function buildRenderPerfSummary(input: {
|
||||
peakRssMb: Math.round(input.peakRssBytes / (1024 * 1024)),
|
||||
peakHeapUsedMb: Math.round(input.peakHeapUsedBytes / (1024 * 1024)),
|
||||
staticDedup: aggregateDedup(input.dedupPerfs),
|
||||
chromeMemory: aggregateChromeMemory(input.dedupPerfs),
|
||||
beginFrameReuse: aggregateBeginFrameReuse(input.dedupPerfs),
|
||||
drawElement: aggregateDrawElement(
|
||||
input.dedupPerfs,
|
||||
|
||||
@@ -133,6 +133,7 @@ import {
|
||||
commitArtifactTransaction,
|
||||
} from "./render/artifactTransaction.js";
|
||||
import {
|
||||
capturePathForPlanKind,
|
||||
createCapturePlan,
|
||||
replanAfterFailure,
|
||||
streamingCaptureFailure,
|
||||
@@ -492,6 +493,14 @@ export interface RenderPerfSummary {
|
||||
* inside the orchestrator. Optional for the same back-compat reason.
|
||||
*/
|
||||
peakHeapUsedMb?: number;
|
||||
/** Chrome process memory aggregated across capture sessions (max of peaks, sum of samples). */
|
||||
chromeMemory?: {
|
||||
browserRssPeakMb?: number;
|
||||
rendererRssPeakMb?: number;
|
||||
rssLastMb?: number;
|
||||
gpuProcessSeenLastSample?: boolean;
|
||||
samples: number;
|
||||
};
|
||||
hdrDiagnostics?: HdrDiagnostics;
|
||||
hdrPerf?: HdrPerfSummary;
|
||||
/**
|
||||
@@ -3291,6 +3300,20 @@ async function executeRenderPipeline(input: {
|
||||
// sample frame indices from this so they land inside the drained range.
|
||||
compositionDurationSeconds: job.duration,
|
||||
requiresWebGpu: compositionRequiresWebGpu(compiled.html),
|
||||
// Live route for Chrome memory (spec §5 Phase −1). The aggregate route
|
||||
// through CapturePerfSummary only exists on success; a mid-capture
|
||||
// target loss never builds one, and that is the case this telemetry is
|
||||
// for. With parallel workers each session reports through the same
|
||||
// callback, so the record holds the most recent session's stats.
|
||||
onMemorySample: (stats) => {
|
||||
updateCaptureObservability({
|
||||
chromeBrowserRssPeakMb: stats.browserRssPeakMb,
|
||||
chromeRendererRssPeakMb: stats.rendererRssPeakMb,
|
||||
chromeRssLastMb: stats.rssLastMb,
|
||||
chromeGpuProcessSeenLastSample: stats.gpuProcessSeenLastSample,
|
||||
chromeMemorySamples: stats.samples,
|
||||
});
|
||||
},
|
||||
});
|
||||
// The URL-served frame path (PR #596) hands each injected `<img>` a
|
||||
// fileServer URL instead of a base64 data URI, on the theory that
|
||||
@@ -4056,6 +4079,10 @@ async function executeRenderPipeline(input: {
|
||||
if (capturePlan.routing.kind === "parallel_router") {
|
||||
deParallelRouter = capturePlan.routing.state === "active" ? "routed" : "reverted";
|
||||
}
|
||||
// Recorded here rather than beside the streaming-encode gate log: this
|
||||
// runs for the initial plan AND every replan, so a render that falls
|
||||
// back to disk reports the path it actually captured on.
|
||||
updateCaptureObservability({ capturePath: capturePathForPlanKind(capturePlan.kind) });
|
||||
};
|
||||
syncCapturePlan();
|
||||
updateCaptureObservability({
|
||||
|
||||
Reference in New Issue
Block a user