Merge c3f279d299 into 3337cc8990

packages/cli/src/capture/index.ts

@@ -42,6 +42,7 @@ import {
 import type { VisionCaptionOutcome } from "./contentExtractor.js";
 import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js";
 import { detectBlockedPage } from "./pageBlockDetection.js";
+import { writeResponseRecord } from "./responseRecord.js";
 import { navigateForCapture } from "./navigateForCapture.js";
 import {
   captureProtocolTimeoutMs,
@@ -295,8 +296,14 @@ export async function captureWebsite(
       progress("warn", message);
     }
 
+    // Persisted before the blocked-page check, so a capture that reaches navigation always leaves
+    // a record of what the server said. That makes the file's ABSENCE mean "capture never got a
+    // response", which is a third state distinct from a status of 404 and from a status of null.
+    const httpStatus = navigationResponse?.status() ?? null;
+    writeResponseRecord(join(outputDir, "extracted"), { status: httpStatus });
+
     const blockedReason = detectBlockedPage({
-      httpStatus: navigationResponse?.status() ?? null,
+      httpStatus,
       ...(contentCheckTimedOut
         ? {
             title: "",
@@ -874,6 +881,7 @@ export async function captureWebsite(
       ok: true,
       projectDir: outputDir,
       url,
+      httpStatus,
       title: tokens.title,
       extracted,
       screenshots,

packages/cli/src/capture/responseRecord.test.ts

@@ -0,0 +1,44 @@
+import { mkdtempSync, mkdirSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import {
+  RESPONSE_RECORD_FILENAME,
+  writeResponseRecord,
+  type CaptureResponseRecord,
+} from "./responseRecord.js";
+
+describe("writeResponseRecord", () => {
+  let extractedDir: string;
+  let projectDir: string;
+
+  beforeEach(() => {
+    projectDir = mkdtempSync(join(tmpdir(), "hf-response-record-"));
+    extractedDir = join(projectDir, "extracted");
+    mkdirSync(extractedDir);
+  });
+
+  afterEach(() => {
+    rmSync(projectDir, { recursive: true, force: true });
+  });
+
+  const readBack = (): CaptureResponseRecord =>
+    JSON.parse(readFileSync(join(extractedDir, RESPONSE_RECORD_FILENAME), "utf-8"));
+
+  it("records the status the server answered", () => {
+    writeResponseRecord(extractedDir, { status: 404 });
+
+    expect(readBack()).toEqual({ status: 404 });
+  });
+
+  it("keeps a missing status distinguishable from a status of zero and from success", () => {
+    writeResponseRecord(extractedDir, { status: null });
+    const record = readBack();
+
+    // The three states a consumer must be able to separate: the server answered, the server
+    // answered nothing, and — never — a falsy stand-in that reads as either.
+    expect(record.status).toBeNull();
+    expect(record.status).not.toBe(0);
+    expect("status" in record).toBe(true);
+  });
+});

packages/cli/src/capture/responseRecord.ts

@@ -0,0 +1,32 @@
+import { writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+/**
+ * What the server answered for the captured page, written beside the extraction.
+ *
+ * A capture that renders is not the same fact as a capture that succeeded: an error page has a
+ * title, colors, typefaces and a DOM, so every extractor downstream reads it happily and produces
+ * a design system belonging to whoever wrote the error page. `detectBlockedPage` cannot answer
+ * this — it decides whether the page LOOKS like a protection wall, which is a heuristic over the
+ * rendered document, and a rich 404 passes it. So the status is persisted as its own plain fact
+ * and consumers decide for themselves what a non-success response means for their product.
+ */
+export const RESPONSE_RECORD_FILENAME = "response.json";
+
+export interface CaptureResponseRecord {
+  /**
+   * The final response's status after redirects, or null when navigation produced no response at
+   * all. Null is NOT "fine": it means we never learned what the server said, which is a different
+   * fact from a 200 and from a 404, and a consumer must be able to tell the three apart.
+   */
+  status: number | null;
+}
+
+/** Writes the record into an already-created `extracted/` directory. */
+export function writeResponseRecord(extractedDir: string, record: CaptureResponseRecord): void {
+  writeFileSync(
+    join(extractedDir, RESPONSE_RECORD_FILENAME),
+    JSON.stringify(record, null, 2),
+    "utf-8",
+  );
+}

packages/cli/src/capture/types.ts

@@ -69,6 +69,11 @@ export interface CaptureResult {
   projectDir: string;
   /** Source URL */
   url: string;
+  /**
+   * What the server answered for `url`, after redirects; null when navigation produced no
+   * response. Also persisted to `extracted/response.json` for out-of-process consumers.
+   */
+  httpStatus: number | null;
   /** Page title */
   title: string;
   /** Extracted HTML data */

packages/cli/src/commands/capture.ts

@@ -204,6 +204,9 @@ export default defineCommand({
               ok: result.ok,
               projectDir: result.projectDir,
               url: result.url,
+              // Reported beside `ok`, because they answer different questions: a capture of an
+              // error page is `ok: true` with a status the caller has to see to know it.
+              httpStatus: result.httpStatus,
               title: result.title,
               screenshots: result.screenshots.length,
               assets: result.assets.length,