openworker/deploy/reference/qualify.mjs

// Destructive-to-OWNED-PROCESSES qualification; never target customer state.
// Leaves example sessions/artifacts and one explicitly shared test environment.
import assert from "node:assert/strict";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { randomUUID, createHash } from "node:crypto";
import { Readable } from "node:stream";
import { load, up, launch, until, command, compose, delay } from "./stack.mjs";
const m = await load(process.argv[2]);
await up(m);
const running = await launch(m);
const a = `http://127.0.0.1:${m.ports.apiA}`,
  b = `http://127.0.0.1:${m.ports.apiB}`;
const headers = { Authorization: `Bearer ${running.token}` },
  key = randomUUID();
const report = [];
const pass = (text) => {
  report.push(text);
  console.log("PASS " + text);
};
async function request(base, path, body) {
  const r = await fetch(base + path, {
    method: body ? "POST" : "GET",
    headers: { ...headers, "Content-Type": "application/json" },
    body: body ? JSON.stringify(body) : undefined,
    signal: AbortSignal.timeout(15000),
  });
  assert.ok(r.ok, `${path}: ${r.status}`);
  return r.json();
}
const snap = (id) => request(b, `/sessions/${id}`);
const plan = (name, args) =>
  "TEST_RESOURCE_GRANTS " + JSON.stringify([{ name, arguments: args }]);
async function wait(id, status) {
  return until(async () => {
    const s = await snap(id);
    return s.turn?.status === status ? s : false;
  }, 90000);
}
async function approve(id, s) {
  const i = s.interactions[0];
  assert.ok(i);
  await request(a, `/sessions/${id}/controls`, {
    request_id: "answer-" + i.id,
    turn_id: i.turn_id,
    action: {
      kind: "response",
      interaction_id: i.id,
      response: "once",
      binding: Object.fromEntries(
        [
          "bundle_digest",
          "schema_digest",
          "policy_digest",
          "resource_policy_digest",
        ].map((k) => [k, i[k]]),
      ),
    },
  });
}
async function events(id, through) {
  const abort = new AbortController(),
    timer = setTimeout(() => abort.abort(), 30000);
  const out = [];
  try {
    const r = await fetch(`${a}/sessions/${id}/events?after=0`, {
      headers,
      signal: abort.signal,
    });
    assert.ok(r.ok);
    let buffer = "";
    for await (const chunk of r.body) {
      buffer += Buffer.from(chunk).toString();
      let end;
      while ((end = buffer.indexOf("\n\n")) >= 0) {
        const frame = buffer.slice(0, end);
        buffer = buffer.slice(end + 2);
        const text = frame.split("\n").find((l) => l.startsWith("data: "));
        if (text) {
          const e = JSON.parse(text.slice(6));
          assert.equal(e.event_id, out.length + 1);
          out.push(e);
          if (e.event_id >= through) return out;
        }
      }
    }
    throw Error("SSE ended early");
  } finally {
    abort.abort();
    clearTimeout(timer);
  }
}
let env, checkpoint;
async function create(name, permissions) {
  const made = await request(a, "/sessions", {
    request_id: "create-" + randomUUID(),
    parameters: {
      name,
      coworker: "assistant-example",
      workspace: "/sandbox",
      environment_id: env.id,
      ...(permissions ? { permissions } : {}),
    },
  });
  return made.session_id;
}
async function submit(id, text) {
  return request(a, `/sessions/${id}/turns`, {
    submission_id: randomUUID(),
    text,
  });
}
async function restart(role) {
  await running.stop(role);
  await running.start(role);
}
try {
  env = JSON.parse(
    await command(
      running.bin,
      [
        "--config",
        running.config,
        "environment-create",
        "--request-id",
        "reference-sandbox",
        "--shared",
      ],
      { env: running.env },
    ),
  );
  assert.equal(env.phase, "ready");
  const sid = await create("Weekly release brief");
  await submit(
    sid,
    plan("write_file", {
      path: `brief-${key}.txt`,
      content:
        "Release readiness\n\n• Verify the migration\n• Review the deployment\n• Confirm rollback ownership\n",
    }),
  );
  const pending = await wait(sid, "waiting"),
    interaction = pending.interactions[0].id;
  await running.stop("worker-a", "SIGKILL");
  await running.stop("worker-b", "SIGKILL");
  await restart("api-a");
  await until(async () => {
    try {
      return (
        (await request(a, `/sessions/${sid}`)).interactions[0].id ===
        interaction
      );
    } catch {
      return false;
    }
  });
  await running.start("worker-a");
  await running.start("worker-b");
  await approve(sid, pending);
  await wait(sid, "completed");
  pass(
    "Approval identity survives worker crash and API replacement; response through another node resumes work",
  );
  await submit(sid, plan("read_file", { path: `brief-${key}.txt` }));
  const readPending = await wait(sid, "waiting");
  await approve(sid, readPending);
  const complete = await wait(sid, "completed");
  const history = await events(sid, complete.cursor);
  const result = history
    .filter((e) => e.type === "tool_finished" && e.data.name === "read_file")
    .at(-1).data.result;
  const bytes = Buffer.from(JSON.parse(result.content[0].text).text);
  assert.ok(bytes.includes("rollback ownership"));
  pass("Contiguous SSE replay and actual OpenShell file write/read");
  const hash = (v) => createHash("sha256").update(v).digest("hex");
  async function register(id, data, name) {
    return request(a, `/sessions/${id}/artifacts`, {
      request_id: randomUUID(),
      turn_id: complete.turn.id,
      name,
      content_type: "text/plain",
      size: data.length,
      sha256: hash(data),
    });
  }
  const artifact = await register(sid, bytes, "release-brief.txt");
  const content = `/sessions/${sid}/artifacts/${artifact.id}/content`;
  assert.equal((await fetch(a + content, { headers })).status, 409);
  assert.equal(
    (await fetch(a + content, { method: "PUT", headers, body: bytes })).status,
    200,
  );
  assert.deepEqual(
    Buffer.from(await (await fetch(b + content, { headers })).arrayBuffer()),
    bytes,
  );
  const large = Buffer.alloc(2 * 1024 * 1024 + 37, 6),
    partial = await register(sid, large, "upload-recovery.bin"),
    path = `/sessions/${sid}/artifacts/${partial.id}/content`;
  assert.equal(
    (
      await fetch(a + path, {
        method: "PUT",
        headers,
        body: Readable.from([large.subarray(0, 1024 * 1024 + 99)]),
        duplex: "half",
      })
    ).status,
    409,
  );
  assert.equal((await fetch(b + path, { headers })).status, 409);
  assert.equal(
    (await fetch(b + path, { method: "PUT", headers, body: large })).status,
    200,
  );
  assert.equal(
    hash(Buffer.from(await (await fetch(a + path, { headers })).arrayBuffer())),
    hash(large),
  );
  pass(
    "S3 artifacts: pending refusal, interrupted streamed upload, exact retry and cross-API download",
  );
  const question = await create("Review the deployment checklist");
  await submit(
    question,
    plan("write_file", {
      path: `checklist-${key}.txt`,
      content: "Deployment checklist for review",
    }),
  );
  const waiting = await wait(question, "waiting");
  const consoleURL = `http://127.0.0.1:${m.ports.console}`,
    consoleKey = (
      await readFile(join(m.root, "private/console-key"), "utf8")
    ).trim();
  const login = await fetch(consoleURL + "/login", {
    method: "POST",
    headers: { Origin: consoleURL, "Content-Type": "application/json" },
    body: JSON.stringify({ key: consoleKey }),
  });
  assert.equal(login.status, 200);
  const cookie = login.headers.get("set-cookie").split(";")[0];
  const observer = (
    await readFile(join(m.root, "private/observer-token"), "utf8")
  ).trim();
  for (const port of [m.ports.apiA, m.ports.apiB]) {
    assert.equal(
      (
        await fetch(`http://127.0.0.1:${port}/sessions/${question}/controls`, {
          method: "POST",
          headers: {
            Authorization: `Bearer ${observer}`,
            "Content-Type": "application/json",
          },
          body: "{}",
        })
      ).status,
      403,
    );
  }
  await running.stop("api-a");
  const observed = await fetch(consoleURL + `/api/sessions/${question}`, {
    headers: { Cookie: cookie },
  });
  assert.equal(observed.status, 200);
  assert.equal(
    (await observed.json()).interactions[0].id,
    waiting.interactions[0].id,
  );
  assert.equal(
    (
      await fetch(consoleURL + "/api/sessions", {
        method: "POST",
        headers: { Cookie: cookie, "Content-Type": "application/json" },
        body: "{}",
      })
    ).status,
    405,
  );
  await running.start("api-a");
  pass(
    "Console authenticates, fails over reads, preserves pending decision and refuses mutations",
  );
  // PostgreSQL outage poisons old connections; explicit process replacement
  // reconciles durable identity rather than silently retrying mutations.
  await compose(m, ["stop", "postgres"]);
  assert.equal(
    (
      await fetch(a + `/sessions/${sid}`, {
        headers,
        signal: AbortSignal.timeout(15000),
      })
    ).status,
    503,
  );
  for (const role of ["worker-a", "worker-b", "api-a", "api-b"])
    await running.stop(role);
  await compose(m, ["up", "-d", "--wait", "postgres"]);
  for (const role of ["api-a", "api-b", "worker-a", "worker-b"])
    await running.start(role);
  await until(async () => {
    const s = await snap(question);
    return s.interactions[0]?.id === waiting.interactions[0].id;
  });
  assert.deepEqual(
    Buffer.from(await (await fetch(b + content, { headers })).arrayBuffer()),
    bytes,
  );
  await compose(m, ["restart", "objects"]);
  await until(async () => {
    const r = await fetch(b + content, { headers });
    return r.ok && Buffer.from(await r.arrayBuffer()).equals(bytes);
  });
  pass(
    "PostgreSQL and S3 restarts preserve session, approval and artifact bytes; explicit API/worker reconnect",
  );
  if (m.observability) {
    await compose(m, ["stop", "observability"]);
    const outage = await create("Collector outage · execution check", {
      mode: "bypass",
    });
    await submit(
      outage,
      plan("example_echo", {
        text: "Harness remains available without telemetry",
      }),
    );
    await wait(outage, "completed");
    const health = await request(a, "/readyz");
    assert.equal(health.status, "ready");
    await until(async () => {
      const h = await request(a, "/readyz");
      return h.telemetry.export_failures > 0;
    }, 20000);
    await compose(m, ["up", "-d", "--wait", "observability"]);
    const password = (
        await readFile(join(m.root, "private/grafana-password"), "utf8")
      ).trim(),
      g = `http://127.0.0.1:${m.ports.grafana}`;
    const gh = {
      Authorization:
        "Basic " + Buffer.from("admin:" + password).toString("base64"),
    };
    assert.equal((await fetch(g + "/api/search")).status, 401);
    assert.equal(
      (
        await fetch(g + "/api/dashboards/uid/openworker-harness", {
          headers: gh,
        })
      ).status,
      200,
    );
    await until(async () => {
      await request(a, "/readyz");
      const r = await fetch(
        g +
          "/api/datasources/proxy/uid/prometheus/api/v1/query?query=" +
          encodeURIComponent(
            'sum(openworker_operation_count_total{openworker_operation="tool.execute"})',
          ),
        { headers: gh },
      );
      const j = await r.json();
      return j.data?.result?.some((x) => +x.value[1] > 0);
    }, 60000);
    for (const [path, nonempty] of [
      [
        "/api/datasources/proxy/uid/tempo/api/search?q=" +
          encodeURIComponent('{ resource.service.name = "openworker" }'),
        (j) => j.traces?.length > 0,
      ],
      [
        "/api/datasources/proxy/uid/loki/loki/api/v1/query_range?query=" +
          encodeURIComponent('{service_name="openworker"}') +
          "&limit=1",
        (j) => j.data?.result?.length > 0,
      ],
    ])
      await until(async () => {
        const r = await fetch(g + path, { headers: gh });
        return r.ok && nonempty(await r.json());
      }, 60000);
    pass(
      "Collector outage does not block work; authenticated Grafana queries real metrics, traces and logs",
    );
  }
  checkpoint = {
    session_id: sid,
    waiting_session_id: question,
    interaction_id: waiting.interactions[0].id,
    artifact_id: artifact.id,
    artifact_sha256: hash(bytes),
    environment_id: env.id,
    scripted_model: true,
  };
} finally {
  await running.close();
}
await compose(m, ["down"]); // Deliberately no --volumes.
await up(m);
const replacement = await launch(m);
try {
  const pending = await snap(checkpoint.waiting_session_id);
  assert.equal(pending.interactions[0].id, checkpoint.interaction_id);
  const data = await fetch(
    `${b}/sessions/${checkpoint.session_id}/artifacts/${checkpoint.artifact_id}/content`,
    { headers },
  );
  assert.equal(data.status, 200);
  assert.equal(
    createHash("sha256")
      .update(Buffer.from(await data.arrayBuffer()))
      .digest("hex"),
    checkpoint.artifact_sha256,
  );
  pass(
    "Full Compose down/up preserves original approval identity and verified artifact bytes",
  );
  await writeFile(
    join(m.root, "qualification.json"),
    JSON.stringify(
      { ...checkpoint, time: new Date().toISOString(), checks: report },
      null,
      2,
    ),
    { mode: 0o600 },
  );
  console.log(
    "Qualification complete. Run stack.mjs run to inspect the retained examples.",
  );
} finally {
  await replacement.close();
}