Files
openworker/deploy/helm/local.mjs

706 lines
21 KiB
JavaScript

// Local kind qualification only. No cloud access or default kubecontext mutation.
import { spawn } from "node:child_process";
import { readFile, writeFile, mkdir, access } from "node:fs/promises";
import { resolve, join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { randomBytes, randomUUID, createHash } from "node:crypto";
export const repo = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
export const root = join(repo, ".ow-helm");
export const namespace = "ow-helm-eval",
shellNamespace = "ow-helm-shell";
export async function run(program, args, { input, timeout = 240000 } = {}) {
return new Promise((resolve, reject) => {
const child = spawn(program, args, { stdio: ["pipe", "pipe", "pipe"] });
let out = "",
err = "";
child.stdout.on("data", (b) => (out += b));
child.stderr.on("data", (b) => {
if (err.length < 4096) err += b;
});
child.on("error", () => reject(Error(program + " could not start")));
child.stdin.on("error", () => {});
child.stdin.end(input);
const timer = setTimeout(() => child.kill("SIGTERM"), timeout);
child.on("close", (code) => {
clearTimeout(timer);
code === 0
? resolve(out)
: reject(
Error(
program +
" failed; inspect the scoped pod/job (output withheld to protect secrets)",
),
);
});
});
}
export async function settings() {
const m = JSON.parse(await readFile(join(root, "local.json"), "utf8"));
if (!m.context.startsWith("kind-") || !m.kubeconfig || !m.helm || !m.image)
throw Error("Explicit kind context, kubeconfig, Helm and image required");
const current = (
await run("kubectl", [
"--kubeconfig",
m.kubeconfig,
"config",
"current-context",
])
).trim();
if (current !== m.context) throw Error("Refusing unexpected kubecontext");
return m;
}
export const kubectl = (m, args, input) =>
run(
"kubectl",
["--kubeconfig", m.kubeconfig, "--context", m.context, ...args],
{ input },
);
export const apply = (m, items) =>
kubectl(
m,
["apply", "-f", "-"],
JSON.stringify({ apiVersion: "v1", kind: "List", items }),
);
export async function ownNamespace(m, name) {
const all = JSON.parse(
await kubectl(m, ["get", "namespaces", "-o", "json"]),
).items;
const existing = all.find((n) => n.metadata.name === name);
if (
existing &&
existing.metadata.annotations?.["openworker.dev/deployment"] !== m.id
)
throw Error(
"Refusing namespace not owned by this local deployment: " + name,
);
await apply(m, [
{
apiVersion: "v1",
kind: "Namespace",
metadata: {
name,
labels: { "openworker.dev/fixture": "helm-local" },
annotations: { "openworker.dev/deployment": m.id },
},
},
]);
}
export async function openshell(m) {
await ownNamespace(m, shellNamespace);
const directory = join(root, "charts"),
archive = join(directory, "helm-chart-0.1.3.tgz");
await mkdir(directory, { recursive: true });
try {
await access(archive);
} catch {
await run(m.helm, [
"pull",
"oci://ghcr.io/nvidia/openshell/helm-chart",
"--version",
"0.1.3",
"--destination",
directory,
]);
}
if (
createHash("sha256")
.update(await readFile(archive))
.digest("hex") !==
"5a82e27d6cf4089fa16d96c87d9fc0fe62f65fa05341465bc749ba5ab91abb33"
)
throw Error("OpenShell archive differs from the qualified pin");
await run(m.helm, [
"upgrade",
"--install",
"ow-shell",
archive,
"--kubeconfig",
m.kubeconfig,
"--kube-context",
m.context,
"--namespace",
shellNamespace,
"-f",
join(repo, "deploy/helm/openshell-local.yaml"),
"--wait",
"--timeout",
"180s",
]);
}
const meta = (name) => ({
name,
namespace,
labels: { "openworker.dev/fixture": "helm-local" },
});
const secret = (name, data) => ({
apiVersion: "v1",
kind: "Secret",
metadata: meta(name),
type: "Opaque",
stringData: data,
});
const configmap = (name, data) => ({
apiVersion: "v1",
kind: "ConfigMap",
metadata: meta(name),
data,
});
const svc = (name, port, targetPort = port) => ({
apiVersion: "v1",
kind: "Service",
metadata: meta(name),
spec: { selector: { app: name }, ports: [{ port, targetPort }] },
});
const pvc = (name) => ({
apiVersion: "v1",
kind: "PersistentVolumeClaim",
metadata: meta(name),
spec: {
accessModes: ["ReadWriteOnce"],
resources: { requests: { storage: "1Gi" } },
},
});
export async function certificate(name, dns) {
const folder = join(root, "private"),
key = join(folder, name + ".key"),
crt = join(folder, name + ".crt");
await run("openssl", [
"req",
"-x509",
"-newkey",
"rsa:2048",
"-nodes",
"-days",
"30",
"-keyout",
join(folder, name + ".ca.key"),
"-out",
join(folder, name + ".ca.crt"),
"-subj",
"/CN=" + name + "-local-CA",
"-addext",
"basicConstraints=critical,CA:TRUE",
]);
const csr = join(folder, name + ".csr"),
ext = join(folder, name + ".ext");
await run("openssl", [
"req",
"-new",
"-newkey",
"rsa:2048",
"-nodes",
"-keyout",
key,
"-out",
csr,
"-subj",
"/CN=" + dns[0],
]);
await writeFile(
ext,
"basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=" +
dns.map((d) => "DNS:" + d).join(",") +
"\n",
{ mode: 0o600 },
);
await run("openssl", [
"x509",
"-req",
"-in",
csr,
"-CA",
join(folder, name + ".ca.crt"),
"-CAkey",
join(folder, name + ".ca.key"),
"-CAcreateserial",
"-days",
"30",
"-out",
crt,
"-extfile",
ext,
]);
return { key: await readFile(key, "utf8"), crt: await readFile(crt, "utf8") };
}
export async function init(m) {
await mkdir(root, { recursive: true, mode: 0o700 });
await mkdir(join(root, "private"), { mode: 0o700 }); // Refuse regeneration.
for (const field of ["kubeconfig", "helm"]) m[field] = resolve(m[field]);
m.id = randomUUID();
await writeFile(join(root, "local.json"), JSON.stringify(m, null, 2), {
mode: 0o600,
flag: "wx",
});
await settings();
for (const [name, value] of Object.entries({
admin: randomBytes(32).toString("hex"),
probe: randomBytes(32).toString("hex"),
pg: randomBytes(32).toString("hex"),
s3: randomBytes(32).toString("hex"),
})) {
await writeFile(join(root, "private", name), value, {
mode: 0o600,
flag: "wx",
});
}
for (const [name, dns] of [
["pg", ["postgres", `postgres.${namespace}.svc.cluster.local`]],
["objects", ["objects", `objects.${namespace}.svc.cluster.local`]],
["api", ["localhost", `ow-openworker.${namespace}.svc.cluster.local`]],
])
await certificate(name, dns);
}
export async function fixtures(m) {
const read = (name) => readFile(join(root, "private", name), "utf8");
const pg = await read("pg"),
admin = await read("admin"),
probe = await read("probe"),
s3 = await read("s3");
await ownNamespace(m, namespace);
const shell = JSON.parse(
await kubectl(m, [
"-n",
shellNamespace,
"get",
"secret",
"openshell-client-tls",
"-o",
"json",
]),
);
const shellData = Object.fromEntries(
Object.entries(shell.data).map(([k, v]) => [
k,
Buffer.from(v, "base64").toString(),
]),
);
const objectsConf = `pid /tmp/nginx.pid; error_log /dev/stderr warn; events {} http { access_log off; client_body_temp_path /tmp/body; proxy_temp_path /tmp/proxy; server { listen 8443 ssl; ssl_certificate /cert/tls.crt; ssl_certificate_key /cert/tls.key; ssl_protocols TLSv1.2 TLSv1.3; client_max_body_size 65m; location / { proxy_pass http://127.0.0.1:8333; proxy_set_header Host $http_host; proxy_request_buffering off; proxy_buffering off; } } }`;
await apply(m, [
secret("database", {
url: `postgresql://openworker:${pg}@postgres.${namespace}.svc.cluster.local:5432/openworker`,
}),
secret("postgres-login", {
POSTGRES_PASSWORD: pg,
POSTGRES_USER: "openworker",
POSTGRES_DB: "openworker",
}),
secret("database-ca", { "ca.crt": await read("pg.ca.crt") }),
secret("postgres-tls", {
"tls.crt": await read("pg.crt"),
"tls.key": await read("pg.key"),
}),
secret("objects-tls", {
"tls.crt": await read("objects.crt"),
"tls.key": await read("objects.key"),
}),
secret("object-ca", { "ca.crt": await read("objects.ca.crt") }),
secret("object-login", {
AWS_ACCESS_KEY_ID: "local-harness",
AWS_SECRET_ACCESS_KEY: s3,
}),
secret("object-config", {
"s3.json": JSON.stringify({
identities: [
{
name: "harness",
credentials: [{ accessKey: "local-harness", secretKey: s3 }],
actions: [
"Read:openworker-artifacts",
"Write:openworker-artifacts",
"List:openworker-artifacts",
],
},
],
}),
}),
secret("api-tls", {
"tls.crt": await read("api.crt"),
"tls.key": await read("api.key"),
}),
secret("openshell-client", shellData),
secret("probe", {
"curl.conf": `header = "Authorization: Bearer ${probe}"\n`,
}),
secret("bootstrap", {
admin: admin,
probe: probe,
"probe.json": JSON.stringify({
principal: "health-probe",
role: "observer",
scope: { kind: "owned" },
}),
}),
configmap("objects-proxy", { "nginx.conf": objectsConf }),
configmap("postgres-config", {
"pg_hba.conf":
"local all all trust\nhostssl all all all scram-sha-256\nhostnossl all all all reject\n",
}),
pvc("postgres-data"),
pvc("objects-data"),
svc("postgres", 5432),
svc("objects", 8443),
{
apiVersion: "apps/v1",
kind: "Deployment",
metadata: meta("postgres"),
spec: {
replicas: 1,
strategy: { type: "Recreate" },
selector: { matchLabels: { app: "postgres" } },
template: {
metadata: { labels: { app: "postgres" } },
spec: {
containers: [
{
name: "postgres",
image:
"postgres@sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685",
command: [
"sh",
"-ec",
"cp /cert/* /tmp/; chown postgres:postgres /tmp/tls.*; chmod 600 /tmp/tls.key; exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/tmp/tls.crt -c ssl_key_file=/tmp/tls.key -c hba_file=/pg-config/pg_hba.conf",
],
envFrom: [{ secretRef: { name: "postgres-login" } }],
env: [
{ name: "PGDATA", value: "/var/lib/postgresql/data/pgdata" },
],
readinessProbe: {
exec: { command: ["pg_isready", "-U", "openworker"] },
periodSeconds: 3,
},
resources: {
requests: { cpu: "50m", memory: "64Mi" },
limits: { cpu: "500m", memory: "256Mi" },
},
volumeMounts: [
{ name: "data", mountPath: "/var/lib/postgresql/data" },
{ name: "cert", mountPath: "/cert", readOnly: true },
{ name: "config", mountPath: "/pg-config", readOnly: true },
],
},
],
volumes: [
{
name: "data",
persistentVolumeClaim: { claimName: "postgres-data" },
},
{ name: "cert", secret: { secretName: "postgres-tls" } },
{ name: "config", configMap: { name: "postgres-config" } },
],
},
},
},
},
{
apiVersion: "apps/v1",
kind: "Deployment",
metadata: meta("objects"),
spec: {
replicas: 1,
strategy: { type: "Recreate" },
selector: { matchLabels: { app: "objects" } },
template: {
metadata: { labels: { app: "objects" } },
spec: {
containers: [
{
name: "objects",
image:
"chrislusf/seaweedfs@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d",
args: [
"mini",
"-dir=/data",
"-s3.config=/auth/s3.json",
"-bucket=openworker-artifacts",
"-admin.ui=false",
"-webdav=false",
"-volume.max=4",
],
resources: {
requests: { cpu: "50m", memory: "128Mi" },
limits: { cpu: "500m", memory: "512Mi" },
},
volumeMounts: [
{ name: "data", mountPath: "/data" },
{ name: "auth", mountPath: "/auth", readOnly: true },
],
},
{
name: "tls",
image:
"nginx@sha256:a8b39bd9cf0f83869a2162827a0caf6137ddf759d50a171451b335cecc87d236",
command: [
"nginx",
"-c",
"/config/nginx.conf",
"-g",
"daemon off;",
],
readinessProbe: { tcpSocket: { port: 8443 } },
volumeMounts: [
{ name: "cert", mountPath: "/cert", readOnly: true },
{ name: "config", mountPath: "/config", readOnly: true },
],
},
],
volumes: [
{
name: "data",
persistentVolumeClaim: { claimName: "objects-data" },
},
{ name: "auth", secret: { secretName: "object-config" } },
{ name: "cert", secret: { secretName: "objects-tls" } },
{ name: "config", configMap: { name: "objects-proxy" } },
],
},
},
},
},
{
apiVersion: "networking.k8s.io/v1",
kind: "NetworkPolicy",
metadata: meta("backing-services"),
spec: {
podSelector: {
matchExpressions: [
{ key: "app", operator: "In", values: ["postgres", "objects"] },
],
},
policyTypes: ["Ingress"],
ingress: [
{
from: [{ podSelector: {} }],
ports: [
{ protocol: "TCP", port: 5432 },
{ protocol: "TCP", port: 8443 },
],
},
],
},
},
]);
await kubectl(m, [
"-n",
namespace,
"rollout",
"status",
"deployment/postgres",
"--timeout=180s",
]);
await kubectl(m, [
"-n",
namespace,
"rollout",
"status",
"deployment/objects",
"--timeout=180s",
]);
// Separate local fixtures, not bundled production dependencies.
const config = {
database_env: "OW_SERVER_DATABASE_URL",
schema: "ow_helm",
database_ca: "/database-ca/ca.crt",
state_root: "/state",
bundle: "/opt/openworker/bundle",
runner: "/opt/openworker/openworker",
model: "fixture",
scripted: true,
api_token_file: "/bootstrap/admin",
actor: "local-admin",
openshell: {
identity: "helm-local",
endpoint: `https://ow-helm-gateway.${shellNamespace}.svc.cluster.local:8080`,
certificates: "/openshell",
workspace: "default",
image: m.runnerImage,
},
artifacts: {
kind: "s3",
deployment: m.id,
bucket: "openworker-artifacts",
region: "us-east-1",
endpoint: `https://objects.${namespace}.svc.cluster.local:8443`,
ca_file: "/object-ca/ca.crt",
},
};
const values = {
image: m.image,
config,
secrets: {
database: "database",
databaseCA: "database-ca",
openshell: "openshell-client",
apiTLS: "api-tls",
probe: "probe",
objectStore: "object-login",
objectStoreCA: "object-ca",
},
network: { serviceNamespaces: [shellNamespace] },
};
await writeFile(join(root, "values.json"), JSON.stringify(values, null, 2), {
mode: 0o600,
});
}
export async function helm(m, extra = []) {
return run(m.helm, [
"upgrade",
"--install",
"ow",
join(repo, "deploy/helm/openworker"),
"--kubeconfig",
m.kubeconfig,
"--kube-context",
m.context,
"--namespace",
namespace,
"-f",
join(root, "values.json"),
...extra,
]);
}
export async function job(m, name, args, bootstrap = false) {
const dep = JSON.parse(
await kubectl(m, [
"-n",
namespace,
"get",
"deployment",
"ow-openworker-worker",
"-o",
"json",
]),
);
const spec = dep.spec.template.spec;
const c = spec.containers[0];
c.name = "admin";
c.args = args;
delete c.livenessProbe;
delete c.readinessProbe;
delete c.envFrom;
if (!args.includes("environment-create")) {
c.volumeMounts = c.volumeMounts.filter((v) => v.name !== "openshell");
spec.volumes = spec.volumes.filter((v) => v.name !== "openshell");
}
// Controller-local state is ephemeral; DB, artifacts and sandbox PVC are durable.
if (bootstrap) {
spec.volumes.push({
name: "bootstrap",
secret: { secretName: "bootstrap", defaultMode: 288 },
});
for (const key of ["admin", "probe", "probe.json"])
c.volumeMounts.push({
name: "bootstrap",
mountPath: "/bootstrap/" + key,
subPath: key,
readOnly: true,
});
}
c.env = c.env.filter((e) => e.name !== "POD_NAME");
spec.restartPolicy = "Never";
await apply(m, [
{
apiVersion: "batch/v1",
kind: "Job",
metadata: meta(name),
spec: {
backoffLimit: 0,
template: {
metadata: { labels: dep.spec.template.metadata.labels },
spec,
},
},
},
]);
await kubectl(m, [
"-n",
namespace,
"wait",
"--for=condition=complete",
"job/" + name,
"--timeout=150s",
]);
return kubectl(m, ["-n", namespace, "logs", "job/" + name, "-c", "admin"]);
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const action = process.argv[2];
if (action === "init") {
await init(JSON.parse(await readFile(resolve(process.argv[3]), "utf8")));
console.log(
"Private local state created. Existing credentials will never be regenerated.",
);
} else {
const m = await settings();
if (action === "openshell") {
await openshell(m);
console.log("Pinned authenticated OpenShell release ready.");
} else if (action === "upgrade") {
await helm(m, ["--wait", "--timeout", "180s"]);
console.log("OpenWorker upgraded; bootstrap was not rerun.");
} else if (action === "fixtures") {
await fixtures(m);
console.log("Local backing fixtures ready.");
} else if (action === "install") {
const deployments = JSON.parse(
await kubectl(m, [
"-n",
namespace,
"get",
"deployments",
"-l",
"app.kubernetes.io/instance=ow",
"-o",
"json",
]),
).items;
if (deployments.some((d) => (d.spec.replicas || 0) > 0))
throw Error(
"Use upgrade for a running installation; install will not scale it down",
);
await helm(m, ["--set", "api.replicas=0", "--set", "worker.replicas=0"]);
await job(m, "ow-migrate", [
"--config",
"/config/reference.json",
"migrate",
]);
await job(
m,
"ow-bootstrap",
["--config", "/config/reference.json", "auth-bootstrap"],
true,
);
await job(
m,
"ow-probe-key",
[
"--config",
"/config/reference.json",
"auth-issue",
"--spec",
"/bootstrap/probe.json",
"--token-file",
"/bootstrap/probe",
"--request-id",
"health-probe-001",
],
true,
);
await helm(m, ["--wait", "--timeout", "180s"]);
console.log("OpenWorker installed; APIs/workers run as Kubernetes pods.");
} else if (action === "environment") {
const result = await job(m, "ow-environment", [
"--config",
"/config/reference.json",
"environment-create",
"--request-id",
"helm-environment-001",
"--shared",
]);
await writeFile(join(root, "environment.json"), result, { mode: 0o600 });
console.log("Environment receipt saved privately.");
} else
throw Error(
"Use init CONFIG.json | openshell | fixtures | install | upgrade | environment",
);
}
}