From abcbeadc84e0a676aa64bb083a69a77dc84d3526 Mon Sep 17 00:00:00 2001 From: Rohit P Date: Wed, 11 Nov 2026 10:01:35 -0800 Subject: [PATCH 01/11] Add CLI controls for management UI and trace storage --- deploy/service/chart.test.mjs | 26 + deploy/service/chart/templates/config.yaml | 7 + .../chart/templates/management-ui.yaml | 43 +- deploy/service/chart/templates/workloads.yaml | 17 + deploy/service/chart/values.yaml | 2 + deploy/service/proxy.test.mjs | 59 ++ deploy/service/stack.mjs | 6 +- deploy/service/up/README.md | 78 +++ docs/harness/guides/observability.md | 18 + harness/server-reference/src/main.rs | 20 +- harness/service-cli/README.md | 20 + harness/service-cli/src/deployment.rs | 2 +- .../service-cli/src/deployment_features.rs | 617 ++++++++++++++++++ harness/service-cli/src/deployment_local.rs | 34 +- harness/service-cli/src/deployment_up.rs | 5 + .../service-cli/src/deployment_up_config.rs | 36 +- .../service-cli/src/deployment_up_install.rs | 47 +- harness/service-cli/src/main.rs | 19 +- .../tests/fixtures/kind-installer.py | 9 +- harness/service-cli/tests/local_installer.rs | 137 ++++ 20 files changed, 1177 insertions(+), 25 deletions(-) create mode 100644 deploy/service/proxy.test.mjs create mode 100644 harness/service-cli/src/deployment_features.rs diff --git a/deploy/service/chart.test.mjs b/deploy/service/chart.test.mjs index f5e3e0f1..923f65e7 100644 --- a/deploy/service/chart.test.mjs +++ b/deploy/service/chart.test.mjs @@ -229,3 +229,29 @@ test("exporter authentication mounts only into harness roles and remains opt-in" assert.match(bootstrap, /OW_TELEMETRY_AUTH/); assert.match(bootstrap, /secretName: export-auth/); }); +test("attachment chunks have a scoped 1 MiB limit, separate from metadata and artifacts",async()=>{ + const out=await render({}); + const block=out.match(/location ~ \^\/v1\/sessions\/\[A-Za-z0-9_-\]\+\/attachments\/\[A-Za-z0-9_-\]\+\$ \{[\s\S]*?\}/)?.[0]; + assert.ok(block,"Only the exact authenticated attachment upload path gets a larger bound"); + assert.match(block,/client_max_body_size 1m/); + assert.match(block,/proxy_request_buffering off/); + assert.match(out,/ssl_protocols TLSv1.2 TLSv1.3; client_max_body_size 160k/); +}); +test("recording enable runs only advisory-locked schema initialization, disable removes it",async()=>{ + const off=await render({});assert.doesNotMatch(off,/name: recording-schema/); + const on=await render({config:{...base.config,observability:{retention_days:14}}}); + for(const role of ["api","worker","controller","platform"]){ + const d=deployment(on,role);const init=d.split("initContainers:")[1].split(" containers:")[0]; + assert.match(init,/args: \[recording-migrate\]/); + assert.doesNotMatch(init,/bootstrap|\/models|\/openshell/); + } +}); +test("private management UI preview terminates TLS without giving the UI server the private key",async()=>{ + const out=await render({managementUI:{enabled:true,image:"ui@sha256:"+"b".repeat(64),publicOrigin:"https://localhost:8444",serviceUrl:"https://test-api.default.svc",caSecret:"ow-service-tls",caKey:"tls.crt",tlsSecret:"ow-service-tls"}}); + const d=deployment(out,"ui"); + assert.match(d,/name: tls/);assert.match(d,/key: "tls.crt", path: ca.crt/); + const ui=d.split("- name: ui")[1].split("- name: tls")[0]; + assert.doesNotMatch(ui,/mountPath: \/tls/); + assert.match(out,/name: https, port: 443, targetPort: https/); + assert.match(out,/proxy_set_header Host \$http_host/); +}); diff --git a/deploy/service/chart/templates/config.yaml b/deploy/service/chart/templates/config.yaml index c756b25f..40194ac4 100644 --- a/deploy/service/chart/templates/config.yaml +++ b/deploy/service/chart/templates/config.yaml @@ -54,6 +54,13 @@ data: proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_request_buffering off; proxy_buffering off; proxy_read_timeout 120s; proxy_send_timeout 120s; } + # SDK chunks contain base64 bytes plus metadata. Match only the upload + # route and the backend's 1 MiB bound; ordinary JSON stays at 160 KiB. + location ~ ^/v1/sessions/[A-Za-z0-9_-]+/attachments/[A-Za-z0-9_-]+$ { + client_max_body_size 1m; client_body_timeout 120s; + proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1; + proxy_set_header Connection ""; proxy_request_buffering off; + proxy_buffering off; proxy_read_timeout 120s; proxy_send_timeout 120s; } location / { proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_buffering off; proxy_read_timeout 3600s; } } diff --git a/deploy/service/chart/templates/management-ui.yaml b/deploy/service/chart/templates/management-ui.yaml index 70c1948f..eb2b519b 100644 --- a/deploy/service/chart/templates/management-ui.yaml +++ b/deploy/service/chart/templates/management-ui.yaml @@ -8,6 +8,27 @@ metadata: {name: {{ .Release.Name }}-ui} data: ui.json: | {{- dict "host" "0.0.0.0" "port" 8080 "publicOrigin" .Values.managementUI.publicOrigin "serviceUrl" .Values.managementUI.serviceUrl "serviceId" .Values.config.service.id "namespace" .Values.config.service.namespace "sessionSeconds" 1800 | toPrettyJson | nindent 4 }} + {{- if .Values.managementUI.tlsSecret }} + nginx.conf: | + pid /tmp/nginx.pid; + error_log /dev/stderr warn; + events { worker_connections 128; } + http { + access_log off; + client_body_temp_path /tmp/body; proxy_temp_path /tmp/proxy; + fastcgi_temp_path /tmp/fastcgi; uwsgi_temp_path /tmp/uwsgi; scgi_temp_path /tmp/scgi; + server { + listen 8443 ssl; + ssl_certificate /tls/tls.crt; ssl_certificate_key /tls/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; client_max_body_size 160k; + location / { + proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1; + proxy_set_header Host $http_host; proxy_set_header Connection ""; + proxy_buffering off; proxy_read_timeout 3600s; + } + } + } + {{- end }} --- apiVersion: apps/v1 kind: Deployment @@ -22,7 +43,7 @@ spec: metadata: labels: {app: {{ .Release.Name }}-ui} annotations: - checksum/config: {{ .Values.managementUI | toJson | sha256sum | quote }} + checksum/config: {{ list .Values.managementUI .Values.config.service | toJson | sha256sum | quote }} spec: automountServiceAccountToken: false securityContext: {runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, fsGroup: 1000} @@ -39,12 +60,28 @@ spec: - {name: ca, mountPath: /trust, readOnly: true} env: [{name: NODE_EXTRA_CA_CERTS, value: /trust/ca.crt}] {{- end }} + {{- if .Values.managementUI.tlsSecret }} + - name: tls + image: {{ .Values.proxyImage | quote }} + command: [nginx, -c, /config/nginx.conf, -g, "daemon off;"] + ports: [{name: https, containerPort: 8443}] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}, seccompProfile: {type: RuntimeDefault}} + volumeMounts: + - {name: config, mountPath: /config, readOnly: true} + - {name: tls, mountPath: /tls, readOnly: true} + - {name: tmp, mountPath: /tmp} + {{- end }} volumes: - name: config configMap: {name: {{ .Release.Name }}-ui} {{- if .Values.managementUI.caSecret }} - name: ca - secret: {secretName: {{ .Values.managementUI.caSecret }}, items: [{key: ca.crt, path: ca.crt}]} + secret: {secretName: {{ .Values.managementUI.caSecret }}, items: [{key: {{ .Values.managementUI.caKey | quote }}, path: ca.crt}]} + {{- end }} + {{- if .Values.managementUI.tlsSecret }} + - name: tls + secret: {secretName: {{ .Values.managementUI.tlsSecret }}, defaultMode: 0440} + - {name: tmp, emptyDir: {sizeLimit: 64Mi}} {{- end }} --- apiVersion: v1 @@ -52,5 +89,5 @@ kind: Service metadata: {name: {{ .Release.Name }}-ui} spec: selector: {app: {{ .Release.Name }}-ui} - ports: [{name: http, port: 80, targetPort: http}] + ports: [{{ if .Values.managementUI.tlsSecret }}{name: https, port: 443, targetPort: https}{{ else }}{name: http, port: 80, targetPort: http}{{ end }}] {{- end }} diff --git a/deploy/service/chart/templates/workloads.yaml b/deploy/service/chart/templates/workloads.yaml index f3f4798a..d255fb1f 100644 --- a/deploy/service/chart/templates/workloads.yaml +++ b/deploy/service/chart/templates/workloads.yaml @@ -48,6 +48,23 @@ spec: {{- end }} terminationGracePeriodSeconds: {{ $.Values.terminationGracePeriodSeconds }} securityContext: {fsGroup: 1000} + {{- if hasKey $.Values.config "observability" }} + # Explicit enable-time initialization; advisory-locked and idempotent. + # No first-admin bootstrap, provider key or sandbox credential required. + initContainers: + - name: recording-schema + image: {{ $.Values.image | quote }} + command: [/bin/sh, /opt/openworker/entrypoint.sh] + args: [recording-migrate] + securityContext: {runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}, seccompProfile: {type: RuntimeDefault}} + volumeMounts: + - {name: service, mountPath: /run/service/database-url, subPath: database-url, readOnly: true} + - {name: service, mountPath: /run/service/pg-ca.crt, subPath: pg-ca.crt, readOnly: true} + - {name: service, mountPath: /run/service/config.json, subPath: config.json, readOnly: true} + {{- if $.Values.secrets.observability }} + - {name: observability, mountPath: /run/observability, readOnly: true} + {{- end }} + {{- end }} containers: - name: harness image: {{ $.Values.image | quote }} diff --git a/deploy/service/chart/values.yaml b/deploy/service/chart/values.yaml index c2d86d1b..7c716ea1 100644 --- a/deploy/service/chart/values.yaml +++ b/deploy/service/chart/values.yaml @@ -31,3 +31,5 @@ managementUI: # Must match the API TLS certificate (may use the cluster Service DNS name). serviceUrl: "" caSecret: "" # optional Secret containing ca.crt + caKey: ca.crt + tlsSecret: "" # optional private HTTPS preview using an existing API certificate diff --git a/deploy/service/proxy.test.mjs b/deploy/service/proxy.test.mjs new file mode 100644 index 00000000..4323a626 --- /dev/null +++ b/deploy/service/proxy.test.mjs @@ -0,0 +1,59 @@ +// Opt-in real nginx test: no cloud access, credentials or model calls. +// OWS_PROXY_TEST=1 node --test deploy/service/proxy.test.mjs +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {createServer} from 'node:http'; +import https from 'node:https'; +import {mkdtemp,writeFile,readFile,rm} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import {join,resolve} from 'node:path'; +import {randomUUID,createHash} from 'node:crypto'; +import {execFile} from 'node:child_process'; +import {promisify} from 'node:util'; +const exec=promisify(execFile); +const run=async(program,args)=>String((await exec(program,args,{maxBuffer:8*1024*1024})).stdout); +test('real TLS proxy accepts SDK attachment chunks and preserves scoped limits/authentication',{skip:process.env.OWS_PROXY_TEST!=='1',timeout:60000},async()=>{ + const root=await mkdtemp(join(tmpdir(),'ows-proxy-')); + const name='ows-proxy-'+randomUUID().slice(0,8); + let forwarded=0; + const upstream=createServer(async(req,res)=>{ + forwarded++; + const chunks=[];for await(const c of req)chunks.push(c); + if(req.headers.authorization!=='Bearer fixture'){res.writeHead(401).end();return;} + const body=Buffer.concat(chunks); + res.writeHead(200,{'content-type':'application/json'}).end(JSON.stringify({bytes:body.length,sha256:createHash('sha256').update(body).digest('hex')})); + }); + await new Promise(r=>upstream.listen(0,'0.0.0.0',r)); + try{ + const values=join(root,'values.json'); + await writeFile(values,JSON.stringify({image:'fixture@sha256:'+'a'.repeat(64),config:{service:{id:'proxy-test',namespace:'test',agent:'fixture'}}})); + const rendered=await run('helm',['template','proxy-test',resolve('deploy/service/chart'),'-f',values]); + const raw=rendered.split(' nginx.conf: |\n')[1].split('\n---')[0].replace(/^ /gm,''); + const conf=raw.replaceAll('127.0.0.1:8080',`host.docker.internal:${upstream.address().port}`); + await writeFile(join(root,'nginx.conf'),conf); + await run('openssl',['req','-x509','-newkey','rsa:2048','-nodes','-days','1','-subj','/CN=localhost','-addext','subjectAltName=DNS:localhost,IP:127.0.0.1','-addext','basicConstraints=critical,CA:TRUE','-addext','keyUsage=critical,digitalSignature,keyCertSign,cRLSign','-addext','extendedKeyUsage=serverAuth','-keyout',join(root,'tls.key'),'-out',join(root,'tls.crt')]); + await run('openssl',['verify','-x509_strict','-purpose','sslserver','-CAfile',join(root,'tls.crt'),join(root,'tls.crt')]); + const ca=await readFile(join(root,'tls.crt')); + const image=(await readFile('deploy/service/chart/values.yaml','utf8')).match(/^proxyImage: (.+)$/m)[1]; + await run('docker',['run','--rm','-d','--name',name,'-p','127.0.0.1::8443','-v',`${root}:/tls:ro`,image,'nginx','-c','/tls/nginx.conf','-g','daemon off;']); + const port=Number((await run('docker',['port',name,'8443/tcp'])).trim().split(':').at(-1)); + const request=(path,body,auth=true)=>new Promise((resolve,reject)=>{ + const q=https.request({hostname:'127.0.0.1',port,path,method:'POST',ca,agent:false,headers:{expect:'100-continue','content-type':'application/json','content-length':body.length,...auth?{authorization:'Bearer fixture'}:{}}},r=>{let b='';r.on('data',c=>b+=c);r.on('end',()=>resolve({status:r.statusCode,body:b}));}); + q.on('error',reject);q.on('continue',()=>q.end(body));q.flushHeaders(); + }); + for(let i=0;;i++){try{await request('/healthz',Buffer.alloc(0));break;}catch(e){if(i===30)throw e;await new Promise(r=>setTimeout(r,100));}} + const path='/v1/sessions/session_123/attachments/file_123'; + const chunk=Buffer.from(JSON.stringify({offset:0,data:Buffer.alloc(256*1024,97).toString('base64')})); + assert(chunk.length>160*1024&&chunk.length<1024*1024); + const ok=await request(path,chunk);assert.equal(ok.status,200); + assert.equal(JSON.parse(ok.body).sha256,createHash('sha256').update(chunk).digest('hex')); + assert.equal((await request(path,chunk,false)).status,401); + const count=forwarded; + for(const p of ['/v1/sessions/session_123/turns','/v1/sessions/session_123/attachments','/v1/sessions/session_123/attachments/file_123/extra'])assert.equal((await request(p,chunk)).status,413); + assert.equal((await request(path,Buffer.alloc(1024*1024+1))).status,413); + assert.equal(forwarded,count,'oversized requests must not reach the application'); + }finally{ + await run('docker',['rm','-f',name]).catch(()=>{}); + await new Promise(r=>upstream.close(r));await rm(root,{recursive:true,force:true}); + } +}); diff --git a/deploy/service/stack.mjs b/deploy/service/stack.mjs index e18513dc..289c8341 100644 --- a/deploy/service/stack.mjs +++ b/deploy/service/stack.mjs @@ -70,6 +70,8 @@ async function certificate(input, name, cn, san) { "365", "-subj", `/CN=${cn} development CA`, + "-addext", "basicConstraints=critical,CA:TRUE", + "-addext", "keyUsage=critical,keyCertSign,cRLSign", ]); await run("openssl", [ "req", @@ -86,7 +88,7 @@ async function certificate(input, name, cn, san) { ]); await privateWrite( cert + ".ext", - `subjectAltName=${san}\nextendedKeyUsage=serverAuth\n`, + `subjectAltName=${san}\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\n`, ); await run("openssl", [ "x509", @@ -224,7 +226,7 @@ export async function init(root, connectionFile, image, port = 58443) { ); await privateWrite( join(input, "nginx.conf"), - `pid /tmp/nginx.pid;\nerror_log /dev/stderr warn;\nevents {worker_connections 256;}\nhttp {access_log off;client_body_temp_path /tmp/body;proxy_temp_path /tmp/proxy;fastcgi_temp_path /tmp/fastcgi;uwsgi_temp_path /tmp/uwsgi;scgi_temp_path /tmp/scgi;\nserver {listen 8443 ssl;ssl_certificate /tls/tls.crt;ssl_certificate_key /tls/tls.key;ssl_protocols TLSv1.2 TLSv1.3;client_max_body_size 160k;\nlocation ~ ^/(v1/)?sessions/[A-Za-z0-9_-]+/artifacts/[A-Za-z0-9_-]+/content$ {client_max_body_size 64m;client_body_timeout 120s;proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_request_buffering off;proxy_buffering off;proxy_read_timeout 120s;proxy_send_timeout 120s;}\nlocation / {proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_buffering off;proxy_read_timeout 3600s;}}}\n`, + `pid /tmp/nginx.pid;\nerror_log /dev/stderr warn;\nevents {worker_connections 256;}\nhttp {access_log off;client_body_temp_path /tmp/body;proxy_temp_path /tmp/proxy;fastcgi_temp_path /tmp/fastcgi;uwsgi_temp_path /tmp/uwsgi;scgi_temp_path /tmp/scgi;\nserver {listen 8443 ssl;ssl_certificate /tls/tls.crt;ssl_certificate_key /tls/tls.key;ssl_protocols TLSv1.2 TLSv1.3;client_max_body_size 160k;\nlocation ~ ^/(v1/)?sessions/[A-Za-z0-9_-]+/artifacts/[A-Za-z0-9_-]+/content$ {client_max_body_size 64m;client_body_timeout 120s;proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_request_buffering off;proxy_buffering off;proxy_read_timeout 120s;proxy_send_timeout 120s;}\nlocation ~ ^/v1/sessions/[A-Za-z0-9_-]+/attachments/[A-Za-z0-9_-]+$ {client_max_body_size 1m;client_body_timeout 120s;proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_request_buffering off;proxy_buffering off;proxy_read_timeout 120s;proxy_send_timeout 120s;}\nlocation / {proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_buffering off;proxy_read_timeout 3600s;}}}\n`, ); const manifest = { id, diff --git a/deploy/service/up/README.md b/deploy/service/up/README.md index 82fd5540..eda5a302 100644 --- a/deploy/service/up/README.md +++ b/deploy/service/up/README.md @@ -101,3 +101,81 @@ python3 deploy/service/up/test-plan.py \ ``` That check uses the real pinned OpenShell archive, fixture image references and catalog responses. It performs no real cloud calls, does not install a cluster and must not be presented as cloud qualification. + +## Management UI and trace storage + +Both features are optional and independent. Select them before the first `up`: + +```yaml +managementUI: + enabled: true + publicOrigin: https://localhost:8444 +observability: + enabled: true + retentionDays: 14 +``` + +The reviewed release manifest must include a digest-pinned `service-ui` image. +The default private preview uses the API certificate and a loopback-only Kubernetes +port-forward; `connection.json` contains its command, URL and CA file. Trust that +certificate locally, then sign in with a current service token. No token is put in +the browser URL. Keep the cloud's private management route open as described above. +A non-loopback `publicOrigin` requires your own TLS ingress; `up` does not create +public exposure or DNS records. + +After installation, use the original **installer** state directory (the +`installation/` child of the `up --state` directory): + +```sh +ows management-ui enable --state .ows/my-deployment/installation +ows management-ui status --state .ows/my-deployment/installation +ows management-ui disable --state .ows/my-deployment/installation + +ows observability enable --state .ows/my-deployment/installation --retention-days 14 +ows observability status --state .ows/my-deployment/installation +ows observability disable --state .ows/my-deployment/installation +``` + +If the original release did not include a UI image, pass +`--image registry.example.com/ui@sha256:YOUR_VERIFIED_DIGEST` on enable. +`--public-origin` overrides its origin. These commands use the original Kubernetes +credentials, verify cluster/namespace/release identities, and perform a Helm upgrade +of the saved chart. They do not use the service REST token. Features are saved in +`journal.json` with their original Helm operation and revision receipts; the initial +provisioning review and `deployment.yaml` stay immutable. Rerunning `up` does not +reset these saved feature overrides. Do not edit old installer values in place. + +Trace storage uses a separate `ow_observability` schema and connection to the +execution PostgreSQL by default. For a separate database, create an existing +Kubernetes Secret containing `database-url` and `ca.crt`, then select it with +`observability.databaseSecret` or `ows observability enable --database-secret NAME`. +Verified database TLS remains required. Enabling initializes the recording schema +before workloads start; feature changes roll the affected pods, so allow active +turns to drain. The first-admin bootstrap is not rerun. + +Disabling recording stops new recordings as the rollout completes. It does not +delete traces, databases or credentials. While disabled, the recorder's retention +worker is also stopped; retained data remains until recording resumes or an operator +performs a separate cleanup. Security audit, durable execution state and separately +configured OTLP exporters are unaffected. This is deployment-level control, not a +new per-agent trace policy API. + +A lost Helm response never causes an automatic repeat. `status` can reconcile the +original successful revision; an unresolved/pending/failed result blocks another +upgrade or teardown until the operator inspects that original operation. + +Local regression checks: + +```sh +cargo test --manifest-path harness/Cargo.toml -p openworker-service-cli +node --test deploy/service/chart.test.mjs +OWS_PROXY_TEST=1 node --test deploy/service/proxy.test.mjs +``` + +If the API uses a certificate signed by a separate CA, provide its public trust +bundle using `managementUI.caSecret` / `managementUI.caKey` in deployment YAML, +or `ows management-ui enable --ca-secret API_CA_SECRET --ca-key ca.crt`. +The Secret must already exist in the service namespace. Without an override, +the CLI trusts the API TLS Secret's `tls.crt`, suitable for the self-signed +certificate produced by `up`. The UI process receives only that public certificate; +the HTTPS proxy sidecar receives the separate TLS key mount. diff --git a/docs/harness/guides/observability.md b/docs/harness/guides/observability.md index fb193e2b..8b91e8a4 100644 --- a/docs/harness/guides/observability.md +++ b/docs/harness/guides/observability.md @@ -165,3 +165,21 @@ proof of compliance. A metadata reader is not a content auditor. In server apps, authorize evidence access separately before returning it, and keep database/host APIs in the trusted backend. The service's existing compliance content permission continues to control audit reads. Cluster experiment/review tooling comes later. + +## Manage recording on a service cluster + +The source CLI supports independent UI and trace-storage controls: + +```sh +ows observability enable --state ./deployment-state/installation --retention-days 14 +ows observability status --state ./deployment-state/installation +ows observability disable --state ./deployment-state/installation +ows management-ui enable --state ./deployment-state/installation +``` + +Use the installer state created by `ows deploy aws up` or `ows deploy azure up`. +Enabling recording initializes its separate PostgreSQL schema before workloads +restart. Disabling stops collection after the rollout and retains stored traces; +it does not disable security audit or a separately configured exporter. +See the [deployment configuration and private UI access](../../../deploy/service/up/README.md#management-ui-and-trace-storage) +for the required pinned UI image, CA, database Secret and enable/disable options. diff --git a/harness/server-reference/src/main.rs b/harness/server-reference/src/main.rs index dc9a5703..a9d1e36b 100644 --- a/harness/server-reference/src/main.rs +++ b/harness/server-reference/src/main.rs @@ -29,6 +29,8 @@ enum Command { drain_timeout_seconds: u64, }, Migrate, + /// Initialize only the optional recording schema, including on feature enable. + RecordingMigrate, /// Trusted installer aggregate check; no public/session API equivalent. ServiceInstallStatus, /// Trusted-host operational catalog, not a session-authorized API. @@ -116,7 +118,8 @@ async fn main() -> Result<()> { _ => "admin", }; let recording_config = config.clone(); - let record = !matches!(args.command, Command::Migrate) && !config.recovery_read_only; + let record = !matches!(args.command, Command::Migrate | Command::RecordingMigrate) + && !config.recovery_read_only; let started = tokio::task::spawn_blocking(move || { openworker_server_reference::recording::start(&recording_config, role, record) }) @@ -161,6 +164,19 @@ async fn run(args: Args, config: Config, telemetry: openworker_telemetry::Teleme ) { config.require_execution()?; } + if matches!(args.command, Command::RecordingMigrate) { + let options = config + .observability + .as_ref() + .context("Recording is disabled")?; + let database = options.database(&config)?; + tokio::task::spawn_blocking(move || { + openworker_observability::postgres::Postgres::migrate(&database) + }) + .await??; + println!("Recording schema ready"); + return Ok(()); + } let mut db = config.database()?; db.telemetry = telemetry; if matches!(args.command, Command::Migrate) { @@ -594,7 +610,7 @@ async fn run(args: Args, config: Config, telemetry: openworker_telemetry::Teleme | Command::EnvironmentReconcile { .. } => { bail!("This server reference requires the qualified Kubernetes provider") } - Command::Migrate => unreachable!(), + Command::Migrate | Command::RecordingMigrate => unreachable!(), } Ok(()) } diff --git a/harness/service-cli/README.md b/harness/service-cli/README.md index 38eac233..4e1fb773 100644 --- a/harness/service-cli/README.md +++ b/harness/service-cli/README.md @@ -96,3 +96,23 @@ See [AWS deployment](../../docs/harness/guides/aws.md) and with an explicit budget and independent cleanup before production deployment. `owcluster` is the primary command. `ows` remains an alias, using the same commands, configuration and saved deployment state. Existing `.ows` paths and `OWS_*` settings remain valid. + +## Optional management UI and trace storage + +Select these features in `deployment.yaml` for the AWS/Azure `up` commands, or +change them on an existing installation: + +```sh +ows management-ui enable --state ./deployment-state/installation +ows management-ui disable --state ./deployment-state/installation +ows management-ui status --state ./deployment-state/installation +ows observability enable --state ./deployment-state/installation --retention-days 14 +ows observability disable --state ./deployment-state/installation +ows observability status --state ./deployment-state/installation +``` + +`--state` is the original installer journal, using its Kubernetes credentials and +verified cluster identity. UI and trace storage are independent. Disabling tracing +retains data and leaves security audit and separately configured exporters intact. +The CLI saves feature changes and reconciles lost Helm responses without repeating +unknown upgrades. See [setup, UI image, private HTTPS and database options](../../deploy/service/up/README.md#management-ui-and-trace-storage). diff --git a/harness/service-cli/src/deployment.rs b/harness/service-cli/src/deployment.rs index 75f79a68..347c4c70 100644 --- a/harness/service-cli/src/deployment.rs +++ b/harness/service-cli/src/deployment.rs @@ -13,7 +13,7 @@ mod guard; #[path = "deployment_infrastructure.rs"] mod infrastructure; #[path = "deployment_local.rs"] -mod local; +pub(crate) mod local; #[path = "deployment_preflight.rs"] mod preflight; #[path = "deployment_up.rs"] diff --git a/harness/service-cli/src/deployment_features.rs b/harness/service-cli/src/deployment_features.rs new file mode 100644 index 00000000..4d0e2e16 --- /dev/null +++ b/harness/service-cli/src/deployment_features.rs @@ -0,0 +1,617 @@ +//! Optional service features over the original installer identity and journal. +//! Unknown Helm outcomes are observed, never automatically retried. +use super::*; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct ManagementUI { + #[serde(default)] + pub enabled: bool, + #[serde(default = "origin")] + pub public_origin: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ca_secret: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ca_key: Option, +} +fn origin() -> String { + "https://localhost:8444".into() +} +impl Default for ManagementUI { + fn default() -> Self { + Self { + enabled: false, + public_origin: origin(), + ca_secret: None, + ca_key: None, + } + } +} +impl ManagementUI { + pub(crate) fn validate(&self) -> Result<()> { + if let Some(s) = &self.ca_secret { + ensure!(dns_name(s), "Invalid UI CA Secret name"); + } + if let Some(s) = &self.ca_key { + ensure!( + !s.is_empty() + && s.len() <= 253 + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b)), + "Invalid UI CA Secret key" + ); + } + let u = reqwest::Url::parse(&self.public_origin).context("Invalid management UI origin")?; + ensure!(u.scheme()=="https" && u.host_str().is_some() && u.username().is_empty() && u.password().is_none() && u.path()=="/" && u.query().is_none() && u.fragment().is_none(), "Management UI publicOrigin must be an HTTPS origin without credentials, path, query or fragment"); + Ok(()) + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct Observability { + #[serde(default)] + pub enabled: bool, + #[serde(default = "retention")] + pub retention_days: u16, + /// Existing Kubernetes Secret containing database-url and ca.crt. No raw credentials. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub database_secret: Option, +} +fn retention() -> u16 { + 30 +} +impl Default for Observability { + fn default() -> Self { + Self { + enabled: false, + retention_days: retention(), + database_secret: None, + } + } +} +impl Observability { + pub(crate) fn validate(&self) -> Result<()> { + ensure!( + (1..=365).contains(&self.retention_days), + "Trace retention must be 1..365 days" + ); + if let Some(s) = &self.database_secret { + ensure!(dns_name(s), "Invalid recording database Secret name"); + } + Ok(()) + } +} +fn dns_name(s: &str) -> bool { + !s.is_empty() + && s.len() <= 63 + && s.starts_with(|c: char| c.is_ascii_lowercase() || c.is_ascii_digit()) + && s.ends_with(|c: char| c.is_ascii_lowercase() || c.is_ascii_digit()) + && s.bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} +#[derive(clap::Args)] +pub(crate) struct Target { + /// Original installer journal (local-plan.json and journal.json), not cloud lifecycle state. + #[arg(long)] + state: PathBuf, +} +#[derive(Subcommand)] +pub(crate) enum UICommand { + Enable { + #[command(flatten)] + target: Target, + /// Digest-pinned UI image. Omit when already recorded in Helm values. + #[arg(long)] + image: Option, + #[arg(long)] + public_origin: Option, + /// Existing Secret containing the CA that signs the service API certificate. + #[arg(long)] + ca_secret: Option, + #[arg(long, requires = "ca_secret")] + ca_key: Option, + }, + Disable { + #[command(flatten)] + target: Target, + }, + Status { + #[command(flatten)] + target: Target, + }, +} +#[derive(Subcommand)] +pub(crate) enum ObservationCommand { + Enable { + #[command(flatten)] + target: Target, + #[arg(long, value_parser=clap::value_parser!(u16).range(1..=365))] + retention_days: Option, + /// Existing Secret with database-url and ca.crt; omit to reuse execution PostgreSQL. + #[arg(long)] + database_secret: Option, + }, + Disable { + #[command(flatten)] + target: Target, + }, + Status { + #[command(flatten)] + target: Target, + }, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Update { + request_id: String, + before: Value, + values: Value, + #[serde(default)] + receipt: Option, +} +pub(super) fn latest_receipt(j: &Journal) -> Option<&Value> { + j.feature_updates.last().and_then(|u| u.receipt.as_ref()) +} +pub(super) fn effective_values(p: &Plan, j: &Journal) -> Result { + j.feature_updates + .last() + .map(|u| Ok(u.values.clone())) + .unwrap_or_else(|| read_json(&p.config.service.values_file)) +} +fn confirmed_receipt(j: &Journal) -> Result<&Value> { + latest_receipt(j) + .or_else(|| { + j.operations + .get("service-release") + .and_then(|o| o.receipt.as_ref()) + }) + .context("Original service installation receipt missing") +} +fn same_receipt(a: &Value, b: &Value) -> bool { + a["uid"] == b["uid"] && a["revision"] == b["revision"] && a["operationId"] == b["operationId"] +} +async fn live_values(client: &Client<'_>, p: &Plan) -> Result { + serde_json::from_slice( + &client + .helm(args(&[ + "get", + "values", + &p.config.service.release, + "-n", + &p.config.service.namespace, + "-o", + "json", + ])) + .await?, + ) + .context("Invalid release values") +} +pub(super) async fn verify_current(client: &Client<'_>, p: &Plan, j: &Journal) -> Result { + ensure!( + j.feature_updates.last().is_none_or(|u| u.receipt.is_some()), + "Feature upgrade outcome unknown; rerun status to reconcile, never replay the upgrade" + ); + let r = client + .release(&p.config.service.namespace, &p.config.service.release) + .await? + .context("Original service release absent")?; + ensure!( + same_receipt(&r, confirmed_receipt(j)?) && r["status"] == "deployed", + "Service Helm release identity/revision changed; refusing feature mutation" + ); + Ok(r) +} +pub(super) async fn reconcile( + client: &Client<'_>, + p: &Plan, + root: &Path, + j: &mut Journal, +) -> Result<()> { + let Some(update) = j + .feature_updates + .last() + .filter(|u| u.receipt.is_none()) + .cloned() + else { + return Ok(()); + }; + ensure!( + client.cluster().await? == p.cluster_uid, + "Cluster identity changed" + ); + owned(client, p, j, &p.config.service.namespace) + .await? + .context("Owned namespace missing")?; + let r = client + .release(&p.config.service.namespace, &p.config.service.release) + .await? + .context("Upgrade outcome unknown: original service release absent")?; + ensure!(r["installerId"]==p.id && r["operationId"]==update.request_id && r["revision"].as_u64()==update.before["revision"].as_u64().and_then(|n|n.checked_add(1)) && r["status"]=="deployed", "Feature upgrade outcome unknown/pending/failed; inspect the original Helm operation. It will not be retried automatically"); + ensure!( + live_values(client, p).await? == update.values, + "Upgraded release values differ from the saved intent" + ); + ready_service(client, p, j).await?; + j.feature_updates.last_mut().unwrap().receipt = Some(r); + save(root, "journal.json", j) +} +/// Pure rendering shared by initial cloud deployment and later CLI controls. +pub(crate) fn configure_ui( + values: &mut Value, + ui: &ManagementUI, + image: Option<&str>, + release: &str, + namespace: &str, +) -> Result<()> { + ui.validate()?; + if values["managementUI"].is_null() { + values["managementUI"] = json!({}); + } + values["managementUI"]["enabled"] = json!(ui.enabled); + if let Some(image) = image { + values["managementUI"]["image"] = json!(image); + } + values["managementUI"]["publicOrigin"] = json!(ui.public_origin); + values["managementUI"]["serviceUrl"] = json!(format!("https://{release}-api.{namespace}.svc")); + if ui.enabled { + let image = values["managementUI"]["image"] + .as_str() + .context("Provide --image with a digest-pinned management UI image")?; + ensure!( + image + .rsplit_once("@sha256:") + .is_some_and(|(name, d)| !name.is_empty() + && d.len() == 64 + && d.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))), + "Management UI image must use a sha256 digest" + ); + let api_tls = values["secrets"]["apiTLS"] + .as_str() + .unwrap_or("ow-service-tls") + .to_string(); + let ca_secret = ui + .ca_secret + .clone() + .or_else(|| { + values["managementUI"]["caSecret"] + .as_str() + .filter(|s| !s.is_empty()) + .map(str::to_owned) + }) + .unwrap_or_else(|| api_tls.clone()); + let ca_key = ui + .ca_key + .clone() + .or_else(|| { + if ui.ca_secret.is_none() { + values["managementUI"]["caKey"].as_str().map(str::to_owned) + } else { + None + } + }) + .unwrap_or_else(|| { + if ui.ca_secret.is_some() { + "ca.crt".into() + } else { + "tls.crt".into() + } + }); + values["managementUI"]["caSecret"] = json!(ca_secret); + values["managementUI"]["caKey"] = json!(ca_key); + // Private loopback preview is self-contained. Public origins require the + // operator's TLS ingress; never create public exposure implicitly. + let u = reqwest::Url::parse(&ui.public_origin)?; + values["managementUI"]["tlsSecret"] = + if matches!(u.host_str(), Some("localhost" | "127.0.0.1")) { + json!(api_tls) + } else { + json!("") + }; + } + Ok(()) +} +pub(crate) fn configure_observability(values: &mut Value, obs: &Observability) -> Result<()> { + obs.validate()?; + if values["secrets"].is_null() { + values["secrets"] = json!({}); + } + if obs.enabled { + if values["config"]["observability"].is_null() { + values["config"]["observability"] = json!({"schema":"ow_observability"}); + } + values["config"]["observability"]["retention_days"] = json!(obs.retention_days); + if let Some(s) = &obs.database_secret { + values["secrets"]["observability"] = json!(s); + } + } else { + values["config"] + .as_object_mut() + .context("Service configuration missing")? + .remove("observability"); + values["secrets"]["observability"] = json!(""); + } + Ok(()) +} +async fn change( + root: &Path, + mutation: impl FnOnce(&mut Value, &Plan) -> Result<()>, + status_only: bool, +) -> Result<()> { + let (p, mut j, _lock) = load(root)?; + ensure!( + j.phase == "installed", + "Feature controls require a completed service installation" + ); + ensure!( + inputs(&p.config)? == p.hashes, + "Original installer inputs changed" + ); + let client = Client { c: &p.config }; + ensure!( + client.cluster().await? == p.cluster_uid, + "Cluster identity changed" + ); + owned(&client, &p, &j, &p.config.service.namespace) + .await? + .context("Owned namespace missing")?; + reconcile(&client, &p, root, &mut j).await?; + let before = verify_current(&client, &p, &j).await?; + let mut values = effective_values(&p, &j)?; + // First update reads the live explicit values to preserve defaults/previous + // operator settings, but refuses any unreviewed drift. + ensure!( + live_values(&client, &p).await? == values, + "Service values changed outside the installer" + ); + if !status_only { + mutation(&mut values, &p)?; + if values != effective_values(&p, &j)? { + // Never rerun first-admin bootstrap on a feature upgrade. + values["bootstrapEnabled"] = json!(false); + let id = uuid::Uuid::new_v4().to_string(); + let file = format!("feature-values-{id}.json"); + save(root, &file, &values)?; + // Render validation happens before recording/sending any mutation. + client + .helm(args(&[ + "template", + &p.config.service.release, + p.config + .service + .chart + .to_str() + .context("Invalid chart path")?, + "-n", + &p.config.service.namespace, + "-f", + root.join(&file) + .to_str() + .context("Invalid feature values path")?, + ])) + .await?; + j.feature_updates.push(Update { + request_id: id.clone(), + before, + values: values.clone(), + receipt: None, + }); + save(root, "journal.json", &j)?; + let result = client + .helm(args(&[ + "upgrade", + &p.config.service.release, + p.config.service.chart.to_str().unwrap(), + "-n", + &p.config.service.namespace, + "-f", + root.join(&file).to_str().unwrap(), + "--labels", + &format!("{OWNER}={},openworker.io/operation-id={id}", p.id), + "--description", + &format!("ows:{}:{id}", p.id), + "--wait", + "--timeout", + "300s", + ])) + .await; + if result.is_err() { + // A response can be lost after success. Only observation of this + // exact revision and intent may recover it; never replay Helm. + reconcile(&client, &p, root, &mut j).await.context( + "Helm response failed; saved feature intent retained for status reconciliation", + )?; + } else { + reconcile(&client, &p, root, &mut j).await?; + } + } + } + let values = effective_values(&p, &j)?; + let ui = &values["managementUI"]; + let obs = &values["config"]["observability"]; + let preview = if ui["enabled"] == true + && ui["tlsSecret"].as_str().is_some_and(|s| !s.is_empty()) + { + let origin = ui["publicOrigin"] + .as_str() + .unwrap_or("https://localhost:8444"); + let u = reqwest::Url::parse(origin)?; + json!({"url":origin,"command":"kubectl","arguments":["--kubeconfig",p.config.kubeconfig,"--context",p.config.context,"-n",p.config.service.namespace,"port-forward",format!("service/{}-ui",p.config.service.release),format!("{}:443",u.port_or_known_default().unwrap()),"--address=127.0.0.1"],"caFile":"Use the API certificate from the deployment credentials; do not disable certificate verification","login":"existing service token; UI does not hold a bootstrap credential"}) + } else { + Value::Null + }; + crate::output( + json!({"deploymentId":p.id,"managementUI":{"enabled":ui["enabled"]==true,"publicOrigin":ui["publicOrigin"],"privatePreview":preview},"observability":{"enabled":!obs.is_null(),"configuration":obs,"databaseSecret":values["secrets"]["observability"],"storedData":"retained; no deletion performed","exporters":"configured independently"},"savedConfiguration":root.join("journal.json"),"helmRevision":confirmed_receipt(&j)?["revision"]}), + ) +} +pub(crate) async fn ui(command: &UICommand) -> Result<()> { + match command { + UICommand::Status { target } => change(&target.state, |_, _| Ok(()), true).await, + UICommand::Disable { target } => { + change( + &target.state, + |v, _| { + if v["managementUI"].is_null() { + v["managementUI"] = json!({}); + } + v["managementUI"]["enabled"] = json!(false); + Ok(()) + }, + false, + ) + .await + } + UICommand::Enable { + target, + image, + public_origin, + ca_secret, + ca_key, + } => { + change( + &target.state, + |v, p| { + let ui = ManagementUI { + enabled: true, + public_origin: public_origin + .clone() + .or_else(|| { + v["managementUI"]["publicOrigin"] + .as_str() + .map(str::to_owned) + }) + .unwrap_or_else(origin), + ca_secret: ca_secret.clone(), + ca_key: ca_key.clone(), + }; + configure_ui( + v, + &ui, + image.as_deref(), + &p.config.service.release, + &p.config.service.namespace, + ) + }, + false, + ) + .await + } + } +} +pub(crate) async fn observability(command: &ObservationCommand) -> Result<()> { + match command { + ObservationCommand::Status { target } => change(&target.state, |_, _| Ok(()), true).await, + ObservationCommand::Disable { target } => { + change( + &target.state, + |v, _| configure_observability(v, &Observability::default()), + false, + ) + .await + } + ObservationCommand::Enable { + target, + retention_days, + database_secret, + } => { + change( + &target.state, + |v, _| { + configure_observability( + v, + &Observability { + enabled: true, + retention_days: retention_days.unwrap_or_else(|| { + v["config"]["observability"]["retention_days"] + .as_u64() + .unwrap_or(30) as u16 + }), + database_secret: database_secret.clone(), + }, + ) + }, + false, + ) + .await + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn rendering_is_opt_in_independent_and_preserves_other_settings() { + let original = json!({"image":"service","config":{"schema":"execution","telemetry":{"enabled":true}},"secrets":{"apiTLS":"api-cert"}}); + let mut values = original.clone(); + let ui = ManagementUI { + enabled: true, + ..Default::default() + }; + configure_ui( + &mut values, + &ui, + Some(&format!("ui@sha256:{}", "b".repeat(64))), + "ows", + "example", + ) + .unwrap(); + assert!(values["config"].get("observability").is_none()); + assert_eq!(values["managementUI"]["tlsSecret"], "api-cert"); + assert_eq!( + values["managementUI"]["serviceUrl"], + "https://ows-api.example.svc" + ); + configure_observability( + &mut values, + &Observability { + enabled: true, + retention_days: 14, + database_secret: Some("recording-db".into()), + }, + ) + .unwrap(); + assert_eq!(values["secrets"]["observability"], "recording-db"); + configure_observability(&mut values, &Observability::default()).unwrap(); + assert!(values["config"].get("observability").is_none()); + assert_eq!( + values["config"]["telemetry"], + original["config"]["telemetry"] + ); + assert_eq!(values["image"], original["image"]); + assert_eq!(values["managementUI"]["enabled"], true); + } + #[test] + fn invalid_origins_retention_and_inline_database_values_fail_before_mutation() { + for origin in [ + "http://localhost:8444", + "https://user:secret@example.com", + "https://example.com/path", + "https://example.com/?token=secret", + ] { + assert!(ManagementUI { + enabled: true, + public_origin: origin.into(), + ..Default::default() + } + .validate() + .is_err()); + } + for retention_days in [0, 366] { + assert!(Observability { + enabled: true, + retention_days, + database_secret: None + } + .validate() + .is_err()); + } + assert!(Observability { + enabled: true, + retention_days: 14, + database_secret: Some("postgres://secret".into()) + } + .validate() + .is_err()); + } +} diff --git a/harness/service-cli/src/deployment_local.rs b/harness/service-cli/src/deployment_local.rs index 3a56a3cd..c7ef2914 100644 --- a/harness/service-cli/src/deployment_local.rs +++ b/harness/service-cli/src/deployment_local.rs @@ -6,6 +6,8 @@ use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, process::Command as Process, }; +#[path = "deployment_features.rs"] +pub(crate) mod features; const OWNER: &str = "openworker.io/installer-id"; #[path = "deployment_cluster_platform.rs"] mod platform; @@ -151,6 +153,8 @@ struct Journal { operations: BTreeMap, #[serde(default)] confirmed_no_effect: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + feature_updates: Vec, } #[derive(Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -651,6 +655,10 @@ async fn install_release( owned(client, p, j, ns) .await? .context("Owned namespace missing")?; + if key == "service-release" && !j.feature_updates.is_empty() { + features::verify_current(client, p, j).await?; + return Ok(()); + } if let Some(mut r) = client.release(ns, release).await? { let op = j .operations @@ -692,7 +700,7 @@ async fn install_release( .is_none_or(|v| v.is_null() || v.as_array().is_some_and(Vec::is_empty)), "Hooks may have unknown effects; failed release requires manual review" ); - ready_service(client, p).await?; + ready_service(client, p, j).await?; let job = client .get("job", &format!("{release}-bootstrap"), Some(ns)) .await? @@ -822,9 +830,12 @@ async fn prerequisites(client: &Client<'_>, eks: bool) -> Result<()> { ); Ok(()) } -async fn ready_service(client: &Client<'_>, p: &Plan) -> Result<()> { - let values: Value = read_json(&p.config.service.values_file)?; - let mut roles = vec!["api", "worker", "controller", "platform"]; +async fn ready_service(client: &Client<'_>, p: &Plan, j: &Journal) -> Result<()> { + let values = features::effective_values(p, j)?; + let mut roles = vec!["api", "worker", "controller"]; + if values["platformControllerEnabled"] != false { + roles.push("platform"); + } if values["managementUI"]["enabled"] == true { roles.push("ui"); } @@ -1236,11 +1247,13 @@ async fn uninstall( .context("Owned namespace missing before uninstall")?; let delete = format!("uninstall:{key}"); if let Some(r) = client.release(ns, release).await? { - let receipt = j - .operations - .get(key) - .and_then(|o| o.receipt.as_ref()) - .context("Release identity was never confirmed")?; + let receipt = if key == "service-release" { + features::latest_receipt(j) + } else { + None + } + .or_else(|| j.operations.get(key).and_then(|o| o.receipt.as_ref())) + .context("Release identity was never confirmed")?; ensure!( r["uid"] == receipt["uid"] && r["revision"] == receipt["revision"], "Release identity changed before uninstall" @@ -1447,6 +1460,7 @@ async fn run_scoped(command: &Command, local_only: bool) -> Result<()> { .await?; } let client = Client { c: &p.config }; + features::reconcile(&client, &p, state, &mut j).await?; ensure!( client.cluster().await? == p.cluster_uid, "Cluster identity changed" @@ -1709,7 +1723,7 @@ async fn run_scoped(command: &Command, local_only: bool) -> Result<()> { }, ) .await?; - ready_service(&client, &p).await?; + ready_service(&client, &p, &j).await?; j.phase = "installed".into(); save(state, "journal.json", &j)?; } diff --git a/harness/service-cli/src/deployment_up.rs b/harness/service-cli/src/deployment_up.rs index 667e852d..ec3c99b8 100644 --- a/harness/service-cli/src/deployment_up.rs +++ b/harness/service-cli/src/deployment_up.rs @@ -156,6 +156,11 @@ pub(super) fn operation_lock(root: &Path, name: &str) -> Result { Ok(f) } async fn check_local_tools(c: &Config, package: &package::Snapshot) -> Result<()> { + ensure!( + !c.management_ui.as_ref().is_some_and(|ui| ui.enabled) + || package.release.images.service_ui.is_some(), + "Enabling managementUI requires a digest-pinned service-ui image in the release manifest" + ); let mut tools: Vec<(&str, Vec<&str>)> = vec![ ("kubectl", vec!["version", "--client", "--output=json"]), ("helm", vec!["version", "--short"]), diff --git a/harness/service-cli/src/deployment_up_config.rs b/harness/service-cli/src/deployment_up_config.rs index 7356f097..bfe0d7f9 100644 --- a/harness/service-cli/src/deployment_up_config.rs +++ b/harness/service-cli/src/deployment_up_config.rs @@ -17,6 +17,11 @@ pub(super) struct Config { pub models: BTreeMap, #[serde(default)] pub advanced: Advanced, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(rename = "managementUI")] + pub management_ui: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observability: Option, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -170,6 +175,12 @@ impl Config { Ok(config) } pub fn validate(&self, provider: &str) -> Result<()> { + if let Some(ui) = &self.management_ui { + ui.validate()?; + } + if let Some(obs) = &self.observability { + obs.validate()?; + } if let Some(t) = &self.trial { ensure!((1..=72).contains(&t.expires_after_hours)&&t.delete_data,"A disposable trial requires 1..72 hours and explicit deleteData:true; omit trial for persistent deployment"); } @@ -310,7 +321,7 @@ impl Config { #[cfg(test)] mod tests { use super::*; - pub(super) fn yaml() -> &'static str { + pub(crate) fn yaml() -> &'static str { "name: example\ncloud:\n provider: azure\n subscriptionId: 11111111-2222-4333-8444-555555555555\n region: westus2\nnodes:\n count: 2\n requirements:\n cpu: {min: 4, max: 8}\n memoryGiB: {min: 16, max: 32}\nbudget:\n limitUSD: 100\n" } #[test] @@ -357,3 +368,26 @@ mod tests { c.validate("azure").unwrap(); } } + +#[cfg(test)] +mod feature_config_tests { + use super::*; + #[test] + fn cloud_settings_accept_independent_features_and_reject_unknown_inline_options() { + let base = tests::yaml(); + for options in [ + "managementUI: {enabled: true}\n", + "observability: {enabled: true, retentionDays: 14, databaseSecret: recording-db}\n", + ] { + let c: Config = serde_yaml::from_str(&format!("{base}{options}")).unwrap(); + c.validate("azure").unwrap(); + } + let c: Config = serde_yaml::from_str(base).unwrap(); + let value = serde_json::to_value(c).unwrap(); + assert!(value.get("managementUI").is_none() && value.get("observability").is_none()); + assert!(serde_yaml::from_str::(&format!( + "{base}observability: {{enabled: true, connection: secret}}\n" + )) + .is_err()); + } +} diff --git a/harness/service-cli/src/deployment_up_install.rs b/harness/service-cli/src/deployment_up_install.rs index 9aecd42e..b9dacaa9 100644 --- a/harness/service-cli/src/deployment_up_install.rs +++ b/harness/service-cli/src/deployment_up_install.rs @@ -89,7 +89,21 @@ impl Native<'_> { let id=if i==0{"primary"}else{p.as_str()}; json!({"id":id,"provider":p,"identity":format!("{}-{id}",plan.deployment_id),"key_env":if p=="anthropic"{"ANTHROPIC_API_KEY"}else{"OPENAI_API_KEY"}}) }).collect(); - let values = json!({"image":harness,"apiReplicas":1,"worker":{"replicas":1,"ownership":"manual","maxReplicas":2,"slots":2,"drainSeconds":30},"terminationGracePeriodSeconds":90,"secrets":{"model":if c.models.is_empty(){""}else{"ow-service-model"}},"config":{"service":{"id":plan.deployment_id,"namespace":"local","agent":"installed-tools","legacy_compatibility":false},"database_env":"OW_PG_URL","schema":"ow_service","database_loopback_plaintext":false,"database_ca":"/run/service/pg-ca.crt","state_root":"/state","bundle":"/opt/openworker/bundle","runner":"/opt/openworker/openworker","model":model,"scripted":false,"model_provider":provider,"model_connections":connections,"api_token_file":"/bootstrap/admin-token","actor":"deployment-admin","openshell":{"identity":plan.deployment_id,"endpoint":endpoint,"tls_server_name":format!("{fullname}.{ns}.svc.cluster.local"),"certificates":"/openshell","workspace":"default","image":runner},"artifacts":artifacts}}); + let mut values = json!({"image":harness,"apiReplicas":1,"worker":{"replicas":1,"ownership":"manual","maxReplicas":2,"slots":2,"drainSeconds":30},"terminationGracePeriodSeconds":90,"secrets":{"model":if c.models.is_empty(){""}else{"ow-service-model"}},"config":{"service":{"id":plan.deployment_id,"namespace":"local","agent":"installed-tools","legacy_compatibility":false},"database_env":"OW_PG_URL","schema":"ow_service","database_loopback_plaintext":false,"database_ca":"/run/service/pg-ca.crt","state_root":"/state","bundle":"/opt/openworker/bundle","runner":"/opt/openworker/openworker","model":model,"scripted":false,"model_provider":provider,"model_connections":connections,"api_token_file":"/bootstrap/admin-token","actor":"deployment-admin","openshell":{"identity":plan.deployment_id,"endpoint":endpoint,"tls_server_name":format!("{fullname}.{ns}.svc.cluster.local"),"certificates":"/openshell","workspace":"default","image":runner},"artifacts":artifacts}}); + if let Some(ui) = &c.management_ui { + super::super::super::local::features::configure_ui( + &mut values, + ui, + images["service-ui"]["image"].as_str(), + "ows", + "ow-service", + )?; + } else if let Some(image) = images["service-ui"]["image"].as_str() { + values["managementUI"] = json!({"enabled":false,"image":image}); + } + if let Some(obs) = &c.observability { + super::super::super::local::features::configure_observability(&mut values, obs)?; + } self.exact_file("service-values.json", &values)?; let mut shell: Value = serde_json::from_slice(&fs::read(&self.package.release.openshell_values.file)?)?; @@ -193,7 +207,7 @@ impl Native<'_> { !cert.exists() && !key.exists(), "Partial TLS generation; inspect private credential files before continuing" ); - self.tool("openssl",vec!["req".into(),"-x509".into(),"-newkey".into(),"rsa:3072".into(),"-nodes".into(),"-days".into(),"30".into(),"-subj".into(),"/CN=ows-api.ow-service.svc.cluster.local".into(),"-addext".into(),"subjectAltName=DNS:localhost,IP:127.0.0.1,DNS:ows-api.ow-service.svc.cluster.local,DNS:ows-api.ow-service.svc".into(),"-keyout".into(),arg(&key)?,"-out".into(),arg(&cert)?],None).await?; + self.tool("openssl",vec!["req".into(),"-x509".into(),"-newkey".into(),"rsa:3072".into(),"-nodes".into(),"-days".into(),"30".into(),"-subj".into(),"/CN=ows-api.ow-service.svc.cluster.local".into(),"-addext".into(),"basicConstraints=critical,CA:TRUE".into(),"-addext".into(),"keyUsage=critical,digitalSignature,keyCertSign,cRLSign".into(),"-addext".into(),"extendedKeyUsage=serverAuth".into(),"-addext".into(),"subjectAltName=DNS:localhost,IP:127.0.0.1,DNS:ows-api.ow-service.svc.cluster.local,DNS:ows-api.ow-service.svc".into(),"-keyout".into(),arg(&key)?,"-out".into(),arg(&cert)?],None).await?; } #[cfg(unix)] { @@ -241,7 +255,17 @@ impl Native<'_> { pub(super) async fn verify(&self) -> Result { let result = self.installer("status", &[]).await?; let kube = self.root.join("handoff/kubeconfig.json"); - for component in ["api", "worker", "controller", "platform"] { + let mut components = vec!["api", "worker", "controller", "platform"]; + if self + .review + .config + .management_ui + .as_ref() + .is_some_and(|ui| ui.enabled) + { + components.push("ui"); + } + for component in components { self.tool( "kubectl", vec![ @@ -258,7 +282,22 @@ impl Native<'_> { ) .await?; } - let connection = json!({"deploymentId":self.review.deployment_id,"serviceId":self.review.deployment_id,"namespace":"local","url":"https://ows-api.ow-service.svc.cluster.local","access":"private cluster network","kubeconfig":kube,"caFile":self.root.join("credentials/api.crt"),"tokenFile":self.install_state().join("bootstrap-token"),"privatePreview":{"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-api","8443:443","--address=127.0.0.1"],"url":"https://localhost:8443"},"modelSmokeTest":"not_run"}); + let mut connection = json!({"deploymentId":self.review.deployment_id,"serviceId":self.review.deployment_id,"namespace":"local","url":"https://ows-api.ow-service.svc.cluster.local","access":"private cluster network","kubeconfig":kube,"caFile":self.root.join("credentials/api.crt"),"tokenFile":self.install_state().join("bootstrap-token"),"privatePreview":{"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-api","8443:443","--address=127.0.0.1"],"url":"https://localhost:8443"},"modelSmokeTest":"not_run"}); + if let Some(ui) = self + .review + .config + .management_ui + .as_ref() + .filter(|ui| ui.enabled) + { + connection["managementUI"] = + json!({"url":ui.public_origin,"login":"existing service token"}); + let url = reqwest::Url::parse(&ui.public_origin)?; + if matches!(url.host_str(), Some("localhost" | "127.0.0.1")) { + connection["managementUI"]["privatePreview"] = json!({"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-ui",format!("{}:443",url.port_or_known_default().unwrap()),"--address=127.0.0.1"],"caFile":self.root.join("credentials/api.crt")}); + } + } + connection["observability"] = json!({"enabled":self.review.config.observability.as_ref().is_some_and(|obs|obs.enabled)}); write_private(&self.root.join("connection.json"), &connection)?; Ok(Observation::Complete( json!({"nativeStatus":result,"rolloutsReady":true,"modelSmokeTest":"not_run","connectionFile":self.root.join("connection.json")}), diff --git a/harness/service-cli/src/main.rs b/harness/service-cli/src/main.rs index 645a43f3..21637e29 100644 --- a/harness/service-cli/src/main.rs +++ b/harness/service-cli/src/main.rs @@ -39,6 +39,16 @@ enum Command { #[command(subcommand)] command: deployment::Command, }, + /// Configure the optional browser UI on an installed cluster. + ManagementUi { + #[command(subcommand)] + command: deployment::local::features::UICommand, + }, + /// Configure optional trace storage; disabling never deletes stored traces. + Observability { + #[command(subcommand)] + command: deployment::local::features::ObservationCommand, + }, Context { #[command(subcommand)] command: ContextCommand, @@ -370,6 +380,13 @@ async fn run(args: Args) -> Result<()> { if let Command::Deploy { command } = &args.command { return deployment::dispatch(command).await; } + match &args.command { + Command::ManagementUi { command } => return deployment::local::features::ui(command).await, + Command::Observability { command } => { + return deployment::local::features::observability(command).await + } + _ => {} + } let mut settings = settings::Settings::load(args.config.as_ref())?; if let Command::Context { command } = &args.command { return match command { @@ -650,7 +667,7 @@ async fn run(args: Args) -> Result<()> { .control_receipt(&receipt) .await?, )?, - Command::Context { .. } | Command::Login | Command::Deploy { .. } => unreachable!(), + Command::Context { .. } | Command::Login | Command::Deploy { .. } | Command::ManagementUi { .. } | Command::Observability { .. } => unreachable!(), } service.disconnect(); Ok(()) diff --git a/harness/service-cli/tests/fixtures/kind-installer.py b/harness/service-cli/tests/fixtures/kind-installer.py index 5d423d9e..4bc82f2e 100644 --- a/harness/service-cli/tests/fixtures/kind-installer.py +++ b/harness/service-cli/tests/fixtures/kind-installer.py @@ -13,12 +13,19 @@ if tool=='helm': ns=arg('-n');releases=state['releases'].setdefault(ns,{}) if a[0]=='list':finish([{'name':n} for n in releases]) if a[0]=='status':finish(releases[a[1]]) + if a[:2]==['get','values']:finish(releases[a[2]].get('values',{})) + if a[0]=='template':finish('rendered') if a[0]=='install': if os.environ.get('FAIL_BEFORE')=='yes':sys.exit(77) - name=a[1];labels=dict(pair.split('=',1) for pair in arg('--labels').split(','));releases[name]={'version':1,'info':{'status':'failed' if os.environ.get('FAIL_RELEASE') else 'deployed','description':'context deadline exceeded' if os.environ.get('FAIL_RELEASE') else arg('--description')},'uid':'release-uid','labels':labels};store.write_text(json.dumps(state)) + name=a[1];labels=dict(pair.split('=',1) for pair in arg('--labels').split(','));releases[name]={'version':1,'info':{'status':'failed' if os.environ.get('FAIL_RELEASE') else 'deployed','description':'context deadline exceeded' if os.environ.get('FAIL_RELEASE') else arg('--description')},'uid':'release-uid','labels':labels,'values':json.loads(pathlib.Path(arg('-f')).read_text())};store.write_text(json.dumps(state)) if os.environ.get('FAIL_RELEASE'):sys.exit(81) if os.environ.get('FAIL_AFTER')=='yes':sys.exit(78) finish('installed') + if a[0]=='upgrade': + if os.environ.get('FAIL_UPGRADE_BEFORE')=='yes':sys.exit(77) + name=a[1];r=releases[name];r['version']+=1;r['uid']='release-uid-'+str(r['version']);r['labels']=dict(pair.split('=',1) for pair in arg('--labels').split(','));r['info']={'status':'deployed','description':arg('--description')};r['values']=json.loads(pathlib.Path(arg('-f')).read_text());store.write_text(json.dumps(state)) + if os.environ.get('FAIL_UPGRADE_AFTER')=='yes':sys.exit(78) + finish('upgraded') if a[0]=='uninstall':del releases[a[1]];finish('uninstalled') sys.exit(99) # kubectl global options precede its command. diff --git a/harness/service-cli/tests/local_installer.rs b/harness/service-cli/tests/local_installer.rs index ec2ea638..940a4e58 100644 --- a/harness/service-cli/tests/local_installer.rs +++ b/harness/service-cli/tests/local_installer.rs @@ -451,3 +451,140 @@ fn failed_original_release_receipt_is_cleanup_only_and_rejects_drift() { assert_eq!(f.installs(), 1); assert_eq!(parsed(&f.destroy())["phase"], "destroyed"); } + +fn feature( + f: &Fixture, + name: &str, + action: &str, + options: &[&str], + env: &[(&str, &str)], +) -> Output { + let mut args = vec![name, action, "--state", f.state.to_str().unwrap()]; + args.extend_from_slice(options); + Fixture::call(f.dir.path(), &args, env) +} +fn upgrades(f: &Fixture) -> usize { + fs::read_to_string(f.dir.path().join("calls")) + .unwrap() + .lines() + .filter(|l| { + let v: Value = serde_json::from_str(l).unwrap(); + v[0] == "helm" && v[1][0] == "upgrade" + }) + .count() +} +#[test] +fn feature_controls_preserve_values_disable_without_data_deletion_and_cleanup_latest_revision() { + let f = Fixture::new(); + parsed(&f.apply(&[], false)); + let image = format!("ui@sha256:{}", "b".repeat(64)); + let enabled = parsed(&feature( + &f, + "management-ui", + "enable", + &["--image", &image], + &[], + )); + assert_eq!(enabled["managementUI"]["enabled"], true); + assert_eq!( + enabled["managementUI"]["privatePreview"]["url"], + "https://localhost:8444" + ); + assert_eq!(enabled["observability"]["enabled"], false); + let obs = parsed(&feature( + &f, + "observability", + "enable", + &["--retention-days", "14"], + &[], + )); + assert_eq!(obs["observability"]["configuration"]["retention_days"], 14); + assert_eq!(obs["managementUI"]["enabled"], true); + let count = upgrades(&f); + parsed(&feature( + &f, + "observability", + "enable", + &["--retention-days", "14"], + &[], + )); + parsed(&feature(&f, "management-ui", "status", &[], &[])); + assert_eq!(upgrades(&f), count); + parsed(&feature(&f, "management-ui", "disable", &[], &[])); + let disabled = parsed(&feature(&f, "observability", "disable", &[], &[])); + assert_eq!(disabled["observability"]["enabled"], false); + assert_eq!(disabled["managementUI"]["enabled"], false); + let v = &f.remote()["releases"]["ow-service-m4-installer"]["ows"]["values"]; + assert_eq!(v["bootstrapEnabled"], false); + assert_eq!(v["image"], format!("image@sha256:{}", "a".repeat(64))); + assert!(v["config"].get("observability").is_none()); + assert_eq!(f.installs(), 1); + // A normal installer resume and teardown still recognize the updated release. + parsed(&f.apply(&[], false)); + parsed(&f.destroy()); +} +#[test] +fn feature_upgrade_lost_response_is_observed_without_a_second_upgrade() { + let f = Fixture::new(); + parsed(&f.apply(&[], false)); + let out = feature( + &f, + "observability", + "enable", + &[], + &[("FAIL_UPGRADE_AFTER", "yes")], + ); + assert_eq!(parsed(&out)["observability"]["enabled"], true); + parsed(&feature(&f, "observability", "status", &[], &[])); + assert_eq!(upgrades(&f), 1); +} +#[test] +fn unknown_feature_outcome_is_not_replayed_and_blocks_cleanup() { + let f = Fixture::new(); + parsed(&f.apply(&[], false)); + assert!(!feature( + &f, + "observability", + "enable", + &[], + &[("FAIL_UPGRADE_BEFORE", "yes")] + ) + .status + .success()); + for action in ["enable", "status", "disable"] { + assert!(!feature(&f, "observability", action, &[], &[]) + .status + .success()); + } + assert!(!f.destroy().status.success()); + assert_eq!(upgrades(&f), 1); +} +#[test] +fn feature_controls_reject_replaced_release_or_namespace_and_mutable_images() { + let f = Fixture::new(); + parsed(&f.apply(&[], false)); + assert!(!feature( + &f, + "management-ui", + "enable", + &["--image", "ui:latest"], + &[] + ) + .status + .success()); + assert_eq!(upgrades(&f), 0); + let original = f.remote(); + let mut changed = original.clone(); + changed["releases"]["ow-service-m4-installer"]["ows"]["version"] = json!(8); + f.set_remote(&changed); + assert!(!feature(&f, "observability", "enable", &[], &[]) + .status + .success()); + let mut changed = original; + changed["namespaces"]["ow-service-m4-installer"]["metadata"]["uid"] = json!("replacement"); + f.set_remote(&changed); + assert!(!feature(&f, "observability", "enable", &[], &[]) + .status + .success()); + assert_eq!(upgrades(&f), 0); +} From 9fda56b2346e0fa2321c6fa356ac76585c6bb95b Mon Sep 17 00:00:00 2001 From: Rohit P Date: Wed, 11 Nov 2026 11:25:17 -0800 Subject: [PATCH 02/11] Add Azure AWS and local cluster configuration examples --- deploy/service/chart.test.mjs | 7 + deploy/service/examples/README.md | 131 ++++++++++++++++++ deploy/service/examples/aws/deployment.yaml | 30 ++++ deploy/service/examples/aws/setup.json | 37 +++++ deploy/service/examples/azure/deployment.yaml | 29 ++++ deploy/service/examples/azure/setup.json | 30 ++++ .../installation-secrets.example.json | 14 ++ .../examples/local-cluster/installer.json | 19 +++ .../local-cluster/service-values.json | 63 +++++++++ deploy/service/up/README.md | 4 +- deploy/service/up/aws.yaml | 28 ++-- deploy/service/up/azure.yaml | 28 ++-- harness/service-cli/README.md | 4 + harness/service-cli/src/deployment_local.rs | 30 ++++ .../service-cli/src/deployment_up_config.rs | 32 +++++ 15 files changed, 466 insertions(+), 20 deletions(-) create mode 100644 deploy/service/examples/README.md create mode 100644 deploy/service/examples/aws/deployment.yaml create mode 100644 deploy/service/examples/aws/setup.json create mode 100644 deploy/service/examples/azure/deployment.yaml create mode 100644 deploy/service/examples/azure/setup.json create mode 100644 deploy/service/examples/local-cluster/installation-secrets.example.json create mode 100644 deploy/service/examples/local-cluster/installer.json create mode 100644 deploy/service/examples/local-cluster/service-values.json diff --git a/deploy/service/chart.test.mjs b/deploy/service/chart.test.mjs index 923f65e7..a179f86a 100644 --- a/deploy/service/chart.test.mjs +++ b/deploy/service/chart.test.mjs @@ -255,3 +255,10 @@ test("private management UI preview terminates TLS without giving the UI server assert.match(out,/name: https, port: 443, targetPort: https/); assert.match(out,/proxy_set_header Host \$http_host/); }); +test("local cluster reference renders with management UI and trace storage",async()=>{ + const out=await run("helm",["template","ows",resolve("deploy/service/chart"),"--namespace","ow-service-m4-local","-f",resolve("deploy/service/examples/local-cluster/service-values.json")]); + assert.match(out,/name: ows-ui/); + assert.match(out,/"retention_days": 14/); + assert.match(out,/args: \[recording-migrate\]/); + assert.match(out,/name: https, port: 443, targetPort: https/); +}); diff --git a/deploy/service/examples/README.md b/deploy/service/examples/README.md new file mode 100644 index 00000000..0042055b --- /dev/null +++ b/deploy/service/examples/README.md @@ -0,0 +1,131 @@ +# Deployment examples + +Reference files for the configuration formats accepted by `ows`. The cloud +examples enable the management UI and trace storage, select two fixed-size nodes, +and use a six-hour disposable trial with a US$100 planning limit. + +These files contain deliberately fake identities, paths, versions and image +references. Replace them and complete preflight before approving provisioning. +No credentials are embedded. A planning limit is not a hard cloud spending cap. +Trial expiry deletes workload data; use these examples only for disposable data. + +| Target | Application/deployment choices | Installer/account choices | Command | +| --- | --- | --- | --- | +| Azure | [azure/deployment.yaml](azure/deployment.yaml) | [azure/setup.json](azure/setup.json) | `ows deploy azure up --config azure/deployment.yaml --keep-access` | +| AWS | [aws/deployment.yaml](aws/deployment.yaml) | [aws/setup.json](aws/setup.json) | `ows deploy aws up --config aws/deployment.yaml --keep-access` | +| Existing local kind cluster | [local-cluster/service-values.json](local-cluster/service-values.json) | [local-cluster/installer.json](local-cluster/installer.json) | `ows deploy local plan --config local-cluster/installer.json --out .ows/local` | + +Commands assume you copied this directory and run from its root. Neither cloud +CLI login nor this example folder replaces the required provisioning permissions. +No `ows deploy local up` command exists yet; the local installer targets an existing +kind cluster, OpenShell release and staged PostgreSQL/TLS fixtures. It does not +create a Kubernetes cluster or database for you. + +## Azure + +1. In `deployment.yaml`, set your subscription, region and desired node count/size. + `Standard_D4as_v7` is a supported reference choice, subject to regional + availability, subscription restrictions and quota checks. +2. In `setup.json`, use the same subscription (`account`) and region. Set + `setupObjectId` to the object ID of your signed-in deployment identity in the + subscription's Entra tenant; it is not an application/client ID. `User` is for + interactive Azure CLI login; an approved service principal uses + `ServicePrincipal` and its own object ID. +3. Supply the existing workload/control resource groups, Entra administrator group, + SSH public key (no comment), versioned Key Vault password reference and a + digest-pinned cleanup image in the independent control registry. Replace the + example secret version, image digest, available AKS version and Ubuntu image + version. The `203.0.113.10` operator IPv4 is a documentation address: replace it + with the authorized operator's real IPv4. The SSH private key stays with you; + select it using `advanced.azure.sshPrivateKeyFile` in deployment YAML or your + SSH agent. Key Vault needs RBAC and ARM template deployment access. +4. Point `release` to the absolute path of the prepared **Linux AMD64** + `release.json`, including a pinned `service-ui` image. + +The resource groups, vault/secret, administrator group and independent cleanup +image are one-time prerequisites in the current installer. The reviewed bootstrap +handles its supported identity/grant creation; this example does not mean every +Azure prerequisite is created automatically. See the [installer prerequisites](../up/README.md#one-time-prerequisites). + +## AWS + +1. Set your account, region and authenticated AWS CLI profile in `deployment.yaml`. +2. Use the matching account/region and approved installer role in `setup.json`. +3. Replace the availability zones and verify the pinned Kubernetes, node, database + and add-on versions against your region. These are reference pins from the + supported infrastructure format, not a claim of current regional availability. +4. Replace `guardCode` with the actual versioned S3 object and checksum of this + release's packaged cleanup guard. Replace `operatorAccess.amiId` with a reviewed + regional ARM64 AMI supporting the SSM operator bridge; the example AMI is fake. + The bridge keeps the Kubernetes management API private. Install the AWS Session + Manager plugin for this mode. +5. Point `release` to the absolute path of a prepared **Linux ARM64** `release.json` + that includes a pinned `service-ui` image. + +Both clouds use `advanced.setupProfile: ./setup.json`. That path is resolved +relative to `deployment.yaml`. Paths **inside** the profile/release files must be +absolute with the current loader. The setup profile is an OpenWorker configuration +file, not another cloud login or a place to store access tokens. + +Export your model key in your shell before deployment; the examples reference +`ANTHROPIC_API_KEY` without embedding its value. To deploy without a model key, +remove `models` and configure the provider later. Installation readiness does not +perform an actual model call or create an agent. + +Run a read-only review first, using the same arguments with `--plan-only`. Remove +that flag to proceed through the CLI's scoped approval gates. The command saves +`connection.json` with the API connection, UI preview URL, certificate and +port-forward instructions. Keep the private route open using `--keep-access`. +Register/activate a RuntimeProfile and agent release, then submit your agent using +the service SDK. + +The samples create disposable trials. For persistent deployments omit `trial` and +review a realistic monthly planning budget and a separate data-retention/teardown +policy. The current destructive teardown route refuses persistent data. + +## Existing local kind cluster + +Edit both JSON files together: + +- Replace the absolute kubeconfig, chart and values paths, and select your explicit + `kind-*` context. The current local installer does not target Docker Desktop or + arbitrary private-cluster contexts directly. +- Replace **every** example service/runner/UI digest with an actual locally imported + or pullable image for the node architecture. The repeated `a` digest is fake. +- Use a dedicated service namespace; keep its current `ow-service-m4-` prefix for + this local installer. Match it in `managementUI.serviceUrl`. +- Update the existing OpenShell namespace, release and fullname together with + `config.openshell.endpoint` / `tls_server_name`. Its client TLS Secret must be + available to the trusted installer. +- Stage the fixture PostgreSQL deployment (named `postgres`) and TLS/model Secrets + in the installer's owned namespace. `fixtureDatabase: true` identifies this local + test database; it does not install one or disable database TLS verification. + `ca.crt` in the API TLS Secret must trust the service certificate, whose SANs + include localhost and the configured API Service DNS name. + +The existing lower-level sequence is: + +```sh +ows deploy local plan --config local-cluster/installer.json --out .ows/local +ows deploy local apply --state .ows/local --accept-plan PLAN_DIGEST --openshell-only +# Stage the disposable PostgreSQL fixture in the newly owned namespace. +ows deploy local secrets --state .ows/local --accept-plan PLAN_DIGEST \ + --input /absolute/private/installation-secrets.json +ows deploy local apply --state .ows/local --accept-plan PLAN_DIGEST +ows management-ui status --state .ows/local +ows observability status --state .ows/local +``` + +Use the actual `planDigest` printed by `plan`. Copy [the placeholder secrets-file shape](local-cluster/installation-secrets.example.json) +to a private location outside your checkout, restrict it to mode0600, and fill in +the actual certificate/key/model values there. Never commit the filled file. The +installer reads OpenShell client TLS from the existing gateway Secret itself; +those credentials are not part of this file. See the [shared installer CLI](../../../harness/service-cli/README.md) +for command context. +The existing reference scripts are `deploy/service/prepare-local-install.mjs` and +`deploy/service/kubernetes-local.mjs`; inspect their required paths before use. +They are qualification helpers, not a standalone local-cluster bootstrap command. + +A local cluster has no cloud node count or cloud budget in its installer JSON: +capacity belongs to the existing cluster. UI/storage toggles use the same +`ows management-ui` and `ows observability` commands as cloud installations. diff --git a/deploy/service/examples/aws/deployment.yaml b/deploy/service/examples/aws/deployment.yaml new file mode 100644 index 00000000..6c3652e4 --- /dev/null +++ b/deploy/service/examples/aws/deployment.yaml @@ -0,0 +1,30 @@ +# Replace the account ID and choose an authenticated AWS CLI profile. +apiVersion: ows/v1alpha1 +name: aws-agents +cloud: + provider: aws + accountId: "123456789012" + region: us-west-2 + profile: default +nodes: + count: 2 + instanceType: t4g.large +# Disposable six-hour trial. Expiry deletes workload data; keep no valuable data here. +trial: + expiresAfterHours: 6 + deleteData: true +# Planning limit, not a hard billing cap. Set cloud billing alerts separately. +budget: + limitUSD: 100 +managementUI: + enabled: true + publicOrigin: https://localhost:8444 +observability: + enabled: true + retentionDays: 14 +# Export this variable in your shell; never put a key in this file. +models: + anthropic: {env: ANTHROPIC_API_KEY} +# This is an OpenWorker settings file, separate from your cloud CLI login. +advanced: + setupProfile: ./setup.json diff --git a/deploy/service/examples/aws/setup.json b/deploy/service/examples/aws/setup.json new file mode 100644 index 00000000..74eb4cf8 --- /dev/null +++ b/deploy/service/examples/aws/setup.json @@ -0,0 +1,37 @@ +{ + "schemaVersion": 1, + "provider": "aws", + "account": "123456789012", + "region": "us-west-2", + "operatorArn": "arn:aws:iam::123456789012:role/REPLACE_WITH_APPROVED_INSTALLER", + "infrastructure": { + "schemaVersion": 1, + "availabilityZones": [ + "us-west-2a", + "us-west-2b" + ], + "vpcCidr": "10.94.0.0/16", + "kubernetesVersion": "1.36", + "nodeReleaseVersion": "1.36.4-20261007", + "postgresVersion": "16.13", + "addons": { + "vpcCni": "v1.21.1-eksbuild.1", + "kubeProxy": "v1.36.4-eksbuild.1", + "coreDns": "v1.13.1-eksbuild.1", + "podIdentity": "v1.3.8-eksbuild.1", + "ebsCsi": "v1.48.0-eksbuild.1" + }, + "guardCode": { + "bucket": "example-ows-qualified-artifacts", + "key": "guard/cleanup.zip", + "versionId": "EXAMPLE-not-an-uploaded-object", + "sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + }, + "acknowledgeExperimentDataDeletion": true, + "operatorAccess": { + "mode": "ssm", + "amiId": "ami-0123456789abcdef0" + } + }, + "release": "/absolute/path/to/arm64-release/release.json" +} diff --git a/deploy/service/examples/azure/deployment.yaml b/deploy/service/examples/azure/deployment.yaml new file mode 100644 index 00000000..e9d24690 --- /dev/null +++ b/deploy/service/examples/azure/deployment.yaml @@ -0,0 +1,29 @@ +# Replace the subscription UUID. The setup profile must match it and this region. +apiVersion: ows/v1alpha1 +name: azure-agents +cloud: + provider: azure + subscriptionId: "11111111-2222-4333-8444-555555555555" + region: westus2 +nodes: + count: 2 + instanceType: Standard_D4as_v7 +# Disposable six-hour trial. Expiry deletes workload data; keep no valuable data here. +trial: + expiresAfterHours: 6 + deleteData: true +# Planning limit, not a hard billing cap. Set cloud billing alerts separately. +budget: + limitUSD: 100 +managementUI: + enabled: true + publicOrigin: https://localhost:8444 +observability: + enabled: true + retentionDays: 14 +# Export this variable in your shell; never put a key in this file. +models: + anthropic: {env: ANTHROPIC_API_KEY} +# This is an OpenWorker settings file, separate from your cloud CLI login. +advanced: + setupProfile: ./setup.json diff --git a/deploy/service/examples/azure/setup.json b/deploy/service/examples/azure/setup.json new file mode 100644 index 00000000..b933e4a0 --- /dev/null +++ b/deploy/service/examples/azure/setup.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "provider": "azure", + "account": "11111111-2222-4333-8444-555555555555", + "region": "westus2", + "setupObjectId": "22222222-3333-4444-8555-666666666666", + "setupPrincipalType": "User", + "resourceGroup": "rg-ow-example-workload", + "infrastructure": { + "kubernetesVersion": "REPLACE_WITH_AVAILABLE_AKS_VERSION", + "nodeVmSize": "Standard_D4as_v7", + "sshPublicKey": "REPLACE_WITH_SSH_PUBLIC_KEY_WITHOUT_COMMENT", + "adminGroupObjectIds": [ + "33333333-4444-4555-8666-777777777777" + ], + "controlResourceGroup": "rg-ow-example-control", + "guardImage": "examplecontrol.azurecr.io/cleanup@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "guardRegistryResourceId": "/subscriptions/11111111-2222-4333-8444-555555555555/resourceGroups/rg-ow-example-control/providers/Microsoft.ContainerRegistry/registries/examplecontrol", + "managementAccess": { + "operatorIpv4": "203.0.113.10", + "ubuntuImageVersion": "REPLACE_WITH_PINNED_UBUNTU_IMAGE_VERSION" + }, + "postgresPassword": { + "keyVaultId": "/subscriptions/11111111-2222-4333-8444-555555555555/resourceGroups/rg-ow-example-control/providers/Microsoft.KeyVault/vaults/ow-example-vault", + "secretName": "postgres-password", + "secretVersion": "0123456789abcdef0123456789abcdef" + } + }, + "release": "/absolute/path/to/amd64-release/release.json" +} diff --git a/deploy/service/examples/local-cluster/installation-secrets.example.json b/deploy/service/examples/local-cluster/installation-secrets.example.json new file mode 100644 index 00000000..533147f6 --- /dev/null +++ b/deploy/service/examples/local-cluster/installation-secrets.example.json @@ -0,0 +1,14 @@ +{ + "database": { + "database-url": "postgresql://openworker:REPLACE_WITH_FIXTURE_PASSWORD@postgres:5432/openworker", + "pg-ca.crt": "REPLACE_WITH_POSTGRES_CA_PEM" + }, + "apiTLS": { + "tls.crt": "REPLACE_WITH_API_CERTIFICATE_PEM", + "tls.key": "REPLACE_WITH_API_PRIVATE_KEY_PEM", + "ca.crt": "REPLACE_WITH_API_CA_PEM" + }, + "model": { + "anthropic": "REPLACE_WITH_MODEL_KEY" + } +} diff --git a/deploy/service/examples/local-cluster/installer.json b/deploy/service/examples/local-cluster/installer.json new file mode 100644 index 00000000..2edf508d --- /dev/null +++ b/deploy/service/examples/local-cluster/installer.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 1, + "kubeconfig": "/absolute/path/to/local.kubeconfig", + "context": "kind-openworker", + "helm": "helm", + "service": { + "namespace": "ow-service-m4-local", + "release": "ows", + "chart": "/absolute/path/to/openworker/deploy/service/chart", + "valuesFile": "/absolute/path/to/openworker/deploy/service/examples/local-cluster/service-values.json", + "fixtureDatabase": true + }, + "openshell": { + "mode": "existing", + "namespace": "ow-helm-shell", + "release": "ow-shell", + "fullname": "ow-helm-gateway" + } +} diff --git a/deploy/service/examples/local-cluster/service-values.json b/deploy/service/examples/local-cluster/service-values.json new file mode 100644 index 00000000..d67f946f --- /dev/null +++ b/deploy/service/examples/local-cluster/service-values.json @@ -0,0 +1,63 @@ +{ + "image": "registry.example.com/openworker/service@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "secrets": { + "model": "ow-service-model" + }, + "worker": { + "replicas": 1, + "ownership": "manual", + "maxReplicas": 2, + "slots": 2, + "drainSeconds": 30 + }, + "terminationGracePeriodSeconds": 90, + "managementUI": { + "enabled": true, + "image": "registry.example.com/openworker/ui@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "publicOrigin": "https://localhost:8444", + "serviceUrl": "https://ows-api.ow-service-m4-local.svc", + "caSecret": "ow-service-tls", + "caKey": "ca.crt", + "tlsSecret": "ow-service-tls" + }, + "config": { + "service": { + "id": "ows-local-example", + "namespace": "local", + "agent": "installed-tools", + "legacy_compatibility": false + }, + "database_env": "OW_PG_URL", + "schema": "ow_service", + "database_loopback_plaintext": false, + "database_ca": "/run/service/pg-ca.crt", + "state_root": "/state", + "bundle": "/opt/openworker/bundle", + "runner": "/opt/openworker/openworker", + "model": "claude-sonnet-4-6", + "scripted": false, + "model_provider": "anthropic", + "model_connections": [ + { + "id": "primary", + "provider": "anthropic", + "identity": "local-primary", + "key_env": "ANTHROPIC_API_KEY" + } + ], + "api_token_file": "/bootstrap/admin-token", + "actor": "deployment-admin", + "openshell": { + "identity": "ows-local-example", + "endpoint": "https://ow-helm-gateway.ow-helm-shell.svc.cluster.local:8080", + "tls_server_name": "ow-helm-gateway.ow-helm-shell.svc.cluster.local", + "certificates": "/openshell", + "workspace": "default", + "image": "registry.example.com/openworker/runner@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "observability": { + "schema": "ow_observability", + "retention_days": 14 + } + } +} diff --git a/deploy/service/up/README.md b/deploy/service/up/README.md index eda5a302..b1006c37 100644 --- a/deploy/service/up/README.md +++ b/deploy/service/up/README.md @@ -2,7 +2,7 @@ This experimental coordinator installs a private service cluster, OpenShell and the OpenWorker service using a reviewed account setup profile. It resumes the same deployment when rerun. Fresh AWS and Azure installs with this coordinator still require live qualification; do not treat local tests as a production certificate. -Once the prerequisites below are installed, copy `aws.yaml` or `azure.yaml` to `deployment.yaml`, set your account/subscription and region, then run: +Complete commented examples, including companion setup profiles and an existing local kind cluster, are in [deployment examples](../examples/README.md). Copy the selected cloud folder, replace its documented placeholders and prepare the reviewed release bundle before running: ```sh owcluster deploy aws up --config deployment.yaml @@ -41,7 +41,7 @@ The profile contains **references and version pins**, not passwords: Replace `infrastructure` with the reviewed native AWS input described in [`harness/service-cli/examples/README.md`](../../../harness/service-cli/examples/README.md). It includes zone/version pins and a versioned S3 reference to **this CLI release's packaged cleanup guard**. For Azure, use the existing Azure infrastructure input (Kubernetes version, supported VM size, SSH public key, administrator group IDs, control group, pinned external cleanup image and Key Vault secret reference). Its profile uses `provider: azure`, the subscription as `account`, and `resourceGroup`, `setupObjectId`, `setupPrincipalType` instead of `operatorArn`. Package the matching native cleanup image before running the installer. Existing control assets are retained, not silently taken over or deleted. -This release does not assume public OpenWorker service images or a published Helm chart exist. Prepare a reviewed release manifest with `package.py --help`: it copies a chart, pinned OpenShell archive and values, platform manifest and database CA bundle, then hashes them. Supply immutable harness/runner image digests for Linux ARM64 on AWS or AMD64 on Azure. The supported OpenShell archive is 0.1.7. The CA bundle must contain 1–32 certificates. The installer publishes the selected images into its owned ECR/ACR registry. An optional UI image can be packaged, but browser ingress/UI exposure is not enabled by this initial coordinator. +This release does not assume public OpenWorker service images or a published Helm chart exist. Prepare a reviewed release manifest with `package.py --help`: it copies a chart, pinned OpenShell archive and values, platform manifest and database CA bundle, then hashes them. Supply immutable harness/runner image digests for Linux ARM64 on AWS or AMD64 on Azure. The supported OpenShell archive is 0.1.7. The CA bundle must contain 1–32 certificates. The installer publishes the selected images into its owned ECR/ACR registry. Package a UI image when enabling `managementUI`. The CLI can configure a private HTTPS preview; public ingress remains operator-managed. ## Review, budget and resume diff --git a/deploy/service/up/aws.yaml b/deploy/service/up/aws.yaml index 2b29c14a..48a003c2 100644 --- a/deploy/service/up/aws.yaml +++ b/deploy/service/up/aws.yaml @@ -1,5 +1,7 @@ +# Full reference and companion setup.json: ../examples/aws/ +# Replace the account ID and choose an authenticated AWS CLI profile. apiVersion: ows/v1alpha1 -name: my-agents +name: aws-agents cloud: provider: aws accountId: "123456789012" @@ -7,12 +9,20 @@ cloud: profile: default nodes: count: 2 - requirements: - cpu: {min: 2, max: 4} - memoryGiB: {min: 8, max: 16} + instanceType: t4g.large +# Disposable six-hour trial. Expiry deletes workload data; keep no valuable data here. +trial: + expiresAfterHours: 6 + deleteData: true +# Planning limit, not a hard billing cap. Set cloud billing alerts separately. budget: - limitUSD: 1000 -# Omit trial to keep the deployment running. For disposable test data only: -# trial: {expiresAfterHours: 6, deleteData: true} -# models: -# anthropic: {env: ANTHROPIC_API_KEY} + limitUSD: 100 +managementUI: + enabled: true + publicOrigin: https://localhost:8444 +observability: + enabled: true + retentionDays: 14 +# Export this variable in your shell; never put a key in this file. +models: + anthropic: {env: ANTHROPIC_API_KEY} diff --git a/deploy/service/up/azure.yaml b/deploy/service/up/azure.yaml index 9d1be8e0..f4200de6 100644 --- a/deploy/service/up/azure.yaml +++ b/deploy/service/up/azure.yaml @@ -1,17 +1,27 @@ +# Full reference and companion setup.json: ../examples/azure/ +# Replace the subscription UUID. The setup profile must match it and this region. apiVersion: ows/v1alpha1 -name: my-agents +name: azure-agents cloud: provider: azure subscriptionId: "11111111-2222-4333-8444-555555555555" region: westus2 nodes: count: 2 - requirements: - cpu: {min: 4, max: 8} - memoryGiB: {min: 16, max: 32} + instanceType: Standard_D4as_v7 +# Disposable six-hour trial. Expiry deletes workload data; keep no valuable data here. +trial: + expiresAfterHours: 6 + deleteData: true +# Planning limit, not a hard billing cap. Set cloud billing alerts separately. budget: - limitUSD: 1000 -# Omit trial to keep the deployment running. For disposable test data only: -# trial: {expiresAfterHours: 6, deleteData: true} -# models: -# anthropic: {env: ANTHROPIC_API_KEY} + limitUSD: 100 +managementUI: + enabled: true + publicOrigin: https://localhost:8444 +observability: + enabled: true + retentionDays: 14 +# Export this variable in your shell; never put a key in this file. +models: + anthropic: {env: ANTHROPIC_API_KEY} diff --git a/harness/service-cli/README.md b/harness/service-cli/README.md index 4e1fb773..95fe0754 100644 --- a/harness/service-cli/README.md +++ b/harness/service-cli/README.md @@ -116,3 +116,7 @@ verified cluster identity. UI and trace storage are independent. Disabling traci retains data and leaves security audit and separately configured exporters intact. The CLI saves feature changes and reconciles lost Helm responses without repeating unknown upgrades. See [setup, UI image, private HTTPS and database options](../../deploy/service/up/README.md#management-ui-and-trace-storage). + +Complete [Azure, AWS and local kind reference files](../../deploy/service/examples/README.md) +include deployment choices, companion cloud setup profiles, local chart values and +placeholder secrets-file shape. Replace their documented placeholders before use. diff --git a/harness/service-cli/src/deployment_local.rs b/harness/service-cli/src/deployment_local.rs index c7ef2914..a6289aa3 100644 --- a/harness/service-cli/src/deployment_local.rs +++ b/harness/service-cli/src/deployment_local.rs @@ -1874,3 +1874,33 @@ pub(crate) async fn verify_azure_teardown( } Ok(()) } + +#[cfg(test)] +mod reference_examples { + use super::*; + #[test] + fn local_reference_uses_supported_existing_kind_installer_and_valid_chart_values() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../deploy/service/examples/local-cluster"); + let c: Config = + serde_json::from_slice(&fs::read(root.join("installer.json")).unwrap()).unwrap(); + assert!(c.context.starts_with("kind-") && c.service.fixture_database); + let v: Value = + serde_json::from_slice(&fs::read(root.join("service-values.json")).unwrap()).unwrap(); + let (ns, _, fullname) = c.openshell.scope(); + assert_eq!( + v["config"]["openshell"]["endpoint"], + format!("https://{fullname}.{ns}.svc.cluster.local:8080") + ); + assert_eq!( + v["managementUI"]["serviceUrl"], + format!( + "https://{}-api.{}.svc", + c.service.release, c.service.namespace + ) + ); + assert!(pinned(v["image"].as_str().unwrap())); + assert_eq!(v["managementUI"]["enabled"], true); + assert_eq!(v["config"]["observability"]["retention_days"], 14); + } +} diff --git a/harness/service-cli/src/deployment_up_config.rs b/harness/service-cli/src/deployment_up_config.rs index bfe0d7f9..d0b7898e 100644 --- a/harness/service-cli/src/deployment_up_config.rs +++ b/harness/service-cli/src/deployment_up_config.rs @@ -391,3 +391,35 @@ mod feature_config_tests { .is_err()); } } + +#[cfg(test)] +mod reference_examples { + use super::*; + #[test] + fn cloud_reference_files_match_cli_schema_and_account_profiles() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../deploy/service/examples"); + for provider in ["azure", "aws"] { + let c = Config::load(&root.join(provider).join("deployment.yaml"), provider).unwrap(); + assert_eq!(c.budget.alert_usd, 100); + assert_eq!(c.trial.as_ref().unwrap().expires_after_hours, 6); + assert!( + c.management_ui.as_ref().unwrap().enabled + && c.observability.as_ref().unwrap().enabled + ); + let setup: super::super::package::Setup = serde_json::from_slice( + &fs::read(c.advanced.setup_profile.as_ref().unwrap()).unwrap(), + ) + .unwrap(); + assert_eq!(setup.provider, provider); + assert_eq!(setup.account, c.cloud.account()); + assert_eq!(setup.region, c.cloud.region()); + if provider == "azure" { + let _: super::super::super::azure::Inputs = + serde_json::from_value(setup.infrastructure).unwrap(); + } else { + let _: super::super::super::infrastructure::Inputs = + serde_json::from_value(setup.infrastructure).unwrap(); + } + } + } +} From 9cd45522aa91666ba196381d907f746df9970b78 Mon Sep 17 00:00:00 2001 From: Rohit P Date: Wed, 11 Nov 2026 16:46:58 -0800 Subject: [PATCH 03/11] Simplify deployment configuration and qualify local Kubernetes --- deploy/service/chart.test.mjs | 7 - deploy/service/examples/README.md | 212 ++++------ deploy/service/examples/aws/deployment.yaml | 59 ++- deploy/service/examples/aws/setup.json | 37 -- deploy/service/examples/azure/deployment.yaml | 51 ++- deploy/service/examples/azure/setup.json | 30 -- .../installation-secrets.example.json | 14 - .../examples/local-cluster/installer.json | 19 - .../local-cluster/service-values.json | 63 --- .../examples/local-k8s/deployment.yaml | 34 ++ deploy/service/up/README.md | 275 ++++++------ deploy/service/up/aws.yaml | 59 ++- deploy/service/up/azure.yaml | 51 ++- deploy/service/up/test-plan.py | 16 +- harness/service-cli/README.md | 9 +- harness/service-cli/src/deployment.rs | 55 ++- .../service-cli/src/deployment_features.rs | 78 +++- .../src/deployment_infrastructure.rs | 8 +- .../service-cli/src/deployment_kubernetes.rs | 397 ++++++++++++++++++ harness/service-cli/src/deployment_local.rs | 32 +- .../src/deployment_preflight_checks.rs | 9 +- harness/service-cli/src/deployment_up.rs | 23 +- .../service-cli/src/deployment_up_config.rs | 38 +- harness/service-cli/src/deployment_up_cost.rs | 165 ++++++++ .../service-cli/src/deployment_up_install.rs | 4 +- .../service-cli/src/deployment_up_native.rs | 8 +- .../service-cli/src/deployment_up_package.rs | 41 +- harness/service-cli/tests/deployment_up.rs | 39 +- harness/service-cli/tests/gcp.rs | 14 +- 29 files changed, 1234 insertions(+), 613 deletions(-) delete mode 100644 deploy/service/examples/aws/setup.json delete mode 100644 deploy/service/examples/azure/setup.json delete mode 100644 deploy/service/examples/local-cluster/installation-secrets.example.json delete mode 100644 deploy/service/examples/local-cluster/installer.json delete mode 100644 deploy/service/examples/local-cluster/service-values.json create mode 100644 deploy/service/examples/local-k8s/deployment.yaml create mode 100644 harness/service-cli/src/deployment_kubernetes.rs create mode 100644 harness/service-cli/src/deployment_up_cost.rs diff --git a/deploy/service/chart.test.mjs b/deploy/service/chart.test.mjs index a179f86a..923f65e7 100644 --- a/deploy/service/chart.test.mjs +++ b/deploy/service/chart.test.mjs @@ -255,10 +255,3 @@ test("private management UI preview terminates TLS without giving the UI server assert.match(out,/name: https, port: 443, targetPort: https/); assert.match(out,/proxy_set_header Host \$http_host/); }); -test("local cluster reference renders with management UI and trace storage",async()=>{ - const out=await run("helm",["template","ows",resolve("deploy/service/chart"),"--namespace","ow-service-m4-local","-f",resolve("deploy/service/examples/local-cluster/service-values.json")]); - assert.match(out,/name: ows-ui/); - assert.match(out,/"retention_days": 14/); - assert.match(out,/args: \[recording-migrate\]/); - assert.match(out,/name: https, port: 443, targetPort: https/); -}); diff --git a/deploy/service/examples/README.md b/deploy/service/examples/README.md index 0042055b..6f6cb2a6 100644 --- a/deploy/service/examples/README.md +++ b/deploy/service/examples/README.md @@ -1,131 +1,99 @@ -# Deployment examples +# Deployment reference files -Reference files for the configuration formats accepted by `ows`. The cloud -examples enable the management UI and trace storage, select two fixed-size nodes, -and use a six-hour disposable trial with a US$100 planning limit. +Use one authored YAML per environment: `staging-deploy.yaml`, `prod-deploy.yaml`, +or any filename. Account/infrastructure settings live under `advanced`; no separate +setup-profile or local values/installer JSON is required. The CLI generates its +own private plan, values, credentials and recovery files. Never discard those to +retry an uncertain operation. -These files contain deliberately fake identities, paths, versions and image -references. Replace them and complete preflight before approving provisioning. -No credentials are embedded. A planning limit is not a hard cloud spending cap. -Trial expiry deletes workload data; use these examples only for disposable data. +| Mode | Reference | Current status | +| --- | --- | --- | +| Azure | [azure/deployment.yaml](azure/deployment.yaml) | Experimental coordinator; fresh combined cloud qualification pending | +| AWS | [aws/deployment.yaml](aws/deployment.yaml) | Experimental coordinator; fresh combined cloud qualification pending | +| local-k8s | [local-k8s/deployment.yaml](local-k8s/deployment.yaml) | Existing local kind cluster; reuse journaled installer | +| local-docker | Existing `deploy/service/stack.mjs` / Compose workflow | One-YAML CLI orchestration is roadmap; command refuses effects | +| remote-k8s | Existing cluster + explicit kubeconfig/network route required | Roadmap until implemented and tested on a real remote cluster | -| Target | Application/deployment choices | Installer/account choices | Command | -| --- | --- | --- | --- | -| Azure | [azure/deployment.yaml](azure/deployment.yaml) | [azure/setup.json](azure/setup.json) | `ows deploy azure up --config azure/deployment.yaml --keep-access` | -| AWS | [aws/deployment.yaml](aws/deployment.yaml) | [aws/setup.json](aws/setup.json) | `ows deploy aws up --config aws/deployment.yaml --keep-access` | -| Existing local kind cluster | [local-cluster/service-values.json](local-cluster/service-values.json) | [local-cluster/installer.json](local-cluster/installer.json) | `ows deploy local plan --config local-cluster/installer.json --out .ows/local` | +Reference identities, images, versions and paths are placeholders. Replace them. +Examples do not authorize provisioning, provide real image digests or promise a +spending cap. They omit `trial`: no automatic expiry by default. For disposable +data only, optionally add `trial: {expiresAfterHours: 6, deleteData: true}`. -Commands assume you copied this directory and run from its root. Neither cloud -CLI login nor this example folder replaces the required provisioning permissions. -No `ows deploy local up` command exists yet; the local installer targets an existing -kind cluster, OpenShell release and staged PostgreSQL/TLS fixtures. It does not -create a Kubernetes cluster or database for you. - -## Azure - -1. In `deployment.yaml`, set your subscription, region and desired node count/size. - `Standard_D4as_v7` is a supported reference choice, subject to regional - availability, subscription restrictions and quota checks. -2. In `setup.json`, use the same subscription (`account`) and region. Set - `setupObjectId` to the object ID of your signed-in deployment identity in the - subscription's Entra tenant; it is not an application/client ID. `User` is for - interactive Azure CLI login; an approved service principal uses - `ServicePrincipal` and its own object ID. -3. Supply the existing workload/control resource groups, Entra administrator group, - SSH public key (no comment), versioned Key Vault password reference and a - digest-pinned cleanup image in the independent control registry. Replace the - example secret version, image digest, available AKS version and Ubuntu image - version. The `203.0.113.10` operator IPv4 is a documentation address: replace it - with the authorized operator's real IPv4. The SSH private key stays with you; - select it using `advanced.azure.sshPrivateKeyFile` in deployment YAML or your - SSH agent. Key Vault needs RBAC and ARM template deployment access. -4. Point `release` to the absolute path of the prepared **Linux AMD64** - `release.json`, including a pinned `service-ui` image. - -The resource groups, vault/secret, administrator group and independent cleanup -image are one-time prerequisites in the current installer. The reviewed bootstrap -handles its supported identity/grant creation; this example does not mean every -Azure prerequisite is created automatically. See the [installer prerequisites](../up/README.md#one-time-prerequisites). - -## AWS - -1. Set your account, region and authenticated AWS CLI profile in `deployment.yaml`. -2. Use the matching account/region and approved installer role in `setup.json`. -3. Replace the availability zones and verify the pinned Kubernetes, node, database - and add-on versions against your region. These are reference pins from the - supported infrastructure format, not a claim of current regional availability. -4. Replace `guardCode` with the actual versioned S3 object and checksum of this - release's packaged cleanup guard. Replace `operatorAccess.amiId` with a reviewed - regional ARM64 AMI supporting the SSM operator bridge; the example AMI is fake. - The bridge keeps the Kubernetes management API private. Install the AWS Session - Manager plugin for this mode. -5. Point `release` to the absolute path of a prepared **Linux ARM64** `release.json` - that includes a pinned `service-ui` image. - -Both clouds use `advanced.setupProfile: ./setup.json`. That path is resolved -relative to `deployment.yaml`. Paths **inside** the profile/release files must be -absolute with the current loader. The setup profile is an OpenWorker configuration -file, not another cloud login or a place to store access tokens. - -Export your model key in your shell before deployment; the examples reference -`ANTHROPIC_API_KEY` without embedding its value. To deploy without a model key, -remove `models` and configure the provider later. Installation readiness does not -perform an actual model call or create an agent. - -Run a read-only review first, using the same arguments with `--plan-only`. Remove -that flag to proceed through the CLI's scoped approval gates. The command saves -`connection.json` with the API connection, UI preview URL, certificate and -port-forward instructions. Keep the private route open using `--keep-access`. -Register/activate a RuntimeProfile and agent release, then submit your agent using -the service SDK. - -The samples create disposable trials. For persistent deployments omit `trial` and -review a realistic monthly planning budget and a separate data-retention/teardown -policy. The current destructive teardown route refuses persistent data. - -## Existing local kind cluster - -Edit both JSON files together: - -- Replace the absolute kubeconfig, chart and values paths, and select your explicit - `kind-*` context. The current local installer does not target Docker Desktop or - arbitrary private-cluster contexts directly. -- Replace **every** example service/runner/UI digest with an actual locally imported - or pullable image for the node architecture. The repeated `a` digest is fake. -- Use a dedicated service namespace; keep its current `ow-service-m4-` prefix for - this local installer. Match it in `managementUI.serviceUrl`. -- Update the existing OpenShell namespace, release and fullname together with - `config.openshell.endpoint` / `tls_server_name`. Its client TLS Secret must be - available to the trusted installer. -- Stage the fixture PostgreSQL deployment (named `postgres`) and TLS/model Secrets - in the installer's owned namespace. `fixtureDatabase: true` identifies this local - test database; it does not install one or disable database TLS verification. - `ca.crt` in the API TLS Secret must trust the service certificate, whose SANs - include localhost and the configured API Service DNS name. - -The existing lower-level sequence is: +## Azure and AWS ```sh -ows deploy local plan --config local-cluster/installer.json --out .ows/local -ows deploy local apply --state .ows/local --accept-plan PLAN_DIGEST --openshell-only -# Stage the disposable PostgreSQL fixture in the newly owned namespace. -ows deploy local secrets --state .ows/local --accept-plan PLAN_DIGEST \ - --input /absolute/private/installation-secrets.json -ows deploy local apply --state .ows/local --accept-plan PLAN_DIGEST -ows management-ui status --state .ows/local -ows observability status --state .ows/local +ows deploy azure up --config azure/deployment.yaml --plan-only \ + --max-estimated-cost-usd 100 +ows deploy aws up --config aws/deployment.yaml --plan-only ``` -Use the actual `planDigest` printed by `plan`. Copy [the placeholder secrets-file shape](local-cluster/installation-secrets.example.json) -to a private location outside your checkout, restrict it to mode0600, and fill in -the actual certificate/key/model values there. Never commit the filled file. The -installer reads OpenShell client TLS from the existing gateway Secret itself; -those credentials are not part of this file. See the [shared installer CLI](../../../harness/service-cli/README.md) -for command context. -The existing reference scripts are `deploy/service/prepare-local-install.mjs` and -`deploy/service/kubernetes-local.mjs`; inspect their required paths before use. -They are qualification helpers, not a standalone local-cluster bootstrap command. +A cost ceiling is optional, is plan-only, and checks an estimate rather than actual +billing. It fails when modeled prices are unknown. Review the price timestamp, +assumptions and excluded variable costs. Omit `--plan-only` after reviewing the +provisioning/grant bundle. Noninteractive use requires its exact `--accept-plan` +and `--accept-bootstrap` digests. Add `--keep-access` for the private management +route and use the generated `connection.json` to open the UI. -A local cluster has no cloud node count or cloud budget in its installer JSON: -capacity belongs to the existing cluster. UI/storage toggles use the same -`ows management-ui` and `ows observability` commands as cloud installations. +Replace `advanced.infrastructure` pins and identity/resource references in the same +YAML. Azure needs its actual subscription, installer object ID (not app/client ID), +existing workload/control groups, Entra administrator group, SSH public key, +operator IPv4, AKS/Ubuntu versions and versioned Key Vault password reference. +The documentation IPv4 and image digests are fake. The Key Vault uses RBAC and ARM +deployment access. Workload/control prerequisites and the cleanup image must exist. +The private key stays local; select `advanced.azure.sshPrivateKeyFile` if needed. + +AWS needs its actual account/profile, installer ARN, region/availability zones, +reviewed versions/add-ons, versioned S3 cleanup package/checksum and regional ARM64 +AMI for the optional SSM bridge. Replace the fake AMI and cleanup object reference. +Use the matching reviewed identity scopes and current regional preflight. + +Set `advanced.release` to a generated release manifest from `up/package.py`: +ARM64 for AWS, AMD64 for Azure, including a UI image if enabled. Release manifests +are generated artifacts, not another hand-authored environment config. Versions +and package distribution remain subject to qualification; a CLI login alone is +not permission to provision. Export model keys using the named environment +variables or use file references. No secret values belong in deployment YAML. + +[Installer details](../up/README.md) explain current prerequisites, pricing scope, +trial data deletion, private access and mode limitations. + +## Existing local Kubernetes + +Both local and remote Kubernetes require an already running cluster, a working +kubeconfig/context, permissions and network access. Current `local-k8s` is qualified +only for explicit `kind-*` contexts and dedicated `ow-service-m4-*` namespaces. It +does not create Kubernetes from SSH machines or install a database. + +Edit the one local YAML: kubeconfig/context/namespace, service/runner/UI image +digests, chart path, model/provider references, existing OpenShell coordinates and +an existing PostgreSQL URL reference + public CA file. The database connection +verifies TLS; the credentials and model keys are read privately only after plan +approval. Kubernetes must be able to reach PostgreSQL and pull/use those images. +`scripted` is a local fixture option, not a substitute for live-provider qualification. + +```sh +ows deploy local-k8s plan --config local-k8s/deployment.yaml --state .ows/local +ows deploy local-k8s up --config local-k8s/deployment.yaml --state .ows/local \ + --accept-plan PLAN_DIGEST +ows deploy local-k8s status --state .ows/local +ows management-ui status --state .ows/local/installation +``` + +Use the actual digest printed by plan. The shared installer checks cluster and +namespace identities, OpenShell compatibility/storage/network prerequisites and +original Helm receipts. `advanced.openshell.mode: existing` reuses the named +installation; `install` accepts a reviewed pinned archive and inline `values` +that the CLI materializes internally. It never silently upgrades customer-managed +OpenShell. The existing installer enforces its pinned archive/image requirements. +No new sandbox/model run is implied by successful installation readiness. + +The CLI generates the private API certificate and maps `managementUI.port` to the +loopback preview. Keep and trust that CA; do not disable TLS verification. +Set UI/recording preferences in YAML or use the independent feature commands. +Native lower-level `ows deploy local` / `deploy cluster` remain compatibility +interfaces for existing journals; new examples use `local-k8s`. + +The new `local-docker` and `remote-k8s` command names are discoverable but report +`phase: roadmap`, return failure, and create no resources. They must not be used +as proof of successful deployment or qualification. Docker's existing separate +scripts remain available pending a complete one-file CLI adapter. diff --git a/deploy/service/examples/aws/deployment.yaml b/deploy/service/examples/aws/deployment.yaml index 6c3652e4..91f3d845 100644 --- a/deploy/service/examples/aws/deployment.yaml +++ b/deploy/service/examples/aws/deployment.yaml @@ -1,30 +1,51 @@ -# Replace the account ID and choose an authenticated AWS CLI profile. -apiVersion: ows/v1alpha1 -name: aws-agents +# Single authored deployment file. Replace documented identity/artifact placeholders. +# No trial by default: this deployment does not expire automatically. +# For disposable data only, add trial: {expiresAfterHours: 6, deleteData: true}. +# Cost estimates are checked with --plan-only --max-estimated-cost-usd, not a spending cap. +apiVersion: "ows/v1alpha1" +name: "aws-agents" cloud: - provider: aws + provider: "aws" accountId: "123456789012" - region: us-west-2 - profile: default + region: "us-west-2" + profile: "default" nodes: count: 2 - instanceType: t4g.large -# Disposable six-hour trial. Expiry deletes workload data; keep no valuable data here. -trial: - expiresAfterHours: 6 - deleteData: true -# Planning limit, not a hard billing cap. Set cloud billing alerts separately. -budget: - limitUSD: 100 + instanceType: "t4g.large" managementUI: enabled: true - publicOrigin: https://localhost:8444 + port: 8444 observability: enabled: true retentionDays: 14 -# Export this variable in your shell; never put a key in this file. models: - anthropic: {env: ANTHROPIC_API_KEY} -# This is an OpenWorker settings file, separate from your cloud CLI login. + anthropic: + env: "ANTHROPIC_API_KEY" advanced: - setupProfile: ./setup.json + release: "/absolute/path/to/arm64-release/release.json" + infrastructure: + schemaVersion: 1 + availabilityZones: + - "us-west-2a" + - "us-west-2b" + vpcCidr: "10.94.0.0/16" + kubernetesVersion: "1.36" + nodeReleaseVersion: "1.36.4-20261007" + postgresVersion: "16.13" + addons: + vpcCni: "v1.21.1-eksbuild.1" + kubeProxy: "v1.36.4-eksbuild.1" + coreDns: "v1.13.1-eksbuild.1" + podIdentity: "v1.3.8-eksbuild.1" + ebsCsi: "v1.48.0-eksbuild.1" + guardCode: + bucket: "example-ows-qualified-artifacts" + key: "guard/cleanup.zip" + versionId: "EXAMPLE-not-an-uploaded-object" + sha256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + acknowledgeExperimentDataDeletion: true + operatorAccess: + mode: "ssm" + amiId: "ami-0123456789abcdef0" + aws: + operatorArn: "arn:aws:iam::123456789012:role/REPLACE_WITH_APPROVED_INSTALLER" diff --git a/deploy/service/examples/aws/setup.json b/deploy/service/examples/aws/setup.json deleted file mode 100644 index 74eb4cf8..00000000 --- a/deploy/service/examples/aws/setup.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "schemaVersion": 1, - "provider": "aws", - "account": "123456789012", - "region": "us-west-2", - "operatorArn": "arn:aws:iam::123456789012:role/REPLACE_WITH_APPROVED_INSTALLER", - "infrastructure": { - "schemaVersion": 1, - "availabilityZones": [ - "us-west-2a", - "us-west-2b" - ], - "vpcCidr": "10.94.0.0/16", - "kubernetesVersion": "1.36", - "nodeReleaseVersion": "1.36.4-20261007", - "postgresVersion": "16.13", - "addons": { - "vpcCni": "v1.21.1-eksbuild.1", - "kubeProxy": "v1.36.4-eksbuild.1", - "coreDns": "v1.13.1-eksbuild.1", - "podIdentity": "v1.3.8-eksbuild.1", - "ebsCsi": "v1.48.0-eksbuild.1" - }, - "guardCode": { - "bucket": "example-ows-qualified-artifacts", - "key": "guard/cleanup.zip", - "versionId": "EXAMPLE-not-an-uploaded-object", - "sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" - }, - "acknowledgeExperimentDataDeletion": true, - "operatorAccess": { - "mode": "ssm", - "amiId": "ami-0123456789abcdef0" - } - }, - "release": "/absolute/path/to/arm64-release/release.json" -} diff --git a/deploy/service/examples/azure/deployment.yaml b/deploy/service/examples/azure/deployment.yaml index e9d24690..e6e36afd 100644 --- a/deploy/service/examples/azure/deployment.yaml +++ b/deploy/service/examples/azure/deployment.yaml @@ -1,29 +1,44 @@ -# Replace the subscription UUID. The setup profile must match it and this region. -apiVersion: ows/v1alpha1 -name: azure-agents +# Single authored deployment file. Replace documented identity/artifact placeholders. +# No trial by default: this deployment does not expire automatically. +# For disposable data only, add trial: {expiresAfterHours: 6, deleteData: true}. +# Cost estimates are checked with --plan-only --max-estimated-cost-usd, not a spending cap. +apiVersion: "ows/v1alpha1" +name: "azure-agents" cloud: - provider: azure + provider: "azure" subscriptionId: "11111111-2222-4333-8444-555555555555" - region: westus2 + region: "westus2" nodes: count: 2 - instanceType: Standard_D4as_v7 -# Disposable six-hour trial. Expiry deletes workload data; keep no valuable data here. -trial: - expiresAfterHours: 6 - deleteData: true -# Planning limit, not a hard billing cap. Set cloud billing alerts separately. -budget: - limitUSD: 100 + instanceType: "Standard_D4as_v7" managementUI: enabled: true - publicOrigin: https://localhost:8444 + port: 8444 observability: enabled: true retentionDays: 14 -# Export this variable in your shell; never put a key in this file. models: - anthropic: {env: ANTHROPIC_API_KEY} -# This is an OpenWorker settings file, separate from your cloud CLI login. + anthropic: + env: "ANTHROPIC_API_KEY" advanced: - setupProfile: ./setup.json + release: "/absolute/path/to/amd64-release/release.json" + infrastructure: + kubernetesVersion: "REPLACE_WITH_AVAILABLE_AKS_VERSION" + nodeVmSize: "Standard_D4as_v7" + sshPublicKey: "REPLACE_WITH_SSH_PUBLIC_KEY_WITHOUT_COMMENT" + adminGroupObjectIds: + - "33333333-4444-4555-8666-777777777777" + controlResourceGroup: "rg-ow-example-control" + guardImage: "examplecontrol.azurecr.io/cleanup@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + guardRegistryResourceId: "/subscriptions/11111111-2222-4333-8444-555555555555/resourceGroups/rg-ow-example-control/providers/Microsoft.ContainerRegistry/registries/examplecontrol" + managementAccess: + operatorIpv4: "203.0.113.10" + ubuntuImageVersion: "REPLACE_WITH_PINNED_UBUNTU_IMAGE_VERSION" + postgresPassword: + keyVaultId: "/subscriptions/11111111-2222-4333-8444-555555555555/resourceGroups/rg-ow-example-control/providers/Microsoft.KeyVault/vaults/ow-example-vault" + secretName: "postgres-password" + secretVersion: "0123456789abcdef0123456789abcdef" + azure: + resourceGroup: "rg-ow-example-workload" + setupObjectId: "22222222-3333-4444-8555-666666666666" + setupPrincipalType: "User" diff --git a/deploy/service/examples/azure/setup.json b/deploy/service/examples/azure/setup.json deleted file mode 100644 index b933e4a0..00000000 --- a/deploy/service/examples/azure/setup.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "schemaVersion": 1, - "provider": "azure", - "account": "11111111-2222-4333-8444-555555555555", - "region": "westus2", - "setupObjectId": "22222222-3333-4444-8555-666666666666", - "setupPrincipalType": "User", - "resourceGroup": "rg-ow-example-workload", - "infrastructure": { - "kubernetesVersion": "REPLACE_WITH_AVAILABLE_AKS_VERSION", - "nodeVmSize": "Standard_D4as_v7", - "sshPublicKey": "REPLACE_WITH_SSH_PUBLIC_KEY_WITHOUT_COMMENT", - "adminGroupObjectIds": [ - "33333333-4444-4555-8666-777777777777" - ], - "controlResourceGroup": "rg-ow-example-control", - "guardImage": "examplecontrol.azurecr.io/cleanup@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "guardRegistryResourceId": "/subscriptions/11111111-2222-4333-8444-555555555555/resourceGroups/rg-ow-example-control/providers/Microsoft.ContainerRegistry/registries/examplecontrol", - "managementAccess": { - "operatorIpv4": "203.0.113.10", - "ubuntuImageVersion": "REPLACE_WITH_PINNED_UBUNTU_IMAGE_VERSION" - }, - "postgresPassword": { - "keyVaultId": "/subscriptions/11111111-2222-4333-8444-555555555555/resourceGroups/rg-ow-example-control/providers/Microsoft.KeyVault/vaults/ow-example-vault", - "secretName": "postgres-password", - "secretVersion": "0123456789abcdef0123456789abcdef" - } - }, - "release": "/absolute/path/to/amd64-release/release.json" -} diff --git a/deploy/service/examples/local-cluster/installation-secrets.example.json b/deploy/service/examples/local-cluster/installation-secrets.example.json deleted file mode 100644 index 533147f6..00000000 --- a/deploy/service/examples/local-cluster/installation-secrets.example.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "database": { - "database-url": "postgresql://openworker:REPLACE_WITH_FIXTURE_PASSWORD@postgres:5432/openworker", - "pg-ca.crt": "REPLACE_WITH_POSTGRES_CA_PEM" - }, - "apiTLS": { - "tls.crt": "REPLACE_WITH_API_CERTIFICATE_PEM", - "tls.key": "REPLACE_WITH_API_PRIVATE_KEY_PEM", - "ca.crt": "REPLACE_WITH_API_CA_PEM" - }, - "model": { - "anthropic": "REPLACE_WITH_MODEL_KEY" - } -} diff --git a/deploy/service/examples/local-cluster/installer.json b/deploy/service/examples/local-cluster/installer.json deleted file mode 100644 index 2edf508d..00000000 --- a/deploy/service/examples/local-cluster/installer.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "schemaVersion": 1, - "kubeconfig": "/absolute/path/to/local.kubeconfig", - "context": "kind-openworker", - "helm": "helm", - "service": { - "namespace": "ow-service-m4-local", - "release": "ows", - "chart": "/absolute/path/to/openworker/deploy/service/chart", - "valuesFile": "/absolute/path/to/openworker/deploy/service/examples/local-cluster/service-values.json", - "fixtureDatabase": true - }, - "openshell": { - "mode": "existing", - "namespace": "ow-helm-shell", - "release": "ow-shell", - "fullname": "ow-helm-gateway" - } -} diff --git a/deploy/service/examples/local-cluster/service-values.json b/deploy/service/examples/local-cluster/service-values.json deleted file mode 100644 index d67f946f..00000000 --- a/deploy/service/examples/local-cluster/service-values.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "image": "registry.example.com/openworker/service@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "secrets": { - "model": "ow-service-model" - }, - "worker": { - "replicas": 1, - "ownership": "manual", - "maxReplicas": 2, - "slots": 2, - "drainSeconds": 30 - }, - "terminationGracePeriodSeconds": 90, - "managementUI": { - "enabled": true, - "image": "registry.example.com/openworker/ui@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "publicOrigin": "https://localhost:8444", - "serviceUrl": "https://ows-api.ow-service-m4-local.svc", - "caSecret": "ow-service-tls", - "caKey": "ca.crt", - "tlsSecret": "ow-service-tls" - }, - "config": { - "service": { - "id": "ows-local-example", - "namespace": "local", - "agent": "installed-tools", - "legacy_compatibility": false - }, - "database_env": "OW_PG_URL", - "schema": "ow_service", - "database_loopback_plaintext": false, - "database_ca": "/run/service/pg-ca.crt", - "state_root": "/state", - "bundle": "/opt/openworker/bundle", - "runner": "/opt/openworker/openworker", - "model": "claude-sonnet-4-6", - "scripted": false, - "model_provider": "anthropic", - "model_connections": [ - { - "id": "primary", - "provider": "anthropic", - "identity": "local-primary", - "key_env": "ANTHROPIC_API_KEY" - } - ], - "api_token_file": "/bootstrap/admin-token", - "actor": "deployment-admin", - "openshell": { - "identity": "ows-local-example", - "endpoint": "https://ow-helm-gateway.ow-helm-shell.svc.cluster.local:8080", - "tls_server_name": "ow-helm-gateway.ow-helm-shell.svc.cluster.local", - "certificates": "/openshell", - "workspace": "default", - "image": "registry.example.com/openworker/runner@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - }, - "observability": { - "schema": "ow_observability", - "retention_days": 14 - } - } -} diff --git a/deploy/service/examples/local-k8s/deployment.yaml b/deploy/service/examples/local-k8s/deployment.yaml new file mode 100644 index 00000000..2449b4e6 --- /dev/null +++ b/deploy/service/examples/local-k8s/deployment.yaml @@ -0,0 +1,34 @@ +# Existing local kind cluster and PostgreSQL; does not create Kubernetes or a database. +# Replace all example image digests and paths. No secrets belong in this file. +apiVersion: "ows/v1alpha1" +name: "local-agents" +managementUI: + enabled: true + port: 8444 +observability: + enabled: true + retentionDays: 14 +models: + anthropic: + env: "ANTHROPIC_API_KEY" +advanced: + kubernetes: + kubeconfig: "/absolute/path/to/local.kubeconfig" + context: "kind-openworker" + namespace: "ow-service-m4-local" + helm: "helm" + service: + chart: "../../chart" + image: "registry.example.com/openworker/service@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + runnerImage: "registry.example.com/openworker/runner@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + uiImage: "registry.example.com/openworker/ui@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + model: "claude-sonnet-4-6" + openshell: + mode: "existing" + namespace: "ow-helm-shell" + release: "ow-shell" + fullname: "ow-helm-gateway" + database: + url: + env: "OW_DATABASE_URL" + caFile: "/absolute/path/to/postgres-ca.crt" diff --git a/deploy/service/up/README.md b/deploy/service/up/README.md index b1006c37..c8d26614 100644 --- a/deploy/service/up/README.md +++ b/deploy/service/up/README.md @@ -1,84 +1,120 @@ -# Deploy OpenWorker with one command +# Deploy OpenWorker from one YAML -This experimental coordinator installs a private service cluster, OpenShell and the OpenWorker service using a reviewed account setup profile. It resumes the same deployment when rerun. Fresh AWS and Azure installs with this coordinator still require live qualification; do not treat local tests as a production certificate. +Use one `staging-deploy.yaml` or `prod-deploy.yaml` for deployment choices and +advanced account/infrastructure overrides. There is no required setup-profile +file. Generated release manifests, plans, credentials and recovery journals are +separate installer artifacts, not additional configuration you must hand-author. -Complete commented examples, including companion setup profiles and an existing local kind cluster, are in [deployment examples](../examples/README.md). Copy the selected cloud folder, replace its documented placeholders and prepare the reviewed release bundle before running: +Complete [Azure, AWS and local examples](../examples/README.md) show the supported +schema. The service installer remains experimental until the current release +passes a fresh cloud install, agent/recovery and cleanup qualification. ```sh -owcluster deploy aws up --config deployment.yaml -# or -owcluster deploy azure up --config deployment.yaml +ows deploy azure up --config staging-deploy.yaml --plan-only +ows deploy aws up --config prod-deploy.yaml --plan-only ``` -The command resolves a supported VM size, writes a cost/permission review, and asks for approval before making cloud changes. Fixed node counts are supported; autoscaling is not. Requests with an uncertain outcome are observed using their original identity rather than blindly resubmitted. +Remove `--plan-only` to review/approve provisioning. `--accept-plan DIGEST` and +`--accept-bootstrap DIGEST` provide separate noninteractive approvals of the exact +saved review. The command never treats an uncertain result as permission to +recreate resources. Rerun with the original state to reconcile unfinished work. -## One-time prerequisites +## Application choices -Install `owcluster`, your cloud CLI, Docker, Helm, kubectl and OpenSSL. Azure also needs kubelogin and, for the management VM, SSH. AWS's optional operator bridge needs the Session Manager plugin. Authenticate with your own cloud identity. The command never creates long-lived cloud access keys. - -An administrator prepares a target-bound setup profile at: - -```text -~/.config/ows/deployments/aws//.json -~/.config/ows/deployments/azure//.json +```yaml +apiVersion: ows/v1alpha1 +name: my-agents +cloud: + provider: azure + subscriptionId: "11111111-2222-4333-8444-555555555555" + region: westus2 +nodes: + count: 2 + instanceType: Standard_D4as_v7 +managementUI: + enabled: true + port: 8444 +observability: + enabled: true + retentionDays: 14 +models: + anthropic: {env: ANTHROPIC_API_KEY} +advanced: + # Inline the provider/identity overrides shown in the complete example. + release: /absolute/path/to/generated-release/release.json + infrastructure: {} + azure: + resourceGroup: rg-my-agents + setupObjectId: "22222222-3333-4444-8555-666666666666" + setupPrincipalType: User ``` -Use `advanced.setupProfile` in YAML to select another file. The administrator reviews the scoped native bootstrap permission exports. AWS needs permission to create the reviewed bootstrap identity stack. Azure needs the reviewed RBAC authority at the workload/control groups and exact vault/secret scopes; its active login must match `setupObjectId`. Normal workload identities never receive these installation privileges. Azure resource groups, the Entra administrator group and the versioned database password in Key Vault already exist. The vault must use RBAC and allow ARM template deployment. +The snippet omits infrastructure detail for readability; copy a complete example. +Fixed node count or a CPU/memory size range is supported; autoscaling is roadmap. +`models` supplies env/file credential references, currently OpenAI and Anthropic. +It is not a general provider-routing configuration API. The installer stages keys +privately after approval. Completion reports `modelSmokeTest: not_run`; deploy and +submit an agent separately through the service SDK. -The profile contains **references and version pins**, not passwords: +## Advanced inputs and prerequisites -```json -{ - "schemaVersion": 1, - "provider": "aws", - "account": "123456789012", - "region": "us-west-2", - "operatorArn": "arn:aws:iam::123456789012:role/approved-installer", - "infrastructure": {}, - "release": "/absolute/path/to/release/release.json" -} -``` +`advanced.infrastructure` embeds the reviewed provider inputs previously kept in +`setup.json`. AWS uses `advanced.aws.operatorArn`; Azure uses +`advanced.azure.resourceGroup`, `setupObjectId` and `setupPrincipalType`. +`advanced.release` selects a generated, checksum-verified release manifest with +pinned service/runner/UI images and chart/CA/OpenShell artifacts. Relative paths +in deployment YAML resolve against that YAML's directory. Generated release +manifest paths are absolute. Legacy explicit `advanced.setupProfile` import is +compatibility-only; new files do not use it or a hidden default account profile. -Replace `infrastructure` with the reviewed native AWS input described in [`harness/service-cli/examples/README.md`](../../../harness/service-cli/examples/README.md). It includes zone/version pins and a versioned S3 reference to **this CLI release's packaged cleanup guard**. For Azure, use the existing Azure infrastructure input (Kubernetes version, supported VM size, SSH public key, administrator group IDs, control group, pinned external cleanup image and Key Vault secret reference). Its profile uses `provider: azure`, the subscription as `account`, and `resourceGroup`, `setupObjectId`, `setupPrincipalType` instead of `operatorArn`. Package the matching native cleanup image before running the installer. Existing control assets are retained, not silently taken over or deleted. +Install `ows`, the cloud CLI, Docker, Helm, kubectl and OpenSSL. Azure also needs +kubelogin and SSH for its optional management VM; AWS's SSM bridge needs the +Session Manager plugin. Sign in using your existing cloud identity. The installer +requires the reviewed provisioning/RBAC scopes, not merely a successful login. -This release does not assume public OpenWorker service images or a published Helm chart exist. Prepare a reviewed release manifest with `package.py --help`: it copies a chart, pinned OpenShell archive and values, platform manifest and database CA bundle, then hashes them. Supply immutable harness/runner image digests for Linux ARM64 on AWS or AMD64 on Azure. The supported OpenShell archive is 0.1.7. The CA bundle must contain 1–32 certificates. The installer publishes the selected images into its owned ECR/ACR registry. Package a UI image when enabling `managementUI`. The CLI can configure a private HTTPS preview; public ingress remains operator-managed. +Current Azure prerequisites still include existing workload/control resource +groups, an Entra administrator group, versioned database password in an RBAC Key +Vault with ARM deployment access, and an independent pinned cleanup image. +AWS requires the reviewed installer identity and a versioned S3 cleanup package. +The native approval/bootstrap flow handles its supported grants; it does not +magically create all account prerequisites. Default-pin discovery, account setup +wizard and publicly distributed release artifacts remain separate launch work. -## Review, budget and resume +Package the matching release using `package.py --help`: Linux ARM64 for AWS, +AMD64 for Azure, including `service-ui` when enabling the UI. New cloud deployments +require the reviewed OpenShell0.1.7 archive. The current version does not assume +public service images or a published Helm chart are already available. + +## Cost estimates, not spending caps + +There is no required `budget` field in new deployment YAML. Check an optional +estimated-cost ceiling in plan mode: ```sh -owcluster deploy azure up --config deployment.yaml --plan-only +ows deploy azure up --config staging-deploy.yaml --plan-only \ + --max-estimated-cost-usd 100 ``` -The private `.ows/deployments//` directory contains `review.json`, the exact native `infrastructure/` permission/template bundle and the durable journal. Keep it: it holds the original deployment identities. Do not delete or edit it to retry an uncertain cloud operation. +The CLI queries provider retail catalogs for the selected resources and saves a +separate `cost-estimate.json` with timestamp, sources, quantities, missing prices, +planned hours and ceiling result. The estimate covers modeled fixed infrastructure +plus an explicit allowance; model usage, variable traffic/storage/requests, taxes, +discounts and unmodeled resources are not a bill prediction or hard cap. A ceiling +check fails if modeled prices are incomplete or exceed the ceiling. Without a +ceiling, unknown prices remain visible rather than becoming zero or a passing +budget claim. The option requires `--plan-only`. -`budget.limitUSD` is the reviewed experiment/planning budget used for cost checks. **It does not create cloud billing alerts and is not a hard cap.** Fixed costs, missing catalog rates and a separate variable-cost allowance are reported. Storage, logs, traffic, model usage and retained control resources can add costs; the allowance is not an upper bound. Persistent reviews use a 730-hour planning horizon; disposable trials use their specified lifetime. Set account billing alerts separately. +Quotes expire after30minutes. Rerun plan mode to refresh the quote for the same +frozen resource intent; its review digest and original operations do not change. +Before a new deployment is admitted, expired or failed quote checks require a +fresh plan review. Already dispatched deployments retain their original approval +for recovery; price expiry never causes replacement/recreation. Estimates are not +continuously enforced during execution. Cloud billing alerts are separate. +Earlier experimental YAML/native CLI budgeting is compatibility behavior only. -For automation, review the generated files, then supply their exact review digest: +## Optional trial expiry -```sh -owcluster deploy azure up --config deployment.yaml \ - --accept-plan --accept-bootstrap -``` - -AWS accepts the same options. Saved approvals apply only to that exact configuration, release and generated cloud graph. Rerunning observes unfinished operations and retains their receipts. Changed configuration requires a separate reviewed deployment/update; it is not applied in place. - -## Private management access - -The command can use an existing private route. If your account input enables the operator bridge, it creates the native bridge resource and connects automatically: AWS SSM port forwarding or Azure SSH restricted to the configured operator IPv4. Azure's `advanced.azure.sshPrivateKeyFile` selects the matching local key; otherwise SSH uses your existing key/agent. SSH host keys are saved in the private deployment directory. The original Kubernetes hostname and CA remain verified through a loopback proxy. - -Add `--keep-access` to keep that route open after readiness checks. In another terminal use `connection.json` for the private API URL, CA, token-file path and port-forward instructions. Ctrl-C closes the local route; subsequent commands can recreate it after reconciling the original SSM session. A changed operator address, unavailable SSM session, occupied local proxy port or missing private route produces an actionable error rather than making the Kubernetes management API public. - -## Model credentials and completion - -Use `models.openai: {env: OPENAI_API_KEY}` or `models.anthropic: {env: ANTHROPIC_API_KEY}` (a `file` reference also works). Keys are read only when staging Kubernetes Secrets and are never written to the immutable review. The private local credential staging directory contains secrets and must not be committed. - -An installation can complete without model credentials. Completion verifies component readiness; it explicitly reports `modelSmokeTest: not_run`. Submit an agent separately to qualify your chosen provider/model and application. - -## Persistent deployments and disposable trials - -Omitting `trial` disables automatic workload expiry. The older destructive teardown route refuses persistent deployments until a separate retention workflow is approved. It will not delete your database as a side effect of an expired provisioning deadline. - -For disposable tests only, set: +Without `trial`, the workload does not automatically expire. For disposable data: ```yaml trial: @@ -86,96 +122,47 @@ trial: deleteData: true ``` -This enables the existing independent cleanup controller. It retains the control/audit resources documented in the native review. A timeout or failure does not authorize deleting/recreating resources with uncertain outcomes. Read the original receipts and use the native lifecycle reconciliation/cleanup commands for the same deployment. +This enables independent expiry cleanup and explicitly permits workload data +deletion. It is not a guaranteed cost cap; failures and retained control/audit +resources can still incur costs. Persistent destructive teardown remains refused +pending a separately reviewed data-retention workflow. -## Local checks +## UI, trace storage and private access -Run the CLI unit/integration suite with `cargo test --manifest-path harness/Cargo.toml -p openworker-service-cli`. Its cloud operations use fixtures. The cleanup guard and Helm rendering checks run with `node --test deploy/service/guard/guard.test.cjs deploy/service/chart.test.mjs`. +`managementUI.port` is the operator's loopback HTTPS preview port. The CLI derives +`https://localhost:PORT`; internal browser origin/CSRF checks remain strict. It +emits port-forward commands and CA information, not tokens in URLs. Use a current +service token to sign in. This does not create public ingress, DNS or a certificate +for a company domain. Existing ingress users can retain the legacy explicit origin; +`ows management-ui enable --ingress-url https://manage.example.com` selects it. +Do not combine an explicit ingress origin and preview port. -To exercise the actual `up --plan-only` entry point, package pins, approval gates and resume handling with trapped AWS commands: +Use `--keep-access` to retain AWS SSM/Azure SSH private management connectivity. +The original Kubernetes endpoint/hostname/CA stay verified through the loopback +route. `connection.json` records API and enabled UI connection instructions. + +UI and recording are independent and disabled unless configured. Recording uses +its own connection/schema on execution PostgreSQL by default; an existing Secret +selected by `observability.databaseSecret` may provide `database-url` and `ca.crt` +for a separate database. That Secret must be staged in the owned namespace before +service installation. Enabling initializes its schema before startup. ```sh -python3 deploy/service/up/test-plan.py \ - --ows /absolute/path/to/ows \ - --openshell-archive /absolute/path/to/helm-chart-0.1.7.tgz +ows management-ui enable --state ./deployment-state/installation --port 8444 +ows management-ui disable --state ./deployment-state/installation +ows management-ui status --state ./deployment-state/installation +ows observability enable --state ./deployment-state/installation --retention-days 14 +ows observability disable --state ./deployment-state/installation +ows observability status --state ./deployment-state/installation ``` -That check uses the real pinned OpenShell archive, fixture image references and catalog responses. It performs no real cloud calls, does not install a cluster and must not be presented as cloud qualification. +The commands save feature overrides in the installer journal, retain the original +review and verify cluster/namespace/Helm identities. Unknown upgrades are observed, +never replayed. Disabling recording retains stored data; its retention worker is +also stopped until recording resumes. Execution state, security audit and independent +OTLP exporters remain separate. A UI rollout requires sign-in again. -## Management UI and trace storage - -Both features are optional and independent. Select them before the first `up`: - -```yaml -managementUI: - enabled: true - publicOrigin: https://localhost:8444 -observability: - enabled: true - retentionDays: 14 -``` - -The reviewed release manifest must include a digest-pinned `service-ui` image. -The default private preview uses the API certificate and a loopback-only Kubernetes -port-forward; `connection.json` contains its command, URL and CA file. Trust that -certificate locally, then sign in with a current service token. No token is put in -the browser URL. Keep the cloud's private management route open as described above. -A non-loopback `publicOrigin` requires your own TLS ingress; `up` does not create -public exposure or DNS records. - -After installation, use the original **installer** state directory (the -`installation/` child of the `up --state` directory): - -```sh -ows management-ui enable --state .ows/my-deployment/installation -ows management-ui status --state .ows/my-deployment/installation -ows management-ui disable --state .ows/my-deployment/installation - -ows observability enable --state .ows/my-deployment/installation --retention-days 14 -ows observability status --state .ows/my-deployment/installation -ows observability disable --state .ows/my-deployment/installation -``` - -If the original release did not include a UI image, pass -`--image registry.example.com/ui@sha256:YOUR_VERIFIED_DIGEST` on enable. -`--public-origin` overrides its origin. These commands use the original Kubernetes -credentials, verify cluster/namespace/release identities, and perform a Helm upgrade -of the saved chart. They do not use the service REST token. Features are saved in -`journal.json` with their original Helm operation and revision receipts; the initial -provisioning review and `deployment.yaml` stay immutable. Rerunning `up` does not -reset these saved feature overrides. Do not edit old installer values in place. - -Trace storage uses a separate `ow_observability` schema and connection to the -execution PostgreSQL by default. For a separate database, create an existing -Kubernetes Secret containing `database-url` and `ca.crt`, then select it with -`observability.databaseSecret` or `ows observability enable --database-secret NAME`. -Verified database TLS remains required. Enabling initializes the recording schema -before workloads start; feature changes roll the affected pods, so allow active -turns to drain. The first-admin bootstrap is not rerun. - -Disabling recording stops new recordings as the rollout completes. It does not -delete traces, databases or credentials. While disabled, the recorder's retention -worker is also stopped; retained data remains until recording resumes or an operator -performs a separate cleanup. Security audit, durable execution state and separately -configured OTLP exporters are unaffected. This is deployment-level control, not a -new per-agent trace policy API. - -A lost Helm response never causes an automatic repeat. `status` can reconcile the -original successful revision; an unresolved/pending/failed result blocks another -upgrade or teardown until the operator inspects that original operation. - -Local regression checks: - -```sh -cargo test --manifest-path harness/Cargo.toml -p openworker-service-cli -node --test deploy/service/chart.test.mjs -OWS_PROXY_TEST=1 node --test deploy/service/proxy.test.mjs -``` - -If the API uses a certificate signed by a separate CA, provide its public trust -bundle using `managementUI.caSecret` / `managementUI.caKey` in deployment YAML, -or `ows management-ui enable --ca-secret API_CA_SECRET --ca-key ca.crt`. -The Secret must already exist in the service namespace. Without an override, -the CLI trusts the API TLS Secret's `tls.crt`, suitable for the self-signed -certificate produced by `up`. The UI process receives only that public certificate; -the HTTPS proxy sidecar receives the separate TLS key mount. +For a separately signed API certificate, the UI can select a public CA Secret/key +through `caSecret`/`caKey` or `--ca-secret`/`--ca-key`. Without an override it trusts +the self-signed API certificate generated by the installer. The UI process gets +only public trust material; its TLS sidecar gets the separate private-key mount. diff --git a/deploy/service/up/aws.yaml b/deploy/service/up/aws.yaml index 48a003c2..91f3d845 100644 --- a/deploy/service/up/aws.yaml +++ b/deploy/service/up/aws.yaml @@ -1,28 +1,51 @@ -# Full reference and companion setup.json: ../examples/aws/ -# Replace the account ID and choose an authenticated AWS CLI profile. -apiVersion: ows/v1alpha1 -name: aws-agents +# Single authored deployment file. Replace documented identity/artifact placeholders. +# No trial by default: this deployment does not expire automatically. +# For disposable data only, add trial: {expiresAfterHours: 6, deleteData: true}. +# Cost estimates are checked with --plan-only --max-estimated-cost-usd, not a spending cap. +apiVersion: "ows/v1alpha1" +name: "aws-agents" cloud: - provider: aws + provider: "aws" accountId: "123456789012" - region: us-west-2 - profile: default + region: "us-west-2" + profile: "default" nodes: count: 2 - instanceType: t4g.large -# Disposable six-hour trial. Expiry deletes workload data; keep no valuable data here. -trial: - expiresAfterHours: 6 - deleteData: true -# Planning limit, not a hard billing cap. Set cloud billing alerts separately. -budget: - limitUSD: 100 + instanceType: "t4g.large" managementUI: enabled: true - publicOrigin: https://localhost:8444 + port: 8444 observability: enabled: true retentionDays: 14 -# Export this variable in your shell; never put a key in this file. models: - anthropic: {env: ANTHROPIC_API_KEY} + anthropic: + env: "ANTHROPIC_API_KEY" +advanced: + release: "/absolute/path/to/arm64-release/release.json" + infrastructure: + schemaVersion: 1 + availabilityZones: + - "us-west-2a" + - "us-west-2b" + vpcCidr: "10.94.0.0/16" + kubernetesVersion: "1.36" + nodeReleaseVersion: "1.36.4-20261007" + postgresVersion: "16.13" + addons: + vpcCni: "v1.21.1-eksbuild.1" + kubeProxy: "v1.36.4-eksbuild.1" + coreDns: "v1.13.1-eksbuild.1" + podIdentity: "v1.3.8-eksbuild.1" + ebsCsi: "v1.48.0-eksbuild.1" + guardCode: + bucket: "example-ows-qualified-artifacts" + key: "guard/cleanup.zip" + versionId: "EXAMPLE-not-an-uploaded-object" + sha256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + acknowledgeExperimentDataDeletion: true + operatorAccess: + mode: "ssm" + amiId: "ami-0123456789abcdef0" + aws: + operatorArn: "arn:aws:iam::123456789012:role/REPLACE_WITH_APPROVED_INSTALLER" diff --git a/deploy/service/up/azure.yaml b/deploy/service/up/azure.yaml index f4200de6..e6e36afd 100644 --- a/deploy/service/up/azure.yaml +++ b/deploy/service/up/azure.yaml @@ -1,27 +1,44 @@ -# Full reference and companion setup.json: ../examples/azure/ -# Replace the subscription UUID. The setup profile must match it and this region. -apiVersion: ows/v1alpha1 -name: azure-agents +# Single authored deployment file. Replace documented identity/artifact placeholders. +# No trial by default: this deployment does not expire automatically. +# For disposable data only, add trial: {expiresAfterHours: 6, deleteData: true}. +# Cost estimates are checked with --plan-only --max-estimated-cost-usd, not a spending cap. +apiVersion: "ows/v1alpha1" +name: "azure-agents" cloud: - provider: azure + provider: "azure" subscriptionId: "11111111-2222-4333-8444-555555555555" - region: westus2 + region: "westus2" nodes: count: 2 - instanceType: Standard_D4as_v7 -# Disposable six-hour trial. Expiry deletes workload data; keep no valuable data here. -trial: - expiresAfterHours: 6 - deleteData: true -# Planning limit, not a hard billing cap. Set cloud billing alerts separately. -budget: - limitUSD: 100 + instanceType: "Standard_D4as_v7" managementUI: enabled: true - publicOrigin: https://localhost:8444 + port: 8444 observability: enabled: true retentionDays: 14 -# Export this variable in your shell; never put a key in this file. models: - anthropic: {env: ANTHROPIC_API_KEY} + anthropic: + env: "ANTHROPIC_API_KEY" +advanced: + release: "/absolute/path/to/amd64-release/release.json" + infrastructure: + kubernetesVersion: "REPLACE_WITH_AVAILABLE_AKS_VERSION" + nodeVmSize: "Standard_D4as_v7" + sshPublicKey: "REPLACE_WITH_SSH_PUBLIC_KEY_WITHOUT_COMMENT" + adminGroupObjectIds: + - "33333333-4444-4555-8666-777777777777" + controlResourceGroup: "rg-ow-example-control" + guardImage: "examplecontrol.azurecr.io/cleanup@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + guardRegistryResourceId: "/subscriptions/11111111-2222-4333-8444-555555555555/resourceGroups/rg-ow-example-control/providers/Microsoft.ContainerRegistry/registries/examplecontrol" + managementAccess: + operatorIpv4: "203.0.113.10" + ubuntuImageVersion: "REPLACE_WITH_PINNED_UBUNTU_IMAGE_VERSION" + postgresPassword: + keyVaultId: "/subscriptions/11111111-2222-4333-8444-555555555555/resourceGroups/rg-ow-example-control/providers/Microsoft.KeyVault/vaults/ow-example-vault" + secretName: "postgres-password" + secretVersion: "0123456789abcdef0123456789abcdef" + azure: + resourceGroup: "rg-ow-example-workload" + setupObjectId: "22222222-3333-4444-8555-666666666666" + setupPrincipalType: "User" diff --git a/deploy/service/up/test-plan.py b/deploy/service/up/test-plan.py index 0fcf9b51..326f9384 100644 --- a/deploy/service/up/test-plan.py +++ b/deploy/service/up/test-plan.py @@ -58,10 +58,7 @@ print(json.dumps(v)) '--database-ca',str(repo/'harness/store-postgres/tests/fixtures/public-ca.pem'),'--out',str(root/'release')], check=True, capture_output=True) infrastructure = json.loads((repo/'harness/service-cli/examples/aws-infrastructure.json').read_text()) infrastructure['guardCode']['sha256'] = json.loads((root/'guard/package-manifest.json').read_text())['archiveSha256'] - setup = {'schemaVersion':1,'provider':'aws','account':'123456789012','region':'us-west-2', - 'operatorArn':'arn:aws:iam::123456789012:role/operator','infrastructure':infrastructure,'release':str(root/'release/release.json')} - (root/'setup.json').write_text(json.dumps(setup)) - yaml = "name: fixture\ncloud: {provider: aws, accountId: '123456789012', region: us-west-2}\nnodes: {count: 2, instanceType: t4g.large}\nbudget: {limitUSD: 1000}\nadvanced:\n setupProfile: " + str(root/'setup.json') + '\n' + yaml = "name: fixture\ncloud: {provider: aws, accountId: '123456789012', region: us-west-2}\nnodes: {count: 2, instanceType: t4g.large}\nadvanced:\n release: " + json.dumps(str(root/'release/release.json')) + '\n aws: {operatorArn: "arn:aws:iam::123456789012:role/operator"}\n infrastructure: ' + json.dumps(infrastructure) + '\n' (root/'deployment.yaml').write_text(yaml) args = ['deploy','aws','up','--config',str(root/'deployment.yaml'),'--state',str(root/'state')] run(args+['--plan-only']) @@ -70,8 +67,17 @@ print(json.dumps(v)) assert review['config']['nodes']['count'] == 2 plan = json.loads((root/'state/infrastructure/deployment-plan.json').read_text()) assert plan['autoExpire'] is False + assert plan['budget']['estimateCheckDisabled'] is True + assert not (root/'setup.json').exists() run(args+['--plan-only']) assert (root/'state/review.json').read_bytes() == first + too_small=run(args+['--plan-only','--max-estimated-cost-usd','1'],ok=False) + assert 'exceeds' in too_small.stderr + run(args+['--plan-only','--max-estimated-cost-usd','100000']) + quote=json.loads((root/'state/cost-estimate.json').read_text()) + assert quote['ceilingCheck']=='within_estimated_ceiling' + assert quote['reviewDigest']==review['digest'] + assert (root/'state/review.json').read_bytes()==first run(args+['--accept-plan','wrong'], ok=False) run(args+['--accept-plan',review['digest']], ok=False) # bootstrap still unapproved kubectl = tools / 'kubectl' @@ -85,4 +91,4 @@ print(json.dumps(v)) assert 'changed' in result.stderr assert (root/'calls.jsonl').read_bytes() == calls assert all(json.loads(line) in [['sts','get-caller-identity'],['ec2','describe-instance-types'],['ec2','describe-instance-type-offerings'],['pricing','get-products']] for line in calls.splitlines()) - print(json.dumps({'readOnlyPlan':True,'stableResume':True,'changedConfigRejected':True,'unapprovedBootstrapBlocked':True,'missingLocalToolBlocked':True,'cloudMutations':0,'installationQualified':False})) + print(json.dumps({'oneAuthoredYaml':True,'planCeilingChecked':True,'quoteRefreshPreservesIntent':True,'readOnlyPlan':True,'stableResume':True,'changedConfigRejected':True,'unapprovedBootstrapBlocked':True,'missingLocalToolBlocked':True,'cloudMutations':0,'installationQualified':False})) diff --git a/harness/service-cli/README.md b/harness/service-cli/README.md index 95fe0754..ae50844b 100644 --- a/harness/service-cli/README.md +++ b/harness/service-cli/README.md @@ -103,7 +103,7 @@ Select these features in `deployment.yaml` for the AWS/Azure `up` commands, or change them on an existing installation: ```sh -ows management-ui enable --state ./deployment-state/installation +ows management-ui enable --state ./deployment-state/installation --port 8444 ows management-ui disable --state ./deployment-state/installation ows management-ui status --state ./deployment-state/installation ows observability enable --state ./deployment-state/installation --retention-days 14 @@ -118,5 +118,8 @@ The CLI saves feature changes and reconciles lost Helm responses without repeati unknown upgrades. See [setup, UI image, private HTTPS and database options](../../deploy/service/up/README.md#management-ui-and-trace-storage). Complete [Azure, AWS and local kind reference files](../../deploy/service/examples/README.md) -include deployment choices, companion cloud setup profiles, local chart values and -placeholder secrets-file shape. Replace their documented placeholders before use. +put deployment choices and advanced account/cluster inputs in one YAML. The CLI +generates internal values/installer files; replace the documented placeholders +before use. `local-docker` and `remote-k8s` are explicit roadmap modes, not +qualified installers. New cloud configs omit budget; optional estimated-cost +ceilings use `up --plan-only --max-estimated-cost-usd`. diff --git a/harness/service-cli/src/deployment.rs b/harness/service-cli/src/deployment.rs index 347c4c70..d2e53c20 100644 --- a/harness/service-cli/src/deployment.rs +++ b/harness/service-cli/src/deployment.rs @@ -33,6 +33,21 @@ use std::{ #[derive(Subcommand)] pub enum Command { + /// Install from one YAML into an existing local kind cluster; requires kubeconfig. + LocalK8s { + #[command(subcommand)] + command: local::single_file::Command, + }, + /// Roadmap: one-file Docker orchestration; existing Compose scripts remain available. + LocalDocker { + #[command(subcommand)] + command: RoadmapCommand, + }, + /// Roadmap: customer-managed remote Kubernetes; not qualified yet. + RemoteK8s { + #[command(subcommand)] + command: RoadmapCommand, + }, Guard { #[command(subcommand)] command: guard::Command, @@ -90,6 +105,19 @@ pub enum Command { }, } #[derive(Subcommand)] +pub enum RoadmapCommand { + /// Unavailable until the mode is implemented/qualified. Never creates resources. + Up { + #[arg(long)] + config: PathBuf, + }, + /// Show that the mode is roadmap; does not validate a runnable deployment. + Plan { + #[arg(long)] + config: PathBuf, + }, +} +#[derive(Subcommand)] pub enum AwsCommand { /// Provision and install from one reviewed deployment.yaml; resumes original state. Up(up::Options), @@ -226,6 +254,9 @@ enum Target { #[serde(rename_all = "camelCase", deny_unknown_fields)] struct Budget { authorized_usd: u32, + /// New high-level deployments use plan-mode estimates, not a runtime budget. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + estimate_check_disabled: bool, hard_cap: bool, estimate: Option, } @@ -429,7 +460,10 @@ impl Plan { "Expiry must be within 72 hours of plan creation" ); ensure!( - (1..=100_000).contains(&self.budget.authorized_usd) + ((1..=100_000).contains(&self.budget.authorized_usd) + || (self.budget.estimate_check_disabled + && self.budget.authorized_usd == 0 + && !matches!(self.target, Target::Gcp { .. }))) && !self.budget.hard_cap && self.budget.estimate.is_none(), "Offline plans require a positive budget, unknown estimate, and no hard-cap claim" @@ -632,6 +666,18 @@ fn result(p: &Plan, m: &Manifest) -> Value { } pub async fn dispatch(command: &Command) -> Result<()> { match command { + Command::LocalK8s { command } => return local::single_file::run(command).await, + Command::LocalDocker { .. } | Command::RemoteK8s { .. } => { + let mode = if matches!(command, Command::LocalDocker { .. }) { + "local-docker" + } else { + "remote-k8s" + }; + crate::output( + json!({"mode":mode,"phase":"roadmap","supported":false,"mutations":0,"clusterCreated":false}), + )?; + bail!("{mode} is roadmap, not a qualified deployment mode. No resources were created. See deploy/service/examples/README.md"); + } Command::Aws { command: AwsCommand::Up(o), } => return up::run("aws", o).await, @@ -750,7 +796,10 @@ pub fn run(command: &Command) -> Result<()> { command: AwsCommand::Lifecycle { .. }, } => bail!("Cloud lifecycle requires asynchronous dispatch"), Command::Guard { command } => guard::run(command), - Command::Local { .. } => bail!("Local installer requires asynchronous dispatch"), + Command::Local { .. } + | Command::LocalK8s { .. } + | Command::LocalDocker { .. } + | Command::RemoteK8s { .. } => bail!("Deployment mode requires asynchronous dispatch"), Command::Cluster { .. } => bail!("Cluster installer requires asynchronous dispatch"), Command::Aws { command: AwsCommand::Preflight(_), @@ -891,6 +940,7 @@ fn create( auto_expire: None, budget: Budget { authorized_usd: a.budget, + estimate_check_disabled: false, hard_cap: false, estimate: None, }, @@ -992,6 +1042,7 @@ mod tests { auto_expire: None, budget: Budget { authorized_usd: 100, + estimate_check_disabled: false, hard_cap: false, estimate: None, }, diff --git a/harness/service-cli/src/deployment_features.rs b/harness/service-cli/src/deployment_features.rs index 4d0e2e16..f1a74425 100644 --- a/harness/service-cli/src/deployment_features.rs +++ b/harness/service-cli/src/deployment_features.rs @@ -7,6 +7,9 @@ use super::*; pub(crate) struct ManagementUI { #[serde(default)] pub enabled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub port: Option, + /// Legacy explicit origin for existing ingress deployments. Basic YAML uses port. #[serde(default = "origin")] pub public_origin: String, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -21,6 +24,7 @@ impl Default for ManagementUI { fn default() -> Self { Self { enabled: false, + port: None, public_origin: origin(), ca_secret: None, ca_key: None, @@ -28,6 +32,21 @@ impl Default for ManagementUI { } } impl ManagementUI { + pub(crate) fn browser_origin(&self) -> Result { + if let Some(port) = self.port { + ensure!( + port >= 1024, + "Management UI port must be between 1024 and 65535" + ); + ensure!( + self.public_origin == origin(), + "Choose a preview port or a legacy explicit ingress origin, not both" + ); + Ok(format!("https://localhost:{port}")) + } else { + Ok(self.public_origin.clone()) + } + } pub(crate) fn validate(&self) -> Result<()> { if let Some(s) = &self.ca_secret { ensure!(dns_name(s), "Invalid UI CA Secret name"); @@ -41,7 +60,8 @@ impl ManagementUI { "Invalid UI CA Secret key" ); } - let u = reqwest::Url::parse(&self.public_origin).context("Invalid management UI origin")?; + let u = + reqwest::Url::parse(&self.browser_origin()?).context("Invalid management UI origin")?; ensure!(u.scheme()=="https" && u.host_str().is_some() && u.username().is_empty() && u.password().is_none() && u.path()=="/" && u.query().is_none() && u.fragment().is_none(), "Management UI publicOrigin must be an HTTPS origin without credentials, path, query or fragment"); Ok(()) } @@ -93,7 +113,7 @@ fn dns_name(s: &str) -> bool { pub(crate) struct Target { /// Original installer journal (local-plan.json and journal.json), not cloud lifecycle state. #[arg(long)] - state: PathBuf, + pub(super) state: PathBuf, } #[derive(Subcommand)] pub(crate) enum UICommand { @@ -103,8 +123,11 @@ pub(crate) enum UICommand { /// Digest-pinned UI image. Omit when already recorded in Helm values. #[arg(long)] image: Option, - #[arg(long)] + #[arg(long = "ingress-url", alias = "public-origin", conflicts_with = "port")] public_origin: Option, + /// Loopback HTTPS preview port; defaults to 8444. + #[arg(long, value_parser=clap::value_parser!(u16).range(1024..=65535))] + port: Option, /// Existing Secret containing the CA that signs the service API certificate. #[arg(long)] ca_secret: Option, @@ -251,7 +274,7 @@ pub(crate) fn configure_ui( if let Some(image) = image { values["managementUI"]["image"] = json!(image); } - values["managementUI"]["publicOrigin"] = json!(ui.public_origin); + values["managementUI"]["publicOrigin"] = json!(ui.browser_origin()?); values["managementUI"]["serviceUrl"] = json!(format!("https://{release}-api.{namespace}.svc")); if ui.enabled { let image = values["managementUI"]["image"] @@ -301,7 +324,7 @@ pub(crate) fn configure_ui( values["managementUI"]["caKey"] = json!(ca_key); // Private loopback preview is self-contained. Public origins require the // operator's TLS ingress; never create public exposure implicitly. - let u = reqwest::Url::parse(&ui.public_origin)?; + let u = reqwest::Url::parse(&ui.browser_origin()?)?; values["managementUI"]["tlsSecret"] = if matches!(u.host_str(), Some("localhost" | "127.0.0.1")) { json!(api_tls) @@ -441,7 +464,7 @@ async fn change( Value::Null }; crate::output( - json!({"deploymentId":p.id,"managementUI":{"enabled":ui["enabled"]==true,"publicOrigin":ui["publicOrigin"],"privatePreview":preview},"observability":{"enabled":!obs.is_null(),"configuration":obs,"databaseSecret":values["secrets"]["observability"],"storedData":"retained; no deletion performed","exporters":"configured independently"},"savedConfiguration":root.join("journal.json"),"helmRevision":confirmed_receipt(&j)?["revision"]}), + json!({"deploymentId":p.id,"managementUI":{"enabled":ui["enabled"]==true,"port":ui["publicOrigin"].as_str().and_then(|s|reqwest::Url::parse(s).ok()).and_then(|u|u.port_or_known_default()),"url":ui["publicOrigin"],"publicOrigin":ui["publicOrigin"],"privatePreview":preview},"observability":{"enabled":!obs.is_null(),"configuration":obs,"databaseSecret":values["secrets"]["observability"],"storedData":"retained; no deletion performed","exporters":"configured independently"},"savedConfiguration":root.join("journal.json"),"helmRevision":confirmed_receipt(&j)?["revision"]}), ) } pub(crate) async fn ui(command: &UICommand) -> Result<()> { @@ -465,6 +488,7 @@ pub(crate) async fn ui(command: &UICommand) -> Result<()> { target, image, public_origin, + port, ca_secret, ca_key, } => { @@ -473,14 +497,18 @@ pub(crate) async fn ui(command: &UICommand) -> Result<()> { |v, p| { let ui = ManagementUI { enabled: true, - public_origin: public_origin - .clone() - .or_else(|| { - v["managementUI"]["publicOrigin"] - .as_str() - .map(str::to_owned) - }) - .unwrap_or_else(origin), + port: *port, + public_origin: if port.is_some() { + Some(origin()) + } else { + public_origin.clone() + } + .or_else(|| { + v["managementUI"]["publicOrigin"] + .as_str() + .map(str::to_owned) + }) + .unwrap_or_else(origin), ca_secret: ca_secret.clone(), ca_key: ca_key.clone(), }; @@ -615,3 +643,25 @@ mod tests { .is_err()); } } + +#[cfg(test)] +mod port_tests { + use super::*; + #[test] + fn port_derives_trusted_loopback_origin_and_rejects_ambiguous_ingress() { + let mut ui = ManagementUI { + port: Some(9000), + ..Default::default() + }; + ui.validate().unwrap(); + assert_eq!(ui.browser_origin().unwrap(), "https://localhost:9000"); + ui.port = Some(80); + assert!(ui.validate().is_err()); + ui.port = Some(9000); + ui.public_origin = "https://manage.example.com".into(); + assert!(ui.validate().is_err()); + ui.port = None; + ui.validate().unwrap(); + assert_eq!(ui.browser_origin().unwrap(), "https://manage.example.com"); + } +} diff --git a/harness/service-cli/src/deployment_infrastructure.rs b/harness/service-cli/src/deployment_infrastructure.rs index d927987b..6a661673 100644 --- a/harness/service-cli/src/deployment_infrastructure.rs +++ b/harness/service-cli/src/deployment_infrastructure.rs @@ -673,7 +673,7 @@ fn review(p: &Plan, i: &Inputs, policies: &BTreeMap) -> String { r#"# {} AWS experiment review Deployment `{}`; account `{account}`; region `{region}`; resource prefix `{prefix}`. -Budget authorization ${}; no verified estimate and no spending cap. Fixed expiry {}. +Cost policy {}; no spending cap. Provisioning deadline {}. ## Generated, not provisioned @@ -750,7 +750,11 @@ of the entire bundle. No cloud execution was performed to generate this review. "#, p.name, p.deployment_id, - p.budget.authorized_usd, + if p.budget.estimate_check_disabled { + "plan-only estimates; no runtime budget".into() + } else { + format!("legacy authorization ${}", p.budget.authorized_usd) + }, p.expires_at_unix, i.vpc_cidr, i.availability_zones, diff --git a/harness/service-cli/src/deployment_kubernetes.rs b/harness/service-cli/src/deployment_kubernetes.rs new file mode 100644 index 00000000..602a369f --- /dev/null +++ b/harness/service-cli/src/deployment_kubernetes.rs @@ -0,0 +1,397 @@ +//! One-file front end for the existing identity/journal-aware kind installer. +//! This does not bootstrap Kubernetes or claim qualification on arbitrary clusters. +use super::super::up::config::Secret; +use super::features::{ManagementUI, Observability}; +use super::*; +#[derive(Subcommand)] +pub(crate) enum Command { + /// Resolve one YAML into a bound installer plan; no cluster mutations. + Plan { + #[arg(long)] + config: PathBuf, + #[arg(long)] + state: PathBuf, + }, + /// Apply the exact reviewed local-kind plan. Kubernetes/OpenShell/DB prerequisites must be ready. + Up { + #[arg(long)] + config: PathBuf, + #[arg(long)] + state: PathBuf, + #[arg(long)] + accept_plan: String, + }, + Status { + #[arg(long)] + state: PathBuf, + }, + Destroy { + #[arg(long)] + state: PathBuf, + #[arg(long)] + deployment_id: String, + }, +} +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Deployment { + api_version: String, + name: String, + #[serde(default, rename = "managementUI")] + management_ui: ManagementUI, + #[serde(default)] + observability: Observability, + #[serde(default)] + models: BTreeMap, + advanced: Advanced, +} +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Advanced { + kubernetes: Kubernetes, + service: Images, + openshell: OpenShell, + database: Database, +} +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Kubernetes { + kubeconfig: PathBuf, + context: String, + namespace: String, + #[serde(default = "helm")] + helm: PathBuf, +} +fn helm() -> PathBuf { + "helm".into() +} +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Images { + chart: PathBuf, + image: String, + runner_image: String, + ui_image: Option, + model: Option, + #[serde(default)] + scripted: bool, +} +#[derive(Deserialize, Serialize)] +#[serde(tag = "mode", rename_all = "lowercase", deny_unknown_fields)] +enum OpenShell { + Existing { + namespace: String, + release: String, + fullname: String, + }, + #[serde(rename_all = "camelCase")] + Install { + namespace: String, + release: String, + fullname: String, + archive: PathBuf, + values: Value, + }, +} +impl OpenShell { + fn scope(&self) -> (&str, &str, &str) { + match self { + Self::Existing { + namespace, + release, + fullname, + } + | Self::Install { + namespace, + release, + fullname, + .. + } => (namespace, release, fullname), + } + } +} +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Database { + url: Secret, + ca_file: PathBuf, +} +fn load_deployment(path: &Path) -> Result { + let bytes = fs::read(path)?; + ensure!(bytes.len() <= 65536, "Deployment YAML exceeds 64 KiB"); + let mut c: Deployment = serde_yaml::from_slice(&bytes).map_err(|_| { + anyhow::anyhow!("Invalid local-k8s YAML schema; use env/file credential references") + })?; + ensure!( + c.api_version == "ows/v1alpha1" && super::super::name_ok(&c.name), + "Invalid deployment version/name" + ); + let k = &c.advanced.kubernetes; + ensure!(k.context.starts_with("kind-") && k.namespace.starts_with("ow-service-m4-") && bounded_name(&k.namespace),"local-k8s is qualified only for an explicit kind-* context and dedicated ow-service-m4-* namespace; remote-k8s remains roadmap"); + c.management_ui.validate()?; + c.observability.validate()?; + ensure!( + pinned(&c.advanced.service.image) && pinned(&c.advanced.service.runner_image), + "Pin service and runner images by digest" + ); + if c.management_ui.enabled { + ensure!( + c.advanced.service.ui_image.as_deref().is_some_and(pinned), + "Pin the enabled management UI image by digest" + ); + } + let base = path + .canonicalize()? + .parent() + .context("Deployment directory missing")? + .to_owned(); + let resolve = |p: &mut PathBuf| { + if p.is_relative() { + *p = base.join(&*p) + } + }; + resolve(&mut c.advanced.kubernetes.kubeconfig); + resolve(&mut c.advanced.service.chart); + resolve(&mut c.advanced.database.ca_file); + if c.advanced.kubernetes.helm.components().count() > 1 { + resolve(&mut c.advanced.kubernetes.helm); + } + if let OpenShell::Install { archive, .. } = &mut c.advanced.openshell { + resolve(archive) + } + for (name, secret) in c.models.iter_mut().chain(std::iter::once(( + &"database".to_owned(), + &mut c.advanced.database.url, + ))) { + ensure!( + name == "database" || ["anthropic", "openai"].contains(&name.as_str()), + "Only OpenAI/Anthropic deployment credential wiring is supported currently" + ); + ensure!( + secret.env.is_some() != secret.file.is_some(), + "Use exactly one env/file reference per credential" + ); + if let Some(env) = &secret.env { + ensure!( + !env.is_empty() + && env.len() <= 128 + && env + .bytes() + .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_'), + "Invalid credential variable name" + ); + } + if let Some(p) = &mut secret.file { + resolve(p) + } + } + Ok(c) +} +fn exact(root: &Path, name: &str, value: &Value) -> Result<()> { + let path = root.join(name); + if path.exists() { + ensure!( + read_json(&path)? == *value, + "Saved deployment inputs changed; preserve the original plan and journal" + ); + } else { + super::super::up::write_private(&path, value)?; + } + Ok(()) +} +fn secret(s: &Secret) -> Result> { + let value = if let Some(env) = &s.env { + std::env::var(env).context("Credential environment variable unavailable")? + } else { + fs::read_to_string(s.file.as_ref().context("Credential reference missing")?)? + }; + ensure!( + !value.trim().is_empty() && value.len() <= 32768 && !value.trim().contains(['\n', '\r']), + "Invalid credential format" + ); + Ok(zeroize::Zeroizing::new(value.trim().to_string())) +} +fn render(c: &Deployment, root: &Path) -> Result<()> { + let k = &c.advanced.kubernetes; + let (shell_ns, _, fullname) = c.advanced.openshell.scope(); + let service_id = if root.join("service-values.json").exists() { + read_json(&root.join("service-values.json"))?["config"]["service"]["id"] + .as_str() + .context("Original service identity missing")? + .to_string() + } else { + format!("ows-{}", &uuid::Uuid::new_v4().simple().to_string()[..12]) + }; + let provider = if c.models.contains_key("anthropic") { + "anthropic" + } else { + "openai" + }; + let connections:Vec<_>=c.models.keys().enumerate().map(|(i,p)|json!({"id":if i==0{"primary"}else{p},"provider":p,"identity":format!("{service_id}-{p}"),"key_env":if p=="anthropic"{"ANTHROPIC_API_KEY"}else{"OPENAI_API_KEY"}})).collect(); + let mut values = json!({"image":c.advanced.service.image,"secrets":{"model":if c.models.is_empty(){""}else{"ow-service-model"}},"worker":{"replicas":1,"ownership":"manual","maxReplicas":2,"slots":2,"drainSeconds":30},"terminationGracePeriodSeconds":90,"config":{"service":{"id":service_id,"namespace":"local","agent":"installed-tools","legacy_compatibility":false},"database_env":"OW_PG_URL","schema":format!("ow_{}",c.name.replace('-',"_")),"database_loopback_plaintext":false,"database_ca":"/run/service/pg-ca.crt","state_root":"/state","bundle":"/opt/openworker/bundle","runner":"/opt/openworker/openworker","model":c.advanced.service.model.as_deref().unwrap_or("unconfigured"),"scripted":c.advanced.service.scripted,"model_provider":provider,"model_connections":connections,"api_token_file":"/bootstrap/admin-token","actor":"deployment-admin","openshell":{"identity":service_id,"endpoint":format!("https://{fullname}.{shell_ns}.svc.cluster.local:8080"),"tls_server_name":format!("{fullname}.{shell_ns}.svc.cluster.local"),"certificates":"/openshell","workspace":"default","image":c.advanced.service.runner_image}}}); + let mut ui = c.management_ui.clone(); + // The generated self-signed API certificate is also the public trust anchor. + if ui.ca_secret.is_none() { + ui.ca_secret = Some("ow-service-tls".into()); + ui.ca_key = Some("tls.crt".into()); + } + features::configure_ui( + &mut values, + &ui, + c.advanced.service.ui_image.as_deref(), + "ows", + &k.namespace, + )?; + if c.observability.enabled { + values["config"]["observability"] = + json!({"schema":format!("ow_observability_{}",c.name.replace('-',"_"))}); + } + features::configure_observability(&mut values, &c.observability)?; + exact(root, "service-values.json", &values)?; + let shell = match &c.advanced.openshell { + OpenShell::Existing { + namespace, + release, + fullname, + } => json!({"mode":"existing","namespace":namespace,"release":release,"fullname":fullname}), + OpenShell::Install { + namespace, + release, + fullname, + archive, + values, + } => { + exact(root, "openshell-values.json", values)?; + json!({"mode":"install","namespace":namespace,"release":release,"fullname":fullname,"archive":archive,"valuesFile":root.join("openshell-values.json")}) + } + }; + let installer = json!({"schemaVersion":1,"kubeconfig":k.kubeconfig,"context":k.context,"helm":k.helm,"service":{"namespace":k.namespace,"release":"ows","chart":c.advanced.service.chart,"valuesFile":root.join("service-values.json"),"fixtureDatabase":true},"openshell":shell}); + exact(root, "installer.json", &installer)?; + exact(root, "deployment-input.json", &serde_json::to_value(c)?) +} +async fn credentials(c: &Deployment, root: &Path) -> Result { + let cert = root.join("api.crt"); + let key = root.join("api.key"); + if !cert.exists() || !key.exists() { + ensure!( + !cert.exists() && !key.exists(), + "Partial certificate generation requires inspection" + ); + exec(Path::new("openssl"),&args(&["req","-x509","-newkey","rsa:3072","-nodes","-days","30","-subj","/CN=localhost","-addext","basicConstraints=critical,CA:TRUE","-addext","keyUsage=critical,digitalSignature,keyCertSign,cRLSign","-addext","extendedKeyUsage=serverAuth","-addext",&format!("subjectAltName=DNS:localhost,IP:127.0.0.1,DNS:ows-api.{}.svc,DNS:ows-api.{}.svc.cluster.local",c.advanced.kubernetes.namespace,c.advanced.kubernetes.namespace),"-keyout",key.to_str().context("Invalid key path")?,"-out",cert.to_str().context("Invalid certificate path")?]),None).await?; + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&key, fs::Permissions::from_mode(0o600))?; + } + let models: serde_json::Map = c + .models + .iter() + .map(|(p, s)| Ok((p.clone(), json!(secret(s)?.as_str())))) + .collect::>()?; + let input = json!({"database":{"database-url":secret(&c.advanced.database.url)?.as_str(),"pg-ca.crt":fs::read_to_string(&c.advanced.database.ca_file)?},"apiTLS":{"tls.crt":fs::read_to_string(cert)?,"tls.key":fs::read_to_string(key)?},"model":if models.is_empty(){Value::Null}else{json!(models)}}); + let path = root.join("installation-secrets.json"); + super::super::up::write_private(&path, &input)?; + Ok(path) +} +pub(crate) async fn run(command: &Command) -> Result<()> { + let (config, root, accept) = match command { + Command::Plan { config, state } => (Some(config), state, None), + Command::Up { + config, + state, + accept_plan, + } => (Some(config), state, Some(accept_plan)), + Command::Status { state } => (None, state, None), + Command::Destroy { + state, + deployment_id, + } => { + return super::run(&super::Command::Destroy { + state: state.join("installation"), + deployment_id: deployment_id.clone(), + }) + .await + } + }; + if let Command::Status { .. } = command { + return super::run(&super::Command::Status { + state: root.join("installation"), + }) + .await; + } + let c = load_deployment(config.unwrap())?; + let _lock = super::super::up::operation_lock(root, "configuration.lock")?; + let root = root.canonicalize()?; + render(&c, &root)?; + let state = root.join("installation"); + if !state.exists() { + super::run(&super::Command::Plan { + config: root.join("installer.json"), + out: state.clone(), + }) + .await?; + } + let plan: Plan = read_json(&state.join("local-plan.json")) + .and_then(|v| serde_json::from_value(v).map_err(Into::into))?; + if matches!(command, Command::Plan { .. }) { + return crate::output( + json!({"mode":"local-k8s","phase":"review","state":root,"planDigest":plan.digest,"deploymentId":plan.id,"clusterAlreadyExists":true,"mutations":0}), + ); + } + let accept = accept.unwrap(); + ensure!( + accept == &plan.digest, + "Approval digest differs from original installer plan" + ); + super::run(&super::Command::Apply { + state: state.clone(), + accept_plan: accept.clone(), + openshell_only: true, + confirm_no_effect: None, + reconcile_ready_release: None, + }) + .await?; + let input = credentials(&c, &root).await?; + super::run(&super::Command::Secrets { + state: state.clone(), + input, + accept_plan: accept.clone(), + }) + .await?; + super::run(&super::Command::Apply { + state: state.clone(), + accept_plan: accept.clone(), + openshell_only: false, + confirm_no_effect: None, + reconcile_ready_release: None, + }) + .await?; + features::ui(&features::UICommand::Status { + target: features::Target { state }, + }) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn one_yaml_reference_renders_private_origin_and_installer_files_without_secrets_or_effects() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../deploy/service/examples/local-k8s/deployment.yaml"); + let c = load_deployment(&path).unwrap(); + let d = tempfile::tempdir().unwrap(); + render(&c, d.path()).unwrap(); + let v = read_json(&d.path().join("service-values.json")).unwrap(); + assert_eq!(v["managementUI"]["publicOrigin"], "https://localhost:8444"); + assert_eq!(v["config"]["observability"]["retention_days"], 14); + assert!(!d.path().join("installation-secrets.json").exists()); + let _: Config = + serde_json::from_value(read_json(&d.path().join("installer.json")).unwrap()).unwrap(); + render(&c, d.path()).unwrap(); + let mut changed = c; + changed.management_ui.port = Some(9000); + assert!(render(&changed, d.path()).is_err()); + } +} diff --git a/harness/service-cli/src/deployment_local.rs b/harness/service-cli/src/deployment_local.rs index a6289aa3..c6c0e7fe 100644 --- a/harness/service-cli/src/deployment_local.rs +++ b/harness/service-cli/src/deployment_local.rs @@ -8,6 +8,8 @@ use tokio::{ }; #[path = "deployment_features.rs"] pub(crate) mod features; +#[path = "deployment_kubernetes.rs"] +pub(crate) mod single_file; const OWNER: &str = "openworker.io/installer-id"; #[path = "deployment_cluster_platform.rs"] mod platform; @@ -1874,33 +1876,3 @@ pub(crate) async fn verify_azure_teardown( } Ok(()) } - -#[cfg(test)] -mod reference_examples { - use super::*; - #[test] - fn local_reference_uses_supported_existing_kind_installer_and_valid_chart_values() { - let root = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../deploy/service/examples/local-cluster"); - let c: Config = - serde_json::from_slice(&fs::read(root.join("installer.json")).unwrap()).unwrap(); - assert!(c.context.starts_with("kind-") && c.service.fixture_database); - let v: Value = - serde_json::from_slice(&fs::read(root.join("service-values.json")).unwrap()).unwrap(); - let (ns, _, fullname) = c.openshell.scope(); - assert_eq!( - v["config"]["openshell"]["endpoint"], - format!("https://{fullname}.{ns}.svc.cluster.local:8080") - ); - assert_eq!( - v["managementUI"]["serviceUrl"], - format!( - "https://{}-api.{}.svc", - c.service.release, c.service.namespace - ) - ); - assert!(pinned(v["image"].as_str().unwrap())); - assert_eq!(v["managementUI"]["enabled"], true); - assert_eq!(v["config"]["observability"]["retention_days"], 14); - } -} diff --git a/harness/service-cli/src/deployment_preflight_checks.rs b/harness/service-cli/src/deployment_preflight_checks.rs index 3e42ad1c..e55b54a3 100644 --- a/harness/service-cli/src/deployment_preflight_checks.rs +++ b/harness/service-cli/src/deployment_preflight_checks.rs @@ -703,6 +703,11 @@ async fn price(aws: &mut Aws, p: &Plan, kind: Product<'_>) -> Observed { Ok(selected.remove(0)) } async fn cost(aws: &mut Aws, p: &Plan, allowance: u32) -> Observed { + if p.budget.estimate_check_disabled { + return Ok( + json!({"estimateCheck":"plan_mode_only","estimateAvailable":false,"withinAuthorizedBudget":null,"hardCap":false,"note":"This high-level deployment has no runtime spending cap; see its separate cost-estimate.json"}), + ); + } let mut components = BTreeMap::new(); let mut lines = vec![ ( @@ -749,8 +754,8 @@ async fn cost(aws: &mut Aws, p: &Plan, allowance: u32) -> Observed { expect(fixed.is_finite())?; let fixed = (fixed * 100.0).ceil() / 100.0; let estimate = fixed + f64::from(allowance); - let mut detail = json!({"rateSource":"AWS Price List GetProducts OnDemand USD","components":components,"plannedHours":hours,"fixedCostEstimateUsd":fixed,"variableAllowanceUsd":allowance,"planningEstimateUsd":estimate,"authorizedBudgetUsd":p.budget.authorized_usd,"withinAuthorizedBudget":estimate<=f64::from(p.budget.authorized_usd),"hardCap":false,"monthHoursForProration":730,"excludedVariableCharges":["NAT processed bytes","data transfer","T4g CPU credits","S3/ECR storage and requests","guard, logs and control-plane requests","OpenShell workspace volume usage"],"allowanceIsNotAnUpperBound":true}); - if estimate > f64::from(p.budget.authorized_usd) { + let mut detail = json!({"rateSource":"AWS Price List GetProducts OnDemand USD","components":components,"plannedHours":hours,"fixedCostEstimateUsd":fixed,"variableAllowanceUsd":allowance,"planningEstimateUsd":estimate,"authorizedBudgetUsd":if p.budget.estimate_check_disabled{Value::Null}else{json!(p.budget.authorized_usd)},"withinAuthorizedBudget":if p.budget.estimate_check_disabled{Value::Null}else{json!(estimate<=f64::from(p.budget.authorized_usd))},"hardCap":false,"monthHoursForProration":730,"excludedVariableCharges":["NAT processed bytes","data transfer","T4g CPU credits","S3/ECR storage and requests","guard, logs and control-plane requests","OpenShell workspace volume usage"],"allowanceIsNotAnUpperBound":true}); + if !p.budget.estimate_check_disabled && estimate > f64::from(p.budget.authorized_usd) { detail["blockingReason"] = json!("budget_exceeded"); } Ok(detail) diff --git a/harness/service-cli/src/deployment_up.rs b/harness/service-cli/src/deployment_up.rs index ec3c99b8..b8d451e6 100644 --- a/harness/service-cli/src/deployment_up.rs +++ b/harness/service-cli/src/deployment_up.rs @@ -2,18 +2,20 @@ //! Desired state, reviewed observations and effect receipts remain distinct. use super::*; #[path = "deployment_up_config.rs"] -mod config; +pub(super) mod config; #[path = "deployment_up_native.rs"] mod native; #[path = "deployment_up_package.rs"] mod package; #[path = "deployment_up_pricing.rs"] -mod pricing; +pub(super) mod pricing; #[path = "deployment_up_resolve.rs"] mod resolve; #[path = "deployment_up_state.rs"] mod state; use config::Config; +#[path = "deployment_up_cost.rs"] +mod cost; #[derive(Args)] pub struct Options { @@ -25,6 +27,9 @@ pub struct Options { /// Resolve and write the review without provisioning. #[arg(long)] plan_only: bool, + /// Check the priced fixed infrastructure estimate plus explicit allowance, never actual spend. + #[arg(long, requires = "plan_only", value_parser=clap::value_parser!(u32).range(1..=100000))] + max_estimated_cost_usd: Option, /// Noninteractive approval of this exact review digest, never a blanket yes. #[arg(long)] accept_plan: Option, @@ -267,7 +272,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { "Generated infrastructure/permissions changed since review; do not reuse the old approval" ); crate::output( - json!({"deploymentId":review.deployment_id,"state":state,"planDigest":review.digest,"nodeCount":config.nodes.count,"instanceType":review.resolution.node.instance_type,"price":review.resolution.price,"phase":"review","mutations":0}), + json!({"deploymentId":review.deployment_id,"state":state,"planDigest":review.digest,"nodeCount":config.nodes.count,"instanceType":review.resolution.node.instance_type,"price":review.resolution.price,"priceObservedAtUnix":review.resolution.observed_at_unix,"priceSnapshot":"original planning observation; refreshed prices live in cost-estimate.json","phase":"review","mutations":0}), )?; if o.plan_only { native::Native { @@ -279,12 +284,18 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { } .review_bundle() .await?; - return Ok(()); + return cost::review(&state, &review, &package, o.max_estimated_cost_usd).await; } - if let Some(subtotal) = review.resolution.price.known_hourly_subtotal_usd { + cost::validate_approval(&state, &review)?; + if let Some(subtotal) = review + .resolution + .price + .known_hourly_subtotal_usd + .filter(|_| config.budget.is_some()) + { let minimum = subtotal * f64::from(review.resolution.price.planned_hours) + f64::from(review.resolution.price.variable_allowance_usd); - ensure!(minimum<=f64::from(config.budget.alert_usd),"Known planning estimate already exceeds budget.limitUSD. Review a new budget/size/lifetime before provisioning; no mutations performed."); + ensure!(minimum<=f64::from(config.budget.as_ref().unwrap().alert_usd),"Known planning estimate already exceeds budget.limitUSD. Review a new budget/size/lifetime before provisioning; no mutations performed."); } // Render the exact permission payloads before either approval prompt. let preview = native::Native { diff --git a/harness/service-cli/src/deployment_up_config.rs b/harness/service-cli/src/deployment_up_config.rs index d0b7898e..4a4eeff2 100644 --- a/harness/service-cli/src/deployment_up_config.rs +++ b/harness/service-cli/src/deployment_up_config.rs @@ -10,7 +10,9 @@ pub(super) struct Config { pub name: String, pub cloud: Cloud, pub nodes: Nodes, - pub budget: Budget, + /// Compatibility with earlier experimental YAML only. New configs omit budget. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub budget: Option, #[serde(default)] pub trial: Option, #[serde(default)] @@ -102,7 +104,7 @@ pub(super) struct Budget { } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -pub(super) struct Secret { +pub(crate) struct Secret { #[serde(default, skip_serializing_if = "Option::is_none")] pub env: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -113,7 +115,10 @@ pub(super) struct Secret { pub(super) struct Advanced { /// A reviewed release package can replace locally packaged repository assets. pub release: Option, + /// Legacy profile import. New YAML embeds these fields below. pub setup_profile: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub infrastructure: Option, pub network_cidr: Option, pub kubernetes_version: Option, pub azure: Option, @@ -123,6 +128,10 @@ pub(super) struct Advanced { #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(super) struct AzureOptions { pub tenant_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub setup_object_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub setup_principal_type: Option, pub resource_group: Option, pub control_resource_group: Option, #[serde(default)] @@ -247,9 +256,12 @@ impl Config { ); } } + if let Some(b) = &self.budget { + ensure!((1..=100_000).contains(&b.alert_usd), "Legacy budget.limitUSD must be positive; omit it and use plan-only --max-estimated-cost-usd instead"); + } ensure!( - (1..=100_000).contains(&self.budget.alert_usd), - "budget.limitUSD must be between 1 and 100000; it is not a hard cap" + self.advanced.setup_profile.is_none() || self.advanced.infrastructure.is_none(), + "Choose inline advanced.infrastructure, not both it and a legacy setupProfile" ); ensure!( self.advanced.azure.is_none() || provider == "azure", @@ -400,25 +412,21 @@ mod reference_examples { let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../deploy/service/examples"); for provider in ["azure", "aws"] { let c = Config::load(&root.join(provider).join("deployment.yaml"), provider).unwrap(); - assert_eq!(c.budget.alert_usd, 100); - assert_eq!(c.trial.as_ref().unwrap().expires_after_hours, 6); + assert!(c.budget.is_none()); + assert!(c.trial.is_none()); assert!( c.management_ui.as_ref().unwrap().enabled && c.observability.as_ref().unwrap().enabled ); - let setup: super::super::package::Setup = serde_json::from_slice( - &fs::read(c.advanced.setup_profile.as_ref().unwrap()).unwrap(), - ) - .unwrap(); - assert_eq!(setup.provider, provider); - assert_eq!(setup.account, c.cloud.account()); - assert_eq!(setup.region, c.cloud.region()); + assert!(c.advanced.setup_profile.is_none()); + let infrastructure = c.advanced.infrastructure.clone().unwrap(); + assert!(c.advanced.release.as_ref().unwrap().is_absolute()); if provider == "azure" { let _: super::super::super::azure::Inputs = - serde_json::from_value(setup.infrastructure).unwrap(); + serde_json::from_value(infrastructure).unwrap(); } else { let _: super::super::super::infrastructure::Inputs = - serde_json::from_value(setup.infrastructure).unwrap(); + serde_json::from_value(infrastructure).unwrap(); } } } diff --git a/harness/service-cli/src/deployment_up_cost.rs b/harness/service-cli/src/deployment_up_cost.rs new file mode 100644 index 00000000..cb06c544 --- /dev/null +++ b/harness/service-cli/src/deployment_up_cost.rs @@ -0,0 +1,165 @@ +//! Mutable price observations are separate from immutable provisioning intent. +//! Refreshing a quote must never regenerate resource identities or replay writes. +use super::*; +const QUOTE_SECONDS: u64 = 1800; +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Quote { + review_digest: String, + observed_at_unix: u64, + valid_until_unix: u64, + price: resolve::Price, + max_estimated_cost_usd: Option, + estimate_usd: Option, + ceiling_check: String, + digest: String, +} +impl Quote { + fn digest(&self) -> Result { + let mut v = serde_json::to_value(self)?; + v.as_object_mut().unwrap().remove("digest"); + Ok(hash(&serde_json::to_vec(&v)?)) + } +} +fn check(price: &resolve::Price, ceiling: Option) -> (Option, &'static str) { + let estimate = price + .total_hourly_usd + .filter(|n| n.is_finite() && *n > 0.0) + .filter(|_| price.unpriced_components.is_empty()) + .map(|n| n * f64::from(price.planned_hours) + f64::from(price.variable_allowance_usd)) + .filter(|n| n.is_finite() && *n > 0.0); + let result = match (ceiling, estimate) { + (None, _) => "not_requested", + (Some(_), None) => "incomplete", + (Some(c), Some(v)) if v > f64::from(c) => "exceeded", + _ => "within_estimated_ceiling", + }; + (estimate, result) +} +pub(super) async fn review( + root: &Path, + review: &Review, + package: &package::Snapshot, + ceiling: Option, +) -> Result<()> { + let mut nodes = vec![review.resolution.node.clone()]; + pricing::price_nodes(&review.config, &mut nodes).await; + let price = pricing::estimate(&review.config, &nodes[0], &package.setup.infrastructure).await; + let (estimate, status) = check(&price, ceiling); + let at = now()?; + let mut quote = Quote { + review_digest: review.digest.clone(), + observed_at_unix: at, + valid_until_unix: at + QUOTE_SECONDS, + price, + max_estimated_cost_usd: ceiling, + estimate_usd: estimate, + ceiling_check: status.into(), + digest: String::new(), + }; + quote.digest = quote.digest()?; + write_private(&root.join("cost-estimate.json"), "e)?; + crate::output( + json!({"phase":"cost_review","quote":quote,"mutations":0,"hardSpendingCap":false,"scope":"modeled fixed infrastructure and explicit usage allowance only","excludedCosts":["actual LLM usage","variable traffic/storage/requests/logging","discounts, taxes and unmodeled services"],"unknownPricesAreZero":false}), + )?; + ensure!(status!="incomplete","Estimated-cost ceiling cannot be checked: modeled prices are missing or ambiguous. No cloud mutation performed"); + ensure!( + status != "exceeded", + "Estimated cost exceeds the plan-mode ceiling. No cloud mutation performed" + ); + Ok(()) +} +pub(super) fn validate_approval(root: &Path, review: &Review) -> Result<()> { + // An already dispatched deployment retains its original approval. Quote + // freshness is an admission check, never an excuse to recreate resources. + if root.join("up-journal.json").exists() { + return Ok(()); + } + let path = root.join("cost-estimate.json"); + if !path.exists() { + let at = now()?; + ensure!(review.resolution.observed_at_unix<=at && at-review.resolution.observed_at_unix<=QUOTE_SECONDS,"Saved price observation is stale. Rerun --plan-only to refresh costs without changing the deployment intent"); + return Ok(()); + } + let quote: Quote = serde_json::from_slice(&fs::read(path)?)?; + validate("e, review, now()?) +} +fn validate(q: &Quote, r: &Review, at: u64) -> Result<()> { + ensure!( + q.digest == q.digest()? && q.review_digest == r.digest, + "Cost quote does not match the reviewed deployment" + ); + ensure!( + q.observed_at_unix <= at + && at <= q.valid_until_unix + && q.valid_until_unix - q.observed_at_unix == QUOTE_SECONDS, + "Cost quote expired; rerun --plan-only. Resource and operation identities are unchanged" + ); + let (estimate, status) = check(&q.price, q.max_estimated_cost_usd); + ensure!( + estimate == q.estimate_usd && status == q.ceiling_check, + "Cost quote result changed" + ); + ensure!( + ["not_requested", "within_estimated_ceiling"].contains(&status), + "Plan-mode estimated-cost check did not pass; inspect cost-estimate.json" + ); + Ok(()) +} +#[cfg(test)] +mod tests { + use super::*; + fn price() -> resolve::Price { + serde_json::from_value(json!({"currency":"USD","workerHourlyUsd":0.2,"totalHourlyUsd":0.5,"knownHourlySubtotalUsd":0.5,"components":{},"variableAllowanceUsd":25,"plannedHours":6,"unpricedComponents":[],"hardCap":false})).unwrap() + } + #[test] + fn ceiling_is_plan_estimate_not_zero_for_unknown_or_a_spending_cap() { + let mut p = price(); + assert_eq!( + check(&p, Some(100)), + (Some(28.0), "within_estimated_ceiling") + ); + assert_eq!(check(&p, Some(27)).1, "exceeded"); + p.unpriced_components.push("network".into()); + assert_eq!(check(&p, Some(100)), (None, "incomplete")); + assert_eq!(check(&p, None).1, "not_requested"); + p.unpriced_components.clear(); + p.total_hourly_usd = None; + assert_eq!(check(&p, Some(100)).1, "incomplete"); + } +} + +#[cfg(test)] +mod receipt_tests { + use super::*; + #[test] + fn quote_refresh_preserves_intent_and_expired_failed_or_changed_receipts_cannot_admit() { + let config:Config=serde_yaml::from_str("name: example\ncloud: {provider: aws, accountId: '123456789012', region: us-west-2}\nnodes: {count: 2, instanceType: t4g.large}\n").unwrap(); + let price:resolve::Price=serde_json::from_value(json!({"currency":"USD","workerHourlyUsd":0.2,"totalHourlyUsd":0.5,"knownHourlySubtotalUsd":0.5,"components":{},"variableAllowanceUsd":25,"plannedHours":6,"unpricedComponents":[],"hardCap":false})).unwrap(); + let r=Review{schema_version:1,config_digest:config.digest().unwrap(),config,deployment_id:"original".into(),resolution:serde_json::from_value(json!({"node":{"instanceType":"t4g.large","cpu":2,"memoryGiB":8,"architecture":"arm64","eligible":true,"hourlyUsd":0.1},"price":price,"observedAtUnix":1000,"identity":{}})).unwrap(),package_digest:"release".into(),native_bundle_digest:"original-bundle".into(),digest:"original-review".into()}; + let mut q = Quote { + review_digest: r.digest.clone(), + observed_at_unix: 2000, + valid_until_unix: 2000 + QUOTE_SECONDS, + price, + max_estimated_cost_usd: Some(100), + estimate_usd: Some(28.0), + ceiling_check: "within_estimated_ceiling".into(), + digest: String::new(), + }; + q.digest = q.digest().unwrap(); + validate(&q, &r, 2001).unwrap(); + assert!(validate(&q, &r, 1999).is_err()); + assert!(validate(&q, &r, 4000).is_err()); + q.review_digest = "other-plan".into(); + q.digest = q.digest().unwrap(); + assert!(validate(&q, &r, 2001).is_err()); + q.review_digest = r.digest.clone(); + q.max_estimated_cost_usd = Some(1); + q.ceiling_check = "exceeded".into(); + q.digest = q.digest().unwrap(); + assert!(validate(&q, &r, 2001).is_err()); + assert_eq!(r.deployment_id, "original"); + assert_eq!(r.native_bundle_digest, "original-bundle"); + } +} diff --git a/harness/service-cli/src/deployment_up_install.rs b/harness/service-cli/src/deployment_up_install.rs index b9dacaa9..0ca2ea79 100644 --- a/harness/service-cli/src/deployment_up_install.rs +++ b/harness/service-cli/src/deployment_up_install.rs @@ -291,8 +291,8 @@ impl Native<'_> { .filter(|ui| ui.enabled) { connection["managementUI"] = - json!({"url":ui.public_origin,"login":"existing service token"}); - let url = reqwest::Url::parse(&ui.public_origin)?; + json!({"url":ui.browser_origin()?,"login":"existing service token"}); + let url = reqwest::Url::parse(&ui.browser_origin()?)?; if matches!(url.host_str(), Some("localhost" | "127.0.0.1")) { connection["managementUI"]["privatePreview"] = json!({"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-ui",format!("{}:443",url.port_or_known_default().unwrap()),"--address=127.0.0.1"],"caFile":self.root.join("credentials/api.crt")}); } diff --git a/harness/service-cli/src/deployment_up_native.rs b/harness/service-cli/src/deployment_up_native.rs index 6a0bc3e9..4691a385 100644 --- a/harness/service-cli/src/deployment_up_native.rs +++ b/harness/service-cli/src/deployment_up_native.rs @@ -92,7 +92,7 @@ impl Native<'_> { let c = &self.review.config; let package = &self.package; let created = self.review.resolution.observed_at_unix; - let mut value = json!({"schemaVersion":if self.provider()=="aws"{3}else{2},"deploymentId":self.review.deployment_id,"name":c.name,"createdAtUnix":created,"expiresAtUnix":created+u64::from(c.trial.as_ref().map(|t|t.expires_after_hours).unwrap_or(72))*3600,"autoExpire":c.trial.is_some(),"budget":{"authorizedUsd":c.budget.alert_usd,"hardCap":false,"estimate":null},"topology":{"clusterMode":"new","nodeCount":c.nodes.count,"maxWorkerReplicas":2,"architecture":self.review.resolution.node.architecture,"access":"private","nodeAutoscaling":false}}); + let mut value = json!({"schemaVersion":if self.provider()=="aws"{3}else{2},"deploymentId":self.review.deployment_id,"name":c.name,"createdAtUnix":created,"expiresAtUnix":created+u64::from(c.trial.as_ref().map(|t|t.expires_after_hours).unwrap_or(72))*3600,"autoExpire":c.trial.is_some(),"budget":{"authorizedUsd":c.budget.as_ref().map(|b|b.alert_usd).unwrap_or(0),"estimateCheckDisabled":c.budget.is_none(),"hardCap":false,"estimate":null},"topology":{"clusterMode":"new","nodeCount":c.nodes.count,"maxWorkerReplicas":2,"architecture":self.review.resolution.node.architecture,"access":"private","nodeAutoscaling":false}}); let mut inputs = package.setup.infrastructure.clone(); if let Some(version) = &c.advanced.kubernetes_version { inputs["kubernetesVersion"] = json!(version); @@ -115,7 +115,9 @@ impl Native<'_> { .as_ref() .and_then(|a| a.operator_arn.as_ref()) .or(package.setup.operator_arn.as_ref()) - .context("AWS setup profile needs its approved IAM operatorArn")?; + .context( + "Set advanced.aws.operatorArn to the approved installer IAM identity", + )?; let prefix = format!("ows-{}-{}", c.name, &self.review.deployment_id[4..16]); value["bootstrap"] = json!({"operatorArn":operator,"executionRoleArn":format!("arn:aws:iam::{account_id}:role/{prefix}-executor")}); inputs["nodeInstanceType"] = json!(self.review.resolution.node.instance_type); @@ -139,7 +141,7 @@ impl Native<'_> { inputs["sshPublicKey"] = json!(fs::read_to_string(key)?.trim()); } } - value["azureBootstrap"] = json!({"resourceGroup":azure.and_then(|a| a.resource_group.as_ref()).or(package.setup.resource_group.as_ref()).context("Azure setup profile needs an approved workload resourceGroup")?,"setupObjectId":package.setup.setup_object_id.as_ref().context("Azure setup profile needs an exact setupObjectId")?,"setupPrincipalType":package.setup.setup_principal_type.as_deref().unwrap_or("ServicePrincipal")}); + value["azureBootstrap"] = json!({"resourceGroup":azure.and_then(|a| a.resource_group.as_ref()).or(package.setup.resource_group.as_ref()).context("Set advanced.azure.resourceGroup to the approved workload group")?,"setupObjectId":package.setup.setup_object_id.as_ref().context("Set advanced.azure.setupObjectId to the approved installer object ID")?,"setupPrincipalType":package.setup.setup_principal_type.as_deref().unwrap_or("ServicePrincipal")}); inputs["nodeVmSize"] = json!(self.review.resolution.node.instance_type); value["azureInfrastructure"] = inputs; } diff --git a/harness/service-cli/src/deployment_up_package.rs b/harness/service-cli/src/deployment_up_package.rs index 3a967875..bd02404c 100644 --- a/harness/service-cli/src/deployment_up_package.rs +++ b/harness/service-cli/src/deployment_up_package.rs @@ -187,25 +187,32 @@ impl Snapshot { } } pub(super) fn load(c: &Config) -> Result { - let path = if let Some(p) = &c.advanced.setup_profile { - p.clone() + let setup: Setup = if let Some(infrastructure) = &c.advanced.infrastructure { + let azure = c.advanced.azure.as_ref(); + Setup { + schema_version: 1, + provider: c.cloud.provider().into(), + account: c.cloud.account().into(), + region: c.cloud.region().into(), + infrastructure: infrastructure.clone(), + operator_arn: c.advanced.aws.as_ref().and_then(|a| a.operator_arn.clone()), + setup_object_id: azure.and_then(|a| a.setup_object_id.clone()), + setup_principal_type: azure.and_then(|a| a.setup_principal_type.clone()), + resource_group: azure.and_then(|a| a.resource_group.clone()), + release: c.advanced.release.clone().context( + "Set advanced.release to a generated release manifest in this deployment YAML", + )?, + } + } else if let Some(path) = &c.advanced.setup_profile { + ensure!( + fs::metadata(path)?.len() <= 65536, + "Legacy setup profile exceeds 64 KiB" + ); + serde_json::from_slice(&fs::read(path)?) + .map_err(|_| anyhow::anyhow!("Invalid legacy setup profile"))? } else { - let home = - std::env::var_os("HOME").context("HOME unavailable; set advanced.setupProfile")?; - PathBuf::from(home) - .join(".config/ows/deployments") - .join(c.cloud.provider()) - .join(c.cloud.account()) - .join(format!("{}.json", c.cloud.region())) + bail!("Set advanced.infrastructure, advanced.release and cloud identity overrides in deployment.yaml. No separate setup profile is required and no cloud resources were created.") }; - ensure!(path.is_file(),"Account setup profile is missing at {}. Install the reviewed account/release prerequisites once (see deploy/service/up/README.md), then rerun the same up command. No cloud resources were created.",path.display()); - ensure!( - fs::metadata(&path)?.len() <= 65536, - "Setup profile exceeds 64 KiB" - ); - let setup: Setup = serde_json::from_slice(&fs::read(&path)?).map_err(|_| { - anyhow::anyhow!("Invalid account setup profile; use references, not inline credentials") - })?; let release_path = c.advanced.release.as_ref().unwrap_or(&setup.release); ensure!( fs::metadata(release_path)?.len() <= 65536, diff --git a/harness/service-cli/tests/deployment_up.rs b/harness/service-cli/tests/deployment_up.rs index 0600fd50..74090179 100644 --- a/harness/service-cli/tests/deployment_up.rs +++ b/harness/service-cli/tests/deployment_up.rs @@ -49,7 +49,7 @@ fn inline_secret_or_wrong_cloud_fails_without_login_or_secret_echo() { } } #[test] -fn absent_account_profile_produces_prerequisite_instructions_without_cloud_calls() { +fn missing_inline_infrastructure_produces_instructions_without_cloud_calls() { let temp = tempfile::tempdir().unwrap(); let config = temp.path().join("deployment.yaml"); fs::write(&config,"name: example\ncloud: {provider: azure, subscriptionId: '11111111-2222-4333-8444-555555555555', region: westus2}\nnodes: {count: 2, instanceType: Standard_D4as_v7}\nbudget: {limitUSD: 100}\n").unwrap(); @@ -69,6 +69,41 @@ fn absent_account_profile_produces_prerequisite_instructions_without_cloud_calls .unwrap(); assert!(!r.status.success()); let error = String::from_utf8_lossy(&r.stderr); - assert!(error.contains("setup profile is missing"), "{error}"); + assert!(error.contains("advanced.infrastructure"), "{error}"); assert!(!temp.path().join(".ows").exists()); } + +#[test] +fn estimated_ceiling_is_explicitly_plan_only_and_unqualified_modes_never_execute_tools() { + let out = Command::new(BIN) + .args([ + "deploy", + "aws", + "up", + "--config", + "unused.yaml", + "--max-estimated-cost-usd", + "100", + ]) + .output() + .unwrap(); + assert!(!out.status.success()); + assert!(String::from_utf8_lossy(&out.stderr).contains("--plan-only")); + let d = tempfile::tempdir().unwrap(); + for mode in ["local-docker", "remote-k8s"] { + for action in ["plan", "up"] { + let o = Command::new(BIN) + .env_clear() + .env("HOME", d.path()) + .env("PATH", d.path()) + .args(["deploy", mode, action, "--config", "not-read.yaml"]) + .output() + .unwrap(); + assert!(!o.status.success()); + let status: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap(); + assert_eq!(status["supported"], false); + assert_eq!(status["mutations"], 0); + assert!(String::from_utf8_lossy(&o.stderr).contains("roadmap")); + } + } +} diff --git a/harness/service-cli/tests/gcp.rs b/harness/service-cli/tests/gcp.rs index fc07d811..94b35df2 100644 --- a/harness/service-cli/tests/gcp.rs +++ b/harness/service-cli/tests/gcp.rs @@ -103,7 +103,9 @@ impl Fixture { "--out", out.to_str().unwrap(), "--timeout-seconds", - "1", + // Python startup is not the timeout behavior being tested in + // ordinary scenarios. Keep the intentional timeout case at1s. + if scenario == "timeout" { "1" } else { "3" }, "--overall-timeout-seconds", // Leave room for Python fixture startup on loaded CI hosts. // Timeout scenarios still have a one-second per-call deadline. @@ -688,7 +690,15 @@ fn blocked_identity_policy_api_and_transport_cases_are_scrubbed_and_bounded() { ] .contains(&scenario) { - let calls = fs::read_to_string(f.home.path().join("calls.jsonl")).unwrap(); + let calls = fs::read_to_string(f.home.path().join("calls.jsonl")).unwrap_or_else(|e| { + // A bounded failure before the fixture starts means zero calls, + // not an absent authorization gate. Other I/O errors still fail. + if e.kind() == std::io::ErrorKind::NotFound { + String::new() + } else { + panic!("Cannot read fixture calls for {scenario}: {e}") + } + }); assert!(!calls.contains("get-iam-policy"), "{scenario}"); } } From d47ceb69efd09a146fc9f88e1c3b30356e6aebe0 Mon Sep 17 00:00:00 2001 From: Rohit P Date: Wed, 11 Nov 2026 19:07:57 -0800 Subject: [PATCH 04/11] Support reviewed OpenShell 0.1.8 deployments --- deploy/service/up/README.md | 2 +- harness/runtime/src/environment_openshell.rs | 14 ++--- .../src/deployment_azure_installation.rs | 2 +- harness/service-cli/src/deployment_local.rs | 3 +- .../service-cli/src/deployment_up_package.rs | 54 +++++++++++++++---- 5 files changed, 55 insertions(+), 20 deletions(-) diff --git a/deploy/service/up/README.md b/deploy/service/up/README.md index c8d26614..3686bcaf 100644 --- a/deploy/service/up/README.md +++ b/deploy/service/up/README.md @@ -82,7 +82,7 @@ wizard and publicly distributed release artifacts remain separate launch work. Package the matching release using `package.py --help`: Linux ARM64 for AWS, AMD64 for Azure, including `service-ui` when enabling the UI. New cloud deployments -require the reviewed OpenShell0.1.7 archive. The current version does not assume +require the reviewed OpenShell0.1.7 or0.1.8 archive. The current version does not assume public service images or a published Helm chart are already available. ## Cost estimates, not spending caps diff --git a/harness/runtime/src/environment_openshell.rs b/harness/runtime/src/environment_openshell.rs index 35a7dfae..ef5831a7 100644 --- a/harness/runtime/src/environment_openshell.rs +++ b/harness/runtime/src/environment_openshell.rs @@ -1,4 +1,4 @@ -//! Reviewed OpenShell 0.1.3/0.1.7 server provider. No CLI, local Docker fallback, host +//! Reviewed OpenShell 0.1.3/0.1.7/0.1.8 server provider. No CLI, local Docker fallback, host //! mounts or automatic lifecycle retry. Credential providers require an explicit //! original provider/profile pin; they are absent from the default profile. The gateway and //! cluster operator remain trusted; app-owned objects must use this exact profile. @@ -598,20 +598,20 @@ impl Connection for Stream { } // The lifecycle, authority, workspace and Exec wire fields used here are identical -// in the official v0.1.3 and v0.1.7 protobuf contracts. Keep an exact allowlist; +// in the official v0.1.3, v0.1.7 and v0.1.8 protobuf contracts. Keep an exact allowlist; // accepting a version does not replace functional qualification on each platform. fn checked_gateway(info: &Gateway) -> Result<()> { - if !matches!(info.version.as_str(), "0.1.3" | "0.1.7") + if !matches!(info.version.as_str(), "0.1.3" | "0.1.7" | "0.1.8") || info.drivers.len() != 1 || info.drivers[0].name != "kubernetes" { bail!( - "Server provider requires reviewed OpenShell 0.1.3 or 0.1.7 with the Kubernetes driver" + "Server provider requires reviewed OpenShell 0.1.3, 0.1.7 or 0.1.8 with the Kubernetes driver" ); } Ok(()) } -// Minimal wire messages shared by the reviewed public 0.1.3/0.1.7 protocols. Presence-only +// Minimal wire messages shared by the reviewed public 0.1.3/0.1.7/0.1.8 protocols. Presence-only // fields use bytes/Empty when their contents are never authorized by this profile. #[derive(Clone, PartialEq, Message)] struct Empty {} @@ -858,7 +858,7 @@ mod tests { use super::*; #[test] fn gateway_versions_are_exact_and_driver_authority_is_not_broadened() { - for version in ["0.1.3", "0.1.7"] { + for version in ["0.1.3", "0.1.7", "0.1.8"] { let mut info = Gateway { version: version.into(), drivers: vec![Driver { @@ -877,7 +877,7 @@ mod tests { info.drivers.clear(); assert!(checked_gateway(&info).is_err()); } - for version in ["0.1.4", "0.1.6", "0.1.8", "0.1.7-dev", "latest", ""] { + for version in ["0.1.4", "0.1.6", "0.1.9", "0.1.8-dev", "latest", ""] { assert!(checked_gateway(&Gateway { version: version.into(), drivers: vec![Driver { diff --git a/harness/service-cli/src/deployment_azure_installation.rs b/harness/service-cli/src/deployment_azure_installation.rs index c207b962..6dd03335 100644 --- a/harness/service-cli/src/deployment_azure_installation.rs +++ b/harness/service-cli/src/deployment_azure_installation.rs @@ -488,7 +488,7 @@ not contain admin certificates, bearer tokens, database passwords or model keys. The installer needs a private network/DNS path to AKS, `kubelogin` with Azure CLI login in the pinned tenant, `kubectl`, Helm, a reviewed platform manifest and the -reviewed, pinned OpenShell archive/images (0.1.3 or 0.1.7). Installing OpenShell/Agent Sandbox requires +reviewed, pinned OpenShell archive/images (0.1.3, 0.1.7 or 0.1.8). Installing OpenShell/Agent Sandbox requires reviewed cluster-scoped Kubernetes rights; namespace RBAC alone is insufficient. No provider/plugin/component is installed implicitly by preparing this handoff. diff --git a/harness/service-cli/src/deployment_local.rs b/harness/service-cli/src/deployment_local.rs index c6c0e7fe..e8e25cce 100644 --- a/harness/service-cli/src/deployment_local.rs +++ b/harness/service-cli/src/deployment_local.rs @@ -16,6 +16,7 @@ mod platform; const CHART_PINS: &[&str] = &[ "5a82e27d6cf4089fa16d96c87d9fc0fe62f65fa05341465bc749ba5ab91abb33", // 0.1.3 "5ce3e63648d1a80f96f748063e44444069667ae9049457b8aded35cdb2a6171a", // 0.1.7 official OCI archive + "271368019433b7e5c0668757cdd880302a044924eb37077aef82a95d814efd84", // 0.1.8 official OCI archive ]; #[derive(Subcommand)] pub enum Command { @@ -456,7 +457,7 @@ fn inputs(c: &Config) -> Result> { let sha = hash_path(archive)?; ensure!( CHART_PINS.contains(&sha.as_str()), - "OpenShell archive differs from reviewed 0.1.3/0.1.7 pins" + "OpenShell archive differs from reviewed 0.1.3/0.1.7/0.1.8 pins" ); let v = read_json(values_file)?; ensure!( diff --git a/harness/service-cli/src/deployment_up_package.rs b/harness/service-cli/src/deployment_up_package.rs index bd02404c..0a1695d9 100644 --- a/harness/service-cli/src/deployment_up_package.rs +++ b/harness/service-cli/src/deployment_up_package.rs @@ -156,18 +156,9 @@ impl Snapshot { ] { f.validate()?; } - ensure!( - self.release.openshell_archive.sha256 - == "5ce3e63648d1a80f96f748063e44444069667ae9049457b8aded35cdb2a6171a", - "New deployment requires the reviewed OpenShell 0.1.7 archive" - ); let values: Value = serde_json::from_slice(&fs::read(&self.release.openshell_values.file)?)?; - ensure!( - values["global"]["image"]["tag"] == "0.1.7" - && values["server"]["auth"]["allowUnauthenticatedUsers"] == false, - "OpenShell release/authentication mismatch" - ); + check_openshell_release(&self.release.openshell_archive.sha256, &values)?; let ca = fs::read(&self.release.database_ca.file)?; let certificates = reqwest::Certificate::from_pem_bundle(&ca).context("Invalid database CA bundle")?; @@ -263,3 +254,46 @@ mod tests { assert!(tree(d.path()).is_err()); } } + +fn check_openshell_release(sha: &str, values: &Value) -> Result<()> { + let version = match sha { + "5ce3e63648d1a80f96f748063e44444069667ae9049457b8aded35cdb2a6171a" => "0.1.7", + "271368019433b7e5c0668757cdd880302a044924eb37077aef82a95d814efd84" => "0.1.8", + _ => anyhow::bail!("New deployment requires a reviewed OpenShell 0.1.7/0.1.8 archive"), + }; + ensure!( + values["global"]["image"]["tag"] == version + && values["server"]["auth"]["allowUnauthenticatedUsers"] == false, + "OpenShell release/authentication mismatch" + ); + Ok(()) +} + +#[cfg(test)] +mod release_pin_tests { + use super::*; + #[test] + fn chart_hash_matches_exact_authenticated_release() { + for (hash, version) in [ + ( + "5ce3e63648d1a80f96f748063e44444069667ae9049457b8aded35cdb2a6171a", + "0.1.7", + ), + ( + "271368019433b7e5c0668757cdd880302a044924eb37077aef82a95d814efd84", + "0.1.8", + ), + ] { + let mut v = json!({"global":{"image":{"tag":version}},"server":{"auth":{"allowUnauthenticatedUsers":false}}}); + check_openshell_release(hash, &v).unwrap(); + v["global"]["image"]["tag"] = json!("latest"); + assert!(check_openshell_release(hash, &v).is_err()); + v["global"]["image"]["tag"] = json!(if version == "0.1.7" { "0.1.8" } else { "0.1.7" }); + assert!(check_openshell_release(hash, &v).is_err()); + v["global"]["image"]["tag"] = json!(version); + v["server"]["auth"]["allowUnauthenticatedUsers"] = json!(true); + assert!(check_openshell_release(hash, &v).is_err()); + } + assert!(check_openshell_release(&"a".repeat(64), &json!({})).is_err()); + } +} From 7a0ea3c203deefc82c95b64dfdc59c6777d1e9e8 Mon Sep 17 00:00:00 2001 From: Rohit P Date: Wed, 11 Nov 2026 20:31:42 -0800 Subject: [PATCH 05/11] Fix private Azure image publication across Docker image stores --- harness/service-cli/src/deployment_up.rs | 1 + .../src/deployment_up_azure_images.rs | 193 +++++++++++++++--- 2 files changed, 166 insertions(+), 28 deletions(-) diff --git a/harness/service-cli/src/deployment_up.rs b/harness/service-cli/src/deployment_up.rs index b8d451e6..5d772b9d 100644 --- a/harness/service-cli/src/deployment_up.rs +++ b/harness/service-cli/src/deployment_up.rs @@ -173,6 +173,7 @@ async fn check_local_tools(c: &Config, package: &package::Snapshot) -> Result<() ("openssl", vec!["version"]), ]; if c.cloud.provider() == "azure" { + tools.push(("docker", vec!["buildx", "version"])); tools.push(("kubelogin", vec!["--version"])); if package .setup diff --git a/harness/service-cli/src/deployment_up_azure_images.rs b/harness/service-cli/src/deployment_up_azure_images.rs index 89c16d46..aaa4cfb0 100644 --- a/harness/service-cli/src/deployment_up_azure_images.rs +++ b/harness/service-cli/src/deployment_up_azure_images.rs @@ -3,6 +3,52 @@ use super::*; use zeroize::Zeroizing; struct Temporary(PathBuf); +fn source_acr(source: &str) -> Result> { + let host = source.split('/').next().context("Image registry missing")?; + let Some(name) = host.strip_suffix(".azurecr.io") else { + return Ok(None); + }; + ensure!( + (5..=50).contains(&name.len()) + && name + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit()), + "Invalid private source registry" + ); + Ok(Some((name.into(), host.into()))) +} +fn verified_publication(source: &str, local_id: &str, digest: &str, raw: &[u8]) -> bool { + let digest_matches = format!("sha256:{}", hash(raw)) == digest + || raw + .strip_suffix(b"\n") + .is_some_and(|bytes| format!("sha256:{}", hash(bytes)) == digest); + let Ok(manifest) = serde_json::from_slice::(raw) else { + return false; + }; + // Containerd image stores may expose an index digest as Docker's image Id. + // Exact preservation of the reviewed source manifest/index is strongest; + // legacy stores may instead publish a manifest with the same config Id. + digest_matches + && (source + .rsplit_once('@') + .is_some_and(|(_, pin)| pin == digest) + || manifest["config"]["digest"] == local_id) +} +fn plugin_config(plugins: &Value) -> Result { + let entries: Vec<_> = plugins + .as_array() + .context("Docker plugin inventory missing")? + .iter() + .filter(|p| p["Name"] == "buildx") + .collect(); + ensure!( + entries.len() == 1, + "Install one working Docker Buildx plugin" + ); + let path = Path::new(entries[0]["Path"].as_str().context("Buildx path missing")?); + ensure!(path.is_absolute(), "Buildx plugin path must be absolute"); + Ok(json!({"cliPluginsExtraDirs":[path.parent().context("Buildx directory missing")?]})) +} impl Drop for Temporary { fn drop(&mut self) { let _ = fs::remove_dir_all(&self.0); @@ -65,28 +111,18 @@ impl Native<'_> { ); Ok(Some(digest.into())) } - pub(super) async fn azure_images(&self) -> Result { - let material = self.azure_material().await?; - let registry = text(&material["registry"], "name")?; - let host = text(&material["registry"], "loginServer")?; - ensure!( - host == format!("{registry}.azurecr.io") - && material["registry"]["createdAt"].is_string(), - "Owned ACR identity missing" - ); - let directory_path = self - .root - .join(format!(".registry-auth-{}", uuid::Uuid::new_v4().simple())); - directory(&directory_path)?; - let _temporary = Temporary(directory_path.clone()); - let mut auth_env = self.environment()?; - auth_env.insert("DOCKER_CONFIG".into(), arg(&directory_path)?); + async fn login_acr( + &self, + registry: &str, + host: &str, + auth_env: &BTreeMap, + ) -> Result<()> { let mut login = self .az_json(strings(&[ "acr", "login", "--name", - ®istry, + registry, "--expose-token", ])) .await?; @@ -112,15 +148,50 @@ impl Native<'_> { Path::new("docker"), &strings(&[ "login", - &host, + host, "--username", "00000000-0000-0000-0000-000000000000", "--password-stdin", ]), Some(token.as_bytes().to_vec()), - &auth_env, + auth_env, ) .await?; + Ok(()) + } + pub(super) async fn azure_images(&self) -> Result { + let material = self.azure_material().await?; + let registry = text(&material["registry"], "name")?; + let host = text(&material["registry"], "loginServer")?; + ensure!( + host == format!("{registry}.azurecr.io") + && material["registry"]["createdAt"].is_string(), + "Owned ACR identity missing" + ); + let directory_path = self + .root + .join(format!(".registry-auth-{}", uuid::Uuid::new_v4().simple())); + directory(&directory_path)?; + let _temporary = Temporary(directory_path.clone()); + // Private auth must not hide user-installed CLI plugins. Copy only the + // discovered plugin directory, never the global Docker auth/config file. + let plugins: Value = serde_json::from_slice( + &self + .tool( + "docker", + strings(&["info", "--format", "{{json .ClientInfo.Plugins}}"]), + None, + ) + .await?, + )?; + write_private( + &directory_path.join("config.json"), + &plugin_config(&plugins)?, + )?; + let mut auth_env = self.environment()?; + auth_env.insert("DOCKER_CONFIG".into(), arg(&directory_path)?); + self.login_acr(®istry, &host, &auth_env).await?; + let mut authenticated = std::collections::BTreeSet::from([host.clone()]); let mut results = BTreeMap::::new(); for (role, source) in [ ("harness", Some(&self.package.release.images.harness)), @@ -131,10 +202,16 @@ impl Native<'_> { ), ] { let Some(source) = source else { continue }; - self.tool( - "docker", - strings(&["pull", "--platform", "linux/amd64", source]), + if let Some((name, source_host)) = source_acr(source)? { + if authenticated.insert(source_host.clone()) { + self.login_acr(&name, &source_host, &auth_env).await?; + } + } + super::super::super::local::exec_with_env( + Path::new("docker"), + &strings(&["pull", "--platform", "linux/amd64", source]), None, + &auth_env, ) .await?; let local: Value = serde_json::from_slice( @@ -199,13 +276,8 @@ impl Native<'_> { &auth_env, ) .await?; - let manifest: Value = serde_json::from_slice(&raw)?; ensure!( - manifest["config"]["digest"] == id - && (format!("sha256:{}", hash(&raw)) == digest - || raw - .strip_suffix(b"\n") - .is_some_and(|b| format!("sha256:{}", hash(b)) == digest)), + verified_publication(source, &id, &digest, &raw), "Remote image content differs from the reviewed local image" ); receipt["image"] = json!(pinned); @@ -263,3 +335,68 @@ impl Native<'_> { ))) } } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn isolated_auth_preserves_only_buildx_discovery_without_copying_global_credentials() { + let value = plugin_config(&json!([{"Name":"buildx","Path":"/opt/docker/cli-plugins/docker-buildx","secret":"must-not-copy"}])).unwrap(); + assert_eq!( + value, + json!({"cliPluginsExtraDirs":["/opt/docker/cli-plugins"]}) + ); + assert!(plugin_config(&json!([])).is_err()); + assert!(plugin_config(&json!([{"Name":"buildx","Path":"relative/plugin"}])).is_err()); + } + #[test] + fn private_source_registry_is_exact_and_public_sources_need_no_login() { + assert_eq!( + source_acr("control123.azurecr.io/runtime@sha256:abc").unwrap(), + Some(("control123".into(), "control123.azurecr.io".into())) + ); + assert_eq!( + source_acr("ghcr.io/nvidia/runtime@sha256:abc").unwrap(), + None + ); + assert!(source_acr("wrong.nested.azurecr.io/image").is_err()); + assert!(source_acr("user@control123.azurecr.io/image").is_err()); + } + #[test] + fn publication_accepts_exact_reviewed_index_or_verified_legacy_config_only() { + let index = br#"{"schemaVersion":2,"manifests":[]}"#; + let digest = format!("sha256:{}", hash(index)); + assert!(verified_publication( + &format!("registry/image@{digest}"), + &digest, + &digest, + index + )); + assert!(!verified_publication( + "registry/image@sha256:other", + &digest, + &digest, + index + )); + let manifest = br#"{"schemaVersion":2,"config":{"digest":"sha256:config"}}"#; + let digest = format!("sha256:{}", hash(manifest)); + assert!(verified_publication( + "registry/image@sha256:source", + "sha256:config", + &digest, + manifest + )); + assert!(!verified_publication( + "registry/image@sha256:source", + "sha256:wrong", + &digest, + manifest + )); + assert!(!verified_publication( + &format!("registry/image@{digest}"), + "sha256:config", + &digest, + index + )); + } +} From 95d175a2c7a2a0f09514b4cfe05cc2d55a0dede5 Mon Sep 17 00:00:00 2001 From: Rohit P Date: Fri, 13 Nov 2026 12:20:33 -0800 Subject: [PATCH 06/11] fix: preserve private installation access and recording schema setup --- deploy/service/chart.test.mjs | 11 +++ deploy/service/chart/templates/workloads.yaml | 7 +- deploy/service/examples/README.md | 14 ++-- deploy/service/up/README.md | 32 ++++++--- docs/harness/guides/observability.md | 10 +-- harness/service-cli/README.md | 14 ++-- harness/service-cli/src/deployment_up.rs | 34 +++++++++ .../src/deployment_up_azure_images.rs | 71 ++++++++++++++++++- .../service-cli/src/deployment_up_native.rs | 5 ++ 9 files changed, 165 insertions(+), 33 deletions(-) diff --git a/deploy/service/chart.test.mjs b/deploy/service/chart.test.mjs index 923f65e7..f672703c 100644 --- a/deploy/service/chart.test.mjs +++ b/deploy/service/chart.test.mjs @@ -173,6 +173,17 @@ test("invalid ownership, mutable image and inadequate drain grace fail rendering }), ); }); +test("Azure trace schema initializer is excluded from workload credential injection", async()=>{ + const artifacts={kind:"azure",deployment:"ows-test",account:"owteststorage",container:"artifacts",tenant_id:"11111111-1111-4111-8111-111111111111",client_id:"22222222-2222-4222-8222-222222222222",federated_token_file:"/var/run/secrets/azure/tokens/azure-identity-token"}; + const out=await render({config:{...base.config,artifacts,observability:{retention_days:14}}}); + for(const role of ["api","worker"]){ + const d=deployment(out,role); + assert.match(d,/azure.workload.identity\/skip-containers: "harness(?:;tls)?;recording-schema"/); + const init=d.split("initContainers:")[1].split(" containers:")[0]; + assert.doesNotMatch(init,/azure-artifact-token|azure-identity-token|AZURE_/); + assert.equal((d.match(/name: azure-artifact-token\n/g)||[]).length,1); + } +}); test("management UI has no execution, database, or cluster authority", async () => { const uiValues = { enabled: true, diff --git a/deploy/service/chart/templates/workloads.yaml b/deploy/service/chart/templates/workloads.yaml index d255fb1f..2e3615bd 100644 --- a/deploy/service/chart/templates/workloads.yaml +++ b/deploy/service/chart/templates/workloads.yaml @@ -36,8 +36,11 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/config.yaml") $ | sha256sum }} {{- if $azure }} # This chart owns its explicit projection; prevent webhook injection of - # another token path or credentials into the TLS sidecar. - azure.workload.identity/skip-containers: {{ if eq $role "api" }}"harness;tls"{{ else }}"harness"{{ end }} + # another token path or credentials into TLS/schema-only containers. + {{- $skip := list "harness" -}} + {{- if eq $role "api" }}{{ $skip = append $skip "tls" }}{{ end -}} + {{- if hasKey $.Values.config "observability" }}{{ $skip = append $skip "recording-schema" }}{{ end }} + azure.workload.identity/skip-containers: {{ join ";" $skip | quote }} {{- end }} spec: automountServiceAccountToken: {{ eq $role "platform" }} diff --git a/deploy/service/examples/README.md b/deploy/service/examples/README.md index 6f6cb2a6..2a5438ec 100644 --- a/deploy/service/examples/README.md +++ b/deploy/service/examples/README.md @@ -22,9 +22,9 @@ data only, optionally add `trial: {expiresAfterHours: 6, deleteData: true}`. ## Azure and AWS ```sh -ows deploy azure up --config azure/deployment.yaml --plan-only \ +owcluster deploy azure up --config azure/deployment.yaml --plan-only \ --max-estimated-cost-usd 100 -ows deploy aws up --config aws/deployment.yaml --plan-only +owcluster deploy aws up --config aws/deployment.yaml --plan-only ``` A cost ceiling is optional, is plan-only, and checks an estimate rather than actual @@ -72,11 +72,11 @@ approval. Kubernetes must be able to reach PostgreSQL and pull/use those images. `scripted` is a local fixture option, not a substitute for live-provider qualification. ```sh -ows deploy local-k8s plan --config local-k8s/deployment.yaml --state .ows/local -ows deploy local-k8s up --config local-k8s/deployment.yaml --state .ows/local \ +owcluster deploy local-k8s plan --config local-k8s/deployment.yaml --state .ows/local +owcluster deploy local-k8s up --config local-k8s/deployment.yaml --state .ows/local \ --accept-plan PLAN_DIGEST -ows deploy local-k8s status --state .ows/local -ows management-ui status --state .ows/local/installation +owcluster deploy local-k8s status --state .ows/local +owcluster management-ui status --state .ows/local/installation ``` Use the actual digest printed by plan. The shared installer checks cluster and @@ -90,7 +90,7 @@ No new sandbox/model run is implied by successful installation readiness. The CLI generates the private API certificate and maps `managementUI.port` to the loopback preview. Keep and trust that CA; do not disable TLS verification. Set UI/recording preferences in YAML or use the independent feature commands. -Native lower-level `ows deploy local` / `deploy cluster` remain compatibility +Native lower-level `owcluster deploy local` / `deploy cluster` remain compatibility interfaces for existing journals; new examples use `local-k8s`. The new `local-docker` and `remote-k8s` command names are discoverable but report diff --git a/deploy/service/up/README.md b/deploy/service/up/README.md index 3686bcaf..f8ddffe5 100644 --- a/deploy/service/up/README.md +++ b/deploy/service/up/README.md @@ -10,8 +10,8 @@ schema. The service installer remains experimental until the current release passes a fresh cloud install, agent/recovery and cleanup qualification. ```sh -ows deploy azure up --config staging-deploy.yaml --plan-only -ows deploy aws up --config prod-deploy.yaml --plan-only +owcluster deploy azure up --config staging-deploy.yaml --plan-only +owcluster deploy aws up --config prod-deploy.yaml --plan-only ``` Remove `--plan-only` to review/approve provisioning. `--accept-plan DIGEST` and @@ -67,7 +67,7 @@ in deployment YAML resolve against that YAML's directory. Generated release manifest paths are absolute. Legacy explicit `advanced.setupProfile` import is compatibility-only; new files do not use it or a hidden default account profile. -Install `ows`, the cloud CLI, Docker, Helm, kubectl and OpenSSL. Azure also needs +Install `owcluster`, the cloud CLI, Docker, Helm, kubectl and OpenSSL. Azure also needs kubelogin and SSH for its optional management VM; AWS's SSM bridge needs the Session Manager plugin. Sign in using your existing cloud identity. The installer requires the reviewed provisioning/RBAC scopes, not merely a successful login. @@ -91,7 +91,7 @@ There is no required `budget` field in new deployment YAML. Check an optional estimated-cost ceiling in plan mode: ```sh -ows deploy azure up --config staging-deploy.yaml --plan-only \ +owcluster deploy azure up --config staging-deploy.yaml --plan-only \ --max-estimated-cost-usd 100 ``` @@ -134,13 +134,23 @@ pending a separately reviewed data-retention workflow. emits port-forward commands and CA information, not tokens in URLs. Use a current service token to sign in. This does not create public ingress, DNS or a certificate for a company domain. Existing ingress users can retain the legacy explicit origin; -`ows management-ui enable --ingress-url https://manage.example.com` selects it. +`owcluster management-ui enable --ingress-url https://manage.example.com` selects it. Do not combine an explicit ingress origin and preview port. Use `--keep-access` to retain AWS SSM/Azure SSH private management connectivity. The original Kubernetes endpoint/hostname/CA stay verified through the loopback route. `connection.json` records API and enabled UI connection instructions. +After an installation has recorded its original private-access route, +`--access-only --keep-access` reopens that route for diagnosis without advancing +installation or replacing resources. Stop the command to close the route. + +For Azure operators whose installation host cannot reach the database bootstrap +secret in Key Vault, `--database-password-file /private/database-password` can +supply that same database password. The file must be a bounded regular UTF-8 file +with private permissions (0600 on Unix). Its content is not printed or saved in +the installation review; the database host and TLS checks remain unchanged. + UI and recording are independent and disabled unless configured. Recording uses its own connection/schema on execution PostgreSQL by default; an existing Secret selected by `observability.databaseSecret` may provide `database-url` and `ca.crt` @@ -148,12 +158,12 @@ for a separate database. That Secret must be staged in the owned namespace befor service installation. Enabling initializes its schema before startup. ```sh -ows management-ui enable --state ./deployment-state/installation --port 8444 -ows management-ui disable --state ./deployment-state/installation -ows management-ui status --state ./deployment-state/installation -ows observability enable --state ./deployment-state/installation --retention-days 14 -ows observability disable --state ./deployment-state/installation -ows observability status --state ./deployment-state/installation +owcluster management-ui enable --state ./deployment-state/installation --port 8444 +owcluster management-ui disable --state ./deployment-state/installation +owcluster management-ui status --state ./deployment-state/installation +owcluster observability enable --state ./deployment-state/installation --retention-days 14 +owcluster observability disable --state ./deployment-state/installation +owcluster observability status --state ./deployment-state/installation ``` The commands save feature overrides in the installer journal, retain the original diff --git a/docs/harness/guides/observability.md b/docs/harness/guides/observability.md index 8b91e8a4..7a149281 100644 --- a/docs/harness/guides/observability.md +++ b/docs/harness/guides/observability.md @@ -171,13 +171,13 @@ continues to control audit reads. Cluster experiment/review tooling comes later. The source CLI supports independent UI and trace-storage controls: ```sh -ows observability enable --state ./deployment-state/installation --retention-days 14 -ows observability status --state ./deployment-state/installation -ows observability disable --state ./deployment-state/installation -ows management-ui enable --state ./deployment-state/installation +owcluster observability enable --state ./deployment-state/installation --retention-days 14 +owcluster observability status --state ./deployment-state/installation +owcluster observability disable --state ./deployment-state/installation +owcluster management-ui enable --state ./deployment-state/installation ``` -Use the installer state created by `ows deploy aws up` or `ows deploy azure up`. +Use the installer state created by `owcluster deploy aws up` or `owcluster deploy azure up`. Enabling recording initializes its separate PostgreSQL schema before workloads restart. Disabling stops collection after the rollout and retains stored traces; it does not disable security audit or a separately configured exporter. diff --git a/harness/service-cli/README.md b/harness/service-cli/README.md index ae50844b..cbb12647 100644 --- a/harness/service-cli/README.md +++ b/harness/service-cli/README.md @@ -95,7 +95,7 @@ See [AWS deployment](../../docs/harness/guides/aws.md) and [deployment options](../../docs/harness/guides/deployment.md). Use an owned trial with an explicit budget and independent cleanup before production deployment. -`owcluster` is the primary command. `ows` remains an alias, using the same commands, configuration and saved deployment state. Existing `.ows` paths and `OWS_*` settings remain valid. +`owcluster` is the primary command. `owcluster` remains an alias, using the same commands, configuration and saved deployment state. Existing `.ows` paths and `OWS_*` settings remain valid. ## Optional management UI and trace storage @@ -103,12 +103,12 @@ Select these features in `deployment.yaml` for the AWS/Azure `up` commands, or change them on an existing installation: ```sh -ows management-ui enable --state ./deployment-state/installation --port 8444 -ows management-ui disable --state ./deployment-state/installation -ows management-ui status --state ./deployment-state/installation -ows observability enable --state ./deployment-state/installation --retention-days 14 -ows observability disable --state ./deployment-state/installation -ows observability status --state ./deployment-state/installation +owcluster management-ui enable --state ./deployment-state/installation --port 8444 +owcluster management-ui disable --state ./deployment-state/installation +owcluster management-ui status --state ./deployment-state/installation +owcluster observability enable --state ./deployment-state/installation --retention-days 14 +owcluster observability disable --state ./deployment-state/installation +owcluster observability status --state ./deployment-state/installation ``` `--state` is the original installer journal, using its Kubernetes credentials and diff --git a/harness/service-cli/src/deployment_up.rs b/harness/service-cli/src/deployment_up.rs index 5d772b9d..2815a829 100644 --- a/harness/service-cli/src/deployment_up.rs +++ b/harness/service-cli/src/deployment_up.rs @@ -36,11 +36,18 @@ pub struct Options { /// Separate approval of scoped identity grants in the exact reviewed plan. #[arg(long)] accept_bootstrap: Option, + /// Azure only: private password file matching the reviewed database secret. + /// Avoids reading Key Vault data plane from the installer network. + #[arg(long)] + database_password_file: Option, #[arg(long, default_value_t = 3600)] timeout_seconds: u64, /// Keep the private management route open after installation; Ctrl-C closes it. #[arg(long)] keep_access: bool, + /// Reopen the original private route for diagnosis without advancing installation. + #[arg(long, requires = "keep_access", conflicts_with = "plan_only")] + access_only: bool, } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -199,6 +206,10 @@ async fn check_local_tools(c: &Config, package: &package::Snapshot) -> Result<() } pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { let config = Config::load(&o.config, provider)?; + ensure!( + o.database_password_file.is_none() || provider == "azure", + "database-password-file is Azure-only" + ); let package = package::load(&config)?; ensure!( (10..=14400).contains(&o.timeout_seconds), @@ -252,6 +263,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { review: &value, package: package.clone(), bootstrap_approved: false, + database_password_file: None, bridge: Default::default(), } .artifact_digest()?; @@ -266,6 +278,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { review: &review, package: package.clone(), bootstrap_approved: false, + database_password_file: None, bridge: Default::default() } .artifact_digest()? @@ -281,12 +294,16 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { review: &review, package: package.clone(), bootstrap_approved: false, + database_password_file: None, bridge: Default::default(), } .review_bundle() .await?; return cost::review(&state, &review, &package, o.max_estimated_cost_usd).await; } + if let Some(path) = &o.database_password_file { + native::validate_password_input(path)?; + } cost::validate_approval(&state, &review)?; if let Some(subtotal) = review .resolution @@ -304,6 +321,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { review: &review, package: package.clone(), bootstrap_approved: false, + database_password_file: None, bridge: Default::default(), }; preview.review_bundle().await?; @@ -326,6 +344,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { review: &review, package, bootstrap_approved: true, + database_password_file: o.database_password_file.clone(), bridge: Default::default(), }; let path = state.join("up-journal.json"); @@ -337,6 +356,21 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { { driver.restore_access().await?; } + if o.access_only { + ensure!( + journal + .stages + .get(&state::Stage::PrivateAccess) + .is_some_and(|s| s.phase == state::Phase::Complete), + "Original private access receipt is required" + ); + crate::output( + json!({"phase":"access_ready","state":state,"installationAdvanced":false,"close":"Ctrl-C"}), + )?; + tokio::signal::ctrl_c().await?; + driver.close_access().await?; + return Ok(()); + } let previously_complete = journal.complete(); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(o.timeout_seconds); loop { diff --git a/harness/service-cli/src/deployment_up_azure_images.rs b/harness/service-cli/src/deployment_up_azure_images.rs index aaa4cfb0..7e0783f2 100644 --- a/harness/service-cli/src/deployment_up_azure_images.rs +++ b/harness/service-cli/src/deployment_up_azure_images.rs @@ -3,6 +3,52 @@ use super::*; use zeroize::Zeroizing; struct Temporary(PathBuf); +pub(super) fn private_password_file(path: &Path) -> Result>> { + use std::io::Read; + let meta = fs::symlink_metadata(path).context("Private database password file unavailable")?; + ensure!( + meta.is_file() && !meta.file_type().is_symlink() && meta.len() > 0 && meta.len() <= 4098, + "Use a bounded regular private password file" + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + ensure!( + meta.permissions().mode() & 0o077 == 0, + "Database password file must be private (chmod 600)" + ); + } + let file = fs::File::open(path).context("Private database password read failed")?; + let opened = file.metadata()?; + ensure!( + opened.is_file() && opened.len() == meta.len(), + "Database password file changed while opening" + ); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + ensure!( + opened.dev() == meta.dev() && opened.ino() == meta.ino() && opened.mode() & 0o077 == 0, + "Database password file identity changed" + ); + } + let mut bytes = Zeroizing::new(vec![]); + file.take(4099) + .read_to_end(&mut bytes) + .context("Private database password read failed")?; + ensure!( + bytes.len() <= 4098 && bytes.len() as u64 == opened.len(), + "Database password file changed while reading" + ); + let text = std::str::from_utf8(&bytes) + .context("Invalid database password encoding")? + .trim_end_matches(['\r', '\n']); + ensure!( + !text.is_empty() && text.len() <= 4096 && !text.contains(['\r', '\n', '\0']), + "Invalid private database password" + ); + Ok(bytes) +} fn source_acr(source: &str) -> Result> { let host = source.split('/').next().context("Image registry missing")?; let Some(name) = host.strip_suffix(".azurecr.io") else { @@ -311,7 +357,11 @@ impl Native<'_> { "--subscription", self.review.config.cloud.account(), ]); - let raw = Zeroizing::new(self.tool("az", std::mem::take(&mut args), None).await?); + let raw = if let Some(path) = &self.database_password_file { + private_password_file(path)? + } else { + Zeroizing::new(self.tool("az", std::mem::take(&mut args), None).await?) + }; let password = std::str::from_utf8(&raw) .context("Invalid secret encoding")? .trim_end_matches(['\r', '\n']); @@ -339,6 +389,25 @@ impl Native<'_> { #[cfg(test)] mod tests { use super::*; + #[cfg(unix)] + #[test] + fn password_file_requires_private_regular_bounded_input_without_echoing_values() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("password"); + fs::write(&path, b"private-canary\n").unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + assert!(private_password_file(&path).is_ok()); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + let error = private_password_file(&path).unwrap_err().to_string(); + assert!(!error.contains("private-canary")); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + fs::write(&path, b"multiple\nlines").unwrap(); + assert!(private_password_file(&path).is_err()); + let link = temp.path().join("link"); + std::os::unix::fs::symlink(&path, &link).unwrap(); + assert!(private_password_file(&link).is_err()); + } #[test] fn isolated_auth_preserves_only_buildx_discovery_without_copying_global_credentials() { let value = plugin_config(&json!([{"Name":"buildx","Path":"/opt/docker/cli-plugins/docker-buildx","secret":"must-not-copy"}])).unwrap(); diff --git a/harness/service-cli/src/deployment_up_native.rs b/harness/service-cli/src/deployment_up_native.rs index 4691a385..38abb9db 100644 --- a/harness/service-cli/src/deployment_up_native.rs +++ b/harness/service-cli/src/deployment_up_native.rs @@ -9,11 +9,15 @@ mod access; mod azure_images; #[path = "deployment_up_install.rs"] mod install; +pub(super) fn validate_password_input(path: &Path) -> Result<()> { + azure_images::private_password_file(path).map(|_| ()) +} pub(super) struct Native<'a> { pub root: PathBuf, pub review: &'a Review, pub package: package::Snapshot, pub bootstrap_approved: bool, + pub database_password_file: Option, pub bridge: std::cell::RefCell>, } fn arg(path: &Path) -> Result { @@ -654,6 +658,7 @@ mod contract_tests { review: &review, package, bootstrap_approved: false, + database_password_file: None, bridge: Default::default(), }; let plan = native.native_plan().unwrap(); From 6f8eec295fd05a8cb0c8af2bccbdeb99b181140d Mon Sep 17 00:00:00 2001 From: Rohit P Date: Fri, 13 Nov 2026 12:41:09 -0800 Subject: [PATCH 07/11] test: keep unrelated GCP fixtures unchanged --- harness/service-cli/tests/gcp.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/harness/service-cli/tests/gcp.rs b/harness/service-cli/tests/gcp.rs index 94b35df2..fc07d811 100644 --- a/harness/service-cli/tests/gcp.rs +++ b/harness/service-cli/tests/gcp.rs @@ -103,9 +103,7 @@ impl Fixture { "--out", out.to_str().unwrap(), "--timeout-seconds", - // Python startup is not the timeout behavior being tested in - // ordinary scenarios. Keep the intentional timeout case at1s. - if scenario == "timeout" { "1" } else { "3" }, + "1", "--overall-timeout-seconds", // Leave room for Python fixture startup on loaded CI hosts. // Timeout scenarios still have a one-second per-call deadline. @@ -690,15 +688,7 @@ fn blocked_identity_policy_api_and_transport_cases_are_scrubbed_and_bounded() { ] .contains(&scenario) { - let calls = fs::read_to_string(f.home.path().join("calls.jsonl")).unwrap_or_else(|e| { - // A bounded failure before the fixture starts means zero calls, - // not an absent authorization gate. Other I/O errors still fail. - if e.kind() == std::io::ErrorKind::NotFound { - String::new() - } else { - panic!("Cannot read fixture calls for {scenario}: {e}") - } - }); + let calls = fs::read_to_string(f.home.path().join("calls.jsonl")).unwrap(); assert!(!calls.contains("get-iam-policy"), "{scenario}"); } } From c6797d584ab2833cf6f370cc1ab9c5993fb73e88 Mon Sep 17 00:00:00 2001 From: Rohit P Date: Fri, 13 Nov 2026 13:01:45 -0800 Subject: [PATCH 08/11] fix: preserve deployment review prices across restarts --- harness/service-cli/Cargo.toml | 2 +- harness/service-cli/src/deployment_up.rs | 60 ++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/harness/service-cli/Cargo.toml b/harness/service-cli/Cargo.toml index b3308f98..34e27eb8 100644 --- a/harness/service-cli/Cargo.toml +++ b/harness/service-cli/Cargo.toml @@ -18,7 +18,7 @@ openworker-sdk = { path = "../harness-sdk" } anyhow = "1" clap = { version = "4", features = ["derive"] } serde = { version = "1", features = ["derive"] } -serde_json = "1" +serde_json = { version = "1", features = ["float_roundtrip"] } serde_yaml = "0.9" tokio = { version = "1", features = ["full"] } futures-util = "0.3" diff --git a/harness/service-cli/src/deployment_up.rs b/harness/service-cli/src/deployment_up.rs index 2815a829..dcdf984e 100644 --- a/harness/service-cli/src/deployment_up.rs +++ b/harness/service-cli/src/deployment_up.rs @@ -473,6 +473,66 @@ mod route_tests { mod approval_tests { use super::*; #[test] + fn saved_review_preserves_catalog_prices_and_rejects_drift() { + let config: Config = serde_json::from_value(json!({ + "name":"example", + "cloud":{"provider":"aws","accountId":"123456789012","region":"us-west-2"}, + "nodes":{"count":2,"instanceType":"t4g.large"} + })) + .unwrap(); + for rate in [ + 0.00015753424657534247_f64, + 0.5438415068493151, + 0.012712465753424657, + 0.1534027397260274, + ] { + let resolution = serde_json::from_value(json!({ + "node":{"instanceType":"t4g.large","cpu":2,"memoryGiB":8,"architecture":"arm64","eligible":true,"hourlyUsd":0.0672}, + "price":{"currency":"USD","workerHourlyUsd":0.1344,"totalHourlyUsd":null, + "knownHourlySubtotalUsd":rate, + "components":{"databaseDisks":{"hourlyUsd":rate,"quantity":20.0,"source":"catalog fixture","meter":"disk"}}, + "variableAllowanceUsd":15,"plannedHours":4,"unpricedComponents":["cluster"],"hardCap":false}, + "observedAtUnix":1,"identity":{"accountId":"123456789012"} + })) + .unwrap(); + let mut original = Review { + schema_version: 1, + config_digest: config.digest().unwrap(), + config: config.clone(), + deployment_id: "ows-0123456789abcdef0123456789abcdef".into(), + resolution, + package_digest: "a".repeat(64), + native_bundle_digest: "b".repeat(64), + digest: String::new(), + }; + original.digest = original.digest().unwrap(); + original.validate(&config).unwrap(); + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("review.json"); + write_private(&path, &original).unwrap(); + let mut restored: Review = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + assert_eq!( + restored.resolution.price.components["databaseDisks"] + .hourly_usd + .to_bits(), + rate.to_bits() + ); + restored.validate(&config).unwrap(); + assert_eq!(restored.digest, original.digest); + restored + .resolution + .price + .components + .get_mut("databaseDisks") + .unwrap() + .hourly_usd += 0.01; + assert!( + restored.validate(&config).is_err(), + "Actual price drift must still invalidate approval" + ); + } + } + #[test] fn approval_is_durable_scoped_and_rejects_changed_review() { let temp = tempfile::tempdir().unwrap(); let digest = "a".repeat(64); From a67cfad9692223c2a53289622a7104d429ae211b Mon Sep 17 00:00:00 2001 From: Rohit P Date: Fri, 13 Nov 2026 13:39:48 -0800 Subject: [PATCH 09/11] fix: reconcile completed AWS infrastructure before installation --- .../service-cli/src/deployment_up_install.rs | 10 ++++++ .../service-cli/src/deployment_up_native.rs | 13 +++++++- harness/service-cli/tests/cloud_lifecycle.rs | 33 +++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/harness/service-cli/src/deployment_up_install.rs b/harness/service-cli/src/deployment_up_install.rs index 0ca2ea79..9dcc842b 100644 --- a/harness/service-cli/src/deployment_up_install.rs +++ b/harness/service-cli/src/deployment_up_install.rs @@ -19,6 +19,16 @@ impl Native<'_> { if self.provider() != "aws" { return self.azure_images().await; } + // Recover journals written by an older coordinator which completed the + // outer stage from a read-only status without advancing native receipts. + let journal = self.document(&self.cloud_state().join("cloud-journal.json"))?; + if journal["phase"] != "infrastructure_ready" { + let reconciled = self.cloud_apply("foundation").await?; + ensure!( + reconciled["phase"] == "infrastructure_ready", + "Original AWS infrastructure receipts are not ready for image publication" + ); + } let mut receipts = BTreeMap::::new(); for (role, source) in [ ("harness", Some(&self.package.release.images.harness)), diff --git a/harness/service-cli/src/deployment_up_native.rs b/harness/service-cli/src/deployment_up_native.rs index 38abb9db..99990b54 100644 --- a/harness/service-cli/src/deployment_up_native.rs +++ b/harness/service-cli/src/deployment_up_native.rs @@ -485,9 +485,20 @@ impl Driver for Native<'_> { if !self.cloud_state().exists() { return Ok(Observation::Absent); } - let status = self.cloud_status().await?; + let mut status = self.cloud_status().await?; if self.provider() == "aws" { if status["observed"]["infra"]["status"] == "CREATE_COMPLETE" { + // Status is intentionally read-only. Reconcile the original + // create receipts before declaring this stage complete; + // installation requires the durable lifecycle phase too. + if status["phase"] != "infrastructure_ready" { + let reconciled = self.cloud_apply("foundation").await?; + ensure!( + reconciled["phase"] == "infrastructure_ready", + "Original AWS infrastructure receipts are not ready" + ); + status = reconciled; + } return Ok(Observation::Complete(status)); } let journal = self.document(&self.cloud_state().join("cloud-journal.json"))?; diff --git a/harness/service-cli/tests/cloud_lifecycle.rs b/harness/service-cli/tests/cloud_lifecycle.rs index 822bf791..d17850af 100644 --- a/harness/service-cli/tests/cloud_lifecycle.rs +++ b/harness/service-cli/tests/cloud_lifecycle.rs @@ -500,6 +500,39 @@ fn stale_wrong_or_incomplete_preflight_and_locked_state_prevent_all_calls() { assert!(f.calls().is_empty()); } +#[test] +fn completed_remote_stack_requires_receipt_reconciliation_before_image_publication() { + let f = Fixture::new(); + parsed(&f.apply("", false, true)); // guard accepted + parsed(&f.apply("", false, true)); // infrastructure accepted + let journal = f.home.path().join("state/cloud-journal.json"); + let original = fs::read(&journal).unwrap(); + let status = parsed(&f.status("")); + assert_eq!(status["observed"]["infra"]["status"], "CREATE_COMPLETE"); + assert_eq!(status["phase"], "infra_creating"); + assert_eq!(fs::read(&journal).unwrap(), original); // status remains read-only + let refused = f.publish("", false); + assert!(!refused.status.success()); + assert!(String::from_utf8_lossy(&refused.stderr).contains("must be ready")); + let creates = f.creates(); + let before: Value = serde_json::from_slice(&original).unwrap(); + let reconciled = parsed(&f.apply("", false, true)); + assert_eq!(reconciled["phase"], "infrastructure_ready"); + assert_eq!(f.creates(), creates); + let after: Value = serde_json::from_slice(&fs::read(&journal).unwrap()).unwrap(); + for key in ["create:guard", "create:infra"] { + assert_eq!( + before["operations"][key]["token"], + after["operations"][key]["token"] + ); + assert_eq!( + before["operations"][key]["arn"], + after["operations"][key]["arn"] + ); + } + parsed(&f.publish("", false)); + assert_eq!(f.creates(), creates); +} #[test] fn existing_create_is_reconciled_with_stale_preflight_but_next_creation_is_blocked() { let f = Fixture::new(); From c851e6101b81508f61cd5efa286bad1875e0d5b5 Mon Sep 17 00:00:00 2001 From: Rohit P Date: Fri, 13 Nov 2026 13:48:26 -0800 Subject: [PATCH 10/11] fix: accept DNS case variation in private cluster tunnel --- .../service-cli/src/deployment_up_access.rs | 45 ++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/harness/service-cli/src/deployment_up_access.rs b/harness/service-cli/src/deployment_up_access.rs index bbc308f8..310d9b3e 100644 --- a/harness/service-cli/src/deployment_up_access.rs +++ b/harness/service-cli/src/deployment_up_access.rs @@ -38,6 +38,20 @@ fn endpoint(kube: &Value) -> Result { ); Ok(u.host_str().context("Endpoint host missing")?.into()) } +fn validate_connect(line: &str, host: &str) -> Result<()> { + let target = line.strip_prefix("CONNECT ").and_then(|rest| { + rest.strip_suffix(" HTTP/1.1") + .or_else(|| rest.strip_suffix(" HTTP/1.0")) + }); + // DNS names are case-insensitive. URL parsing normalizes the reviewed host, + // while kubectl preserves the uppercase label returned by some EKS endpoints. + // Keep the method, port and exact destination restriction unchanged. + ensure!( + target.is_some_and(|target| target.eq_ignore_ascii_case(&format!("{host}:443"))), + "Proxy destination is outside the reviewed cluster" + ); + Ok(()) +} async fn tunnel(mut incoming: tokio::net::TcpStream, host: &str, port: u16) -> Result<()> { let mut header = Vec::new(); loop { @@ -53,11 +67,7 @@ async fn tunnel(mut incoming: tokio::net::TcpStream, host: &str, port: u16) -> R .split("\r\n") .next() .unwrap_or(""); - ensure!( - line == format!("CONNECT {host}:443 HTTP/1.1") - || line == format!("CONNECT {host}:443 HTTP/1.0"), - "Proxy destination is outside the reviewed cluster" - ); + validate_connect(line, host)?; let mut upstream = tokio::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, port)).await?; incoming @@ -350,6 +360,31 @@ impl Native<'_> { mod tests { use super::*; #[test] + fn connect_accepts_dns_case_variation_but_no_other_destination_changes() { + let host = "3e2d511d.gr7.us-west-2.eks.amazonaws.com"; + for protocol in ["HTTP/1.0", "HTTP/1.1"] { + assert!(validate_connect(&format!("CONNECT {host}:443 {protocol}"), host).is_ok()); + assert!(validate_connect( + &format!("CONNECT {}:443 {protocol}", host.to_ascii_uppercase()), + host + ) + .is_ok()); + } + for target in [ + format!("{host}:8443"), + format!("{host}.evil:443"), + format!("user@{host}:443"), + format!("{host}:443/path"), + format!("{host}:443?query"), + format!("https://{host}:443"), + "127.0.0.1:443".into(), + ] { + assert!(validate_connect(&format!("CONNECT {target} HTTP/1.1"), host).is_err()); + } + assert!(validate_connect(&format!("connect {host}:443 HTTP/1.1"), host).is_err()); + assert!(validate_connect(&format!("CONNECT {host}:443 HTTP/2.0"), host).is_err()); + } + #[test] fn private_endpoint_requires_original_ca_and_https() { let mut v = json!({"clusters":[{"cluster":{"server":"https://original.internal","certificate-authority-data":"CA"}}]}); assert_eq!(endpoint(&v).unwrap(), "original.internal"); From cf2c0790726cd03b349726571598af699ae63586 Mon Sep 17 00:00:00 2001 From: Rohit P Date: Fri, 13 Nov 2026 15:07:58 -0800 Subject: [PATCH 11/11] Wait for the initial UI session check before sign-in --- harness/service-ui/public/app.js | 11 +++++++++-- harness/service-ui/public/index.html | 5 +++-- harness/service-ui/test/live.mjs | 17 ++++++++++++++++- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/harness/service-ui/public/app.js b/harness/service-ui/public/app.js index e622ddae..44dec8ca 100644 --- a/harness/service-ui/public/app.js +++ b/harness/service-ui/public/app.js @@ -1079,14 +1079,21 @@ document.addEventListener("keydown", (e) => { }); window.addEventListener("hashchange", render); wireForm(); -fetch("/session") +// Finish restoring the existing session before accepting a new token. Otherwise +// a delayed initial 401 can clear a token or undo a successful fresh sign-in. +fetch("/session", { signal: AbortSignal.timeout(10000) }) .then(async (r) => { if (r.ok) { auth = await r.json(); showApp(); } else loggedOut(); }) - .catch(() => loggedOut("Service unavailable.")); + .catch(() => loggedOut("Service unavailable.")) + .finally(() => { + $("#token").disabled = false; + $("#login-form button").disabled = false; + $("#login-form").removeAttribute("aria-busy"); + }); window.addEventListener("beforeunload", (event) => { if (pending) { diff --git a/harness/service-ui/public/index.html b/harness/service-ui/public/index.html index d934f27d..7fd578f1 100644 --- a/harness/service-ui/public/index.html +++ b/harness/service-ui/public/index.html @@ -14,14 +14,15 @@ diff --git a/harness/service-ui/test/live.mjs b/harness/service-ui/test/live.mjs index a0742b11..f88839d4 100644 --- a/harness/service-ui/test/live.mjs +++ b/harness/service-ui/test/live.mjs @@ -203,13 +203,28 @@ try { const { chromium, expect } = createRequire(import.meta.url)( process.env.OW_PLAYWRIGHT_MODULE, ); - browser = await chromium.launch({ headless: true }); + browser = await chromium.launch({ headless: true, executablePath: process.env.OW_BROWSER_EXECUTABLE }); const page = await browser.newPage({ viewport: { width: 1440, height: 1000 }, }); const errors = []; page.on("pageerror", (e) => errors.push(e.message)); + // A delayed startup 401 must not clear a token being entered or undo login. + let releaseStartup, startupRequested; + const startupGate = new Promise(resolve => { releaseStartup = resolve; }); + const requested = new Promise(resolve => { startupRequested = resolve; }); + await page.route("**/session", async route => { + startupRequested(); + await startupGate; + await route.fulfill({status:401,contentType:"application/json",body:'{"error":"login_required"}'}); + }); await page.goto(app.url); + await requested; + await expect(page.getByLabel("Service token", { exact: true })).toBeDisabled(); + await expect(page.getByRole("button", { name: "Sign in", exact: true })).toBeDisabled(); + releaseStartup(); + await expect(page.getByLabel("Service token", { exact: true })).toBeEnabled(); + await page.unroute("**/session"); await page .getByLabel("Service token", { exact: true }) .fill(process.env.OW_UI_USER);