diff --git a/deploy/service/chart.test.mjs b/deploy/service/chart.test.mjs index f5e3e0f1..923f65e7 100644 --- a/deploy/service/chart.test.mjs +++ b/deploy/service/chart.test.mjs @@ -229,3 +229,29 @@ test("exporter authentication mounts only into harness roles and remains opt-in" assert.match(bootstrap, /OW_TELEMETRY_AUTH/); assert.match(bootstrap, /secretName: export-auth/); }); +test("attachment chunks have a scoped 1 MiB limit, separate from metadata and artifacts",async()=>{ + const out=await render({}); + const block=out.match(/location ~ \^\/v1\/sessions\/\[A-Za-z0-9_-\]\+\/attachments\/\[A-Za-z0-9_-\]\+\$ \{[\s\S]*?\}/)?.[0]; + assert.ok(block,"Only the exact authenticated attachment upload path gets a larger bound"); + assert.match(block,/client_max_body_size 1m/); + assert.match(block,/proxy_request_buffering off/); + assert.match(out,/ssl_protocols TLSv1.2 TLSv1.3; client_max_body_size 160k/); +}); +test("recording enable runs only advisory-locked schema initialization, disable removes it",async()=>{ + const off=await render({});assert.doesNotMatch(off,/name: recording-schema/); + const on=await render({config:{...base.config,observability:{retention_days:14}}}); + for(const role of ["api","worker","controller","platform"]){ + const d=deployment(on,role);const init=d.split("initContainers:")[1].split(" containers:")[0]; + assert.match(init,/args: \[recording-migrate\]/); + assert.doesNotMatch(init,/bootstrap|\/models|\/openshell/); + } +}); +test("private management UI preview terminates TLS without giving the UI server the private key",async()=>{ + const out=await render({managementUI:{enabled:true,image:"ui@sha256:"+"b".repeat(64),publicOrigin:"https://localhost:8444",serviceUrl:"https://test-api.default.svc",caSecret:"ow-service-tls",caKey:"tls.crt",tlsSecret:"ow-service-tls"}}); + const d=deployment(out,"ui"); + assert.match(d,/name: tls/);assert.match(d,/key: "tls.crt", path: ca.crt/); + const ui=d.split("- name: ui")[1].split("- name: tls")[0]; + assert.doesNotMatch(ui,/mountPath: \/tls/); + assert.match(out,/name: https, port: 443, targetPort: https/); + assert.match(out,/proxy_set_header Host \$http_host/); +}); diff --git a/deploy/service/chart/templates/config.yaml b/deploy/service/chart/templates/config.yaml index c756b25f..40194ac4 100644 --- a/deploy/service/chart/templates/config.yaml +++ b/deploy/service/chart/templates/config.yaml @@ -54,6 +54,13 @@ data: proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_request_buffering off; proxy_buffering off; proxy_read_timeout 120s; proxy_send_timeout 120s; } + # SDK chunks contain base64 bytes plus metadata. Match only the upload + # route and the backend's 1 MiB bound; ordinary JSON stays at 160 KiB. + location ~ ^/v1/sessions/[A-Za-z0-9_-]+/attachments/[A-Za-z0-9_-]+$ { + client_max_body_size 1m; client_body_timeout 120s; + proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1; + proxy_set_header Connection ""; proxy_request_buffering off; + proxy_buffering off; proxy_read_timeout 120s; proxy_send_timeout 120s; } location / { proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1; proxy_set_header Connection ""; proxy_buffering off; proxy_read_timeout 3600s; } } diff --git a/deploy/service/chart/templates/management-ui.yaml b/deploy/service/chart/templates/management-ui.yaml index 70c1948f..eb2b519b 100644 --- a/deploy/service/chart/templates/management-ui.yaml +++ b/deploy/service/chart/templates/management-ui.yaml @@ -8,6 +8,27 @@ metadata: {name: {{ .Release.Name }}-ui} data: ui.json: | {{- dict "host" "0.0.0.0" "port" 8080 "publicOrigin" .Values.managementUI.publicOrigin "serviceUrl" .Values.managementUI.serviceUrl "serviceId" .Values.config.service.id "namespace" .Values.config.service.namespace "sessionSeconds" 1800 | toPrettyJson | nindent 4 }} + {{- if .Values.managementUI.tlsSecret }} + nginx.conf: | + pid /tmp/nginx.pid; + error_log /dev/stderr warn; + events { worker_connections 128; } + http { + access_log off; + client_body_temp_path /tmp/body; proxy_temp_path /tmp/proxy; + fastcgi_temp_path /tmp/fastcgi; uwsgi_temp_path /tmp/uwsgi; scgi_temp_path /tmp/scgi; + server { + listen 8443 ssl; + ssl_certificate /tls/tls.crt; ssl_certificate_key /tls/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; client_max_body_size 160k; + location / { + proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1; + proxy_set_header Host $http_host; proxy_set_header Connection ""; + proxy_buffering off; proxy_read_timeout 3600s; + } + } + } + {{- end }} --- apiVersion: apps/v1 kind: Deployment @@ -22,7 +43,7 @@ spec: metadata: labels: {app: {{ .Release.Name }}-ui} annotations: - checksum/config: {{ .Values.managementUI | toJson | sha256sum | quote }} + checksum/config: {{ list .Values.managementUI .Values.config.service | toJson | sha256sum | quote }} spec: automountServiceAccountToken: false securityContext: {runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, fsGroup: 1000} @@ -39,12 +60,28 @@ spec: - {name: ca, mountPath: /trust, readOnly: true} env: [{name: NODE_EXTRA_CA_CERTS, value: /trust/ca.crt}] {{- end }} + {{- if .Values.managementUI.tlsSecret }} + - name: tls + image: {{ .Values.proxyImage | quote }} + command: [nginx, -c, /config/nginx.conf, -g, "daemon off;"] + ports: [{name: https, containerPort: 8443}] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}, seccompProfile: {type: RuntimeDefault}} + volumeMounts: + - {name: config, mountPath: /config, readOnly: true} + - {name: tls, mountPath: /tls, readOnly: true} + - {name: tmp, mountPath: /tmp} + {{- end }} volumes: - name: config configMap: {name: {{ .Release.Name }}-ui} {{- if .Values.managementUI.caSecret }} - name: ca - secret: {secretName: {{ .Values.managementUI.caSecret }}, items: [{key: ca.crt, path: ca.crt}]} + secret: {secretName: {{ .Values.managementUI.caSecret }}, items: [{key: {{ .Values.managementUI.caKey | quote }}, path: ca.crt}]} + {{- end }} + {{- if .Values.managementUI.tlsSecret }} + - name: tls + secret: {secretName: {{ .Values.managementUI.tlsSecret }}, defaultMode: 0440} + - {name: tmp, emptyDir: {sizeLimit: 64Mi}} {{- end }} --- apiVersion: v1 @@ -52,5 +89,5 @@ kind: Service metadata: {name: {{ .Release.Name }}-ui} spec: selector: {app: {{ .Release.Name }}-ui} - ports: [{name: http, port: 80, targetPort: http}] + ports: [{{ if .Values.managementUI.tlsSecret }}{name: https, port: 443, targetPort: https}{{ else }}{name: http, port: 80, targetPort: http}{{ end }}] {{- end }} diff --git a/deploy/service/chart/templates/workloads.yaml b/deploy/service/chart/templates/workloads.yaml index f3f4798a..d255fb1f 100644 --- a/deploy/service/chart/templates/workloads.yaml +++ b/deploy/service/chart/templates/workloads.yaml @@ -48,6 +48,23 @@ spec: {{- end }} terminationGracePeriodSeconds: {{ $.Values.terminationGracePeriodSeconds }} securityContext: {fsGroup: 1000} + {{- if hasKey $.Values.config "observability" }} + # Explicit enable-time initialization; advisory-locked and idempotent. + # No first-admin bootstrap, provider key or sandbox credential required. + initContainers: + - name: recording-schema + image: {{ $.Values.image | quote }} + command: [/bin/sh, /opt/openworker/entrypoint.sh] + args: [recording-migrate] + securityContext: {runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}, seccompProfile: {type: RuntimeDefault}} + volumeMounts: + - {name: service, mountPath: /run/service/database-url, subPath: database-url, readOnly: true} + - {name: service, mountPath: /run/service/pg-ca.crt, subPath: pg-ca.crt, readOnly: true} + - {name: service, mountPath: /run/service/config.json, subPath: config.json, readOnly: true} + {{- if $.Values.secrets.observability }} + - {name: observability, mountPath: /run/observability, readOnly: true} + {{- end }} + {{- end }} containers: - name: harness image: {{ $.Values.image | quote }} diff --git a/deploy/service/chart/values.yaml b/deploy/service/chart/values.yaml index c2d86d1b..7c716ea1 100644 --- a/deploy/service/chart/values.yaml +++ b/deploy/service/chart/values.yaml @@ -31,3 +31,5 @@ managementUI: # Must match the API TLS certificate (may use the cluster Service DNS name). serviceUrl: "" caSecret: "" # optional Secret containing ca.crt + caKey: ca.crt + tlsSecret: "" # optional private HTTPS preview using an existing API certificate diff --git a/deploy/service/proxy.test.mjs b/deploy/service/proxy.test.mjs new file mode 100644 index 00000000..4323a626 --- /dev/null +++ b/deploy/service/proxy.test.mjs @@ -0,0 +1,59 @@ +// Opt-in real nginx test: no cloud access, credentials or model calls. +// OWS_PROXY_TEST=1 node --test deploy/service/proxy.test.mjs +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {createServer} from 'node:http'; +import https from 'node:https'; +import {mkdtemp,writeFile,readFile,rm} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import {join,resolve} from 'node:path'; +import {randomUUID,createHash} from 'node:crypto'; +import {execFile} from 'node:child_process'; +import {promisify} from 'node:util'; +const exec=promisify(execFile); +const run=async(program,args)=>String((await exec(program,args,{maxBuffer:8*1024*1024})).stdout); +test('real TLS proxy accepts SDK attachment chunks and preserves scoped limits/authentication',{skip:process.env.OWS_PROXY_TEST!=='1',timeout:60000},async()=>{ + const root=await mkdtemp(join(tmpdir(),'ows-proxy-')); + const name='ows-proxy-'+randomUUID().slice(0,8); + let forwarded=0; + const upstream=createServer(async(req,res)=>{ + forwarded++; + const chunks=[];for await(const c of req)chunks.push(c); + if(req.headers.authorization!=='Bearer fixture'){res.writeHead(401).end();return;} + const body=Buffer.concat(chunks); + res.writeHead(200,{'content-type':'application/json'}).end(JSON.stringify({bytes:body.length,sha256:createHash('sha256').update(body).digest('hex')})); + }); + await new Promise(r=>upstream.listen(0,'0.0.0.0',r)); + try{ + const values=join(root,'values.json'); + await writeFile(values,JSON.stringify({image:'fixture@sha256:'+'a'.repeat(64),config:{service:{id:'proxy-test',namespace:'test',agent:'fixture'}}})); + const rendered=await run('helm',['template','proxy-test',resolve('deploy/service/chart'),'-f',values]); + const raw=rendered.split(' nginx.conf: |\n')[1].split('\n---')[0].replace(/^ /gm,''); + const conf=raw.replaceAll('127.0.0.1:8080',`host.docker.internal:${upstream.address().port}`); + await writeFile(join(root,'nginx.conf'),conf); + await run('openssl',['req','-x509','-newkey','rsa:2048','-nodes','-days','1','-subj','/CN=localhost','-addext','subjectAltName=DNS:localhost,IP:127.0.0.1','-addext','basicConstraints=critical,CA:TRUE','-addext','keyUsage=critical,digitalSignature,keyCertSign,cRLSign','-addext','extendedKeyUsage=serverAuth','-keyout',join(root,'tls.key'),'-out',join(root,'tls.crt')]); + await run('openssl',['verify','-x509_strict','-purpose','sslserver','-CAfile',join(root,'tls.crt'),join(root,'tls.crt')]); + const ca=await readFile(join(root,'tls.crt')); + const image=(await readFile('deploy/service/chart/values.yaml','utf8')).match(/^proxyImage: (.+)$/m)[1]; + await run('docker',['run','--rm','-d','--name',name,'-p','127.0.0.1::8443','-v',`${root}:/tls:ro`,image,'nginx','-c','/tls/nginx.conf','-g','daemon off;']); + const port=Number((await run('docker',['port',name,'8443/tcp'])).trim().split(':').at(-1)); + const request=(path,body,auth=true)=>new Promise((resolve,reject)=>{ + const q=https.request({hostname:'127.0.0.1',port,path,method:'POST',ca,agent:false,headers:{expect:'100-continue','content-type':'application/json','content-length':body.length,...auth?{authorization:'Bearer fixture'}:{}}},r=>{let b='';r.on('data',c=>b+=c);r.on('end',()=>resolve({status:r.statusCode,body:b}));}); + q.on('error',reject);q.on('continue',()=>q.end(body));q.flushHeaders(); + }); + for(let i=0;;i++){try{await request('/healthz',Buffer.alloc(0));break;}catch(e){if(i===30)throw e;await new Promise(r=>setTimeout(r,100));}} + const path='/v1/sessions/session_123/attachments/file_123'; + const chunk=Buffer.from(JSON.stringify({offset:0,data:Buffer.alloc(256*1024,97).toString('base64')})); + assert(chunk.length>160*1024&&chunk.length<1024*1024); + const ok=await request(path,chunk);assert.equal(ok.status,200); + assert.equal(JSON.parse(ok.body).sha256,createHash('sha256').update(chunk).digest('hex')); + assert.equal((await request(path,chunk,false)).status,401); + const count=forwarded; + for(const p of ['/v1/sessions/session_123/turns','/v1/sessions/session_123/attachments','/v1/sessions/session_123/attachments/file_123/extra'])assert.equal((await request(p,chunk)).status,413); + assert.equal((await request(path,Buffer.alloc(1024*1024+1))).status,413); + assert.equal(forwarded,count,'oversized requests must not reach the application'); + }finally{ + await run('docker',['rm','-f',name]).catch(()=>{}); + await new Promise(r=>upstream.close(r));await rm(root,{recursive:true,force:true}); + } +}); diff --git a/deploy/service/stack.mjs b/deploy/service/stack.mjs index e18513dc..289c8341 100644 --- a/deploy/service/stack.mjs +++ b/deploy/service/stack.mjs @@ -70,6 +70,8 @@ async function certificate(input, name, cn, san) { "365", "-subj", `/CN=${cn} development CA`, + "-addext", "basicConstraints=critical,CA:TRUE", + "-addext", "keyUsage=critical,keyCertSign,cRLSign", ]); await run("openssl", [ "req", @@ -86,7 +88,7 @@ async function certificate(input, name, cn, san) { ]); await privateWrite( cert + ".ext", - `subjectAltName=${san}\nextendedKeyUsage=serverAuth\n`, + `subjectAltName=${san}\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\n`, ); await run("openssl", [ "x509", @@ -224,7 +226,7 @@ export async function init(root, connectionFile, image, port = 58443) { ); await privateWrite( join(input, "nginx.conf"), - `pid /tmp/nginx.pid;\nerror_log /dev/stderr warn;\nevents {worker_connections 256;}\nhttp {access_log off;client_body_temp_path /tmp/body;proxy_temp_path /tmp/proxy;fastcgi_temp_path /tmp/fastcgi;uwsgi_temp_path /tmp/uwsgi;scgi_temp_path /tmp/scgi;\nserver {listen 8443 ssl;ssl_certificate /tls/tls.crt;ssl_certificate_key /tls/tls.key;ssl_protocols TLSv1.2 TLSv1.3;client_max_body_size 160k;\nlocation ~ ^/(v1/)?sessions/[A-Za-z0-9_-]+/artifacts/[A-Za-z0-9_-]+/content$ {client_max_body_size 64m;client_body_timeout 120s;proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_request_buffering off;proxy_buffering off;proxy_read_timeout 120s;proxy_send_timeout 120s;}\nlocation / {proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_buffering off;proxy_read_timeout 3600s;}}}\n`, + `pid /tmp/nginx.pid;\nerror_log /dev/stderr warn;\nevents {worker_connections 256;}\nhttp {access_log off;client_body_temp_path /tmp/body;proxy_temp_path /tmp/proxy;fastcgi_temp_path /tmp/fastcgi;uwsgi_temp_path /tmp/uwsgi;scgi_temp_path /tmp/scgi;\nserver {listen 8443 ssl;ssl_certificate /tls/tls.crt;ssl_certificate_key /tls/tls.key;ssl_protocols TLSv1.2 TLSv1.3;client_max_body_size 160k;\nlocation ~ ^/(v1/)?sessions/[A-Za-z0-9_-]+/artifacts/[A-Za-z0-9_-]+/content$ {client_max_body_size 64m;client_body_timeout 120s;proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_request_buffering off;proxy_buffering off;proxy_read_timeout 120s;proxy_send_timeout 120s;}\nlocation ~ ^/v1/sessions/[A-Za-z0-9_-]+/attachments/[A-Za-z0-9_-]+$ {client_max_body_size 1m;client_body_timeout 120s;proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_request_buffering off;proxy_buffering off;proxy_read_timeout 120s;proxy_send_timeout 120s;}\nlocation / {proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_buffering off;proxy_read_timeout 3600s;}}}\n`, ); const manifest = { id, diff --git a/deploy/service/up/README.md b/deploy/service/up/README.md index 82fd5540..eda5a302 100644 --- a/deploy/service/up/README.md +++ b/deploy/service/up/README.md @@ -101,3 +101,81 @@ python3 deploy/service/up/test-plan.py \ ``` That check uses the real pinned OpenShell archive, fixture image references and catalog responses. It performs no real cloud calls, does not install a cluster and must not be presented as cloud qualification. + +## Management UI and trace storage + +Both features are optional and independent. Select them before the first `up`: + +```yaml +managementUI: + enabled: true + publicOrigin: https://localhost:8444 +observability: + enabled: true + retentionDays: 14 +``` + +The reviewed release manifest must include a digest-pinned `service-ui` image. +The default private preview uses the API certificate and a loopback-only Kubernetes +port-forward; `connection.json` contains its command, URL and CA file. Trust that +certificate locally, then sign in with a current service token. No token is put in +the browser URL. Keep the cloud's private management route open as described above. +A non-loopback `publicOrigin` requires your own TLS ingress; `up` does not create +public exposure or DNS records. + +After installation, use the original **installer** state directory (the +`installation/` child of the `up --state` directory): + +```sh +ows management-ui enable --state .ows/my-deployment/installation +ows management-ui status --state .ows/my-deployment/installation +ows management-ui disable --state .ows/my-deployment/installation + +ows observability enable --state .ows/my-deployment/installation --retention-days 14 +ows observability status --state .ows/my-deployment/installation +ows observability disable --state .ows/my-deployment/installation +``` + +If the original release did not include a UI image, pass +`--image registry.example.com/ui@sha256:YOUR_VERIFIED_DIGEST` on enable. +`--public-origin` overrides its origin. These commands use the original Kubernetes +credentials, verify cluster/namespace/release identities, and perform a Helm upgrade +of the saved chart. They do not use the service REST token. Features are saved in +`journal.json` with their original Helm operation and revision receipts; the initial +provisioning review and `deployment.yaml` stay immutable. Rerunning `up` does not +reset these saved feature overrides. Do not edit old installer values in place. + +Trace storage uses a separate `ow_observability` schema and connection to the +execution PostgreSQL by default. For a separate database, create an existing +Kubernetes Secret containing `database-url` and `ca.crt`, then select it with +`observability.databaseSecret` or `ows observability enable --database-secret NAME`. +Verified database TLS remains required. Enabling initializes the recording schema +before workloads start; feature changes roll the affected pods, so allow active +turns to drain. The first-admin bootstrap is not rerun. + +Disabling recording stops new recordings as the rollout completes. It does not +delete traces, databases or credentials. While disabled, the recorder's retention +worker is also stopped; retained data remains until recording resumes or an operator +performs a separate cleanup. Security audit, durable execution state and separately +configured OTLP exporters are unaffected. This is deployment-level control, not a +new per-agent trace policy API. + +A lost Helm response never causes an automatic repeat. `status` can reconcile the +original successful revision; an unresolved/pending/failed result blocks another +upgrade or teardown until the operator inspects that original operation. + +Local regression checks: + +```sh +cargo test --manifest-path harness/Cargo.toml -p openworker-service-cli +node --test deploy/service/chart.test.mjs +OWS_PROXY_TEST=1 node --test deploy/service/proxy.test.mjs +``` + +If the API uses a certificate signed by a separate CA, provide its public trust +bundle using `managementUI.caSecret` / `managementUI.caKey` in deployment YAML, +or `ows management-ui enable --ca-secret API_CA_SECRET --ca-key ca.crt`. +The Secret must already exist in the service namespace. Without an override, +the CLI trusts the API TLS Secret's `tls.crt`, suitable for the self-signed +certificate produced by `up`. The UI process receives only that public certificate; +the HTTPS proxy sidecar receives the separate TLS key mount. diff --git a/docs/harness/guides/observability.md b/docs/harness/guides/observability.md index fb193e2b..8b91e8a4 100644 --- a/docs/harness/guides/observability.md +++ b/docs/harness/guides/observability.md @@ -165,3 +165,21 @@ proof of compliance. A metadata reader is not a content auditor. In server apps, authorize evidence access separately before returning it, and keep database/host APIs in the trusted backend. The service's existing compliance content permission continues to control audit reads. Cluster experiment/review tooling comes later. + +## Manage recording on a service cluster + +The source CLI supports independent UI and trace-storage controls: + +```sh +ows observability enable --state ./deployment-state/installation --retention-days 14 +ows observability status --state ./deployment-state/installation +ows observability disable --state ./deployment-state/installation +ows management-ui enable --state ./deployment-state/installation +``` + +Use the installer state created by `ows deploy aws up` or `ows deploy azure up`. +Enabling recording initializes its separate PostgreSQL schema before workloads +restart. Disabling stops collection after the rollout and retains stored traces; +it does not disable security audit or a separately configured exporter. +See the [deployment configuration and private UI access](../../../deploy/service/up/README.md#management-ui-and-trace-storage) +for the required pinned UI image, CA, database Secret and enable/disable options. diff --git a/harness/server-reference/src/main.rs b/harness/server-reference/src/main.rs index dc9a5703..a9d1e36b 100644 --- a/harness/server-reference/src/main.rs +++ b/harness/server-reference/src/main.rs @@ -29,6 +29,8 @@ enum Command { drain_timeout_seconds: u64, }, Migrate, + /// Initialize only the optional recording schema, including on feature enable. + RecordingMigrate, /// Trusted installer aggregate check; no public/session API equivalent. ServiceInstallStatus, /// Trusted-host operational catalog, not a session-authorized API. @@ -116,7 +118,8 @@ async fn main() -> Result<()> { _ => "admin", }; let recording_config = config.clone(); - let record = !matches!(args.command, Command::Migrate) && !config.recovery_read_only; + let record = !matches!(args.command, Command::Migrate | Command::RecordingMigrate) + && !config.recovery_read_only; let started = tokio::task::spawn_blocking(move || { openworker_server_reference::recording::start(&recording_config, role, record) }) @@ -161,6 +164,19 @@ async fn run(args: Args, config: Config, telemetry: openworker_telemetry::Teleme ) { config.require_execution()?; } + if matches!(args.command, Command::RecordingMigrate) { + let options = config + .observability + .as_ref() + .context("Recording is disabled")?; + let database = options.database(&config)?; + tokio::task::spawn_blocking(move || { + openworker_observability::postgres::Postgres::migrate(&database) + }) + .await??; + println!("Recording schema ready"); + return Ok(()); + } let mut db = config.database()?; db.telemetry = telemetry; if matches!(args.command, Command::Migrate) { @@ -594,7 +610,7 @@ async fn run(args: Args, config: Config, telemetry: openworker_telemetry::Teleme | Command::EnvironmentReconcile { .. } => { bail!("This server reference requires the qualified Kubernetes provider") } - Command::Migrate => unreachable!(), + Command::Migrate | Command::RecordingMigrate => unreachable!(), } Ok(()) } diff --git a/harness/service-cli/README.md b/harness/service-cli/README.md index 38eac233..4e1fb773 100644 --- a/harness/service-cli/README.md +++ b/harness/service-cli/README.md @@ -96,3 +96,23 @@ See [AWS deployment](../../docs/harness/guides/aws.md) and with an explicit budget and independent cleanup before production deployment. `owcluster` is the primary command. `ows` remains an alias, using the same commands, configuration and saved deployment state. Existing `.ows` paths and `OWS_*` settings remain valid. + +## Optional management UI and trace storage + +Select these features in `deployment.yaml` for the AWS/Azure `up` commands, or +change them on an existing installation: + +```sh +ows management-ui enable --state ./deployment-state/installation +ows management-ui disable --state ./deployment-state/installation +ows management-ui status --state ./deployment-state/installation +ows observability enable --state ./deployment-state/installation --retention-days 14 +ows observability disable --state ./deployment-state/installation +ows observability status --state ./deployment-state/installation +``` + +`--state` is the original installer journal, using its Kubernetes credentials and +verified cluster identity. UI and trace storage are independent. Disabling tracing +retains data and leaves security audit and separately configured exporters intact. +The CLI saves feature changes and reconciles lost Helm responses without repeating +unknown upgrades. See [setup, UI image, private HTTPS and database options](../../deploy/service/up/README.md#management-ui-and-trace-storage). diff --git a/harness/service-cli/src/deployment.rs b/harness/service-cli/src/deployment.rs index 75f79a68..347c4c70 100644 --- a/harness/service-cli/src/deployment.rs +++ b/harness/service-cli/src/deployment.rs @@ -13,7 +13,7 @@ mod guard; #[path = "deployment_infrastructure.rs"] mod infrastructure; #[path = "deployment_local.rs"] -mod local; +pub(crate) mod local; #[path = "deployment_preflight.rs"] mod preflight; #[path = "deployment_up.rs"] diff --git a/harness/service-cli/src/deployment_features.rs b/harness/service-cli/src/deployment_features.rs new file mode 100644 index 00000000..4d0e2e16 --- /dev/null +++ b/harness/service-cli/src/deployment_features.rs @@ -0,0 +1,617 @@ +//! Optional service features over the original installer identity and journal. +//! Unknown Helm outcomes are observed, never automatically retried. +use super::*; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct ManagementUI { + #[serde(default)] + pub enabled: bool, + #[serde(default = "origin")] + pub public_origin: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ca_secret: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ca_key: Option, +} +fn origin() -> String { + "https://localhost:8444".into() +} +impl Default for ManagementUI { + fn default() -> Self { + Self { + enabled: false, + public_origin: origin(), + ca_secret: None, + ca_key: None, + } + } +} +impl ManagementUI { + pub(crate) fn validate(&self) -> Result<()> { + if let Some(s) = &self.ca_secret { + ensure!(dns_name(s), "Invalid UI CA Secret name"); + } + if let Some(s) = &self.ca_key { + ensure!( + !s.is_empty() + && s.len() <= 253 + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b)), + "Invalid UI CA Secret key" + ); + } + let u = reqwest::Url::parse(&self.public_origin).context("Invalid management UI origin")?; + ensure!(u.scheme()=="https" && u.host_str().is_some() && u.username().is_empty() && u.password().is_none() && u.path()=="/" && u.query().is_none() && u.fragment().is_none(), "Management UI publicOrigin must be an HTTPS origin without credentials, path, query or fragment"); + Ok(()) + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct Observability { + #[serde(default)] + pub enabled: bool, + #[serde(default = "retention")] + pub retention_days: u16, + /// Existing Kubernetes Secret containing database-url and ca.crt. No raw credentials. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub database_secret: Option, +} +fn retention() -> u16 { + 30 +} +impl Default for Observability { + fn default() -> Self { + Self { + enabled: false, + retention_days: retention(), + database_secret: None, + } + } +} +impl Observability { + pub(crate) fn validate(&self) -> Result<()> { + ensure!( + (1..=365).contains(&self.retention_days), + "Trace retention must be 1..365 days" + ); + if let Some(s) = &self.database_secret { + ensure!(dns_name(s), "Invalid recording database Secret name"); + } + Ok(()) + } +} +fn dns_name(s: &str) -> bool { + !s.is_empty() + && s.len() <= 63 + && s.starts_with(|c: char| c.is_ascii_lowercase() || c.is_ascii_digit()) + && s.ends_with(|c: char| c.is_ascii_lowercase() || c.is_ascii_digit()) + && s.bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} +#[derive(clap::Args)] +pub(crate) struct Target { + /// Original installer journal (local-plan.json and journal.json), not cloud lifecycle state. + #[arg(long)] + state: PathBuf, +} +#[derive(Subcommand)] +pub(crate) enum UICommand { + Enable { + #[command(flatten)] + target: Target, + /// Digest-pinned UI image. Omit when already recorded in Helm values. + #[arg(long)] + image: Option, + #[arg(long)] + public_origin: Option, + /// Existing Secret containing the CA that signs the service API certificate. + #[arg(long)] + ca_secret: Option, + #[arg(long, requires = "ca_secret")] + ca_key: Option, + }, + Disable { + #[command(flatten)] + target: Target, + }, + Status { + #[command(flatten)] + target: Target, + }, +} +#[derive(Subcommand)] +pub(crate) enum ObservationCommand { + Enable { + #[command(flatten)] + target: Target, + #[arg(long, value_parser=clap::value_parser!(u16).range(1..=365))] + retention_days: Option, + /// Existing Secret with database-url and ca.crt; omit to reuse execution PostgreSQL. + #[arg(long)] + database_secret: Option, + }, + Disable { + #[command(flatten)] + target: Target, + }, + Status { + #[command(flatten)] + target: Target, + }, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Update { + request_id: String, + before: Value, + values: Value, + #[serde(default)] + receipt: Option, +} +pub(super) fn latest_receipt(j: &Journal) -> Option<&Value> { + j.feature_updates.last().and_then(|u| u.receipt.as_ref()) +} +pub(super) fn effective_values(p: &Plan, j: &Journal) -> Result { + j.feature_updates + .last() + .map(|u| Ok(u.values.clone())) + .unwrap_or_else(|| read_json(&p.config.service.values_file)) +} +fn confirmed_receipt(j: &Journal) -> Result<&Value> { + latest_receipt(j) + .or_else(|| { + j.operations + .get("service-release") + .and_then(|o| o.receipt.as_ref()) + }) + .context("Original service installation receipt missing") +} +fn same_receipt(a: &Value, b: &Value) -> bool { + a["uid"] == b["uid"] && a["revision"] == b["revision"] && a["operationId"] == b["operationId"] +} +async fn live_values(client: &Client<'_>, p: &Plan) -> Result { + serde_json::from_slice( + &client + .helm(args(&[ + "get", + "values", + &p.config.service.release, + "-n", + &p.config.service.namespace, + "-o", + "json", + ])) + .await?, + ) + .context("Invalid release values") +} +pub(super) async fn verify_current(client: &Client<'_>, p: &Plan, j: &Journal) -> Result { + ensure!( + j.feature_updates.last().is_none_or(|u| u.receipt.is_some()), + "Feature upgrade outcome unknown; rerun status to reconcile, never replay the upgrade" + ); + let r = client + .release(&p.config.service.namespace, &p.config.service.release) + .await? + .context("Original service release absent")?; + ensure!( + same_receipt(&r, confirmed_receipt(j)?) && r["status"] == "deployed", + "Service Helm release identity/revision changed; refusing feature mutation" + ); + Ok(r) +} +pub(super) async fn reconcile( + client: &Client<'_>, + p: &Plan, + root: &Path, + j: &mut Journal, +) -> Result<()> { + let Some(update) = j + .feature_updates + .last() + .filter(|u| u.receipt.is_none()) + .cloned() + else { + return Ok(()); + }; + ensure!( + client.cluster().await? == p.cluster_uid, + "Cluster identity changed" + ); + owned(client, p, j, &p.config.service.namespace) + .await? + .context("Owned namespace missing")?; + let r = client + .release(&p.config.service.namespace, &p.config.service.release) + .await? + .context("Upgrade outcome unknown: original service release absent")?; + ensure!(r["installerId"]==p.id && r["operationId"]==update.request_id && r["revision"].as_u64()==update.before["revision"].as_u64().and_then(|n|n.checked_add(1)) && r["status"]=="deployed", "Feature upgrade outcome unknown/pending/failed; inspect the original Helm operation. It will not be retried automatically"); + ensure!( + live_values(client, p).await? == update.values, + "Upgraded release values differ from the saved intent" + ); + ready_service(client, p, j).await?; + j.feature_updates.last_mut().unwrap().receipt = Some(r); + save(root, "journal.json", j) +} +/// Pure rendering shared by initial cloud deployment and later CLI controls. +pub(crate) fn configure_ui( + values: &mut Value, + ui: &ManagementUI, + image: Option<&str>, + release: &str, + namespace: &str, +) -> Result<()> { + ui.validate()?; + if values["managementUI"].is_null() { + values["managementUI"] = json!({}); + } + values["managementUI"]["enabled"] = json!(ui.enabled); + if let Some(image) = image { + values["managementUI"]["image"] = json!(image); + } + values["managementUI"]["publicOrigin"] = json!(ui.public_origin); + values["managementUI"]["serviceUrl"] = json!(format!("https://{release}-api.{namespace}.svc")); + if ui.enabled { + let image = values["managementUI"]["image"] + .as_str() + .context("Provide --image with a digest-pinned management UI image")?; + ensure!( + image + .rsplit_once("@sha256:") + .is_some_and(|(name, d)| !name.is_empty() + && d.len() == 64 + && d.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))), + "Management UI image must use a sha256 digest" + ); + let api_tls = values["secrets"]["apiTLS"] + .as_str() + .unwrap_or("ow-service-tls") + .to_string(); + let ca_secret = ui + .ca_secret + .clone() + .or_else(|| { + values["managementUI"]["caSecret"] + .as_str() + .filter(|s| !s.is_empty()) + .map(str::to_owned) + }) + .unwrap_or_else(|| api_tls.clone()); + let ca_key = ui + .ca_key + .clone() + .or_else(|| { + if ui.ca_secret.is_none() { + values["managementUI"]["caKey"].as_str().map(str::to_owned) + } else { + None + } + }) + .unwrap_or_else(|| { + if ui.ca_secret.is_some() { + "ca.crt".into() + } else { + "tls.crt".into() + } + }); + values["managementUI"]["caSecret"] = json!(ca_secret); + values["managementUI"]["caKey"] = json!(ca_key); + // Private loopback preview is self-contained. Public origins require the + // operator's TLS ingress; never create public exposure implicitly. + let u = reqwest::Url::parse(&ui.public_origin)?; + values["managementUI"]["tlsSecret"] = + if matches!(u.host_str(), Some("localhost" | "127.0.0.1")) { + json!(api_tls) + } else { + json!("") + }; + } + Ok(()) +} +pub(crate) fn configure_observability(values: &mut Value, obs: &Observability) -> Result<()> { + obs.validate()?; + if values["secrets"].is_null() { + values["secrets"] = json!({}); + } + if obs.enabled { + if values["config"]["observability"].is_null() { + values["config"]["observability"] = json!({"schema":"ow_observability"}); + } + values["config"]["observability"]["retention_days"] = json!(obs.retention_days); + if let Some(s) = &obs.database_secret { + values["secrets"]["observability"] = json!(s); + } + } else { + values["config"] + .as_object_mut() + .context("Service configuration missing")? + .remove("observability"); + values["secrets"]["observability"] = json!(""); + } + Ok(()) +} +async fn change( + root: &Path, + mutation: impl FnOnce(&mut Value, &Plan) -> Result<()>, + status_only: bool, +) -> Result<()> { + let (p, mut j, _lock) = load(root)?; + ensure!( + j.phase == "installed", + "Feature controls require a completed service installation" + ); + ensure!( + inputs(&p.config)? == p.hashes, + "Original installer inputs changed" + ); + let client = Client { c: &p.config }; + ensure!( + client.cluster().await? == p.cluster_uid, + "Cluster identity changed" + ); + owned(&client, &p, &j, &p.config.service.namespace) + .await? + .context("Owned namespace missing")?; + reconcile(&client, &p, root, &mut j).await?; + let before = verify_current(&client, &p, &j).await?; + let mut values = effective_values(&p, &j)?; + // First update reads the live explicit values to preserve defaults/previous + // operator settings, but refuses any unreviewed drift. + ensure!( + live_values(&client, &p).await? == values, + "Service values changed outside the installer" + ); + if !status_only { + mutation(&mut values, &p)?; + if values != effective_values(&p, &j)? { + // Never rerun first-admin bootstrap on a feature upgrade. + values["bootstrapEnabled"] = json!(false); + let id = uuid::Uuid::new_v4().to_string(); + let file = format!("feature-values-{id}.json"); + save(root, &file, &values)?; + // Render validation happens before recording/sending any mutation. + client + .helm(args(&[ + "template", + &p.config.service.release, + p.config + .service + .chart + .to_str() + .context("Invalid chart path")?, + "-n", + &p.config.service.namespace, + "-f", + root.join(&file) + .to_str() + .context("Invalid feature values path")?, + ])) + .await?; + j.feature_updates.push(Update { + request_id: id.clone(), + before, + values: values.clone(), + receipt: None, + }); + save(root, "journal.json", &j)?; + let result = client + .helm(args(&[ + "upgrade", + &p.config.service.release, + p.config.service.chart.to_str().unwrap(), + "-n", + &p.config.service.namespace, + "-f", + root.join(&file).to_str().unwrap(), + "--labels", + &format!("{OWNER}={},openworker.io/operation-id={id}", p.id), + "--description", + &format!("ows:{}:{id}", p.id), + "--wait", + "--timeout", + "300s", + ])) + .await; + if result.is_err() { + // A response can be lost after success. Only observation of this + // exact revision and intent may recover it; never replay Helm. + reconcile(&client, &p, root, &mut j).await.context( + "Helm response failed; saved feature intent retained for status reconciliation", + )?; + } else { + reconcile(&client, &p, root, &mut j).await?; + } + } + } + let values = effective_values(&p, &j)?; + let ui = &values["managementUI"]; + let obs = &values["config"]["observability"]; + let preview = if ui["enabled"] == true + && ui["tlsSecret"].as_str().is_some_and(|s| !s.is_empty()) + { + let origin = ui["publicOrigin"] + .as_str() + .unwrap_or("https://localhost:8444"); + let u = reqwest::Url::parse(origin)?; + json!({"url":origin,"command":"kubectl","arguments":["--kubeconfig",p.config.kubeconfig,"--context",p.config.context,"-n",p.config.service.namespace,"port-forward",format!("service/{}-ui",p.config.service.release),format!("{}:443",u.port_or_known_default().unwrap()),"--address=127.0.0.1"],"caFile":"Use the API certificate from the deployment credentials; do not disable certificate verification","login":"existing service token; UI does not hold a bootstrap credential"}) + } else { + Value::Null + }; + crate::output( + json!({"deploymentId":p.id,"managementUI":{"enabled":ui["enabled"]==true,"publicOrigin":ui["publicOrigin"],"privatePreview":preview},"observability":{"enabled":!obs.is_null(),"configuration":obs,"databaseSecret":values["secrets"]["observability"],"storedData":"retained; no deletion performed","exporters":"configured independently"},"savedConfiguration":root.join("journal.json"),"helmRevision":confirmed_receipt(&j)?["revision"]}), + ) +} +pub(crate) async fn ui(command: &UICommand) -> Result<()> { + match command { + UICommand::Status { target } => change(&target.state, |_, _| Ok(()), true).await, + UICommand::Disable { target } => { + change( + &target.state, + |v, _| { + if v["managementUI"].is_null() { + v["managementUI"] = json!({}); + } + v["managementUI"]["enabled"] = json!(false); + Ok(()) + }, + false, + ) + .await + } + UICommand::Enable { + target, + image, + public_origin, + ca_secret, + ca_key, + } => { + change( + &target.state, + |v, p| { + let ui = ManagementUI { + enabled: true, + public_origin: public_origin + .clone() + .or_else(|| { + v["managementUI"]["publicOrigin"] + .as_str() + .map(str::to_owned) + }) + .unwrap_or_else(origin), + ca_secret: ca_secret.clone(), + ca_key: ca_key.clone(), + }; + configure_ui( + v, + &ui, + image.as_deref(), + &p.config.service.release, + &p.config.service.namespace, + ) + }, + false, + ) + .await + } + } +} +pub(crate) async fn observability(command: &ObservationCommand) -> Result<()> { + match command { + ObservationCommand::Status { target } => change(&target.state, |_, _| Ok(()), true).await, + ObservationCommand::Disable { target } => { + change( + &target.state, + |v, _| configure_observability(v, &Observability::default()), + false, + ) + .await + } + ObservationCommand::Enable { + target, + retention_days, + database_secret, + } => { + change( + &target.state, + |v, _| { + configure_observability( + v, + &Observability { + enabled: true, + retention_days: retention_days.unwrap_or_else(|| { + v["config"]["observability"]["retention_days"] + .as_u64() + .unwrap_or(30) as u16 + }), + database_secret: database_secret.clone(), + }, + ) + }, + false, + ) + .await + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn rendering_is_opt_in_independent_and_preserves_other_settings() { + let original = json!({"image":"service","config":{"schema":"execution","telemetry":{"enabled":true}},"secrets":{"apiTLS":"api-cert"}}); + let mut values = original.clone(); + let ui = ManagementUI { + enabled: true, + ..Default::default() + }; + configure_ui( + &mut values, + &ui, + Some(&format!("ui@sha256:{}", "b".repeat(64))), + "ows", + "example", + ) + .unwrap(); + assert!(values["config"].get("observability").is_none()); + assert_eq!(values["managementUI"]["tlsSecret"], "api-cert"); + assert_eq!( + values["managementUI"]["serviceUrl"], + "https://ows-api.example.svc" + ); + configure_observability( + &mut values, + &Observability { + enabled: true, + retention_days: 14, + database_secret: Some("recording-db".into()), + }, + ) + .unwrap(); + assert_eq!(values["secrets"]["observability"], "recording-db"); + configure_observability(&mut values, &Observability::default()).unwrap(); + assert!(values["config"].get("observability").is_none()); + assert_eq!( + values["config"]["telemetry"], + original["config"]["telemetry"] + ); + assert_eq!(values["image"], original["image"]); + assert_eq!(values["managementUI"]["enabled"], true); + } + #[test] + fn invalid_origins_retention_and_inline_database_values_fail_before_mutation() { + for origin in [ + "http://localhost:8444", + "https://user:secret@example.com", + "https://example.com/path", + "https://example.com/?token=secret", + ] { + assert!(ManagementUI { + enabled: true, + public_origin: origin.into(), + ..Default::default() + } + .validate() + .is_err()); + } + for retention_days in [0, 366] { + assert!(Observability { + enabled: true, + retention_days, + database_secret: None + } + .validate() + .is_err()); + } + assert!(Observability { + enabled: true, + retention_days: 14, + database_secret: Some("postgres://secret".into()) + } + .validate() + .is_err()); + } +} diff --git a/harness/service-cli/src/deployment_local.rs b/harness/service-cli/src/deployment_local.rs index 3a56a3cd..c7ef2914 100644 --- a/harness/service-cli/src/deployment_local.rs +++ b/harness/service-cli/src/deployment_local.rs @@ -6,6 +6,8 @@ use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, process::Command as Process, }; +#[path = "deployment_features.rs"] +pub(crate) mod features; const OWNER: &str = "openworker.io/installer-id"; #[path = "deployment_cluster_platform.rs"] mod platform; @@ -151,6 +153,8 @@ struct Journal { operations: BTreeMap, #[serde(default)] confirmed_no_effect: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + feature_updates: Vec, } #[derive(Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -651,6 +655,10 @@ async fn install_release( owned(client, p, j, ns) .await? .context("Owned namespace missing")?; + if key == "service-release" && !j.feature_updates.is_empty() { + features::verify_current(client, p, j).await?; + return Ok(()); + } if let Some(mut r) = client.release(ns, release).await? { let op = j .operations @@ -692,7 +700,7 @@ async fn install_release( .is_none_or(|v| v.is_null() || v.as_array().is_some_and(Vec::is_empty)), "Hooks may have unknown effects; failed release requires manual review" ); - ready_service(client, p).await?; + ready_service(client, p, j).await?; let job = client .get("job", &format!("{release}-bootstrap"), Some(ns)) .await? @@ -822,9 +830,12 @@ async fn prerequisites(client: &Client<'_>, eks: bool) -> Result<()> { ); Ok(()) } -async fn ready_service(client: &Client<'_>, p: &Plan) -> Result<()> { - let values: Value = read_json(&p.config.service.values_file)?; - let mut roles = vec!["api", "worker", "controller", "platform"]; +async fn ready_service(client: &Client<'_>, p: &Plan, j: &Journal) -> Result<()> { + let values = features::effective_values(p, j)?; + let mut roles = vec!["api", "worker", "controller"]; + if values["platformControllerEnabled"] != false { + roles.push("platform"); + } if values["managementUI"]["enabled"] == true { roles.push("ui"); } @@ -1236,11 +1247,13 @@ async fn uninstall( .context("Owned namespace missing before uninstall")?; let delete = format!("uninstall:{key}"); if let Some(r) = client.release(ns, release).await? { - let receipt = j - .operations - .get(key) - .and_then(|o| o.receipt.as_ref()) - .context("Release identity was never confirmed")?; + let receipt = if key == "service-release" { + features::latest_receipt(j) + } else { + None + } + .or_else(|| j.operations.get(key).and_then(|o| o.receipt.as_ref())) + .context("Release identity was never confirmed")?; ensure!( r["uid"] == receipt["uid"] && r["revision"] == receipt["revision"], "Release identity changed before uninstall" @@ -1447,6 +1460,7 @@ async fn run_scoped(command: &Command, local_only: bool) -> Result<()> { .await?; } let client = Client { c: &p.config }; + features::reconcile(&client, &p, state, &mut j).await?; ensure!( client.cluster().await? == p.cluster_uid, "Cluster identity changed" @@ -1709,7 +1723,7 @@ async fn run_scoped(command: &Command, local_only: bool) -> Result<()> { }, ) .await?; - ready_service(&client, &p).await?; + ready_service(&client, &p, &j).await?; j.phase = "installed".into(); save(state, "journal.json", &j)?; } diff --git a/harness/service-cli/src/deployment_up.rs b/harness/service-cli/src/deployment_up.rs index 667e852d..ec3c99b8 100644 --- a/harness/service-cli/src/deployment_up.rs +++ b/harness/service-cli/src/deployment_up.rs @@ -156,6 +156,11 @@ pub(super) fn operation_lock(root: &Path, name: &str) -> Result { Ok(f) } async fn check_local_tools(c: &Config, package: &package::Snapshot) -> Result<()> { + ensure!( + !c.management_ui.as_ref().is_some_and(|ui| ui.enabled) + || package.release.images.service_ui.is_some(), + "Enabling managementUI requires a digest-pinned service-ui image in the release manifest" + ); let mut tools: Vec<(&str, Vec<&str>)> = vec![ ("kubectl", vec!["version", "--client", "--output=json"]), ("helm", vec!["version", "--short"]), diff --git a/harness/service-cli/src/deployment_up_config.rs b/harness/service-cli/src/deployment_up_config.rs index 7356f097..bfe0d7f9 100644 --- a/harness/service-cli/src/deployment_up_config.rs +++ b/harness/service-cli/src/deployment_up_config.rs @@ -17,6 +17,11 @@ pub(super) struct Config { pub models: BTreeMap, #[serde(default)] pub advanced: Advanced, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(rename = "managementUI")] + pub management_ui: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observability: Option, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -170,6 +175,12 @@ impl Config { Ok(config) } pub fn validate(&self, provider: &str) -> Result<()> { + if let Some(ui) = &self.management_ui { + ui.validate()?; + } + if let Some(obs) = &self.observability { + obs.validate()?; + } if let Some(t) = &self.trial { ensure!((1..=72).contains(&t.expires_after_hours)&&t.delete_data,"A disposable trial requires 1..72 hours and explicit deleteData:true; omit trial for persistent deployment"); } @@ -310,7 +321,7 @@ impl Config { #[cfg(test)] mod tests { use super::*; - pub(super) fn yaml() -> &'static str { + pub(crate) fn yaml() -> &'static str { "name: example\ncloud:\n provider: azure\n subscriptionId: 11111111-2222-4333-8444-555555555555\n region: westus2\nnodes:\n count: 2\n requirements:\n cpu: {min: 4, max: 8}\n memoryGiB: {min: 16, max: 32}\nbudget:\n limitUSD: 100\n" } #[test] @@ -357,3 +368,26 @@ mod tests { c.validate("azure").unwrap(); } } + +#[cfg(test)] +mod feature_config_tests { + use super::*; + #[test] + fn cloud_settings_accept_independent_features_and_reject_unknown_inline_options() { + let base = tests::yaml(); + for options in [ + "managementUI: {enabled: true}\n", + "observability: {enabled: true, retentionDays: 14, databaseSecret: recording-db}\n", + ] { + let c: Config = serde_yaml::from_str(&format!("{base}{options}")).unwrap(); + c.validate("azure").unwrap(); + } + let c: Config = serde_yaml::from_str(base).unwrap(); + let value = serde_json::to_value(c).unwrap(); + assert!(value.get("managementUI").is_none() && value.get("observability").is_none()); + assert!(serde_yaml::from_str::(&format!( + "{base}observability: {{enabled: true, connection: secret}}\n" + )) + .is_err()); + } +} diff --git a/harness/service-cli/src/deployment_up_install.rs b/harness/service-cli/src/deployment_up_install.rs index 9aecd42e..b9dacaa9 100644 --- a/harness/service-cli/src/deployment_up_install.rs +++ b/harness/service-cli/src/deployment_up_install.rs @@ -89,7 +89,21 @@ impl Native<'_> { let id=if i==0{"primary"}else{p.as_str()}; json!({"id":id,"provider":p,"identity":format!("{}-{id}",plan.deployment_id),"key_env":if p=="anthropic"{"ANTHROPIC_API_KEY"}else{"OPENAI_API_KEY"}}) }).collect(); - let values = json!({"image":harness,"apiReplicas":1,"worker":{"replicas":1,"ownership":"manual","maxReplicas":2,"slots":2,"drainSeconds":30},"terminationGracePeriodSeconds":90,"secrets":{"model":if c.models.is_empty(){""}else{"ow-service-model"}},"config":{"service":{"id":plan.deployment_id,"namespace":"local","agent":"installed-tools","legacy_compatibility":false},"database_env":"OW_PG_URL","schema":"ow_service","database_loopback_plaintext":false,"database_ca":"/run/service/pg-ca.crt","state_root":"/state","bundle":"/opt/openworker/bundle","runner":"/opt/openworker/openworker","model":model,"scripted":false,"model_provider":provider,"model_connections":connections,"api_token_file":"/bootstrap/admin-token","actor":"deployment-admin","openshell":{"identity":plan.deployment_id,"endpoint":endpoint,"tls_server_name":format!("{fullname}.{ns}.svc.cluster.local"),"certificates":"/openshell","workspace":"default","image":runner},"artifacts":artifacts}}); + let mut values = json!({"image":harness,"apiReplicas":1,"worker":{"replicas":1,"ownership":"manual","maxReplicas":2,"slots":2,"drainSeconds":30},"terminationGracePeriodSeconds":90,"secrets":{"model":if c.models.is_empty(){""}else{"ow-service-model"}},"config":{"service":{"id":plan.deployment_id,"namespace":"local","agent":"installed-tools","legacy_compatibility":false},"database_env":"OW_PG_URL","schema":"ow_service","database_loopback_plaintext":false,"database_ca":"/run/service/pg-ca.crt","state_root":"/state","bundle":"/opt/openworker/bundle","runner":"/opt/openworker/openworker","model":model,"scripted":false,"model_provider":provider,"model_connections":connections,"api_token_file":"/bootstrap/admin-token","actor":"deployment-admin","openshell":{"identity":plan.deployment_id,"endpoint":endpoint,"tls_server_name":format!("{fullname}.{ns}.svc.cluster.local"),"certificates":"/openshell","workspace":"default","image":runner},"artifacts":artifacts}}); + if let Some(ui) = &c.management_ui { + super::super::super::local::features::configure_ui( + &mut values, + ui, + images["service-ui"]["image"].as_str(), + "ows", + "ow-service", + )?; + } else if let Some(image) = images["service-ui"]["image"].as_str() { + values["managementUI"] = json!({"enabled":false,"image":image}); + } + if let Some(obs) = &c.observability { + super::super::super::local::features::configure_observability(&mut values, obs)?; + } self.exact_file("service-values.json", &values)?; let mut shell: Value = serde_json::from_slice(&fs::read(&self.package.release.openshell_values.file)?)?; @@ -193,7 +207,7 @@ impl Native<'_> { !cert.exists() && !key.exists(), "Partial TLS generation; inspect private credential files before continuing" ); - self.tool("openssl",vec!["req".into(),"-x509".into(),"-newkey".into(),"rsa:3072".into(),"-nodes".into(),"-days".into(),"30".into(),"-subj".into(),"/CN=ows-api.ow-service.svc.cluster.local".into(),"-addext".into(),"subjectAltName=DNS:localhost,IP:127.0.0.1,DNS:ows-api.ow-service.svc.cluster.local,DNS:ows-api.ow-service.svc".into(),"-keyout".into(),arg(&key)?,"-out".into(),arg(&cert)?],None).await?; + self.tool("openssl",vec!["req".into(),"-x509".into(),"-newkey".into(),"rsa:3072".into(),"-nodes".into(),"-days".into(),"30".into(),"-subj".into(),"/CN=ows-api.ow-service.svc.cluster.local".into(),"-addext".into(),"basicConstraints=critical,CA:TRUE".into(),"-addext".into(),"keyUsage=critical,digitalSignature,keyCertSign,cRLSign".into(),"-addext".into(),"extendedKeyUsage=serverAuth".into(),"-addext".into(),"subjectAltName=DNS:localhost,IP:127.0.0.1,DNS:ows-api.ow-service.svc.cluster.local,DNS:ows-api.ow-service.svc".into(),"-keyout".into(),arg(&key)?,"-out".into(),arg(&cert)?],None).await?; } #[cfg(unix)] { @@ -241,7 +255,17 @@ impl Native<'_> { pub(super) async fn verify(&self) -> Result { let result = self.installer("status", &[]).await?; let kube = self.root.join("handoff/kubeconfig.json"); - for component in ["api", "worker", "controller", "platform"] { + let mut components = vec!["api", "worker", "controller", "platform"]; + if self + .review + .config + .management_ui + .as_ref() + .is_some_and(|ui| ui.enabled) + { + components.push("ui"); + } + for component in components { self.tool( "kubectl", vec![ @@ -258,7 +282,22 @@ impl Native<'_> { ) .await?; } - let connection = json!({"deploymentId":self.review.deployment_id,"serviceId":self.review.deployment_id,"namespace":"local","url":"https://ows-api.ow-service.svc.cluster.local","access":"private cluster network","kubeconfig":kube,"caFile":self.root.join("credentials/api.crt"),"tokenFile":self.install_state().join("bootstrap-token"),"privatePreview":{"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-api","8443:443","--address=127.0.0.1"],"url":"https://localhost:8443"},"modelSmokeTest":"not_run"}); + let mut connection = json!({"deploymentId":self.review.deployment_id,"serviceId":self.review.deployment_id,"namespace":"local","url":"https://ows-api.ow-service.svc.cluster.local","access":"private cluster network","kubeconfig":kube,"caFile":self.root.join("credentials/api.crt"),"tokenFile":self.install_state().join("bootstrap-token"),"privatePreview":{"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-api","8443:443","--address=127.0.0.1"],"url":"https://localhost:8443"},"modelSmokeTest":"not_run"}); + if let Some(ui) = self + .review + .config + .management_ui + .as_ref() + .filter(|ui| ui.enabled) + { + connection["managementUI"] = + json!({"url":ui.public_origin,"login":"existing service token"}); + let url = reqwest::Url::parse(&ui.public_origin)?; + if matches!(url.host_str(), Some("localhost" | "127.0.0.1")) { + connection["managementUI"]["privatePreview"] = json!({"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-ui",format!("{}:443",url.port_or_known_default().unwrap()),"--address=127.0.0.1"],"caFile":self.root.join("credentials/api.crt")}); + } + } + connection["observability"] = json!({"enabled":self.review.config.observability.as_ref().is_some_and(|obs|obs.enabled)}); write_private(&self.root.join("connection.json"), &connection)?; Ok(Observation::Complete( json!({"nativeStatus":result,"rolloutsReady":true,"modelSmokeTest":"not_run","connectionFile":self.root.join("connection.json")}), diff --git a/harness/service-cli/src/main.rs b/harness/service-cli/src/main.rs index 645a43f3..21637e29 100644 --- a/harness/service-cli/src/main.rs +++ b/harness/service-cli/src/main.rs @@ -39,6 +39,16 @@ enum Command { #[command(subcommand)] command: deployment::Command, }, + /// Configure the optional browser UI on an installed cluster. + ManagementUi { + #[command(subcommand)] + command: deployment::local::features::UICommand, + }, + /// Configure optional trace storage; disabling never deletes stored traces. + Observability { + #[command(subcommand)] + command: deployment::local::features::ObservationCommand, + }, Context { #[command(subcommand)] command: ContextCommand, @@ -370,6 +380,13 @@ async fn run(args: Args) -> Result<()> { if let Command::Deploy { command } = &args.command { return deployment::dispatch(command).await; } + match &args.command { + Command::ManagementUi { command } => return deployment::local::features::ui(command).await, + Command::Observability { command } => { + return deployment::local::features::observability(command).await + } + _ => {} + } let mut settings = settings::Settings::load(args.config.as_ref())?; if let Command::Context { command } = &args.command { return match command { @@ -650,7 +667,7 @@ async fn run(args: Args) -> Result<()> { .control_receipt(&receipt) .await?, )?, - Command::Context { .. } | Command::Login | Command::Deploy { .. } => unreachable!(), + Command::Context { .. } | Command::Login | Command::Deploy { .. } | Command::ManagementUi { .. } | Command::Observability { .. } => unreachable!(), } service.disconnect(); Ok(()) diff --git a/harness/service-cli/tests/fixtures/kind-installer.py b/harness/service-cli/tests/fixtures/kind-installer.py index 5d423d9e..4bc82f2e 100644 --- a/harness/service-cli/tests/fixtures/kind-installer.py +++ b/harness/service-cli/tests/fixtures/kind-installer.py @@ -13,12 +13,19 @@ if tool=='helm': ns=arg('-n');releases=state['releases'].setdefault(ns,{}) if a[0]=='list':finish([{'name':n} for n in releases]) if a[0]=='status':finish(releases[a[1]]) + if a[:2]==['get','values']:finish(releases[a[2]].get('values',{})) + if a[0]=='template':finish('rendered') if a[0]=='install': if os.environ.get('FAIL_BEFORE')=='yes':sys.exit(77) - name=a[1];labels=dict(pair.split('=',1) for pair in arg('--labels').split(','));releases[name]={'version':1,'info':{'status':'failed' if os.environ.get('FAIL_RELEASE') else 'deployed','description':'context deadline exceeded' if os.environ.get('FAIL_RELEASE') else arg('--description')},'uid':'release-uid','labels':labels};store.write_text(json.dumps(state)) + name=a[1];labels=dict(pair.split('=',1) for pair in arg('--labels').split(','));releases[name]={'version':1,'info':{'status':'failed' if os.environ.get('FAIL_RELEASE') else 'deployed','description':'context deadline exceeded' if os.environ.get('FAIL_RELEASE') else arg('--description')},'uid':'release-uid','labels':labels,'values':json.loads(pathlib.Path(arg('-f')).read_text())};store.write_text(json.dumps(state)) if os.environ.get('FAIL_RELEASE'):sys.exit(81) if os.environ.get('FAIL_AFTER')=='yes':sys.exit(78) finish('installed') + if a[0]=='upgrade': + if os.environ.get('FAIL_UPGRADE_BEFORE')=='yes':sys.exit(77) + name=a[1];r=releases[name];r['version']+=1;r['uid']='release-uid-'+str(r['version']);r['labels']=dict(pair.split('=',1) for pair in arg('--labels').split(','));r['info']={'status':'deployed','description':arg('--description')};r['values']=json.loads(pathlib.Path(arg('-f')).read_text());store.write_text(json.dumps(state)) + if os.environ.get('FAIL_UPGRADE_AFTER')=='yes':sys.exit(78) + finish('upgraded') if a[0]=='uninstall':del releases[a[1]];finish('uninstalled') sys.exit(99) # kubectl global options precede its command. diff --git a/harness/service-cli/tests/local_installer.rs b/harness/service-cli/tests/local_installer.rs index ec2ea638..940a4e58 100644 --- a/harness/service-cli/tests/local_installer.rs +++ b/harness/service-cli/tests/local_installer.rs @@ -451,3 +451,140 @@ fn failed_original_release_receipt_is_cleanup_only_and_rejects_drift() { assert_eq!(f.installs(), 1); assert_eq!(parsed(&f.destroy())["phase"], "destroyed"); } + +fn feature( + f: &Fixture, + name: &str, + action: &str, + options: &[&str], + env: &[(&str, &str)], +) -> Output { + let mut args = vec![name, action, "--state", f.state.to_str().unwrap()]; + args.extend_from_slice(options); + Fixture::call(f.dir.path(), &args, env) +} +fn upgrades(f: &Fixture) -> usize { + fs::read_to_string(f.dir.path().join("calls")) + .unwrap() + .lines() + .filter(|l| { + let v: Value = serde_json::from_str(l).unwrap(); + v[0] == "helm" && v[1][0] == "upgrade" + }) + .count() +} +#[test] +fn feature_controls_preserve_values_disable_without_data_deletion_and_cleanup_latest_revision() { + let f = Fixture::new(); + parsed(&f.apply(&[], false)); + let image = format!("ui@sha256:{}", "b".repeat(64)); + let enabled = parsed(&feature( + &f, + "management-ui", + "enable", + &["--image", &image], + &[], + )); + assert_eq!(enabled["managementUI"]["enabled"], true); + assert_eq!( + enabled["managementUI"]["privatePreview"]["url"], + "https://localhost:8444" + ); + assert_eq!(enabled["observability"]["enabled"], false); + let obs = parsed(&feature( + &f, + "observability", + "enable", + &["--retention-days", "14"], + &[], + )); + assert_eq!(obs["observability"]["configuration"]["retention_days"], 14); + assert_eq!(obs["managementUI"]["enabled"], true); + let count = upgrades(&f); + parsed(&feature( + &f, + "observability", + "enable", + &["--retention-days", "14"], + &[], + )); + parsed(&feature(&f, "management-ui", "status", &[], &[])); + assert_eq!(upgrades(&f), count); + parsed(&feature(&f, "management-ui", "disable", &[], &[])); + let disabled = parsed(&feature(&f, "observability", "disable", &[], &[])); + assert_eq!(disabled["observability"]["enabled"], false); + assert_eq!(disabled["managementUI"]["enabled"], false); + let v = &f.remote()["releases"]["ow-service-m4-installer"]["ows"]["values"]; + assert_eq!(v["bootstrapEnabled"], false); + assert_eq!(v["image"], format!("image@sha256:{}", "a".repeat(64))); + assert!(v["config"].get("observability").is_none()); + assert_eq!(f.installs(), 1); + // A normal installer resume and teardown still recognize the updated release. + parsed(&f.apply(&[], false)); + parsed(&f.destroy()); +} +#[test] +fn feature_upgrade_lost_response_is_observed_without_a_second_upgrade() { + let f = Fixture::new(); + parsed(&f.apply(&[], false)); + let out = feature( + &f, + "observability", + "enable", + &[], + &[("FAIL_UPGRADE_AFTER", "yes")], + ); + assert_eq!(parsed(&out)["observability"]["enabled"], true); + parsed(&feature(&f, "observability", "status", &[], &[])); + assert_eq!(upgrades(&f), 1); +} +#[test] +fn unknown_feature_outcome_is_not_replayed_and_blocks_cleanup() { + let f = Fixture::new(); + parsed(&f.apply(&[], false)); + assert!(!feature( + &f, + "observability", + "enable", + &[], + &[("FAIL_UPGRADE_BEFORE", "yes")] + ) + .status + .success()); + for action in ["enable", "status", "disable"] { + assert!(!feature(&f, "observability", action, &[], &[]) + .status + .success()); + } + assert!(!f.destroy().status.success()); + assert_eq!(upgrades(&f), 1); +} +#[test] +fn feature_controls_reject_replaced_release_or_namespace_and_mutable_images() { + let f = Fixture::new(); + parsed(&f.apply(&[], false)); + assert!(!feature( + &f, + "management-ui", + "enable", + &["--image", "ui:latest"], + &[] + ) + .status + .success()); + assert_eq!(upgrades(&f), 0); + let original = f.remote(); + let mut changed = original.clone(); + changed["releases"]["ow-service-m4-installer"]["ows"]["version"] = json!(8); + f.set_remote(&changed); + assert!(!feature(&f, "observability", "enable", &[], &[]) + .status + .success()); + let mut changed = original; + changed["namespaces"]["ow-service-m4-installer"]["metadata"]["uid"] = json!("replacement"); + f.set_remote(&changed); + assert!(!feature(&f, "observability", "enable", &[], &[]) + .status + .success()); + assert_eq!(upgrades(&f), 0); +}