mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
Add CLI controls for management UI and trace storage
This commit is contained in:
@@ -229,3 +229,29 @@ test("exporter authentication mounts only into harness roles and remains opt-in"
|
||||
assert.match(bootstrap, /OW_TELEMETRY_AUTH/);
|
||||
assert.match(bootstrap, /secretName: export-auth/);
|
||||
});
|
||||
test("attachment chunks have a scoped 1 MiB limit, separate from metadata and artifacts",async()=>{
|
||||
const out=await render({});
|
||||
const block=out.match(/location ~ \^\/v1\/sessions\/\[A-Za-z0-9_-\]\+\/attachments\/\[A-Za-z0-9_-\]\+\$ \{[\s\S]*?\}/)?.[0];
|
||||
assert.ok(block,"Only the exact authenticated attachment upload path gets a larger bound");
|
||||
assert.match(block,/client_max_body_size 1m/);
|
||||
assert.match(block,/proxy_request_buffering off/);
|
||||
assert.match(out,/ssl_protocols TLSv1.2 TLSv1.3; client_max_body_size 160k/);
|
||||
});
|
||||
test("recording enable runs only advisory-locked schema initialization, disable removes it",async()=>{
|
||||
const off=await render({});assert.doesNotMatch(off,/name: recording-schema/);
|
||||
const on=await render({config:{...base.config,observability:{retention_days:14}}});
|
||||
for(const role of ["api","worker","controller","platform"]){
|
||||
const d=deployment(on,role);const init=d.split("initContainers:")[1].split(" containers:")[0];
|
||||
assert.match(init,/args: \[recording-migrate\]/);
|
||||
assert.doesNotMatch(init,/bootstrap|\/models|\/openshell/);
|
||||
}
|
||||
});
|
||||
test("private management UI preview terminates TLS without giving the UI server the private key",async()=>{
|
||||
const out=await render({managementUI:{enabled:true,image:"ui@sha256:"+"b".repeat(64),publicOrigin:"https://localhost:8444",serviceUrl:"https://test-api.default.svc",caSecret:"ow-service-tls",caKey:"tls.crt",tlsSecret:"ow-service-tls"}});
|
||||
const d=deployment(out,"ui");
|
||||
assert.match(d,/name: tls/);assert.match(d,/key: "tls.crt", path: ca.crt/);
|
||||
const ui=d.split("- name: ui")[1].split("- name: tls")[0];
|
||||
assert.doesNotMatch(ui,/mountPath: \/tls/);
|
||||
assert.match(out,/name: https, port: 443, targetPort: https/);
|
||||
assert.match(out,/proxy_set_header Host \$http_host/);
|
||||
});
|
||||
|
||||
@@ -54,6 +54,13 @@ data:
|
||||
proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1;
|
||||
proxy_set_header Connection ""; proxy_request_buffering off;
|
||||
proxy_buffering off; proxy_read_timeout 120s; proxy_send_timeout 120s; }
|
||||
# SDK chunks contain base64 bytes plus metadata. Match only the upload
|
||||
# route and the backend's 1 MiB bound; ordinary JSON stays at 160 KiB.
|
||||
location ~ ^/v1/sessions/[A-Za-z0-9_-]+/attachments/[A-Za-z0-9_-]+$ {
|
||||
client_max_body_size 1m; client_body_timeout 120s;
|
||||
proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1;
|
||||
proxy_set_header Connection ""; proxy_request_buffering off;
|
||||
proxy_buffering off; proxy_read_timeout 120s; proxy_send_timeout 120s; }
|
||||
location / { proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1;
|
||||
proxy_set_header Connection ""; proxy_buffering off; proxy_read_timeout 3600s; }
|
||||
}
|
||||
|
||||
@@ -8,6 +8,27 @@ metadata: {name: {{ .Release.Name }}-ui}
|
||||
data:
|
||||
ui.json: |
|
||||
{{- dict "host" "0.0.0.0" "port" 8080 "publicOrigin" .Values.managementUI.publicOrigin "serviceUrl" .Values.managementUI.serviceUrl "serviceId" .Values.config.service.id "namespace" .Values.config.service.namespace "sessionSeconds" 1800 | toPrettyJson | nindent 4 }}
|
||||
{{- if .Values.managementUI.tlsSecret }}
|
||||
nginx.conf: |
|
||||
pid /tmp/nginx.pid;
|
||||
error_log /dev/stderr warn;
|
||||
events { worker_connections 128; }
|
||||
http {
|
||||
access_log off;
|
||||
client_body_temp_path /tmp/body; proxy_temp_path /tmp/proxy;
|
||||
fastcgi_temp_path /tmp/fastcgi; uwsgi_temp_path /tmp/uwsgi; scgi_temp_path /tmp/scgi;
|
||||
server {
|
||||
listen 8443 ssl;
|
||||
ssl_certificate /tls/tls.crt; ssl_certificate_key /tls/tls.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.3; client_max_body_size 160k;
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1;
|
||||
proxy_set_header Host $http_host; proxy_set_header Connection "";
|
||||
proxy_buffering off; proxy_read_timeout 3600s;
|
||||
}
|
||||
}
|
||||
}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
@@ -22,7 +43,7 @@ spec:
|
||||
metadata:
|
||||
labels: {app: {{ .Release.Name }}-ui}
|
||||
annotations:
|
||||
checksum/config: {{ .Values.managementUI | toJson | sha256sum | quote }}
|
||||
checksum/config: {{ list .Values.managementUI .Values.config.service | toJson | sha256sum | quote }}
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
securityContext: {runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, fsGroup: 1000}
|
||||
@@ -39,12 +60,28 @@ spec:
|
||||
- {name: ca, mountPath: /trust, readOnly: true}
|
||||
env: [{name: NODE_EXTRA_CA_CERTS, value: /trust/ca.crt}]
|
||||
{{- end }}
|
||||
{{- if .Values.managementUI.tlsSecret }}
|
||||
- name: tls
|
||||
image: {{ .Values.proxyImage | quote }}
|
||||
command: [nginx, -c, /config/nginx.conf, -g, "daemon off;"]
|
||||
ports: [{name: https, containerPort: 8443}]
|
||||
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}, seccompProfile: {type: RuntimeDefault}}
|
||||
volumeMounts:
|
||||
- {name: config, mountPath: /config, readOnly: true}
|
||||
- {name: tls, mountPath: /tls, readOnly: true}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap: {name: {{ .Release.Name }}-ui}
|
||||
{{- if .Values.managementUI.caSecret }}
|
||||
- name: ca
|
||||
secret: {secretName: {{ .Values.managementUI.caSecret }}, items: [{key: ca.crt, path: ca.crt}]}
|
||||
secret: {secretName: {{ .Values.managementUI.caSecret }}, items: [{key: {{ .Values.managementUI.caKey | quote }}, path: ca.crt}]}
|
||||
{{- end }}
|
||||
{{- if .Values.managementUI.tlsSecret }}
|
||||
- name: tls
|
||||
secret: {secretName: {{ .Values.managementUI.tlsSecret }}, defaultMode: 0440}
|
||||
- {name: tmp, emptyDir: {sizeLimit: 64Mi}}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: v1
|
||||
@@ -52,5 +89,5 @@ kind: Service
|
||||
metadata: {name: {{ .Release.Name }}-ui}
|
||||
spec:
|
||||
selector: {app: {{ .Release.Name }}-ui}
|
||||
ports: [{name: http, port: 80, targetPort: http}]
|
||||
ports: [{{ if .Values.managementUI.tlsSecret }}{name: https, port: 443, targetPort: https}{{ else }}{name: http, port: 80, targetPort: http}{{ end }}]
|
||||
{{- end }}
|
||||
|
||||
@@ -48,6 +48,23 @@ spec:
|
||||
{{- end }}
|
||||
terminationGracePeriodSeconds: {{ $.Values.terminationGracePeriodSeconds }}
|
||||
securityContext: {fsGroup: 1000}
|
||||
{{- if hasKey $.Values.config "observability" }}
|
||||
# Explicit enable-time initialization; advisory-locked and idempotent.
|
||||
# No first-admin bootstrap, provider key or sandbox credential required.
|
||||
initContainers:
|
||||
- name: recording-schema
|
||||
image: {{ $.Values.image | quote }}
|
||||
command: [/bin/sh, /opt/openworker/entrypoint.sh]
|
||||
args: [recording-migrate]
|
||||
securityContext: {runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}, seccompProfile: {type: RuntimeDefault}}
|
||||
volumeMounts:
|
||||
- {name: service, mountPath: /run/service/database-url, subPath: database-url, readOnly: true}
|
||||
- {name: service, mountPath: /run/service/pg-ca.crt, subPath: pg-ca.crt, readOnly: true}
|
||||
- {name: service, mountPath: /run/service/config.json, subPath: config.json, readOnly: true}
|
||||
{{- if $.Values.secrets.observability }}
|
||||
- {name: observability, mountPath: /run/observability, readOnly: true}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: harness
|
||||
image: {{ $.Values.image | quote }}
|
||||
|
||||
@@ -31,3 +31,5 @@ managementUI:
|
||||
# Must match the API TLS certificate (may use the cluster Service DNS name).
|
||||
serviceUrl: ""
|
||||
caSecret: "" # optional Secret containing ca.crt
|
||||
caKey: ca.crt
|
||||
tlsSecret: "" # optional private HTTPS preview using an existing API certificate
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
// Opt-in real nginx test: no cloud access, credentials or model calls.
|
||||
// OWS_PROXY_TEST=1 node --test deploy/service/proxy.test.mjs
|
||||
import {test} from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createServer} from 'node:http';
|
||||
import https from 'node:https';
|
||||
import {mkdtemp,writeFile,readFile,rm} from 'node:fs/promises';
|
||||
import {tmpdir} from 'node:os';
|
||||
import {join,resolve} from 'node:path';
|
||||
import {randomUUID,createHash} from 'node:crypto';
|
||||
import {execFile} from 'node:child_process';
|
||||
import {promisify} from 'node:util';
|
||||
const exec=promisify(execFile);
|
||||
const run=async(program,args)=>String((await exec(program,args,{maxBuffer:8*1024*1024})).stdout);
|
||||
test('real TLS proxy accepts SDK attachment chunks and preserves scoped limits/authentication',{skip:process.env.OWS_PROXY_TEST!=='1',timeout:60000},async()=>{
|
||||
const root=await mkdtemp(join(tmpdir(),'ows-proxy-'));
|
||||
const name='ows-proxy-'+randomUUID().slice(0,8);
|
||||
let forwarded=0;
|
||||
const upstream=createServer(async(req,res)=>{
|
||||
forwarded++;
|
||||
const chunks=[];for await(const c of req)chunks.push(c);
|
||||
if(req.headers.authorization!=='Bearer fixture'){res.writeHead(401).end();return;}
|
||||
const body=Buffer.concat(chunks);
|
||||
res.writeHead(200,{'content-type':'application/json'}).end(JSON.stringify({bytes:body.length,sha256:createHash('sha256').update(body).digest('hex')}));
|
||||
});
|
||||
await new Promise(r=>upstream.listen(0,'0.0.0.0',r));
|
||||
try{
|
||||
const values=join(root,'values.json');
|
||||
await writeFile(values,JSON.stringify({image:'fixture@sha256:'+'a'.repeat(64),config:{service:{id:'proxy-test',namespace:'test',agent:'fixture'}}}));
|
||||
const rendered=await run('helm',['template','proxy-test',resolve('deploy/service/chart'),'-f',values]);
|
||||
const raw=rendered.split(' nginx.conf: |\n')[1].split('\n---')[0].replace(/^ /gm,'');
|
||||
const conf=raw.replaceAll('127.0.0.1:8080',`host.docker.internal:${upstream.address().port}`);
|
||||
await writeFile(join(root,'nginx.conf'),conf);
|
||||
await run('openssl',['req','-x509','-newkey','rsa:2048','-nodes','-days','1','-subj','/CN=localhost','-addext','subjectAltName=DNS:localhost,IP:127.0.0.1','-addext','basicConstraints=critical,CA:TRUE','-addext','keyUsage=critical,digitalSignature,keyCertSign,cRLSign','-addext','extendedKeyUsage=serverAuth','-keyout',join(root,'tls.key'),'-out',join(root,'tls.crt')]);
|
||||
await run('openssl',['verify','-x509_strict','-purpose','sslserver','-CAfile',join(root,'tls.crt'),join(root,'tls.crt')]);
|
||||
const ca=await readFile(join(root,'tls.crt'));
|
||||
const image=(await readFile('deploy/service/chart/values.yaml','utf8')).match(/^proxyImage: (.+)$/m)[1];
|
||||
await run('docker',['run','--rm','-d','--name',name,'-p','127.0.0.1::8443','-v',`${root}:/tls:ro`,image,'nginx','-c','/tls/nginx.conf','-g','daemon off;']);
|
||||
const port=Number((await run('docker',['port',name,'8443/tcp'])).trim().split(':').at(-1));
|
||||
const request=(path,body,auth=true)=>new Promise((resolve,reject)=>{
|
||||
const q=https.request({hostname:'127.0.0.1',port,path,method:'POST',ca,agent:false,headers:{expect:'100-continue','content-type':'application/json','content-length':body.length,...auth?{authorization:'Bearer fixture'}:{}}},r=>{let b='';r.on('data',c=>b+=c);r.on('end',()=>resolve({status:r.statusCode,body:b}));});
|
||||
q.on('error',reject);q.on('continue',()=>q.end(body));q.flushHeaders();
|
||||
});
|
||||
for(let i=0;;i++){try{await request('/healthz',Buffer.alloc(0));break;}catch(e){if(i===30)throw e;await new Promise(r=>setTimeout(r,100));}}
|
||||
const path='/v1/sessions/session_123/attachments/file_123';
|
||||
const chunk=Buffer.from(JSON.stringify({offset:0,data:Buffer.alloc(256*1024,97).toString('base64')}));
|
||||
assert(chunk.length>160*1024&&chunk.length<1024*1024);
|
||||
const ok=await request(path,chunk);assert.equal(ok.status,200);
|
||||
assert.equal(JSON.parse(ok.body).sha256,createHash('sha256').update(chunk).digest('hex'));
|
||||
assert.equal((await request(path,chunk,false)).status,401);
|
||||
const count=forwarded;
|
||||
for(const p of ['/v1/sessions/session_123/turns','/v1/sessions/session_123/attachments','/v1/sessions/session_123/attachments/file_123/extra'])assert.equal((await request(p,chunk)).status,413);
|
||||
assert.equal((await request(path,Buffer.alloc(1024*1024+1))).status,413);
|
||||
assert.equal(forwarded,count,'oversized requests must not reach the application');
|
||||
}finally{
|
||||
await run('docker',['rm','-f',name]).catch(()=>{});
|
||||
await new Promise(r=>upstream.close(r));await rm(root,{recursive:true,force:true});
|
||||
}
|
||||
});
|
||||
@@ -70,6 +70,8 @@ async function certificate(input, name, cn, san) {
|
||||
"365",
|
||||
"-subj",
|
||||
`/CN=${cn} development CA`,
|
||||
"-addext", "basicConstraints=critical,CA:TRUE",
|
||||
"-addext", "keyUsage=critical,keyCertSign,cRLSign",
|
||||
]);
|
||||
await run("openssl", [
|
||||
"req",
|
||||
@@ -86,7 +88,7 @@ async function certificate(input, name, cn, san) {
|
||||
]);
|
||||
await privateWrite(
|
||||
cert + ".ext",
|
||||
`subjectAltName=${san}\nextendedKeyUsage=serverAuth\n`,
|
||||
`subjectAltName=${san}\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\n`,
|
||||
);
|
||||
await run("openssl", [
|
||||
"x509",
|
||||
@@ -224,7 +226,7 @@ export async function init(root, connectionFile, image, port = 58443) {
|
||||
);
|
||||
await privateWrite(
|
||||
join(input, "nginx.conf"),
|
||||
`pid /tmp/nginx.pid;\nerror_log /dev/stderr warn;\nevents {worker_connections 256;}\nhttp {access_log off;client_body_temp_path /tmp/body;proxy_temp_path /tmp/proxy;fastcgi_temp_path /tmp/fastcgi;uwsgi_temp_path /tmp/uwsgi;scgi_temp_path /tmp/scgi;\nserver {listen 8443 ssl;ssl_certificate /tls/tls.crt;ssl_certificate_key /tls/tls.key;ssl_protocols TLSv1.2 TLSv1.3;client_max_body_size 160k;\nlocation ~ ^/(v1/)?sessions/[A-Za-z0-9_-]+/artifacts/[A-Za-z0-9_-]+/content$ {client_max_body_size 64m;client_body_timeout 120s;proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_request_buffering off;proxy_buffering off;proxy_read_timeout 120s;proxy_send_timeout 120s;}\nlocation / {proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_buffering off;proxy_read_timeout 3600s;}}}\n`,
|
||||
`pid /tmp/nginx.pid;\nerror_log /dev/stderr warn;\nevents {worker_connections 256;}\nhttp {access_log off;client_body_temp_path /tmp/body;proxy_temp_path /tmp/proxy;fastcgi_temp_path /tmp/fastcgi;uwsgi_temp_path /tmp/uwsgi;scgi_temp_path /tmp/scgi;\nserver {listen 8443 ssl;ssl_certificate /tls/tls.crt;ssl_certificate_key /tls/tls.key;ssl_protocols TLSv1.2 TLSv1.3;client_max_body_size 160k;\nlocation ~ ^/(v1/)?sessions/[A-Za-z0-9_-]+/artifacts/[A-Za-z0-9_-]+/content$ {client_max_body_size 64m;client_body_timeout 120s;proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_request_buffering off;proxy_buffering off;proxy_read_timeout 120s;proxy_send_timeout 120s;}\nlocation ~ ^/v1/sessions/[A-Za-z0-9_-]+/attachments/[A-Za-z0-9_-]+$ {client_max_body_size 1m;client_body_timeout 120s;proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_request_buffering off;proxy_buffering off;proxy_read_timeout 120s;proxy_send_timeout 120s;}\nlocation / {proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_buffering off;proxy_read_timeout 3600s;}}}\n`,
|
||||
);
|
||||
const manifest = {
|
||||
id,
|
||||
|
||||
@@ -101,3 +101,81 @@ python3 deploy/service/up/test-plan.py \
|
||||
```
|
||||
|
||||
That check uses the real pinned OpenShell archive, fixture image references and catalog responses. It performs no real cloud calls, does not install a cluster and must not be presented as cloud qualification.
|
||||
|
||||
## Management UI and trace storage
|
||||
|
||||
Both features are optional and independent. Select them before the first `up`:
|
||||
|
||||
```yaml
|
||||
managementUI:
|
||||
enabled: true
|
||||
publicOrigin: https://localhost:8444
|
||||
observability:
|
||||
enabled: true
|
||||
retentionDays: 14
|
||||
```
|
||||
|
||||
The reviewed release manifest must include a digest-pinned `service-ui` image.
|
||||
The default private preview uses the API certificate and a loopback-only Kubernetes
|
||||
port-forward; `connection.json` contains its command, URL and CA file. Trust that
|
||||
certificate locally, then sign in with a current service token. No token is put in
|
||||
the browser URL. Keep the cloud's private management route open as described above.
|
||||
A non-loopback `publicOrigin` requires your own TLS ingress; `up` does not create
|
||||
public exposure or DNS records.
|
||||
|
||||
After installation, use the original **installer** state directory (the
|
||||
`installation/` child of the `up --state` directory):
|
||||
|
||||
```sh
|
||||
ows management-ui enable --state .ows/my-deployment/installation
|
||||
ows management-ui status --state .ows/my-deployment/installation
|
||||
ows management-ui disable --state .ows/my-deployment/installation
|
||||
|
||||
ows observability enable --state .ows/my-deployment/installation --retention-days 14
|
||||
ows observability status --state .ows/my-deployment/installation
|
||||
ows observability disable --state .ows/my-deployment/installation
|
||||
```
|
||||
|
||||
If the original release did not include a UI image, pass
|
||||
`--image registry.example.com/ui@sha256:YOUR_VERIFIED_DIGEST` on enable.
|
||||
`--public-origin` overrides its origin. These commands use the original Kubernetes
|
||||
credentials, verify cluster/namespace/release identities, and perform a Helm upgrade
|
||||
of the saved chart. They do not use the service REST token. Features are saved in
|
||||
`journal.json` with their original Helm operation and revision receipts; the initial
|
||||
provisioning review and `deployment.yaml` stay immutable. Rerunning `up` does not
|
||||
reset these saved feature overrides. Do not edit old installer values in place.
|
||||
|
||||
Trace storage uses a separate `ow_observability` schema and connection to the
|
||||
execution PostgreSQL by default. For a separate database, create an existing
|
||||
Kubernetes Secret containing `database-url` and `ca.crt`, then select it with
|
||||
`observability.databaseSecret` or `ows observability enable --database-secret NAME`.
|
||||
Verified database TLS remains required. Enabling initializes the recording schema
|
||||
before workloads start; feature changes roll the affected pods, so allow active
|
||||
turns to drain. The first-admin bootstrap is not rerun.
|
||||
|
||||
Disabling recording stops new recordings as the rollout completes. It does not
|
||||
delete traces, databases or credentials. While disabled, the recorder's retention
|
||||
worker is also stopped; retained data remains until recording resumes or an operator
|
||||
performs a separate cleanup. Security audit, durable execution state and separately
|
||||
configured OTLP exporters are unaffected. This is deployment-level control, not a
|
||||
new per-agent trace policy API.
|
||||
|
||||
A lost Helm response never causes an automatic repeat. `status` can reconcile the
|
||||
original successful revision; an unresolved/pending/failed result blocks another
|
||||
upgrade or teardown until the operator inspects that original operation.
|
||||
|
||||
Local regression checks:
|
||||
|
||||
```sh
|
||||
cargo test --manifest-path harness/Cargo.toml -p openworker-service-cli
|
||||
node --test deploy/service/chart.test.mjs
|
||||
OWS_PROXY_TEST=1 node --test deploy/service/proxy.test.mjs
|
||||
```
|
||||
|
||||
If the API uses a certificate signed by a separate CA, provide its public trust
|
||||
bundle using `managementUI.caSecret` / `managementUI.caKey` in deployment YAML,
|
||||
or `ows management-ui enable --ca-secret API_CA_SECRET --ca-key ca.crt`.
|
||||
The Secret must already exist in the service namespace. Without an override,
|
||||
the CLI trusts the API TLS Secret's `tls.crt`, suitable for the self-signed
|
||||
certificate produced by `up`. The UI process receives only that public certificate;
|
||||
the HTTPS proxy sidecar receives the separate TLS key mount.
|
||||
|
||||
@@ -165,3 +165,21 @@ proof of compliance. A metadata reader is not a content auditor. In server apps,
|
||||
authorize evidence access separately before returning it, and keep database/host
|
||||
APIs in the trusted backend. The service's existing compliance content permission
|
||||
continues to control audit reads. Cluster experiment/review tooling comes later.
|
||||
|
||||
## Manage recording on a service cluster
|
||||
|
||||
The source CLI supports independent UI and trace-storage controls:
|
||||
|
||||
```sh
|
||||
ows observability enable --state ./deployment-state/installation --retention-days 14
|
||||
ows observability status --state ./deployment-state/installation
|
||||
ows observability disable --state ./deployment-state/installation
|
||||
ows management-ui enable --state ./deployment-state/installation
|
||||
```
|
||||
|
||||
Use the installer state created by `ows deploy aws up` or `ows deploy azure up`.
|
||||
Enabling recording initializes its separate PostgreSQL schema before workloads
|
||||
restart. Disabling stops collection after the rollout and retains stored traces;
|
||||
it does not disable security audit or a separately configured exporter.
|
||||
See the [deployment configuration and private UI access](../../../deploy/service/up/README.md#management-ui-and-trace-storage)
|
||||
for the required pinned UI image, CA, database Secret and enable/disable options.
|
||||
|
||||
@@ -29,6 +29,8 @@ enum Command {
|
||||
drain_timeout_seconds: u64,
|
||||
},
|
||||
Migrate,
|
||||
/// Initialize only the optional recording schema, including on feature enable.
|
||||
RecordingMigrate,
|
||||
/// Trusted installer aggregate check; no public/session API equivalent.
|
||||
ServiceInstallStatus,
|
||||
/// Trusted-host operational catalog, not a session-authorized API.
|
||||
@@ -116,7 +118,8 @@ async fn main() -> Result<()> {
|
||||
_ => "admin",
|
||||
};
|
||||
let recording_config = config.clone();
|
||||
let record = !matches!(args.command, Command::Migrate) && !config.recovery_read_only;
|
||||
let record = !matches!(args.command, Command::Migrate | Command::RecordingMigrate)
|
||||
&& !config.recovery_read_only;
|
||||
let started = tokio::task::spawn_blocking(move || {
|
||||
openworker_server_reference::recording::start(&recording_config, role, record)
|
||||
})
|
||||
@@ -161,6 +164,19 @@ async fn run(args: Args, config: Config, telemetry: openworker_telemetry::Teleme
|
||||
) {
|
||||
config.require_execution()?;
|
||||
}
|
||||
if matches!(args.command, Command::RecordingMigrate) {
|
||||
let options = config
|
||||
.observability
|
||||
.as_ref()
|
||||
.context("Recording is disabled")?;
|
||||
let database = options.database(&config)?;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
openworker_observability::postgres::Postgres::migrate(&database)
|
||||
})
|
||||
.await??;
|
||||
println!("Recording schema ready");
|
||||
return Ok(());
|
||||
}
|
||||
let mut db = config.database()?;
|
||||
db.telemetry = telemetry;
|
||||
if matches!(args.command, Command::Migrate) {
|
||||
@@ -594,7 +610,7 @@ async fn run(args: Args, config: Config, telemetry: openworker_telemetry::Teleme
|
||||
| Command::EnvironmentReconcile { .. } => {
|
||||
bail!("This server reference requires the qualified Kubernetes provider")
|
||||
}
|
||||
Command::Migrate => unreachable!(),
|
||||
Command::Migrate | Command::RecordingMigrate => unreachable!(),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -96,3 +96,23 @@ See [AWS deployment](../../docs/harness/guides/aws.md) and
|
||||
with an explicit budget and independent cleanup before production deployment.
|
||||
|
||||
`owcluster` is the primary command. `ows` remains an alias, using the same commands, configuration and saved deployment state. Existing `.ows` paths and `OWS_*` settings remain valid.
|
||||
|
||||
## Optional management UI and trace storage
|
||||
|
||||
Select these features in `deployment.yaml` for the AWS/Azure `up` commands, or
|
||||
change them on an existing installation:
|
||||
|
||||
```sh
|
||||
ows management-ui enable --state ./deployment-state/installation
|
||||
ows management-ui disable --state ./deployment-state/installation
|
||||
ows management-ui status --state ./deployment-state/installation
|
||||
ows observability enable --state ./deployment-state/installation --retention-days 14
|
||||
ows observability disable --state ./deployment-state/installation
|
||||
ows observability status --state ./deployment-state/installation
|
||||
```
|
||||
|
||||
`--state` is the original installer journal, using its Kubernetes credentials and
|
||||
verified cluster identity. UI and trace storage are independent. Disabling tracing
|
||||
retains data and leaves security audit and separately configured exporters intact.
|
||||
The CLI saves feature changes and reconciles lost Helm responses without repeating
|
||||
unknown upgrades. See [setup, UI image, private HTTPS and database options](../../deploy/service/up/README.md#management-ui-and-trace-storage).
|
||||
|
||||
@@ -13,7 +13,7 @@ mod guard;
|
||||
#[path = "deployment_infrastructure.rs"]
|
||||
mod infrastructure;
|
||||
#[path = "deployment_local.rs"]
|
||||
mod local;
|
||||
pub(crate) mod local;
|
||||
#[path = "deployment_preflight.rs"]
|
||||
mod preflight;
|
||||
#[path = "deployment_up.rs"]
|
||||
|
||||
@@ -0,0 +1,617 @@
|
||||
//! Optional service features over the original installer identity and journal.
|
||||
//! Unknown Helm outcomes are observed, never automatically retried.
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub(crate) struct ManagementUI {
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
#[serde(default = "origin")]
|
||||
pub public_origin: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub ca_secret: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub ca_key: Option<String>,
|
||||
}
|
||||
fn origin() -> String {
|
||||
"https://localhost:8444".into()
|
||||
}
|
||||
impl Default for ManagementUI {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
public_origin: origin(),
|
||||
ca_secret: None,
|
||||
ca_key: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl ManagementUI {
|
||||
pub(crate) fn validate(&self) -> Result<()> {
|
||||
if let Some(s) = &self.ca_secret {
|
||||
ensure!(dns_name(s), "Invalid UI CA Secret name");
|
||||
}
|
||||
if let Some(s) = &self.ca_key {
|
||||
ensure!(
|
||||
!s.is_empty()
|
||||
&& s.len() <= 253
|
||||
&& s.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b)),
|
||||
"Invalid UI CA Secret key"
|
||||
);
|
||||
}
|
||||
let u = reqwest::Url::parse(&self.public_origin).context("Invalid management UI origin")?;
|
||||
ensure!(u.scheme()=="https" && u.host_str().is_some() && u.username().is_empty() && u.password().is_none() && u.path()=="/" && u.query().is_none() && u.fragment().is_none(), "Management UI publicOrigin must be an HTTPS origin without credentials, path, query or fragment");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub(crate) struct Observability {
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
#[serde(default = "retention")]
|
||||
pub retention_days: u16,
|
||||
/// Existing Kubernetes Secret containing database-url and ca.crt. No raw credentials.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub database_secret: Option<String>,
|
||||
}
|
||||
fn retention() -> u16 {
|
||||
30
|
||||
}
|
||||
impl Default for Observability {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
retention_days: retention(),
|
||||
database_secret: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Observability {
|
||||
pub(crate) fn validate(&self) -> Result<()> {
|
||||
ensure!(
|
||||
(1..=365).contains(&self.retention_days),
|
||||
"Trace retention must be 1..365 days"
|
||||
);
|
||||
if let Some(s) = &self.database_secret {
|
||||
ensure!(dns_name(s), "Invalid recording database Secret name");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
fn dns_name(s: &str) -> bool {
|
||||
!s.is_empty()
|
||||
&& s.len() <= 63
|
||||
&& s.starts_with(|c: char| c.is_ascii_lowercase() || c.is_ascii_digit())
|
||||
&& s.ends_with(|c: char| c.is_ascii_lowercase() || c.is_ascii_digit())
|
||||
&& s.bytes()
|
||||
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
|
||||
}
|
||||
#[derive(clap::Args)]
|
||||
pub(crate) struct Target {
|
||||
/// Original installer journal (local-plan.json and journal.json), not cloud lifecycle state.
|
||||
#[arg(long)]
|
||||
state: PathBuf,
|
||||
}
|
||||
#[derive(Subcommand)]
|
||||
pub(crate) enum UICommand {
|
||||
Enable {
|
||||
#[command(flatten)]
|
||||
target: Target,
|
||||
/// Digest-pinned UI image. Omit when already recorded in Helm values.
|
||||
#[arg(long)]
|
||||
image: Option<String>,
|
||||
#[arg(long)]
|
||||
public_origin: Option<String>,
|
||||
/// Existing Secret containing the CA that signs the service API certificate.
|
||||
#[arg(long)]
|
||||
ca_secret: Option<String>,
|
||||
#[arg(long, requires = "ca_secret")]
|
||||
ca_key: Option<String>,
|
||||
},
|
||||
Disable {
|
||||
#[command(flatten)]
|
||||
target: Target,
|
||||
},
|
||||
Status {
|
||||
#[command(flatten)]
|
||||
target: Target,
|
||||
},
|
||||
}
|
||||
#[derive(Subcommand)]
|
||||
pub(crate) enum ObservationCommand {
|
||||
Enable {
|
||||
#[command(flatten)]
|
||||
target: Target,
|
||||
#[arg(long, value_parser=clap::value_parser!(u16).range(1..=365))]
|
||||
retention_days: Option<u16>,
|
||||
/// Existing Secret with database-url and ca.crt; omit to reuse execution PostgreSQL.
|
||||
#[arg(long)]
|
||||
database_secret: Option<String>,
|
||||
},
|
||||
Disable {
|
||||
#[command(flatten)]
|
||||
target: Target,
|
||||
},
|
||||
Status {
|
||||
#[command(flatten)]
|
||||
target: Target,
|
||||
},
|
||||
}
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub(super) struct Update {
|
||||
request_id: String,
|
||||
before: Value,
|
||||
values: Value,
|
||||
#[serde(default)]
|
||||
receipt: Option<Value>,
|
||||
}
|
||||
pub(super) fn latest_receipt(j: &Journal) -> Option<&Value> {
|
||||
j.feature_updates.last().and_then(|u| u.receipt.as_ref())
|
||||
}
|
||||
pub(super) fn effective_values(p: &Plan, j: &Journal) -> Result<Value> {
|
||||
j.feature_updates
|
||||
.last()
|
||||
.map(|u| Ok(u.values.clone()))
|
||||
.unwrap_or_else(|| read_json(&p.config.service.values_file))
|
||||
}
|
||||
fn confirmed_receipt(j: &Journal) -> Result<&Value> {
|
||||
latest_receipt(j)
|
||||
.or_else(|| {
|
||||
j.operations
|
||||
.get("service-release")
|
||||
.and_then(|o| o.receipt.as_ref())
|
||||
})
|
||||
.context("Original service installation receipt missing")
|
||||
}
|
||||
fn same_receipt(a: &Value, b: &Value) -> bool {
|
||||
a["uid"] == b["uid"] && a["revision"] == b["revision"] && a["operationId"] == b["operationId"]
|
||||
}
|
||||
async fn live_values(client: &Client<'_>, p: &Plan) -> Result<Value> {
|
||||
serde_json::from_slice(
|
||||
&client
|
||||
.helm(args(&[
|
||||
"get",
|
||||
"values",
|
||||
&p.config.service.release,
|
||||
"-n",
|
||||
&p.config.service.namespace,
|
||||
"-o",
|
||||
"json",
|
||||
]))
|
||||
.await?,
|
||||
)
|
||||
.context("Invalid release values")
|
||||
}
|
||||
pub(super) async fn verify_current(client: &Client<'_>, p: &Plan, j: &Journal) -> Result<Value> {
|
||||
ensure!(
|
||||
j.feature_updates.last().is_none_or(|u| u.receipt.is_some()),
|
||||
"Feature upgrade outcome unknown; rerun status to reconcile, never replay the upgrade"
|
||||
);
|
||||
let r = client
|
||||
.release(&p.config.service.namespace, &p.config.service.release)
|
||||
.await?
|
||||
.context("Original service release absent")?;
|
||||
ensure!(
|
||||
same_receipt(&r, confirmed_receipt(j)?) && r["status"] == "deployed",
|
||||
"Service Helm release identity/revision changed; refusing feature mutation"
|
||||
);
|
||||
Ok(r)
|
||||
}
|
||||
pub(super) async fn reconcile(
|
||||
client: &Client<'_>,
|
||||
p: &Plan,
|
||||
root: &Path,
|
||||
j: &mut Journal,
|
||||
) -> Result<()> {
|
||||
let Some(update) = j
|
||||
.feature_updates
|
||||
.last()
|
||||
.filter(|u| u.receipt.is_none())
|
||||
.cloned()
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
ensure!(
|
||||
client.cluster().await? == p.cluster_uid,
|
||||
"Cluster identity changed"
|
||||
);
|
||||
owned(client, p, j, &p.config.service.namespace)
|
||||
.await?
|
||||
.context("Owned namespace missing")?;
|
||||
let r = client
|
||||
.release(&p.config.service.namespace, &p.config.service.release)
|
||||
.await?
|
||||
.context("Upgrade outcome unknown: original service release absent")?;
|
||||
ensure!(r["installerId"]==p.id && r["operationId"]==update.request_id && r["revision"].as_u64()==update.before["revision"].as_u64().and_then(|n|n.checked_add(1)) && r["status"]=="deployed", "Feature upgrade outcome unknown/pending/failed; inspect the original Helm operation. It will not be retried automatically");
|
||||
ensure!(
|
||||
live_values(client, p).await? == update.values,
|
||||
"Upgraded release values differ from the saved intent"
|
||||
);
|
||||
ready_service(client, p, j).await?;
|
||||
j.feature_updates.last_mut().unwrap().receipt = Some(r);
|
||||
save(root, "journal.json", j)
|
||||
}
|
||||
/// Pure rendering shared by initial cloud deployment and later CLI controls.
|
||||
pub(crate) fn configure_ui(
|
||||
values: &mut Value,
|
||||
ui: &ManagementUI,
|
||||
image: Option<&str>,
|
||||
release: &str,
|
||||
namespace: &str,
|
||||
) -> Result<()> {
|
||||
ui.validate()?;
|
||||
if values["managementUI"].is_null() {
|
||||
values["managementUI"] = json!({});
|
||||
}
|
||||
values["managementUI"]["enabled"] = json!(ui.enabled);
|
||||
if let Some(image) = image {
|
||||
values["managementUI"]["image"] = json!(image);
|
||||
}
|
||||
values["managementUI"]["publicOrigin"] = json!(ui.public_origin);
|
||||
values["managementUI"]["serviceUrl"] = json!(format!("https://{release}-api.{namespace}.svc"));
|
||||
if ui.enabled {
|
||||
let image = values["managementUI"]["image"]
|
||||
.as_str()
|
||||
.context("Provide --image with a digest-pinned management UI image")?;
|
||||
ensure!(
|
||||
image
|
||||
.rsplit_once("@sha256:")
|
||||
.is_some_and(|(name, d)| !name.is_empty()
|
||||
&& d.len() == 64
|
||||
&& d.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))),
|
||||
"Management UI image must use a sha256 digest"
|
||||
);
|
||||
let api_tls = values["secrets"]["apiTLS"]
|
||||
.as_str()
|
||||
.unwrap_or("ow-service-tls")
|
||||
.to_string();
|
||||
let ca_secret = ui
|
||||
.ca_secret
|
||||
.clone()
|
||||
.or_else(|| {
|
||||
values["managementUI"]["caSecret"]
|
||||
.as_str()
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_owned)
|
||||
})
|
||||
.unwrap_or_else(|| api_tls.clone());
|
||||
let ca_key = ui
|
||||
.ca_key
|
||||
.clone()
|
||||
.or_else(|| {
|
||||
if ui.ca_secret.is_none() {
|
||||
values["managementUI"]["caKey"].as_str().map(str::to_owned)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_else(|| {
|
||||
if ui.ca_secret.is_some() {
|
||||
"ca.crt".into()
|
||||
} else {
|
||||
"tls.crt".into()
|
||||
}
|
||||
});
|
||||
values["managementUI"]["caSecret"] = json!(ca_secret);
|
||||
values["managementUI"]["caKey"] = json!(ca_key);
|
||||
// Private loopback preview is self-contained. Public origins require the
|
||||
// operator's TLS ingress; never create public exposure implicitly.
|
||||
let u = reqwest::Url::parse(&ui.public_origin)?;
|
||||
values["managementUI"]["tlsSecret"] =
|
||||
if matches!(u.host_str(), Some("localhost" | "127.0.0.1")) {
|
||||
json!(api_tls)
|
||||
} else {
|
||||
json!("")
|
||||
};
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub(crate) fn configure_observability(values: &mut Value, obs: &Observability) -> Result<()> {
|
||||
obs.validate()?;
|
||||
if values["secrets"].is_null() {
|
||||
values["secrets"] = json!({});
|
||||
}
|
||||
if obs.enabled {
|
||||
if values["config"]["observability"].is_null() {
|
||||
values["config"]["observability"] = json!({"schema":"ow_observability"});
|
||||
}
|
||||
values["config"]["observability"]["retention_days"] = json!(obs.retention_days);
|
||||
if let Some(s) = &obs.database_secret {
|
||||
values["secrets"]["observability"] = json!(s);
|
||||
}
|
||||
} else {
|
||||
values["config"]
|
||||
.as_object_mut()
|
||||
.context("Service configuration missing")?
|
||||
.remove("observability");
|
||||
values["secrets"]["observability"] = json!("");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn change(
|
||||
root: &Path,
|
||||
mutation: impl FnOnce(&mut Value, &Plan) -> Result<()>,
|
||||
status_only: bool,
|
||||
) -> Result<()> {
|
||||
let (p, mut j, _lock) = load(root)?;
|
||||
ensure!(
|
||||
j.phase == "installed",
|
||||
"Feature controls require a completed service installation"
|
||||
);
|
||||
ensure!(
|
||||
inputs(&p.config)? == p.hashes,
|
||||
"Original installer inputs changed"
|
||||
);
|
||||
let client = Client { c: &p.config };
|
||||
ensure!(
|
||||
client.cluster().await? == p.cluster_uid,
|
||||
"Cluster identity changed"
|
||||
);
|
||||
owned(&client, &p, &j, &p.config.service.namespace)
|
||||
.await?
|
||||
.context("Owned namespace missing")?;
|
||||
reconcile(&client, &p, root, &mut j).await?;
|
||||
let before = verify_current(&client, &p, &j).await?;
|
||||
let mut values = effective_values(&p, &j)?;
|
||||
// First update reads the live explicit values to preserve defaults/previous
|
||||
// operator settings, but refuses any unreviewed drift.
|
||||
ensure!(
|
||||
live_values(&client, &p).await? == values,
|
||||
"Service values changed outside the installer"
|
||||
);
|
||||
if !status_only {
|
||||
mutation(&mut values, &p)?;
|
||||
if values != effective_values(&p, &j)? {
|
||||
// Never rerun first-admin bootstrap on a feature upgrade.
|
||||
values["bootstrapEnabled"] = json!(false);
|
||||
let id = uuid::Uuid::new_v4().to_string();
|
||||
let file = format!("feature-values-{id}.json");
|
||||
save(root, &file, &values)?;
|
||||
// Render validation happens before recording/sending any mutation.
|
||||
client
|
||||
.helm(args(&[
|
||||
"template",
|
||||
&p.config.service.release,
|
||||
p.config
|
||||
.service
|
||||
.chart
|
||||
.to_str()
|
||||
.context("Invalid chart path")?,
|
||||
"-n",
|
||||
&p.config.service.namespace,
|
||||
"-f",
|
||||
root.join(&file)
|
||||
.to_str()
|
||||
.context("Invalid feature values path")?,
|
||||
]))
|
||||
.await?;
|
||||
j.feature_updates.push(Update {
|
||||
request_id: id.clone(),
|
||||
before,
|
||||
values: values.clone(),
|
||||
receipt: None,
|
||||
});
|
||||
save(root, "journal.json", &j)?;
|
||||
let result = client
|
||||
.helm(args(&[
|
||||
"upgrade",
|
||||
&p.config.service.release,
|
||||
p.config.service.chart.to_str().unwrap(),
|
||||
"-n",
|
||||
&p.config.service.namespace,
|
||||
"-f",
|
||||
root.join(&file).to_str().unwrap(),
|
||||
"--labels",
|
||||
&format!("{OWNER}={},openworker.io/operation-id={id}", p.id),
|
||||
"--description",
|
||||
&format!("ows:{}:{id}", p.id),
|
||||
"--wait",
|
||||
"--timeout",
|
||||
"300s",
|
||||
]))
|
||||
.await;
|
||||
if result.is_err() {
|
||||
// A response can be lost after success. Only observation of this
|
||||
// exact revision and intent may recover it; never replay Helm.
|
||||
reconcile(&client, &p, root, &mut j).await.context(
|
||||
"Helm response failed; saved feature intent retained for status reconciliation",
|
||||
)?;
|
||||
} else {
|
||||
reconcile(&client, &p, root, &mut j).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
let values = effective_values(&p, &j)?;
|
||||
let ui = &values["managementUI"];
|
||||
let obs = &values["config"]["observability"];
|
||||
let preview = if ui["enabled"] == true
|
||||
&& ui["tlsSecret"].as_str().is_some_and(|s| !s.is_empty())
|
||||
{
|
||||
let origin = ui["publicOrigin"]
|
||||
.as_str()
|
||||
.unwrap_or("https://localhost:8444");
|
||||
let u = reqwest::Url::parse(origin)?;
|
||||
json!({"url":origin,"command":"kubectl","arguments":["--kubeconfig",p.config.kubeconfig,"--context",p.config.context,"-n",p.config.service.namespace,"port-forward",format!("service/{}-ui",p.config.service.release),format!("{}:443",u.port_or_known_default().unwrap()),"--address=127.0.0.1"],"caFile":"Use the API certificate from the deployment credentials; do not disable certificate verification","login":"existing service token; UI does not hold a bootstrap credential"})
|
||||
} else {
|
||||
Value::Null
|
||||
};
|
||||
crate::output(
|
||||
json!({"deploymentId":p.id,"managementUI":{"enabled":ui["enabled"]==true,"publicOrigin":ui["publicOrigin"],"privatePreview":preview},"observability":{"enabled":!obs.is_null(),"configuration":obs,"databaseSecret":values["secrets"]["observability"],"storedData":"retained; no deletion performed","exporters":"configured independently"},"savedConfiguration":root.join("journal.json"),"helmRevision":confirmed_receipt(&j)?["revision"]}),
|
||||
)
|
||||
}
|
||||
pub(crate) async fn ui(command: &UICommand) -> Result<()> {
|
||||
match command {
|
||||
UICommand::Status { target } => change(&target.state, |_, _| Ok(()), true).await,
|
||||
UICommand::Disable { target } => {
|
||||
change(
|
||||
&target.state,
|
||||
|v, _| {
|
||||
if v["managementUI"].is_null() {
|
||||
v["managementUI"] = json!({});
|
||||
}
|
||||
v["managementUI"]["enabled"] = json!(false);
|
||||
Ok(())
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await
|
||||
}
|
||||
UICommand::Enable {
|
||||
target,
|
||||
image,
|
||||
public_origin,
|
||||
ca_secret,
|
||||
ca_key,
|
||||
} => {
|
||||
change(
|
||||
&target.state,
|
||||
|v, p| {
|
||||
let ui = ManagementUI {
|
||||
enabled: true,
|
||||
public_origin: public_origin
|
||||
.clone()
|
||||
.or_else(|| {
|
||||
v["managementUI"]["publicOrigin"]
|
||||
.as_str()
|
||||
.map(str::to_owned)
|
||||
})
|
||||
.unwrap_or_else(origin),
|
||||
ca_secret: ca_secret.clone(),
|
||||
ca_key: ca_key.clone(),
|
||||
};
|
||||
configure_ui(
|
||||
v,
|
||||
&ui,
|
||||
image.as_deref(),
|
||||
&p.config.service.release,
|
||||
&p.config.service.namespace,
|
||||
)
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
pub(crate) async fn observability(command: &ObservationCommand) -> Result<()> {
|
||||
match command {
|
||||
ObservationCommand::Status { target } => change(&target.state, |_, _| Ok(()), true).await,
|
||||
ObservationCommand::Disable { target } => {
|
||||
change(
|
||||
&target.state,
|
||||
|v, _| configure_observability(v, &Observability::default()),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
}
|
||||
ObservationCommand::Enable {
|
||||
target,
|
||||
retention_days,
|
||||
database_secret,
|
||||
} => {
|
||||
change(
|
||||
&target.state,
|
||||
|v, _| {
|
||||
configure_observability(
|
||||
v,
|
||||
&Observability {
|
||||
enabled: true,
|
||||
retention_days: retention_days.unwrap_or_else(|| {
|
||||
v["config"]["observability"]["retention_days"]
|
||||
.as_u64()
|
||||
.unwrap_or(30) as u16
|
||||
}),
|
||||
database_secret: database_secret.clone(),
|
||||
},
|
||||
)
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn rendering_is_opt_in_independent_and_preserves_other_settings() {
|
||||
let original = json!({"image":"service","config":{"schema":"execution","telemetry":{"enabled":true}},"secrets":{"apiTLS":"api-cert"}});
|
||||
let mut values = original.clone();
|
||||
let ui = ManagementUI {
|
||||
enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
configure_ui(
|
||||
&mut values,
|
||||
&ui,
|
||||
Some(&format!("ui@sha256:{}", "b".repeat(64))),
|
||||
"ows",
|
||||
"example",
|
||||
)
|
||||
.unwrap();
|
||||
assert!(values["config"].get("observability").is_none());
|
||||
assert_eq!(values["managementUI"]["tlsSecret"], "api-cert");
|
||||
assert_eq!(
|
||||
values["managementUI"]["serviceUrl"],
|
||||
"https://ows-api.example.svc"
|
||||
);
|
||||
configure_observability(
|
||||
&mut values,
|
||||
&Observability {
|
||||
enabled: true,
|
||||
retention_days: 14,
|
||||
database_secret: Some("recording-db".into()),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(values["secrets"]["observability"], "recording-db");
|
||||
configure_observability(&mut values, &Observability::default()).unwrap();
|
||||
assert!(values["config"].get("observability").is_none());
|
||||
assert_eq!(
|
||||
values["config"]["telemetry"],
|
||||
original["config"]["telemetry"]
|
||||
);
|
||||
assert_eq!(values["image"], original["image"]);
|
||||
assert_eq!(values["managementUI"]["enabled"], true);
|
||||
}
|
||||
#[test]
|
||||
fn invalid_origins_retention_and_inline_database_values_fail_before_mutation() {
|
||||
for origin in [
|
||||
"http://localhost:8444",
|
||||
"https://user:secret@example.com",
|
||||
"https://example.com/path",
|
||||
"https://example.com/?token=secret",
|
||||
] {
|
||||
assert!(ManagementUI {
|
||||
enabled: true,
|
||||
public_origin: origin.into(),
|
||||
..Default::default()
|
||||
}
|
||||
.validate()
|
||||
.is_err());
|
||||
}
|
||||
for retention_days in [0, 366] {
|
||||
assert!(Observability {
|
||||
enabled: true,
|
||||
retention_days,
|
||||
database_secret: None
|
||||
}
|
||||
.validate()
|
||||
.is_err());
|
||||
}
|
||||
assert!(Observability {
|
||||
enabled: true,
|
||||
retention_days: 14,
|
||||
database_secret: Some("postgres://secret".into())
|
||||
}
|
||||
.validate()
|
||||
.is_err());
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,8 @@ use tokio::{
|
||||
io::{AsyncReadExt, AsyncWriteExt},
|
||||
process::Command as Process,
|
||||
};
|
||||
#[path = "deployment_features.rs"]
|
||||
pub(crate) mod features;
|
||||
const OWNER: &str = "openworker.io/installer-id";
|
||||
#[path = "deployment_cluster_platform.rs"]
|
||||
mod platform;
|
||||
@@ -151,6 +153,8 @@ struct Journal {
|
||||
operations: BTreeMap<String, Operation>,
|
||||
#[serde(default)]
|
||||
confirmed_no_effect: Vec<Operation>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
feature_updates: Vec<features::Update>,
|
||||
}
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
@@ -651,6 +655,10 @@ async fn install_release(
|
||||
owned(client, p, j, ns)
|
||||
.await?
|
||||
.context("Owned namespace missing")?;
|
||||
if key == "service-release" && !j.feature_updates.is_empty() {
|
||||
features::verify_current(client, p, j).await?;
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(mut r) = client.release(ns, release).await? {
|
||||
let op = j
|
||||
.operations
|
||||
@@ -692,7 +700,7 @@ async fn install_release(
|
||||
.is_none_or(|v| v.is_null() || v.as_array().is_some_and(Vec::is_empty)),
|
||||
"Hooks may have unknown effects; failed release requires manual review"
|
||||
);
|
||||
ready_service(client, p).await?;
|
||||
ready_service(client, p, j).await?;
|
||||
let job = client
|
||||
.get("job", &format!("{release}-bootstrap"), Some(ns))
|
||||
.await?
|
||||
@@ -822,9 +830,12 @@ async fn prerequisites(client: &Client<'_>, eks: bool) -> Result<()> {
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
async fn ready_service(client: &Client<'_>, p: &Plan) -> Result<()> {
|
||||
let values: Value = read_json(&p.config.service.values_file)?;
|
||||
let mut roles = vec!["api", "worker", "controller", "platform"];
|
||||
async fn ready_service(client: &Client<'_>, p: &Plan, j: &Journal) -> Result<()> {
|
||||
let values = features::effective_values(p, j)?;
|
||||
let mut roles = vec!["api", "worker", "controller"];
|
||||
if values["platformControllerEnabled"] != false {
|
||||
roles.push("platform");
|
||||
}
|
||||
if values["managementUI"]["enabled"] == true {
|
||||
roles.push("ui");
|
||||
}
|
||||
@@ -1236,11 +1247,13 @@ async fn uninstall(
|
||||
.context("Owned namespace missing before uninstall")?;
|
||||
let delete = format!("uninstall:{key}");
|
||||
if let Some(r) = client.release(ns, release).await? {
|
||||
let receipt = j
|
||||
.operations
|
||||
.get(key)
|
||||
.and_then(|o| o.receipt.as_ref())
|
||||
.context("Release identity was never confirmed")?;
|
||||
let receipt = if key == "service-release" {
|
||||
features::latest_receipt(j)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
.or_else(|| j.operations.get(key).and_then(|o| o.receipt.as_ref()))
|
||||
.context("Release identity was never confirmed")?;
|
||||
ensure!(
|
||||
r["uid"] == receipt["uid"] && r["revision"] == receipt["revision"],
|
||||
"Release identity changed before uninstall"
|
||||
@@ -1447,6 +1460,7 @@ async fn run_scoped(command: &Command, local_only: bool) -> Result<()> {
|
||||
.await?;
|
||||
}
|
||||
let client = Client { c: &p.config };
|
||||
features::reconcile(&client, &p, state, &mut j).await?;
|
||||
ensure!(
|
||||
client.cluster().await? == p.cluster_uid,
|
||||
"Cluster identity changed"
|
||||
@@ -1709,7 +1723,7 @@ async fn run_scoped(command: &Command, local_only: bool) -> Result<()> {
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
ready_service(&client, &p).await?;
|
||||
ready_service(&client, &p, &j).await?;
|
||||
j.phase = "installed".into();
|
||||
save(state, "journal.json", &j)?;
|
||||
}
|
||||
|
||||
@@ -156,6 +156,11 @@ pub(super) fn operation_lock(root: &Path, name: &str) -> Result<fs::File> {
|
||||
Ok(f)
|
||||
}
|
||||
async fn check_local_tools(c: &Config, package: &package::Snapshot) -> Result<()> {
|
||||
ensure!(
|
||||
!c.management_ui.as_ref().is_some_and(|ui| ui.enabled)
|
||||
|| package.release.images.service_ui.is_some(),
|
||||
"Enabling managementUI requires a digest-pinned service-ui image in the release manifest"
|
||||
);
|
||||
let mut tools: Vec<(&str, Vec<&str>)> = vec![
|
||||
("kubectl", vec!["version", "--client", "--output=json"]),
|
||||
("helm", vec!["version", "--short"]),
|
||||
|
||||
@@ -17,6 +17,11 @@ pub(super) struct Config {
|
||||
pub models: BTreeMap<String, Secret>,
|
||||
#[serde(default)]
|
||||
pub advanced: Advanced,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[serde(rename = "managementUI")]
|
||||
pub management_ui: Option<super::super::local::features::ManagementUI>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub observability: Option<super::super::local::features::Observability>,
|
||||
}
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
@@ -170,6 +175,12 @@ impl Config {
|
||||
Ok(config)
|
||||
}
|
||||
pub fn validate(&self, provider: &str) -> Result<()> {
|
||||
if let Some(ui) = &self.management_ui {
|
||||
ui.validate()?;
|
||||
}
|
||||
if let Some(obs) = &self.observability {
|
||||
obs.validate()?;
|
||||
}
|
||||
if let Some(t) = &self.trial {
|
||||
ensure!((1..=72).contains(&t.expires_after_hours)&&t.delete_data,"A disposable trial requires 1..72 hours and explicit deleteData:true; omit trial for persistent deployment");
|
||||
}
|
||||
@@ -310,7 +321,7 @@ impl Config {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
pub(super) fn yaml() -> &'static str {
|
||||
pub(crate) fn yaml() -> &'static str {
|
||||
"name: example\ncloud:\n provider: azure\n subscriptionId: 11111111-2222-4333-8444-555555555555\n region: westus2\nnodes:\n count: 2\n requirements:\n cpu: {min: 4, max: 8}\n memoryGiB: {min: 16, max: 32}\nbudget:\n limitUSD: 100\n"
|
||||
}
|
||||
#[test]
|
||||
@@ -357,3 +368,26 @@ mod tests {
|
||||
c.validate("azure").unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod feature_config_tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn cloud_settings_accept_independent_features_and_reject_unknown_inline_options() {
|
||||
let base = tests::yaml();
|
||||
for options in [
|
||||
"managementUI: {enabled: true}\n",
|
||||
"observability: {enabled: true, retentionDays: 14, databaseSecret: recording-db}\n",
|
||||
] {
|
||||
let c: Config = serde_yaml::from_str(&format!("{base}{options}")).unwrap();
|
||||
c.validate("azure").unwrap();
|
||||
}
|
||||
let c: Config = serde_yaml::from_str(base).unwrap();
|
||||
let value = serde_json::to_value(c).unwrap();
|
||||
assert!(value.get("managementUI").is_none() && value.get("observability").is_none());
|
||||
assert!(serde_yaml::from_str::<Config>(&format!(
|
||||
"{base}observability: {{enabled: true, connection: secret}}\n"
|
||||
))
|
||||
.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,7 +89,21 @@ impl Native<'_> {
|
||||
let id=if i==0{"primary"}else{p.as_str()};
|
||||
json!({"id":id,"provider":p,"identity":format!("{}-{id}",plan.deployment_id),"key_env":if p=="anthropic"{"ANTHROPIC_API_KEY"}else{"OPENAI_API_KEY"}})
|
||||
}).collect();
|
||||
let values = json!({"image":harness,"apiReplicas":1,"worker":{"replicas":1,"ownership":"manual","maxReplicas":2,"slots":2,"drainSeconds":30},"terminationGracePeriodSeconds":90,"secrets":{"model":if c.models.is_empty(){""}else{"ow-service-model"}},"config":{"service":{"id":plan.deployment_id,"namespace":"local","agent":"installed-tools","legacy_compatibility":false},"database_env":"OW_PG_URL","schema":"ow_service","database_loopback_plaintext":false,"database_ca":"/run/service/pg-ca.crt","state_root":"/state","bundle":"/opt/openworker/bundle","runner":"/opt/openworker/openworker","model":model,"scripted":false,"model_provider":provider,"model_connections":connections,"api_token_file":"/bootstrap/admin-token","actor":"deployment-admin","openshell":{"identity":plan.deployment_id,"endpoint":endpoint,"tls_server_name":format!("{fullname}.{ns}.svc.cluster.local"),"certificates":"/openshell","workspace":"default","image":runner},"artifacts":artifacts}});
|
||||
let mut values = json!({"image":harness,"apiReplicas":1,"worker":{"replicas":1,"ownership":"manual","maxReplicas":2,"slots":2,"drainSeconds":30},"terminationGracePeriodSeconds":90,"secrets":{"model":if c.models.is_empty(){""}else{"ow-service-model"}},"config":{"service":{"id":plan.deployment_id,"namespace":"local","agent":"installed-tools","legacy_compatibility":false},"database_env":"OW_PG_URL","schema":"ow_service","database_loopback_plaintext":false,"database_ca":"/run/service/pg-ca.crt","state_root":"/state","bundle":"/opt/openworker/bundle","runner":"/opt/openworker/openworker","model":model,"scripted":false,"model_provider":provider,"model_connections":connections,"api_token_file":"/bootstrap/admin-token","actor":"deployment-admin","openshell":{"identity":plan.deployment_id,"endpoint":endpoint,"tls_server_name":format!("{fullname}.{ns}.svc.cluster.local"),"certificates":"/openshell","workspace":"default","image":runner},"artifacts":artifacts}});
|
||||
if let Some(ui) = &c.management_ui {
|
||||
super::super::super::local::features::configure_ui(
|
||||
&mut values,
|
||||
ui,
|
||||
images["service-ui"]["image"].as_str(),
|
||||
"ows",
|
||||
"ow-service",
|
||||
)?;
|
||||
} else if let Some(image) = images["service-ui"]["image"].as_str() {
|
||||
values["managementUI"] = json!({"enabled":false,"image":image});
|
||||
}
|
||||
if let Some(obs) = &c.observability {
|
||||
super::super::super::local::features::configure_observability(&mut values, obs)?;
|
||||
}
|
||||
self.exact_file("service-values.json", &values)?;
|
||||
let mut shell: Value =
|
||||
serde_json::from_slice(&fs::read(&self.package.release.openshell_values.file)?)?;
|
||||
@@ -193,7 +207,7 @@ impl Native<'_> {
|
||||
!cert.exists() && !key.exists(),
|
||||
"Partial TLS generation; inspect private credential files before continuing"
|
||||
);
|
||||
self.tool("openssl",vec!["req".into(),"-x509".into(),"-newkey".into(),"rsa:3072".into(),"-nodes".into(),"-days".into(),"30".into(),"-subj".into(),"/CN=ows-api.ow-service.svc.cluster.local".into(),"-addext".into(),"subjectAltName=DNS:localhost,IP:127.0.0.1,DNS:ows-api.ow-service.svc.cluster.local,DNS:ows-api.ow-service.svc".into(),"-keyout".into(),arg(&key)?,"-out".into(),arg(&cert)?],None).await?;
|
||||
self.tool("openssl",vec!["req".into(),"-x509".into(),"-newkey".into(),"rsa:3072".into(),"-nodes".into(),"-days".into(),"30".into(),"-subj".into(),"/CN=ows-api.ow-service.svc.cluster.local".into(),"-addext".into(),"basicConstraints=critical,CA:TRUE".into(),"-addext".into(),"keyUsage=critical,digitalSignature,keyCertSign,cRLSign".into(),"-addext".into(),"extendedKeyUsage=serverAuth".into(),"-addext".into(),"subjectAltName=DNS:localhost,IP:127.0.0.1,DNS:ows-api.ow-service.svc.cluster.local,DNS:ows-api.ow-service.svc".into(),"-keyout".into(),arg(&key)?,"-out".into(),arg(&cert)?],None).await?;
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
@@ -241,7 +255,17 @@ impl Native<'_> {
|
||||
pub(super) async fn verify(&self) -> Result<Observation> {
|
||||
let result = self.installer("status", &[]).await?;
|
||||
let kube = self.root.join("handoff/kubeconfig.json");
|
||||
for component in ["api", "worker", "controller", "platform"] {
|
||||
let mut components = vec!["api", "worker", "controller", "platform"];
|
||||
if self
|
||||
.review
|
||||
.config
|
||||
.management_ui
|
||||
.as_ref()
|
||||
.is_some_and(|ui| ui.enabled)
|
||||
{
|
||||
components.push("ui");
|
||||
}
|
||||
for component in components {
|
||||
self.tool(
|
||||
"kubectl",
|
||||
vec![
|
||||
@@ -258,7 +282,22 @@ impl Native<'_> {
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let connection = json!({"deploymentId":self.review.deployment_id,"serviceId":self.review.deployment_id,"namespace":"local","url":"https://ows-api.ow-service.svc.cluster.local","access":"private cluster network","kubeconfig":kube,"caFile":self.root.join("credentials/api.crt"),"tokenFile":self.install_state().join("bootstrap-token"),"privatePreview":{"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-api","8443:443","--address=127.0.0.1"],"url":"https://localhost:8443"},"modelSmokeTest":"not_run"});
|
||||
let mut connection = json!({"deploymentId":self.review.deployment_id,"serviceId":self.review.deployment_id,"namespace":"local","url":"https://ows-api.ow-service.svc.cluster.local","access":"private cluster network","kubeconfig":kube,"caFile":self.root.join("credentials/api.crt"),"tokenFile":self.install_state().join("bootstrap-token"),"privatePreview":{"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-api","8443:443","--address=127.0.0.1"],"url":"https://localhost:8443"},"modelSmokeTest":"not_run"});
|
||||
if let Some(ui) = self
|
||||
.review
|
||||
.config
|
||||
.management_ui
|
||||
.as_ref()
|
||||
.filter(|ui| ui.enabled)
|
||||
{
|
||||
connection["managementUI"] =
|
||||
json!({"url":ui.public_origin,"login":"existing service token"});
|
||||
let url = reqwest::Url::parse(&ui.public_origin)?;
|
||||
if matches!(url.host_str(), Some("localhost" | "127.0.0.1")) {
|
||||
connection["managementUI"]["privatePreview"] = json!({"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-ui",format!("{}:443",url.port_or_known_default().unwrap()),"--address=127.0.0.1"],"caFile":self.root.join("credentials/api.crt")});
|
||||
}
|
||||
}
|
||||
connection["observability"] = json!({"enabled":self.review.config.observability.as_ref().is_some_and(|obs|obs.enabled)});
|
||||
write_private(&self.root.join("connection.json"), &connection)?;
|
||||
Ok(Observation::Complete(
|
||||
json!({"nativeStatus":result,"rolloutsReady":true,"modelSmokeTest":"not_run","connectionFile":self.root.join("connection.json")}),
|
||||
|
||||
@@ -39,6 +39,16 @@ enum Command {
|
||||
#[command(subcommand)]
|
||||
command: deployment::Command,
|
||||
},
|
||||
/// Configure the optional browser UI on an installed cluster.
|
||||
ManagementUi {
|
||||
#[command(subcommand)]
|
||||
command: deployment::local::features::UICommand,
|
||||
},
|
||||
/// Configure optional trace storage; disabling never deletes stored traces.
|
||||
Observability {
|
||||
#[command(subcommand)]
|
||||
command: deployment::local::features::ObservationCommand,
|
||||
},
|
||||
Context {
|
||||
#[command(subcommand)]
|
||||
command: ContextCommand,
|
||||
@@ -370,6 +380,13 @@ async fn run(args: Args) -> Result<()> {
|
||||
if let Command::Deploy { command } = &args.command {
|
||||
return deployment::dispatch(command).await;
|
||||
}
|
||||
match &args.command {
|
||||
Command::ManagementUi { command } => return deployment::local::features::ui(command).await,
|
||||
Command::Observability { command } => {
|
||||
return deployment::local::features::observability(command).await
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let mut settings = settings::Settings::load(args.config.as_ref())?;
|
||||
if let Command::Context { command } = &args.command {
|
||||
return match command {
|
||||
@@ -650,7 +667,7 @@ async fn run(args: Args) -> Result<()> {
|
||||
.control_receipt(&receipt)
|
||||
.await?,
|
||||
)?,
|
||||
Command::Context { .. } | Command::Login | Command::Deploy { .. } => unreachable!(),
|
||||
Command::Context { .. } | Command::Login | Command::Deploy { .. } | Command::ManagementUi { .. } | Command::Observability { .. } => unreachable!(),
|
||||
}
|
||||
service.disconnect();
|
||||
Ok(())
|
||||
|
||||
+8
-1
@@ -13,12 +13,19 @@ if tool=='helm':
|
||||
ns=arg('-n');releases=state['releases'].setdefault(ns,{})
|
||||
if a[0]=='list':finish([{'name':n} for n in releases])
|
||||
if a[0]=='status':finish(releases[a[1]])
|
||||
if a[:2]==['get','values']:finish(releases[a[2]].get('values',{}))
|
||||
if a[0]=='template':finish('rendered')
|
||||
if a[0]=='install':
|
||||
if os.environ.get('FAIL_BEFORE')=='yes':sys.exit(77)
|
||||
name=a[1];labels=dict(pair.split('=',1) for pair in arg('--labels').split(','));releases[name]={'version':1,'info':{'status':'failed' if os.environ.get('FAIL_RELEASE') else 'deployed','description':'context deadline exceeded' if os.environ.get('FAIL_RELEASE') else arg('--description')},'uid':'release-uid','labels':labels};store.write_text(json.dumps(state))
|
||||
name=a[1];labels=dict(pair.split('=',1) for pair in arg('--labels').split(','));releases[name]={'version':1,'info':{'status':'failed' if os.environ.get('FAIL_RELEASE') else 'deployed','description':'context deadline exceeded' if os.environ.get('FAIL_RELEASE') else arg('--description')},'uid':'release-uid','labels':labels,'values':json.loads(pathlib.Path(arg('-f')).read_text())};store.write_text(json.dumps(state))
|
||||
if os.environ.get('FAIL_RELEASE'):sys.exit(81)
|
||||
if os.environ.get('FAIL_AFTER')=='yes':sys.exit(78)
|
||||
finish('installed')
|
||||
if a[0]=='upgrade':
|
||||
if os.environ.get('FAIL_UPGRADE_BEFORE')=='yes':sys.exit(77)
|
||||
name=a[1];r=releases[name];r['version']+=1;r['uid']='release-uid-'+str(r['version']);r['labels']=dict(pair.split('=',1) for pair in arg('--labels').split(','));r['info']={'status':'deployed','description':arg('--description')};r['values']=json.loads(pathlib.Path(arg('-f')).read_text());store.write_text(json.dumps(state))
|
||||
if os.environ.get('FAIL_UPGRADE_AFTER')=='yes':sys.exit(78)
|
||||
finish('upgraded')
|
||||
if a[0]=='uninstall':del releases[a[1]];finish('uninstalled')
|
||||
sys.exit(99)
|
||||
# kubectl global options precede its command.
|
||||
|
||||
@@ -451,3 +451,140 @@ fn failed_original_release_receipt_is_cleanup_only_and_rejects_drift() {
|
||||
assert_eq!(f.installs(), 1);
|
||||
assert_eq!(parsed(&f.destroy())["phase"], "destroyed");
|
||||
}
|
||||
|
||||
fn feature(
|
||||
f: &Fixture,
|
||||
name: &str,
|
||||
action: &str,
|
||||
options: &[&str],
|
||||
env: &[(&str, &str)],
|
||||
) -> Output {
|
||||
let mut args = vec![name, action, "--state", f.state.to_str().unwrap()];
|
||||
args.extend_from_slice(options);
|
||||
Fixture::call(f.dir.path(), &args, env)
|
||||
}
|
||||
fn upgrades(f: &Fixture) -> usize {
|
||||
fs::read_to_string(f.dir.path().join("calls"))
|
||||
.unwrap()
|
||||
.lines()
|
||||
.filter(|l| {
|
||||
let v: Value = serde_json::from_str(l).unwrap();
|
||||
v[0] == "helm" && v[1][0] == "upgrade"
|
||||
})
|
||||
.count()
|
||||
}
|
||||
#[test]
|
||||
fn feature_controls_preserve_values_disable_without_data_deletion_and_cleanup_latest_revision() {
|
||||
let f = Fixture::new();
|
||||
parsed(&f.apply(&[], false));
|
||||
let image = format!("ui@sha256:{}", "b".repeat(64));
|
||||
let enabled = parsed(&feature(
|
||||
&f,
|
||||
"management-ui",
|
||||
"enable",
|
||||
&["--image", &image],
|
||||
&[],
|
||||
));
|
||||
assert_eq!(enabled["managementUI"]["enabled"], true);
|
||||
assert_eq!(
|
||||
enabled["managementUI"]["privatePreview"]["url"],
|
||||
"https://localhost:8444"
|
||||
);
|
||||
assert_eq!(enabled["observability"]["enabled"], false);
|
||||
let obs = parsed(&feature(
|
||||
&f,
|
||||
"observability",
|
||||
"enable",
|
||||
&["--retention-days", "14"],
|
||||
&[],
|
||||
));
|
||||
assert_eq!(obs["observability"]["configuration"]["retention_days"], 14);
|
||||
assert_eq!(obs["managementUI"]["enabled"], true);
|
||||
let count = upgrades(&f);
|
||||
parsed(&feature(
|
||||
&f,
|
||||
"observability",
|
||||
"enable",
|
||||
&["--retention-days", "14"],
|
||||
&[],
|
||||
));
|
||||
parsed(&feature(&f, "management-ui", "status", &[], &[]));
|
||||
assert_eq!(upgrades(&f), count);
|
||||
parsed(&feature(&f, "management-ui", "disable", &[], &[]));
|
||||
let disabled = parsed(&feature(&f, "observability", "disable", &[], &[]));
|
||||
assert_eq!(disabled["observability"]["enabled"], false);
|
||||
assert_eq!(disabled["managementUI"]["enabled"], false);
|
||||
let v = &f.remote()["releases"]["ow-service-m4-installer"]["ows"]["values"];
|
||||
assert_eq!(v["bootstrapEnabled"], false);
|
||||
assert_eq!(v["image"], format!("image@sha256:{}", "a".repeat(64)));
|
||||
assert!(v["config"].get("observability").is_none());
|
||||
assert_eq!(f.installs(), 1);
|
||||
// A normal installer resume and teardown still recognize the updated release.
|
||||
parsed(&f.apply(&[], false));
|
||||
parsed(&f.destroy());
|
||||
}
|
||||
#[test]
|
||||
fn feature_upgrade_lost_response_is_observed_without_a_second_upgrade() {
|
||||
let f = Fixture::new();
|
||||
parsed(&f.apply(&[], false));
|
||||
let out = feature(
|
||||
&f,
|
||||
"observability",
|
||||
"enable",
|
||||
&[],
|
||||
&[("FAIL_UPGRADE_AFTER", "yes")],
|
||||
);
|
||||
assert_eq!(parsed(&out)["observability"]["enabled"], true);
|
||||
parsed(&feature(&f, "observability", "status", &[], &[]));
|
||||
assert_eq!(upgrades(&f), 1);
|
||||
}
|
||||
#[test]
|
||||
fn unknown_feature_outcome_is_not_replayed_and_blocks_cleanup() {
|
||||
let f = Fixture::new();
|
||||
parsed(&f.apply(&[], false));
|
||||
assert!(!feature(
|
||||
&f,
|
||||
"observability",
|
||||
"enable",
|
||||
&[],
|
||||
&[("FAIL_UPGRADE_BEFORE", "yes")]
|
||||
)
|
||||
.status
|
||||
.success());
|
||||
for action in ["enable", "status", "disable"] {
|
||||
assert!(!feature(&f, "observability", action, &[], &[])
|
||||
.status
|
||||
.success());
|
||||
}
|
||||
assert!(!f.destroy().status.success());
|
||||
assert_eq!(upgrades(&f), 1);
|
||||
}
|
||||
#[test]
|
||||
fn feature_controls_reject_replaced_release_or_namespace_and_mutable_images() {
|
||||
let f = Fixture::new();
|
||||
parsed(&f.apply(&[], false));
|
||||
assert!(!feature(
|
||||
&f,
|
||||
"management-ui",
|
||||
"enable",
|
||||
&["--image", "ui:latest"],
|
||||
&[]
|
||||
)
|
||||
.status
|
||||
.success());
|
||||
assert_eq!(upgrades(&f), 0);
|
||||
let original = f.remote();
|
||||
let mut changed = original.clone();
|
||||
changed["releases"]["ow-service-m4-installer"]["ows"]["version"] = json!(8);
|
||||
f.set_remote(&changed);
|
||||
assert!(!feature(&f, "observability", "enable", &[], &[])
|
||||
.status
|
||||
.success());
|
||||
let mut changed = original;
|
||||
changed["namespaces"]["ow-service-m4-installer"]["metadata"]["uid"] = json!("replacement");
|
||||
f.set_remote(&changed);
|
||||
assert!(!feature(&f, "observability", "enable", &[], &[])
|
||||
.status
|
||||
.success());
|
||||
assert_eq!(upgrades(&f), 0);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user