Add CLI controls for management UI and trace storage

This commit is contained in:
Rohit P
2026-11-13 12:38:44 -08:00
parent 59a17fc9a5
commit abcbeadc84
20 changed files with 1177 additions and 25 deletions
+26
View File
@@ -229,3 +229,29 @@ test("exporter authentication mounts only into harness roles and remains opt-in"
assert.match(bootstrap, /OW_TELEMETRY_AUTH/);
assert.match(bootstrap, /secretName: export-auth/);
});
test("attachment chunks have a scoped 1 MiB limit, separate from metadata and artifacts",async()=>{
const out=await render({});
const block=out.match(/location ~ \^\/v1\/sessions\/\[A-Za-z0-9_-\]\+\/attachments\/\[A-Za-z0-9_-\]\+\$ \{[\s\S]*?\}/)?.[0];
assert.ok(block,"Only the exact authenticated attachment upload path gets a larger bound");
assert.match(block,/client_max_body_size 1m/);
assert.match(block,/proxy_request_buffering off/);
assert.match(out,/ssl_protocols TLSv1.2 TLSv1.3; client_max_body_size 160k/);
});
test("recording enable runs only advisory-locked schema initialization, disable removes it",async()=>{
const off=await render({});assert.doesNotMatch(off,/name: recording-schema/);
const on=await render({config:{...base.config,observability:{retention_days:14}}});
for(const role of ["api","worker","controller","platform"]){
const d=deployment(on,role);const init=d.split("initContainers:")[1].split(" containers:")[0];
assert.match(init,/args: \[recording-migrate\]/);
assert.doesNotMatch(init,/bootstrap|\/models|\/openshell/);
}
});
test("private management UI preview terminates TLS without giving the UI server the private key",async()=>{
const out=await render({managementUI:{enabled:true,image:"ui@sha256:"+"b".repeat(64),publicOrigin:"https://localhost:8444",serviceUrl:"https://test-api.default.svc",caSecret:"ow-service-tls",caKey:"tls.crt",tlsSecret:"ow-service-tls"}});
const d=deployment(out,"ui");
assert.match(d,/name: tls/);assert.match(d,/key: "tls.crt", path: ca.crt/);
const ui=d.split("- name: ui")[1].split("- name: tls")[0];
assert.doesNotMatch(ui,/mountPath: \/tls/);
assert.match(out,/name: https, port: 443, targetPort: https/);
assert.match(out,/proxy_set_header Host \$http_host/);
});
@@ -54,6 +54,13 @@ data:
proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1;
proxy_set_header Connection ""; proxy_request_buffering off;
proxy_buffering off; proxy_read_timeout 120s; proxy_send_timeout 120s; }
# SDK chunks contain base64 bytes plus metadata. Match only the upload
# route and the backend's 1 MiB bound; ordinary JSON stays at 160 KiB.
location ~ ^/v1/sessions/[A-Za-z0-9_-]+/attachments/[A-Za-z0-9_-]+$ {
client_max_body_size 1m; client_body_timeout 120s;
proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1;
proxy_set_header Connection ""; proxy_request_buffering off;
proxy_buffering off; proxy_read_timeout 120s; proxy_send_timeout 120s; }
location / { proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1;
proxy_set_header Connection ""; proxy_buffering off; proxy_read_timeout 3600s; }
}
@@ -8,6 +8,27 @@ metadata: {name: {{ .Release.Name }}-ui}
data:
ui.json: |
{{- dict "host" "0.0.0.0" "port" 8080 "publicOrigin" .Values.managementUI.publicOrigin "serviceUrl" .Values.managementUI.serviceUrl "serviceId" .Values.config.service.id "namespace" .Values.config.service.namespace "sessionSeconds" 1800 | toPrettyJson | nindent 4 }}
{{- if .Values.managementUI.tlsSecret }}
nginx.conf: |
pid /tmp/nginx.pid;
error_log /dev/stderr warn;
events { worker_connections 128; }
http {
access_log off;
client_body_temp_path /tmp/body; proxy_temp_path /tmp/proxy;
fastcgi_temp_path /tmp/fastcgi; uwsgi_temp_path /tmp/uwsgi; scgi_temp_path /tmp/scgi;
server {
listen 8443 ssl;
ssl_certificate /tls/tls.crt; ssl_certificate_key /tls/tls.key;
ssl_protocols TLSv1.2 TLSv1.3; client_max_body_size 160k;
location / {
proxy_pass http://127.0.0.1:8080; proxy_http_version 1.1;
proxy_set_header Host $http_host; proxy_set_header Connection "";
proxy_buffering off; proxy_read_timeout 3600s;
}
}
}
{{- end }}
---
apiVersion: apps/v1
kind: Deployment
@@ -22,7 +43,7 @@ spec:
metadata:
labels: {app: {{ .Release.Name }}-ui}
annotations:
checksum/config: {{ .Values.managementUI | toJson | sha256sum | quote }}
checksum/config: {{ list .Values.managementUI .Values.config.service | toJson | sha256sum | quote }}
spec:
automountServiceAccountToken: false
securityContext: {runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, fsGroup: 1000}
@@ -39,12 +60,28 @@ spec:
- {name: ca, mountPath: /trust, readOnly: true}
env: [{name: NODE_EXTRA_CA_CERTS, value: /trust/ca.crt}]
{{- end }}
{{- if .Values.managementUI.tlsSecret }}
- name: tls
image: {{ .Values.proxyImage | quote }}
command: [nginx, -c, /config/nginx.conf, -g, "daemon off;"]
ports: [{name: https, containerPort: 8443}]
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}, seccompProfile: {type: RuntimeDefault}}
volumeMounts:
- {name: config, mountPath: /config, readOnly: true}
- {name: tls, mountPath: /tls, readOnly: true}
- {name: tmp, mountPath: /tmp}
{{- end }}
volumes:
- name: config
configMap: {name: {{ .Release.Name }}-ui}
{{- if .Values.managementUI.caSecret }}
- name: ca
secret: {secretName: {{ .Values.managementUI.caSecret }}, items: [{key: ca.crt, path: ca.crt}]}
secret: {secretName: {{ .Values.managementUI.caSecret }}, items: [{key: {{ .Values.managementUI.caKey | quote }}, path: ca.crt}]}
{{- end }}
{{- if .Values.managementUI.tlsSecret }}
- name: tls
secret: {secretName: {{ .Values.managementUI.tlsSecret }}, defaultMode: 0440}
- {name: tmp, emptyDir: {sizeLimit: 64Mi}}
{{- end }}
---
apiVersion: v1
@@ -52,5 +89,5 @@ kind: Service
metadata: {name: {{ .Release.Name }}-ui}
spec:
selector: {app: {{ .Release.Name }}-ui}
ports: [{name: http, port: 80, targetPort: http}]
ports: [{{ if .Values.managementUI.tlsSecret }}{name: https, port: 443, targetPort: https}{{ else }}{name: http, port: 80, targetPort: http}{{ end }}]
{{- end }}
@@ -48,6 +48,23 @@ spec:
{{- end }}
terminationGracePeriodSeconds: {{ $.Values.terminationGracePeriodSeconds }}
securityContext: {fsGroup: 1000}
{{- if hasKey $.Values.config "observability" }}
# Explicit enable-time initialization; advisory-locked and idempotent.
# No first-admin bootstrap, provider key or sandbox credential required.
initContainers:
- name: recording-schema
image: {{ $.Values.image | quote }}
command: [/bin/sh, /opt/openworker/entrypoint.sh]
args: [recording-migrate]
securityContext: {runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}, seccompProfile: {type: RuntimeDefault}}
volumeMounts:
- {name: service, mountPath: /run/service/database-url, subPath: database-url, readOnly: true}
- {name: service, mountPath: /run/service/pg-ca.crt, subPath: pg-ca.crt, readOnly: true}
- {name: service, mountPath: /run/service/config.json, subPath: config.json, readOnly: true}
{{- if $.Values.secrets.observability }}
- {name: observability, mountPath: /run/observability, readOnly: true}
{{- end }}
{{- end }}
containers:
- name: harness
image: {{ $.Values.image | quote }}
+2
View File
@@ -31,3 +31,5 @@ managementUI:
# Must match the API TLS certificate (may use the cluster Service DNS name).
serviceUrl: ""
caSecret: "" # optional Secret containing ca.crt
caKey: ca.crt
tlsSecret: "" # optional private HTTPS preview using an existing API certificate
+59
View File
@@ -0,0 +1,59 @@
// Opt-in real nginx test: no cloud access, credentials or model calls.
// OWS_PROXY_TEST=1 node --test deploy/service/proxy.test.mjs
import {test} from 'node:test';
import assert from 'node:assert/strict';
import {createServer} from 'node:http';
import https from 'node:https';
import {mkdtemp,writeFile,readFile,rm} from 'node:fs/promises';
import {tmpdir} from 'node:os';
import {join,resolve} from 'node:path';
import {randomUUID,createHash} from 'node:crypto';
import {execFile} from 'node:child_process';
import {promisify} from 'node:util';
const exec=promisify(execFile);
const run=async(program,args)=>String((await exec(program,args,{maxBuffer:8*1024*1024})).stdout);
test('real TLS proxy accepts SDK attachment chunks and preserves scoped limits/authentication',{skip:process.env.OWS_PROXY_TEST!=='1',timeout:60000},async()=>{
const root=await mkdtemp(join(tmpdir(),'ows-proxy-'));
const name='ows-proxy-'+randomUUID().slice(0,8);
let forwarded=0;
const upstream=createServer(async(req,res)=>{
forwarded++;
const chunks=[];for await(const c of req)chunks.push(c);
if(req.headers.authorization!=='Bearer fixture'){res.writeHead(401).end();return;}
const body=Buffer.concat(chunks);
res.writeHead(200,{'content-type':'application/json'}).end(JSON.stringify({bytes:body.length,sha256:createHash('sha256').update(body).digest('hex')}));
});
await new Promise(r=>upstream.listen(0,'0.0.0.0',r));
try{
const values=join(root,'values.json');
await writeFile(values,JSON.stringify({image:'fixture@sha256:'+'a'.repeat(64),config:{service:{id:'proxy-test',namespace:'test',agent:'fixture'}}}));
const rendered=await run('helm',['template','proxy-test',resolve('deploy/service/chart'),'-f',values]);
const raw=rendered.split(' nginx.conf: |\n')[1].split('\n---')[0].replace(/^ /gm,'');
const conf=raw.replaceAll('127.0.0.1:8080',`host.docker.internal:${upstream.address().port}`);
await writeFile(join(root,'nginx.conf'),conf);
await run('openssl',['req','-x509','-newkey','rsa:2048','-nodes','-days','1','-subj','/CN=localhost','-addext','subjectAltName=DNS:localhost,IP:127.0.0.1','-addext','basicConstraints=critical,CA:TRUE','-addext','keyUsage=critical,digitalSignature,keyCertSign,cRLSign','-addext','extendedKeyUsage=serverAuth','-keyout',join(root,'tls.key'),'-out',join(root,'tls.crt')]);
await run('openssl',['verify','-x509_strict','-purpose','sslserver','-CAfile',join(root,'tls.crt'),join(root,'tls.crt')]);
const ca=await readFile(join(root,'tls.crt'));
const image=(await readFile('deploy/service/chart/values.yaml','utf8')).match(/^proxyImage: (.+)$/m)[1];
await run('docker',['run','--rm','-d','--name',name,'-p','127.0.0.1::8443','-v',`${root}:/tls:ro`,image,'nginx','-c','/tls/nginx.conf','-g','daemon off;']);
const port=Number((await run('docker',['port',name,'8443/tcp'])).trim().split(':').at(-1));
const request=(path,body,auth=true)=>new Promise((resolve,reject)=>{
const q=https.request({hostname:'127.0.0.1',port,path,method:'POST',ca,agent:false,headers:{expect:'100-continue','content-type':'application/json','content-length':body.length,...auth?{authorization:'Bearer fixture'}:{}}},r=>{let b='';r.on('data',c=>b+=c);r.on('end',()=>resolve({status:r.statusCode,body:b}));});
q.on('error',reject);q.on('continue',()=>q.end(body));q.flushHeaders();
});
for(let i=0;;i++){try{await request('/healthz',Buffer.alloc(0));break;}catch(e){if(i===30)throw e;await new Promise(r=>setTimeout(r,100));}}
const path='/v1/sessions/session_123/attachments/file_123';
const chunk=Buffer.from(JSON.stringify({offset:0,data:Buffer.alloc(256*1024,97).toString('base64')}));
assert(chunk.length>160*1024&&chunk.length<1024*1024);
const ok=await request(path,chunk);assert.equal(ok.status,200);
assert.equal(JSON.parse(ok.body).sha256,createHash('sha256').update(chunk).digest('hex'));
assert.equal((await request(path,chunk,false)).status,401);
const count=forwarded;
for(const p of ['/v1/sessions/session_123/turns','/v1/sessions/session_123/attachments','/v1/sessions/session_123/attachments/file_123/extra'])assert.equal((await request(p,chunk)).status,413);
assert.equal((await request(path,Buffer.alloc(1024*1024+1))).status,413);
assert.equal(forwarded,count,'oversized requests must not reach the application');
}finally{
await run('docker',['rm','-f',name]).catch(()=>{});
await new Promise(r=>upstream.close(r));await rm(root,{recursive:true,force:true});
}
});
+4 -2
View File
@@ -70,6 +70,8 @@ async function certificate(input, name, cn, san) {
"365",
"-subj",
`/CN=${cn} development CA`,
"-addext", "basicConstraints=critical,CA:TRUE",
"-addext", "keyUsage=critical,keyCertSign,cRLSign",
]);
await run("openssl", [
"req",
@@ -86,7 +88,7 @@ async function certificate(input, name, cn, san) {
]);
await privateWrite(
cert + ".ext",
`subjectAltName=${san}\nextendedKeyUsage=serverAuth\n`,
`subjectAltName=${san}\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\n`,
);
await run("openssl", [
"x509",
@@ -224,7 +226,7 @@ export async function init(root, connectionFile, image, port = 58443) {
);
await privateWrite(
join(input, "nginx.conf"),
`pid /tmp/nginx.pid;\nerror_log /dev/stderr warn;\nevents {worker_connections 256;}\nhttp {access_log off;client_body_temp_path /tmp/body;proxy_temp_path /tmp/proxy;fastcgi_temp_path /tmp/fastcgi;uwsgi_temp_path /tmp/uwsgi;scgi_temp_path /tmp/scgi;\nserver {listen 8443 ssl;ssl_certificate /tls/tls.crt;ssl_certificate_key /tls/tls.key;ssl_protocols TLSv1.2 TLSv1.3;client_max_body_size 160k;\nlocation ~ ^/(v1/)?sessions/[A-Za-z0-9_-]+/artifacts/[A-Za-z0-9_-]+/content$ {client_max_body_size 64m;client_body_timeout 120s;proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_request_buffering off;proxy_buffering off;proxy_read_timeout 120s;proxy_send_timeout 120s;}\nlocation / {proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_buffering off;proxy_read_timeout 3600s;}}}\n`,
`pid /tmp/nginx.pid;\nerror_log /dev/stderr warn;\nevents {worker_connections 256;}\nhttp {access_log off;client_body_temp_path /tmp/body;proxy_temp_path /tmp/proxy;fastcgi_temp_path /tmp/fastcgi;uwsgi_temp_path /tmp/uwsgi;scgi_temp_path /tmp/scgi;\nserver {listen 8443 ssl;ssl_certificate /tls/tls.crt;ssl_certificate_key /tls/tls.key;ssl_protocols TLSv1.2 TLSv1.3;client_max_body_size 160k;\nlocation ~ ^/(v1/)?sessions/[A-Za-z0-9_-]+/artifacts/[A-Za-z0-9_-]+/content$ {client_max_body_size 64m;client_body_timeout 120s;proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_request_buffering off;proxy_buffering off;proxy_read_timeout 120s;proxy_send_timeout 120s;}\nlocation ~ ^/v1/sessions/[A-Za-z0-9_-]+/attachments/[A-Za-z0-9_-]+$ {client_max_body_size 1m;client_body_timeout 120s;proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_request_buffering off;proxy_buffering off;proxy_read_timeout 120s;proxy_send_timeout 120s;}\nlocation / {proxy_pass http://127.0.0.1:8080;proxy_http_version 1.1;proxy_set_header Connection "";proxy_buffering off;proxy_read_timeout 3600s;}}}\n`,
);
const manifest = {
id,
+78
View File
@@ -101,3 +101,81 @@ python3 deploy/service/up/test-plan.py \
```
That check uses the real pinned OpenShell archive, fixture image references and catalog responses. It performs no real cloud calls, does not install a cluster and must not be presented as cloud qualification.
## Management UI and trace storage
Both features are optional and independent. Select them before the first `up`:
```yaml
managementUI:
enabled: true
publicOrigin: https://localhost:8444
observability:
enabled: true
retentionDays: 14
```
The reviewed release manifest must include a digest-pinned `service-ui` image.
The default private preview uses the API certificate and a loopback-only Kubernetes
port-forward; `connection.json` contains its command, URL and CA file. Trust that
certificate locally, then sign in with a current service token. No token is put in
the browser URL. Keep the cloud's private management route open as described above.
A non-loopback `publicOrigin` requires your own TLS ingress; `up` does not create
public exposure or DNS records.
After installation, use the original **installer** state directory (the
`installation/` child of the `up --state` directory):
```sh
ows management-ui enable --state .ows/my-deployment/installation
ows management-ui status --state .ows/my-deployment/installation
ows management-ui disable --state .ows/my-deployment/installation
ows observability enable --state .ows/my-deployment/installation --retention-days 14
ows observability status --state .ows/my-deployment/installation
ows observability disable --state .ows/my-deployment/installation
```
If the original release did not include a UI image, pass
`--image registry.example.com/ui@sha256:YOUR_VERIFIED_DIGEST` on enable.
`--public-origin` overrides its origin. These commands use the original Kubernetes
credentials, verify cluster/namespace/release identities, and perform a Helm upgrade
of the saved chart. They do not use the service REST token. Features are saved in
`journal.json` with their original Helm operation and revision receipts; the initial
provisioning review and `deployment.yaml` stay immutable. Rerunning `up` does not
reset these saved feature overrides. Do not edit old installer values in place.
Trace storage uses a separate `ow_observability` schema and connection to the
execution PostgreSQL by default. For a separate database, create an existing
Kubernetes Secret containing `database-url` and `ca.crt`, then select it with
`observability.databaseSecret` or `ows observability enable --database-secret NAME`.
Verified database TLS remains required. Enabling initializes the recording schema
before workloads start; feature changes roll the affected pods, so allow active
turns to drain. The first-admin bootstrap is not rerun.
Disabling recording stops new recordings as the rollout completes. It does not
delete traces, databases or credentials. While disabled, the recorder's retention
worker is also stopped; retained data remains until recording resumes or an operator
performs a separate cleanup. Security audit, durable execution state and separately
configured OTLP exporters are unaffected. This is deployment-level control, not a
new per-agent trace policy API.
A lost Helm response never causes an automatic repeat. `status` can reconcile the
original successful revision; an unresolved/pending/failed result blocks another
upgrade or teardown until the operator inspects that original operation.
Local regression checks:
```sh
cargo test --manifest-path harness/Cargo.toml -p openworker-service-cli
node --test deploy/service/chart.test.mjs
OWS_PROXY_TEST=1 node --test deploy/service/proxy.test.mjs
```
If the API uses a certificate signed by a separate CA, provide its public trust
bundle using `managementUI.caSecret` / `managementUI.caKey` in deployment YAML,
or `ows management-ui enable --ca-secret API_CA_SECRET --ca-key ca.crt`.
The Secret must already exist in the service namespace. Without an override,
the CLI trusts the API TLS Secret's `tls.crt`, suitable for the self-signed
certificate produced by `up`. The UI process receives only that public certificate;
the HTTPS proxy sidecar receives the separate TLS key mount.
+18
View File
@@ -165,3 +165,21 @@ proof of compliance. A metadata reader is not a content auditor. In server apps,
authorize evidence access separately before returning it, and keep database/host
APIs in the trusted backend. The service's existing compliance content permission
continues to control audit reads. Cluster experiment/review tooling comes later.
## Manage recording on a service cluster
The source CLI supports independent UI and trace-storage controls:
```sh
ows observability enable --state ./deployment-state/installation --retention-days 14
ows observability status --state ./deployment-state/installation
ows observability disable --state ./deployment-state/installation
ows management-ui enable --state ./deployment-state/installation
```
Use the installer state created by `ows deploy aws up` or `ows deploy azure up`.
Enabling recording initializes its separate PostgreSQL schema before workloads
restart. Disabling stops collection after the rollout and retains stored traces;
it does not disable security audit or a separately configured exporter.
See the [deployment configuration and private UI access](../../../deploy/service/up/README.md#management-ui-and-trace-storage)
for the required pinned UI image, CA, database Secret and enable/disable options.
+18 -2
View File
@@ -29,6 +29,8 @@ enum Command {
drain_timeout_seconds: u64,
},
Migrate,
/// Initialize only the optional recording schema, including on feature enable.
RecordingMigrate,
/// Trusted installer aggregate check; no public/session API equivalent.
ServiceInstallStatus,
/// Trusted-host operational catalog, not a session-authorized API.
@@ -116,7 +118,8 @@ async fn main() -> Result<()> {
_ => "admin",
};
let recording_config = config.clone();
let record = !matches!(args.command, Command::Migrate) && !config.recovery_read_only;
let record = !matches!(args.command, Command::Migrate | Command::RecordingMigrate)
&& !config.recovery_read_only;
let started = tokio::task::spawn_blocking(move || {
openworker_server_reference::recording::start(&recording_config, role, record)
})
@@ -161,6 +164,19 @@ async fn run(args: Args, config: Config, telemetry: openworker_telemetry::Teleme
) {
config.require_execution()?;
}
if matches!(args.command, Command::RecordingMigrate) {
let options = config
.observability
.as_ref()
.context("Recording is disabled")?;
let database = options.database(&config)?;
tokio::task::spawn_blocking(move || {
openworker_observability::postgres::Postgres::migrate(&database)
})
.await??;
println!("Recording schema ready");
return Ok(());
}
let mut db = config.database()?;
db.telemetry = telemetry;
if matches!(args.command, Command::Migrate) {
@@ -594,7 +610,7 @@ async fn run(args: Args, config: Config, telemetry: openworker_telemetry::Teleme
| Command::EnvironmentReconcile { .. } => {
bail!("This server reference requires the qualified Kubernetes provider")
}
Command::Migrate => unreachable!(),
Command::Migrate | Command::RecordingMigrate => unreachable!(),
}
Ok(())
}
+20
View File
@@ -96,3 +96,23 @@ See [AWS deployment](../../docs/harness/guides/aws.md) and
with an explicit budget and independent cleanup before production deployment.
`owcluster` is the primary command. `ows` remains an alias, using the same commands, configuration and saved deployment state. Existing `.ows` paths and `OWS_*` settings remain valid.
## Optional management UI and trace storage
Select these features in `deployment.yaml` for the AWS/Azure `up` commands, or
change them on an existing installation:
```sh
ows management-ui enable --state ./deployment-state/installation
ows management-ui disable --state ./deployment-state/installation
ows management-ui status --state ./deployment-state/installation
ows observability enable --state ./deployment-state/installation --retention-days 14
ows observability disable --state ./deployment-state/installation
ows observability status --state ./deployment-state/installation
```
`--state` is the original installer journal, using its Kubernetes credentials and
verified cluster identity. UI and trace storage are independent. Disabling tracing
retains data and leaves security audit and separately configured exporters intact.
The CLI saves feature changes and reconciles lost Helm responses without repeating
unknown upgrades. See [setup, UI image, private HTTPS and database options](../../deploy/service/up/README.md#management-ui-and-trace-storage).
+1 -1
View File
@@ -13,7 +13,7 @@ mod guard;
#[path = "deployment_infrastructure.rs"]
mod infrastructure;
#[path = "deployment_local.rs"]
mod local;
pub(crate) mod local;
#[path = "deployment_preflight.rs"]
mod preflight;
#[path = "deployment_up.rs"]
@@ -0,0 +1,617 @@
//! Optional service features over the original installer identity and journal.
//! Unknown Helm outcomes are observed, never automatically retried.
use super::*;
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(crate) struct ManagementUI {
#[serde(default)]
pub enabled: bool,
#[serde(default = "origin")]
pub public_origin: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ca_secret: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ca_key: Option<String>,
}
fn origin() -> String {
"https://localhost:8444".into()
}
impl Default for ManagementUI {
fn default() -> Self {
Self {
enabled: false,
public_origin: origin(),
ca_secret: None,
ca_key: None,
}
}
}
impl ManagementUI {
pub(crate) fn validate(&self) -> Result<()> {
if let Some(s) = &self.ca_secret {
ensure!(dns_name(s), "Invalid UI CA Secret name");
}
if let Some(s) = &self.ca_key {
ensure!(
!s.is_empty()
&& s.len() <= 253
&& s.bytes()
.all(|b| b.is_ascii_alphanumeric() || b"._-".contains(&b)),
"Invalid UI CA Secret key"
);
}
let u = reqwest::Url::parse(&self.public_origin).context("Invalid management UI origin")?;
ensure!(u.scheme()=="https" && u.host_str().is_some() && u.username().is_empty() && u.password().is_none() && u.path()=="/" && u.query().is_none() && u.fragment().is_none(), "Management UI publicOrigin must be an HTTPS origin without credentials, path, query or fragment");
Ok(())
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(crate) struct Observability {
#[serde(default)]
pub enabled: bool,
#[serde(default = "retention")]
pub retention_days: u16,
/// Existing Kubernetes Secret containing database-url and ca.crt. No raw credentials.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub database_secret: Option<String>,
}
fn retention() -> u16 {
30
}
impl Default for Observability {
fn default() -> Self {
Self {
enabled: false,
retention_days: retention(),
database_secret: None,
}
}
}
impl Observability {
pub(crate) fn validate(&self) -> Result<()> {
ensure!(
(1..=365).contains(&self.retention_days),
"Trace retention must be 1..365 days"
);
if let Some(s) = &self.database_secret {
ensure!(dns_name(s), "Invalid recording database Secret name");
}
Ok(())
}
}
fn dns_name(s: &str) -> bool {
!s.is_empty()
&& s.len() <= 63
&& s.starts_with(|c: char| c.is_ascii_lowercase() || c.is_ascii_digit())
&& s.ends_with(|c: char| c.is_ascii_lowercase() || c.is_ascii_digit())
&& s.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
}
#[derive(clap::Args)]
pub(crate) struct Target {
/// Original installer journal (local-plan.json and journal.json), not cloud lifecycle state.
#[arg(long)]
state: PathBuf,
}
#[derive(Subcommand)]
pub(crate) enum UICommand {
Enable {
#[command(flatten)]
target: Target,
/// Digest-pinned UI image. Omit when already recorded in Helm values.
#[arg(long)]
image: Option<String>,
#[arg(long)]
public_origin: Option<String>,
/// Existing Secret containing the CA that signs the service API certificate.
#[arg(long)]
ca_secret: Option<String>,
#[arg(long, requires = "ca_secret")]
ca_key: Option<String>,
},
Disable {
#[command(flatten)]
target: Target,
},
Status {
#[command(flatten)]
target: Target,
},
}
#[derive(Subcommand)]
pub(crate) enum ObservationCommand {
Enable {
#[command(flatten)]
target: Target,
#[arg(long, value_parser=clap::value_parser!(u16).range(1..=365))]
retention_days: Option<u16>,
/// Existing Secret with database-url and ca.crt; omit to reuse execution PostgreSQL.
#[arg(long)]
database_secret: Option<String>,
},
Disable {
#[command(flatten)]
target: Target,
},
Status {
#[command(flatten)]
target: Target,
},
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub(super) struct Update {
request_id: String,
before: Value,
values: Value,
#[serde(default)]
receipt: Option<Value>,
}
pub(super) fn latest_receipt(j: &Journal) -> Option<&Value> {
j.feature_updates.last().and_then(|u| u.receipt.as_ref())
}
pub(super) fn effective_values(p: &Plan, j: &Journal) -> Result<Value> {
j.feature_updates
.last()
.map(|u| Ok(u.values.clone()))
.unwrap_or_else(|| read_json(&p.config.service.values_file))
}
fn confirmed_receipt(j: &Journal) -> Result<&Value> {
latest_receipt(j)
.or_else(|| {
j.operations
.get("service-release")
.and_then(|o| o.receipt.as_ref())
})
.context("Original service installation receipt missing")
}
fn same_receipt(a: &Value, b: &Value) -> bool {
a["uid"] == b["uid"] && a["revision"] == b["revision"] && a["operationId"] == b["operationId"]
}
async fn live_values(client: &Client<'_>, p: &Plan) -> Result<Value> {
serde_json::from_slice(
&client
.helm(args(&[
"get",
"values",
&p.config.service.release,
"-n",
&p.config.service.namespace,
"-o",
"json",
]))
.await?,
)
.context("Invalid release values")
}
pub(super) async fn verify_current(client: &Client<'_>, p: &Plan, j: &Journal) -> Result<Value> {
ensure!(
j.feature_updates.last().is_none_or(|u| u.receipt.is_some()),
"Feature upgrade outcome unknown; rerun status to reconcile, never replay the upgrade"
);
let r = client
.release(&p.config.service.namespace, &p.config.service.release)
.await?
.context("Original service release absent")?;
ensure!(
same_receipt(&r, confirmed_receipt(j)?) && r["status"] == "deployed",
"Service Helm release identity/revision changed; refusing feature mutation"
);
Ok(r)
}
pub(super) async fn reconcile(
client: &Client<'_>,
p: &Plan,
root: &Path,
j: &mut Journal,
) -> Result<()> {
let Some(update) = j
.feature_updates
.last()
.filter(|u| u.receipt.is_none())
.cloned()
else {
return Ok(());
};
ensure!(
client.cluster().await? == p.cluster_uid,
"Cluster identity changed"
);
owned(client, p, j, &p.config.service.namespace)
.await?
.context("Owned namespace missing")?;
let r = client
.release(&p.config.service.namespace, &p.config.service.release)
.await?
.context("Upgrade outcome unknown: original service release absent")?;
ensure!(r["installerId"]==p.id && r["operationId"]==update.request_id && r["revision"].as_u64()==update.before["revision"].as_u64().and_then(|n|n.checked_add(1)) && r["status"]=="deployed", "Feature upgrade outcome unknown/pending/failed; inspect the original Helm operation. It will not be retried automatically");
ensure!(
live_values(client, p).await? == update.values,
"Upgraded release values differ from the saved intent"
);
ready_service(client, p, j).await?;
j.feature_updates.last_mut().unwrap().receipt = Some(r);
save(root, "journal.json", j)
}
/// Pure rendering shared by initial cloud deployment and later CLI controls.
pub(crate) fn configure_ui(
values: &mut Value,
ui: &ManagementUI,
image: Option<&str>,
release: &str,
namespace: &str,
) -> Result<()> {
ui.validate()?;
if values["managementUI"].is_null() {
values["managementUI"] = json!({});
}
values["managementUI"]["enabled"] = json!(ui.enabled);
if let Some(image) = image {
values["managementUI"]["image"] = json!(image);
}
values["managementUI"]["publicOrigin"] = json!(ui.public_origin);
values["managementUI"]["serviceUrl"] = json!(format!("https://{release}-api.{namespace}.svc"));
if ui.enabled {
let image = values["managementUI"]["image"]
.as_str()
.context("Provide --image with a digest-pinned management UI image")?;
ensure!(
image
.rsplit_once("@sha256:")
.is_some_and(|(name, d)| !name.is_empty()
&& d.len() == 64
&& d.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))),
"Management UI image must use a sha256 digest"
);
let api_tls = values["secrets"]["apiTLS"]
.as_str()
.unwrap_or("ow-service-tls")
.to_string();
let ca_secret = ui
.ca_secret
.clone()
.or_else(|| {
values["managementUI"]["caSecret"]
.as_str()
.filter(|s| !s.is_empty())
.map(str::to_owned)
})
.unwrap_or_else(|| api_tls.clone());
let ca_key = ui
.ca_key
.clone()
.or_else(|| {
if ui.ca_secret.is_none() {
values["managementUI"]["caKey"].as_str().map(str::to_owned)
} else {
None
}
})
.unwrap_or_else(|| {
if ui.ca_secret.is_some() {
"ca.crt".into()
} else {
"tls.crt".into()
}
});
values["managementUI"]["caSecret"] = json!(ca_secret);
values["managementUI"]["caKey"] = json!(ca_key);
// Private loopback preview is self-contained. Public origins require the
// operator's TLS ingress; never create public exposure implicitly.
let u = reqwest::Url::parse(&ui.public_origin)?;
values["managementUI"]["tlsSecret"] =
if matches!(u.host_str(), Some("localhost" | "127.0.0.1")) {
json!(api_tls)
} else {
json!("")
};
}
Ok(())
}
pub(crate) fn configure_observability(values: &mut Value, obs: &Observability) -> Result<()> {
obs.validate()?;
if values["secrets"].is_null() {
values["secrets"] = json!({});
}
if obs.enabled {
if values["config"]["observability"].is_null() {
values["config"]["observability"] = json!({"schema":"ow_observability"});
}
values["config"]["observability"]["retention_days"] = json!(obs.retention_days);
if let Some(s) = &obs.database_secret {
values["secrets"]["observability"] = json!(s);
}
} else {
values["config"]
.as_object_mut()
.context("Service configuration missing")?
.remove("observability");
values["secrets"]["observability"] = json!("");
}
Ok(())
}
async fn change(
root: &Path,
mutation: impl FnOnce(&mut Value, &Plan) -> Result<()>,
status_only: bool,
) -> Result<()> {
let (p, mut j, _lock) = load(root)?;
ensure!(
j.phase == "installed",
"Feature controls require a completed service installation"
);
ensure!(
inputs(&p.config)? == p.hashes,
"Original installer inputs changed"
);
let client = Client { c: &p.config };
ensure!(
client.cluster().await? == p.cluster_uid,
"Cluster identity changed"
);
owned(&client, &p, &j, &p.config.service.namespace)
.await?
.context("Owned namespace missing")?;
reconcile(&client, &p, root, &mut j).await?;
let before = verify_current(&client, &p, &j).await?;
let mut values = effective_values(&p, &j)?;
// First update reads the live explicit values to preserve defaults/previous
// operator settings, but refuses any unreviewed drift.
ensure!(
live_values(&client, &p).await? == values,
"Service values changed outside the installer"
);
if !status_only {
mutation(&mut values, &p)?;
if values != effective_values(&p, &j)? {
// Never rerun first-admin bootstrap on a feature upgrade.
values["bootstrapEnabled"] = json!(false);
let id = uuid::Uuid::new_v4().to_string();
let file = format!("feature-values-{id}.json");
save(root, &file, &values)?;
// Render validation happens before recording/sending any mutation.
client
.helm(args(&[
"template",
&p.config.service.release,
p.config
.service
.chart
.to_str()
.context("Invalid chart path")?,
"-n",
&p.config.service.namespace,
"-f",
root.join(&file)
.to_str()
.context("Invalid feature values path")?,
]))
.await?;
j.feature_updates.push(Update {
request_id: id.clone(),
before,
values: values.clone(),
receipt: None,
});
save(root, "journal.json", &j)?;
let result = client
.helm(args(&[
"upgrade",
&p.config.service.release,
p.config.service.chart.to_str().unwrap(),
"-n",
&p.config.service.namespace,
"-f",
root.join(&file).to_str().unwrap(),
"--labels",
&format!("{OWNER}={},openworker.io/operation-id={id}", p.id),
"--description",
&format!("ows:{}:{id}", p.id),
"--wait",
"--timeout",
"300s",
]))
.await;
if result.is_err() {
// A response can be lost after success. Only observation of this
// exact revision and intent may recover it; never replay Helm.
reconcile(&client, &p, root, &mut j).await.context(
"Helm response failed; saved feature intent retained for status reconciliation",
)?;
} else {
reconcile(&client, &p, root, &mut j).await?;
}
}
}
let values = effective_values(&p, &j)?;
let ui = &values["managementUI"];
let obs = &values["config"]["observability"];
let preview = if ui["enabled"] == true
&& ui["tlsSecret"].as_str().is_some_and(|s| !s.is_empty())
{
let origin = ui["publicOrigin"]
.as_str()
.unwrap_or("https://localhost:8444");
let u = reqwest::Url::parse(origin)?;
json!({"url":origin,"command":"kubectl","arguments":["--kubeconfig",p.config.kubeconfig,"--context",p.config.context,"-n",p.config.service.namespace,"port-forward",format!("service/{}-ui",p.config.service.release),format!("{}:443",u.port_or_known_default().unwrap()),"--address=127.0.0.1"],"caFile":"Use the API certificate from the deployment credentials; do not disable certificate verification","login":"existing service token; UI does not hold a bootstrap credential"})
} else {
Value::Null
};
crate::output(
json!({"deploymentId":p.id,"managementUI":{"enabled":ui["enabled"]==true,"publicOrigin":ui["publicOrigin"],"privatePreview":preview},"observability":{"enabled":!obs.is_null(),"configuration":obs,"databaseSecret":values["secrets"]["observability"],"storedData":"retained; no deletion performed","exporters":"configured independently"},"savedConfiguration":root.join("journal.json"),"helmRevision":confirmed_receipt(&j)?["revision"]}),
)
}
pub(crate) async fn ui(command: &UICommand) -> Result<()> {
match command {
UICommand::Status { target } => change(&target.state, |_, _| Ok(()), true).await,
UICommand::Disable { target } => {
change(
&target.state,
|v, _| {
if v["managementUI"].is_null() {
v["managementUI"] = json!({});
}
v["managementUI"]["enabled"] = json!(false);
Ok(())
},
false,
)
.await
}
UICommand::Enable {
target,
image,
public_origin,
ca_secret,
ca_key,
} => {
change(
&target.state,
|v, p| {
let ui = ManagementUI {
enabled: true,
public_origin: public_origin
.clone()
.or_else(|| {
v["managementUI"]["publicOrigin"]
.as_str()
.map(str::to_owned)
})
.unwrap_or_else(origin),
ca_secret: ca_secret.clone(),
ca_key: ca_key.clone(),
};
configure_ui(
v,
&ui,
image.as_deref(),
&p.config.service.release,
&p.config.service.namespace,
)
},
false,
)
.await
}
}
}
pub(crate) async fn observability(command: &ObservationCommand) -> Result<()> {
match command {
ObservationCommand::Status { target } => change(&target.state, |_, _| Ok(()), true).await,
ObservationCommand::Disable { target } => {
change(
&target.state,
|v, _| configure_observability(v, &Observability::default()),
false,
)
.await
}
ObservationCommand::Enable {
target,
retention_days,
database_secret,
} => {
change(
&target.state,
|v, _| {
configure_observability(
v,
&Observability {
enabled: true,
retention_days: retention_days.unwrap_or_else(|| {
v["config"]["observability"]["retention_days"]
.as_u64()
.unwrap_or(30) as u16
}),
database_secret: database_secret.clone(),
},
)
},
false,
)
.await
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rendering_is_opt_in_independent_and_preserves_other_settings() {
let original = json!({"image":"service","config":{"schema":"execution","telemetry":{"enabled":true}},"secrets":{"apiTLS":"api-cert"}});
let mut values = original.clone();
let ui = ManagementUI {
enabled: true,
..Default::default()
};
configure_ui(
&mut values,
&ui,
Some(&format!("ui@sha256:{}", "b".repeat(64))),
"ows",
"example",
)
.unwrap();
assert!(values["config"].get("observability").is_none());
assert_eq!(values["managementUI"]["tlsSecret"], "api-cert");
assert_eq!(
values["managementUI"]["serviceUrl"],
"https://ows-api.example.svc"
);
configure_observability(
&mut values,
&Observability {
enabled: true,
retention_days: 14,
database_secret: Some("recording-db".into()),
},
)
.unwrap();
assert_eq!(values["secrets"]["observability"], "recording-db");
configure_observability(&mut values, &Observability::default()).unwrap();
assert!(values["config"].get("observability").is_none());
assert_eq!(
values["config"]["telemetry"],
original["config"]["telemetry"]
);
assert_eq!(values["image"], original["image"]);
assert_eq!(values["managementUI"]["enabled"], true);
}
#[test]
fn invalid_origins_retention_and_inline_database_values_fail_before_mutation() {
for origin in [
"http://localhost:8444",
"https://user:secret@example.com",
"https://example.com/path",
"https://example.com/?token=secret",
] {
assert!(ManagementUI {
enabled: true,
public_origin: origin.into(),
..Default::default()
}
.validate()
.is_err());
}
for retention_days in [0, 366] {
assert!(Observability {
enabled: true,
retention_days,
database_secret: None
}
.validate()
.is_err());
}
assert!(Observability {
enabled: true,
retention_days: 14,
database_secret: Some("postgres://secret".into())
}
.validate()
.is_err());
}
}
+24 -10
View File
@@ -6,6 +6,8 @@ use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
process::Command as Process,
};
#[path = "deployment_features.rs"]
pub(crate) mod features;
const OWNER: &str = "openworker.io/installer-id";
#[path = "deployment_cluster_platform.rs"]
mod platform;
@@ -151,6 +153,8 @@ struct Journal {
operations: BTreeMap<String, Operation>,
#[serde(default)]
confirmed_no_effect: Vec<Operation>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
feature_updates: Vec<features::Update>,
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
@@ -651,6 +655,10 @@ async fn install_release(
owned(client, p, j, ns)
.await?
.context("Owned namespace missing")?;
if key == "service-release" && !j.feature_updates.is_empty() {
features::verify_current(client, p, j).await?;
return Ok(());
}
if let Some(mut r) = client.release(ns, release).await? {
let op = j
.operations
@@ -692,7 +700,7 @@ async fn install_release(
.is_none_or(|v| v.is_null() || v.as_array().is_some_and(Vec::is_empty)),
"Hooks may have unknown effects; failed release requires manual review"
);
ready_service(client, p).await?;
ready_service(client, p, j).await?;
let job = client
.get("job", &format!("{release}-bootstrap"), Some(ns))
.await?
@@ -822,9 +830,12 @@ async fn prerequisites(client: &Client<'_>, eks: bool) -> Result<()> {
);
Ok(())
}
async fn ready_service(client: &Client<'_>, p: &Plan) -> Result<()> {
let values: Value = read_json(&p.config.service.values_file)?;
let mut roles = vec!["api", "worker", "controller", "platform"];
async fn ready_service(client: &Client<'_>, p: &Plan, j: &Journal) -> Result<()> {
let values = features::effective_values(p, j)?;
let mut roles = vec!["api", "worker", "controller"];
if values["platformControllerEnabled"] != false {
roles.push("platform");
}
if values["managementUI"]["enabled"] == true {
roles.push("ui");
}
@@ -1236,11 +1247,13 @@ async fn uninstall(
.context("Owned namespace missing before uninstall")?;
let delete = format!("uninstall:{key}");
if let Some(r) = client.release(ns, release).await? {
let receipt = j
.operations
.get(key)
.and_then(|o| o.receipt.as_ref())
.context("Release identity was never confirmed")?;
let receipt = if key == "service-release" {
features::latest_receipt(j)
} else {
None
}
.or_else(|| j.operations.get(key).and_then(|o| o.receipt.as_ref()))
.context("Release identity was never confirmed")?;
ensure!(
r["uid"] == receipt["uid"] && r["revision"] == receipt["revision"],
"Release identity changed before uninstall"
@@ -1447,6 +1460,7 @@ async fn run_scoped(command: &Command, local_only: bool) -> Result<()> {
.await?;
}
let client = Client { c: &p.config };
features::reconcile(&client, &p, state, &mut j).await?;
ensure!(
client.cluster().await? == p.cluster_uid,
"Cluster identity changed"
@@ -1709,7 +1723,7 @@ async fn run_scoped(command: &Command, local_only: bool) -> Result<()> {
},
)
.await?;
ready_service(&client, &p).await?;
ready_service(&client, &p, &j).await?;
j.phase = "installed".into();
save(state, "journal.json", &j)?;
}
+5
View File
@@ -156,6 +156,11 @@ pub(super) fn operation_lock(root: &Path, name: &str) -> Result<fs::File> {
Ok(f)
}
async fn check_local_tools(c: &Config, package: &package::Snapshot) -> Result<()> {
ensure!(
!c.management_ui.as_ref().is_some_and(|ui| ui.enabled)
|| package.release.images.service_ui.is_some(),
"Enabling managementUI requires a digest-pinned service-ui image in the release manifest"
);
let mut tools: Vec<(&str, Vec<&str>)> = vec![
("kubectl", vec!["version", "--client", "--output=json"]),
("helm", vec!["version", "--short"]),
@@ -17,6 +17,11 @@ pub(super) struct Config {
pub models: BTreeMap<String, Secret>,
#[serde(default)]
pub advanced: Advanced,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[serde(rename = "managementUI")]
pub management_ui: Option<super::super::local::features::ManagementUI>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub observability: Option<super::super::local::features::Observability>,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
@@ -170,6 +175,12 @@ impl Config {
Ok(config)
}
pub fn validate(&self, provider: &str) -> Result<()> {
if let Some(ui) = &self.management_ui {
ui.validate()?;
}
if let Some(obs) = &self.observability {
obs.validate()?;
}
if let Some(t) = &self.trial {
ensure!((1..=72).contains(&t.expires_after_hours)&&t.delete_data,"A disposable trial requires 1..72 hours and explicit deleteData:true; omit trial for persistent deployment");
}
@@ -310,7 +321,7 @@ impl Config {
#[cfg(test)]
mod tests {
use super::*;
pub(super) fn yaml() -> &'static str {
pub(crate) fn yaml() -> &'static str {
"name: example\ncloud:\n provider: azure\n subscriptionId: 11111111-2222-4333-8444-555555555555\n region: westus2\nnodes:\n count: 2\n requirements:\n cpu: {min: 4, max: 8}\n memoryGiB: {min: 16, max: 32}\nbudget:\n limitUSD: 100\n"
}
#[test]
@@ -357,3 +368,26 @@ mod tests {
c.validate("azure").unwrap();
}
}
#[cfg(test)]
mod feature_config_tests {
use super::*;
#[test]
fn cloud_settings_accept_independent_features_and_reject_unknown_inline_options() {
let base = tests::yaml();
for options in [
"managementUI: {enabled: true}\n",
"observability: {enabled: true, retentionDays: 14, databaseSecret: recording-db}\n",
] {
let c: Config = serde_yaml::from_str(&format!("{base}{options}")).unwrap();
c.validate("azure").unwrap();
}
let c: Config = serde_yaml::from_str(base).unwrap();
let value = serde_json::to_value(c).unwrap();
assert!(value.get("managementUI").is_none() && value.get("observability").is_none());
assert!(serde_yaml::from_str::<Config>(&format!(
"{base}observability: {{enabled: true, connection: secret}}\n"
))
.is_err());
}
}
@@ -89,7 +89,21 @@ impl Native<'_> {
let id=if i==0{"primary"}else{p.as_str()};
json!({"id":id,"provider":p,"identity":format!("{}-{id}",plan.deployment_id),"key_env":if p=="anthropic"{"ANTHROPIC_API_KEY"}else{"OPENAI_API_KEY"}})
}).collect();
let values = json!({"image":harness,"apiReplicas":1,"worker":{"replicas":1,"ownership":"manual","maxReplicas":2,"slots":2,"drainSeconds":30},"terminationGracePeriodSeconds":90,"secrets":{"model":if c.models.is_empty(){""}else{"ow-service-model"}},"config":{"service":{"id":plan.deployment_id,"namespace":"local","agent":"installed-tools","legacy_compatibility":false},"database_env":"OW_PG_URL","schema":"ow_service","database_loopback_plaintext":false,"database_ca":"/run/service/pg-ca.crt","state_root":"/state","bundle":"/opt/openworker/bundle","runner":"/opt/openworker/openworker","model":model,"scripted":false,"model_provider":provider,"model_connections":connections,"api_token_file":"/bootstrap/admin-token","actor":"deployment-admin","openshell":{"identity":plan.deployment_id,"endpoint":endpoint,"tls_server_name":format!("{fullname}.{ns}.svc.cluster.local"),"certificates":"/openshell","workspace":"default","image":runner},"artifacts":artifacts}});
let mut values = json!({"image":harness,"apiReplicas":1,"worker":{"replicas":1,"ownership":"manual","maxReplicas":2,"slots":2,"drainSeconds":30},"terminationGracePeriodSeconds":90,"secrets":{"model":if c.models.is_empty(){""}else{"ow-service-model"}},"config":{"service":{"id":plan.deployment_id,"namespace":"local","agent":"installed-tools","legacy_compatibility":false},"database_env":"OW_PG_URL","schema":"ow_service","database_loopback_plaintext":false,"database_ca":"/run/service/pg-ca.crt","state_root":"/state","bundle":"/opt/openworker/bundle","runner":"/opt/openworker/openworker","model":model,"scripted":false,"model_provider":provider,"model_connections":connections,"api_token_file":"/bootstrap/admin-token","actor":"deployment-admin","openshell":{"identity":plan.deployment_id,"endpoint":endpoint,"tls_server_name":format!("{fullname}.{ns}.svc.cluster.local"),"certificates":"/openshell","workspace":"default","image":runner},"artifacts":artifacts}});
if let Some(ui) = &c.management_ui {
super::super::super::local::features::configure_ui(
&mut values,
ui,
images["service-ui"]["image"].as_str(),
"ows",
"ow-service",
)?;
} else if let Some(image) = images["service-ui"]["image"].as_str() {
values["managementUI"] = json!({"enabled":false,"image":image});
}
if let Some(obs) = &c.observability {
super::super::super::local::features::configure_observability(&mut values, obs)?;
}
self.exact_file("service-values.json", &values)?;
let mut shell: Value =
serde_json::from_slice(&fs::read(&self.package.release.openshell_values.file)?)?;
@@ -193,7 +207,7 @@ impl Native<'_> {
!cert.exists() && !key.exists(),
"Partial TLS generation; inspect private credential files before continuing"
);
self.tool("openssl",vec!["req".into(),"-x509".into(),"-newkey".into(),"rsa:3072".into(),"-nodes".into(),"-days".into(),"30".into(),"-subj".into(),"/CN=ows-api.ow-service.svc.cluster.local".into(),"-addext".into(),"subjectAltName=DNS:localhost,IP:127.0.0.1,DNS:ows-api.ow-service.svc.cluster.local,DNS:ows-api.ow-service.svc".into(),"-keyout".into(),arg(&key)?,"-out".into(),arg(&cert)?],None).await?;
self.tool("openssl",vec!["req".into(),"-x509".into(),"-newkey".into(),"rsa:3072".into(),"-nodes".into(),"-days".into(),"30".into(),"-subj".into(),"/CN=ows-api.ow-service.svc.cluster.local".into(),"-addext".into(),"basicConstraints=critical,CA:TRUE".into(),"-addext".into(),"keyUsage=critical,digitalSignature,keyCertSign,cRLSign".into(),"-addext".into(),"extendedKeyUsage=serverAuth".into(),"-addext".into(),"subjectAltName=DNS:localhost,IP:127.0.0.1,DNS:ows-api.ow-service.svc.cluster.local,DNS:ows-api.ow-service.svc".into(),"-keyout".into(),arg(&key)?,"-out".into(),arg(&cert)?],None).await?;
}
#[cfg(unix)]
{
@@ -241,7 +255,17 @@ impl Native<'_> {
pub(super) async fn verify(&self) -> Result<Observation> {
let result = self.installer("status", &[]).await?;
let kube = self.root.join("handoff/kubeconfig.json");
for component in ["api", "worker", "controller", "platform"] {
let mut components = vec!["api", "worker", "controller", "platform"];
if self
.review
.config
.management_ui
.as_ref()
.is_some_and(|ui| ui.enabled)
{
components.push("ui");
}
for component in components {
self.tool(
"kubectl",
vec![
@@ -258,7 +282,22 @@ impl Native<'_> {
)
.await?;
}
let connection = json!({"deploymentId":self.review.deployment_id,"serviceId":self.review.deployment_id,"namespace":"local","url":"https://ows-api.ow-service.svc.cluster.local","access":"private cluster network","kubeconfig":kube,"caFile":self.root.join("credentials/api.crt"),"tokenFile":self.install_state().join("bootstrap-token"),"privatePreview":{"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-api","8443:443","--address=127.0.0.1"],"url":"https://localhost:8443"},"modelSmokeTest":"not_run"});
let mut connection = json!({"deploymentId":self.review.deployment_id,"serviceId":self.review.deployment_id,"namespace":"local","url":"https://ows-api.ow-service.svc.cluster.local","access":"private cluster network","kubeconfig":kube,"caFile":self.root.join("credentials/api.crt"),"tokenFile":self.install_state().join("bootstrap-token"),"privatePreview":{"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-api","8443:443","--address=127.0.0.1"],"url":"https://localhost:8443"},"modelSmokeTest":"not_run"});
if let Some(ui) = self
.review
.config
.management_ui
.as_ref()
.filter(|ui| ui.enabled)
{
connection["managementUI"] =
json!({"url":ui.public_origin,"login":"existing service token"});
let url = reqwest::Url::parse(&ui.public_origin)?;
if matches!(url.host_str(), Some("localhost" | "127.0.0.1")) {
connection["managementUI"]["privatePreview"] = json!({"command":"kubectl","arguments":["--kubeconfig",arg(&kube)?,"-n","ow-service","port-forward","service/ows-ui",format!("{}:443",url.port_or_known_default().unwrap()),"--address=127.0.0.1"],"caFile":self.root.join("credentials/api.crt")});
}
}
connection["observability"] = json!({"enabled":self.review.config.observability.as_ref().is_some_and(|obs|obs.enabled)});
write_private(&self.root.join("connection.json"), &connection)?;
Ok(Observation::Complete(
json!({"nativeStatus":result,"rolloutsReady":true,"modelSmokeTest":"not_run","connectionFile":self.root.join("connection.json")}),
+18 -1
View File
@@ -39,6 +39,16 @@ enum Command {
#[command(subcommand)]
command: deployment::Command,
},
/// Configure the optional browser UI on an installed cluster.
ManagementUi {
#[command(subcommand)]
command: deployment::local::features::UICommand,
},
/// Configure optional trace storage; disabling never deletes stored traces.
Observability {
#[command(subcommand)]
command: deployment::local::features::ObservationCommand,
},
Context {
#[command(subcommand)]
command: ContextCommand,
@@ -370,6 +380,13 @@ async fn run(args: Args) -> Result<()> {
if let Command::Deploy { command } = &args.command {
return deployment::dispatch(command).await;
}
match &args.command {
Command::ManagementUi { command } => return deployment::local::features::ui(command).await,
Command::Observability { command } => {
return deployment::local::features::observability(command).await
}
_ => {}
}
let mut settings = settings::Settings::load(args.config.as_ref())?;
if let Command::Context { command } = &args.command {
return match command {
@@ -650,7 +667,7 @@ async fn run(args: Args) -> Result<()> {
.control_receipt(&receipt)
.await?,
)?,
Command::Context { .. } | Command::Login | Command::Deploy { .. } => unreachable!(),
Command::Context { .. } | Command::Login | Command::Deploy { .. } | Command::ManagementUi { .. } | Command::Observability { .. } => unreachable!(),
}
service.disconnect();
Ok(())
+8 -1
View File
@@ -13,12 +13,19 @@ if tool=='helm':
ns=arg('-n');releases=state['releases'].setdefault(ns,{})
if a[0]=='list':finish([{'name':n} for n in releases])
if a[0]=='status':finish(releases[a[1]])
if a[:2]==['get','values']:finish(releases[a[2]].get('values',{}))
if a[0]=='template':finish('rendered')
if a[0]=='install':
if os.environ.get('FAIL_BEFORE')=='yes':sys.exit(77)
name=a[1];labels=dict(pair.split('=',1) for pair in arg('--labels').split(','));releases[name]={'version':1,'info':{'status':'failed' if os.environ.get('FAIL_RELEASE') else 'deployed','description':'context deadline exceeded' if os.environ.get('FAIL_RELEASE') else arg('--description')},'uid':'release-uid','labels':labels};store.write_text(json.dumps(state))
name=a[1];labels=dict(pair.split('=',1) for pair in arg('--labels').split(','));releases[name]={'version':1,'info':{'status':'failed' if os.environ.get('FAIL_RELEASE') else 'deployed','description':'context deadline exceeded' if os.environ.get('FAIL_RELEASE') else arg('--description')},'uid':'release-uid','labels':labels,'values':json.loads(pathlib.Path(arg('-f')).read_text())};store.write_text(json.dumps(state))
if os.environ.get('FAIL_RELEASE'):sys.exit(81)
if os.environ.get('FAIL_AFTER')=='yes':sys.exit(78)
finish('installed')
if a[0]=='upgrade':
if os.environ.get('FAIL_UPGRADE_BEFORE')=='yes':sys.exit(77)
name=a[1];r=releases[name];r['version']+=1;r['uid']='release-uid-'+str(r['version']);r['labels']=dict(pair.split('=',1) for pair in arg('--labels').split(','));r['info']={'status':'deployed','description':arg('--description')};r['values']=json.loads(pathlib.Path(arg('-f')).read_text());store.write_text(json.dumps(state))
if os.environ.get('FAIL_UPGRADE_AFTER')=='yes':sys.exit(78)
finish('upgraded')
if a[0]=='uninstall':del releases[a[1]];finish('uninstalled')
sys.exit(99)
# kubectl global options precede its command.
@@ -451,3 +451,140 @@ fn failed_original_release_receipt_is_cleanup_only_and_rejects_drift() {
assert_eq!(f.installs(), 1);
assert_eq!(parsed(&f.destroy())["phase"], "destroyed");
}
fn feature(
f: &Fixture,
name: &str,
action: &str,
options: &[&str],
env: &[(&str, &str)],
) -> Output {
let mut args = vec![name, action, "--state", f.state.to_str().unwrap()];
args.extend_from_slice(options);
Fixture::call(f.dir.path(), &args, env)
}
fn upgrades(f: &Fixture) -> usize {
fs::read_to_string(f.dir.path().join("calls"))
.unwrap()
.lines()
.filter(|l| {
let v: Value = serde_json::from_str(l).unwrap();
v[0] == "helm" && v[1][0] == "upgrade"
})
.count()
}
#[test]
fn feature_controls_preserve_values_disable_without_data_deletion_and_cleanup_latest_revision() {
let f = Fixture::new();
parsed(&f.apply(&[], false));
let image = format!("ui@sha256:{}", "b".repeat(64));
let enabled = parsed(&feature(
&f,
"management-ui",
"enable",
&["--image", &image],
&[],
));
assert_eq!(enabled["managementUI"]["enabled"], true);
assert_eq!(
enabled["managementUI"]["privatePreview"]["url"],
"https://localhost:8444"
);
assert_eq!(enabled["observability"]["enabled"], false);
let obs = parsed(&feature(
&f,
"observability",
"enable",
&["--retention-days", "14"],
&[],
));
assert_eq!(obs["observability"]["configuration"]["retention_days"], 14);
assert_eq!(obs["managementUI"]["enabled"], true);
let count = upgrades(&f);
parsed(&feature(
&f,
"observability",
"enable",
&["--retention-days", "14"],
&[],
));
parsed(&feature(&f, "management-ui", "status", &[], &[]));
assert_eq!(upgrades(&f), count);
parsed(&feature(&f, "management-ui", "disable", &[], &[]));
let disabled = parsed(&feature(&f, "observability", "disable", &[], &[]));
assert_eq!(disabled["observability"]["enabled"], false);
assert_eq!(disabled["managementUI"]["enabled"], false);
let v = &f.remote()["releases"]["ow-service-m4-installer"]["ows"]["values"];
assert_eq!(v["bootstrapEnabled"], false);
assert_eq!(v["image"], format!("image@sha256:{}", "a".repeat(64)));
assert!(v["config"].get("observability").is_none());
assert_eq!(f.installs(), 1);
// A normal installer resume and teardown still recognize the updated release.
parsed(&f.apply(&[], false));
parsed(&f.destroy());
}
#[test]
fn feature_upgrade_lost_response_is_observed_without_a_second_upgrade() {
let f = Fixture::new();
parsed(&f.apply(&[], false));
let out = feature(
&f,
"observability",
"enable",
&[],
&[("FAIL_UPGRADE_AFTER", "yes")],
);
assert_eq!(parsed(&out)["observability"]["enabled"], true);
parsed(&feature(&f, "observability", "status", &[], &[]));
assert_eq!(upgrades(&f), 1);
}
#[test]
fn unknown_feature_outcome_is_not_replayed_and_blocks_cleanup() {
let f = Fixture::new();
parsed(&f.apply(&[], false));
assert!(!feature(
&f,
"observability",
"enable",
&[],
&[("FAIL_UPGRADE_BEFORE", "yes")]
)
.status
.success());
for action in ["enable", "status", "disable"] {
assert!(!feature(&f, "observability", action, &[], &[])
.status
.success());
}
assert!(!f.destroy().status.success());
assert_eq!(upgrades(&f), 1);
}
#[test]
fn feature_controls_reject_replaced_release_or_namespace_and_mutable_images() {
let f = Fixture::new();
parsed(&f.apply(&[], false));
assert!(!feature(
&f,
"management-ui",
"enable",
&["--image", "ui:latest"],
&[]
)
.status
.success());
assert_eq!(upgrades(&f), 0);
let original = f.remote();
let mut changed = original.clone();
changed["releases"]["ow-service-m4-installer"]["ows"]["version"] = json!(8);
f.set_remote(&changed);
assert!(!feature(&f, "observability", "enable", &[], &[])
.status
.success());
let mut changed = original;
changed["namespaces"]["ow-service-m4-installer"]["metadata"]["uid"] = json!("replacement");
f.set_remote(&changed);
assert!(!feature(&f, "observability", "enable", &[], &[])
.status
.success());
assert_eq!(upgrades(&f), 0);
}