From 95d175a2c7a2a0f09514b4cfe05cc2d55a0dede5 Mon Sep 17 00:00:00 2001 From: Rohit P Date: Fri, 13 Nov 2026 12:20:33 -0800 Subject: [PATCH] fix: preserve private installation access and recording schema setup --- deploy/service/chart.test.mjs | 11 +++ deploy/service/chart/templates/workloads.yaml | 7 +- deploy/service/examples/README.md | 14 ++-- deploy/service/up/README.md | 32 ++++++--- docs/harness/guides/observability.md | 10 +-- harness/service-cli/README.md | 14 ++-- harness/service-cli/src/deployment_up.rs | 34 +++++++++ .../src/deployment_up_azure_images.rs | 71 ++++++++++++++++++- .../service-cli/src/deployment_up_native.rs | 5 ++ 9 files changed, 165 insertions(+), 33 deletions(-) diff --git a/deploy/service/chart.test.mjs b/deploy/service/chart.test.mjs index 923f65e7..f672703c 100644 --- a/deploy/service/chart.test.mjs +++ b/deploy/service/chart.test.mjs @@ -173,6 +173,17 @@ test("invalid ownership, mutable image and inadequate drain grace fail rendering }), ); }); +test("Azure trace schema initializer is excluded from workload credential injection", async()=>{ + const artifacts={kind:"azure",deployment:"ows-test",account:"owteststorage",container:"artifacts",tenant_id:"11111111-1111-4111-8111-111111111111",client_id:"22222222-2222-4222-8222-222222222222",federated_token_file:"/var/run/secrets/azure/tokens/azure-identity-token"}; + const out=await render({config:{...base.config,artifacts,observability:{retention_days:14}}}); + for(const role of ["api","worker"]){ + const d=deployment(out,role); + assert.match(d,/azure.workload.identity\/skip-containers: "harness(?:;tls)?;recording-schema"/); + const init=d.split("initContainers:")[1].split(" containers:")[0]; + assert.doesNotMatch(init,/azure-artifact-token|azure-identity-token|AZURE_/); + assert.equal((d.match(/name: azure-artifact-token\n/g)||[]).length,1); + } +}); test("management UI has no execution, database, or cluster authority", async () => { const uiValues = { enabled: true, diff --git a/deploy/service/chart/templates/workloads.yaml b/deploy/service/chart/templates/workloads.yaml index d255fb1f..2e3615bd 100644 --- a/deploy/service/chart/templates/workloads.yaml +++ b/deploy/service/chart/templates/workloads.yaml @@ -36,8 +36,11 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/config.yaml") $ | sha256sum }} {{- if $azure }} # This chart owns its explicit projection; prevent webhook injection of - # another token path or credentials into the TLS sidecar. - azure.workload.identity/skip-containers: {{ if eq $role "api" }}"harness;tls"{{ else }}"harness"{{ end }} + # another token path or credentials into TLS/schema-only containers. + {{- $skip := list "harness" -}} + {{- if eq $role "api" }}{{ $skip = append $skip "tls" }}{{ end -}} + {{- if hasKey $.Values.config "observability" }}{{ $skip = append $skip "recording-schema" }}{{ end }} + azure.workload.identity/skip-containers: {{ join ";" $skip | quote }} {{- end }} spec: automountServiceAccountToken: {{ eq $role "platform" }} diff --git a/deploy/service/examples/README.md b/deploy/service/examples/README.md index 6f6cb2a6..2a5438ec 100644 --- a/deploy/service/examples/README.md +++ b/deploy/service/examples/README.md @@ -22,9 +22,9 @@ data only, optionally add `trial: {expiresAfterHours: 6, deleteData: true}`. ## Azure and AWS ```sh -ows deploy azure up --config azure/deployment.yaml --plan-only \ +owcluster deploy azure up --config azure/deployment.yaml --plan-only \ --max-estimated-cost-usd 100 -ows deploy aws up --config aws/deployment.yaml --plan-only +owcluster deploy aws up --config aws/deployment.yaml --plan-only ``` A cost ceiling is optional, is plan-only, and checks an estimate rather than actual @@ -72,11 +72,11 @@ approval. Kubernetes must be able to reach PostgreSQL and pull/use those images. `scripted` is a local fixture option, not a substitute for live-provider qualification. ```sh -ows deploy local-k8s plan --config local-k8s/deployment.yaml --state .ows/local -ows deploy local-k8s up --config local-k8s/deployment.yaml --state .ows/local \ +owcluster deploy local-k8s plan --config local-k8s/deployment.yaml --state .ows/local +owcluster deploy local-k8s up --config local-k8s/deployment.yaml --state .ows/local \ --accept-plan PLAN_DIGEST -ows deploy local-k8s status --state .ows/local -ows management-ui status --state .ows/local/installation +owcluster deploy local-k8s status --state .ows/local +owcluster management-ui status --state .ows/local/installation ``` Use the actual digest printed by plan. The shared installer checks cluster and @@ -90,7 +90,7 @@ No new sandbox/model run is implied by successful installation readiness. The CLI generates the private API certificate and maps `managementUI.port` to the loopback preview. Keep and trust that CA; do not disable TLS verification. Set UI/recording preferences in YAML or use the independent feature commands. -Native lower-level `ows deploy local` / `deploy cluster` remain compatibility +Native lower-level `owcluster deploy local` / `deploy cluster` remain compatibility interfaces for existing journals; new examples use `local-k8s`. The new `local-docker` and `remote-k8s` command names are discoverable but report diff --git a/deploy/service/up/README.md b/deploy/service/up/README.md index 3686bcaf..f8ddffe5 100644 --- a/deploy/service/up/README.md +++ b/deploy/service/up/README.md @@ -10,8 +10,8 @@ schema. The service installer remains experimental until the current release passes a fresh cloud install, agent/recovery and cleanup qualification. ```sh -ows deploy azure up --config staging-deploy.yaml --plan-only -ows deploy aws up --config prod-deploy.yaml --plan-only +owcluster deploy azure up --config staging-deploy.yaml --plan-only +owcluster deploy aws up --config prod-deploy.yaml --plan-only ``` Remove `--plan-only` to review/approve provisioning. `--accept-plan DIGEST` and @@ -67,7 +67,7 @@ in deployment YAML resolve against that YAML's directory. Generated release manifest paths are absolute. Legacy explicit `advanced.setupProfile` import is compatibility-only; new files do not use it or a hidden default account profile. -Install `ows`, the cloud CLI, Docker, Helm, kubectl and OpenSSL. Azure also needs +Install `owcluster`, the cloud CLI, Docker, Helm, kubectl and OpenSSL. Azure also needs kubelogin and SSH for its optional management VM; AWS's SSM bridge needs the Session Manager plugin. Sign in using your existing cloud identity. The installer requires the reviewed provisioning/RBAC scopes, not merely a successful login. @@ -91,7 +91,7 @@ There is no required `budget` field in new deployment YAML. Check an optional estimated-cost ceiling in plan mode: ```sh -ows deploy azure up --config staging-deploy.yaml --plan-only \ +owcluster deploy azure up --config staging-deploy.yaml --plan-only \ --max-estimated-cost-usd 100 ``` @@ -134,13 +134,23 @@ pending a separately reviewed data-retention workflow. emits port-forward commands and CA information, not tokens in URLs. Use a current service token to sign in. This does not create public ingress, DNS or a certificate for a company domain. Existing ingress users can retain the legacy explicit origin; -`ows management-ui enable --ingress-url https://manage.example.com` selects it. +`owcluster management-ui enable --ingress-url https://manage.example.com` selects it. Do not combine an explicit ingress origin and preview port. Use `--keep-access` to retain AWS SSM/Azure SSH private management connectivity. The original Kubernetes endpoint/hostname/CA stay verified through the loopback route. `connection.json` records API and enabled UI connection instructions. +After an installation has recorded its original private-access route, +`--access-only --keep-access` reopens that route for diagnosis without advancing +installation or replacing resources. Stop the command to close the route. + +For Azure operators whose installation host cannot reach the database bootstrap +secret in Key Vault, `--database-password-file /private/database-password` can +supply that same database password. The file must be a bounded regular UTF-8 file +with private permissions (0600 on Unix). Its content is not printed or saved in +the installation review; the database host and TLS checks remain unchanged. + UI and recording are independent and disabled unless configured. Recording uses its own connection/schema on execution PostgreSQL by default; an existing Secret selected by `observability.databaseSecret` may provide `database-url` and `ca.crt` @@ -148,12 +158,12 @@ for a separate database. That Secret must be staged in the owned namespace befor service installation. Enabling initializes its schema before startup. ```sh -ows management-ui enable --state ./deployment-state/installation --port 8444 -ows management-ui disable --state ./deployment-state/installation -ows management-ui status --state ./deployment-state/installation -ows observability enable --state ./deployment-state/installation --retention-days 14 -ows observability disable --state ./deployment-state/installation -ows observability status --state ./deployment-state/installation +owcluster management-ui enable --state ./deployment-state/installation --port 8444 +owcluster management-ui disable --state ./deployment-state/installation +owcluster management-ui status --state ./deployment-state/installation +owcluster observability enable --state ./deployment-state/installation --retention-days 14 +owcluster observability disable --state ./deployment-state/installation +owcluster observability status --state ./deployment-state/installation ``` The commands save feature overrides in the installer journal, retain the original diff --git a/docs/harness/guides/observability.md b/docs/harness/guides/observability.md index 8b91e8a4..7a149281 100644 --- a/docs/harness/guides/observability.md +++ b/docs/harness/guides/observability.md @@ -171,13 +171,13 @@ continues to control audit reads. Cluster experiment/review tooling comes later. The source CLI supports independent UI and trace-storage controls: ```sh -ows observability enable --state ./deployment-state/installation --retention-days 14 -ows observability status --state ./deployment-state/installation -ows observability disable --state ./deployment-state/installation -ows management-ui enable --state ./deployment-state/installation +owcluster observability enable --state ./deployment-state/installation --retention-days 14 +owcluster observability status --state ./deployment-state/installation +owcluster observability disable --state ./deployment-state/installation +owcluster management-ui enable --state ./deployment-state/installation ``` -Use the installer state created by `ows deploy aws up` or `ows deploy azure up`. +Use the installer state created by `owcluster deploy aws up` or `owcluster deploy azure up`. Enabling recording initializes its separate PostgreSQL schema before workloads restart. Disabling stops collection after the rollout and retains stored traces; it does not disable security audit or a separately configured exporter. diff --git a/harness/service-cli/README.md b/harness/service-cli/README.md index ae50844b..cbb12647 100644 --- a/harness/service-cli/README.md +++ b/harness/service-cli/README.md @@ -95,7 +95,7 @@ See [AWS deployment](../../docs/harness/guides/aws.md) and [deployment options](../../docs/harness/guides/deployment.md). Use an owned trial with an explicit budget and independent cleanup before production deployment. -`owcluster` is the primary command. `ows` remains an alias, using the same commands, configuration and saved deployment state. Existing `.ows` paths and `OWS_*` settings remain valid. +`owcluster` is the primary command. `owcluster` remains an alias, using the same commands, configuration and saved deployment state. Existing `.ows` paths and `OWS_*` settings remain valid. ## Optional management UI and trace storage @@ -103,12 +103,12 @@ Select these features in `deployment.yaml` for the AWS/Azure `up` commands, or change them on an existing installation: ```sh -ows management-ui enable --state ./deployment-state/installation --port 8444 -ows management-ui disable --state ./deployment-state/installation -ows management-ui status --state ./deployment-state/installation -ows observability enable --state ./deployment-state/installation --retention-days 14 -ows observability disable --state ./deployment-state/installation -ows observability status --state ./deployment-state/installation +owcluster management-ui enable --state ./deployment-state/installation --port 8444 +owcluster management-ui disable --state ./deployment-state/installation +owcluster management-ui status --state ./deployment-state/installation +owcluster observability enable --state ./deployment-state/installation --retention-days 14 +owcluster observability disable --state ./deployment-state/installation +owcluster observability status --state ./deployment-state/installation ``` `--state` is the original installer journal, using its Kubernetes credentials and diff --git a/harness/service-cli/src/deployment_up.rs b/harness/service-cli/src/deployment_up.rs index 5d772b9d..2815a829 100644 --- a/harness/service-cli/src/deployment_up.rs +++ b/harness/service-cli/src/deployment_up.rs @@ -36,11 +36,18 @@ pub struct Options { /// Separate approval of scoped identity grants in the exact reviewed plan. #[arg(long)] accept_bootstrap: Option, + /// Azure only: private password file matching the reviewed database secret. + /// Avoids reading Key Vault data plane from the installer network. + #[arg(long)] + database_password_file: Option, #[arg(long, default_value_t = 3600)] timeout_seconds: u64, /// Keep the private management route open after installation; Ctrl-C closes it. #[arg(long)] keep_access: bool, + /// Reopen the original private route for diagnosis without advancing installation. + #[arg(long, requires = "keep_access", conflicts_with = "plan_only")] + access_only: bool, } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -199,6 +206,10 @@ async fn check_local_tools(c: &Config, package: &package::Snapshot) -> Result<() } pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { let config = Config::load(&o.config, provider)?; + ensure!( + o.database_password_file.is_none() || provider == "azure", + "database-password-file is Azure-only" + ); let package = package::load(&config)?; ensure!( (10..=14400).contains(&o.timeout_seconds), @@ -252,6 +263,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { review: &value, package: package.clone(), bootstrap_approved: false, + database_password_file: None, bridge: Default::default(), } .artifact_digest()?; @@ -266,6 +278,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { review: &review, package: package.clone(), bootstrap_approved: false, + database_password_file: None, bridge: Default::default() } .artifact_digest()? @@ -281,12 +294,16 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { review: &review, package: package.clone(), bootstrap_approved: false, + database_password_file: None, bridge: Default::default(), } .review_bundle() .await?; return cost::review(&state, &review, &package, o.max_estimated_cost_usd).await; } + if let Some(path) = &o.database_password_file { + native::validate_password_input(path)?; + } cost::validate_approval(&state, &review)?; if let Some(subtotal) = review .resolution @@ -304,6 +321,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { review: &review, package: package.clone(), bootstrap_approved: false, + database_password_file: None, bridge: Default::default(), }; preview.review_bundle().await?; @@ -326,6 +344,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { review: &review, package, bootstrap_approved: true, + database_password_file: o.database_password_file.clone(), bridge: Default::default(), }; let path = state.join("up-journal.json"); @@ -337,6 +356,21 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> { { driver.restore_access().await?; } + if o.access_only { + ensure!( + journal + .stages + .get(&state::Stage::PrivateAccess) + .is_some_and(|s| s.phase == state::Phase::Complete), + "Original private access receipt is required" + ); + crate::output( + json!({"phase":"access_ready","state":state,"installationAdvanced":false,"close":"Ctrl-C"}), + )?; + tokio::signal::ctrl_c().await?; + driver.close_access().await?; + return Ok(()); + } let previously_complete = journal.complete(); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(o.timeout_seconds); loop { diff --git a/harness/service-cli/src/deployment_up_azure_images.rs b/harness/service-cli/src/deployment_up_azure_images.rs index aaa4cfb0..7e0783f2 100644 --- a/harness/service-cli/src/deployment_up_azure_images.rs +++ b/harness/service-cli/src/deployment_up_azure_images.rs @@ -3,6 +3,52 @@ use super::*; use zeroize::Zeroizing; struct Temporary(PathBuf); +pub(super) fn private_password_file(path: &Path) -> Result>> { + use std::io::Read; + let meta = fs::symlink_metadata(path).context("Private database password file unavailable")?; + ensure!( + meta.is_file() && !meta.file_type().is_symlink() && meta.len() > 0 && meta.len() <= 4098, + "Use a bounded regular private password file" + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + ensure!( + meta.permissions().mode() & 0o077 == 0, + "Database password file must be private (chmod 600)" + ); + } + let file = fs::File::open(path).context("Private database password read failed")?; + let opened = file.metadata()?; + ensure!( + opened.is_file() && opened.len() == meta.len(), + "Database password file changed while opening" + ); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + ensure!( + opened.dev() == meta.dev() && opened.ino() == meta.ino() && opened.mode() & 0o077 == 0, + "Database password file identity changed" + ); + } + let mut bytes = Zeroizing::new(vec![]); + file.take(4099) + .read_to_end(&mut bytes) + .context("Private database password read failed")?; + ensure!( + bytes.len() <= 4098 && bytes.len() as u64 == opened.len(), + "Database password file changed while reading" + ); + let text = std::str::from_utf8(&bytes) + .context("Invalid database password encoding")? + .trim_end_matches(['\r', '\n']); + ensure!( + !text.is_empty() && text.len() <= 4096 && !text.contains(['\r', '\n', '\0']), + "Invalid private database password" + ); + Ok(bytes) +} fn source_acr(source: &str) -> Result> { let host = source.split('/').next().context("Image registry missing")?; let Some(name) = host.strip_suffix(".azurecr.io") else { @@ -311,7 +357,11 @@ impl Native<'_> { "--subscription", self.review.config.cloud.account(), ]); - let raw = Zeroizing::new(self.tool("az", std::mem::take(&mut args), None).await?); + let raw = if let Some(path) = &self.database_password_file { + private_password_file(path)? + } else { + Zeroizing::new(self.tool("az", std::mem::take(&mut args), None).await?) + }; let password = std::str::from_utf8(&raw) .context("Invalid secret encoding")? .trim_end_matches(['\r', '\n']); @@ -339,6 +389,25 @@ impl Native<'_> { #[cfg(test)] mod tests { use super::*; + #[cfg(unix)] + #[test] + fn password_file_requires_private_regular_bounded_input_without_echoing_values() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("password"); + fs::write(&path, b"private-canary\n").unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + assert!(private_password_file(&path).is_ok()); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + let error = private_password_file(&path).unwrap_err().to_string(); + assert!(!error.contains("private-canary")); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + fs::write(&path, b"multiple\nlines").unwrap(); + assert!(private_password_file(&path).is_err()); + let link = temp.path().join("link"); + std::os::unix::fs::symlink(&path, &link).unwrap(); + assert!(private_password_file(&link).is_err()); + } #[test] fn isolated_auth_preserves_only_buildx_discovery_without_copying_global_credentials() { let value = plugin_config(&json!([{"Name":"buildx","Path":"/opt/docker/cli-plugins/docker-buildx","secret":"must-not-copy"}])).unwrap(); diff --git a/harness/service-cli/src/deployment_up_native.rs b/harness/service-cli/src/deployment_up_native.rs index 4691a385..38abb9db 100644 --- a/harness/service-cli/src/deployment_up_native.rs +++ b/harness/service-cli/src/deployment_up_native.rs @@ -9,11 +9,15 @@ mod access; mod azure_images; #[path = "deployment_up_install.rs"] mod install; +pub(super) fn validate_password_input(path: &Path) -> Result<()> { + azure_images::private_password_file(path).map(|_| ()) +} pub(super) struct Native<'a> { pub root: PathBuf, pub review: &'a Review, pub package: package::Snapshot, pub bootstrap_approved: bool, + pub database_password_file: Option, pub bridge: std::cell::RefCell>, } fn arg(path: &Path) -> Result { @@ -654,6 +658,7 @@ mod contract_tests { review: &review, package, bootstrap_approved: false, + database_password_file: None, bridge: Default::default(), }; let plan = native.native_plan().unwrap();