fix: preserve private installation access and recording schema setup

This commit is contained in:
Rohit P
2026-11-13 12:38:44 -08:00
parent 7a0ea3c203
commit 95d175a2c7
9 changed files with 165 additions and 33 deletions
+11
View File
@@ -173,6 +173,17 @@ test("invalid ownership, mutable image and inadequate drain grace fail rendering
}),
);
});
test("Azure trace schema initializer is excluded from workload credential injection", async()=>{
const artifacts={kind:"azure",deployment:"ows-test",account:"owteststorage",container:"artifacts",tenant_id:"11111111-1111-4111-8111-111111111111",client_id:"22222222-2222-4222-8222-222222222222",federated_token_file:"/var/run/secrets/azure/tokens/azure-identity-token"};
const out=await render({config:{...base.config,artifacts,observability:{retention_days:14}}});
for(const role of ["api","worker"]){
const d=deployment(out,role);
assert.match(d,/azure.workload.identity\/skip-containers: "harness(?:;tls)?;recording-schema"/);
const init=d.split("initContainers:")[1].split(" containers:")[0];
assert.doesNotMatch(init,/azure-artifact-token|azure-identity-token|AZURE_/);
assert.equal((d.match(/name: azure-artifact-token\n/g)||[]).length,1);
}
});
test("management UI has no execution, database, or cluster authority", async () => {
const uiValues = {
enabled: true,
@@ -36,8 +36,11 @@ spec:
checksum/config: {{ include (print $.Template.BasePath "/config.yaml") $ | sha256sum }}
{{- if $azure }}
# This chart owns its explicit projection; prevent webhook injection of
# another token path or credentials into the TLS sidecar.
azure.workload.identity/skip-containers: {{ if eq $role "api" }}"harness;tls"{{ else }}"harness"{{ end }}
# another token path or credentials into TLS/schema-only containers.
{{- $skip := list "harness" -}}
{{- if eq $role "api" }}{{ $skip = append $skip "tls" }}{{ end -}}
{{- if hasKey $.Values.config "observability" }}{{ $skip = append $skip "recording-schema" }}{{ end }}
azure.workload.identity/skip-containers: {{ join ";" $skip | quote }}
{{- end }}
spec:
automountServiceAccountToken: {{ eq $role "platform" }}
+7 -7
View File
@@ -22,9 +22,9 @@ data only, optionally add `trial: {expiresAfterHours: 6, deleteData: true}`.
## Azure and AWS
```sh
ows deploy azure up --config azure/deployment.yaml --plan-only \
owcluster deploy azure up --config azure/deployment.yaml --plan-only \
--max-estimated-cost-usd 100
ows deploy aws up --config aws/deployment.yaml --plan-only
owcluster deploy aws up --config aws/deployment.yaml --plan-only
```
A cost ceiling is optional, is plan-only, and checks an estimate rather than actual
@@ -72,11 +72,11 @@ approval. Kubernetes must be able to reach PostgreSQL and pull/use those images.
`scripted` is a local fixture option, not a substitute for live-provider qualification.
```sh
ows deploy local-k8s plan --config local-k8s/deployment.yaml --state .ows/local
ows deploy local-k8s up --config local-k8s/deployment.yaml --state .ows/local \
owcluster deploy local-k8s plan --config local-k8s/deployment.yaml --state .ows/local
owcluster deploy local-k8s up --config local-k8s/deployment.yaml --state .ows/local \
--accept-plan PLAN_DIGEST
ows deploy local-k8s status --state .ows/local
ows management-ui status --state .ows/local/installation
owcluster deploy local-k8s status --state .ows/local
owcluster management-ui status --state .ows/local/installation
```
Use the actual digest printed by plan. The shared installer checks cluster and
@@ -90,7 +90,7 @@ No new sandbox/model run is implied by successful installation readiness.
The CLI generates the private API certificate and maps `managementUI.port` to the
loopback preview. Keep and trust that CA; do not disable TLS verification.
Set UI/recording preferences in YAML or use the independent feature commands.
Native lower-level `ows deploy local` / `deploy cluster` remain compatibility
Native lower-level `owcluster deploy local` / `deploy cluster` remain compatibility
interfaces for existing journals; new examples use `local-k8s`.
The new `local-docker` and `remote-k8s` command names are discoverable but report
+21 -11
View File
@@ -10,8 +10,8 @@ schema. The service installer remains experimental until the current release
passes a fresh cloud install, agent/recovery and cleanup qualification.
```sh
ows deploy azure up --config staging-deploy.yaml --plan-only
ows deploy aws up --config prod-deploy.yaml --plan-only
owcluster deploy azure up --config staging-deploy.yaml --plan-only
owcluster deploy aws up --config prod-deploy.yaml --plan-only
```
Remove `--plan-only` to review/approve provisioning. `--accept-plan DIGEST` and
@@ -67,7 +67,7 @@ in deployment YAML resolve against that YAML's directory. Generated release
manifest paths are absolute. Legacy explicit `advanced.setupProfile` import is
compatibility-only; new files do not use it or a hidden default account profile.
Install `ows`, the cloud CLI, Docker, Helm, kubectl and OpenSSL. Azure also needs
Install `owcluster`, the cloud CLI, Docker, Helm, kubectl and OpenSSL. Azure also needs
kubelogin and SSH for its optional management VM; AWS's SSM bridge needs the
Session Manager plugin. Sign in using your existing cloud identity. The installer
requires the reviewed provisioning/RBAC scopes, not merely a successful login.
@@ -91,7 +91,7 @@ There is no required `budget` field in new deployment YAML. Check an optional
estimated-cost ceiling in plan mode:
```sh
ows deploy azure up --config staging-deploy.yaml --plan-only \
owcluster deploy azure up --config staging-deploy.yaml --plan-only \
--max-estimated-cost-usd 100
```
@@ -134,13 +134,23 @@ pending a separately reviewed data-retention workflow.
emits port-forward commands and CA information, not tokens in URLs. Use a current
service token to sign in. This does not create public ingress, DNS or a certificate
for a company domain. Existing ingress users can retain the legacy explicit origin;
`ows management-ui enable --ingress-url https://manage.example.com` selects it.
`owcluster management-ui enable --ingress-url https://manage.example.com` selects it.
Do not combine an explicit ingress origin and preview port.
Use `--keep-access` to retain AWS SSM/Azure SSH private management connectivity.
The original Kubernetes endpoint/hostname/CA stay verified through the loopback
route. `connection.json` records API and enabled UI connection instructions.
After an installation has recorded its original private-access route,
`--access-only --keep-access` reopens that route for diagnosis without advancing
installation or replacing resources. Stop the command to close the route.
For Azure operators whose installation host cannot reach the database bootstrap
secret in Key Vault, `--database-password-file /private/database-password` can
supply that same database password. The file must be a bounded regular UTF-8 file
with private permissions (0600 on Unix). Its content is not printed or saved in
the installation review; the database host and TLS checks remain unchanged.
UI and recording are independent and disabled unless configured. Recording uses
its own connection/schema on execution PostgreSQL by default; an existing Secret
selected by `observability.databaseSecret` may provide `database-url` and `ca.crt`
@@ -148,12 +158,12 @@ for a separate database. That Secret must be staged in the owned namespace befor
service installation. Enabling initializes its schema before startup.
```sh
ows management-ui enable --state ./deployment-state/installation --port 8444
ows management-ui disable --state ./deployment-state/installation
ows management-ui status --state ./deployment-state/installation
ows observability enable --state ./deployment-state/installation --retention-days 14
ows observability disable --state ./deployment-state/installation
ows observability status --state ./deployment-state/installation
owcluster management-ui enable --state ./deployment-state/installation --port 8444
owcluster management-ui disable --state ./deployment-state/installation
owcluster management-ui status --state ./deployment-state/installation
owcluster observability enable --state ./deployment-state/installation --retention-days 14
owcluster observability disable --state ./deployment-state/installation
owcluster observability status --state ./deployment-state/installation
```
The commands save feature overrides in the installer journal, retain the original
+5 -5
View File
@@ -171,13 +171,13 @@ continues to control audit reads. Cluster experiment/review tooling comes later.
The source CLI supports independent UI and trace-storage controls:
```sh
ows observability enable --state ./deployment-state/installation --retention-days 14
ows observability status --state ./deployment-state/installation
ows observability disable --state ./deployment-state/installation
ows management-ui enable --state ./deployment-state/installation
owcluster observability enable --state ./deployment-state/installation --retention-days 14
owcluster observability status --state ./deployment-state/installation
owcluster observability disable --state ./deployment-state/installation
owcluster management-ui enable --state ./deployment-state/installation
```
Use the installer state created by `ows deploy aws up` or `ows deploy azure up`.
Use the installer state created by `owcluster deploy aws up` or `owcluster deploy azure up`.
Enabling recording initializes its separate PostgreSQL schema before workloads
restart. Disabling stops collection after the rollout and retains stored traces;
it does not disable security audit or a separately configured exporter.
+7 -7
View File
@@ -95,7 +95,7 @@ See [AWS deployment](../../docs/harness/guides/aws.md) and
[deployment options](../../docs/harness/guides/deployment.md). Use an owned trial
with an explicit budget and independent cleanup before production deployment.
`owcluster` is the primary command. `ows` remains an alias, using the same commands, configuration and saved deployment state. Existing `.ows` paths and `OWS_*` settings remain valid.
`owcluster` is the primary command. `owcluster` remains an alias, using the same commands, configuration and saved deployment state. Existing `.ows` paths and `OWS_*` settings remain valid.
## Optional management UI and trace storage
@@ -103,12 +103,12 @@ Select these features in `deployment.yaml` for the AWS/Azure `up` commands, or
change them on an existing installation:
```sh
ows management-ui enable --state ./deployment-state/installation --port 8444
ows management-ui disable --state ./deployment-state/installation
ows management-ui status --state ./deployment-state/installation
ows observability enable --state ./deployment-state/installation --retention-days 14
ows observability disable --state ./deployment-state/installation
ows observability status --state ./deployment-state/installation
owcluster management-ui enable --state ./deployment-state/installation --port 8444
owcluster management-ui disable --state ./deployment-state/installation
owcluster management-ui status --state ./deployment-state/installation
owcluster observability enable --state ./deployment-state/installation --retention-days 14
owcluster observability disable --state ./deployment-state/installation
owcluster observability status --state ./deployment-state/installation
```
`--state` is the original installer journal, using its Kubernetes credentials and
+34
View File
@@ -36,11 +36,18 @@ pub struct Options {
/// Separate approval of scoped identity grants in the exact reviewed plan.
#[arg(long)]
accept_bootstrap: Option<String>,
/// Azure only: private password file matching the reviewed database secret.
/// Avoids reading Key Vault data plane from the installer network.
#[arg(long)]
database_password_file: Option<PathBuf>,
#[arg(long, default_value_t = 3600)]
timeout_seconds: u64,
/// Keep the private management route open after installation; Ctrl-C closes it.
#[arg(long)]
keep_access: bool,
/// Reopen the original private route for diagnosis without advancing installation.
#[arg(long, requires = "keep_access", conflicts_with = "plan_only")]
access_only: bool,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
@@ -199,6 +206,10 @@ async fn check_local_tools(c: &Config, package: &package::Snapshot) -> Result<()
}
pub(super) async fn run(provider: &str, o: &Options) -> Result<()> {
let config = Config::load(&o.config, provider)?;
ensure!(
o.database_password_file.is_none() || provider == "azure",
"database-password-file is Azure-only"
);
let package = package::load(&config)?;
ensure!(
(10..=14400).contains(&o.timeout_seconds),
@@ -252,6 +263,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> {
review: &value,
package: package.clone(),
bootstrap_approved: false,
database_password_file: None,
bridge: Default::default(),
}
.artifact_digest()?;
@@ -266,6 +278,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> {
review: &review,
package: package.clone(),
bootstrap_approved: false,
database_password_file: None,
bridge: Default::default()
}
.artifact_digest()?
@@ -281,12 +294,16 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> {
review: &review,
package: package.clone(),
bootstrap_approved: false,
database_password_file: None,
bridge: Default::default(),
}
.review_bundle()
.await?;
return cost::review(&state, &review, &package, o.max_estimated_cost_usd).await;
}
if let Some(path) = &o.database_password_file {
native::validate_password_input(path)?;
}
cost::validate_approval(&state, &review)?;
if let Some(subtotal) = review
.resolution
@@ -304,6 +321,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> {
review: &review,
package: package.clone(),
bootstrap_approved: false,
database_password_file: None,
bridge: Default::default(),
};
preview.review_bundle().await?;
@@ -326,6 +344,7 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> {
review: &review,
package,
bootstrap_approved: true,
database_password_file: o.database_password_file.clone(),
bridge: Default::default(),
};
let path = state.join("up-journal.json");
@@ -337,6 +356,21 @@ pub(super) async fn run(provider: &str, o: &Options) -> Result<()> {
{
driver.restore_access().await?;
}
if o.access_only {
ensure!(
journal
.stages
.get(&state::Stage::PrivateAccess)
.is_some_and(|s| s.phase == state::Phase::Complete),
"Original private access receipt is required"
);
crate::output(
json!({"phase":"access_ready","state":state,"installationAdvanced":false,"close":"Ctrl-C"}),
)?;
tokio::signal::ctrl_c().await?;
driver.close_access().await?;
return Ok(());
}
let previously_complete = journal.complete();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(o.timeout_seconds);
loop {
@@ -3,6 +3,52 @@
use super::*;
use zeroize::Zeroizing;
struct Temporary(PathBuf);
pub(super) fn private_password_file(path: &Path) -> Result<Zeroizing<Vec<u8>>> {
use std::io::Read;
let meta = fs::symlink_metadata(path).context("Private database password file unavailable")?;
ensure!(
meta.is_file() && !meta.file_type().is_symlink() && meta.len() > 0 && meta.len() <= 4098,
"Use a bounded regular private password file"
);
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
ensure!(
meta.permissions().mode() & 0o077 == 0,
"Database password file must be private (chmod 600)"
);
}
let file = fs::File::open(path).context("Private database password read failed")?;
let opened = file.metadata()?;
ensure!(
opened.is_file() && opened.len() == meta.len(),
"Database password file changed while opening"
);
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
ensure!(
opened.dev() == meta.dev() && opened.ino() == meta.ino() && opened.mode() & 0o077 == 0,
"Database password file identity changed"
);
}
let mut bytes = Zeroizing::new(vec![]);
file.take(4099)
.read_to_end(&mut bytes)
.context("Private database password read failed")?;
ensure!(
bytes.len() <= 4098 && bytes.len() as u64 == opened.len(),
"Database password file changed while reading"
);
let text = std::str::from_utf8(&bytes)
.context("Invalid database password encoding")?
.trim_end_matches(['\r', '\n']);
ensure!(
!text.is_empty() && text.len() <= 4096 && !text.contains(['\r', '\n', '\0']),
"Invalid private database password"
);
Ok(bytes)
}
fn source_acr(source: &str) -> Result<Option<(String, String)>> {
let host = source.split('/').next().context("Image registry missing")?;
let Some(name) = host.strip_suffix(".azurecr.io") else {
@@ -311,7 +357,11 @@ impl Native<'_> {
"--subscription",
self.review.config.cloud.account(),
]);
let raw = Zeroizing::new(self.tool("az", std::mem::take(&mut args), None).await?);
let raw = if let Some(path) = &self.database_password_file {
private_password_file(path)?
} else {
Zeroizing::new(self.tool("az", std::mem::take(&mut args), None).await?)
};
let password = std::str::from_utf8(&raw)
.context("Invalid secret encoding")?
.trim_end_matches(['\r', '\n']);
@@ -339,6 +389,25 @@ impl Native<'_> {
#[cfg(test)]
mod tests {
use super::*;
#[cfg(unix)]
#[test]
fn password_file_requires_private_regular_bounded_input_without_echoing_values() {
use std::os::unix::fs::PermissionsExt;
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("password");
fs::write(&path, b"private-canary\n").unwrap();
fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap();
assert!(private_password_file(&path).is_ok());
fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
let error = private_password_file(&path).unwrap_err().to_string();
assert!(!error.contains("private-canary"));
fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap();
fs::write(&path, b"multiple\nlines").unwrap();
assert!(private_password_file(&path).is_err());
let link = temp.path().join("link");
std::os::unix::fs::symlink(&path, &link).unwrap();
assert!(private_password_file(&link).is_err());
}
#[test]
fn isolated_auth_preserves_only_buildx_discovery_without_copying_global_credentials() {
let value = plugin_config(&json!([{"Name":"buildx","Path":"/opt/docker/cli-plugins/docker-buildx","secret":"must-not-copy"}])).unwrap();
@@ -9,11 +9,15 @@ mod access;
mod azure_images;
#[path = "deployment_up_install.rs"]
mod install;
pub(super) fn validate_password_input(path: &Path) -> Result<()> {
azure_images::private_password_file(path).map(|_| ())
}
pub(super) struct Native<'a> {
pub root: PathBuf,
pub review: &'a Review,
pub package: package::Snapshot,
pub bootstrap_approved: bool,
pub database_password_file: Option<PathBuf>,
pub bridge: std::cell::RefCell<Option<access::Bridge>>,
}
fn arg(path: &Path) -> Result<String> {
@@ -654,6 +658,7 @@ mod contract_tests {
review: &review,
package,
bootstrap_approved: false,
database_password_file: None,
bridge: Default::default(),
};
let plan = native.native_plan().unwrap();