Files
Rohit PrasadandDevika Verma d53e058eda Introduce OpenWorker Harness SDK (in beta), agent service cluster and relevant developer documentation (#844)
Allow developers to easily build AI apps without worrying about the harness part.
Bundles the Rust agent runtime as a SDK (Rust and TypeScript) for building desktop and web applications. Applications can manage a local harness or connect to an authenticated harness service on k8s with separately scaled workers and sandbox execution.

The implementation is primarily contained in harness/, with deployment tooling in deploy/, Markdown guides in docs/harness/guides/, and the static documentation site in docs-site/. It does not replace the existing Python harness or migrate OpenWorker Desktop.

Co-authored-by: Devika Verma <devikaverma11@gmail.com>
2026-11-06 09:45:20 -08:00

347 lines
13 KiB
Rust

//! Real processes and PostgreSQL; no live models or OS-sandbox claims.
use super::concurrency::{
cancel, check_effects, effects, evidence, remove_owned_schema, sessions, submit, wait,
};
use super::*;
pub(super) fn pool(
root: &Path,
schema: &str,
name: &str,
sid: &str,
slots: usize,
seconds: u64,
) -> Child {
pool_command(root, schema, name, sid, slots, seconds)
.spawn()
.unwrap()
}
pub(super) fn pool_command(
root: &Path,
schema: &str,
name: &str,
sid: &str,
slots: usize,
seconds: u64,
) -> Command {
let mut command = Command::new(env!("CARGO_BIN_EXE_pg-worker-fixture"));
command
.arg(root)
.args([
schema,
name,
sid,
"drain-service",
&slots.to_string(),
&seconds.to_string(),
])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::inherit())
.kill_on_drop(true);
command
}
async fn signal(child: &Child) {
assert!(Command::new("kill")
.args(["-TERM", &child.id().unwrap().to_string()])
.status()
.await
.unwrap()
.success());
}
async fn exit(child: &mut Child, success: bool) {
let status = tokio::time::timeout(Duration::from_secs(18), child.wait())
.await
.unwrap()
.unwrap();
assert_eq!(status.success(), success);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated OW_PG_TEST_URL and Linux"]
async fn drain_finishes_current_turn_renews_lease_and_leaves_followups_for_replacement() {
let (root, c, api) = service_fixture();
let mut seed = Worker::start(root.path(), &c.schema, "seed", None).await;
let ids = sessions(root.path(), &mut seed, 2, true).await;
seed.kill().await;
for sid in &ids {
submit(&api, sid, "first");
}
let mut a = pool(root.path(), &c.schema, "a", &ids[0], 1, 20);
wait(|| (0..2).map(|i| effects(root.path(), i).len()).sum::<usize>() == 1).await;
let n = usize::from(effects(root.path(), 0).is_empty());
submit(&api, &ids[n], "follow");
signal(&a).await;
wait(|| root.path().join("a-draining").exists()).await;
tokio::time::sleep(Duration::from_secs(7)).await;
assert!(
a.try_wait().unwrap().is_none(),
"drain interrupted current tool"
);
assert_eq!(
api.submission(&ids[n], "first").unwrap().status,
SubmissionStatus::Started
);
assert_eq!(
api.submission(&ids[n], "follow").unwrap().status,
SubmissionStatus::Queued
);
std::fs::write(root.path().join(format!("work{n}/release")), "").unwrap();
exit(&mut a, true).await;
terminal_receipt(&api, &ids[n], "first", SubmissionStatus::Completed).await;
assert_eq!(effects(root.path(), n).len(), 1);
assert!(effects(root.path(), 1 - n).is_empty());
assert_eq!(
api.submission(&ids[n], "follow").unwrap().status,
SubmissionStatus::Queued
);
std::fs::write(root.path().join(format!("work{}/release", 1 - n)), "").unwrap();
let mut b = pool(root.path(), &c.schema, "b", &ids[0], 2, 20);
for sid in &ids {
terminal_receipt(&api, sid, "first", SubmissionStatus::Completed).await;
}
terminal_receipt(&api, &ids[n], "follow", SubmissionStatus::Completed).await;
stop_service(&mut b).await;
let mut expected = [1, 1];
expected[n] = 2;
check_effects(&api, root.path(), &ids, &expected);
evidence(
&c,
"drain-finish",
json!({"current_turn_completed":true,"lease_renewed_beyond_two_periods":true,"followup_left_queued":true,"replacement_effects":3,"duplicates":0}),
);
remove_owned_schema(&c);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated OW_PG_TEST_URL and Linux"]
async fn draining_preserves_approval_and_defers_resume_but_honors_active_cancellation() {
let (root, c, api) = service_fixture();
let mut seed = Worker::start(root.path(), &c.schema, "seed", None).await;
let ids = sessions(root.path(), &mut seed, 2, false).await;
seed.kill().await;
for sid in &ids {
submit(&api, sid, "first");
}
let mut a = pool(root.path(), &c.schema, "a", &ids[0], 2, 20);
wait(|| {
ids.iter().all(|sid| {
api.read_snapshot(sid).unwrap()["interactions"]
.as_array()
.is_some_and(|a| !a.is_empty())
})
})
.await;
let pending: Vec<_> = ids
.iter()
.map(|sid| api.read_snapshot(sid).unwrap())
.collect();
api.submit_control(decision(&ids[0], &pending[0], "allow-active", "once"))
.unwrap();
wait(|| effects(root.path(), 0).len() == 1).await;
signal(&a).await;
wait(|| root.path().join("a-draining").exists()).await;
let answer = decision(&ids[1], &pending[1], "answer-while-draining", "session");
api.submit_control(answer.clone()).unwrap();
tokio::time::sleep(Duration::from_millis(500)).await;
assert_eq!(
api.read_snapshot(&ids[1]).unwrap()["interactions"],
pending[1]["interactions"]
);
assert_eq!(
api.submit_control(answer.clone()).unwrap().status,
ControlStatus::Pending
);
let cancellation = cancel(&api, &ids[0], "cancel-during-drain");
exit(&mut a, true).await;
terminal_receipt(&api, &ids[0], "first", SubmissionStatus::Interrupted).await;
assert_eq!(
api.submit_control(cancellation).unwrap().status,
ControlStatus::Applied
);
assert!(effects(root.path(), 1).is_empty());
std::fs::write(root.path().join("work1/release"), "").unwrap();
let mut b = pool(root.path(), &c.schema, "b", &ids[0], 2, 20);
terminal_receipt(&api, &ids[1], "first", SubmissionStatus::Completed).await;
assert_eq!(
api.submit_control(answer.clone()).unwrap().status,
ControlStatus::Applied
);
stop_service(&mut b).await;
check_effects(&api, root.path(), &ids, &[1, 1]);
evidence(
&c,
"drain-controls",
json!({"approval_identity_preserved":true,"resume_deferred":true,"active_cancel_applied":true,"duplicates":0}),
);
remove_owned_schema(&c);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated OW_PG_TEST_URL and Linux"]
async fn drain_deadline_interrupts_without_replaying_an_uncertain_effect() {
let (root, c, api) = service_fixture();
let mut seed = Worker::start(root.path(), &c.schema, "seed", None).await;
let ids = sessions(root.path(), &mut seed, 1, true).await;
seed.kill().await;
submit(&api, &ids[0], "first");
let mut a = pool(root.path(), &c.schema, "a", &ids[0], 1, 2);
wait(|| effects(root.path(), 0).len() == 1).await;
signal(&a).await;
exit(&mut a, false).await;
terminal_receipt(&api, &ids[0], "first", SubmissionStatus::Interrupted).await;
assert!(api
.read_events(&ids[0], 0)
.unwrap()
.iter()
.any(|e| e.data["outcome_unknown_count"].as_u64().unwrap_or(0) > 0));
std::fs::write(root.path().join("work0/release"), "").unwrap();
submit(&api, &ids[0], "explicit-followup");
let mut b = pool(root.path(), &c.schema, "b", &ids[0], 1, 20);
terminal_receipt(
&api,
&ids[0],
"explicit-followup",
SubmissionStatus::Completed,
)
.await;
stop_service(&mut b).await;
check_effects(&api, root.path(), &ids, &[2]);
evidence(
&c,
"drain-deadline",
json!({"deadline_seconds":2,"interrupted":true,"unknown_recorded":true,"only_explicit_followup_executed":true,"duplicates":0}),
);
remove_owned_schema(&c);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated PG, Linux and OW_BASELINE_WORKER retained executable"]
async fn mixed_versions_keep_native_identity_pinned_and_old_pool_can_resume() {
let Some(baseline) = std::env::var_os("OW_BASELINE_WORKER") else {
return;
};
let (root, c, api) = service_fixture();
let baseline = std::path::PathBuf::from(baseline);
let mut seed =
Worker::start_binary(&baseline, root.path(), &c.schema, "old-seed", None, false).await;
let workspace = root.path().join("old-work");
std::fs::create_dir(&workspace).unwrap();
let old = seed
.rpc(
"session/create",
json!({"name":"old","coworker":"pg-test","workspace":workspace}),
)
.await["session"]["id"]
.as_str()
.unwrap()
.to_owned();
seed.kill().await;
submit(&api, &old, "first");
let old_pool = |name: &str| {
Command::new(&baseline)
.arg(root.path())
.args([&c.schema, name, &old, "service", "1"])
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::inherit())
.kill_on_drop(true)
.spawn()
.unwrap()
};
let mut old_worker = old_pool("old-worker");
wait(|| {
!api.read_snapshot(&old).unwrap()["interactions"]
.as_array()
.unwrap()
.is_empty()
})
.await;
let pending = api.read_snapshot(&old).unwrap();
stop_service(&mut old_worker).await;
let mut seed = Worker::start(root.path(), &c.schema, "new-seed", None).await;
let ids = sessions(root.path(), &mut seed, 1, true).await;
seed.kill().await;
assert_ne!(
api.inspect(&old).unwrap().bundle_digest,
api.inspect(&ids[0]).unwrap().bundle_digest,
"different host versions must not silently share native authority"
);
std::fs::write(root.path().join("work0/release"), "").unwrap();
submit(&api, &ids[0], "new-turn");
let mut new_worker = pool(root.path(), &c.schema, "new-worker", &ids[0], 1, 20);
terminal_receipt(&api, &ids[0], "new-turn", SubmissionStatus::Completed).await;
let answer = decision(&old, &pending, "old-approval", "once");
api.submit_control(answer.clone()).unwrap();
tokio::time::sleep(Duration::from_millis(700)).await;
assert_eq!(
api.submit_control(answer.clone()).unwrap().status,
ControlStatus::Pending
);
assert!(!workspace.join("effects.jsonl").exists());
assert_eq!(
api.read_snapshot(&old).unwrap()["interactions"],
pending["interactions"]
);
stop_service(&mut new_worker).await;
std::fs::write(workspace.join("release"), "").unwrap();
let mut restored = old_pool("restored-old");
terminal_receipt(&api, &old, "first", SubmissionStatus::Completed).await;
stop_service(&mut restored).await;
assert_eq!(
std::fs::read_to_string(workspace.join("effects.jsonl"))
.unwrap()
.lines()
.count(),
1
);
check_effects(&api, root.path(), &ids, &[1]);
evidence(
&c,
"mixed-version-pins",
json!({"new_native_binary_cannot_consume_old_session":true,"approval_identity_preserved":true,"pinned_baseline_pool_completed_old_work":true,"automatic_cross_version_migration":false}),
);
remove_owned_schema(&c);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "requires isolated OW_PG_TEST_URL and Linux"]
async fn database_loss_during_drain_interrupts_without_waiting_for_drain_deadline() {
let (root, c, api) = service_fixture();
let mut seed = Worker::start(root.path(), &c.schema, "seed", None).await;
let ids = sessions(root.path(), &mut seed, 1, true).await;
seed.kill().await;
submit(&api, &ids[0], "first");
let mut a = pool(root.path(), &c.schema, "a", &ids[0], 1, 30);
wait(|| effects(root.path(), 0).len() == 1).await;
signal(&a).await;
wait(|| root.path().join("a-draining").exists()).await;
tokio::task::block_in_place(|| {
let mut db = postgres::Client::connect(&c.connection, postgres::NoTls).unwrap();
let owner: String = db
.query_one(
&format!("SELECT owner FROM {}.sessions WHERE id=$1", c.schema),
&[&ids[0]],
)
.unwrap()
.get(0);
let rows = db
.query(
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name=$1",
&[&format!("ow-store-{owner}")],
)
.unwrap();
assert_eq!(rows.len(), 1);
});
let start = std::time::Instant::now();
exit(&mut a, false).await;
assert!(start.elapsed() < Duration::from_secs(15));
let mut b = pool(root.path(), &c.schema, "b", &ids[0], 1, 20);
terminal_receipt(&api, &ids[0], "first", SubmissionStatus::Interrupted).await;
stop_service(&mut b).await;
check_effects(&api, root.path(), &ids, &[1]);
evidence(
&c,
"drain-database-loss",
json!({"drain_seconds":30,"failure_exit_seconds":start.elapsed().as_secs_f64(),"effects":1,"replayed":false}),
);
remove_owned_schema(&c);
}