Files
openworker/harness/store-postgres/tests/submissions.rs

612 lines
23 KiB
Rust

use openworker_runtime::store::{Session, StateStore};
use openworker_store_postgres::{
Config, PostgresStore, SubmissionError, SubmissionStatus as Status, Tls, TurnSubmission,
};
use serde_json::json;
use std::sync::{Arc, Barrier};
fn fixture(limit: u16) -> (Config, PostgresStore) {
let mut c = Config::new(
std::env::var("OW_PG_TEST_URL").expect("Use a disposable DB"),
format!("ow_queue_{}", uuid::Uuid::new_v4().simple()),
);
c.tls = Tls::LoopbackPlaintext;
c.max_queued_turns = limit;
PostgresStore::migrate(&c).unwrap();
let mut store = PostgresStore::open(c.clone()).unwrap();
let s: Session=serde_json::from_value(json!({"id":"s_queue","name":"Queue","workspace":"/work","coworker":"test","bundle_digest":"test","schema_digest":null,"interactions":[],"turn":null,"sandbox":null,"closed":false,"resource_policy":openworker_runtime::resource_policy::Resolved::default()})).unwrap();
store.save(&s, "session_created", json!({})).unwrap();
(c, store)
}
fn admin(c: &Config) -> postgres::Client {
let mut db = postgres::Client::connect(&c.connection, postgres::NoTls).unwrap();
db.batch_execute(&format!(
"SET ow.runtime_schema='7'; SET search_path={},pg_catalog",
c.schema
))
.unwrap();
db
}
fn request(id: &str, text: &str) -> TurnSubmission {
TurnSubmission {
limits: None,
session_id: "s_queue".into(),
submission_id: id.into(),
text: text.into(),
}
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn twelve_producers_respect_queue_bound_and_retries_remain_available_when_full() {
let (config, mut owner) = fixture(4);
let barrier = Arc::new(Barrier::new(13));
let producers: Vec<_> = (0..12)
.map(|n| {
let config = config.clone();
let barrier = barrier.clone();
std::thread::spawn(move || {
let api = PostgresStore::open(config).unwrap();
let request = request(&format!("burst-{n}"), "bounded task");
barrier.wait();
(request.clone(), api.submit_turn(request))
})
})
.collect();
barrier.wait();
let outcomes: Vec<_> = producers.into_iter().map(|p| p.join().unwrap()).collect();
let admitted: Vec<_> = outcomes
.iter()
.filter_map(|(r, s)| s.as_ref().ok().map(|s| (r, s)))
.collect();
assert_eq!(admitted.len(), 4);
for (_, result) in outcomes.iter().filter(|(_, s)| s.is_err()) {
assert_eq!(
result
.as_ref()
.unwrap_err()
.downcast_ref::<SubmissionError>(),
Some(&SubmissionError::QueueFull)
);
}
let mut sequence: Vec<_> = admitted.iter().map(|(_, s)| s.sequence).collect();
sequence.sort();
assert_eq!(sequence, vec![1, 2, 3, 4]);
let api = PostgresStore::open(config.clone()).unwrap();
for (original, receipt) in admitted {
assert_eq!(
api.submit_turn(original.clone()).unwrap().turn_id,
receipt.turn_id
);
let mut changed = original.clone();
changed.text = "changed payload".into();
error(api.submit_turn(changed), SubmissionError::Conflict);
}
error(
api.submit_turn(request("fifth", "new task")),
SubmissionError::QueueFull,
);
let mut other = owner.get("s_queue").unwrap();
other.id = "s_other_burst".into();
other.storage_revision = 0;
owner.save(&other, "session_created", json!({})).unwrap();
let mut independent = request("independent", "other session");
independent.session_id = other.id.clone();
assert_eq!(api.submit_turn(independent).unwrap().sequence, 1);
assert_eq!(
api.read_events("s_queue", 0)
.unwrap()
.iter()
.filter(|e| e.kind == "command_accepted")
.count(),
4
);
}
fn error<T>(result: anyhow::Result<T>, expected: SubmissionError) {
let e = result.err().expect("operation unexpectedly succeeded");
assert_eq!(e.downcast_ref::<SubmissionError>(), Some(&expected), "{e}");
}
fn complete(owner: &mut PostgresStore) {
let mut s = owner.get("s_queue").unwrap();
s.turn.as_mut().unwrap().status = "completed".into();
owner
.save(&s, "turn_completed", json!({"status":"completed"}))
.unwrap();
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn competing_payloads_and_capacity_are_serialized_and_session_scoped() {
let (c, mut owner) = fixture(1);
let a = PostgresStore::open(c.clone()).unwrap();
let b = PostgresStore::open(c.clone()).unwrap();
let gate = Arc::new(Barrier::new(3));
let ga = gate.clone();
let gb = gate.clone();
let first = std::thread::spawn(move || {
ga.wait();
a.submit_turn(request("stable", "one"))
});
let second = std::thread::spawn(move || {
gb.wait();
b.submit_turn(request("stable", "two"))
});
gate.wait();
let outcomes = [first.join().unwrap(), second.join().unwrap()];
assert_eq!(outcomes.iter().filter(|r| r.is_ok()).count(), 1);
assert!(outcomes
.iter()
.filter_map(|r| r.as_ref().err())
.all(|e| e.downcast_ref::<SubmissionError>() == Some(&SubmissionError::Conflict)));
let api = PostgresStore::open(c.clone()).unwrap();
error(
api.submit_turn(request("other", "third")),
SubmissionError::QueueFull,
);
let mut other = owner.get("s_queue").unwrap();
other.id = "s_other".into();
other.storage_revision = 0;
owner.save(&other, "session_created", json!({})).unwrap();
let mut req = request("stable", "independent");
req.session_id = "s_other".into();
let other_receipt = api.submit_turn(req).unwrap();
assert_ne!(
api.submission("s_queue", "stable").unwrap().turn_id,
other_receipt.turn_id
);
assert_eq!(other_receipt.sequence, 1);
assert!(owner.next_turn("s_queue").unwrap().is_some());
assert!(owner.next_turn("s_other").unwrap().is_some());
let (c, _owner) = fixture(1);
let a = PostgresStore::open(c.clone()).unwrap();
let b = PostgresStore::open(c).unwrap();
let gate = Arc::new(Barrier::new(3));
let ga = gate.clone();
let gb = gate.clone();
let first = std::thread::spawn(move || {
ga.wait();
a.submit_turn(request("different-a", "A"))
});
let second = std::thread::spawn(move || {
gb.wait();
b.submit_turn(request("different-b", "B"))
});
gate.wait();
let outcomes = [first.join().unwrap(), second.join().unwrap()];
assert_eq!(outcomes.iter().filter(|r| r.is_ok()).count(), 1);
assert!(outcomes
.iter()
.filter_map(|r| r.as_ref().err())
.all(|e| e.downcast_ref::<SubmissionError>() == Some(&SubmissionError::QueueFull)));
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn concurrent_api_nodes_create_one_receipt_and_one_turn() {
let (c, mut owner) = fixture(100);
let a = PostgresStore::open(c.clone()).unwrap();
let b = PostgresStore::open(c.clone()).unwrap();
let gate = Arc::new(Barrier::new(3));
let ga = gate.clone();
let gb = gate.clone();
let first = std::thread::spawn(move || {
ga.wait();
a.submit_turn(request("stable", "sensitive prompt"))
.unwrap()
});
let second = std::thread::spawn(move || {
gb.wait();
b.submit_turn(request("stable", "sensitive prompt"))
.unwrap()
});
gate.wait();
let receipt = first.join().unwrap();
assert_eq!(receipt, second.join().unwrap());
assert_eq!(receipt.status, Status::Queued);
assert_eq!(receipt.sequence, 1);
assert!(owner.get("s_queue").unwrap().turn.is_none());
assert_eq!(
owner
.events("s_queue", 0)
.unwrap()
.iter()
.filter(|e| e.kind == "command_accepted")
.count(),
1
);
let api = PostgresStore::open(c).unwrap();
error(
api.submit_turn(request("stable", "changed")),
SubmissionError::Conflict,
);
assert!(api.next_turn("s_queue").is_err());
let next = owner.next_turn("s_queue").unwrap().unwrap();
assert_eq!(next.submission_id, "stable");
let (started, fresh, _) = owner
.accept("s_queue", &next.submission_id, &next.text)
.unwrap();
assert!(fresh);
assert_eq!(started.turn.unwrap().id, receipt.turn_id);
assert_eq!(
api.submission("s_queue", "stable").unwrap().status,
Status::Started
);
assert!(
!owner
.accept("s_queue", "stable", "sensitive prompt")
.unwrap()
.1
);
assert_eq!(owner.get("s_queue").unwrap().history.len(), 1);
complete(&mut owner);
let final_receipt = api
.submit_turn(request("stable", "sensitive prompt"))
.unwrap();
assert_eq!(final_receipt.status, Status::Completed);
assert_eq!(receipt.turn_id, final_receipt.turn_id);
assert!(!serde_json::to_string(&owner.audit("s_queue", 0).unwrap())
.unwrap()
.contains("sensitive prompt"));
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn fifo_capacity_and_interleaved_ingress_do_not_invalidate_worker_state() {
let (c, mut owner) = fixture(2);
let api = PostgresStore::open(c).unwrap();
let mut snapshot = owner.get("s_queue").unwrap();
let first = api.submit_turn(request("one", "First")).unwrap();
api.submit_turn(request("two", "Second")).unwrap();
error(
api.submit_turn(request("three", "Third")),
SubmissionError::QueueFull,
);
assert_eq!(api.submit_turn(request("one", "First")).unwrap(), first);
// Queue events share the cursor, not canonical metadata revision.
snapshot.name = "Worker update".into();
owner.save(&snapshot, "name_changed", json!({})).unwrap();
assert_eq!(owner.get("s_queue").unwrap().name, "Worker update");
error(
owner.accept("s_queue", "two", "Second"),
SubmissionError::OutOfOrder,
);
error(
owner.accept("s_queue", "bypass", "Unqueued"),
SubmissionError::OutOfOrder,
);
owner.accept("s_queue", "one", "First").unwrap();
assert!(owner.next_turn("s_queue").unwrap().is_none());
api.submit_turn(request("three", "Third")).unwrap();
complete(&mut owner);
assert_eq!(
owner.next_turn("s_queue").unwrap().unwrap().submission_id,
"two"
);
owner.accept("s_queue", "two", "Second").unwrap();
complete(&mut owner);
owner.accept("s_queue", "three", "Third").unwrap();
complete(&mut owner);
assert!(owner.next_turn("s_queue").unwrap().is_none());
let events = owner.events("s_queue", 0).unwrap();
assert!(events
.iter()
.enumerate()
.all(|(i, e)| e.event_id == i as i64 + 1));
assert_eq!(events.len(), owner.audit("s_queue", 0).unwrap().len());
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn lost_receipts_reconnect_and_takeover_preserve_queue_without_replay() {
let (c, mut old) = fixture(100);
{
let api = PostgresStore::open(c.clone()).unwrap();
// Simulate a committed acceptance whose transport response is lost.
let _ = api.submit_turn(request("one", "First")).unwrap();
let _ = api.submit_turn(request("two", "Second")).unwrap();
}
let api = PostgresStore::open(c.clone()).unwrap();
let receipt = api.submission("s_queue", "one").unwrap();
assert_eq!(api.submit_turn(request("one", "First")).unwrap(), receipt);
old.accept("s_queue", "one", "First").unwrap();
admin(&c)
.execute(
"UPDATE sessions SET expires=clock_timestamp()-interval '1 second' WHERE id='s_queue'",
&[],
)
.unwrap();
let mut new = PostgresStore::open(c).unwrap();
new.claim("s_queue").unwrap();
new.recover().unwrap();
assert_eq!(
api.submission("s_queue", "one").unwrap().status,
Status::Interrupted
);
assert_eq!(
api.submit_turn(request("one", "First")).unwrap().turn_id,
receipt.turn_id
);
assert_eq!(
new.next_turn("s_queue").unwrap().unwrap().submission_id,
"two"
);
assert!(old.accept("s_queue", "two", "Second").is_err());
new.accept("s_queue", "two", "Second").unwrap();
assert_eq!(new.get("s_queue").unwrap().history.len(), 2);
assert_eq!(
api.submission("s_queue", "two").unwrap().owner_epoch,
Some(2)
);
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn receipt_event_audit_and_queue_position_roll_back_together() {
let (c, mut owner) = fixture(100);
let api = PostgresStore::open(c.clone()).unwrap();
admin(&c).batch_execute("CREATE FUNCTION fail_ingress() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'SECRET_SQL_DETAIL'; END $$; CREATE TRIGGER fail_ingress BEFORE INSERT ON audit FOR EACH ROW WHEN (NEW.kind='command_accepted') EXECUTE FUNCTION fail_ingress()").unwrap();
let e = api
.submit_turn(request("one", "secret text"))
.unwrap_err()
.to_string();
assert!(!e.contains("SECRET_SQL_DETAIL"));
let other = PostgresStore::open(c.clone()).unwrap();
error(
other.submission("s_queue", "one"),
SubmissionError::NotFound,
);
assert!(owner.next_turn("s_queue").unwrap().is_none());
assert_eq!(owner.cursor("s_queue").unwrap(), 1);
admin(&c)
.batch_execute("DROP TRIGGER fail_ingress ON audit")
.unwrap();
assert_eq!(
other
.submit_turn(request("one", "secret text"))
.unwrap()
.sequence,
1
);
// The same atomicity applies when the owner promotes a queued submission.
admin(&c).batch_execute("CREATE TRIGGER fail_start BEFORE INSERT ON audit FOR EACH ROW WHEN (NEW.kind='turn_accepted') EXECUTE FUNCTION fail_ingress()").unwrap();
assert!(owner.accept("s_queue", "one", "secret text").is_err());
assert_eq!(
other.submission("s_queue", "one").unwrap().status,
Status::Queued
);
assert!(other.inspect("s_queue").unwrap().history.is_empty());
assert!(other.inspect("s_queue").unwrap().turn.is_none());
admin(&c).batch_execute("DROP TRIGGER fail_start ON audit; UPDATE sessions SET expires=clock_timestamp()-interval '1 second'").unwrap();
let mut replacement = PostgresStore::open(c.clone()).unwrap();
replacement.claim("s_queue").unwrap();
replacement.accept("s_queue", "one", "secret text").unwrap();
admin(&c).batch_execute("CREATE TRIGGER fail_finish BEFORE INSERT ON audit FOR EACH ROW WHEN (NEW.kind='turn_completed') EXECUTE FUNCTION fail_ingress()").unwrap();
let mut finishing = replacement.get("s_queue").unwrap();
finishing.turn.as_mut().unwrap().status = "completed".into();
finishing
.history
.push(json!({"type":"message","role":"assistant","content":"done"}));
assert!(replacement
.save(&finishing, "turn_completed", json!({"status":"completed"}))
.is_err());
assert_eq!(
other.submission("s_queue", "one").unwrap().status,
Status::Started
);
let unchanged = other.inspect("s_queue").unwrap();
assert_eq!(unchanged.turn.unwrap().status, "accepted");
assert_eq!(unchanged.history.len(), 1);
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn validation_lifecycle_and_old_schema_are_explicit() {
let (c, mut owner) = fixture(100);
let api = PostgresStore::open(c.clone()).unwrap();
error(
api.submit_turn(request("", "Hello")),
SubmissionError::InvalidRequest,
);
error(
api.submit_turn(request("id", &"a".repeat(32001))),
SubmissionError::InvalidRequest,
);
error(
api.submit_turn(request("id", "\0")),
SubmissionError::InvalidRequest,
);
let mut missing = request("id", "Hi");
missing.session_id = "s_missing".into();
error(api.submit_turn(missing), SubmissionError::NotFound);
api.submit_turn(request("one", "First")).unwrap();
let mut s = owner.get("s_queue").unwrap();
s.closed = true;
assert!(owner.save(&s, "session_closed", json!({})).is_err());
assert!(!owner.get("s_queue").unwrap().closed);
owner.accept("s_queue", "one", "First").unwrap();
complete(&mut owner);
let mut s = owner.get("s_queue").unwrap();
s.provider_blocked = true;
owner.save(&s, "blocked", json!({})).unwrap();
error(
api.submit_turn(request("two", "Second")),
SubmissionError::ProviderBlocked,
);
let mut s = owner.get("s_queue").unwrap();
s.closed = true;
owner.save(&s, "closed", json!({})).unwrap();
error(
api.submit_turn(request("two", "Second")),
SubmissionError::Closed,
);
assert_eq!(
api.submit_turn(request("one", "First")).unwrap().status,
Status::Completed
);
admin(&c)
.execute("UPDATE version SET version=1", &[])
.unwrap();
assert!(PostgresStore::open(c.clone()).is_err());
assert!(PostgresStore::migrate(&c).is_err());
let mut bad = c;
bad.max_queued_turns = 101;
assert!(PostgresStore::open(bad).is_err());
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn deleted_receipts_do_not_turn_a_duplicate_into_new_work() {
let (c, mut owner) = fixture(100);
let api = PostgresStore::open(c.clone()).unwrap();
api.submit_turn(request("one", "First")).unwrap();
owner.accept("s_queue", "one", "First").unwrap();
complete(&mut owner);
let length = owner.get("s_queue").unwrap().history.len();
admin(&c)
.execute("DELETE FROM submissions WHERE session_id='s_queue'", &[])
.unwrap();
error(
api.submission("s_queue", "one"),
SubmissionError::IncompleteState,
);
error(
api.submit_turn(request("one", "First")),
SubmissionError::IncompleteState,
);
error(
owner.accept("s_queue", "one", "First"),
SubmissionError::IncompleteState,
);
error(owner.next_turn("s_queue"), SubmissionError::IncompleteState);
assert_eq!(owner.get("s_queue").unwrap().history.len(), length);
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn turn_limits_survive_queue_and_bind_idempotency() {
use openworker_runtime::budget::Limits;
let (config, mut owner) = fixture(4);
let api = PostgresStore::open(config).unwrap();
let mut request = request("limited", "Build only");
request.limits = Some(Limits {
max_model_calls: 2,
max_output_tokens: 2048,
summary_tokens: 256,
..Default::default()
});
let receipt = api.submit_turn(request.clone()).unwrap();
assert_eq!(
api.submit_turn(request.clone()).unwrap().turn_id,
receipt.turn_id
);
let mut changed = request.clone();
changed.limits.as_mut().unwrap().max_model_calls = 3;
assert!(api.submit_turn(changed).is_err());
let queued = owner.next_turn("s_queue").unwrap().unwrap();
assert_eq!(queued.limits, request.limits);
let (state, fresh, _) = owner
.accept_with_limits(
"s_queue",
&queued.submission_id,
&queued.text,
queued.limits.clone(),
)
.unwrap();
assert!(fresh);
assert_eq!(
state.turn.as_ref().unwrap().budget.limits,
request.limits.unwrap()
);
let mut state = state;
state.turn.as_mut().unwrap().budget.model_calls = 1;
state.turn.as_mut().unwrap().budget.charged_output_tokens = 17;
owner
.save(&state, "budget_test_checkpoint", json!({}))
.unwrap();
let copy = owner.get("s_queue").unwrap();
assert_eq!(copy.turn.unwrap().budget.model_calls, 1);
complete(&mut owner);
let (next, fresh, _) = owner.accept("s_queue", "next", "Next task").unwrap();
assert!(fresh);
assert_eq!(next.turn.unwrap().budget.limits, Limits::default());
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn resilience_service_result_preserves_interruption_for_current_and_older_runs() {
let (_config, mut owner) = fixture(4);
let (mut state, _, _) = owner
.accept("s_queue", "crashed", "Streaming task")
.unwrap();
state.turn.as_mut().unwrap().status = "running".into();
owner.save(&state, "model_started", json!({})).unwrap();
owner.recover().unwrap();
let original = owner.service_run("s_queue", "crashed").unwrap();
assert_eq!(original["status"], "interrupted");
assert_eq!(original["failure"], "interrupted");
assert!(original["finalOutput"].is_null());
assert_eq!(original["interruptions"], json!([]));
let (_next, _, _) = owner
.accept("s_queue", "next", "Explicit continuation")
.unwrap();
complete(&mut owner);
assert_eq!(
owner.service_run("s_queue", "next").unwrap()["status"],
"completed"
);
let older = owner.service_run("s_queue", "crashed").unwrap();
assert_eq!(older["status"], "interrupted");
assert_eq!(older["runId"], original["runId"]);
assert!(older["finalOutput"].is_null());
}
#[test]
#[ignore = "requires isolated OW_PG_TEST_URL"]
fn resilience_database_actor_recovery_events_keep_the_explicit_turn() {
use openworker_runtime::telemetry::{Controller, Operation, Record, Sink, Snapshot, Telemetry};
use std::{sync::Mutex, time::Duration};
struct Capture(Arc<Mutex<Vec<Record>>>);
impl Sink for Capture {
fn export(&mut self, records: &[Record], _: Snapshot) -> bool {
self.0.lock().unwrap().extend_from_slice(records);
true
}
}
let (mut c, mut owner) = fixture(4);
let (mut s, _, _) = owner
.accept("s_queue", "unknown", "PRIVATE_PROMPT")
.unwrap();
let call = json!({"type":"function_call","call_id":"unknown-call","name":"write_file","arguments":"{\"private\":\"PRIVATE_ARGUMENT\"}"});
let t = s.turn.as_mut().unwrap();
t.status = "executing".into();
t.queue = vec![call.clone()];
let tid = t.id.clone();
s.history.push(call);
owner
.save(&s, "tool_started", json!({"call_id":"unknown-call"}))
.unwrap();
owner.release("s_queue").unwrap();
let rows = Arc::new(Mutex::new(vec![]));
let (telemetry, control): (Telemetry, Controller) =
Telemetry::start(Capture(rows.clone()), 256).unwrap();
c.telemetry = telemetry;
let mut replacement = PostgresStore::open(c).unwrap();
replacement.claim("s_queue").unwrap();
replacement.recover().unwrap();
assert!(control.shutdown(Duration::from_secs(3)));
let rows = rows.lock().unwrap();
for op in [Operation::Recovery, Operation::UnknownOutcome] {
let events: Vec<_> = rows.iter().filter(|r| r.operation == op).collect();
assert_eq!(events.len(), 1);
let hash = events[0]
.turn_hash
.expect("Database actor may not rely on task-local context");
assert_eq!(
hash.iter().map(|n| format!("{n:02x}")).collect::<String>(),
openworker_runtime::bundle::digest(tid.as_bytes())
);
}
let debug = format!("{rows:?}");
assert!(!debug.contains("PRIVATE_PROMPT"));
assert!(!debug.contains("PRIVATE_ARGUMENT"));
}