mirror of
https://github.com/andrewyng/openworker.git
synced 2026-11-20 20:37:44 +00:00
fix(azure): reconcile proven DNS dependency rejections before bounded retry
This commit is contained in:
@@ -116,6 +116,19 @@ struct Owned {
|
||||
delete_request: Option<String>,
|
||||
delete_accepted: bool,
|
||||
operation_url: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
rejected_dns_deletes: Vec<RejectedDnsDelete>,
|
||||
}
|
||||
// A bounded, provider-confirmed no-effect response is different from an unknown
|
||||
// mutation. Keep every original request; old ambiguous receipts stay parked.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
struct RejectedDnsDelete {
|
||||
request_id: String,
|
||||
observed_at_unix: u64,
|
||||
retry_after_unix: u64,
|
||||
status: u16,
|
||||
code: String,
|
||||
}
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
@@ -345,6 +358,23 @@ impl State {
|
||||
"Invalid deletion request UUID"
|
||||
);
|
||||
}
|
||||
let mut rejected = std::collections::BTreeSet::new();
|
||||
ensure!(
|
||||
r.rejected_dns_deletes.len() <= 5,
|
||||
"DNS dependency retry limit exceeded"
|
||||
);
|
||||
for rejection in &r.rejected_dns_deletes {
|
||||
ensure!(
|
||||
r.id.contains("/providers/Microsoft.Network/privateDnsZones/")
|
||||
&& rejection.status == 409
|
||||
&& rejection.code == "CannotDeleteResource"
|
||||
&& uuid::Uuid::parse_str(&rejection.request_id).is_ok_and(|u| !u.is_nil())
|
||||
&& rejected.insert(&rejection.request_id)
|
||||
&& rejection.retry_after_unix
|
||||
== rejection.observed_at_unix.saturating_add(60),
|
||||
"Invalid DNS no-effect receipt"
|
||||
);
|
||||
}
|
||||
if r.deleted {
|
||||
ensure!(
|
||||
r.delete_request.is_some(),
|
||||
@@ -581,6 +611,7 @@ async fn observe(c: &mut impl Cloud, s: &mut State, e: &mut String) -> Result<()
|
||||
delete_request: None,
|
||||
delete_accepted: false,
|
||||
operation_url: None,
|
||||
rejected_dns_deletes: vec![],
|
||||
});
|
||||
save(c, s, e).await?;
|
||||
}
|
||||
@@ -703,9 +734,22 @@ async fn destroy_one(c: &mut impl Cloud, s: &mut State, e: &mut String, at: u64)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if s.resources[i].delete_request.is_some() {
|
||||
return Ok(());
|
||||
} // GET only on subsequent ticks; even a lost DELETE is not replay permission.
|
||||
if let Some(request) = &s.resources[i].delete_request {
|
||||
let Some(rejected) = s.resources[i]
|
||||
.rejected_dns_deletes
|
||||
.last()
|
||||
.filter(|r| &r.request_id == request)
|
||||
else {
|
||||
// GET only after an accepted or unknown mutation, including old
|
||||
// receipts. A lost response must never authorize another DELETE.
|
||||
return Ok(());
|
||||
};
|
||||
ensure!(s.resources[i].rejected_dns_deletes.len() < 5,
|
||||
"DNS dependencies repeatedly rejected deletion; inspect original resource and receipts");
|
||||
if at < rejected.retry_after_unix {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
let id = s.resources[i].id.clone();
|
||||
let api = expected["apiVersion"].as_str().unwrap();
|
||||
let r = c.arm(Method::GET, &id, api, None, &s.nonce).await?;
|
||||
@@ -749,9 +793,32 @@ async fn destroy_one(c: &mut impl Cloud, s: &mut State, e: &mut String, at: u64)
|
||||
}
|
||||
let request = uuid::Uuid::new_v4().to_string();
|
||||
s.resources[i].delete_request = Some(request.clone());
|
||||
s.resources[i].delete_accepted = false;
|
||||
s.resources[i].operation_url = None;
|
||||
save(c, s, e).await?;
|
||||
let r = c.arm(Method::DELETE, &id, api, None, &request).await;
|
||||
match r {Ok(r) if [200,202,204].contains(&r.status)=>{s.resources[i].delete_accepted=true;s.resources[i].operation_url=r.operation;save(c,s,e).await?;},_=>bail!("Delete outcome unknown/rejected. Observe this exact resource; no automatic replay.")}
|
||||
match r {
|
||||
Ok(r) if [200, 202, 204].contains(&r.status) => {
|
||||
s.resources[i].delete_accepted = true;
|
||||
s.resources[i].operation_url = r.operation;
|
||||
save(c, s, e).await?;
|
||||
}
|
||||
Ok(r) if id.contains("/providers/Microsoft.Network/privateDnsZones/")
|
||||
&& r.status == 409
|
||||
&& r.body.as_ref().and_then(|v| v.pointer("/error/code")).and_then(Value::as_str)
|
||||
== Some("CannotDeleteResource") => {
|
||||
// Azure's empty link listing can precede dependency deletion.
|
||||
// Only this exact rejected response permits a future NEW attempt,
|
||||
// after backoff and fresh ownership/dependency checks above.
|
||||
s.resources[i].rejected_dns_deletes.push(RejectedDnsDelete {
|
||||
request_id: request, observed_at_unix: at,
|
||||
retry_after_unix: at.saturating_add(60), status: 409,
|
||||
code: "CannotDeleteResource".into(),
|
||||
});
|
||||
save(c, s, e).await?;
|
||||
}
|
||||
_ => bail!("Delete outcome unknown/rejected. Observe this exact resource; no automatic replay."),
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
s.completed_unix = Some(at);
|
||||
@@ -1261,6 +1328,8 @@ mod tests {
|
||||
lose_put: bool,
|
||||
lose_delete: bool,
|
||||
conflict: bool,
|
||||
dns_rejections: usize,
|
||||
dns_error_code: String,
|
||||
}
|
||||
impl Fake {
|
||||
fn new(s: State) -> Self {
|
||||
@@ -1274,6 +1343,8 @@ mod tests {
|
||||
lose_put: false,
|
||||
lose_delete: false,
|
||||
conflict: false,
|
||||
dns_rejections: 0,
|
||||
dns_error_code: "CannotDeleteResource".into(),
|
||||
}
|
||||
}
|
||||
fn etag(&self) -> String {
|
||||
@@ -1374,6 +1445,13 @@ mod tests {
|
||||
"Delete intent must be durable"
|
||||
);
|
||||
}
|
||||
if id.contains("/privateDnsZones/") && self.dns_rejections > 0 {
|
||||
self.dns_rejections -= 1;
|
||||
return Ok(reply(
|
||||
409,
|
||||
Some(json!({"error":{"code":self.dns_error_code}})),
|
||||
));
|
||||
}
|
||||
self.objects.remove(id);
|
||||
if self.lose_delete {
|
||||
self.lose_delete = false;
|
||||
@@ -1541,6 +1619,109 @@ mod tests {
|
||||
assert!(s.completed_unix.is_some());
|
||||
}
|
||||
|
||||
async fn blocked_dns() -> (Fake, State, String, String) {
|
||||
let (mut c, mut s, mut e) = active().await;
|
||||
c.dns_rejections = 1;
|
||||
for _ in 0..15 {
|
||||
destroy_one(&mut c, &mut s, &mut e, 201).await.unwrap();
|
||||
if s.resources
|
||||
.iter()
|
||||
.any(|r| !r.rejected_dns_deletes.is_empty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
let zone = s
|
||||
.resources
|
||||
.iter()
|
||||
.find(|r| !r.rejected_dns_deletes.is_empty())
|
||||
.unwrap()
|
||||
.id
|
||||
.clone();
|
||||
(c, s, e, zone)
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn dns_proven_dependency_rejection_survives_restart_and_rechecks_before_new_request() {
|
||||
let (mut c, s, _, zone) = blocked_dns().await;
|
||||
let old = s
|
||||
.resources
|
||||
.iter()
|
||||
.find(|r| r.id == zone)
|
||||
.unwrap()
|
||||
.delete_request
|
||||
.clone();
|
||||
let (mut e, mut s) = c.read_control("").await.unwrap().unwrap();
|
||||
let before = c.deletes();
|
||||
destroy_one(&mut c, &mut s, &mut e, 260).await.unwrap();
|
||||
assert_eq!(c.deletes(), before, "backoff must survive restart");
|
||||
c.objects.insert(
|
||||
format!("{zone}/virtualNetworkLinks"),
|
||||
json!({"value":[{"id":"still-there"}]}),
|
||||
);
|
||||
destroy_one(&mut c, &mut s, &mut e, 261).await.unwrap();
|
||||
assert_eq!(c.deletes(), before);
|
||||
c.objects.remove(&format!("{zone}/virtualNetworkLinks"));
|
||||
destroy_one(&mut c, &mut s, &mut e, 262).await.unwrap();
|
||||
let resource = s.resources.iter().find(|r| r.id == zone).unwrap();
|
||||
assert_ne!(resource.delete_request, old);
|
||||
assert_eq!(resource.rejected_dns_deletes[0].request_id, old.unwrap());
|
||||
assert!(resource.delete_accepted);
|
||||
assert_eq!(c.deletes(), before + 1);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn dns_retry_refuses_replacement_unknown_response_and_other_conflicts() {
|
||||
let (mut c, mut s, mut e, zone) = blocked_dns().await;
|
||||
c.objects.get_mut(&zone).unwrap()["properties"]["internalId"] = json!("replacement");
|
||||
let count = c.deletes();
|
||||
assert!(destroy_one(&mut c, &mut s, &mut e, 300).await.is_err());
|
||||
assert_eq!(c.deletes(), count);
|
||||
let (mut c, mut s, mut e, zone) = blocked_dns().await;
|
||||
// Model a lost/legacy response: no proven-rejection receipt exists.
|
||||
s.resources
|
||||
.iter_mut()
|
||||
.find(|r| r.id == zone)
|
||||
.unwrap()
|
||||
.rejected_dns_deletes
|
||||
.clear();
|
||||
save(&mut c, &s, &mut e).await.unwrap();
|
||||
let count = c.deletes();
|
||||
destroy_one(&mut c, &mut s, &mut e, 400).await.unwrap();
|
||||
assert_eq!(c.deletes(), count);
|
||||
let (mut c, mut s, mut e) = active().await;
|
||||
c.dns_rejections = 1;
|
||||
c.dns_error_code = "AnotherConflict".into();
|
||||
for _ in 0..15 {
|
||||
if destroy_one(&mut c, &mut s, &mut e, 201).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(s
|
||||
.resources
|
||||
.iter()
|
||||
.all(|r| r.rejected_dns_deletes.is_empty()));
|
||||
let count = c.deletes();
|
||||
destroy_one(&mut c, &mut s, &mut e, 400).await.unwrap();
|
||||
assert_eq!(c.deletes(), count);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn dns_dependency_attempts_are_bounded_and_keep_all_receipts() {
|
||||
let (mut c, mut s, mut e, _) = blocked_dns().await;
|
||||
c.dns_rejections = 20;
|
||||
for at in [261, 321, 381, 441] {
|
||||
destroy_one(&mut c, &mut s, &mut e, at).await.unwrap();
|
||||
}
|
||||
let count = c.deletes();
|
||||
assert!(destroy_one(&mut c, &mut s, &mut e, 501).await.is_err());
|
||||
assert_eq!(c.deletes(), count);
|
||||
assert_eq!(
|
||||
s.resources
|
||||
.iter()
|
||||
.map(|r| r.rejected_dns_deletes.len())
|
||||
.sum::<usize>(),
|
||||
5
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aks_resource_uid_is_required_when_creation_metadata_is_absent() {
|
||||
let (mut c, mut s, mut e) = active().await;
|
||||
|
||||
Reference in New Issue
Block a user