receipt_delivery_faults.rs (20335B)
1 use super::*; 2 3 #[tokio::test] 4 async fn a_retry_cannot_replace_the_original_publication_target_policy() { 5 let ready = Ready::new( 6 &[ 7 String::from("ws://127.0.0.1:9"), 8 String::from("ws://127.0.0.1:10"), 9 ], 10 243, 11 ) 12 .await; 13 let original = ready.retain(1).await; 14 let narrowed = TargetSet::new(vec![original.intent().targets().targets()[0].clone()]).unwrap(); 15 let replacement = DeliveryIntent::new( 16 &"02".repeat(32), 17 original.intent().event_id(), 18 &narrowed, 19 original.intent().policy(), 20 original.intent().deadline_unix_ms(), 21 ) 22 .unwrap(); 23 let request = 24 DeliveryAttemptRequest::new(ready.binding.clone(), ready.work, replacement).unwrap(); 25 assert!(matches!( 26 ready 27 .store 28 .prepare_delivery_attempt(request, copy(&ready.proof), None) 29 .await, 30 Err(OperationError::IdempotencyConflict) 31 )); 32 ready.store.close().await.unwrap(); 33 } 34 35 fn selected_request( 36 ready: &Ready, 37 id: u8, 38 policy: &SatisfactionPolicy, 39 index: usize, 40 ) -> DeliveryAttemptRequest { 41 let original = ready.request(id); 42 let selected = 43 TargetSet::new(vec![original.intent().targets().targets()[index].clone()]).unwrap(); 44 let intent = DeliveryIntent::new( 45 original.intent().request_id().as_str(), 46 original.intent().event_id(), 47 original.intent().targets(), 48 policy, 49 original.intent().deadline_unix_ms(), 50 ) 51 .unwrap() 52 .with_selection(&selected) 53 .unwrap(); 54 DeliveryAttemptRequest::new(ready.binding.clone(), ready.work, intent).unwrap() 55 } 56 57 // Synthetic normalized adapter data exercises storage/policy composition here. 58 // The separate actual loopback cases qualify remote frame and ACK behavior. 59 fn synthetic_accepted(attempt: &DeliveryAttempt) -> DeliveryObservation { 60 use radroots_transport::{ 61 DeliveryReceipt, outcome::DeliveryOutcome, sink::DeliveryTargetReceipt, 62 }; 63 let request = attempt.transport_request().unwrap(); 64 let selected = &attempt.intent().selection()[0]; 65 let receipts = request 66 .target_set() 67 .targets() 68 .iter() 69 .map(|target| { 70 if target.fingerprint() == selected { 71 DeliveryTargetReceipt::attempted(target.clone(), DeliveryOutcome::accepted()) 72 } else { 73 DeliveryTargetReceipt::skipped(target.clone(), DeliveryOutcome::unavailable()) 74 .unwrap() 75 } 76 }) 77 .collect(); 78 let receipt = DeliveryReceipt::for_request(&request, receipts).unwrap(); 79 DeliveryObservation::from_receipt(attempt, &receipt, selected, None).unwrap() 80 } 81 82 #[tokio::test] 83 async fn exhausted_optional_endpoint_does_not_starve_required_endpoint_or_weaken_the_original_policy() 84 { 85 let ready = Ready::new( 86 &[ 87 String::from("ws://127.0.0.1:9"), 88 String::from("ws://127.0.0.1:10"), 89 ], 90 244, 91 ) 92 .await; 93 let base = ready.request(1); 94 let required = base.intent().targets().targets()[1].fingerprint().clone(); 95 let policy = SatisfactionPolicy::new( 96 SatisfactionClass::Accepted, 97 TargetPolicy::required(vec![required]).unwrap(), 98 ); 99 for ordinal in 1..=12u8 { 100 let selected = selected_request(&ready, ordinal, &policy, 0); 101 let candidate = if ordinal == 1 { 102 Some(&ready.candidate) 103 } else { 104 None 105 }; 106 let DeliveryAttemptOutcome::Claimed(attempt) = ready 107 .store 108 .prepare_delivery_attempt(selected, copy(&ready.proof), candidate) 109 .await 110 .unwrap() 111 else { 112 panic!("new selected potential attempt expected") 113 }; 114 assert_eq!(attempt.target_ordinals(), &[ordinal]); 115 assert_eq!(attempt.intent().targets().len(), 2); 116 assert_eq!(attempt.intent().selection().len(), 1); 117 } 118 assert!(matches!( 119 ready 120 .store 121 .prepare_delivery_attempt( 122 selected_request(&ready, 13, &policy, 0), 123 copy(&ready.proof), 124 None 125 ) 126 .await, 127 Err(OperationError::DeliveryAttemptLimit) 128 )); 129 let DeliveryAttemptOutcome::Claimed(required) = ready 130 .store 131 .prepare_delivery_attempt( 132 selected_request(&ready, 14, &policy, 1), 133 copy(&ready.proof), 134 None, 135 ) 136 .await 137 .unwrap() 138 else { 139 panic!("required endpoint must progress independently") 140 }; 141 assert_eq!(required.target_ordinals(), &[1]); 142 assert_eq!(required.raw_json(), ready.candidate.raw_json()); 143 captured( 144 ready 145 .store 146 .record_delivery_observation( 147 ready.binding.clone(), 148 copy(&ready.proof), 149 synthetic_accepted(&required), 150 ) 151 .await 152 .unwrap(), 153 ); 154 let e = ready.work.evidence(); 155 let snapshot = ready 156 .store 157 .operation_snapshot( 158 ready.binding.clone(), 159 e.operation_id(), 160 None, 161 copy(&ready.proof), 162 ) 163 .await 164 .unwrap() 165 .unwrap(); 166 assert_eq!(snapshot.publication_accepted(ready.work), Ok(true)); 167 let receipt = 168 radroots_execution::OperationReceipt::<radroots_domain::ChatMessage>::try_from_snapshot( 169 &snapshot, 170 ) 171 .unwrap(); 172 assert_eq!(receipt.delivery(), radroots_execution::Delivery::Unknown); 173 assert!( 174 !receipt 175 .recovery() 176 .contains(&radroots_execution::RecoveryRequirement::ResumeDelivery) 177 ); 178 ready.store.close().await.unwrap(); 179 } 180 181 #[tokio::test] 182 async fn original_quorum_accumulates_acceptance_across_independent_selected_attempts() { 183 let ready = Ready::new( 184 &[ 185 String::from("ws://127.0.0.1:9"), 186 String::from("ws://127.0.0.1:10"), 187 String::from("ws://127.0.0.1:11"), 188 ], 189 245, 190 ) 191 .await; 192 let policy = SatisfactionPolicy::new( 193 SatisfactionClass::Accepted, 194 TargetPolicy::quorum(2).unwrap(), 195 ); 196 for index in 0..2 { 197 let selected = selected_request(&ready, index as u8 + 1, &policy, index); 198 let candidate = if index == 0 { 199 Some(&ready.candidate) 200 } else { 201 None 202 }; 203 let DeliveryAttemptOutcome::Claimed(attempt) = ready 204 .store 205 .prepare_delivery_attempt(selected, copy(&ready.proof), candidate) 206 .await 207 .unwrap() 208 else { 209 panic!("fresh selected attempt required") 210 }; 211 captured( 212 ready 213 .store 214 .record_delivery_observation( 215 ready.binding.clone(), 216 copy(&ready.proof), 217 synthetic_accepted(&attempt), 218 ) 219 .await 220 .unwrap(), 221 ); 222 let e = ready.work.evidence(); 223 let snapshot = ready 224 .store 225 .operation_snapshot( 226 ready.binding.clone(), 227 e.operation_id(), 228 None, 229 copy(&ready.proof), 230 ) 231 .await 232 .unwrap() 233 .unwrap(); 234 assert_eq!(snapshot.publication_accepted(ready.work), Ok(index == 1)); 235 let receipt = radroots_execution::OperationReceipt::<radroots_domain::ChatMessage>::try_from_snapshot(&snapshot).unwrap(); 236 assert_eq!( 237 receipt 238 .recovery() 239 .contains(&radroots_execution::RecoveryRequirement::ResumeDelivery), 240 index == 0 241 ); 242 } 243 ready.store.close().await.unwrap(); 244 } 245 246 #[tokio::test] 247 async fn a_fresh_delivery_deadline_after_first_issuance_expiry_reuses_original_bytes_and_policy() { 248 let ready = Ready::new(&[String::from("ws://127.0.0.1:9")], 246).await; 249 let original = ready.retain(1).await; 250 ready 251 .fixture 252 .clock 253 .now 254 .store(ready.work.queued_at().get() + 121, Ordering::SeqCst); 255 let next = DeliveryIntent::new( 256 &"02".repeat(32), 257 original.intent().event_id(), 258 original.intent().targets(), 259 original.intent().policy(), 260 NonZeroU64::new((ready.work.queued_at().get() + 180) * 1000).unwrap(), 261 ) 262 .unwrap(); 263 assert_eq!( 264 next.publication_policy_digest(), 265 original.intent().publication_policy_digest() 266 ); 267 assert_ne!(next.digest(), original.intent().digest()); 268 let DeliveryAttemptOutcome::Claimed(next) = ready 269 .store 270 .prepare_delivery_attempt( 271 DeliveryAttemptRequest::new(ready.binding.clone(), ready.work, next).unwrap(), 272 copy(&ready.proof), 273 None, 274 ) 275 .await 276 .unwrap() 277 else { 278 panic!("current-access delivery may retain a fresh attempt without reissuing S") 279 }; 280 assert_eq!(next.raw_json(), original.raw_json()); 281 assert_eq!(next.target_ordinals(), &[2]); 282 let retained = ready 283 .query(&ready.store, original.intent().request_id()) 284 .await; 285 assert_eq!( 286 retained.intent().deadline_unix_ms(), 287 original.intent().deadline_unix_ms() 288 ); 289 let mut database = ready.fixture.database().await; 290 let counts:(i64,i64,i64)=sqlx::query_as("SELECT (SELECT count(*) FROM signing_attempts),(SELECT count(*) FROM artifact_evidence),(SELECT count(*) FROM chat_messages)").fetch_one(&mut database).await.unwrap(); 291 assert_eq!(counts, (1, 1, 1)); 292 database.close().await.unwrap(); 293 ready.store.close().await.unwrap(); 294 } 295 296 #[tokio::test] 297 async fn actual_loopback_wrong_ack_timeout_disconnect_and_rejection_remain_distinct_after_reopen() { 298 for (case, expected) in [ 299 (0, DeliveryObservationKind::TimedOut), 300 (1, DeliveryObservationKind::Disconnected), 301 (2, DeliveryObservationKind::Rejected), 302 ] { 303 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); 304 let ready = Ready::new( 305 &[format!("ws://{}", listener.local_addr().unwrap())], 306 237 + case, 307 ) 308 .await; 309 let attempt = ready.retain(1).await; 310 let exact = String::from(attempt.raw_json()); 311 let (release, released) = tokio::sync::oneshot::channel(); 312 let server = tokio::spawn(async move { 313 let (tcp, _) = listener.accept().await.unwrap(); 314 let mut socket = accept_async(tcp).await.unwrap(); 315 let raw = tokio::time::timeout(Duration::from_secs(5), async { 316 loop { 317 if let Message::Text(raw) = socket.next().await.unwrap().unwrap() { 318 break raw.to_string(); 319 } 320 } 321 }) 322 .await 323 .unwrap(); 324 assert_eq!(raw, format!("[\"EVENT\",{exact}]")); 325 let event: serde_json::Value = serde_json::from_str(&raw).unwrap(); 326 match case { 327 0 => { 328 socket 329 .send(Message::Text( 330 json!(["OK", "ff".repeat(32), true, ""]).to_string().into(), 331 )) 332 .await 333 .unwrap(); 334 let _ = released.await; 335 } 336 1 => { 337 socket.close(None).await.unwrap(); 338 } 339 2 => { 340 socket 341 .send(Message::Text( 342 json!(["OK", event[1]["id"], false, "restricted: synthetic refusal"]) 343 .to_string() 344 .into(), 345 )) 346 .await 347 .unwrap(); 348 } 349 _ => unreachable!(), 350 } 351 }); 352 let request = attempt.transport_request().unwrap(); 353 let target = request.target_set().targets()[0].fingerprint(); 354 let transport = radroots_transport_nostr_native::create_transport(ready.config.clone()); 355 let result = transport.deliver(request.clone()).await; 356 let observation = match result { 357 Ok(receipt) => { 358 assert!(!receipt.is_satisfied(&request).unwrap()); 359 DeliveryObservation::from_receipt(&attempt, &receipt, target, None).unwrap() 360 } 361 Err(failure) => { 362 DeliveryObservation::from_failure(&attempt, &failure, target, None).unwrap() 363 } 364 }; 365 let _ = release.send(()); 366 server.await.unwrap(); 367 assert_eq!(observation.kind(), expected, "actual case {case}"); 368 let first = captured( 369 ready 370 .store 371 .record_delivery_observation(ready.binding.clone(), copy(&ready.proof), observation) 372 .await 373 .unwrap(), 374 ); 375 ready.store.close().await.unwrap(); 376 let reopened = ready.fixture.reopen(ready.input.clone()).await; 377 assert_eq!( 378 ready 379 .observations(&reopened, attempt.intent().request_id()) 380 .await, 381 vec![first] 382 ); 383 let e = ready.work.evidence(); 384 let snapshot = reopened 385 .operation_snapshot( 386 ready.binding.clone(), 387 e.operation_id(), 388 None, 389 copy(&ready.proof), 390 ) 391 .await 392 .unwrap() 393 .unwrap(); 394 assert!(snapshot.installed_message().is_some()); 395 let view = radroots_execution::OperationReceipt::<radroots_domain::ChatMessage>::try_from_snapshot(&snapshot).unwrap(); 396 assert!( 397 view.recovery() 398 .contains(&radroots_execution::RecoveryRequirement::ResumeDelivery) 399 ); 400 assert_eq!(view.delivery_observations().len(), 1); 401 reopened.close().await.unwrap(); 402 } 403 } 404 405 fn not_attempted(attempt: &DeliveryAttempt) -> DeliveryObservation { 406 let request = attempt.transport_request().unwrap(); 407 DeliveryObservation::from_failure( 408 attempt, 409 &radroots_transport::SinkFailure::transport_closed(&request), 410 request.target_set().targets()[0].fingerprint(), 411 None, 412 ) 413 .unwrap() 414 } 415 416 #[tokio::test] 417 async fn actual_observation_rollback_and_commit_uncertainty_preserve_retained_outer_and_business_facts() 418 { 419 for commit in [false, true] { 420 let ready = Ready::new(&[String::from("ws://127.0.0.1:9")], 240 + u8::from(commit)).await; 421 let attempt = ready.retain(1).await; 422 let observation = not_attempted(&attempt); 423 let mut database = ready.fixture.database().await; 424 if commit { 425 sqlx::raw_sql("CREATE TABLE observation_commit_fault (parent INTEGER REFERENCES workspace_grant(singleton) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER observation_commit_fault_trigger AFTER INSERT ON artifact_delivery_observations BEGIN INSERT INTO observation_commit_fault VALUES(2); END").execute(&mut database).await.unwrap(); 426 } else { 427 sqlx::raw_sql("CREATE TRIGGER observation_insert_fault BEFORE INSERT ON artifact_delivery_observations BEGIN SELECT RAISE(ABORT,'synthetic observation fault'); END").execute(&mut database).await.unwrap(); 428 } 429 database.close().await.unwrap(); 430 let result = ready 431 .store 432 .record_delivery_observation( 433 ready.binding.clone(), 434 copy(&ready.proof), 435 observation.clone(), 436 ) 437 .await; 438 if commit { 439 assert!(matches!(result, Ok(DeliveryObservationOutcome::Unknown(_)))); 440 } else { 441 assert!(matches!(result, Err(OperationError::StorageFailure))); 442 } 443 ready.store.close().await.unwrap(); 444 let mut database = ready.fixture.database().await; 445 if commit { 446 sqlx::raw_sql("DROP TRIGGER observation_commit_fault_trigger; DROP TABLE observation_commit_fault").execute(&mut database).await.unwrap(); 447 } else { 448 sqlx::query("DROP TRIGGER observation_insert_fault") 449 .execute(&mut database) 450 .await 451 .unwrap(); 452 } 453 let counts:(i64,i64,i64)=sqlx::query_as("SELECT (SELECT count(*) FROM artifact_delivery_attempts),(SELECT count(*) FROM artifact_delivery_observations),(SELECT count(*) FROM chat_messages)").fetch_one(&mut database).await.unwrap(); 454 assert_eq!(counts, (1, 0, 1)); 455 database.close().await.unwrap(); 456 let reopened = ready.fixture.reopen(ready.input.clone()).await; 457 assert_eq!( 458 ready 459 .query(&reopened, attempt.intent().request_id()) 460 .await 461 .raw_json(), 462 attempt.raw_json() 463 ); 464 assert!( 465 ready 466 .observations(&reopened, attempt.intent().request_id()) 467 .await 468 .is_empty() 469 ); 470 assert_eq!( 471 captured( 472 reopened 473 .record_delivery_observation( 474 ready.binding.clone(), 475 copy(&ready.proof), 476 observation.clone() 477 ) 478 .await 479 .unwrap() 480 ), 481 observation 482 ); 483 reopened.close().await.unwrap(); 484 } 485 } 486 487 #[tokio::test] 488 async fn corrupted_delivery_ciphertext_and_oversized_observation_refuse_the_complete_snapshot() { 489 let ready = Ready::new(&[String::from("ws://127.0.0.1:9")], 242).await; 490 let attempt = ready.retain(1).await; 491 captured( 492 ready 493 .store 494 .record_delivery_observation( 495 ready.binding.clone(), 496 copy(&ready.proof), 497 not_attempted(&attempt), 498 ) 499 .await 500 .unwrap(), 501 ); 502 let mut database = ready.fixture.database().await; 503 let original: Vec<u8> = 504 sqlx::query_scalar("SELECT protected FROM artifact_delivery_parts WHERE part=1") 505 .fetch_one(&mut database) 506 .await 507 .unwrap(); 508 let mut wrong = original.clone(); 509 wrong[0] ^= 1; 510 sqlx::query("UPDATE artifact_delivery_parts SET protected=? WHERE part=1") 511 .bind(wrong) 512 .execute(&mut database) 513 .await 514 .unwrap(); 515 let e = ready.work.evidence(); 516 assert!( 517 ready 518 .store 519 .operation_snapshot( 520 ready.binding.clone(), 521 e.operation_id(), 522 None, 523 copy(&ready.proof) 524 ) 525 .await 526 .is_err() 527 ); 528 sqlx::query("UPDATE artifact_delivery_parts SET protected=? WHERE part=1") 529 .bind(original) 530 .execute(&mut database) 531 .await 532 .unwrap(); 533 let original: Vec<u8> = 534 sqlx::query_scalar("SELECT protected FROM artifact_delivery_observations") 535 .fetch_one(&mut database) 536 .await 537 .unwrap(); 538 sqlx::query("PRAGMA ignore_check_constraints=ON") 539 .execute(&mut database) 540 .await 541 .unwrap(); 542 sqlx::query("UPDATE artifact_delivery_observations SET protected=zeroblob(262145)") 543 .execute(&mut database) 544 .await 545 .unwrap(); 546 assert!( 547 ready 548 .store 549 .operation_snapshot( 550 ready.binding.clone(), 551 e.operation_id(), 552 None, 553 copy(&ready.proof) 554 ) 555 .await 556 .is_err() 557 ); 558 sqlx::query("UPDATE artifact_delivery_observations SET protected=?") 559 .bind(original) 560 .execute(&mut database) 561 .await 562 .unwrap(); 563 database.close().await.unwrap(); 564 let snapshot = ready 565 .store 566 .operation_snapshot( 567 ready.binding.clone(), 568 e.operation_id(), 569 None, 570 copy(&ready.proof), 571 ) 572 .await 573 .unwrap() 574 .unwrap(); 575 assert_eq!(snapshot.delivery_attempts().len(), 1); 576 assert_eq!(snapshot.delivery_observations().len(), 1); 577 ready.store.close().await.unwrap(); 578 }