receipt_workflow_faults.rs (23483B)
1 use super::*; 2 use radroots_execution::{Admission, Code, session::SessionState}; 3 use radroots_signing::{ 4 issuance::{SigningIntent, SigningOutcome}, 5 provider::{PrivateCiphertext, PrivateText, ProviderCall, ProviderState}, 6 }; 7 8 type SigningPause = (oneshot::Sender<()>, oneshot::Receiver<()>); 9 #[derive(Clone, Copy, Default)] 10 enum CryptoFault { 11 #[default] 12 None, 13 EncryptionUnavailable, 14 DecryptionUnavailable, 15 WrongPlaintext, 16 } 17 struct RecordedSigner { 18 local: LocalSigner, 19 calls: AtomicU64, 20 uncertain: Option<PublicKey>, 21 pause: Mutex<Option<SigningPause>>, 22 crypto_fault: Mutex<CryptoFault>, 23 crypto_calls: AtomicU64, 24 } 25 impl RecordedSigner { 26 fn new(uncertain: Option<PublicKey>) -> Arc<Self> { 27 Arc::new(Self { 28 local: LocalSigner::new( 29 parse_secret_key( 30 "0000000000000000000000000000000000000000000000000000000000000001", 31 ) 32 .unwrap(), 33 ) 34 .unwrap() 35 .with_nonce_source(Arc::new(OsNonceSource)), 36 calls: AtomicU64::new(0), 37 uncertain, 38 pause: Mutex::new(None), 39 crypto_fault: Mutex::new(CryptoFault::None), 40 crypto_calls: AtomicU64::new(0), 41 }) 42 } 43 } 44 impl Signer for RecordedSigner { 45 fn status( 46 &self, 47 ) -> BoxFuture<'_, Result<radroots_signing::SignerStatus, radroots_signing::Error>> { 48 self.local.status() 49 } 50 fn sign( 51 &self, 52 request: radroots_signing::SignRequest, 53 ) -> BoxFuture<'_, Result<radroots_signing::SignReceipt, radroots_signing::Error>> { 54 self.local.sign(request) 55 } 56 } 57 impl ParticipantProvider for RecordedSigner { 58 fn provider_state(&self) -> BoxFuture<'_, Result<ProviderState, ProviderError>> { 59 Box::pin(async move { 60 use radroots_signing::provider::PrivateCryptoCapability::Unavailable; 61 let state = self.local.provider_state().await?; 62 let fault = *self.crypto_fault.lock().unwrap(); 63 let (encryption, decryption) = match fault { 64 CryptoFault::EncryptionUnavailable => (Unavailable, state.decryption()), 65 CryptoFault::DecryptionUnavailable => (state.encryption(), Unavailable), 66 CryptoFault::None | CryptoFault::WrongPlaintext => return Ok(state), 67 }; 68 Ok(ProviderState::new( 69 state.identity(), 70 encryption, 71 decryption, 72 state.signing_replay(), 73 ) 74 .with_exact_issuance(state.exact_issuance())) 75 }) 76 } 77 fn encrypt_private( 78 &self, 79 call: ProviderCall, 80 recipient: PublicKey, 81 text: PrivateText, 82 ) -> BoxFuture<'_, Result<PrivateCiphertext, ProviderError>> { 83 self.crypto_calls.fetch_add(1, Ordering::SeqCst); 84 self.local.encrypt_private(call, recipient, text) 85 } 86 fn decrypt_private( 87 &self, 88 call: ProviderCall, 89 sender: PublicKey, 90 text: PrivateCiphertext, 91 ) -> BoxFuture<'_, Result<PrivateText, ProviderError>> { 92 self.crypto_calls.fetch_add(1, Ordering::SeqCst); 93 Box::pin(async move { 94 let actual = self.local.decrypt_private(call, sender, text).await?; 95 if matches!( 96 *self.crypto_fault.lock().unwrap(), 97 CryptoFault::WrongPlaintext 98 ) { 99 PrivateText::new("injected post-decryption mismatch") 100 } else { 101 Ok(actual) 102 } 103 }) 104 } 105 fn sign_exact(&self, intent: SigningIntent) -> BoxFuture<'_, SigningOutcome> { 106 Box::pin(async move { 107 self.calls.fetch_add(1, Ordering::SeqCst); 108 if intent.recipient() == self.uncertain { 109 return SigningOutcome::unknown(intent.binding(), ProviderError::ProviderFailure); 110 } 111 let outcome = self.local.sign_exact(intent).await; 112 let pause = self.pause.lock().unwrap().take(); 113 if let Some((reached, resume)) = pause { 114 reached.send(()).unwrap(); 115 resume.await.unwrap(); 116 } 117 outcome 118 }) 119 } 120 } 121 struct Ready { 122 fixture: Fixture, 123 store: Arc<NativeWorkspace>, 124 client: Client, 125 scope: radroots::WorkspaceSession, 126 input: AttachmentInput, 127 proof: AttachmentProof, 128 signer: Arc<RecordedSigner>, 129 } 130 impl Ready { 131 async fn new(relay: &Relay, mode: StartMode, signer: Arc<RecordedSigner>) -> Self { 132 let fixture = Fixture::new(); 133 fixture.clock.live.store(true, Ordering::SeqCst); 134 let (store, input, proof, _) = fixture.create_with_expiry(now() + 600).await; 135 let cache = fixture.path.parent().unwrap().join("cache"); 136 std::fs::create_dir(&cache).unwrap(); 137 let options = OpenOptions::new(Paths::from_directory(cache).unwrap(), OpenMode::Create) 138 .with_source_generation(SourceGeneration::new([19; 32]).unwrap(), 1) 139 .unwrap(); 140 let client = Client::open( 141 NativeConfig::public_read(options) 142 .with_workspace(store.as_ref().clone()) 143 .with_nostr( 144 relay.config(), 145 SatisfactionPolicy::new(SatisfactionClass::Accepted, TargetPolicy::all()), 146 ) 147 .unwrap(), 148 ) 149 .await 150 .unwrap(); 151 let scope = client.attach(input.clone(), copy(&proof)).await.unwrap(); 152 client 153 .attach_signer(SignerHandle::new(signer.clone(), Arc::new(Callbacks))) 154 .await 155 .unwrap(); 156 client.start(mode).await.unwrap(); 157 Self { 158 fixture, 159 store, 160 client, 161 scope, 162 input, 163 proof, 164 signer, 165 } 166 } 167 fn context(&self, id: u8, not_after: u64) -> radroots::CommandContext { 168 self.scope 169 .command_context_with_correlation( 170 OperationId::from_bytes([id; 32]), 171 None, 172 None, 173 UnixSeconds::new(not_after).unwrap(), 174 Some(CorrelationId::new("workflow-fault").unwrap()), 175 ) 176 .unwrap() 177 } 178 async fn snapshot(&self, id: u8) -> radroots_storage::command::OperationSnapshot { 179 self.store 180 .operation_snapshot( 181 self.scope.binding().clone(), 182 OperationId::from_bytes([id; 32]), 183 None, 184 copy(&self.proof), 185 ) 186 .await 187 .unwrap() 188 .unwrap() 189 } 190 async fn writer_is_retained(&self) { 191 assert!(matches!( 192 NativeWorkspace::open_existing( 193 &self.fixture.path, 194 self.input.clone(), 195 self.fixture.crypto(), 196 self.fixture.clock.clone() 197 ) 198 .await, 199 Err(WorkspaceError::WriterActive) 200 )); 201 } 202 } 203 fn inquiry() -> InquireInput { 204 InquireInput::new( 205 ThreadId::from_bytes([34; 32]), 206 PublicKey::from_bytes(route_key(2).public_key().to_bytes()).unwrap(), 207 None, 208 "ordinary fault path", 209 ) 210 .unwrap() 211 } 212 async fn stop(relay: Relay) { 213 relay.stop.send(()).unwrap(); 214 relay.task.await.unwrap(); 215 } 216 217 #[tokio::test] 218 async fn planning_refuses_wrong_participant_missing_private_methods_and_altered_decryption_before_t1() 219 { 220 let relay = Relay::new().await; 221 for (index, fault) in [ 222 CryptoFault::None, 223 CryptoFault::EncryptionUnavailable, 224 CryptoFault::DecryptionUnavailable, 225 CryptoFault::WrongPlaintext, 226 ] 227 .into_iter() 228 .enumerate() 229 { 230 let mut signer = RecordedSigner::new(None); 231 if index == 0 { 232 Arc::get_mut(&mut signer).unwrap().local = LocalSigner::new( 233 parse_secret_key( 234 "0000000000000000000000000000000000000000000000000000000000000002", 235 ) 236 .unwrap(), 237 ) 238 .unwrap() 239 .with_nonce_source(Arc::new(OsNonceSource)); 240 } 241 *signer.crypto_fault.lock().unwrap() = fault; 242 let ready = Ready::new(&relay, StartMode::Online, signer).await; 243 let id = 210 + index as u8; 244 let error = ready 245 .client 246 .inbox() 247 .inquire(ready.context(id, now() + 120), inquiry()) 248 .await 249 .unwrap_err(); 250 assert_eq!(error.context().admission(), Admission::Confirmed); 251 assert_eq!( 252 error.code(), 253 if index == 3 { 254 Code::RecoveryRequired 255 } else { 256 Code::Unauthorized 257 } 258 ); 259 assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 0); 260 assert_eq!( 261 ready.signer.crypto_calls.load(Ordering::SeqCst), 262 if index == 3 { 2 } else { 0 } 263 ); 264 assert!(ready.snapshot(id).await.bound_plan().is_none()); 265 assert_eq!(relay.count.load(Ordering::SeqCst), 0); 266 ready.client.close().await.unwrap(); 267 } 268 stop(relay).await; 269 } 270 271 #[tokio::test] 272 async fn explicit_resume_refuses_foreign_and_closed_attachments_before_read_or_dispatch() { 273 let relay = Relay::new().await; 274 let ready = Ready::new(&relay, StartMode::Online, RecordedSigner::new(None)).await; 275 let foreign = Ready::new(&relay, StartMode::Online, RecordedSigner::new(None)).await; 276 let operation = OperationId::from_bytes([214; 32]); 277 let error = ready 278 .client 279 .operations(&foreign.scope) 280 .resume(operation, None) 281 .await 282 .unwrap_err(); 283 assert_eq!(error.code(), Code::Unauthorized); 284 assert_eq!(error.context().admission(), Admission::NotAdmitted); 285 ready.scope.close(); 286 let error = ready 287 .client 288 .operations(&ready.scope) 289 .resume(operation, None) 290 .await 291 .unwrap_err(); 292 assert_eq!(error.code(), Code::CapabilityUnavailable); 293 assert_eq!(error.context().admission(), Admission::NotAdmitted); 294 assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 0); 295 assert_eq!(relay.count.load(Ordering::SeqCst), 0); 296 ready.client.close().await.unwrap(); 297 foreign.client.close().await.unwrap(); 298 stop(relay).await; 299 } 300 301 #[tokio::test] 302 async fn local_only_admits_without_crypto_network_or_mode_escalation() { 303 let relay = Relay::new().await; 304 let ready = Ready::new(&relay, StartMode::LocalOnly, RecordedSigner::new(None)).await; 305 let absent = ready 306 .client 307 .operations(&ready.scope) 308 .resume(OperationId::from_bytes([202; 32]), None) 309 .await 310 .unwrap_err(); 311 assert_eq!(absent.code(), Code::MissingHistory); 312 assert_eq!(absent.context().admission(), Admission::NotAdmitted); 313 let receipt = ready 314 .client 315 .inbox() 316 .inquire(ready.context(202, now() + 120), inquiry()) 317 .await 318 .unwrap(); 319 assert!(receipt.durable()); 320 assert!(receipt.business_result().is_none()); 321 assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 0); 322 assert_eq!(relay.count.load(Ordering::SeqCst), 0); 323 assert!(ready.client.start(StartMode::Online).await.is_err()); 324 assert!( 325 ready 326 .client 327 .operations(&ready.scope) 328 .get(OperationId::from_bytes([202; 32])) 329 .await 330 .unwrap() 331 .is_some() 332 ); 333 ready.client.close().await.unwrap(); 334 stop(relay).await; 335 } 336 337 #[tokio::test] 338 async fn explicit_delivery_retry_after_first_issue_expiry_preserves_original_bytes_and_criterion() { 339 let mut relay = Relay::new().await; 340 relay.accepting.store(false, Ordering::SeqCst); 341 let ready = Ready::new(&relay, StartMode::Online, RecordedSigner::new(None)).await; 342 let not_after = now() + 10; 343 let first = ready 344 .client 345 .inbox() 346 .inquire(ready.context(203, not_after), inquiry()) 347 .await 348 .unwrap(); 349 assert!(first.business_result().is_some()); 350 let initial_count = relay.count.load(Ordering::SeqCst); 351 assert!((1..=2).contains(&initial_count)); 352 let snapshot = ready.snapshot(203).await; 353 assert!( 354 snapshot 355 .publications() 356 .iter() 357 .all(|work| !snapshot.publication_accepted(*work).unwrap()) 358 ); 359 let mut original = Vec::new(); 360 assert_eq!(snapshot.delivery_attempts().len(), 2); 361 for attempt in snapshot.delivery_attempts() { 362 let retained = ready 363 .store 364 .delivery_attempt( 365 ready.scope.binding().clone(), 366 OperationId::from_bytes([203; 32]), 367 first.command_digest(), 368 attempt.work().target(), 369 attempt.intent().request_id().clone(), 370 copy(&ready.proof), 371 ) 372 .await 373 .unwrap() 374 .unwrap(); 375 original.push(retained.raw_json().to_owned()); 376 } 377 while relay.events.try_recv().is_ok() {} 378 let replay = ready 379 .client 380 .inbox() 381 .inquire(ready.context(203, not_after), inquiry()) 382 .await 383 .unwrap(); 384 assert_eq!(replay.operation_checkpoint(), first.operation_checkpoint()); 385 assert_eq!(relay.count.load(Ordering::SeqCst), initial_count); 386 tokio::time::sleep(Duration::from_secs(not_after.saturating_sub(now()) + 1)).await; 387 relay.accepting.store(true, Ordering::SeqCst); 388 let resumed = ready 389 .client 390 .operations(&ready.scope) 391 .resume( 392 OperationId::from_bytes([203; 32]), 393 Some(first.command_digest()), 394 ) 395 .await 396 .unwrap(); 397 assert_eq!(resumed.issuance_window(), first.issuance_window()); 398 assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 2); 399 let mut retried = Vec::new(); 400 for _ in 0..2 { 401 retried.push(relay.events.recv().await.unwrap()); 402 } 403 retried.sort(); 404 original.sort(); 405 assert!( 406 retried == original, 407 "retry changed the exact original outer representation" 408 ); 409 let snapshot = ready.snapshot(203).await; 410 assert_eq!(snapshot.delivery_attempts().len(), 4); 411 assert!( 412 snapshot 413 .publications() 414 .iter() 415 .all(|work| snapshot.publication_accepted(*work).unwrap()) 416 ); 417 for work in snapshot.publications() { 418 let attempts: Vec<_> = snapshot 419 .delivery_attempts() 420 .iter() 421 .filter(|attempt| { 422 attempt.work() == radroots_storage::command::PublicationArtifact::Private(*work) 423 }) 424 .collect(); 425 assert_eq!( 426 attempts[0].intent().publication_policy_digest(), 427 attempts[1].intent().publication_policy_digest() 428 ); 429 assert_eq!(attempts[0].envelope_digest(), attempts[1].envelope_digest()); 430 } 431 ready 432 .client 433 .operations(&ready.scope) 434 .resume(OperationId::from_bytes([203; 32]), None) 435 .await 436 .unwrap(); 437 assert_eq!(relay.count.load(Ordering::SeqCst), initial_count + 2); 438 ready.client.close().await.unwrap(); 439 stop(relay).await; 440 } 441 442 #[tokio::test] 443 async fn uncertain_second_recipient_never_reissues_and_preserves_actual_first_copy_installation() { 444 let relay = Relay::new().await; 445 let uncertain = PublicKey::from_bytes(route_key(2).public_key().to_bytes()).unwrap(); 446 let ready = Ready::new( 447 &relay, 448 StartMode::Online, 449 RecordedSigner::new(Some(uncertain)), 450 ) 451 .await; 452 let first = ready 453 .client 454 .inbox() 455 .inquire(ready.context(204, now() + 120), inquiry()) 456 .await 457 .unwrap(); 458 assert!(first.business_result().is_some()); 459 let snapshot = ready.snapshot(204).await; 460 assert_eq!( 461 snapshot 462 .artifacts() 463 .iter() 464 .filter(|recipient| recipient.obtained().is_some()) 465 .count(), 466 1 467 ); 468 assert_eq!(snapshot.publications().len(), 1); 469 assert_eq!(relay.count.load(Ordering::SeqCst), 1); 470 ready 471 .client 472 .operations(&ready.scope) 473 .resume(OperationId::from_bytes([204; 32]), None) 474 .await 475 .unwrap(); 476 assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 2); 477 assert_eq!(relay.count.load(Ordering::SeqCst), 1); 478 ready.client.close().await.unwrap(); 479 stop(relay).await; 480 } 481 482 #[tokio::test] 483 async fn failed_obtained_capture_blocks_close_retains_writer_and_retries_without_signing_again() { 484 let relay = Relay::new().await; 485 let ready = Ready::new(&relay, StartMode::Online, RecordedSigner::new(None)).await; 486 let mut db = ready.fixture.database().await; 487 sqlx::query("CREATE TRIGGER capture_insert_fault BEFORE INSERT ON artifact_evidence BEGIN SELECT RAISE(ABORT, 'capture_insert_fault'); END").execute(&mut db).await.unwrap(); 488 let error = ready 489 .client 490 .inbox() 491 .inquire(ready.context(205, now() + 120), inquiry()) 492 .await 493 .unwrap_err(); 494 assert_eq!(error.code(), Code::StorageFailure); 495 assert_eq!(error.retry(), radroots::RetryClass::ManualReconciliation); 496 assert_eq!(error.context().admission(), Admission::Confirmed); 497 assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 1); 498 let blocked = ready 499 .client 500 .inbox() 501 .inquire(ready.context(209, now() + 120), inquiry()) 502 .await 503 .unwrap_err(); 504 assert_eq!(blocked.code(), Code::StorageFailure); 505 assert_eq!(blocked.retry(), radroots::RetryClass::ManualReconciliation); 506 assert_eq!( 507 blocked.context().operation_id(), 508 Some(OperationId::from_bytes([205; 32])) 509 ); 510 assert_eq!(blocked.context().admission(), Admission::Confirmed); 511 assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 1); 512 let error = ready.client.close().await.unwrap_err(); 513 assert_eq!(error.code(), Code::StorageFailure); 514 assert_eq!(error.retry(), radroots::RetryClass::ManualReconciliation); 515 assert_eq!( 516 error.context().operation_id(), 517 Some(OperationId::from_bytes([205; 32])) 518 ); 519 assert_eq!(error.context().admission(), Admission::Confirmed); 520 assert_eq!(ready.client.state(), SessionState::Closing); 521 ready.writer_is_retained().await; 522 sqlx::query("DROP TRIGGER capture_insert_fault") 523 .execute(&mut db) 524 .await 525 .unwrap(); 526 db.close().await.unwrap(); 527 ready.client.close().await.unwrap(); 528 assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 1); 529 assert_eq!(relay.count.load(Ordering::SeqCst), 0); 530 let reopened = ready.fixture.reopen(ready.input).await; 531 let snapshot = reopened 532 .operation_snapshot( 533 ready.scope.binding().clone(), 534 OperationId::from_bytes([205; 32]), 535 None, 536 copy(&ready.proof), 537 ) 538 .await 539 .unwrap() 540 .unwrap(); 541 assert_eq!( 542 snapshot 543 .artifacts() 544 .iter() 545 .filter(|recipient| recipient.obtained().is_some()) 546 .count(), 547 1 548 ); 549 reopened.close_workspace().await.unwrap(); 550 stop(relay).await; 551 } 552 553 #[tokio::test] 554 async fn explicit_resume_recovers_unknown_capture_under_same_writer_before_continuing_original() { 555 let relay = Relay::new().await; 556 let ready = Ready::new(&relay, StartMode::Online, RecordedSigner::new(None)).await; 557 let mut db = ready.fixture.database().await; 558 sqlx::query("PRAGMA foreign_keys=ON") 559 .execute(&mut db) 560 .await 561 .unwrap(); 562 sqlx::query("CREATE TABLE capture_commit_fault (parent INTEGER REFERENCES workspace_grant(singleton) DEFERRABLE INITIALLY DEFERRED)").execute(&mut db).await.unwrap(); 563 sqlx::query("CREATE TRIGGER capture_commit_fault AFTER INSERT ON artifact_evidence BEGIN INSERT INTO capture_commit_fault(parent) VALUES(2); END").execute(&mut db).await.unwrap(); 564 let error = ready 565 .client 566 .inbox() 567 .inquire(ready.context(207, now() + 120), inquiry()) 568 .await 569 .unwrap_err(); 570 assert_eq!(error.code(), Code::RecoveryRequired); 571 assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 1); 572 assert!( 573 ready 574 .store 575 .operation_snapshot( 576 ready.scope.binding().clone(), 577 OperationId::from_bytes([207; 32]), 578 None, 579 copy(&ready.proof) 580 ) 581 .await 582 .is_err() 583 ); 584 assert_eq!( 585 ready 586 .client 587 .operations(&ready.scope) 588 .resume(OperationId::from_bytes([207; 32]), None) 589 .await 590 .unwrap_err() 591 .code(), 592 Code::RecoveryRequired 593 ); 594 ready.writer_is_retained().await; 595 sqlx::query("DROP TRIGGER capture_commit_fault") 596 .execute(&mut db) 597 .await 598 .unwrap(); 599 sqlx::query("DROP TABLE capture_commit_fault") 600 .execute(&mut db) 601 .await 602 .unwrap(); 603 db.close().await.unwrap(); 604 let receipt = ready 605 .client 606 .operations(&ready.scope) 607 .resume(OperationId::from_bytes([207; 32]), None) 608 .await 609 .unwrap(); 610 assert!(receipt.business_result().is_some()); 611 assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 2); 612 let snapshot = ready.snapshot(207).await; 613 assert!( 614 snapshot 615 .artifacts() 616 .iter() 617 .all(|recipient| recipient.obtained().is_some()) 618 ); 619 assert!( 620 snapshot 621 .publications() 622 .iter() 623 .all(|work| snapshot.publication_accepted(*work).unwrap()) 624 ); 625 ready.client.close().await.unwrap(); 626 stop(relay).await; 627 } 628 629 #[tokio::test] 630 async fn lost_observer_and_cancelled_close_preserve_actual_obtained_signature_until_capture() { 631 let relay = Relay::new().await; 632 let signer = RecordedSigner::new(None); 633 let (reached, produced) = oneshot::channel(); 634 let (resume, released) = oneshot::channel(); 635 *signer.pause.lock().unwrap() = Some((reached, released)); 636 let ready = Ready::new(&relay, StartMode::Online, signer).await; 637 let context = ready.context(206, now() + 120); 638 let client = ready.client.clone(); 639 let observer = tokio::spawn(async move { client.inbox().inquire(context, inquiry()).await }); 640 tokio::time::timeout(Duration::from_secs(5), produced) 641 .await 642 .unwrap() 643 .unwrap(); 644 observer.abort(); 645 assert!(observer.await.unwrap_err().is_cancelled()); 646 { 647 let mut close = Box::pin(ready.client.close()); 648 assert!(futures::poll!(&mut close).is_pending()); 649 } 650 assert_eq!(ready.client.state(), SessionState::Closing); 651 ready.writer_is_retained().await; 652 resume.send(()).unwrap(); 653 tokio::time::timeout(Duration::from_secs(5), ready.client.close()) 654 .await 655 .unwrap() 656 .unwrap(); 657 assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 1); 658 let reopened = ready.fixture.reopen(ready.input).await; 659 let snapshot = reopened 660 .operation_snapshot( 661 ready.scope.binding().clone(), 662 OperationId::from_bytes([206; 32]), 663 None, 664 copy(&ready.proof), 665 ) 666 .await 667 .unwrap() 668 .unwrap(); 669 assert!( 670 snapshot 671 .artifacts() 672 .iter() 673 .any(|recipient| recipient.obtained().is_some()) 674 ); 675 reopened.close_workspace().await.unwrap(); 676 stop(relay).await; 677 }