lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

receipt_workflow_faults.rs (23483B)


      1 use super::*;
      2 use radroots_execution::{Admission, Code, session::SessionState};
      3 use radroots_signing::{
      4     issuance::{SigningIntent, SigningOutcome},
      5     provider::{PrivateCiphertext, PrivateText, ProviderCall, ProviderState},
      6 };
      7 
      8 type SigningPause = (oneshot::Sender<()>, oneshot::Receiver<()>);
      9 #[derive(Clone, Copy, Default)]
     10 enum CryptoFault {
     11     #[default]
     12     None,
     13     EncryptionUnavailable,
     14     DecryptionUnavailable,
     15     WrongPlaintext,
     16 }
     17 struct RecordedSigner {
     18     local: LocalSigner,
     19     calls: AtomicU64,
     20     uncertain: Option<PublicKey>,
     21     pause: Mutex<Option<SigningPause>>,
     22     crypto_fault: Mutex<CryptoFault>,
     23     crypto_calls: AtomicU64,
     24 }
     25 impl RecordedSigner {
     26     fn new(uncertain: Option<PublicKey>) -> Arc<Self> {
     27         Arc::new(Self {
     28             local: LocalSigner::new(
     29                 parse_secret_key(
     30                     "0000000000000000000000000000000000000000000000000000000000000001",
     31                 )
     32                 .unwrap(),
     33             )
     34             .unwrap()
     35             .with_nonce_source(Arc::new(OsNonceSource)),
     36             calls: AtomicU64::new(0),
     37             uncertain,
     38             pause: Mutex::new(None),
     39             crypto_fault: Mutex::new(CryptoFault::None),
     40             crypto_calls: AtomicU64::new(0),
     41         })
     42     }
     43 }
     44 impl Signer for RecordedSigner {
     45     fn status(
     46         &self,
     47     ) -> BoxFuture<'_, Result<radroots_signing::SignerStatus, radroots_signing::Error>> {
     48         self.local.status()
     49     }
     50     fn sign(
     51         &self,
     52         request: radroots_signing::SignRequest,
     53     ) -> BoxFuture<'_, Result<radroots_signing::SignReceipt, radroots_signing::Error>> {
     54         self.local.sign(request)
     55     }
     56 }
     57 impl ParticipantProvider for RecordedSigner {
     58     fn provider_state(&self) -> BoxFuture<'_, Result<ProviderState, ProviderError>> {
     59         Box::pin(async move {
     60             use radroots_signing::provider::PrivateCryptoCapability::Unavailable;
     61             let state = self.local.provider_state().await?;
     62             let fault = *self.crypto_fault.lock().unwrap();
     63             let (encryption, decryption) = match fault {
     64                 CryptoFault::EncryptionUnavailable => (Unavailable, state.decryption()),
     65                 CryptoFault::DecryptionUnavailable => (state.encryption(), Unavailable),
     66                 CryptoFault::None | CryptoFault::WrongPlaintext => return Ok(state),
     67             };
     68             Ok(ProviderState::new(
     69                 state.identity(),
     70                 encryption,
     71                 decryption,
     72                 state.signing_replay(),
     73             )
     74             .with_exact_issuance(state.exact_issuance()))
     75         })
     76     }
     77     fn encrypt_private(
     78         &self,
     79         call: ProviderCall,
     80         recipient: PublicKey,
     81         text: PrivateText,
     82     ) -> BoxFuture<'_, Result<PrivateCiphertext, ProviderError>> {
     83         self.crypto_calls.fetch_add(1, Ordering::SeqCst);
     84         self.local.encrypt_private(call, recipient, text)
     85     }
     86     fn decrypt_private(
     87         &self,
     88         call: ProviderCall,
     89         sender: PublicKey,
     90         text: PrivateCiphertext,
     91     ) -> BoxFuture<'_, Result<PrivateText, ProviderError>> {
     92         self.crypto_calls.fetch_add(1, Ordering::SeqCst);
     93         Box::pin(async move {
     94             let actual = self.local.decrypt_private(call, sender, text).await?;
     95             if matches!(
     96                 *self.crypto_fault.lock().unwrap(),
     97                 CryptoFault::WrongPlaintext
     98             ) {
     99                 PrivateText::new("injected post-decryption mismatch")
    100             } else {
    101                 Ok(actual)
    102             }
    103         })
    104     }
    105     fn sign_exact(&self, intent: SigningIntent) -> BoxFuture<'_, SigningOutcome> {
    106         Box::pin(async move {
    107             self.calls.fetch_add(1, Ordering::SeqCst);
    108             if intent.recipient() == self.uncertain {
    109                 return SigningOutcome::unknown(intent.binding(), ProviderError::ProviderFailure);
    110             }
    111             let outcome = self.local.sign_exact(intent).await;
    112             let pause = self.pause.lock().unwrap().take();
    113             if let Some((reached, resume)) = pause {
    114                 reached.send(()).unwrap();
    115                 resume.await.unwrap();
    116             }
    117             outcome
    118         })
    119     }
    120 }
    121 struct Ready {
    122     fixture: Fixture,
    123     store: Arc<NativeWorkspace>,
    124     client: Client,
    125     scope: radroots::WorkspaceSession,
    126     input: AttachmentInput,
    127     proof: AttachmentProof,
    128     signer: Arc<RecordedSigner>,
    129 }
    130 impl Ready {
    131     async fn new(relay: &Relay, mode: StartMode, signer: Arc<RecordedSigner>) -> Self {
    132         let fixture = Fixture::new();
    133         fixture.clock.live.store(true, Ordering::SeqCst);
    134         let (store, input, proof, _) = fixture.create_with_expiry(now() + 600).await;
    135         let cache = fixture.path.parent().unwrap().join("cache");
    136         std::fs::create_dir(&cache).unwrap();
    137         let options = OpenOptions::new(Paths::from_directory(cache).unwrap(), OpenMode::Create)
    138             .with_source_generation(SourceGeneration::new([19; 32]).unwrap(), 1)
    139             .unwrap();
    140         let client = Client::open(
    141             NativeConfig::public_read(options)
    142                 .with_workspace(store.as_ref().clone())
    143                 .with_nostr(
    144                     relay.config(),
    145                     SatisfactionPolicy::new(SatisfactionClass::Accepted, TargetPolicy::all()),
    146                 )
    147                 .unwrap(),
    148         )
    149         .await
    150         .unwrap();
    151         let scope = client.attach(input.clone(), copy(&proof)).await.unwrap();
    152         client
    153             .attach_signer(SignerHandle::new(signer.clone(), Arc::new(Callbacks)))
    154             .await
    155             .unwrap();
    156         client.start(mode).await.unwrap();
    157         Self {
    158             fixture,
    159             store,
    160             client,
    161             scope,
    162             input,
    163             proof,
    164             signer,
    165         }
    166     }
    167     fn context(&self, id: u8, not_after: u64) -> radroots::CommandContext {
    168         self.scope
    169             .command_context_with_correlation(
    170                 OperationId::from_bytes([id; 32]),
    171                 None,
    172                 None,
    173                 UnixSeconds::new(not_after).unwrap(),
    174                 Some(CorrelationId::new("workflow-fault").unwrap()),
    175             )
    176             .unwrap()
    177     }
    178     async fn snapshot(&self, id: u8) -> radroots_storage::command::OperationSnapshot {
    179         self.store
    180             .operation_snapshot(
    181                 self.scope.binding().clone(),
    182                 OperationId::from_bytes([id; 32]),
    183                 None,
    184                 copy(&self.proof),
    185             )
    186             .await
    187             .unwrap()
    188             .unwrap()
    189     }
    190     async fn writer_is_retained(&self) {
    191         assert!(matches!(
    192             NativeWorkspace::open_existing(
    193                 &self.fixture.path,
    194                 self.input.clone(),
    195                 self.fixture.crypto(),
    196                 self.fixture.clock.clone()
    197             )
    198             .await,
    199             Err(WorkspaceError::WriterActive)
    200         ));
    201     }
    202 }
    203 fn inquiry() -> InquireInput {
    204     InquireInput::new(
    205         ThreadId::from_bytes([34; 32]),
    206         PublicKey::from_bytes(route_key(2).public_key().to_bytes()).unwrap(),
    207         None,
    208         "ordinary fault path",
    209     )
    210     .unwrap()
    211 }
    212 async fn stop(relay: Relay) {
    213     relay.stop.send(()).unwrap();
    214     relay.task.await.unwrap();
    215 }
    216 
    217 #[tokio::test]
    218 async fn planning_refuses_wrong_participant_missing_private_methods_and_altered_decryption_before_t1()
    219  {
    220     let relay = Relay::new().await;
    221     for (index, fault) in [
    222         CryptoFault::None,
    223         CryptoFault::EncryptionUnavailable,
    224         CryptoFault::DecryptionUnavailable,
    225         CryptoFault::WrongPlaintext,
    226     ]
    227     .into_iter()
    228     .enumerate()
    229     {
    230         let mut signer = RecordedSigner::new(None);
    231         if index == 0 {
    232             Arc::get_mut(&mut signer).unwrap().local = LocalSigner::new(
    233                 parse_secret_key(
    234                     "0000000000000000000000000000000000000000000000000000000000000002",
    235                 )
    236                 .unwrap(),
    237             )
    238             .unwrap()
    239             .with_nonce_source(Arc::new(OsNonceSource));
    240         }
    241         *signer.crypto_fault.lock().unwrap() = fault;
    242         let ready = Ready::new(&relay, StartMode::Online, signer).await;
    243         let id = 210 + index as u8;
    244         let error = ready
    245             .client
    246             .inbox()
    247             .inquire(ready.context(id, now() + 120), inquiry())
    248             .await
    249             .unwrap_err();
    250         assert_eq!(error.context().admission(), Admission::Confirmed);
    251         assert_eq!(
    252             error.code(),
    253             if index == 3 {
    254                 Code::RecoveryRequired
    255             } else {
    256                 Code::Unauthorized
    257             }
    258         );
    259         assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 0);
    260         assert_eq!(
    261             ready.signer.crypto_calls.load(Ordering::SeqCst),
    262             if index == 3 { 2 } else { 0 }
    263         );
    264         assert!(ready.snapshot(id).await.bound_plan().is_none());
    265         assert_eq!(relay.count.load(Ordering::SeqCst), 0);
    266         ready.client.close().await.unwrap();
    267     }
    268     stop(relay).await;
    269 }
    270 
    271 #[tokio::test]
    272 async fn explicit_resume_refuses_foreign_and_closed_attachments_before_read_or_dispatch() {
    273     let relay = Relay::new().await;
    274     let ready = Ready::new(&relay, StartMode::Online, RecordedSigner::new(None)).await;
    275     let foreign = Ready::new(&relay, StartMode::Online, RecordedSigner::new(None)).await;
    276     let operation = OperationId::from_bytes([214; 32]);
    277     let error = ready
    278         .client
    279         .operations(&foreign.scope)
    280         .resume(operation, None)
    281         .await
    282         .unwrap_err();
    283     assert_eq!(error.code(), Code::Unauthorized);
    284     assert_eq!(error.context().admission(), Admission::NotAdmitted);
    285     ready.scope.close();
    286     let error = ready
    287         .client
    288         .operations(&ready.scope)
    289         .resume(operation, None)
    290         .await
    291         .unwrap_err();
    292     assert_eq!(error.code(), Code::CapabilityUnavailable);
    293     assert_eq!(error.context().admission(), Admission::NotAdmitted);
    294     assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 0);
    295     assert_eq!(relay.count.load(Ordering::SeqCst), 0);
    296     ready.client.close().await.unwrap();
    297     foreign.client.close().await.unwrap();
    298     stop(relay).await;
    299 }
    300 
    301 #[tokio::test]
    302 async fn local_only_admits_without_crypto_network_or_mode_escalation() {
    303     let relay = Relay::new().await;
    304     let ready = Ready::new(&relay, StartMode::LocalOnly, RecordedSigner::new(None)).await;
    305     let absent = ready
    306         .client
    307         .operations(&ready.scope)
    308         .resume(OperationId::from_bytes([202; 32]), None)
    309         .await
    310         .unwrap_err();
    311     assert_eq!(absent.code(), Code::MissingHistory);
    312     assert_eq!(absent.context().admission(), Admission::NotAdmitted);
    313     let receipt = ready
    314         .client
    315         .inbox()
    316         .inquire(ready.context(202, now() + 120), inquiry())
    317         .await
    318         .unwrap();
    319     assert!(receipt.durable());
    320     assert!(receipt.business_result().is_none());
    321     assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 0);
    322     assert_eq!(relay.count.load(Ordering::SeqCst), 0);
    323     assert!(ready.client.start(StartMode::Online).await.is_err());
    324     assert!(
    325         ready
    326             .client
    327             .operations(&ready.scope)
    328             .get(OperationId::from_bytes([202; 32]))
    329             .await
    330             .unwrap()
    331             .is_some()
    332     );
    333     ready.client.close().await.unwrap();
    334     stop(relay).await;
    335 }
    336 
    337 #[tokio::test]
    338 async fn explicit_delivery_retry_after_first_issue_expiry_preserves_original_bytes_and_criterion() {
    339     let mut relay = Relay::new().await;
    340     relay.accepting.store(false, Ordering::SeqCst);
    341     let ready = Ready::new(&relay, StartMode::Online, RecordedSigner::new(None)).await;
    342     let not_after = now() + 10;
    343     let first = ready
    344         .client
    345         .inbox()
    346         .inquire(ready.context(203, not_after), inquiry())
    347         .await
    348         .unwrap();
    349     assert!(first.business_result().is_some());
    350     let initial_count = relay.count.load(Ordering::SeqCst);
    351     assert!((1..=2).contains(&initial_count));
    352     let snapshot = ready.snapshot(203).await;
    353     assert!(
    354         snapshot
    355             .publications()
    356             .iter()
    357             .all(|work| !snapshot.publication_accepted(*work).unwrap())
    358     );
    359     let mut original = Vec::new();
    360     assert_eq!(snapshot.delivery_attempts().len(), 2);
    361     for attempt in snapshot.delivery_attempts() {
    362         let retained = ready
    363             .store
    364             .delivery_attempt(
    365                 ready.scope.binding().clone(),
    366                 OperationId::from_bytes([203; 32]),
    367                 first.command_digest(),
    368                 attempt.work().target(),
    369                 attempt.intent().request_id().clone(),
    370                 copy(&ready.proof),
    371             )
    372             .await
    373             .unwrap()
    374             .unwrap();
    375         original.push(retained.raw_json().to_owned());
    376     }
    377     while relay.events.try_recv().is_ok() {}
    378     let replay = ready
    379         .client
    380         .inbox()
    381         .inquire(ready.context(203, not_after), inquiry())
    382         .await
    383         .unwrap();
    384     assert_eq!(replay.operation_checkpoint(), first.operation_checkpoint());
    385     assert_eq!(relay.count.load(Ordering::SeqCst), initial_count);
    386     tokio::time::sleep(Duration::from_secs(not_after.saturating_sub(now()) + 1)).await;
    387     relay.accepting.store(true, Ordering::SeqCst);
    388     let resumed = ready
    389         .client
    390         .operations(&ready.scope)
    391         .resume(
    392             OperationId::from_bytes([203; 32]),
    393             Some(first.command_digest()),
    394         )
    395         .await
    396         .unwrap();
    397     assert_eq!(resumed.issuance_window(), first.issuance_window());
    398     assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 2);
    399     let mut retried = Vec::new();
    400     for _ in 0..2 {
    401         retried.push(relay.events.recv().await.unwrap());
    402     }
    403     retried.sort();
    404     original.sort();
    405     assert!(
    406         retried == original,
    407         "retry changed the exact original outer representation"
    408     );
    409     let snapshot = ready.snapshot(203).await;
    410     assert_eq!(snapshot.delivery_attempts().len(), 4);
    411     assert!(
    412         snapshot
    413             .publications()
    414             .iter()
    415             .all(|work| snapshot.publication_accepted(*work).unwrap())
    416     );
    417     for work in snapshot.publications() {
    418         let attempts: Vec<_> = snapshot
    419             .delivery_attempts()
    420             .iter()
    421             .filter(|attempt| {
    422                 attempt.work() == radroots_storage::command::PublicationArtifact::Private(*work)
    423             })
    424             .collect();
    425         assert_eq!(
    426             attempts[0].intent().publication_policy_digest(),
    427             attempts[1].intent().publication_policy_digest()
    428         );
    429         assert_eq!(attempts[0].envelope_digest(), attempts[1].envelope_digest());
    430     }
    431     ready
    432         .client
    433         .operations(&ready.scope)
    434         .resume(OperationId::from_bytes([203; 32]), None)
    435         .await
    436         .unwrap();
    437     assert_eq!(relay.count.load(Ordering::SeqCst), initial_count + 2);
    438     ready.client.close().await.unwrap();
    439     stop(relay).await;
    440 }
    441 
    442 #[tokio::test]
    443 async fn uncertain_second_recipient_never_reissues_and_preserves_actual_first_copy_installation() {
    444     let relay = Relay::new().await;
    445     let uncertain = PublicKey::from_bytes(route_key(2).public_key().to_bytes()).unwrap();
    446     let ready = Ready::new(
    447         &relay,
    448         StartMode::Online,
    449         RecordedSigner::new(Some(uncertain)),
    450     )
    451     .await;
    452     let first = ready
    453         .client
    454         .inbox()
    455         .inquire(ready.context(204, now() + 120), inquiry())
    456         .await
    457         .unwrap();
    458     assert!(first.business_result().is_some());
    459     let snapshot = ready.snapshot(204).await;
    460     assert_eq!(
    461         snapshot
    462             .artifacts()
    463             .iter()
    464             .filter(|recipient| recipient.obtained().is_some())
    465             .count(),
    466         1
    467     );
    468     assert_eq!(snapshot.publications().len(), 1);
    469     assert_eq!(relay.count.load(Ordering::SeqCst), 1);
    470     ready
    471         .client
    472         .operations(&ready.scope)
    473         .resume(OperationId::from_bytes([204; 32]), None)
    474         .await
    475         .unwrap();
    476     assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 2);
    477     assert_eq!(relay.count.load(Ordering::SeqCst), 1);
    478     ready.client.close().await.unwrap();
    479     stop(relay).await;
    480 }
    481 
    482 #[tokio::test]
    483 async fn failed_obtained_capture_blocks_close_retains_writer_and_retries_without_signing_again() {
    484     let relay = Relay::new().await;
    485     let ready = Ready::new(&relay, StartMode::Online, RecordedSigner::new(None)).await;
    486     let mut db = ready.fixture.database().await;
    487     sqlx::query("CREATE TRIGGER capture_insert_fault BEFORE INSERT ON artifact_evidence BEGIN SELECT RAISE(ABORT, 'capture_insert_fault'); END").execute(&mut db).await.unwrap();
    488     let error = ready
    489         .client
    490         .inbox()
    491         .inquire(ready.context(205, now() + 120), inquiry())
    492         .await
    493         .unwrap_err();
    494     assert_eq!(error.code(), Code::StorageFailure);
    495     assert_eq!(error.retry(), radroots::RetryClass::ManualReconciliation);
    496     assert_eq!(error.context().admission(), Admission::Confirmed);
    497     assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 1);
    498     let blocked = ready
    499         .client
    500         .inbox()
    501         .inquire(ready.context(209, now() + 120), inquiry())
    502         .await
    503         .unwrap_err();
    504     assert_eq!(blocked.code(), Code::StorageFailure);
    505     assert_eq!(blocked.retry(), radroots::RetryClass::ManualReconciliation);
    506     assert_eq!(
    507         blocked.context().operation_id(),
    508         Some(OperationId::from_bytes([205; 32]))
    509     );
    510     assert_eq!(blocked.context().admission(), Admission::Confirmed);
    511     assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 1);
    512     let error = ready.client.close().await.unwrap_err();
    513     assert_eq!(error.code(), Code::StorageFailure);
    514     assert_eq!(error.retry(), radroots::RetryClass::ManualReconciliation);
    515     assert_eq!(
    516         error.context().operation_id(),
    517         Some(OperationId::from_bytes([205; 32]))
    518     );
    519     assert_eq!(error.context().admission(), Admission::Confirmed);
    520     assert_eq!(ready.client.state(), SessionState::Closing);
    521     ready.writer_is_retained().await;
    522     sqlx::query("DROP TRIGGER capture_insert_fault")
    523         .execute(&mut db)
    524         .await
    525         .unwrap();
    526     db.close().await.unwrap();
    527     ready.client.close().await.unwrap();
    528     assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 1);
    529     assert_eq!(relay.count.load(Ordering::SeqCst), 0);
    530     let reopened = ready.fixture.reopen(ready.input).await;
    531     let snapshot = reopened
    532         .operation_snapshot(
    533             ready.scope.binding().clone(),
    534             OperationId::from_bytes([205; 32]),
    535             None,
    536             copy(&ready.proof),
    537         )
    538         .await
    539         .unwrap()
    540         .unwrap();
    541     assert_eq!(
    542         snapshot
    543             .artifacts()
    544             .iter()
    545             .filter(|recipient| recipient.obtained().is_some())
    546             .count(),
    547         1
    548     );
    549     reopened.close_workspace().await.unwrap();
    550     stop(relay).await;
    551 }
    552 
    553 #[tokio::test]
    554 async fn explicit_resume_recovers_unknown_capture_under_same_writer_before_continuing_original() {
    555     let relay = Relay::new().await;
    556     let ready = Ready::new(&relay, StartMode::Online, RecordedSigner::new(None)).await;
    557     let mut db = ready.fixture.database().await;
    558     sqlx::query("PRAGMA foreign_keys=ON")
    559         .execute(&mut db)
    560         .await
    561         .unwrap();
    562     sqlx::query("CREATE TABLE capture_commit_fault (parent INTEGER REFERENCES workspace_grant(singleton) DEFERRABLE INITIALLY DEFERRED)").execute(&mut db).await.unwrap();
    563     sqlx::query("CREATE TRIGGER capture_commit_fault AFTER INSERT ON artifact_evidence BEGIN INSERT INTO capture_commit_fault(parent) VALUES(2); END").execute(&mut db).await.unwrap();
    564     let error = ready
    565         .client
    566         .inbox()
    567         .inquire(ready.context(207, now() + 120), inquiry())
    568         .await
    569         .unwrap_err();
    570     assert_eq!(error.code(), Code::RecoveryRequired);
    571     assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 1);
    572     assert!(
    573         ready
    574             .store
    575             .operation_snapshot(
    576                 ready.scope.binding().clone(),
    577                 OperationId::from_bytes([207; 32]),
    578                 None,
    579                 copy(&ready.proof)
    580             )
    581             .await
    582             .is_err()
    583     );
    584     assert_eq!(
    585         ready
    586             .client
    587             .operations(&ready.scope)
    588             .resume(OperationId::from_bytes([207; 32]), None)
    589             .await
    590             .unwrap_err()
    591             .code(),
    592         Code::RecoveryRequired
    593     );
    594     ready.writer_is_retained().await;
    595     sqlx::query("DROP TRIGGER capture_commit_fault")
    596         .execute(&mut db)
    597         .await
    598         .unwrap();
    599     sqlx::query("DROP TABLE capture_commit_fault")
    600         .execute(&mut db)
    601         .await
    602         .unwrap();
    603     db.close().await.unwrap();
    604     let receipt = ready
    605         .client
    606         .operations(&ready.scope)
    607         .resume(OperationId::from_bytes([207; 32]), None)
    608         .await
    609         .unwrap();
    610     assert!(receipt.business_result().is_some());
    611     assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 2);
    612     let snapshot = ready.snapshot(207).await;
    613     assert!(
    614         snapshot
    615             .artifacts()
    616             .iter()
    617             .all(|recipient| recipient.obtained().is_some())
    618     );
    619     assert!(
    620         snapshot
    621             .publications()
    622             .iter()
    623             .all(|work| snapshot.publication_accepted(*work).unwrap())
    624     );
    625     ready.client.close().await.unwrap();
    626     stop(relay).await;
    627 }
    628 
    629 #[tokio::test]
    630 async fn lost_observer_and_cancelled_close_preserve_actual_obtained_signature_until_capture() {
    631     let relay = Relay::new().await;
    632     let signer = RecordedSigner::new(None);
    633     let (reached, produced) = oneshot::channel();
    634     let (resume, released) = oneshot::channel();
    635     *signer.pause.lock().unwrap() = Some((reached, released));
    636     let ready = Ready::new(&relay, StartMode::Online, signer).await;
    637     let context = ready.context(206, now() + 120);
    638     let client = ready.client.clone();
    639     let observer = tokio::spawn(async move { client.inbox().inquire(context, inquiry()).await });
    640     tokio::time::timeout(Duration::from_secs(5), produced)
    641         .await
    642         .unwrap()
    643         .unwrap();
    644     observer.abort();
    645     assert!(observer.await.unwrap_err().is_cancelled());
    646     {
    647         let mut close = Box::pin(ready.client.close());
    648         assert!(futures::poll!(&mut close).is_pending());
    649     }
    650     assert_eq!(ready.client.state(), SessionState::Closing);
    651     ready.writer_is_retained().await;
    652     resume.send(()).unwrap();
    653     tokio::time::timeout(Duration::from_secs(5), ready.client.close())
    654         .await
    655         .unwrap()
    656         .unwrap();
    657     assert_eq!(ready.signer.calls.load(Ordering::SeqCst), 1);
    658     let reopened = ready.fixture.reopen(ready.input).await;
    659     let snapshot = reopened
    660         .operation_snapshot(
    661             ready.scope.binding().clone(),
    662             OperationId::from_bytes([206; 32]),
    663             None,
    664             copy(&ready.proof),
    665         )
    666         .await
    667         .unwrap()
    668         .unwrap();
    669     assert!(
    670         snapshot
    671             .artifacts()
    672             .iter()
    673             .any(|recipient| recipient.obtained().is_some())
    674     );
    675     reopened.close_workspace().await.unwrap();
    676     stop(relay).await;
    677 }