lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

session.rs (22059B)


      1 #![cfg(feature = "session")]
      2 
      3 use std::{
      4     future::Future,
      5     num::NonZeroU16,
      6     pin::pin,
      7     sync::{
      8         Arc, Mutex,
      9         atomic::{AtomicBool, AtomicUsize, Ordering},
     10     },
     11     task::{Context, Poll},
     12 };
     13 
     14 use futures::{
     15     executor::block_on,
     16     future::poll_fn,
     17     task::{AtomicWaker, noop_waker},
     18 };
     19 use radroots_execution::{
     20     Code, Error, OperationContext, Result, RetryClass,
     21     session::{HostFuture, Session, SessionHost, SessionState},
     22 };
     23 
     24 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
     25 #[repr(usize)]
     26 enum Hook {
     27     Start,
     28     Quiesce,
     29     Disconnect,
     30     Callbacks,
     31     Storage,
     32 }
     33 
     34 #[derive(Default)]
     35 struct Observations {
     36     calls: Mutex<Vec<Hook>>,
     37     completed: AtomicUsize,
     38     blocked: AtomicUsize,
     39     fail: AtomicUsize,
     40     passive: AtomicBool,
     41     waker: AtomicWaker,
     42 }
     43 
     44 struct Host(Arc<Observations>);
     45 
     46 impl Host {
     47     fn run(&self, hook: Hook) -> HostFuture<'_, Result<()>> {
     48         Box::pin(async move {
     49             self.0.calls.lock().unwrap().push(hook);
     50             let bit = 1 << hook as usize;
     51             poll_fn(|context| {
     52                 if !self.0.passive.load(Ordering::SeqCst) {
     53                     self.0.waker.register(context.waker());
     54                 }
     55                 if self.0.blocked.load(Ordering::SeqCst) & bit != 0 {
     56                     return Poll::Pending;
     57                 }
     58                 if self.0.fail.fetch_and(!bit, Ordering::SeqCst) & bit != 0 {
     59                     return Poll::Ready(Err(Error::new(
     60                         Code::StorageFailure,
     61                         OperationContext::proposed(None),
     62                         RetryClass::Never,
     63                         None,
     64                     )
     65                     .unwrap()));
     66                 }
     67                 self.0.completed.fetch_or(bit, Ordering::SeqCst);
     68                 Poll::Ready(Ok(()))
     69             })
     70             .await
     71         })
     72     }
     73 }
     74 
     75 impl SessionHost for Host {
     76     fn start(&self) -> HostFuture<'_, Result<()>> {
     77         self.run(Hook::Start)
     78     }
     79     fn quiesce(&self) -> HostFuture<'_, Result<()>> {
     80         self.run(Hook::Quiesce)
     81     }
     82     fn disconnect(&self) -> HostFuture<'_, Result<()>> {
     83         self.run(Hook::Disconnect)
     84     }
     85     fn dispose_callbacks(&self) -> HostFuture<'_, Result<()>> {
     86         self.run(Hook::Callbacks)
     87     }
     88     fn close_storage(&self) -> HostFuture<'_, Result<()>> {
     89         self.run(Hook::Storage)
     90     }
     91 }
     92 
     93 fn fixture(limit: u16) -> (Session<Host>, Arc<Observations>) {
     94     let observations = Arc::new(Observations::default());
     95     (
     96         Session::new(Host(observations.clone()), NonZeroU16::new(limit).unwrap()),
     97         observations,
     98     )
     99 }
    100 
    101 fn poll<F: Future>(future: std::pin::Pin<&mut F>) -> Poll<F::Output> {
    102     future.poll(&mut Context::from_waker(&noop_waker()))
    103 }
    104 
    105 #[test]
    106 fn construction_and_unpolled_futures_have_no_effects() {
    107     let (session, observed) = fixture(2);
    108     assert_eq!(session.state(), SessionState::Ready);
    109     drop(session.start());
    110     drop(session.close());
    111     assert_eq!(session.state(), SessionState::Ready);
    112     assert!(observed.calls.lock().unwrap().is_empty());
    113     assert_eq!(
    114         session.admit().err().unwrap().code(),
    115         Code::CapabilityUnavailable
    116     );
    117     drop(session);
    118     assert!(observed.calls.lock().unwrap().is_empty());
    119 }
    120 
    121 #[test]
    122 fn startup_and_close_are_shared_idempotent_and_storage_is_last() {
    123     let (session, observed) = fixture(1);
    124     let clone = session.clone();
    125     block_on(async {
    126         let (one, two) = futures::join!(session.start(), clone.start());
    127         one.unwrap();
    128         two.unwrap();
    129         assert_eq!(session.state(), SessionState::Running);
    130         let (one, two) = futures::join!(session.close(), clone.close());
    131         one.unwrap();
    132         two.unwrap();
    133     });
    134     assert_eq!(
    135         observed.calls.lock().unwrap().as_slice(),
    136         &[
    137             Hook::Start,
    138             Hook::Quiesce,
    139             Hook::Disconnect,
    140             Hook::Callbacks,
    141             Hook::Storage
    142         ]
    143     );
    144     assert_eq!(clone.state(), SessionState::Closed);
    145     assert!(block_on(clone.start()).is_err());
    146     assert!(clone.admit().is_err());
    147     block_on(clone.close()).unwrap();
    148     assert_eq!(observed.calls.lock().unwrap().len(), 5);
    149 }
    150 
    151 #[test]
    152 fn admission_is_bounded_and_close_waits_for_owned_work() {
    153     let (session, observed) = fixture(2);
    154     block_on(session.start()).unwrap();
    155     let first = session.admit().unwrap();
    156     let second = session.clone().admit().unwrap();
    157     let error = session.admit().err().unwrap();
    158     assert_eq!(error.code(), Code::CapabilityUnavailable);
    159     assert_eq!(error.context(), OperationContext::proposed(None));
    160     let mut closing = pin!(session.close());
    161     assert!(poll(closing.as_mut()).is_pending());
    162     assert_eq!(session.state(), SessionState::Closing);
    163     assert!(session.admit().is_err());
    164     assert_eq!(
    165         observed.calls.lock().unwrap().as_slice(),
    166         &[Hook::Start, Hook::Quiesce]
    167     );
    168     drop(first);
    169     assert!(poll(closing.as_mut()).is_pending());
    170     drop(second);
    171     assert_eq!(poll(closing.as_mut()), Poll::Ready(Ok(())));
    172     assert_eq!(observed.calls.lock().unwrap().last(), Some(&Hook::Storage));
    173 }
    174 
    175 #[test]
    176 fn cancellation_and_known_failure_resume_only_the_unfinished_hook() {
    177     for hook in [
    178         Hook::Quiesce,
    179         Hook::Disconnect,
    180         Hook::Callbacks,
    181         Hook::Storage,
    182     ] {
    183         for failure in [false, true] {
    184             let (session, observed) = fixture(1);
    185             block_on(session.start()).unwrap();
    186             let bit = 1 << hook as usize;
    187             if failure {
    188                 observed.fail.store(bit, Ordering::SeqCst);
    189                 assert_eq!(
    190                     block_on(session.close()).unwrap_err().code(),
    191                     Code::StorageFailure
    192                 );
    193             } else {
    194                 observed.blocked.store(bit, Ordering::SeqCst);
    195                 let mut close = Box::pin(session.close());
    196                 assert!(poll(close.as_mut()).is_pending());
    197                 drop(close);
    198                 observed.blocked.store(0, Ordering::SeqCst);
    199             }
    200             assert_eq!(session.state(), SessionState::Closing);
    201             assert!(session.admit().is_err());
    202             assert!(block_on(session.start()).is_err());
    203             block_on(session.close()).unwrap();
    204             let calls = observed.calls.lock().unwrap();
    205             let expected: Vec<_> = [
    206                 Hook::Start,
    207                 Hook::Quiesce,
    208                 Hook::Disconnect,
    209                 Hook::Callbacks,
    210                 Hook::Storage,
    211             ]
    212             .into_iter()
    213             .flat_map(|value| {
    214                 if value == hook {
    215                     vec![value, value]
    216                 } else {
    217                     vec![value]
    218                 }
    219             })
    220             .collect();
    221             assert_eq!(*calls, expected);
    222             assert_eq!(session.state(), SessionState::Closed);
    223         }
    224     }
    225 }
    226 
    227 #[test]
    228 fn canceled_drain_resumes_and_concurrent_closers_share_completion() {
    229     let (session, observed) = fixture(1);
    230     block_on(session.start()).unwrap();
    231     let work = session.admit().unwrap();
    232     let mut close = Box::pin(session.close());
    233     assert!(poll(close.as_mut()).is_pending());
    234     drop(close);
    235     drop(work);
    236     let clone = session.clone();
    237     block_on(async {
    238         let (a, b) = futures::join!(clone.close(), session.close());
    239         a.unwrap();
    240         b.unwrap();
    241     });
    242     assert_eq!(
    243         observed.calls.lock().unwrap().as_slice(),
    244         &[
    245             Hook::Start,
    246             Hook::Quiesce,
    247             Hook::Disconnect,
    248             Hook::Callbacks,
    249             Hook::Storage
    250         ]
    251     );
    252 }
    253 
    254 #[test]
    255 fn close_interrupts_pending_start_and_never_reopens_admission() {
    256     let (session, observed) = fixture(1);
    257     observed
    258         .blocked
    259         .store(1 << Hook::Start as usize, Ordering::SeqCst);
    260     let mut start = Box::pin(session.start());
    261     assert!(poll(start.as_mut()).is_pending());
    262     assert_eq!(session.state(), SessionState::Starting);
    263     assert!(session.admit().is_err());
    264     let mut close = Box::pin(session.close());
    265     assert!(poll(close.as_mut()).is_pending());
    266     assert_eq!(session.state(), SessionState::Closing);
    267     assert!(matches!(poll(start.as_mut()), Poll::Ready(Err(_))));
    268     assert_eq!(poll(close.as_mut()), Poll::Ready(Ok(())));
    269     assert_eq!(observed.completed.load(Ordering::SeqCst) & 1, 0);
    270     assert_eq!(observed.calls.lock().unwrap().last(), Some(&Hook::Storage));
    271 }
    272 
    273 #[test]
    274 fn canceled_and_failed_startup_cannot_admit_and_may_be_resumed_or_closed() {
    275     let (session, observed) = fixture(1);
    276     observed.blocked.store(1, Ordering::SeqCst);
    277     let mut start = Box::pin(session.start());
    278     assert!(poll(start.as_mut()).is_pending());
    279     drop(start);
    280     assert!(session.admit().is_err());
    281     observed.blocked.store(0, Ordering::SeqCst);
    282     observed.fail.store(1, Ordering::SeqCst);
    283     assert!(block_on(session.start()).is_err());
    284     assert_eq!(session.state(), SessionState::Starting);
    285     assert!(session.admit().is_err());
    286     block_on(session.start()).unwrap();
    287     drop(session.admit().unwrap());
    288     block_on(session.close()).unwrap();
    289     let (unstarted, _) = fixture(1);
    290     block_on(unstarted.close()).unwrap();
    291     assert_eq!(unstarted.state(), SessionState::Closed);
    292 }
    293 
    294 #[test]
    295 fn work_permit_retains_resources_after_last_client_handle() {
    296     let (session, observed) = fixture(1);
    297     block_on(session.start()).unwrap();
    298     let permit = session.admit().unwrap();
    299     assert_eq!(Arc::strong_count(&observed), 2);
    300     drop(session);
    301     assert_eq!(Arc::strong_count(&observed), 2);
    302     drop(permit);
    303     assert_eq!(Arc::strong_count(&observed), 1);
    304     assert_eq!(observed.calls.lock().unwrap().as_slice(), &[Hook::Start]);
    305 }
    306 
    307 #[test]
    308 fn permit_completion_and_close_wake_their_pending_observers() {
    309     struct WakeCount(AtomicUsize);
    310     impl futures::task::ArcWake for WakeCount {
    311         fn wake_by_ref(arc: &Arc<Self>) {
    312             arc.0.fetch_add(1, Ordering::SeqCst);
    313         }
    314     }
    315     let counter = Arc::new(WakeCount(AtomicUsize::new(0)));
    316     let waker = futures::task::waker(counter.clone());
    317     let mut context = Context::from_waker(&waker);
    318     let (session, _) = fixture(1);
    319     block_on(session.start()).unwrap();
    320     let work = session.admit().unwrap();
    321     let mut close = Box::pin(session.close());
    322     assert!(close.as_mut().poll(&mut context).is_pending());
    323     drop(work);
    324     assert_eq!(counter.0.load(Ordering::SeqCst), 1);
    325     assert_eq!(close.as_mut().poll(&mut context), Poll::Ready(Ok(())));
    326 
    327     let (session, observed) = fixture(1);
    328     observed.blocked.store(1, Ordering::SeqCst);
    329     let mut start = Box::pin(session.start());
    330     assert!(start.as_mut().poll(&mut context).is_pending());
    331     let before = counter.0.load(Ordering::SeqCst);
    332     let mut close = Box::pin(session.close());
    333     assert!(poll(close.as_mut()).is_pending());
    334     assert!(counter.0.load(Ordering::SeqCst) > before);
    335     assert!(matches!(poll(start.as_mut()), Poll::Ready(Err(_))));
    336     assert_eq!(poll(close.as_mut()), Poll::Ready(Ok(())));
    337 }
    338 
    339 #[cfg(not(target_arch = "wasm32"))]
    340 #[test]
    341 fn replacing_or_clearing_wakers_allows_reentrant_resource_observation() {
    342     use std::{sync::mpsc, task::Wake, time::Duration};
    343 
    344     struct InspectOnDrop {
    345         session: Session<Host>,
    346         drops: Arc<AtomicUsize>,
    347         close: bool,
    348     }
    349     impl Wake for InspectOnDrop {
    350         fn wake(self: Arc<Self>) {
    351             let _state = self.session.state();
    352         }
    353     }
    354     impl Drop for InspectOnDrop {
    355         fn drop(&mut self) {
    356             let _state = self.session.state();
    357             if self.close {
    358                 let mut closing = Box::pin(self.session.close());
    359                 assert!(poll(closing.as_mut()).is_pending());
    360             }
    361             self.drops.fetch_add(1, Ordering::SeqCst);
    362         }
    363     }
    364 
    365     let (done, result) = mpsc::channel();
    366     let worker = std::thread::spawn(move || {
    367         let (session, observed) = fixture(1);
    368         let drops = Arc::new(AtomicUsize::new(0));
    369         observed.passive.store(true, Ordering::SeqCst);
    370         observed.blocked.store(1, Ordering::SeqCst);
    371         let mut start = Box::pin(session.start());
    372         {
    373             let waker = Arc::new(InspectOnDrop {
    374                 session: session.clone(),
    375                 drops: drops.clone(),
    376                 close: false,
    377             })
    378             .into();
    379             assert!(
    380                 start
    381                     .as_mut()
    382                     .poll(&mut Context::from_waker(&waker))
    383                     .is_pending()
    384             );
    385         }
    386         // Replacing the last reference to this waker runs its destructor.
    387         assert!(poll(start.as_mut()).is_pending());
    388         assert_eq!(drops.load(Ordering::SeqCst), 1);
    389         assert!(poll(start.as_mut()).is_pending());
    390         {
    391             let waker = Arc::new(InspectOnDrop {
    392                 session: session.clone(),
    393                 drops: drops.clone(),
    394                 close: false,
    395             })
    396             .into();
    397             assert!(
    398                 start
    399                     .as_mut()
    400                     .poll(&mut Context::from_waker(&waker))
    401                     .is_pending()
    402             );
    403         }
    404         observed.blocked.store(0, Ordering::SeqCst);
    405         let waker = Arc::new(InspectOnDrop {
    406             session: session.clone(),
    407             drops: drops.clone(),
    408             close: false,
    409         })
    410         .into();
    411         assert_eq!(
    412             start.as_mut().poll(&mut Context::from_waker(&waker)),
    413             Poll::Ready(Ok(()))
    414         );
    415         drop(waker);
    416         assert_eq!(drops.load(Ordering::SeqCst), 3);
    417         drop(start);
    418 
    419         let work = session.admit().unwrap();
    420         let mut close = Box::pin(session.close());
    421         {
    422             let waker = Arc::new(InspectOnDrop {
    423                 session: session.clone(),
    424                 drops: drops.clone(),
    425                 close: false,
    426             })
    427             .into();
    428             assert!(
    429                 close
    430                     .as_mut()
    431                     .poll(&mut Context::from_waker(&waker))
    432                     .is_pending()
    433             );
    434         }
    435         assert!(poll(close.as_mut()).is_pending());
    436         assert_eq!(drops.load(Ordering::SeqCst), 4);
    437         drop(work);
    438         assert_eq!(poll(close.as_mut()), Poll::Ready(Ok(())));
    439 
    440         let (session, observed) = fixture(1);
    441         observed.passive.store(true, Ordering::SeqCst);
    442         observed.blocked.store(1, Ordering::SeqCst);
    443         let mut start = Box::pin(session.start());
    444         {
    445             let waker = Arc::new(InspectOnDrop {
    446                 session: session.clone(),
    447                 drops: drops.clone(),
    448                 close: true,
    449             })
    450             .into();
    451             assert!(
    452                 start
    453                     .as_mut()
    454                     .poll(&mut Context::from_waker(&waker))
    455                     .is_pending()
    456             );
    457         }
    458         observed.blocked.store(0, Ordering::SeqCst);
    459         // Reentrant close races with a successful host startup result.
    460         assert!(matches!(poll(start.as_mut()), Poll::Ready(Err(_))));
    461         assert_eq!(session.state(), SessionState::Closing);
    462         assert!(session.admit().is_err());
    463         block_on(session.close()).unwrap();
    464 
    465         let (session, observed) = fixture(1);
    466         observed.passive.store(true, Ordering::SeqCst);
    467         observed.blocked.store(1, Ordering::SeqCst);
    468         let mut start = Box::pin(session.start());
    469         {
    470             let waker = Arc::new(InspectOnDrop {
    471                 session: session.clone(),
    472                 drops: drops.clone(),
    473                 close: false,
    474             })
    475             .into();
    476             assert!(
    477                 start
    478                     .as_mut()
    479                     .poll(&mut Context::from_waker(&waker))
    480                     .is_pending()
    481             );
    482         }
    483         let mut close = Box::pin(session.close());
    484         // Both wake and final destructor may observe the session reentrantly.
    485         assert!(poll(close.as_mut()).is_pending());
    486         assert!(matches!(poll(start.as_mut()), Poll::Ready(Err(_))));
    487         assert_eq!(poll(close.as_mut()), Poll::Ready(Ok(())));
    488         done.send(()).unwrap();
    489     });
    490     // A regression must fail within a bound rather than hang the test process.
    491     result
    492         .recv_timeout(Duration::from_secs(5))
    493         .expect("waker destructor deadlocked session bookkeeping");
    494     worker.join().unwrap();
    495 }
    496 
    497 #[cfg(not(target_arch = "wasm32"))]
    498 #[test]
    499 fn native_handles_and_futures_are_send_and_concurrent_admission_is_bounded() {
    500     fn send_sync<T: Send + Sync>() {}
    501     fn send<T: Send>(_: T) {}
    502     send_sync::<Session<Host>>();
    503     let (session, _) = fixture(3);
    504     send(session.start());
    505     send(session.close());
    506     block_on(session.start()).unwrap();
    507     let admitted: Vec<_> = std::thread::scope(|scope| {
    508         let attempts: Vec<_> = (0..32).map(|_| scope.spawn(|| session.admit())).collect();
    509         attempts
    510             .into_iter()
    511             .filter_map(|attempt| attempt.join().unwrap().ok())
    512             .collect()
    513     });
    514     assert_eq!(admitted.len(), 3);
    515     drop(admitted);
    516     block_on(session.close()).unwrap();
    517 }
    518 
    519 #[cfg(target_arch = "wasm32")]
    520 #[test]
    521 fn browser_host_futures_accept_thread_local_callback_state() {
    522     use std::{cell::Cell, rc::Rc};
    523     struct Local(Rc<Cell<usize>>);
    524     impl SessionHost for Local {
    525         fn start(&self) -> HostFuture<'_, Result<()>> {
    526             Box::pin(async {
    527                 self.0.set(1);
    528                 Ok(())
    529             })
    530         }
    531         fn quiesce(&self) -> HostFuture<'_, Result<()>> {
    532             Box::pin(async { Ok(()) })
    533         }
    534         fn disconnect(&self) -> HostFuture<'_, Result<()>> {
    535             Box::pin(async { Ok(()) })
    536         }
    537         fn dispose_callbacks(&self) -> HostFuture<'_, Result<()>> {
    538             Box::pin(async { Ok(()) })
    539         }
    540         fn close_storage(&self) -> HostFuture<'_, Result<()>> {
    541             Box::pin(async { Ok(()) })
    542         }
    543     }
    544     let calls = Rc::new(Cell::new(0));
    545     let client = Session::new(Local(calls.clone()), NonZeroU16::new(1).unwrap());
    546     block_on(client.start()).unwrap();
    547     assert_eq!(calls.get(), 1);
    548     block_on(client.close()).unwrap();
    549 }
    550 
    551 #[test]
    552 fn explicit_quiesce_stops_admission_without_draining_or_disposing_resources() {
    553     let (session, observed) = fixture(1);
    554     block_on(session.start()).unwrap();
    555     let permit = session.admit().unwrap();
    556     drop(session.quiesce());
    557     assert_eq!(session.state(), SessionState::Running);
    558     block_on(session.quiesce()).unwrap();
    559     assert_eq!(session.state(), SessionState::Closing);
    560     assert!(session.admit().is_err());
    561     assert_eq!(
    562         *observed.calls.lock().unwrap(),
    563         [Hook::Start, Hook::Quiesce]
    564     );
    565     block_on(session.quiesce()).unwrap();
    566     assert_eq!(
    567         *observed.calls.lock().unwrap(),
    568         [Hook::Start, Hook::Quiesce]
    569     );
    570     drop(permit);
    571     block_on(session.close()).unwrap();
    572     block_on(session.quiesce()).unwrap();
    573     assert_eq!(
    574         *observed.calls.lock().unwrap(),
    575         [
    576             Hook::Start,
    577             Hook::Quiesce,
    578             Hook::Disconnect,
    579             Hook::Callbacks,
    580             Hook::Storage
    581         ]
    582     );
    583 }
    584 
    585 struct RetainedJobsHost {
    586     host: Host,
    587     drains: Arc<AtomicUsize>,
    588     retained: Arc<AtomicBool>,
    589 }
    590 impl SessionHost for RetainedJobsHost {
    591     fn start(&self) -> HostFuture<'_, Result<()>> {
    592         self.host.start()
    593     }
    594     fn quiesce(&self) -> HostFuture<'_, Result<()>> {
    595         self.host.quiesce()
    596     }
    597     fn drain(&self) -> HostFuture<'_, Result<()>> {
    598         Box::pin(async move {
    599             self.drains.fetch_add(1, Ordering::SeqCst);
    600             if self.retained.load(Ordering::SeqCst) {
    601                 return Err(Error::new(
    602                     Code::RecoveryRequired,
    603                     OperationContext::proposed(None),
    604                     RetryClass::ManualReconciliation,
    605                     None,
    606                 )
    607                 .unwrap());
    608             }
    609             Ok(())
    610         })
    611     }
    612     fn disconnect(&self) -> HostFuture<'_, Result<()>> {
    613         self.host.disconnect()
    614     }
    615     fn dispose_callbacks(&self) -> HostFuture<'_, Result<()>> {
    616         self.host.dispose_callbacks()
    617     }
    618     fn close_storage(&self) -> HostFuture<'_, Result<()>> {
    619         self.host.close_storage()
    620     }
    621 }
    622 #[test]
    623 fn owned_drain_failure_preserves_close_stage_and_resources_until_safe_completion() {
    624     let observations = Arc::new(Observations::default());
    625     let drains = Arc::new(AtomicUsize::new(0));
    626     let retained = Arc::new(AtomicBool::new(true));
    627     let session = Session::new(
    628         RetainedJobsHost {
    629             host: Host(observations.clone()),
    630             drains: drains.clone(),
    631             retained: retained.clone(),
    632         },
    633         NonZeroU16::MIN,
    634     );
    635     block_on(session.start()).unwrap();
    636     let permit = session.admit().unwrap();
    637     assert_eq!(
    638         block_on(session.close()).unwrap_err().code(),
    639         Code::RecoveryRequired
    640     );
    641     assert_eq!(session.state(), SessionState::Closing);
    642     assert_eq!(
    643         *observations.calls.lock().unwrap(),
    644         vec![Hook::Start, Hook::Quiesce]
    645     );
    646     assert!(session.admit().is_err());
    647     assert_eq!(
    648         block_on(session.close()).unwrap_err().code(),
    649         Code::RecoveryRequired
    650     );
    651     assert_eq!(drains.load(Ordering::SeqCst), 2);
    652     retained.store(false, Ordering::SeqCst);
    653     let mut wait = Box::pin(session.close());
    654     assert!(poll(wait.as_mut()).is_pending());
    655     drop(wait);
    656     assert_eq!(drains.load(Ordering::SeqCst), 3);
    657     drop(permit);
    658     block_on(session.close()).unwrap();
    659     assert_eq!(drains.load(Ordering::SeqCst), 3);
    660     assert_eq!(
    661         *observations.calls.lock().unwrap(),
    662         vec![
    663             Hook::Start,
    664             Hook::Quiesce,
    665             Hook::Disconnect,
    666             Hook::Callbacks,
    667             Hook::Storage
    668         ]
    669     );
    670     assert_eq!(session.state(), SessionState::Closed);
    671 }