lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

supervision_retention.rs (12086B)


      1 use std::{
      2     future::{Future, pending, ready},
      3     pin::pin,
      4     task::Poll,
      5     time::Duration,
      6 };
      7 
      8 use radroots_service_host::{
      9     GracefulShutdown, HostError, HostErrorKind, MonotonicClock, MonotonicTime, ShutdownDisposition,
     10     ShutdownPhase, ShutdownPhaseFuture, ShutdownPhaseHandler, SupervisedTaskExit,
     11     SupervisedTaskExitStatus, SupervisionFailure, SupervisionFailureKind, TaskClassification,
     12     TaskMetadata, TaskName, TaskSupervisor, UnfinishedWork,
     13 };
     14 use tokio::sync::oneshot;
     15 
     16 fn metadata(name: &str, classification: TaskClassification) -> TaskMetadata {
     17     let phase = classification
     18         .requires_shutdown_phase()
     19         .then_some(ShutdownPhase::CancelIngress);
     20     TaskMetadata::new(TaskName::new(name).unwrap(), classification, phase).unwrap()
     21 }
     22 
     23 async fn consume_ready_outcome_and_cancel_bulk_wait(supervisor: &mut TaskSupervisor) {
     24     {
     25         let mut waiting = pin!(supervisor.supervise());
     26         std::future::poll_fn(|context| {
     27             assert!(waiting.as_mut().poll(context).is_pending());
     28             Poll::Ready(())
     29         })
     30         .await;
     31     }
     32     // The first join was consumed before the waiting future was dropped.
     33     assert_eq!(supervisor.task_count(), 1);
     34 }
     35 
     36 #[tokio::test]
     37 async fn cancelled_bulk_wait_retains_already_joined_exit_on_retry() {
     38     let mut supervisor = TaskSupervisor::new();
     39     let (finished, observed) = oneshot::channel();
     40     let (release, held) = oneshot::channel();
     41     supervisor
     42         .spawn(
     43             metadata("finished", TaskClassification::OneShot),
     44             |_| async {
     45                 finished.send(()).unwrap();
     46                 Ok(())
     47             },
     48         )
     49         .unwrap();
     50     supervisor
     51         .spawn(
     52             metadata("waiting", TaskClassification::OneShot),
     53             |_| async {
     54                 held.await.unwrap();
     55                 Ok(())
     56             },
     57         )
     58         .unwrap();
     59     observed.await.unwrap();
     60     consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await;
     61     assert!(!supervisor.cancellation_token().is_cancelled());
     62     release.send(()).unwrap();
     63     let exits = supervisor.supervise().await.unwrap();
     64     assert!(supervisor.is_empty());
     65     let names = exits
     66         .iter()
     67         .map(|exit| exit.metadata().name().as_str())
     68         .collect::<Vec<_>>();
     69     assert_eq!(names, ["finished", "waiting"]);
     70     assert!(supervisor.supervise().await.unwrap().is_empty());
     71 }
     72 
     73 #[tokio::test]
     74 async fn cancelled_bulk_wait_retains_first_fatal_failure_on_retry() {
     75     let mut supervisor = TaskSupervisor::new();
     76     let (finished, observed) = oneshot::channel();
     77     let (release, held) = oneshot::channel();
     78     supervisor
     79         .spawn(
     80             metadata("failed", TaskClassification::Critical),
     81             |_| async {
     82                 finished.send(()).unwrap();
     83                 Err(HostError::new(HostErrorKind::TaskFailure))
     84             },
     85         )
     86         .unwrap();
     87     supervisor
     88         .spawn(
     89             metadata("waiting", TaskClassification::OneShot),
     90             |_| async {
     91                 held.await.unwrap();
     92                 Ok(())
     93             },
     94         )
     95         .unwrap();
     96     observed.await.unwrap();
     97     consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await;
     98     assert!(supervisor.cancellation_token().is_cancelled());
     99     release.send(()).unwrap();
    100     let failure = supervisor.supervise().await.unwrap_err();
    101     assert!(supervisor.is_empty());
    102     assert_eq!(failure.kind(), SupervisionFailureKind::TaskReturnedError);
    103     assert_eq!(failure.metadata().unwrap().name().as_str(), "failed");
    104     assert!(supervisor.supervise().await.unwrap().is_empty());
    105 }
    106 
    107 async fn completed_pair(
    108     classification: TaskClassification,
    109     first_result: Result<(), HostError>,
    110     held_result: Result<(), HostError>,
    111 ) -> (TaskSupervisor, oneshot::Sender<()>) {
    112     let mut supervisor = TaskSupervisor::new();
    113     let (finished, observed) = oneshot::channel();
    114     let (release, held) = oneshot::channel();
    115     supervisor
    116         .spawn(metadata("observed", classification), |_| async move {
    117             finished.send(()).unwrap();
    118             first_result
    119         })
    120         .unwrap();
    121     supervisor
    122         .spawn(
    123             metadata("held", TaskClassification::OneShot),
    124             |_| async move {
    125                 held.await.unwrap();
    126                 held_result
    127             },
    128         )
    129         .unwrap();
    130     observed.await.unwrap();
    131     (supervisor, release)
    132 }
    133 
    134 async fn observe_retained(
    135     supervisor: &mut TaskSupervisor,
    136 ) -> Result<SupervisedTaskExit, SupervisionFailure> {
    137     let mut reading = pin!(supervisor.join_next());
    138     std::future::poll_fn(|context| match reading.as_mut().poll(context) {
    139         Poll::Ready(Some(outcome)) => Poll::Ready(outcome),
    140         _ => panic!("an already joined observation must be immediately available"),
    141     })
    142     .await
    143 }
    144 
    145 #[tokio::test]
    146 async fn cancelled_bulk_optional_failure_transfers_once_to_per_join_observation() {
    147     let (mut supervisor, release) = completed_pair(
    148         TaskClassification::Optional,
    149         Err(HostError::new(HostErrorKind::TaskFailure)),
    150         Ok(()),
    151     )
    152     .await;
    153     consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await;
    154 
    155     let exit = observe_retained(&mut supervisor).await.unwrap();
    156     assert_eq!(exit.metadata().name().as_str(), "observed");
    157     assert_eq!(exit.status(), SupervisedTaskExitStatus::OptionalFailure);
    158     assert!(exit.source().is_some());
    159     assert!(!supervisor.cancellation_token().is_cancelled());
    160     release.send(()).unwrap();
    161     let remaining = supervisor.supervise().await.unwrap();
    162     assert_eq!(remaining.len(), 1);
    163     assert_eq!(remaining[0].metadata().name().as_str(), "held");
    164     assert_eq!(
    165         remaining[0].status(),
    166         SupervisedTaskExitStatus::ExpectedCompletion
    167     );
    168     assert!(supervisor.join_next().await.is_none());
    169 }
    170 
    171 #[tokio::test]
    172 async fn cancelled_bulk_fatal_transfers_once_before_a_later_fatal_join() {
    173     let (mut supervisor, release) = completed_pair(
    174         TaskClassification::Critical,
    175         Err(HostError::new(HostErrorKind::TaskFailure)),
    176         Err(HostError::new(HostErrorKind::Lifecycle)),
    177     )
    178     .await;
    179     consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await;
    180 
    181     let first = observe_retained(&mut supervisor).await.unwrap_err();
    182     assert_eq!(first.kind(), SupervisionFailureKind::TaskReturnedError);
    183     assert_eq!(first.metadata().unwrap().name().as_str(), "observed");
    184     assert!(supervisor.cancellation_token().is_cancelled());
    185     release.send(()).unwrap();
    186     let later = supervisor.join_next().await.unwrap().unwrap_err();
    187     assert_eq!(later.kind(), SupervisionFailureKind::TaskReturnedError);
    188     assert_eq!(later.metadata().unwrap().name().as_str(), "held");
    189     assert!(supervisor.join_next().await.is_none());
    190     assert!(supervisor.supervise().await.unwrap().is_empty());
    191 }
    192 
    193 struct FixedClock(Duration);
    194 
    195 impl MonotonicClock for FixedClock {
    196     fn now_monotonic(&self) -> MonotonicTime {
    197         MonotonicTime::from_duration_since_origin(self.0)
    198     }
    199 }
    200 
    201 #[derive(Default)]
    202 struct RecordingHandler(Vec<ShutdownPhase>);
    203 
    204 impl ShutdownPhaseHandler for RecordingHandler {
    205     fn enter(&mut self, phase: ShutdownPhase) -> ShutdownPhaseFuture<'_> {
    206         self.0.push(phase);
    207         Box::pin(ready(Ok(())))
    208     }
    209 }
    210 
    211 const PHASES: [ShutdownPhase; 7] = [
    212     ShutdownPhase::RejectNewMutations,
    213     ShutdownPhase::CancelIngress,
    214     ShutdownPhase::DrainOperations,
    215     ShutdownPhase::PersistRecoverableWork,
    216     ShutdownPhase::CloseNetwork,
    217     ShutdownPhase::CloseSqlite,
    218     ShutdownPhase::CloseSockets,
    219 ];
    220 
    221 #[tokio::test]
    222 async fn cancelled_bulk_fatal_is_retained_by_graceful_shutdown_before_empty_phase() {
    223     let (mut supervisor, release) = completed_pair(
    224         TaskClassification::Critical,
    225         Err(HostError::new(HostErrorKind::TaskFailure)),
    226         Ok(()),
    227     )
    228     .await;
    229     consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await;
    230     release.send(()).unwrap();
    231     let mut shutdown = GracefulShutdown::new(Duration::from_secs(30)).unwrap();
    232     let mut handler = RecordingHandler::default();
    233     let clock = FixedClock(Duration::from_secs(5));
    234 
    235     let summary = shutdown
    236         .run(&clock, &mut supervisor, &mut handler, pending())
    237         .await
    238         .unwrap();
    239     assert_eq!(
    240         summary.disposition(),
    241         ShutdownDisposition::TaskFailed {
    242             kind: SupervisionFailureKind::TaskReturnedError,
    243         }
    244     );
    245     assert_eq!(
    246         shutdown
    247             .task_failure()
    248             .unwrap()
    249             .metadata()
    250             .unwrap()
    251             .name()
    252             .as_str(),
    253         "observed"
    254     );
    255     assert_eq!(handler.0, PHASES);
    256     assert!(supervisor.is_empty());
    257     assert!(supervisor.join_next().await.is_none());
    258     assert_eq!(
    259         shutdown
    260             .run(&clock, &mut supervisor, &mut handler, ready(()))
    261             .await
    262             .unwrap(),
    263         summary
    264     );
    265     assert_eq!(handler.0, PHASES);
    266 }
    267 
    268 #[tokio::test]
    269 async fn cancelled_phase_drain_transfers_exit_and_resumes_original_deadline() {
    270     let (mut supervisor, release) =
    271         completed_pair(TaskClassification::OneShot, Ok(()), Ok(())).await;
    272     let mut shutdown = GracefulShutdown::new(Duration::from_secs(30)).unwrap();
    273     let mut handler = RecordingHandler::default();
    274     let clock = FixedClock(Duration::from_secs(5));
    275     {
    276         let mut waiting = pin!(shutdown.run(&clock, &mut supervisor, &mut handler, pending()));
    277         std::future::poll_fn(|context| {
    278             assert!(waiting.as_mut().poll(context).is_pending());
    279             Poll::Ready(())
    280         })
    281         .await;
    282     }
    283     assert_eq!(supervisor.task_count(), 1);
    284     assert_eq!(handler.0, PHASES[..3]);
    285     let retained = observe_retained(&mut supervisor).await.unwrap();
    286     assert_eq!(retained.metadata().name().as_str(), "observed");
    287     assert_eq!(
    288         retained.status(),
    289         SupervisedTaskExitStatus::ExpectedCompletion
    290     );
    291     assert!(!supervisor.cancellation_token().is_cancelled());
    292     release.send(()).unwrap();
    293     let summary = shutdown
    294         .run(
    295             &FixedClock(Duration::from_secs(250)),
    296             &mut supervisor,
    297             &mut handler,
    298             pending(),
    299         )
    300         .await
    301         .unwrap();
    302     assert_eq!(summary.disposition(), ShutdownDisposition::Completed);
    303     assert_eq!(
    304         summary.deadline().time().duration_since_origin(),
    305         Duration::from_secs(35)
    306     );
    307     assert_eq!(handler.0, PHASES);
    308     assert!(supervisor.is_empty());
    309     assert!(supervisor.join_next().await.is_none());
    310 }
    311 
    312 #[tokio::test]
    313 async fn forced_abort_drain_retains_previously_joined_fatal_and_aborted_peer() {
    314     let (mut supervisor, _release) = completed_pair(
    315         TaskClassification::Critical,
    316         Err(HostError::new(HostErrorKind::TaskFailure)),
    317         Ok(()),
    318     )
    319     .await;
    320     consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await;
    321     let mut shutdown = GracefulShutdown::new(Duration::from_secs(30)).unwrap();
    322     let mut handler = RecordingHandler::default();
    323     let clock = FixedClock(Duration::from_secs(5));
    324     let summary = shutdown
    325         .run(&clock, &mut supervisor, &mut handler, ready(()))
    326         .await
    327         .unwrap();
    328     assert_eq!(
    329         summary.disposition(),
    330         ShutdownDisposition::Forced {
    331             unfinished: UnfinishedWork::FatalAuthoritative,
    332         }
    333     );
    334     assert!(handler.0.is_empty());
    335     assert!(supervisor.is_empty());
    336     assert_eq!(supervisor.task_count(), 0);
    337     let retained = observe_retained(&mut supervisor).await.unwrap_err();
    338     assert_eq!(retained.kind(), SupervisionFailureKind::TaskReturnedError);
    339     assert_eq!(retained.metadata().unwrap().name().as_str(), "observed");
    340     let aborted = observe_retained(&mut supervisor).await.unwrap();
    341     assert_eq!(aborted.metadata().name().as_str(), "held");
    342     assert_eq!(
    343         aborted.status(),
    344         SupervisedTaskExitStatus::ExpectedCompletion
    345     );
    346     assert!(supervisor.join_next().await.is_none());
    347     assert!(supervisor.supervise().await.unwrap().is_empty());
    348 }