lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

supervisor.rs (27270B)


      1 //! Join-owned task supervision with typed completion policy.
      2 
      3 use core::{fmt, future::Future};
      4 use std::{
      5     collections::{HashMap, VecDeque},
      6     error::Error,
      7 };
      8 
      9 use tokio::task::{Id, JoinError, JoinSet};
     10 
     11 use crate::HostError;
     12 
     13 use super::{CancellationToken, ShutdownPhase, TaskClassification, TaskMetadata, UnfinishedWork};
     14 
     15 /// Owns every spawned service task until its join result is observed.
     16 #[must_use = "a task supervisor must be run or drained so authoritative tasks are joined"]
     17 pub struct TaskSupervisor {
     18     cancellation: CancellationToken,
     19     tasks: JoinSet<TaskCompletion>,
     20     controls: HashMap<Id, TaskControl>,
     21     observed: VecDeque<Result<SupervisedTaskExit, SupervisionFailure>>,
     22 }
     23 
     24 struct TaskControl {
     25     metadata: TaskMetadata,
     26     cancellation: CancellationToken,
     27 }
     28 
     29 impl TaskSupervisor {
     30     pub fn new() -> Self {
     31         Self {
     32             cancellation: CancellationToken::new(),
     33             tasks: JoinSet::new(),
     34             controls: HashMap::new(),
     35             observed: VecDeque::new(),
     36         }
     37     }
     38 
     39     /// Returns a cloneable observer and cancellation authority for composition boundaries.
     40     #[must_use]
     41     pub fn cancellation_token(&self) -> CancellationToken {
     42         self.cancellation.clone()
     43     }
     44 
     45     /// Requests cooperative cancellation of every task child token.
     46     pub fn request_cancellation(&self) {
     47         self.cancellation.cancel();
     48     }
     49 
     50     /// Returns the number of tasks whose join results have not yet been collected.
     51     #[must_use]
     52     pub fn task_count(&self) -> usize {
     53         self.tasks.len()
     54     }
     55 
     56     /// Returns whether every task has been joined; unconsumed observations may remain.
     57     #[must_use]
     58     pub fn is_empty(&self) -> bool {
     59         self.tasks.is_empty()
     60     }
     61 
     62     /// Registers and spawns one task on the current runtime without exposing a detachable handle.
     63     pub fn spawn<F, Fut>(
     64         &mut self,
     65         metadata: TaskMetadata,
     66         task: F,
     67     ) -> Result<(), TaskRegistrationError>
     68     where
     69         F: FnOnce(CancellationToken) -> Fut + Send + 'static,
     70         Fut: Future<Output = Result<(), HostError>> + Send + 'static,
     71     {
     72         if self
     73             .controls
     74             .values()
     75             .any(|active| active.metadata.name() == metadata.name())
     76         {
     77             return Err(TaskRegistrationError::DuplicateName);
     78         }
     79         let runtime =
     80             tokio::runtime::Handle::try_current().map_err(|_| TaskRegistrationError::NoRuntime)?;
     81         let task_metadata = metadata.clone();
     82         let child = self.cancellation.child_token();
     83         let completion_observer = child.clone();
     84         let phase_cancellation = child.clone();
     85         let abort = self.tasks.spawn_on(
     86             async move {
     87                 let result = task(child).await;
     88                 TaskCompletion {
     89                     metadata: task_metadata,
     90                     cancelled_at_return: completion_observer.is_cancelled(),
     91                     result,
     92                 }
     93             },
     94             &runtime,
     95         );
     96         self.controls.insert(
     97             abort.id(),
     98             TaskControl {
     99                 metadata,
    100                 cancellation: phase_cancellation,
    101             },
    102         );
    103         Ok(())
    104     }
    105 
    106     pub(crate) fn request_phase_cancellation(&self, phase: ShutdownPhase) {
    107         for control in self.controls.values() {
    108             if control.metadata.shutdown_phase() == Some(phase) {
    109                 control.cancellation.cancel();
    110             }
    111         }
    112     }
    113 
    114     pub(crate) async fn supervise_phase(
    115         &mut self,
    116         phase: ShutdownPhase,
    117     ) -> Result<Vec<SupervisedTaskExit>, SupervisionFailure> {
    118         if let Some(index) = self.observed.iter().position(Result::is_err) {
    119             return Err(self
    120                 .observed
    121                 .remove(index)
    122                 .expect("retained fatal outcome must exist")
    123                 .expect_err("retained fatal outcome must be a failure"));
    124         }
    125         while self.has_phase_work(phase) {
    126             let outcome = self
    127                 .join_task()
    128                 .await
    129                 .expect("phase work must retain a join-owned task");
    130             match outcome {
    131                 Ok(exit) => self.observed.push_back(Ok(exit)),
    132                 Err(failure) => return Err(failure),
    133             }
    134         }
    135         self.take_observed()
    136     }
    137 
    138     /// Observes all task exits, cancels peers on the first fatal outcome, and drains every join.
    139     ///
    140     /// Cancelling this wait retains already joined outcomes. A subsequent bulk,
    141     /// per-join or phase wait consumes those observations exactly once.
    142     pub async fn supervise(&mut self) -> Result<Vec<SupervisedTaskExit>, SupervisionFailure> {
    143         while let Some(outcome) = self.join_task().await {
    144             self.observed.push_back(outcome);
    145         }
    146         self.take_observed()
    147     }
    148 
    149     fn take_observed(&mut self) -> Result<Vec<SupervisedTaskExit>, SupervisionFailure> {
    150         let mut exits = Vec::with_capacity(self.observed.len());
    151         let mut first_failure = None;
    152         for outcome in self.observed.drain(..) {
    153             match outcome {
    154                 Ok(exit) => exits.push(exit),
    155                 Err(failure) => {
    156                     if first_failure.is_none() {
    157                         first_failure = Some(failure);
    158                     }
    159                 }
    160             }
    161         }
    162         if let Some(failure) = first_failure {
    163             Err(failure)
    164         } else {
    165             Ok(exits)
    166         }
    167     }
    168 
    169     /// Returns the next classified task outcome as soon as it is joined.
    170     ///
    171     /// Optional failures are returned without cancelling peers. A fatal
    172     /// outcome requests cancellation before it is returned. Cancelling this
    173     /// wait does not remove a task or lose its later outcome. Previously joined
    174     /// outcomes retained by a cancelled bulk or phase wait are returned first.
    175     pub async fn join_next(&mut self) -> Option<Result<SupervisedTaskExit, SupervisionFailure>> {
    176         if let Some(outcome) = self.observed.pop_front() {
    177             return Some(outcome);
    178         }
    179         self.join_task().await
    180     }
    181 
    182     async fn join_task(&mut self) -> Option<Result<SupervisedTaskExit, SupervisionFailure>> {
    183         let joined = self.tasks.join_next_with_id().await?;
    184         let outcome = self.classify_join(joined);
    185         if outcome.is_err() {
    186             self.cancellation.cancel();
    187         }
    188         Some(outcome)
    189     }
    190 
    191     fn classify_join(
    192         &mut self,
    193         joined: Result<(Id, TaskCompletion), JoinError>,
    194     ) -> Result<SupervisedTaskExit, SupervisionFailure> {
    195         match joined {
    196             Ok((id, completion)) => {
    197                 self.controls.remove(&id);
    198                 classify_completion(completion)
    199             }
    200             Err(error) => {
    201                 let metadata = self
    202                     .controls
    203                     .remove(&error.id())
    204                     .map(|control| control.metadata);
    205                 let cancelled_during_shutdown =
    206                     error.is_cancelled() && self.cancellation.is_cancelled();
    207                 if cancelled_during_shutdown && let Some(metadata) = metadata {
    208                     return Ok(SupervisedTaskExit::expected(metadata));
    209                 }
    210                 let kind = if error.is_panic() {
    211                     SupervisionFailureKind::TaskPanicked
    212                 } else if error.is_cancelled() {
    213                     SupervisionFailureKind::UnexpectedCancellation
    214                 } else {
    215                     SupervisionFailureKind::JoinFailed
    216                 };
    217                 match metadata {
    218                     Some(metadata) if !metadata.classification().failure_is_fatal() => Ok(
    219                         SupervisedTaskExit::optional_failure(metadata, Box::new(error)),
    220                     ),
    221                     metadata => Err(SupervisionFailure::new(
    222                         metadata,
    223                         kind,
    224                         Some(Box::new(error)),
    225                     )),
    226                 }
    227             }
    228         }
    229     }
    230 
    231     pub(crate) fn unfinished_work(&self) -> UnfinishedWork {
    232         if self.controls.is_empty() {
    233             UnfinishedWork::None
    234         } else if self
    235             .controls
    236             .values()
    237             .any(|control| control.metadata.classification().failure_is_fatal())
    238         {
    239             UnfinishedWork::FatalAuthoritative
    240         } else {
    241             UnfinishedWork::RecoverableOptional
    242         }
    243     }
    244 
    245     /// Joins aborted work while retaining unconsumed observations for an explicit read.
    246     pub(crate) async fn abort_and_drain(&mut self) {
    247         self.cancellation.cancel();
    248         self.tasks.abort_all();
    249         while let Some(outcome) = self.join_task().await {
    250             self.observed.push_back(outcome);
    251         }
    252     }
    253 
    254     fn has_phase_work(&self, phase: ShutdownPhase) -> bool {
    255         self.controls.values().any(|control| {
    256             control.metadata.shutdown_phase() == Some(phase)
    257                 || (phase == ShutdownPhase::DrainOperations
    258                     && control.metadata.classification() == TaskClassification::OneShot)
    259         })
    260     }
    261 }
    262 
    263 impl Default for TaskSupervisor {
    264     fn default() -> Self {
    265         Self::new()
    266     }
    267 }
    268 
    269 fn classify_completion(
    270     completion: TaskCompletion,
    271 ) -> Result<SupervisedTaskExit, SupervisionFailure> {
    272     match completion.result {
    273         Ok(())
    274             if completion.metadata.classification() == TaskClassification::Critical
    275                 && !completion.cancelled_at_return =>
    276         {
    277             Err(SupervisionFailure::new(
    278                 Some(completion.metadata),
    279                 SupervisionFailureKind::UnexpectedCompletion,
    280                 None,
    281             ))
    282         }
    283         Ok(()) => Ok(SupervisedTaskExit::expected(completion.metadata)),
    284         Err(error) if completion.metadata.classification().failure_is_fatal() => {
    285             Err(SupervisionFailure::new(
    286                 Some(completion.metadata),
    287                 SupervisionFailureKind::TaskReturnedError,
    288                 Some(Box::new(error)),
    289             ))
    290         }
    291         Err(error) => Ok(SupervisedTaskExit::optional_failure(
    292             completion.metadata,
    293             Box::new(error),
    294         )),
    295     }
    296 }
    297 
    298 struct TaskCompletion {
    299     metadata: TaskMetadata,
    300     cancelled_at_return: bool,
    301     result: Result<(), HostError>,
    302 }
    303 
    304 /// Nonfatal observed task completion retained for metrics and status consumers.
    305 pub struct SupervisedTaskExit {
    306     metadata: TaskMetadata,
    307     status: SupervisedTaskExitStatus,
    308     source: Option<Box<dyn Error + Send + Sync + 'static>>,
    309 }
    310 
    311 impl SupervisedTaskExit {
    312     fn expected(metadata: TaskMetadata) -> Self {
    313         Self {
    314             metadata,
    315             status: SupervisedTaskExitStatus::ExpectedCompletion,
    316             source: None,
    317         }
    318     }
    319 
    320     fn optional_failure(
    321         metadata: TaskMetadata,
    322         source: Box<dyn Error + Send + Sync + 'static>,
    323     ) -> Self {
    324         Self {
    325             metadata,
    326             status: SupervisedTaskExitStatus::OptionalFailure,
    327             source: Some(source),
    328         }
    329     }
    330 
    331     #[must_use]
    332     pub const fn metadata(&self) -> &TaskMetadata {
    333         &self.metadata
    334     }
    335 
    336     #[must_use]
    337     pub const fn status(&self) -> SupervisedTaskExitStatus {
    338         self.status
    339     }
    340 
    341     /// Returns the trusted internal cause for an optional failure.
    342     #[must_use]
    343     pub fn source(&self) -> Option<&(dyn Error + Send + Sync + 'static)> {
    344         self.source.as_deref()
    345     }
    346 }
    347 
    348 impl fmt::Debug for SupervisedTaskExit {
    349     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
    350         formatter
    351             .debug_struct("SupervisedTaskExit")
    352             .field("metadata", &self.metadata)
    353             .field("status", &self.status)
    354             .field("source", &self.source.as_ref().map(|_| "<redacted>"))
    355             .finish()
    356     }
    357 }
    358 
    359 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
    360 pub enum SupervisedTaskExitStatus {
    361     ExpectedCompletion,
    362     OptionalFailure,
    363 }
    364 
    365 /// Fatal supervisor outcome after every remaining task has been joined.
    366 pub struct SupervisionFailure {
    367     metadata: Option<TaskMetadata>,
    368     kind: SupervisionFailureKind,
    369     source: Option<Box<dyn Error + Send + Sync + 'static>>,
    370 }
    371 
    372 impl SupervisionFailure {
    373     fn new(
    374         metadata: Option<TaskMetadata>,
    375         kind: SupervisionFailureKind,
    376         source: Option<Box<dyn Error + Send + Sync + 'static>>,
    377     ) -> Self {
    378         Self {
    379             metadata,
    380             kind,
    381             source,
    382         }
    383     }
    384 
    385     #[must_use]
    386     pub const fn metadata(&self) -> Option<&TaskMetadata> {
    387         self.metadata.as_ref()
    388     }
    389 
    390     #[must_use]
    391     pub const fn kind(&self) -> SupervisionFailureKind {
    392         self.kind
    393     }
    394 }
    395 
    396 impl fmt::Debug for SupervisionFailure {
    397     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
    398         formatter
    399             .debug_struct("SupervisionFailure")
    400             .field("metadata", &self.metadata)
    401             .field("kind", &self.kind)
    402             .field("source", &self.source.as_ref().map(|_| "<redacted>"))
    403             .finish()
    404     }
    405 }
    406 
    407 impl fmt::Display for SupervisionFailure {
    408     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
    409         formatter.write_str("authoritative service task supervision failed")
    410     }
    411 }
    412 
    413 impl Error for SupervisionFailure {
    414     fn source(&self) -> Option<&(dyn Error + 'static)> {
    415         self.source
    416             .as_deref()
    417             .map(|source| source as &(dyn Error + 'static))
    418     }
    419 }
    420 
    421 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
    422 pub enum SupervisionFailureKind {
    423     TaskReturnedError,
    424     TaskPanicked,
    425     UnexpectedCompletion,
    426     UnexpectedCancellation,
    427     JoinFailed,
    428 }
    429 
    430 /// Failure to register a task without spawning it.
    431 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
    432 pub enum TaskRegistrationError {
    433     DuplicateName,
    434     NoRuntime,
    435 }
    436 
    437 impl fmt::Display for TaskRegistrationError {
    438     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
    439         formatter.write_str(match self {
    440             Self::DuplicateName => "supervised task name is already active",
    441             Self::NoRuntime => "supervised task registration requires an active runtime",
    442         })
    443     }
    444 }
    445 
    446 impl Error for TaskRegistrationError {}
    447 
    448 #[cfg(test)]
    449 mod tests {
    450     use std::sync::{
    451         Arc,
    452         atomic::{AtomicUsize, Ordering},
    453     };
    454 
    455     use crate::{HostErrorKind, ShutdownPhase, TaskClassification, TaskName};
    456 
    457     use super::*;
    458 
    459     fn metadata(name: &str, classification: TaskClassification) -> TaskMetadata {
    460         let shutdown_phase = classification
    461             .requires_shutdown_phase()
    462             .then_some(ShutdownPhase::CancelIngress);
    463         TaskMetadata::new(TaskName::new(name).unwrap(), classification, shutdown_phase).unwrap()
    464     }
    465 
    466     fn metadata_at(
    467         name: &str,
    468         classification: TaskClassification,
    469         shutdown_phase: Option<ShutdownPhase>,
    470     ) -> TaskMetadata {
    471         TaskMetadata::new(TaskName::new(name).unwrap(), classification, shutdown_phase).unwrap()
    472     }
    473 
    474     #[test]
    475     fn registration_without_a_runtime_fails_before_spawning() {
    476         let mut supervisor = TaskSupervisor::new();
    477         assert_eq!(
    478             supervisor.spawn(
    479                 metadata("critical_worker", TaskClassification::Critical),
    480                 |_| async { Ok(()) },
    481             ),
    482             Err(TaskRegistrationError::NoRuntime)
    483         );
    484         assert!(supervisor.is_empty());
    485     }
    486 
    487     #[tokio::test]
    488     async fn critical_error_cancels_peers_propagates_and_drains_all_joins() {
    489         let drained = Arc::new(AtomicUsize::new(0));
    490         let mut supervisor = TaskSupervisor::new();
    491         let peer_drained = Arc::clone(&drained);
    492         supervisor
    493             .spawn(
    494                 metadata("peer_worker", TaskClassification::Critical),
    495                 move |token| async move {
    496                     token.cancelled().await;
    497                     peer_drained.fetch_add(1, Ordering::SeqCst);
    498                     Ok(())
    499                 },
    500             )
    501             .unwrap();
    502         supervisor
    503             .spawn(
    504                 metadata("failing_worker", TaskClassification::Critical),
    505                 |_| async { Err(HostError::new(HostErrorKind::TaskFailure)) },
    506             )
    507             .unwrap();
    508 
    509         let error = supervisor.supervise().await.unwrap_err();
    510         assert_eq!(error.kind(), SupervisionFailureKind::TaskReturnedError);
    511         assert_eq!(error.metadata().unwrap().name().as_str(), "failing_worker");
    512         assert!(error.source().is_some());
    513         assert_eq!(drained.load(Ordering::SeqCst), 1);
    514         assert!(supervisor.is_empty());
    515         assert!(supervisor.cancellation_token().is_cancelled());
    516     }
    517 
    518     #[tokio::test]
    519     async fn phase_cancellation_stops_and_joins_only_the_assigned_tasks() {
    520         let ingress_stopped = Arc::new(AtomicUsize::new(0));
    521         let network_stopped = Arc::new(AtomicUsize::new(0));
    522         let mut supervisor = TaskSupervisor::new();
    523         let ingress = Arc::clone(&ingress_stopped);
    524         supervisor
    525             .spawn(
    526                 metadata_at(
    527                     "ingress_worker",
    528                     TaskClassification::Critical,
    529                     Some(ShutdownPhase::CancelIngress),
    530                 ),
    531                 move |token| async move {
    532                     token.cancelled().await;
    533                     ingress.fetch_add(1, Ordering::SeqCst);
    534                     Ok(())
    535                 },
    536             )
    537             .unwrap();
    538         let network = Arc::clone(&network_stopped);
    539         supervisor
    540             .spawn(
    541                 metadata_at(
    542                     "network_worker",
    543                     TaskClassification::Critical,
    544                     Some(ShutdownPhase::CloseNetwork),
    545                 ),
    546                 move |token| async move {
    547                     token.cancelled().await;
    548                     network.fetch_add(1, Ordering::SeqCst);
    549                     Ok(())
    550                 },
    551             )
    552             .unwrap();
    553 
    554         supervisor.request_phase_cancellation(ShutdownPhase::CancelIngress);
    555         let ingress_exits = supervisor
    556             .supervise_phase(ShutdownPhase::CancelIngress)
    557             .await
    558             .unwrap();
    559         assert_eq!(ingress_exits.len(), 1);
    560         assert_eq!(ingress_stopped.load(Ordering::SeqCst), 1);
    561         assert_eq!(network_stopped.load(Ordering::SeqCst), 0);
    562         assert_eq!(supervisor.task_count(), 1);
    563 
    564         supervisor.request_phase_cancellation(ShutdownPhase::CloseNetwork);
    565         let network_exits = supervisor
    566             .supervise_phase(ShutdownPhase::CloseNetwork)
    567             .await
    568             .unwrap();
    569         assert_eq!(network_exits.len(), 1);
    570         assert_eq!(network_stopped.load(Ordering::SeqCst), 1);
    571         assert!(supervisor.is_empty());
    572     }
    573 
    574     #[tokio::test]
    575     async fn drain_operations_joins_one_shot_work_without_cancelling_it() {
    576         let mut supervisor = TaskSupervisor::new();
    577         supervisor
    578             .spawn(
    579                 metadata_at("bounded_operation", TaskClassification::OneShot, None),
    580                 |token| async move {
    581                     assert!(!token.is_cancelled());
    582                     Ok(())
    583                 },
    584             )
    585             .unwrap();
    586 
    587         supervisor.request_phase_cancellation(ShutdownPhase::DrainOperations);
    588         let exits = supervisor
    589             .supervise_phase(ShutdownPhase::DrainOperations)
    590             .await
    591             .unwrap();
    592         assert_eq!(exits.len(), 1);
    593         assert_eq!(
    594             exits[0].status(),
    595             SupervisedTaskExitStatus::ExpectedCompletion
    596         );
    597         assert!(supervisor.is_empty());
    598     }
    599 
    600     #[tokio::test]
    601     async fn critical_panic_and_early_success_are_fatal() {
    602         for (name, expected, panic_task) in [
    603             ("panic_worker", SupervisionFailureKind::TaskPanicked, true),
    604             (
    605                 "early_worker",
    606                 SupervisionFailureKind::UnexpectedCompletion,
    607                 false,
    608             ),
    609         ] {
    610             let mut supervisor = TaskSupervisor::new();
    611             supervisor
    612                 .spawn(
    613                     metadata(name, TaskClassification::Critical),
    614                     move |_| async move {
    615                         assert!(!panic_task, "sensitive panic payload");
    616                         Ok(())
    617                     },
    618                 )
    619                 .unwrap();
    620             let error = supervisor.supervise().await.unwrap_err();
    621             assert_eq!(error.kind(), expected);
    622             assert!(!error.to_string().contains("sensitive"));
    623             assert!(supervisor.is_empty());
    624         }
    625     }
    626 
    627     #[tokio::test]
    628     async fn optional_error_and_one_shot_success_are_observed_without_failure() {
    629         let mut supervisor = TaskSupervisor::new();
    630         supervisor
    631             .spawn(
    632                 metadata("optional_worker", TaskClassification::Optional),
    633                 |_| async { Err(HostError::new(HostErrorKind::TaskFailure)) },
    634             )
    635             .unwrap();
    636         supervisor
    637             .spawn(
    638                 metadata("startup_once", TaskClassification::OneShot),
    639                 |_| async { Ok(()) },
    640             )
    641             .unwrap();
    642 
    643         let mut exits = supervisor.supervise().await.unwrap();
    644         exits.sort_by(|left, right| left.metadata().name().cmp(right.metadata().name()));
    645         assert_eq!(exits.len(), 2);
    646         assert_eq!(exits[0].status(), SupervisedTaskExitStatus::OptionalFailure);
    647         assert!(exits[0].source().is_some());
    648         assert_eq!(
    649             exits[1].status(),
    650             SupervisedTaskExitStatus::ExpectedCompletion
    651         );
    652         assert!(supervisor.is_empty());
    653     }
    654 
    655     #[tokio::test]
    656     async fn optional_failure_is_observable_while_a_critical_peer_is_alive() {
    657         let mut supervisor = TaskSupervisor::new();
    658         supervisor
    659             .spawn(
    660                 metadata("critical_worker", TaskClassification::Critical),
    661                 |token| async move {
    662                     token.cancelled().await;
    663                     Ok(())
    664                 },
    665             )
    666             .unwrap();
    667         supervisor
    668             .spawn(
    669                 metadata("optional_worker", TaskClassification::Optional),
    670                 |_| async { Err(HostError::new(HostErrorKind::TaskFailure)) },
    671             )
    672             .unwrap();
    673 
    674         let exit = supervisor.join_next().await.unwrap().unwrap();
    675         assert_eq!(exit.metadata().name().as_str(), "optional_worker");
    676         assert_eq!(exit.status(), SupervisedTaskExitStatus::OptionalFailure);
    677         assert_eq!(supervisor.task_count(), 1);
    678         assert!(!supervisor.cancellation_token().is_cancelled());
    679 
    680         supervisor.request_cancellation();
    681         assert!(supervisor.join_next().await.unwrap().is_ok());
    682         assert!(supervisor.is_empty());
    683     }
    684 
    685     #[tokio::test]
    686     async fn optional_failure_remains_observed_before_a_later_fatal_exit() {
    687         let (release_fatal, wait_for_release) = tokio::sync::oneshot::channel();
    688         let mut supervisor = TaskSupervisor::new();
    689         supervisor
    690             .spawn(
    691                 metadata("fatal_worker", TaskClassification::Critical),
    692                 |_| async move {
    693                     let _ = wait_for_release.await;
    694                     Err(HostError::new(HostErrorKind::TaskFailure))
    695                 },
    696             )
    697             .unwrap();
    698         supervisor
    699             .spawn(
    700                 metadata("optional_worker", TaskClassification::Optional),
    701                 |_| async { Err(HostError::new(HostErrorKind::TaskFailure)) },
    702             )
    703             .unwrap();
    704 
    705         let optional = supervisor.join_next().await.unwrap().unwrap();
    706         assert_eq!(optional.metadata().name().as_str(), "optional_worker");
    707         assert_eq!(optional.status(), SupervisedTaskExitStatus::OptionalFailure);
    708         release_fatal.send(()).unwrap();
    709 
    710         let fatal = supervisor.join_next().await.unwrap().unwrap_err();
    711         assert_eq!(fatal.kind(), SupervisionFailureKind::TaskReturnedError);
    712         assert_eq!(fatal.metadata().unwrap().name().as_str(), "fatal_worker");
    713         assert!(supervisor.cancellation_token().is_cancelled());
    714         assert!(supervisor.is_empty());
    715     }
    716 
    717     #[tokio::test]
    718     async fn supervision_retains_only_the_first_of_multiple_fatal_failures() {
    719         let mut supervisor = TaskSupervisor::new();
    720         for name in ["first_failure", "second_failure"] {
    721             supervisor
    722                 .spawn(metadata(name, TaskClassification::Critical), |_| async {
    723                     Err(HostError::new(HostErrorKind::TaskFailure))
    724                 })
    725                 .unwrap();
    726         }
    727 
    728         let error = supervisor.supervise().await.unwrap_err();
    729         assert_eq!(error.kind(), SupervisionFailureKind::TaskReturnedError);
    730         assert!(matches!(
    731             error.metadata().unwrap().name().as_str(),
    732             "first_failure" | "second_failure"
    733         ));
    734         assert!(supervisor.is_empty());
    735     }
    736 
    737     #[tokio::test]
    738     async fn optional_panic_is_an_observable_nonfatal_exit() {
    739         let mut supervisor = TaskSupervisor::new();
    740         supervisor
    741             .spawn(
    742                 metadata("optional_panic", TaskClassification::Optional),
    743                 |_| async {
    744                     panic!("redacted optional panic");
    745                     #[allow(unreachable_code)]
    746                     Ok(())
    747                 },
    748             )
    749             .unwrap();
    750 
    751         let exit = supervisor.join_next().await.unwrap().unwrap();
    752         assert_eq!(exit.status(), SupervisedTaskExitStatus::OptionalFailure);
    753         assert!(exit.source().is_some());
    754         assert!(supervisor.is_empty());
    755     }
    756 
    757     #[tokio::test]
    758     async fn externally_cancelled_critical_task_may_complete_successfully() {
    759         let mut supervisor = TaskSupervisor::new();
    760         supervisor
    761             .spawn(
    762                 metadata("critical_worker", TaskClassification::Critical),
    763                 |token| async move {
    764                     token.cancelled().await;
    765                     Ok(())
    766                 },
    767             )
    768             .unwrap();
    769         supervisor.request_cancellation();
    770 
    771         let exits = supervisor.supervise().await.unwrap();
    772         assert_eq!(exits.len(), 1);
    773         assert_eq!(
    774             exits[0].status(),
    775             SupervisedTaskExitStatus::ExpectedCompletion
    776         );
    777         assert!(supervisor.is_empty());
    778     }
    779 
    780     #[tokio::test]
    781     async fn duplicate_active_names_are_rejected_without_detaching_work() {
    782         let mut supervisor = TaskSupervisor::new();
    783         supervisor
    784             .spawn(
    785                 metadata("same_worker", TaskClassification::Critical),
    786                 |token| async move {
    787                     token.cancelled().await;
    788                     Ok(())
    789                 },
    790             )
    791             .unwrap();
    792         assert_eq!(
    793             supervisor.spawn(
    794                 metadata("same_worker", TaskClassification::Critical),
    795                 |_| async { Ok(()) },
    796             ),
    797             Err(TaskRegistrationError::DuplicateName)
    798         );
    799         assert_eq!(supervisor.task_count(), 1);
    800         supervisor.request_cancellation();
    801         assert!(supervisor.supervise().await.is_ok());
    802     }
    803 }