supervisor.rs (27270B)
1 //! Join-owned task supervision with typed completion policy. 2 3 use core::{fmt, future::Future}; 4 use std::{ 5 collections::{HashMap, VecDeque}, 6 error::Error, 7 }; 8 9 use tokio::task::{Id, JoinError, JoinSet}; 10 11 use crate::HostError; 12 13 use super::{CancellationToken, ShutdownPhase, TaskClassification, TaskMetadata, UnfinishedWork}; 14 15 /// Owns every spawned service task until its join result is observed. 16 #[must_use = "a task supervisor must be run or drained so authoritative tasks are joined"] 17 pub struct TaskSupervisor { 18 cancellation: CancellationToken, 19 tasks: JoinSet<TaskCompletion>, 20 controls: HashMap<Id, TaskControl>, 21 observed: VecDeque<Result<SupervisedTaskExit, SupervisionFailure>>, 22 } 23 24 struct TaskControl { 25 metadata: TaskMetadata, 26 cancellation: CancellationToken, 27 } 28 29 impl TaskSupervisor { 30 pub fn new() -> Self { 31 Self { 32 cancellation: CancellationToken::new(), 33 tasks: JoinSet::new(), 34 controls: HashMap::new(), 35 observed: VecDeque::new(), 36 } 37 } 38 39 /// Returns a cloneable observer and cancellation authority for composition boundaries. 40 #[must_use] 41 pub fn cancellation_token(&self) -> CancellationToken { 42 self.cancellation.clone() 43 } 44 45 /// Requests cooperative cancellation of every task child token. 46 pub fn request_cancellation(&self) { 47 self.cancellation.cancel(); 48 } 49 50 /// Returns the number of tasks whose join results have not yet been collected. 51 #[must_use] 52 pub fn task_count(&self) -> usize { 53 self.tasks.len() 54 } 55 56 /// Returns whether every task has been joined; unconsumed observations may remain. 57 #[must_use] 58 pub fn is_empty(&self) -> bool { 59 self.tasks.is_empty() 60 } 61 62 /// Registers and spawns one task on the current runtime without exposing a detachable handle. 63 pub fn spawn<F, Fut>( 64 &mut self, 65 metadata: TaskMetadata, 66 task: F, 67 ) -> Result<(), TaskRegistrationError> 68 where 69 F: FnOnce(CancellationToken) -> Fut + Send + 'static, 70 Fut: Future<Output = Result<(), HostError>> + Send + 'static, 71 { 72 if self 73 .controls 74 .values() 75 .any(|active| active.metadata.name() == metadata.name()) 76 { 77 return Err(TaskRegistrationError::DuplicateName); 78 } 79 let runtime = 80 tokio::runtime::Handle::try_current().map_err(|_| TaskRegistrationError::NoRuntime)?; 81 let task_metadata = metadata.clone(); 82 let child = self.cancellation.child_token(); 83 let completion_observer = child.clone(); 84 let phase_cancellation = child.clone(); 85 let abort = self.tasks.spawn_on( 86 async move { 87 let result = task(child).await; 88 TaskCompletion { 89 metadata: task_metadata, 90 cancelled_at_return: completion_observer.is_cancelled(), 91 result, 92 } 93 }, 94 &runtime, 95 ); 96 self.controls.insert( 97 abort.id(), 98 TaskControl { 99 metadata, 100 cancellation: phase_cancellation, 101 }, 102 ); 103 Ok(()) 104 } 105 106 pub(crate) fn request_phase_cancellation(&self, phase: ShutdownPhase) { 107 for control in self.controls.values() { 108 if control.metadata.shutdown_phase() == Some(phase) { 109 control.cancellation.cancel(); 110 } 111 } 112 } 113 114 pub(crate) async fn supervise_phase( 115 &mut self, 116 phase: ShutdownPhase, 117 ) -> Result<Vec<SupervisedTaskExit>, SupervisionFailure> { 118 if let Some(index) = self.observed.iter().position(Result::is_err) { 119 return Err(self 120 .observed 121 .remove(index) 122 .expect("retained fatal outcome must exist") 123 .expect_err("retained fatal outcome must be a failure")); 124 } 125 while self.has_phase_work(phase) { 126 let outcome = self 127 .join_task() 128 .await 129 .expect("phase work must retain a join-owned task"); 130 match outcome { 131 Ok(exit) => self.observed.push_back(Ok(exit)), 132 Err(failure) => return Err(failure), 133 } 134 } 135 self.take_observed() 136 } 137 138 /// Observes all task exits, cancels peers on the first fatal outcome, and drains every join. 139 /// 140 /// Cancelling this wait retains already joined outcomes. A subsequent bulk, 141 /// per-join or phase wait consumes those observations exactly once. 142 pub async fn supervise(&mut self) -> Result<Vec<SupervisedTaskExit>, SupervisionFailure> { 143 while let Some(outcome) = self.join_task().await { 144 self.observed.push_back(outcome); 145 } 146 self.take_observed() 147 } 148 149 fn take_observed(&mut self) -> Result<Vec<SupervisedTaskExit>, SupervisionFailure> { 150 let mut exits = Vec::with_capacity(self.observed.len()); 151 let mut first_failure = None; 152 for outcome in self.observed.drain(..) { 153 match outcome { 154 Ok(exit) => exits.push(exit), 155 Err(failure) => { 156 if first_failure.is_none() { 157 first_failure = Some(failure); 158 } 159 } 160 } 161 } 162 if let Some(failure) = first_failure { 163 Err(failure) 164 } else { 165 Ok(exits) 166 } 167 } 168 169 /// Returns the next classified task outcome as soon as it is joined. 170 /// 171 /// Optional failures are returned without cancelling peers. A fatal 172 /// outcome requests cancellation before it is returned. Cancelling this 173 /// wait does not remove a task or lose its later outcome. Previously joined 174 /// outcomes retained by a cancelled bulk or phase wait are returned first. 175 pub async fn join_next(&mut self) -> Option<Result<SupervisedTaskExit, SupervisionFailure>> { 176 if let Some(outcome) = self.observed.pop_front() { 177 return Some(outcome); 178 } 179 self.join_task().await 180 } 181 182 async fn join_task(&mut self) -> Option<Result<SupervisedTaskExit, SupervisionFailure>> { 183 let joined = self.tasks.join_next_with_id().await?; 184 let outcome = self.classify_join(joined); 185 if outcome.is_err() { 186 self.cancellation.cancel(); 187 } 188 Some(outcome) 189 } 190 191 fn classify_join( 192 &mut self, 193 joined: Result<(Id, TaskCompletion), JoinError>, 194 ) -> Result<SupervisedTaskExit, SupervisionFailure> { 195 match joined { 196 Ok((id, completion)) => { 197 self.controls.remove(&id); 198 classify_completion(completion) 199 } 200 Err(error) => { 201 let metadata = self 202 .controls 203 .remove(&error.id()) 204 .map(|control| control.metadata); 205 let cancelled_during_shutdown = 206 error.is_cancelled() && self.cancellation.is_cancelled(); 207 if cancelled_during_shutdown && let Some(metadata) = metadata { 208 return Ok(SupervisedTaskExit::expected(metadata)); 209 } 210 let kind = if error.is_panic() { 211 SupervisionFailureKind::TaskPanicked 212 } else if error.is_cancelled() { 213 SupervisionFailureKind::UnexpectedCancellation 214 } else { 215 SupervisionFailureKind::JoinFailed 216 }; 217 match metadata { 218 Some(metadata) if !metadata.classification().failure_is_fatal() => Ok( 219 SupervisedTaskExit::optional_failure(metadata, Box::new(error)), 220 ), 221 metadata => Err(SupervisionFailure::new( 222 metadata, 223 kind, 224 Some(Box::new(error)), 225 )), 226 } 227 } 228 } 229 } 230 231 pub(crate) fn unfinished_work(&self) -> UnfinishedWork { 232 if self.controls.is_empty() { 233 UnfinishedWork::None 234 } else if self 235 .controls 236 .values() 237 .any(|control| control.metadata.classification().failure_is_fatal()) 238 { 239 UnfinishedWork::FatalAuthoritative 240 } else { 241 UnfinishedWork::RecoverableOptional 242 } 243 } 244 245 /// Joins aborted work while retaining unconsumed observations for an explicit read. 246 pub(crate) async fn abort_and_drain(&mut self) { 247 self.cancellation.cancel(); 248 self.tasks.abort_all(); 249 while let Some(outcome) = self.join_task().await { 250 self.observed.push_back(outcome); 251 } 252 } 253 254 fn has_phase_work(&self, phase: ShutdownPhase) -> bool { 255 self.controls.values().any(|control| { 256 control.metadata.shutdown_phase() == Some(phase) 257 || (phase == ShutdownPhase::DrainOperations 258 && control.metadata.classification() == TaskClassification::OneShot) 259 }) 260 } 261 } 262 263 impl Default for TaskSupervisor { 264 fn default() -> Self { 265 Self::new() 266 } 267 } 268 269 fn classify_completion( 270 completion: TaskCompletion, 271 ) -> Result<SupervisedTaskExit, SupervisionFailure> { 272 match completion.result { 273 Ok(()) 274 if completion.metadata.classification() == TaskClassification::Critical 275 && !completion.cancelled_at_return => 276 { 277 Err(SupervisionFailure::new( 278 Some(completion.metadata), 279 SupervisionFailureKind::UnexpectedCompletion, 280 None, 281 )) 282 } 283 Ok(()) => Ok(SupervisedTaskExit::expected(completion.metadata)), 284 Err(error) if completion.metadata.classification().failure_is_fatal() => { 285 Err(SupervisionFailure::new( 286 Some(completion.metadata), 287 SupervisionFailureKind::TaskReturnedError, 288 Some(Box::new(error)), 289 )) 290 } 291 Err(error) => Ok(SupervisedTaskExit::optional_failure( 292 completion.metadata, 293 Box::new(error), 294 )), 295 } 296 } 297 298 struct TaskCompletion { 299 metadata: TaskMetadata, 300 cancelled_at_return: bool, 301 result: Result<(), HostError>, 302 } 303 304 /// Nonfatal observed task completion retained for metrics and status consumers. 305 pub struct SupervisedTaskExit { 306 metadata: TaskMetadata, 307 status: SupervisedTaskExitStatus, 308 source: Option<Box<dyn Error + Send + Sync + 'static>>, 309 } 310 311 impl SupervisedTaskExit { 312 fn expected(metadata: TaskMetadata) -> Self { 313 Self { 314 metadata, 315 status: SupervisedTaskExitStatus::ExpectedCompletion, 316 source: None, 317 } 318 } 319 320 fn optional_failure( 321 metadata: TaskMetadata, 322 source: Box<dyn Error + Send + Sync + 'static>, 323 ) -> Self { 324 Self { 325 metadata, 326 status: SupervisedTaskExitStatus::OptionalFailure, 327 source: Some(source), 328 } 329 } 330 331 #[must_use] 332 pub const fn metadata(&self) -> &TaskMetadata { 333 &self.metadata 334 } 335 336 #[must_use] 337 pub const fn status(&self) -> SupervisedTaskExitStatus { 338 self.status 339 } 340 341 /// Returns the trusted internal cause for an optional failure. 342 #[must_use] 343 pub fn source(&self) -> Option<&(dyn Error + Send + Sync + 'static)> { 344 self.source.as_deref() 345 } 346 } 347 348 impl fmt::Debug for SupervisedTaskExit { 349 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 350 formatter 351 .debug_struct("SupervisedTaskExit") 352 .field("metadata", &self.metadata) 353 .field("status", &self.status) 354 .field("source", &self.source.as_ref().map(|_| "<redacted>")) 355 .finish() 356 } 357 } 358 359 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 360 pub enum SupervisedTaskExitStatus { 361 ExpectedCompletion, 362 OptionalFailure, 363 } 364 365 /// Fatal supervisor outcome after every remaining task has been joined. 366 pub struct SupervisionFailure { 367 metadata: Option<TaskMetadata>, 368 kind: SupervisionFailureKind, 369 source: Option<Box<dyn Error + Send + Sync + 'static>>, 370 } 371 372 impl SupervisionFailure { 373 fn new( 374 metadata: Option<TaskMetadata>, 375 kind: SupervisionFailureKind, 376 source: Option<Box<dyn Error + Send + Sync + 'static>>, 377 ) -> Self { 378 Self { 379 metadata, 380 kind, 381 source, 382 } 383 } 384 385 #[must_use] 386 pub const fn metadata(&self) -> Option<&TaskMetadata> { 387 self.metadata.as_ref() 388 } 389 390 #[must_use] 391 pub const fn kind(&self) -> SupervisionFailureKind { 392 self.kind 393 } 394 } 395 396 impl fmt::Debug for SupervisionFailure { 397 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 398 formatter 399 .debug_struct("SupervisionFailure") 400 .field("metadata", &self.metadata) 401 .field("kind", &self.kind) 402 .field("source", &self.source.as_ref().map(|_| "<redacted>")) 403 .finish() 404 } 405 } 406 407 impl fmt::Display for SupervisionFailure { 408 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 409 formatter.write_str("authoritative service task supervision failed") 410 } 411 } 412 413 impl Error for SupervisionFailure { 414 fn source(&self) -> Option<&(dyn Error + 'static)> { 415 self.source 416 .as_deref() 417 .map(|source| source as &(dyn Error + 'static)) 418 } 419 } 420 421 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 422 pub enum SupervisionFailureKind { 423 TaskReturnedError, 424 TaskPanicked, 425 UnexpectedCompletion, 426 UnexpectedCancellation, 427 JoinFailed, 428 } 429 430 /// Failure to register a task without spawning it. 431 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 432 pub enum TaskRegistrationError { 433 DuplicateName, 434 NoRuntime, 435 } 436 437 impl fmt::Display for TaskRegistrationError { 438 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 439 formatter.write_str(match self { 440 Self::DuplicateName => "supervised task name is already active", 441 Self::NoRuntime => "supervised task registration requires an active runtime", 442 }) 443 } 444 } 445 446 impl Error for TaskRegistrationError {} 447 448 #[cfg(test)] 449 mod tests { 450 use std::sync::{ 451 Arc, 452 atomic::{AtomicUsize, Ordering}, 453 }; 454 455 use crate::{HostErrorKind, ShutdownPhase, TaskClassification, TaskName}; 456 457 use super::*; 458 459 fn metadata(name: &str, classification: TaskClassification) -> TaskMetadata { 460 let shutdown_phase = classification 461 .requires_shutdown_phase() 462 .then_some(ShutdownPhase::CancelIngress); 463 TaskMetadata::new(TaskName::new(name).unwrap(), classification, shutdown_phase).unwrap() 464 } 465 466 fn metadata_at( 467 name: &str, 468 classification: TaskClassification, 469 shutdown_phase: Option<ShutdownPhase>, 470 ) -> TaskMetadata { 471 TaskMetadata::new(TaskName::new(name).unwrap(), classification, shutdown_phase).unwrap() 472 } 473 474 #[test] 475 fn registration_without_a_runtime_fails_before_spawning() { 476 let mut supervisor = TaskSupervisor::new(); 477 assert_eq!( 478 supervisor.spawn( 479 metadata("critical_worker", TaskClassification::Critical), 480 |_| async { Ok(()) }, 481 ), 482 Err(TaskRegistrationError::NoRuntime) 483 ); 484 assert!(supervisor.is_empty()); 485 } 486 487 #[tokio::test] 488 async fn critical_error_cancels_peers_propagates_and_drains_all_joins() { 489 let drained = Arc::new(AtomicUsize::new(0)); 490 let mut supervisor = TaskSupervisor::new(); 491 let peer_drained = Arc::clone(&drained); 492 supervisor 493 .spawn( 494 metadata("peer_worker", TaskClassification::Critical), 495 move |token| async move { 496 token.cancelled().await; 497 peer_drained.fetch_add(1, Ordering::SeqCst); 498 Ok(()) 499 }, 500 ) 501 .unwrap(); 502 supervisor 503 .spawn( 504 metadata("failing_worker", TaskClassification::Critical), 505 |_| async { Err(HostError::new(HostErrorKind::TaskFailure)) }, 506 ) 507 .unwrap(); 508 509 let error = supervisor.supervise().await.unwrap_err(); 510 assert_eq!(error.kind(), SupervisionFailureKind::TaskReturnedError); 511 assert_eq!(error.metadata().unwrap().name().as_str(), "failing_worker"); 512 assert!(error.source().is_some()); 513 assert_eq!(drained.load(Ordering::SeqCst), 1); 514 assert!(supervisor.is_empty()); 515 assert!(supervisor.cancellation_token().is_cancelled()); 516 } 517 518 #[tokio::test] 519 async fn phase_cancellation_stops_and_joins_only_the_assigned_tasks() { 520 let ingress_stopped = Arc::new(AtomicUsize::new(0)); 521 let network_stopped = Arc::new(AtomicUsize::new(0)); 522 let mut supervisor = TaskSupervisor::new(); 523 let ingress = Arc::clone(&ingress_stopped); 524 supervisor 525 .spawn( 526 metadata_at( 527 "ingress_worker", 528 TaskClassification::Critical, 529 Some(ShutdownPhase::CancelIngress), 530 ), 531 move |token| async move { 532 token.cancelled().await; 533 ingress.fetch_add(1, Ordering::SeqCst); 534 Ok(()) 535 }, 536 ) 537 .unwrap(); 538 let network = Arc::clone(&network_stopped); 539 supervisor 540 .spawn( 541 metadata_at( 542 "network_worker", 543 TaskClassification::Critical, 544 Some(ShutdownPhase::CloseNetwork), 545 ), 546 move |token| async move { 547 token.cancelled().await; 548 network.fetch_add(1, Ordering::SeqCst); 549 Ok(()) 550 }, 551 ) 552 .unwrap(); 553 554 supervisor.request_phase_cancellation(ShutdownPhase::CancelIngress); 555 let ingress_exits = supervisor 556 .supervise_phase(ShutdownPhase::CancelIngress) 557 .await 558 .unwrap(); 559 assert_eq!(ingress_exits.len(), 1); 560 assert_eq!(ingress_stopped.load(Ordering::SeqCst), 1); 561 assert_eq!(network_stopped.load(Ordering::SeqCst), 0); 562 assert_eq!(supervisor.task_count(), 1); 563 564 supervisor.request_phase_cancellation(ShutdownPhase::CloseNetwork); 565 let network_exits = supervisor 566 .supervise_phase(ShutdownPhase::CloseNetwork) 567 .await 568 .unwrap(); 569 assert_eq!(network_exits.len(), 1); 570 assert_eq!(network_stopped.load(Ordering::SeqCst), 1); 571 assert!(supervisor.is_empty()); 572 } 573 574 #[tokio::test] 575 async fn drain_operations_joins_one_shot_work_without_cancelling_it() { 576 let mut supervisor = TaskSupervisor::new(); 577 supervisor 578 .spawn( 579 metadata_at("bounded_operation", TaskClassification::OneShot, None), 580 |token| async move { 581 assert!(!token.is_cancelled()); 582 Ok(()) 583 }, 584 ) 585 .unwrap(); 586 587 supervisor.request_phase_cancellation(ShutdownPhase::DrainOperations); 588 let exits = supervisor 589 .supervise_phase(ShutdownPhase::DrainOperations) 590 .await 591 .unwrap(); 592 assert_eq!(exits.len(), 1); 593 assert_eq!( 594 exits[0].status(), 595 SupervisedTaskExitStatus::ExpectedCompletion 596 ); 597 assert!(supervisor.is_empty()); 598 } 599 600 #[tokio::test] 601 async fn critical_panic_and_early_success_are_fatal() { 602 for (name, expected, panic_task) in [ 603 ("panic_worker", SupervisionFailureKind::TaskPanicked, true), 604 ( 605 "early_worker", 606 SupervisionFailureKind::UnexpectedCompletion, 607 false, 608 ), 609 ] { 610 let mut supervisor = TaskSupervisor::new(); 611 supervisor 612 .spawn( 613 metadata(name, TaskClassification::Critical), 614 move |_| async move { 615 assert!(!panic_task, "sensitive panic payload"); 616 Ok(()) 617 }, 618 ) 619 .unwrap(); 620 let error = supervisor.supervise().await.unwrap_err(); 621 assert_eq!(error.kind(), expected); 622 assert!(!error.to_string().contains("sensitive")); 623 assert!(supervisor.is_empty()); 624 } 625 } 626 627 #[tokio::test] 628 async fn optional_error_and_one_shot_success_are_observed_without_failure() { 629 let mut supervisor = TaskSupervisor::new(); 630 supervisor 631 .spawn( 632 metadata("optional_worker", TaskClassification::Optional), 633 |_| async { Err(HostError::new(HostErrorKind::TaskFailure)) }, 634 ) 635 .unwrap(); 636 supervisor 637 .spawn( 638 metadata("startup_once", TaskClassification::OneShot), 639 |_| async { Ok(()) }, 640 ) 641 .unwrap(); 642 643 let mut exits = supervisor.supervise().await.unwrap(); 644 exits.sort_by(|left, right| left.metadata().name().cmp(right.metadata().name())); 645 assert_eq!(exits.len(), 2); 646 assert_eq!(exits[0].status(), SupervisedTaskExitStatus::OptionalFailure); 647 assert!(exits[0].source().is_some()); 648 assert_eq!( 649 exits[1].status(), 650 SupervisedTaskExitStatus::ExpectedCompletion 651 ); 652 assert!(supervisor.is_empty()); 653 } 654 655 #[tokio::test] 656 async fn optional_failure_is_observable_while_a_critical_peer_is_alive() { 657 let mut supervisor = TaskSupervisor::new(); 658 supervisor 659 .spawn( 660 metadata("critical_worker", TaskClassification::Critical), 661 |token| async move { 662 token.cancelled().await; 663 Ok(()) 664 }, 665 ) 666 .unwrap(); 667 supervisor 668 .spawn( 669 metadata("optional_worker", TaskClassification::Optional), 670 |_| async { Err(HostError::new(HostErrorKind::TaskFailure)) }, 671 ) 672 .unwrap(); 673 674 let exit = supervisor.join_next().await.unwrap().unwrap(); 675 assert_eq!(exit.metadata().name().as_str(), "optional_worker"); 676 assert_eq!(exit.status(), SupervisedTaskExitStatus::OptionalFailure); 677 assert_eq!(supervisor.task_count(), 1); 678 assert!(!supervisor.cancellation_token().is_cancelled()); 679 680 supervisor.request_cancellation(); 681 assert!(supervisor.join_next().await.unwrap().is_ok()); 682 assert!(supervisor.is_empty()); 683 } 684 685 #[tokio::test] 686 async fn optional_failure_remains_observed_before_a_later_fatal_exit() { 687 let (release_fatal, wait_for_release) = tokio::sync::oneshot::channel(); 688 let mut supervisor = TaskSupervisor::new(); 689 supervisor 690 .spawn( 691 metadata("fatal_worker", TaskClassification::Critical), 692 |_| async move { 693 let _ = wait_for_release.await; 694 Err(HostError::new(HostErrorKind::TaskFailure)) 695 }, 696 ) 697 .unwrap(); 698 supervisor 699 .spawn( 700 metadata("optional_worker", TaskClassification::Optional), 701 |_| async { Err(HostError::new(HostErrorKind::TaskFailure)) }, 702 ) 703 .unwrap(); 704 705 let optional = supervisor.join_next().await.unwrap().unwrap(); 706 assert_eq!(optional.metadata().name().as_str(), "optional_worker"); 707 assert_eq!(optional.status(), SupervisedTaskExitStatus::OptionalFailure); 708 release_fatal.send(()).unwrap(); 709 710 let fatal = supervisor.join_next().await.unwrap().unwrap_err(); 711 assert_eq!(fatal.kind(), SupervisionFailureKind::TaskReturnedError); 712 assert_eq!(fatal.metadata().unwrap().name().as_str(), "fatal_worker"); 713 assert!(supervisor.cancellation_token().is_cancelled()); 714 assert!(supervisor.is_empty()); 715 } 716 717 #[tokio::test] 718 async fn supervision_retains_only_the_first_of_multiple_fatal_failures() { 719 let mut supervisor = TaskSupervisor::new(); 720 for name in ["first_failure", "second_failure"] { 721 supervisor 722 .spawn(metadata(name, TaskClassification::Critical), |_| async { 723 Err(HostError::new(HostErrorKind::TaskFailure)) 724 }) 725 .unwrap(); 726 } 727 728 let error = supervisor.supervise().await.unwrap_err(); 729 assert_eq!(error.kind(), SupervisionFailureKind::TaskReturnedError); 730 assert!(matches!( 731 error.metadata().unwrap().name().as_str(), 732 "first_failure" | "second_failure" 733 )); 734 assert!(supervisor.is_empty()); 735 } 736 737 #[tokio::test] 738 async fn optional_panic_is_an_observable_nonfatal_exit() { 739 let mut supervisor = TaskSupervisor::new(); 740 supervisor 741 .spawn( 742 metadata("optional_panic", TaskClassification::Optional), 743 |_| async { 744 panic!("redacted optional panic"); 745 #[allow(unreachable_code)] 746 Ok(()) 747 }, 748 ) 749 .unwrap(); 750 751 let exit = supervisor.join_next().await.unwrap().unwrap(); 752 assert_eq!(exit.status(), SupervisedTaskExitStatus::OptionalFailure); 753 assert!(exit.source().is_some()); 754 assert!(supervisor.is_empty()); 755 } 756 757 #[tokio::test] 758 async fn externally_cancelled_critical_task_may_complete_successfully() { 759 let mut supervisor = TaskSupervisor::new(); 760 supervisor 761 .spawn( 762 metadata("critical_worker", TaskClassification::Critical), 763 |token| async move { 764 token.cancelled().await; 765 Ok(()) 766 }, 767 ) 768 .unwrap(); 769 supervisor.request_cancellation(); 770 771 let exits = supervisor.supervise().await.unwrap(); 772 assert_eq!(exits.len(), 1); 773 assert_eq!( 774 exits[0].status(), 775 SupervisedTaskExitStatus::ExpectedCompletion 776 ); 777 assert!(supervisor.is_empty()); 778 } 779 780 #[tokio::test] 781 async fn duplicate_active_names_are_rejected_without_detaching_work() { 782 let mut supervisor = TaskSupervisor::new(); 783 supervisor 784 .spawn( 785 metadata("same_worker", TaskClassification::Critical), 786 |token| async move { 787 token.cancelled().await; 788 Ok(()) 789 }, 790 ) 791 .unwrap(); 792 assert_eq!( 793 supervisor.spawn( 794 metadata("same_worker", TaskClassification::Critical), 795 |_| async { Ok(()) }, 796 ), 797 Err(TaskRegistrationError::DuplicateName) 798 ); 799 assert_eq!(supervisor.task_count(), 1); 800 supervisor.request_cancellation(); 801 assert!(supervisor.supervise().await.is_ok()); 802 } 803 }