supervision_retention.rs (12086B)
1 use std::{ 2 future::{Future, pending, ready}, 3 pin::pin, 4 task::Poll, 5 time::Duration, 6 }; 7 8 use radroots_service_host::{ 9 GracefulShutdown, HostError, HostErrorKind, MonotonicClock, MonotonicTime, ShutdownDisposition, 10 ShutdownPhase, ShutdownPhaseFuture, ShutdownPhaseHandler, SupervisedTaskExit, 11 SupervisedTaskExitStatus, SupervisionFailure, SupervisionFailureKind, TaskClassification, 12 TaskMetadata, TaskName, TaskSupervisor, UnfinishedWork, 13 }; 14 use tokio::sync::oneshot; 15 16 fn metadata(name: &str, classification: TaskClassification) -> TaskMetadata { 17 let phase = classification 18 .requires_shutdown_phase() 19 .then_some(ShutdownPhase::CancelIngress); 20 TaskMetadata::new(TaskName::new(name).unwrap(), classification, phase).unwrap() 21 } 22 23 async fn consume_ready_outcome_and_cancel_bulk_wait(supervisor: &mut TaskSupervisor) { 24 { 25 let mut waiting = pin!(supervisor.supervise()); 26 std::future::poll_fn(|context| { 27 assert!(waiting.as_mut().poll(context).is_pending()); 28 Poll::Ready(()) 29 }) 30 .await; 31 } 32 // The first join was consumed before the waiting future was dropped. 33 assert_eq!(supervisor.task_count(), 1); 34 } 35 36 #[tokio::test] 37 async fn cancelled_bulk_wait_retains_already_joined_exit_on_retry() { 38 let mut supervisor = TaskSupervisor::new(); 39 let (finished, observed) = oneshot::channel(); 40 let (release, held) = oneshot::channel(); 41 supervisor 42 .spawn( 43 metadata("finished", TaskClassification::OneShot), 44 |_| async { 45 finished.send(()).unwrap(); 46 Ok(()) 47 }, 48 ) 49 .unwrap(); 50 supervisor 51 .spawn( 52 metadata("waiting", TaskClassification::OneShot), 53 |_| async { 54 held.await.unwrap(); 55 Ok(()) 56 }, 57 ) 58 .unwrap(); 59 observed.await.unwrap(); 60 consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await; 61 assert!(!supervisor.cancellation_token().is_cancelled()); 62 release.send(()).unwrap(); 63 let exits = supervisor.supervise().await.unwrap(); 64 assert!(supervisor.is_empty()); 65 let names = exits 66 .iter() 67 .map(|exit| exit.metadata().name().as_str()) 68 .collect::<Vec<_>>(); 69 assert_eq!(names, ["finished", "waiting"]); 70 assert!(supervisor.supervise().await.unwrap().is_empty()); 71 } 72 73 #[tokio::test] 74 async fn cancelled_bulk_wait_retains_first_fatal_failure_on_retry() { 75 let mut supervisor = TaskSupervisor::new(); 76 let (finished, observed) = oneshot::channel(); 77 let (release, held) = oneshot::channel(); 78 supervisor 79 .spawn( 80 metadata("failed", TaskClassification::Critical), 81 |_| async { 82 finished.send(()).unwrap(); 83 Err(HostError::new(HostErrorKind::TaskFailure)) 84 }, 85 ) 86 .unwrap(); 87 supervisor 88 .spawn( 89 metadata("waiting", TaskClassification::OneShot), 90 |_| async { 91 held.await.unwrap(); 92 Ok(()) 93 }, 94 ) 95 .unwrap(); 96 observed.await.unwrap(); 97 consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await; 98 assert!(supervisor.cancellation_token().is_cancelled()); 99 release.send(()).unwrap(); 100 let failure = supervisor.supervise().await.unwrap_err(); 101 assert!(supervisor.is_empty()); 102 assert_eq!(failure.kind(), SupervisionFailureKind::TaskReturnedError); 103 assert_eq!(failure.metadata().unwrap().name().as_str(), "failed"); 104 assert!(supervisor.supervise().await.unwrap().is_empty()); 105 } 106 107 async fn completed_pair( 108 classification: TaskClassification, 109 first_result: Result<(), HostError>, 110 held_result: Result<(), HostError>, 111 ) -> (TaskSupervisor, oneshot::Sender<()>) { 112 let mut supervisor = TaskSupervisor::new(); 113 let (finished, observed) = oneshot::channel(); 114 let (release, held) = oneshot::channel(); 115 supervisor 116 .spawn(metadata("observed", classification), |_| async move { 117 finished.send(()).unwrap(); 118 first_result 119 }) 120 .unwrap(); 121 supervisor 122 .spawn( 123 metadata("held", TaskClassification::OneShot), 124 |_| async move { 125 held.await.unwrap(); 126 held_result 127 }, 128 ) 129 .unwrap(); 130 observed.await.unwrap(); 131 (supervisor, release) 132 } 133 134 async fn observe_retained( 135 supervisor: &mut TaskSupervisor, 136 ) -> Result<SupervisedTaskExit, SupervisionFailure> { 137 let mut reading = pin!(supervisor.join_next()); 138 std::future::poll_fn(|context| match reading.as_mut().poll(context) { 139 Poll::Ready(Some(outcome)) => Poll::Ready(outcome), 140 _ => panic!("an already joined observation must be immediately available"), 141 }) 142 .await 143 } 144 145 #[tokio::test] 146 async fn cancelled_bulk_optional_failure_transfers_once_to_per_join_observation() { 147 let (mut supervisor, release) = completed_pair( 148 TaskClassification::Optional, 149 Err(HostError::new(HostErrorKind::TaskFailure)), 150 Ok(()), 151 ) 152 .await; 153 consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await; 154 155 let exit = observe_retained(&mut supervisor).await.unwrap(); 156 assert_eq!(exit.metadata().name().as_str(), "observed"); 157 assert_eq!(exit.status(), SupervisedTaskExitStatus::OptionalFailure); 158 assert!(exit.source().is_some()); 159 assert!(!supervisor.cancellation_token().is_cancelled()); 160 release.send(()).unwrap(); 161 let remaining = supervisor.supervise().await.unwrap(); 162 assert_eq!(remaining.len(), 1); 163 assert_eq!(remaining[0].metadata().name().as_str(), "held"); 164 assert_eq!( 165 remaining[0].status(), 166 SupervisedTaskExitStatus::ExpectedCompletion 167 ); 168 assert!(supervisor.join_next().await.is_none()); 169 } 170 171 #[tokio::test] 172 async fn cancelled_bulk_fatal_transfers_once_before_a_later_fatal_join() { 173 let (mut supervisor, release) = completed_pair( 174 TaskClassification::Critical, 175 Err(HostError::new(HostErrorKind::TaskFailure)), 176 Err(HostError::new(HostErrorKind::Lifecycle)), 177 ) 178 .await; 179 consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await; 180 181 let first = observe_retained(&mut supervisor).await.unwrap_err(); 182 assert_eq!(first.kind(), SupervisionFailureKind::TaskReturnedError); 183 assert_eq!(first.metadata().unwrap().name().as_str(), "observed"); 184 assert!(supervisor.cancellation_token().is_cancelled()); 185 release.send(()).unwrap(); 186 let later = supervisor.join_next().await.unwrap().unwrap_err(); 187 assert_eq!(later.kind(), SupervisionFailureKind::TaskReturnedError); 188 assert_eq!(later.metadata().unwrap().name().as_str(), "held"); 189 assert!(supervisor.join_next().await.is_none()); 190 assert!(supervisor.supervise().await.unwrap().is_empty()); 191 } 192 193 struct FixedClock(Duration); 194 195 impl MonotonicClock for FixedClock { 196 fn now_monotonic(&self) -> MonotonicTime { 197 MonotonicTime::from_duration_since_origin(self.0) 198 } 199 } 200 201 #[derive(Default)] 202 struct RecordingHandler(Vec<ShutdownPhase>); 203 204 impl ShutdownPhaseHandler for RecordingHandler { 205 fn enter(&mut self, phase: ShutdownPhase) -> ShutdownPhaseFuture<'_> { 206 self.0.push(phase); 207 Box::pin(ready(Ok(()))) 208 } 209 } 210 211 const PHASES: [ShutdownPhase; 7] = [ 212 ShutdownPhase::RejectNewMutations, 213 ShutdownPhase::CancelIngress, 214 ShutdownPhase::DrainOperations, 215 ShutdownPhase::PersistRecoverableWork, 216 ShutdownPhase::CloseNetwork, 217 ShutdownPhase::CloseSqlite, 218 ShutdownPhase::CloseSockets, 219 ]; 220 221 #[tokio::test] 222 async fn cancelled_bulk_fatal_is_retained_by_graceful_shutdown_before_empty_phase() { 223 let (mut supervisor, release) = completed_pair( 224 TaskClassification::Critical, 225 Err(HostError::new(HostErrorKind::TaskFailure)), 226 Ok(()), 227 ) 228 .await; 229 consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await; 230 release.send(()).unwrap(); 231 let mut shutdown = GracefulShutdown::new(Duration::from_secs(30)).unwrap(); 232 let mut handler = RecordingHandler::default(); 233 let clock = FixedClock(Duration::from_secs(5)); 234 235 let summary = shutdown 236 .run(&clock, &mut supervisor, &mut handler, pending()) 237 .await 238 .unwrap(); 239 assert_eq!( 240 summary.disposition(), 241 ShutdownDisposition::TaskFailed { 242 kind: SupervisionFailureKind::TaskReturnedError, 243 } 244 ); 245 assert_eq!( 246 shutdown 247 .task_failure() 248 .unwrap() 249 .metadata() 250 .unwrap() 251 .name() 252 .as_str(), 253 "observed" 254 ); 255 assert_eq!(handler.0, PHASES); 256 assert!(supervisor.is_empty()); 257 assert!(supervisor.join_next().await.is_none()); 258 assert_eq!( 259 shutdown 260 .run(&clock, &mut supervisor, &mut handler, ready(())) 261 .await 262 .unwrap(), 263 summary 264 ); 265 assert_eq!(handler.0, PHASES); 266 } 267 268 #[tokio::test] 269 async fn cancelled_phase_drain_transfers_exit_and_resumes_original_deadline() { 270 let (mut supervisor, release) = 271 completed_pair(TaskClassification::OneShot, Ok(()), Ok(())).await; 272 let mut shutdown = GracefulShutdown::new(Duration::from_secs(30)).unwrap(); 273 let mut handler = RecordingHandler::default(); 274 let clock = FixedClock(Duration::from_secs(5)); 275 { 276 let mut waiting = pin!(shutdown.run(&clock, &mut supervisor, &mut handler, pending())); 277 std::future::poll_fn(|context| { 278 assert!(waiting.as_mut().poll(context).is_pending()); 279 Poll::Ready(()) 280 }) 281 .await; 282 } 283 assert_eq!(supervisor.task_count(), 1); 284 assert_eq!(handler.0, PHASES[..3]); 285 let retained = observe_retained(&mut supervisor).await.unwrap(); 286 assert_eq!(retained.metadata().name().as_str(), "observed"); 287 assert_eq!( 288 retained.status(), 289 SupervisedTaskExitStatus::ExpectedCompletion 290 ); 291 assert!(!supervisor.cancellation_token().is_cancelled()); 292 release.send(()).unwrap(); 293 let summary = shutdown 294 .run( 295 &FixedClock(Duration::from_secs(250)), 296 &mut supervisor, 297 &mut handler, 298 pending(), 299 ) 300 .await 301 .unwrap(); 302 assert_eq!(summary.disposition(), ShutdownDisposition::Completed); 303 assert_eq!( 304 summary.deadline().time().duration_since_origin(), 305 Duration::from_secs(35) 306 ); 307 assert_eq!(handler.0, PHASES); 308 assert!(supervisor.is_empty()); 309 assert!(supervisor.join_next().await.is_none()); 310 } 311 312 #[tokio::test] 313 async fn forced_abort_drain_retains_previously_joined_fatal_and_aborted_peer() { 314 let (mut supervisor, _release) = completed_pair( 315 TaskClassification::Critical, 316 Err(HostError::new(HostErrorKind::TaskFailure)), 317 Ok(()), 318 ) 319 .await; 320 consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await; 321 let mut shutdown = GracefulShutdown::new(Duration::from_secs(30)).unwrap(); 322 let mut handler = RecordingHandler::default(); 323 let clock = FixedClock(Duration::from_secs(5)); 324 let summary = shutdown 325 .run(&clock, &mut supervisor, &mut handler, ready(())) 326 .await 327 .unwrap(); 328 assert_eq!( 329 summary.disposition(), 330 ShutdownDisposition::Forced { 331 unfinished: UnfinishedWork::FatalAuthoritative, 332 } 333 ); 334 assert!(handler.0.is_empty()); 335 assert!(supervisor.is_empty()); 336 assert_eq!(supervisor.task_count(), 0); 337 let retained = observe_retained(&mut supervisor).await.unwrap_err(); 338 assert_eq!(retained.kind(), SupervisionFailureKind::TaskReturnedError); 339 assert_eq!(retained.metadata().unwrap().name().as_str(), "observed"); 340 let aborted = observe_retained(&mut supervisor).await.unwrap(); 341 assert_eq!(aborted.metadata().name().as_str(), "held"); 342 assert_eq!( 343 aborted.status(), 344 SupervisedTaskExitStatus::ExpectedCompletion 345 ); 346 assert!(supervisor.join_next().await.is_none()); 347 assert!(supervisor.supervise().await.unwrap().is_empty()); 348 }