commit 89f1b0f51f0fc38266f31a9a3504755c5834204c
parent c6e895a1060fbaa038f43b0f6cdef236c4289101
Author: triesap <tyson@radroots.org>
Date: Tue, 13 Oct 2026 02:02:48 +0000
service_host: Retain joined outcomes across cancelled supervision
- Preserve joined exits and first fatal outcomes across cancelled waits
- Retain once-only observations through per-join and forced-drain paths
- Add seven public regressions that reproduce the original outcome losses
- Pass all 174 host cases and Rust 1.99 format, check, Clippy and architecture
Diffstat:
4 files changed, 431 insertions(+), 9 deletions(-)
diff --git a/contracts/architecture/deviations.toml b/contracts/architecture/deviations.toml
@@ -2,6 +2,35 @@ schema_version = 1
architecture_id = "radroots.crates.release.v1"
[[deviation]]
+id = "RCRV1-DEV-035"
+date = "2026-10-13"
+status = "closed"
+approval = "The user approved the successor refactor, scoped execution/recovery amendments and all remaining local verified checkpoints; publication and supported consumer qualification remain separate."
+affected_steps = ["010", "040", "070", "130"]
+spec_anchors = ["contracts/crates/release_v1/radroots_crates_release_v1.toml#repository_policy"]
+source_evidence = [
+ "The public bulk supervise future stores already joined exits and its first fatal failure in local variables while awaiting peers. Controlled unchanged-source regressions reproduce loss of both after cancellation and retry.",
+ "The per-join API retains unjoined tasks, while phased supervision also accumulates joined exits locally. GracefulShutdown separately retains its first failure, completed phase progress and original deadline.",
+ "Existing consumer foundations/adapters use bulk supervision and service graphs combine per-join supervision with phased graceful shutdown; this shared-API reproduction does not establish an application or deployed incident.",
+]
+replacement_action = "Bring forward contained cancellation-safe outcome retention. Retain joined observations in private supervisor-owned state until an explicit bulk, per-join or phase observation consumes them. Keep the first fatal outcome on bulk retry, retain phase-drain observations across cancellation, and preserve unobserved outcomes while aborting and draining remaining tasks. Preserve public signatures, task classification/cancellation policy, phase order, original shutdown deadline and task-count semantics without new workers, dependencies, runtime ownership or wire contracts."
+verification = [
+ "Validate this prospective deviation before production edits, then retain unchanged-source public regressions using at most two synthetic tasks, oneshot barriers and explicit polling, with no network, account or persisted-state effects.",
+ "Prove bulk prefix/first-failure retry, exact once per-join transfer, phased shutdown handoff/retry and forced abort-and-drain preservation; retain original fatal/nonfatal/panic/cancellation/name/phase/deadline cases.",
+ "Run exact Rust 1.99.0 scoped formatting, all-feature/all-target package check, complete host package tests and warnings-as-errors Clippy. Validate the final closed deviation with explicit executed closure evidence.",
+ "Review exact source/test/guidance/deviation ownership and actual supported consumer call sites before local commit; supported adoption, graph interruption, blocking-worker/resource and platform qualification remain separate.",
+]
+unresolved_risk = "The contained supervisor retention correction is locally qualified. Exact published supported consumer adoption, complete graph/process interruption, blocking-worker cancellation, resources, platform behavior and whole source closure remain open."
+normative_architecture_change = false
+adr_required = false
+closure_evidence = [
+ "Prospective standalone architecture validation passed before production edits. All seven public regressions compiled and failed against unchanged production source with two synthetic tasks per case, oneshot barriers and explicit polling; no network, account, file or persisted-state effects occurred.",
+ "Private supervisor-owned observations now retain joined prefixes and the first fatal outcome across cancelled bulk waits, provide once-only per-join transfer, preserve phase-drain handoff and the original shutdown deadline, and retain observations while aborting and joining peers. Original classification, phase order, cancellation authority and unjoined-task count semantics are preserved.",
+ "Exact Rust 1.99.0 scoped formatting, all-feature/all-target package check, all 174 host package tests/doctests with zero ignored or filtered cases, warnings-as-errors Clippy and architecture validation passed on local macOS arm64. The same seven unchanged regressions pass with the correction; all 167 original cases remain green.",
+ "Actual Myc bulk foundation/supervision and RHI bulk adapter paths, and their per-join plus graceful-shutdown graph paths, are reviewed as scoped caller windows. Their resolved shared-host source still uses the earlier exact Lib pin; no consumer adoption, complete graph/platform/resource qualification or whole source closure follows from the local package result. Final closed-deviation architecture validation and exact source binding are required before the local owner commit.",
+]
+
+[[deviation]]
id = "RCRV1-DEV-034"
date = "2026-10-12"
status = "closed"
diff --git a/crates/service_host/README.md b/crates/service_host/README.md
@@ -134,6 +134,17 @@ cancelled incomplete handler may be entered again and must be idempotent and
cancellation safe. The first phase or task failure is retained while later
close phases continue as long as the original deadline remains.
+Bulk and phase supervision retain already joined observations in the supervisor
+when their wait is cancelled. Retrying bulk supervision includes that prefix
+and keeps the first fatal outcome; switching to `join_next` returns retained
+observations first in join order. A completed bulk call consumes its batch and
+returns either the nonfatal exits or the first fatal failure; `join_next`
+consumes one observation. Phased shutdown retains its fatal result while
+continuing the remaining close phases. Forced abort-and-drain joins remaining tasks and
+preserves unconsumed observations for a later explicit read. `task_count` and
+`is_empty` describe tasks that have not yet been joined, so an empty supervisor
+can still have retained observations.
+
## Supported targets and publication
The generic host/config/status/lifecycle surfaces support the workspace's
diff --git a/crates/service_host/src/lifecycle/supervisor.rs b/crates/service_host/src/lifecycle/supervisor.rs
@@ -1,7 +1,10 @@
//! Join-owned task supervision with typed completion policy.
use core::{fmt, future::Future};
-use std::{collections::HashMap, error::Error};
+use std::{
+ collections::{HashMap, VecDeque},
+ error::Error,
+};
use tokio::task::{Id, JoinError, JoinSet};
@@ -15,6 +18,7 @@ pub struct TaskSupervisor {
cancellation: CancellationToken,
tasks: JoinSet<TaskCompletion>,
controls: HashMap<Id, TaskControl>,
+ observed: VecDeque<Result<SupervisedTaskExit, SupervisionFailure>>,
}
struct TaskControl {
@@ -28,6 +32,7 @@ impl TaskSupervisor {
cancellation: CancellationToken::new(),
tasks: JoinSet::new(),
controls: HashMap::new(),
+ observed: VecDeque::new(),
}
}
@@ -42,11 +47,13 @@ impl TaskSupervisor {
self.cancellation.cancel();
}
+ /// Returns the number of tasks whose join results have not yet been collected.
#[must_use]
pub fn task_count(&self) -> usize {
self.tasks.len()
}
+ /// Returns whether every task has been joined; unconsumed observations may remain.
#[must_use]
pub fn is_empty(&self) -> bool {
self.tasks.is_empty()
@@ -108,25 +115,41 @@ impl TaskSupervisor {
&mut self,
phase: ShutdownPhase,
) -> Result<Vec<SupervisedTaskExit>, SupervisionFailure> {
- let mut exits = Vec::new();
+ if let Some(index) = self.observed.iter().position(Result::is_err) {
+ return Err(self
+ .observed
+ .remove(index)
+ .expect("retained fatal outcome must exist")
+ .expect_err("retained fatal outcome must be a failure"));
+ }
while self.has_phase_work(phase) {
let outcome = self
- .join_next()
+ .join_task()
.await
.expect("phase work must retain a join-owned task");
match outcome {
- Ok(exit) => exits.push(exit),
+ Ok(exit) => self.observed.push_back(Ok(exit)),
Err(failure) => return Err(failure),
}
}
- Ok(exits)
+ self.take_observed()
}
/// Observes all task exits, cancels peers on the first fatal outcome, and drains every join.
+ ///
+ /// Cancelling this wait retains already joined outcomes. A subsequent bulk,
+ /// per-join or phase wait consumes those observations exactly once.
pub async fn supervise(&mut self) -> Result<Vec<SupervisedTaskExit>, SupervisionFailure> {
- let mut exits = Vec::with_capacity(self.tasks.len());
+ while let Some(outcome) = self.join_task().await {
+ self.observed.push_back(outcome);
+ }
+ self.take_observed()
+ }
+
+ fn take_observed(&mut self) -> Result<Vec<SupervisedTaskExit>, SupervisionFailure> {
+ let mut exits = Vec::with_capacity(self.observed.len());
let mut first_failure = None;
- while let Some(outcome) = self.join_next().await {
+ for outcome in self.observed.drain(..) {
match outcome {
Ok(exit) => exits.push(exit),
Err(failure) => {
@@ -147,8 +170,16 @@ impl TaskSupervisor {
///
/// Optional failures are returned without cancelling peers. A fatal
/// outcome requests cancellation before it is returned. Cancelling this
- /// wait does not remove a task or lose its later outcome.
+ /// wait does not remove a task or lose its later outcome. Previously joined
+ /// outcomes retained by a cancelled bulk or phase wait are returned first.
pub async fn join_next(&mut self) -> Option<Result<SupervisedTaskExit, SupervisionFailure>> {
+ if let Some(outcome) = self.observed.pop_front() {
+ return Some(outcome);
+ }
+ self.join_task().await
+ }
+
+ async fn join_task(&mut self) -> Option<Result<SupervisedTaskExit, SupervisionFailure>> {
let joined = self.tasks.join_next_with_id().await?;
let outcome = self.classify_join(joined);
if outcome.is_err() {
@@ -211,10 +242,13 @@ impl TaskSupervisor {
}
}
+ /// Joins aborted work while retaining unconsumed observations for an explicit read.
pub(crate) async fn abort_and_drain(&mut self) {
self.cancellation.cancel();
self.tasks.abort_all();
- while self.join_next().await.is_some() {}
+ while let Some(outcome) = self.join_task().await {
+ self.observed.push_back(outcome);
+ }
}
fn has_phase_work(&self, phase: ShutdownPhase) -> bool {
diff --git a/crates/service_host/tests/supervision_retention.rs b/crates/service_host/tests/supervision_retention.rs
@@ -0,0 +1,348 @@
+use std::{
+ future::{Future, pending, ready},
+ pin::pin,
+ task::Poll,
+ time::Duration,
+};
+
+use radroots_service_host::{
+ GracefulShutdown, HostError, HostErrorKind, MonotonicClock, MonotonicTime, ShutdownDisposition,
+ ShutdownPhase, ShutdownPhaseFuture, ShutdownPhaseHandler, SupervisedTaskExit,
+ SupervisedTaskExitStatus, SupervisionFailure, SupervisionFailureKind, TaskClassification,
+ TaskMetadata, TaskName, TaskSupervisor, UnfinishedWork,
+};
+use tokio::sync::oneshot;
+
+fn metadata(name: &str, classification: TaskClassification) -> TaskMetadata {
+ let phase = classification
+ .requires_shutdown_phase()
+ .then_some(ShutdownPhase::CancelIngress);
+ TaskMetadata::new(TaskName::new(name).unwrap(), classification, phase).unwrap()
+}
+
+async fn consume_ready_outcome_and_cancel_bulk_wait(supervisor: &mut TaskSupervisor) {
+ {
+ let mut waiting = pin!(supervisor.supervise());
+ std::future::poll_fn(|context| {
+ assert!(waiting.as_mut().poll(context).is_pending());
+ Poll::Ready(())
+ })
+ .await;
+ }
+ // The first join was consumed before the waiting future was dropped.
+ assert_eq!(supervisor.task_count(), 1);
+}
+
+#[tokio::test]
+async fn cancelled_bulk_wait_retains_already_joined_exit_on_retry() {
+ let mut supervisor = TaskSupervisor::new();
+ let (finished, observed) = oneshot::channel();
+ let (release, held) = oneshot::channel();
+ supervisor
+ .spawn(
+ metadata("finished", TaskClassification::OneShot),
+ |_| async {
+ finished.send(()).unwrap();
+ Ok(())
+ },
+ )
+ .unwrap();
+ supervisor
+ .spawn(
+ metadata("waiting", TaskClassification::OneShot),
+ |_| async {
+ held.await.unwrap();
+ Ok(())
+ },
+ )
+ .unwrap();
+ observed.await.unwrap();
+ consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await;
+ assert!(!supervisor.cancellation_token().is_cancelled());
+ release.send(()).unwrap();
+ let exits = supervisor.supervise().await.unwrap();
+ assert!(supervisor.is_empty());
+ let names = exits
+ .iter()
+ .map(|exit| exit.metadata().name().as_str())
+ .collect::<Vec<_>>();
+ assert_eq!(names, ["finished", "waiting"]);
+ assert!(supervisor.supervise().await.unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn cancelled_bulk_wait_retains_first_fatal_failure_on_retry() {
+ let mut supervisor = TaskSupervisor::new();
+ let (finished, observed) = oneshot::channel();
+ let (release, held) = oneshot::channel();
+ supervisor
+ .spawn(
+ metadata("failed", TaskClassification::Critical),
+ |_| async {
+ finished.send(()).unwrap();
+ Err(HostError::new(HostErrorKind::TaskFailure))
+ },
+ )
+ .unwrap();
+ supervisor
+ .spawn(
+ metadata("waiting", TaskClassification::OneShot),
+ |_| async {
+ held.await.unwrap();
+ Ok(())
+ },
+ )
+ .unwrap();
+ observed.await.unwrap();
+ consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await;
+ assert!(supervisor.cancellation_token().is_cancelled());
+ release.send(()).unwrap();
+ let failure = supervisor.supervise().await.unwrap_err();
+ assert!(supervisor.is_empty());
+ assert_eq!(failure.kind(), SupervisionFailureKind::TaskReturnedError);
+ assert_eq!(failure.metadata().unwrap().name().as_str(), "failed");
+ assert!(supervisor.supervise().await.unwrap().is_empty());
+}
+
+async fn completed_pair(
+ classification: TaskClassification,
+ first_result: Result<(), HostError>,
+ held_result: Result<(), HostError>,
+) -> (TaskSupervisor, oneshot::Sender<()>) {
+ let mut supervisor = TaskSupervisor::new();
+ let (finished, observed) = oneshot::channel();
+ let (release, held) = oneshot::channel();
+ supervisor
+ .spawn(metadata("observed", classification), |_| async move {
+ finished.send(()).unwrap();
+ first_result
+ })
+ .unwrap();
+ supervisor
+ .spawn(
+ metadata("held", TaskClassification::OneShot),
+ |_| async move {
+ held.await.unwrap();
+ held_result
+ },
+ )
+ .unwrap();
+ observed.await.unwrap();
+ (supervisor, release)
+}
+
+async fn observe_retained(
+ supervisor: &mut TaskSupervisor,
+) -> Result<SupervisedTaskExit, SupervisionFailure> {
+ let mut reading = pin!(supervisor.join_next());
+ std::future::poll_fn(|context| match reading.as_mut().poll(context) {
+ Poll::Ready(Some(outcome)) => Poll::Ready(outcome),
+ _ => panic!("an already joined observation must be immediately available"),
+ })
+ .await
+}
+
+#[tokio::test]
+async fn cancelled_bulk_optional_failure_transfers_once_to_per_join_observation() {
+ let (mut supervisor, release) = completed_pair(
+ TaskClassification::Optional,
+ Err(HostError::new(HostErrorKind::TaskFailure)),
+ Ok(()),
+ )
+ .await;
+ consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await;
+
+ let exit = observe_retained(&mut supervisor).await.unwrap();
+ assert_eq!(exit.metadata().name().as_str(), "observed");
+ assert_eq!(exit.status(), SupervisedTaskExitStatus::OptionalFailure);
+ assert!(exit.source().is_some());
+ assert!(!supervisor.cancellation_token().is_cancelled());
+ release.send(()).unwrap();
+ let remaining = supervisor.supervise().await.unwrap();
+ assert_eq!(remaining.len(), 1);
+ assert_eq!(remaining[0].metadata().name().as_str(), "held");
+ assert_eq!(
+ remaining[0].status(),
+ SupervisedTaskExitStatus::ExpectedCompletion
+ );
+ assert!(supervisor.join_next().await.is_none());
+}
+
+#[tokio::test]
+async fn cancelled_bulk_fatal_transfers_once_before_a_later_fatal_join() {
+ let (mut supervisor, release) = completed_pair(
+ TaskClassification::Critical,
+ Err(HostError::new(HostErrorKind::TaskFailure)),
+ Err(HostError::new(HostErrorKind::Lifecycle)),
+ )
+ .await;
+ consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await;
+
+ let first = observe_retained(&mut supervisor).await.unwrap_err();
+ assert_eq!(first.kind(), SupervisionFailureKind::TaskReturnedError);
+ assert_eq!(first.metadata().unwrap().name().as_str(), "observed");
+ assert!(supervisor.cancellation_token().is_cancelled());
+ release.send(()).unwrap();
+ let later = supervisor.join_next().await.unwrap().unwrap_err();
+ assert_eq!(later.kind(), SupervisionFailureKind::TaskReturnedError);
+ assert_eq!(later.metadata().unwrap().name().as_str(), "held");
+ assert!(supervisor.join_next().await.is_none());
+ assert!(supervisor.supervise().await.unwrap().is_empty());
+}
+
+struct FixedClock(Duration);
+
+impl MonotonicClock for FixedClock {
+ fn now_monotonic(&self) -> MonotonicTime {
+ MonotonicTime::from_duration_since_origin(self.0)
+ }
+}
+
+#[derive(Default)]
+struct RecordingHandler(Vec<ShutdownPhase>);
+
+impl ShutdownPhaseHandler for RecordingHandler {
+ fn enter(&mut self, phase: ShutdownPhase) -> ShutdownPhaseFuture<'_> {
+ self.0.push(phase);
+ Box::pin(ready(Ok(())))
+ }
+}
+
+const PHASES: [ShutdownPhase; 7] = [
+ ShutdownPhase::RejectNewMutations,
+ ShutdownPhase::CancelIngress,
+ ShutdownPhase::DrainOperations,
+ ShutdownPhase::PersistRecoverableWork,
+ ShutdownPhase::CloseNetwork,
+ ShutdownPhase::CloseSqlite,
+ ShutdownPhase::CloseSockets,
+];
+
+#[tokio::test]
+async fn cancelled_bulk_fatal_is_retained_by_graceful_shutdown_before_empty_phase() {
+ let (mut supervisor, release) = completed_pair(
+ TaskClassification::Critical,
+ Err(HostError::new(HostErrorKind::TaskFailure)),
+ Ok(()),
+ )
+ .await;
+ consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await;
+ release.send(()).unwrap();
+ let mut shutdown = GracefulShutdown::new(Duration::from_secs(30)).unwrap();
+ let mut handler = RecordingHandler::default();
+ let clock = FixedClock(Duration::from_secs(5));
+
+ let summary = shutdown
+ .run(&clock, &mut supervisor, &mut handler, pending())
+ .await
+ .unwrap();
+ assert_eq!(
+ summary.disposition(),
+ ShutdownDisposition::TaskFailed {
+ kind: SupervisionFailureKind::TaskReturnedError,
+ }
+ );
+ assert_eq!(
+ shutdown
+ .task_failure()
+ .unwrap()
+ .metadata()
+ .unwrap()
+ .name()
+ .as_str(),
+ "observed"
+ );
+ assert_eq!(handler.0, PHASES);
+ assert!(supervisor.is_empty());
+ assert!(supervisor.join_next().await.is_none());
+ assert_eq!(
+ shutdown
+ .run(&clock, &mut supervisor, &mut handler, ready(()))
+ .await
+ .unwrap(),
+ summary
+ );
+ assert_eq!(handler.0, PHASES);
+}
+
+#[tokio::test]
+async fn cancelled_phase_drain_transfers_exit_and_resumes_original_deadline() {
+ let (mut supervisor, release) =
+ completed_pair(TaskClassification::OneShot, Ok(()), Ok(())).await;
+ let mut shutdown = GracefulShutdown::new(Duration::from_secs(30)).unwrap();
+ let mut handler = RecordingHandler::default();
+ let clock = FixedClock(Duration::from_secs(5));
+ {
+ let mut waiting = pin!(shutdown.run(&clock, &mut supervisor, &mut handler, pending()));
+ std::future::poll_fn(|context| {
+ assert!(waiting.as_mut().poll(context).is_pending());
+ Poll::Ready(())
+ })
+ .await;
+ }
+ assert_eq!(supervisor.task_count(), 1);
+ assert_eq!(handler.0, PHASES[..3]);
+ let retained = observe_retained(&mut supervisor).await.unwrap();
+ assert_eq!(retained.metadata().name().as_str(), "observed");
+ assert_eq!(
+ retained.status(),
+ SupervisedTaskExitStatus::ExpectedCompletion
+ );
+ assert!(!supervisor.cancellation_token().is_cancelled());
+ release.send(()).unwrap();
+ let summary = shutdown
+ .run(
+ &FixedClock(Duration::from_secs(250)),
+ &mut supervisor,
+ &mut handler,
+ pending(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(summary.disposition(), ShutdownDisposition::Completed);
+ assert_eq!(
+ summary.deadline().time().duration_since_origin(),
+ Duration::from_secs(35)
+ );
+ assert_eq!(handler.0, PHASES);
+ assert!(supervisor.is_empty());
+ assert!(supervisor.join_next().await.is_none());
+}
+
+#[tokio::test]
+async fn forced_abort_drain_retains_previously_joined_fatal_and_aborted_peer() {
+ let (mut supervisor, _release) = completed_pair(
+ TaskClassification::Critical,
+ Err(HostError::new(HostErrorKind::TaskFailure)),
+ Ok(()),
+ )
+ .await;
+ consume_ready_outcome_and_cancel_bulk_wait(&mut supervisor).await;
+ let mut shutdown = GracefulShutdown::new(Duration::from_secs(30)).unwrap();
+ let mut handler = RecordingHandler::default();
+ let clock = FixedClock(Duration::from_secs(5));
+ let summary = shutdown
+ .run(&clock, &mut supervisor, &mut handler, ready(()))
+ .await
+ .unwrap();
+ assert_eq!(
+ summary.disposition(),
+ ShutdownDisposition::Forced {
+ unfinished: UnfinishedWork::FatalAuthoritative,
+ }
+ );
+ assert!(handler.0.is_empty());
+ assert!(supervisor.is_empty());
+ assert_eq!(supervisor.task_count(), 0);
+ let retained = observe_retained(&mut supervisor).await.unwrap_err();
+ assert_eq!(retained.kind(), SupervisionFailureKind::TaskReturnedError);
+ assert_eq!(retained.metadata().unwrap().name().as_str(), "observed");
+ let aborted = observe_retained(&mut supervisor).await.unwrap();
+ assert_eq!(aborted.metadata().name().as_str(), "held");
+ assert_eq!(
+ aborted.status(),
+ SupervisedTaskExitStatus::ExpectedCompletion
+ );
+ assert!(supervisor.join_next().await.is_none());
+ assert!(supervisor.supervise().await.unwrap().is_empty());
+}