doctor_v1.rs (19867B)
1 //! Bounded active-doctor orchestration and safe structured evidence. 2 3 use core::{fmt, future::Future, pin::Pin, time::Duration}; 4 use std::error::Error; 5 6 use radroots_runtime_paths::InstanceId; 7 use serde::Serialize; 8 9 use crate::RhiRuntimeContext; 10 11 /// RHI doctor wire-contract version. 12 pub const RHI_DOCTOR_CONTRACT_VERSION: u32 = 1; 13 /// Exact number of governed RHI doctor checks. 14 pub const RHI_DOCTOR_CHECK_COUNT: usize = 15; 15 /// Maximum encoded size of one safe summary. 16 pub const RHI_DOCTOR_SUMMARY_MAX_UTF8_BYTES: usize = 256; 17 /// Maximum encoded size of the complete canonical doctor report. 18 pub const RHI_DOCTOR_REPORT_MAX_UTF8_BYTES: usize = 8_192; 19 20 const RHI_SERVICE: &str = "rhi"; 21 const DOCTOR_FAILURE_EXIT_CODE: u8 = 6; 22 const _: () = { 23 assert!("check passed".len() <= RHI_DOCTOR_SUMMARY_MAX_UTF8_BYTES); 24 assert!("check failed".len() <= RHI_DOCTOR_SUMMARY_MAX_UTF8_BYTES); 25 assert!("check timed out".len() <= RHI_DOCTOR_SUMMARY_MAX_UTF8_BYTES); 26 assert!("optional check skipped".len() <= RHI_DOCTOR_SUMMARY_MAX_UTF8_BYTES); 27 }; 28 29 /// The closed RHI doctor inventory. 30 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] 31 pub enum RhiDoctorCheckId { 32 PathsPermissions, 33 WriterLock, 34 SqliteSchema, 35 SqliteIntegrity, 36 SqliteFreeSpace, 37 IdentityBinding, 38 AdminBindPolicy, 39 OperationsBindPolicy, 40 NetworkPolicy, 41 RequiredSources, 42 CursorCheckpoint, 43 ReconciliationLeases, 44 ReconciliationBacklog, 45 PublicationInvariants, 46 ClockSkew, 47 } 48 49 impl RhiDoctorCheckId { 50 const fn as_str(self) -> &'static str { 51 match self { 52 Self::PathsPermissions => "paths_permissions", 53 Self::WriterLock => "writer_lock", 54 Self::SqliteSchema => "sqlite_schema", 55 Self::SqliteIntegrity => "sqlite_integrity", 56 Self::SqliteFreeSpace => "sqlite_free_space", 57 Self::IdentityBinding => "identity_binding", 58 Self::AdminBindPolicy => "admin_bind_policy", 59 Self::OperationsBindPolicy => "operations_bind_policy", 60 Self::NetworkPolicy => "network_policy", 61 Self::RequiredSources => "required_sources", 62 Self::CursorCheckpoint => "cursor_checkpoint", 63 Self::ReconciliationLeases => "reconciliation_leases", 64 Self::ReconciliationBacklog => "reconciliation_backlog", 65 Self::PublicationInvariants => "publication_invariants", 66 Self::ClockSkew => "clock_skew", 67 } 68 } 69 } 70 71 /// Stable operator action associated with one doctor check. 72 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 73 pub enum RhiDoctorRemediationCode { 74 CorrectPathPolicy, 75 ReleaseWriterLock, 76 RepairSchema, 77 RestoreVerifiedState, 78 FreeStateDiskSpace, 79 RestoreIdentityBinding, 80 CorrectAdminBindPolicy, 81 CorrectOperationsBindPolicy, 82 CorrectNetworkPolicy, 83 RestoreRequiredSources, 84 RepairCursorCheckpoint, 85 RepairReconciliationLeases, 86 ReduceReconciliationBacklog, 87 RepairPublicationState, 88 CorrectClock, 89 } 90 91 impl RhiDoctorRemediationCode { 92 const fn as_str(self) -> &'static str { 93 match self { 94 Self::CorrectPathPolicy => "correct_path_policy", 95 Self::ReleaseWriterLock => "release_writer_lock", 96 Self::RepairSchema => "repair_schema", 97 Self::RestoreVerifiedState => "restore_verified_state", 98 Self::FreeStateDiskSpace => "free_state_disk_space", 99 Self::RestoreIdentityBinding => "restore_identity_binding", 100 Self::CorrectAdminBindPolicy => "correct_admin_bind_policy", 101 Self::CorrectOperationsBindPolicy => "correct_operations_bind_policy", 102 Self::CorrectNetworkPolicy => "correct_network_policy", 103 Self::RestoreRequiredSources => "restore_required_sources", 104 Self::RepairCursorCheckpoint => "repair_cursor_checkpoint", 105 Self::RepairReconciliationLeases => "repair_reconciliation_leases", 106 Self::ReduceReconciliationBacklog => "reduce_reconciliation_backlog", 107 Self::RepairPublicationState => "repair_publication_state", 108 Self::CorrectClock => "correct_clock", 109 } 110 } 111 } 112 113 /// Immutable authority for one check's requirement, deadline, and remediation. 114 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 115 pub struct RhiDoctorCheckDefinition { 116 id: RhiDoctorCheckId, 117 required: bool, 118 deadline_ms: u64, 119 remediation_code: RhiDoctorRemediationCode, 120 scope: &'static [&'static str], 121 } 122 123 impl RhiDoctorCheckDefinition { 124 const fn new( 125 id: RhiDoctorCheckId, 126 required: bool, 127 deadline_ms: u64, 128 remediation_code: RhiDoctorRemediationCode, 129 scope: &'static [&'static str], 130 ) -> Self { 131 Self { 132 id, 133 required, 134 deadline_ms, 135 remediation_code, 136 scope, 137 } 138 } 139 140 /// Returns the governed check identifier. 141 #[must_use] 142 pub const fn id(self) -> RhiDoctorCheckId { 143 self.id 144 } 145 146 /// Returns whether a non-pass result fails the doctor command. 147 #[must_use] 148 pub const fn required(self) -> bool { 149 self.required 150 } 151 152 /// Returns the exact per-check deadline in milliseconds. 153 #[must_use] 154 pub const fn deadline_ms(self) -> u64 { 155 self.deadline_ms 156 } 157 158 /// Returns the fixed, safe operator remediation classification. 159 #[must_use] 160 pub const fn remediation_code(self) -> RhiDoctorRemediationCode { 161 self.remediation_code 162 } 163 164 /// Returns the exact safe evidence facets owned by this check. 165 #[must_use] 166 pub const fn scope(self) -> &'static [&'static str] { 167 self.scope 168 } 169 } 170 171 const CHECK_DEFINITIONS: [RhiDoctorCheckDefinition; RHI_DOCTOR_CHECK_COUNT] = [ 172 RhiDoctorCheckDefinition::new( 173 RhiDoctorCheckId::PathsPermissions, 174 true, 175 2_000, 176 RhiDoctorRemediationCode::CorrectPathPolicy, 177 &["resolved_path_containment", "owner", "type", "mode"], 178 ), 179 RhiDoctorCheckDefinition::new( 180 RhiDoctorCheckId::WriterLock, 181 true, 182 2_000, 183 RhiDoctorRemediationCode::ReleaseWriterLock, 184 &["state_directory_binding", "writer_lock_state"], 185 ), 186 RhiDoctorCheckDefinition::new( 187 RhiDoctorCheckId::SqliteSchema, 188 true, 189 5_000, 190 RhiDoctorRemediationCode::RepairSchema, 191 &["metadata_identity", "migration_history", "schema_catalog"], 192 ), 193 RhiDoctorCheckDefinition::new( 194 RhiDoctorCheckId::SqliteIntegrity, 195 true, 196 15_000, 197 RhiDoctorRemediationCode::RestoreVerifiedState, 198 &["integrity_check", "foreign_key_check"], 199 ), 200 RhiDoctorCheckDefinition::new( 201 RhiDoctorCheckId::SqliteFreeSpace, 202 true, 203 2_000, 204 RhiDoctorRemediationCode::FreeStateDiskSpace, 205 &["state_filesystem_capacity", "minimum_free_bytes"], 206 ), 207 RhiDoctorCheckDefinition::new( 208 RhiDoctorCheckId::IdentityBinding, 209 true, 210 2_000, 211 RhiDoctorRemediationCode::RestoreIdentityBinding, 212 &[ 213 "envelope_contract", 214 "credential_reference", 215 "public_identity", 216 ], 217 ), 218 RhiDoctorCheckDefinition::new( 219 RhiDoctorCheckId::AdminBindPolicy, 220 true, 221 2_000, 222 RhiDoctorRemediationCode::CorrectAdminBindPolicy, 223 &["unix_socket_path", "socket_mode", "peer_authorization"], 224 ), 225 RhiDoctorCheckDefinition::new( 226 RhiDoctorCheckId::OperationsBindPolicy, 227 true, 228 2_000, 229 RhiDoctorRemediationCode::CorrectOperationsBindPolicy, 230 &["enabled_posture", "listen_address", "bind_policy"], 231 ), 232 RhiDoctorCheckDefinition::new( 233 RhiDoctorCheckId::NetworkPolicy, 234 true, 235 2_000, 236 RhiDoctorRemediationCode::CorrectNetworkPolicy, 237 &["dns_policy", "tls_policy", "relay_url_policy"], 238 ), 239 RhiDoctorCheckDefinition::new( 240 RhiDoctorCheckId::RequiredSources, 241 true, 242 15_000, 243 RhiDoctorRemediationCode::RestoreRequiredSources, 244 &[ 245 "required_source_inventory", 246 "reachability", 247 "source_deadline", 248 ], 249 ), 250 RhiDoctorCheckDefinition::new( 251 RhiDoctorCheckId::CursorCheckpoint, 252 true, 253 5_000, 254 RhiDoctorRemediationCode::RepairCursorCheckpoint, 255 &[ 256 "selector_binding", 257 "cursor_plausibility", 258 "completion_evidence", 259 ], 260 ), 261 RhiDoctorCheckDefinition::new( 262 RhiDoctorCheckId::ReconciliationLeases, 263 true, 264 5_000, 265 RhiDoctorRemediationCode::RepairReconciliationLeases, 266 &["lease_ownership", "lease_expiry", "retry_state"], 267 ), 268 RhiDoctorCheckDefinition::new( 269 RhiDoctorCheckId::ReconciliationBacklog, 270 true, 271 5_000, 272 RhiDoctorRemediationCode::ReduceReconciliationBacklog, 273 &["queue_bound", "attempt_bound", "schedule_plausibility"], 274 ), 275 RhiDoctorCheckDefinition::new( 276 RhiDoctorCheckId::PublicationInvariants, 277 true, 278 5_000, 279 RhiDoctorRemediationCode::RepairPublicationState, 280 &["exact_signed_bytes", "target_inventory", "outbox_schedule"], 281 ), 282 RhiDoctorCheckDefinition::new( 283 RhiDoctorCheckId::ClockSkew, 284 false, 285 5_000, 286 RhiDoctorRemediationCode::CorrectClock, 287 &["wall_clock_skew"], 288 ), 289 ]; 290 291 /// Returns the exact ordered doctor inventory. 292 #[must_use] 293 pub const fn rhi_doctor_check_definitions() 294 -> &'static [RhiDoctorCheckDefinition; RHI_DOCTOR_CHECK_COUNT] { 295 &CHECK_DEFINITIONS 296 } 297 298 /// A closed result supplied by one bounded check implementation. 299 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 300 pub enum RhiDoctorObservation { 301 Pass, 302 Fail, 303 Skipped, 304 } 305 306 /// Future returned by one doctor probe. 307 pub type RhiDoctorFuture<'a> = Pin<Box<dyn Future<Output = RhiDoctorObservation> + Send + 'a>>; 308 309 /// Executes each active check without receiving report-construction authority. 310 /// 311 /// `Pass` is permitted only after every facet in 312 /// [`RhiDoctorCheckDefinition::scope`] is proven. Implementations must be 313 /// cancellation-safe: dropping the future at its deadline must stop work or 314 /// leave synchronous cleanup owned by that future, never detached mutation. 315 pub trait RhiDoctorProbe: Send + Sync { 316 /// Runs one exact check. Raw errors, paths, and arbitrary summaries cannot 317 /// cross this boundary. 318 fn probe(&self, definition: RhiDoctorCheckDefinition) -> RhiDoctorFuture<'_>; 319 } 320 321 /// Stable status of one completed check. 322 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 323 pub enum RhiDoctorCheckStatus { 324 Pass, 325 Fail, 326 Timeout, 327 Skipped, 328 } 329 330 impl RhiDoctorCheckStatus { 331 const fn as_str(self) -> &'static str { 332 match self { 333 Self::Pass => "pass", 334 Self::Fail => "fail", 335 Self::Timeout => "timeout", 336 Self::Skipped => "skipped", 337 } 338 } 339 340 const fn summary(self) -> &'static str { 341 match self { 342 Self::Pass => "check passed", 343 Self::Fail => "check failed", 344 Self::Timeout => "check timed out", 345 Self::Skipped => "optional check skipped", 346 } 347 } 348 } 349 350 /// Stable aggregate doctor status. 351 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 352 pub enum RhiDoctorAggregateStatus { 353 Pass, 354 Degraded, 355 Fail, 356 } 357 358 impl RhiDoctorAggregateStatus { 359 const fn as_str(self) -> &'static str { 360 match self { 361 Self::Pass => "pass", 362 Self::Degraded => "degraded", 363 Self::Fail => "fail", 364 } 365 } 366 } 367 368 /// One sealed structured doctor result. 369 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 370 pub struct RhiDoctorCheckResult { 371 definition: RhiDoctorCheckDefinition, 372 status: RhiDoctorCheckStatus, 373 } 374 375 impl RhiDoctorCheckResult { 376 /// Returns the exact check definition. 377 #[must_use] 378 pub const fn definition(self) -> RhiDoctorCheckDefinition { 379 self.definition 380 } 381 382 /// Returns the admitted check status. 383 #[must_use] 384 pub const fn status(self) -> RhiDoctorCheckStatus { 385 self.status 386 } 387 388 /// Returns the fixed content-free summary. 389 #[must_use] 390 pub const fn summary(self) -> &'static str { 391 self.status.summary() 392 } 393 } 394 395 /// Stable source-free doctor construction failures. 396 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 397 pub enum RhiDoctorErrorKind { 398 Encoding, 399 OutputTooLarge, 400 } 401 402 impl RhiDoctorErrorKind { 403 const fn message(self) -> &'static str { 404 match self { 405 Self::Encoding => "RHI doctor output encoding failed", 406 Self::OutputTooLarge => "RHI doctor output exceeds its byte limit", 407 } 408 } 409 } 410 411 /// One redacted doctor construction failure. 412 #[derive(Clone, Copy, PartialEq, Eq)] 413 pub struct RhiDoctorError { 414 kind: RhiDoctorErrorKind, 415 } 416 417 impl RhiDoctorError { 418 const fn new(kind: RhiDoctorErrorKind) -> Self { 419 Self { kind } 420 } 421 422 /// Returns the stable error classification. 423 #[must_use] 424 pub const fn kind(self) -> RhiDoctorErrorKind { 425 self.kind 426 } 427 } 428 429 impl fmt::Debug for RhiDoctorError { 430 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 431 formatter 432 .debug_struct("RhiDoctorError") 433 .field("kind", &self.kind) 434 .finish() 435 } 436 } 437 438 impl fmt::Display for RhiDoctorError { 439 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 440 formatter.write_str(self.kind.message()) 441 } 442 } 443 444 impl Error for RhiDoctorError {} 445 446 /// One immutable, bounded, canonical RHI doctor report. 447 /// 448 /// Construction remains inside [`run_rhi_doctor`]: 449 /// 450 /// ```compile_fail 451 /// use rhi::{RhiDoctorAggregateStatus, RhiDoctorReport}; 452 /// 453 /// let _ = RhiDoctorReport { 454 /// instance: todo!(), 455 /// status: RhiDoctorAggregateStatus::Pass, 456 /// checks: Box::new([]), 457 /// canonical_json: Box::new([]), 458 /// }; 459 /// ``` 460 pub struct RhiDoctorReport { 461 instance: InstanceId, 462 status: RhiDoctorAggregateStatus, 463 checks: Box<[RhiDoctorCheckResult]>, 464 canonical_json: Box<[u8]>, 465 } 466 467 impl RhiDoctorReport { 468 /// Returns the fixed service identifier. 469 #[must_use] 470 pub const fn service(&self) -> &'static str { 471 RHI_SERVICE 472 } 473 474 /// Returns the validated instance identifier admitted into the report. 475 #[must_use] 476 pub const fn instance(&self) -> &InstanceId { 477 &self.instance 478 } 479 480 /// Returns the aggregate result. 481 #[must_use] 482 pub const fn status(&self) -> RhiDoctorAggregateStatus { 483 self.status 484 } 485 486 /// Returns the ordered complete check inventory. 487 #[must_use] 488 pub fn checks(&self) -> &[RhiDoctorCheckResult] { 489 &self.checks 490 } 491 492 /// Returns exact compact UTF-8 JSON in the shared v1 field order. 493 #[must_use] 494 pub fn canonical_json(&self) -> &[u8] { 495 &self.canonical_json 496 } 497 498 /// Returns exit 6 only when a required check failed or timed out. 499 #[must_use] 500 pub const fn exit_code(&self) -> u8 { 501 match self.status { 502 RhiDoctorAggregateStatus::Fail => DOCTOR_FAILURE_EXIT_CODE, 503 RhiDoctorAggregateStatus::Pass | RhiDoctorAggregateStatus::Degraded => 0, 504 } 505 } 506 } 507 508 impl fmt::Debug for RhiDoctorReport { 509 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 510 formatter 511 .debug_struct("RhiDoctorReport") 512 .field("service", &RHI_SERVICE) 513 .field("instance", &"[redacted]") 514 .field("status", &self.status) 515 .field("check_count", &self.checks.len()) 516 .field("canonical_json", &"[redacted]") 517 .finish() 518 } 519 } 520 521 /// Runs every governed check in exact contract order under its fixed deadline. 522 /// 523 /// Probe implementations retain operation-specific filesystem, SQLite, 524 /// identity, listener, network, source, reconciliation, publication, and clock 525 /// authority. This orchestrator accepts only a closed result and cannot 526 /// serialize their paths or raw errors. 527 pub async fn run_rhi_doctor( 528 context: &RhiRuntimeContext, 529 probe: &(impl RhiDoctorProbe + ?Sized), 530 ) -> Result<RhiDoctorReport, RhiDoctorError> { 531 let mut checks = Vec::with_capacity(RHI_DOCTOR_CHECK_COUNT); 532 for definition in CHECK_DEFINITIONS { 533 let status = match tokio::time::timeout( 534 Duration::from_millis(definition.deadline_ms), 535 probe.probe(definition), 536 ) 537 .await 538 { 539 Ok(RhiDoctorObservation::Pass) => RhiDoctorCheckStatus::Pass, 540 Ok(RhiDoctorObservation::Fail) => RhiDoctorCheckStatus::Fail, 541 Ok(RhiDoctorObservation::Skipped) if !definition.required => { 542 RhiDoctorCheckStatus::Skipped 543 } 544 Ok(RhiDoctorObservation::Skipped) => RhiDoctorCheckStatus::Fail, 545 Err(_) => RhiDoctorCheckStatus::Timeout, 546 }; 547 checks.push(RhiDoctorCheckResult { definition, status }); 548 } 549 let checks = checks.into_boxed_slice(); 550 let status = aggregate_status(&checks); 551 let instance = context.context().instance().clone(); 552 let canonical_json = encode_report(&instance, status, &checks)?; 553 554 Ok(RhiDoctorReport { 555 instance, 556 status, 557 checks, 558 canonical_json, 559 }) 560 } 561 562 fn aggregate_status(checks: &[RhiDoctorCheckResult]) -> RhiDoctorAggregateStatus { 563 if checks 564 .iter() 565 .any(|result| result.definition.required && result.status != RhiDoctorCheckStatus::Pass) 566 { 567 RhiDoctorAggregateStatus::Fail 568 } else if checks 569 .iter() 570 .any(|result| result.status != RhiDoctorCheckStatus::Pass) 571 { 572 RhiDoctorAggregateStatus::Degraded 573 } else { 574 RhiDoctorAggregateStatus::Pass 575 } 576 } 577 578 #[derive(Serialize)] 579 struct DoctorWireReport<'a> { 580 contract_version: u32, 581 service: &'static str, 582 instance: &'a str, 583 status: &'static str, 584 checks: Vec<DoctorWireCheck>, 585 } 586 587 #[derive(Serialize)] 588 struct DoctorWireCheck { 589 id: &'static str, 590 status: &'static str, 591 required: bool, 592 deadline_ms: u64, 593 summary: &'static str, 594 remediation_code: &'static str, 595 } 596 597 fn encode_report( 598 instance: &InstanceId, 599 status: RhiDoctorAggregateStatus, 600 checks: &[RhiDoctorCheckResult], 601 ) -> Result<Box<[u8]>, RhiDoctorError> { 602 let checks = checks 603 .iter() 604 .map(|result| DoctorWireCheck { 605 id: result.definition.id.as_str(), 606 status: result.status.as_str(), 607 required: result.definition.required, 608 deadline_ms: result.definition.deadline_ms, 609 summary: result.status.summary(), 610 remediation_code: result.definition.remediation_code.as_str(), 611 }) 612 .collect(); 613 let encoded = serde_json::to_vec(&DoctorWireReport { 614 contract_version: RHI_DOCTOR_CONTRACT_VERSION, 615 service: RHI_SERVICE, 616 instance: instance.as_str(), 617 status: status.as_str(), 618 checks, 619 }) 620 .map_err(|_| RhiDoctorError::new(RhiDoctorErrorKind::Encoding))?; 621 if encoded.len() > RHI_DOCTOR_REPORT_MAX_UTF8_BYTES { 622 return Err(RhiDoctorError::new(RhiDoctorErrorKind::OutputTooLarge)); 623 } 624 Ok(encoded.into_boxed_slice()) 625 } 626 627 #[cfg(test)] 628 mod tests { 629 use std::error::Error; 630 631 use super::{RhiDoctorError, RhiDoctorErrorKind}; 632 633 #[test] 634 fn errors_are_source_free_and_content_free() { 635 for kind in [ 636 RhiDoctorErrorKind::Encoding, 637 RhiDoctorErrorKind::OutputTooLarge, 638 ] { 639 let error = RhiDoctorError::new(kind); 640 assert!(Error::source(&error).is_none()); 641 let rendered = format!("{error} {error:?}"); 642 for forbidden in ["/private", "secret", "relay", "sqlite"] { 643 assert!(!rendered.to_ascii_lowercase().contains(forbidden)); 644 } 645 } 646 } 647 }