services_hardening_doctor.rs (14345B)
1 #![forbid(unsafe_code)] 2 3 use std::{ 4 collections::{BTreeMap, BTreeSet}, 5 future::pending, 6 process::Command, 7 sync::{ 8 Arc, Mutex, 9 atomic::{AtomicBool, Ordering}, 10 }, 11 }; 12 13 use rhi::{ 14 RHI_DOCTOR_CHECK_COUNT, RHI_DOCTOR_CONTRACT_VERSION, RHI_DOCTOR_REPORT_MAX_UTF8_BYTES, 15 RHI_DOCTOR_SUMMARY_MAX_UTF8_BYTES, RadrootsHostEnvironment, RadrootsPathResolver, 16 RadrootsPlatform, RhiDoctorAggregateStatus, RhiDoctorCheckDefinition, RhiDoctorCheckId, 17 RhiDoctorCheckStatus, RhiDoctorFuture, RhiDoctorObservation, RhiDoctorProbe, 18 RhiDoctorRemediationCode, RhiProcessResult, parse_rhi_cli_v1_from, resolve_rhi_runtime_context, 19 rhi_doctor_check_definitions, run_rhi_doctor, 20 }; 21 use sha2::{Digest, Sha256}; 22 23 const OPERATOR_CONTRACT: &str = 24 include_str!("../contracts/services_hardening/operator_contract.v1.json"); 25 26 struct PendingGuard(Arc<AtomicBool>); 27 28 impl Drop for PendingGuard { 29 fn drop(&mut self) { 30 self.0.store(true, Ordering::SeqCst); 31 } 32 } 33 34 struct TestProbe { 35 outcomes: BTreeMap<RhiDoctorCheckId, RhiDoctorObservation>, 36 pending: Option<RhiDoctorCheckId>, 37 calls: Arc<Mutex<Vec<RhiDoctorCheckId>>>, 38 pending_dropped: Arc<AtomicBool>, 39 } 40 41 impl TestProbe { 42 fn all(outcome: RhiDoctorObservation) -> Self { 43 Self { 44 outcomes: rhi_doctor_check_definitions() 45 .iter() 46 .map(|definition| (definition.id(), outcome)) 47 .collect(), 48 pending: None, 49 calls: Arc::new(Mutex::new(Vec::new())), 50 pending_dropped: Arc::new(AtomicBool::new(false)), 51 } 52 } 53 54 fn with(mut self, id: RhiDoctorCheckId, outcome: RhiDoctorObservation) -> Self { 55 self.outcomes.insert(id, outcome); 56 self 57 } 58 59 fn pending(mut self, id: RhiDoctorCheckId) -> Self { 60 self.pending = Some(id); 61 self 62 } 63 } 64 65 impl RhiDoctorProbe for TestProbe { 66 fn probe(&self, definition: RhiDoctorCheckDefinition) -> RhiDoctorFuture<'_> { 67 let id = definition.id(); 68 self.calls.lock().expect("calls lock").push(id); 69 if self.pending == Some(id) { 70 let dropped = Arc::clone(&self.pending_dropped); 71 return Box::pin(async move { 72 let _guard = PendingGuard(dropped); 73 pending().await 74 }); 75 } 76 let outcome = self.outcomes[&id]; 77 Box::pin(async move { outcome }) 78 } 79 } 80 81 fn runtime() -> (tempfile::TempDir, rhi::RhiRuntimeContext) { 82 let directory = tempfile::tempdir().expect("temporary root"); 83 let invocation = parse_rhi_cli_v1_from([ 84 "rhi", 85 "--profile", 86 "repo-local", 87 "--instance", 88 "primary", 89 "--repo-local-root", 90 directory.path().to_str().expect("UTF-8 path"), 91 "doctor", 92 ]) 93 .expect("doctor invocation"); 94 let context = resolve_rhi_runtime_context( 95 &RadrootsPathResolver::new(RadrootsPlatform::Linux, RadrootsHostEnvironment::default()), 96 &invocation, 97 ) 98 .expect("runtime context"); 99 (directory, context) 100 } 101 102 #[test] 103 fn exact_inventory_and_exit_meanings_match_the_operator_contract() { 104 let contract: serde_json::Value = 105 serde_json::from_str(OPERATOR_CONTRACT).expect("operator contract"); 106 let doctor = contract["doctor"].as_object().expect("doctor"); 107 assert_eq!( 108 doctor.keys().map(String::as_str).collect::<BTreeSet<_>>(), 109 BTreeSet::from([ 110 "aggregate_statuses", 111 "checks", 112 "contract_version", 113 "detached_probe_work", 114 "execution", 115 "pass_requires_all_scope", 116 "probe_future_cancellation", 117 "raw_error_or_path_allowed", 118 "report_max_utf8_bytes", 119 "required_fail_or_timeout_exit", 120 "required_skipped", 121 "shared_schema", 122 "statuses", 123 "summary_max_utf8_bytes", 124 ]) 125 ); 126 assert_eq!(RHI_DOCTOR_CONTRACT_VERSION, 1); 127 assert_eq!(RHI_DOCTOR_CHECK_COUNT, 15); 128 assert_eq!(RHI_DOCTOR_SUMMARY_MAX_UTF8_BYTES, 256); 129 assert_eq!(RHI_DOCTOR_REPORT_MAX_UTF8_BYTES, 8_192); 130 assert_eq!(doctor["execution"], "ordered"); 131 assert_eq!(doctor["pass_requires_all_scope"], true); 132 assert_eq!( 133 doctor["probe_future_cancellation"], 134 "drop_stops_or_owns_cleanup" 135 ); 136 assert_eq!(doctor["detached_probe_work"], false); 137 assert_eq!(doctor["required_skipped"], "forbidden"); 138 assert_eq!(doctor["raw_error_or_path_allowed"], false); 139 assert_eq!(doctor["required_fail_or_timeout_exit"], 6); 140 141 let rows = doctor["checks"].as_array().expect("checks"); 142 assert_eq!(rows.len(), RHI_DOCTOR_CHECK_COUNT); 143 for (definition, row) in rhi_doctor_check_definitions().iter().zip(rows) { 144 assert_eq!(row["id"], id_name(definition.id())); 145 assert_eq!(row["required"], definition.required()); 146 assert_eq!(row["deadline_ms"], definition.deadline_ms()); 147 assert_eq!( 148 row["remediation_code"], 149 remediation_name(definition.remediation_code()) 150 ); 151 assert_eq!( 152 row["scope"], 153 serde_json::to_value(definition.scope()).expect("scope") 154 ); 155 } 156 157 let process_results = [ 158 RhiProcessResult::Success, 159 RhiProcessResult::UnexpectedInternal, 160 RhiProcessResult::InputOrConfiguration, 161 RhiProcessResult::ServiceOrDependencyUnavailable, 162 RhiProcessResult::StateOrIdentityUnavailable, 163 RhiProcessResult::OperationRejectedOrConflict, 164 RhiProcessResult::DoctorRequiredCheckFailed, 165 ]; 166 for (result, row) in process_results 167 .into_iter() 168 .zip(contract["exit_codes"].as_array().expect("exit codes")) 169 { 170 assert_eq!(row["code"], result.exit_code_u8()); 171 assert_eq!(row["name"], result.code()); 172 } 173 } 174 175 #[tokio::test] 176 async fn all_pass_is_canonical_bounded_and_exit_zero() { 177 let (_directory, context) = runtime(); 178 let probe = TestProbe::all(RhiDoctorObservation::Pass); 179 let report = run_rhi_doctor(&context, &probe).await.expect("report"); 180 assert_eq!(report.service(), "rhi"); 181 assert_eq!(report.instance().as_str(), "primary"); 182 assert_eq!(report.status(), RhiDoctorAggregateStatus::Pass); 183 assert_eq!(report.exit_code(), 0); 184 assert_eq!(report.checks().len(), RHI_DOCTOR_CHECK_COUNT); 185 assert!( 186 report 187 .checks() 188 .iter() 189 .all(|result| result.status() == RhiDoctorCheckStatus::Pass) 190 ); 191 assert_eq!( 192 probe.calls.lock().expect("calls").len(), 193 RHI_DOCTOR_CHECK_COUNT 194 ); 195 196 let bytes = report.canonical_json(); 197 assert!(bytes.len() <= RHI_DOCTOR_REPORT_MAX_UTF8_BYTES); 198 assert!(!bytes.contains(&b'\n')); 199 let wire: serde_json::Value = serde_json::from_slice(bytes).expect("JSON"); 200 assert_eq!(wire["contract_version"], 1); 201 assert_eq!(wire["service"], "rhi"); 202 assert_eq!(wire["instance"], "primary"); 203 assert_eq!(wire["status"], "pass"); 204 assert_eq!(wire["checks"].as_array().expect("checks").len(), 15); 205 assert!(String::from_utf8_lossy(bytes).starts_with( 206 "{\"contract_version\":1,\"service\":\"rhi\",\"instance\":\"primary\",\"status\":\"pass\",\"checks\":[" 207 )); 208 assert_eq!( 209 sha256_hex(bytes), 210 "5aabfc84927e36bb877c289f8d0ed2be8f4c3115deedad7def3f3e56daab399b" 211 ); 212 } 213 214 #[tokio::test] 215 async fn optional_nonpass_is_degraded_while_required_nonpass_fails() { 216 let (_directory, context) = runtime(); 217 for outcome in [RhiDoctorObservation::Fail, RhiDoctorObservation::Skipped] { 218 let optional = 219 TestProbe::all(RhiDoctorObservation::Pass).with(RhiDoctorCheckId::ClockSkew, outcome); 220 let report = run_rhi_doctor(&context, &optional) 221 .await 222 .expect("optional report"); 223 assert_eq!(report.status(), RhiDoctorAggregateStatus::Degraded); 224 assert_eq!(report.exit_code(), 0); 225 assert_ne!(report.checks()[14].status(), RhiDoctorCheckStatus::Pass); 226 227 let required = 228 TestProbe::all(RhiDoctorObservation::Pass).with(RhiDoctorCheckId::WriterLock, outcome); 229 let report = run_rhi_doctor(&context, &required) 230 .await 231 .expect("required report"); 232 assert_eq!(report.status(), RhiDoctorAggregateStatus::Fail); 233 assert_eq!(report.exit_code(), 6); 234 assert_eq!(report.checks()[1].status(), RhiDoctorCheckStatus::Fail); 235 } 236 } 237 238 #[tokio::test] 239 async fn required_timeout_drops_work_and_remaining_checks_continue_in_order() { 240 let (_directory, context) = runtime(); 241 let probe = 242 TestProbe::all(RhiDoctorObservation::Pass).pending(RhiDoctorCheckId::PathsPermissions); 243 let report = run_rhi_doctor(&context, &probe).await.expect("report"); 244 assert_eq!(report.status(), RhiDoctorAggregateStatus::Fail); 245 assert_eq!(report.exit_code(), 6); 246 assert_eq!(report.checks()[0].status(), RhiDoctorCheckStatus::Timeout); 247 assert!(probe.pending_dropped.load(Ordering::SeqCst)); 248 assert_eq!( 249 *probe.calls.lock().expect("calls"), 250 rhi_doctor_check_definitions() 251 .iter() 252 .map(|definition| definition.id()) 253 .collect::<Vec<_>>() 254 ); 255 } 256 257 #[tokio::test] 258 async fn report_debug_and_public_errors_retain_no_sensitive_values() { 259 let directory = tempfile::tempdir().expect("temporary root"); 260 let root = directory.path().join("secret-root"); 261 let invocation = parse_rhi_cli_v1_from([ 262 "rhi", 263 "--profile", 264 "repo-local", 265 "--instance", 266 "secret-instance", 267 "--repo-local-root", 268 root.to_str().expect("UTF-8 path"), 269 "doctor", 270 ]) 271 .expect("doctor invocation"); 272 let context = resolve_rhi_runtime_context( 273 &RadrootsPathResolver::new(RadrootsPlatform::Linux, RadrootsHostEnvironment::default()), 274 &invocation, 275 ) 276 .expect("runtime context"); 277 let report = run_rhi_doctor(&context, &TestProbe::all(RhiDoctorObservation::Pass)) 278 .await 279 .expect("report"); 280 let debug = format!("{report:?}"); 281 assert!(!debug.contains("secret-instance")); 282 assert!(!debug.contains("secret-root")); 283 for result in report.checks() { 284 assert!(result.summary().len() <= RHI_DOCTOR_SUMMARY_MAX_UTF8_BYTES); 285 } 286 let rendered = format!("{:?}", rhi::RhiDoctorErrorKind::OutputTooLarge); 287 assert!(!rendered.contains("secret")); 288 } 289 290 #[test] 291 fn binary_uses_stable_safe_nonzero_results() { 292 const INPUT_FAILURE: &str = concat!( 293 r#"{"schema":"radroots.rhi.log.v1","contract_version":1,"service":"rhi","#, 294 r#""level":"error","event":"process_result","code":"input_or_configuration","exit_code":2}"#, 295 "\n" 296 ); 297 let canary = "secret-canary-private-key-path-sql-relay-url"; 298 let invalid = Command::new(env!("CARGO_BIN_EXE_rhi")) 299 .arg(format!("--credential={canary}")) 300 .output() 301 .expect("invalid invocation"); 302 assert_eq!(invalid.status.code(), Some(2)); 303 assert!(invalid.stdout.is_empty()); 304 let stderr = String::from_utf8(invalid.stderr).expect("invalid stderr"); 305 assert_eq!(stderr, INPUT_FAILURE); 306 assert!(!stderr.contains(canary)); 307 308 let repo_local = tempfile::tempdir().expect("repo-local root"); 309 let admitted = Command::new(env!("CARGO_BIN_EXE_rhi")) 310 .args(["--profile", "repo-local", "--instance", "primary"]) 311 .arg("--repo-local-root") 312 .arg(repo_local.path()) 313 .arg("run") 314 .output() 315 .expect("admitted invocation"); 316 assert_eq!(admitted.status.code(), Some(2)); 317 assert!(admitted.stdout.is_empty()); 318 assert_eq!( 319 String::from_utf8(admitted.stderr).expect("admitted stderr"), 320 INPUT_FAILURE 321 ); 322 } 323 324 fn sha256_hex(bytes: &[u8]) -> String { 325 Sha256::digest(bytes) 326 .iter() 327 .map(|byte| format!("{byte:02x}")) 328 .collect() 329 } 330 331 fn id_name(id: RhiDoctorCheckId) -> &'static str { 332 match id { 333 RhiDoctorCheckId::PathsPermissions => "paths_permissions", 334 RhiDoctorCheckId::WriterLock => "writer_lock", 335 RhiDoctorCheckId::SqliteSchema => "sqlite_schema", 336 RhiDoctorCheckId::SqliteIntegrity => "sqlite_integrity", 337 RhiDoctorCheckId::SqliteFreeSpace => "sqlite_free_space", 338 RhiDoctorCheckId::IdentityBinding => "identity_binding", 339 RhiDoctorCheckId::AdminBindPolicy => "admin_bind_policy", 340 RhiDoctorCheckId::OperationsBindPolicy => "operations_bind_policy", 341 RhiDoctorCheckId::NetworkPolicy => "network_policy", 342 RhiDoctorCheckId::RequiredSources => "required_sources", 343 RhiDoctorCheckId::CursorCheckpoint => "cursor_checkpoint", 344 RhiDoctorCheckId::ReconciliationLeases => "reconciliation_leases", 345 RhiDoctorCheckId::ReconciliationBacklog => "reconciliation_backlog", 346 RhiDoctorCheckId::PublicationInvariants => "publication_invariants", 347 RhiDoctorCheckId::ClockSkew => "clock_skew", 348 } 349 } 350 351 fn remediation_name(code: RhiDoctorRemediationCode) -> &'static str { 352 match code { 353 RhiDoctorRemediationCode::CorrectPathPolicy => "correct_path_policy", 354 RhiDoctorRemediationCode::ReleaseWriterLock => "release_writer_lock", 355 RhiDoctorRemediationCode::RepairSchema => "repair_schema", 356 RhiDoctorRemediationCode::RestoreVerifiedState => "restore_verified_state", 357 RhiDoctorRemediationCode::FreeStateDiskSpace => "free_state_disk_space", 358 RhiDoctorRemediationCode::RestoreIdentityBinding => "restore_identity_binding", 359 RhiDoctorRemediationCode::CorrectAdminBindPolicy => "correct_admin_bind_policy", 360 RhiDoctorRemediationCode::CorrectOperationsBindPolicy => "correct_operations_bind_policy", 361 RhiDoctorRemediationCode::CorrectNetworkPolicy => "correct_network_policy", 362 RhiDoctorRemediationCode::RestoreRequiredSources => "restore_required_sources", 363 RhiDoctorRemediationCode::RepairCursorCheckpoint => "repair_cursor_checkpoint", 364 RhiDoctorRemediationCode::RepairReconciliationLeases => "repair_reconciliation_leases", 365 RhiDoctorRemediationCode::ReduceReconciliationBacklog => "reduce_reconciliation_backlog", 366 RhiDoctorRemediationCode::RepairPublicationState => "repair_publication_state", 367 RhiDoctorRemediationCode::CorrectClock => "correct_clock", 368 } 369 }