services_hardening_status.rs (18106B)
1 #![forbid(unsafe_code)] 2 3 use std::error::Error; 4 5 use rhi::{ 6 InstanceId, RHI_DETAILED_STATUS_MAX_UTF8_BYTES, RHI_STATUS_CACHE_CONTRACT_VERSION, 7 RhiEvidenceTransportStatusV1, RhiIdentityHealthV1, RhiIntegrityStateV1, RhiPersistenceHealthV1, 8 RhiPersistenceStatusV1, RhiPresenceStatusV1, RhiProviderStatusV1, RhiPublicationStatusV1, 9 RhiReconciliationStatusV1, RhiServicePhase, RhiStatusBuildInfoV1, RhiStatusBuildMode, 10 RhiStatusCommonV1, RhiStatusConfigurationIdentityV1, RhiStatusConfigurationSource, 11 RhiStatusErrorKind, RhiStatusObservationV1, RhiStatusReasonCode, RhiStatusReasonCodes, 12 RhiStatusUnixSeconds, RhiTransportHealthV1, rhi_status_cache, 13 }; 14 15 const CONTRACT: &str = include_str!("../contracts/services_hardening/status_cache.v1.json"); 16 const SERVICE_REVISION: &str = "0123456789abcdef0123456789abcdef01234567"; 17 const LIB_REVISION: &str = "89abcdef0123456789abcdef0123456789abcdef"; 18 19 fn reasons(values: &[&str]) -> RhiStatusReasonCodes { 20 RhiStatusReasonCodes::new( 21 values 22 .iter() 23 .map(|value| RhiStatusReasonCode::new(value).expect("reason code")), 24 ) 25 .expect("reason codes") 26 } 27 28 fn build_info() -> RhiStatusBuildInfoV1 { 29 RhiStatusBuildInfoV1::new( 30 RhiStatusBuildMode::Release, 31 Some("0.1.0"), 32 Some(SERVICE_REVISION), 33 Some(LIB_REVISION), 34 Some("1.97.1"), 35 Some("x86_64-unknown-linux-gnu"), 36 Some("service-host"), 37 ) 38 .expect("build info") 39 } 40 41 fn identity(configured: bool, available: bool, reason: &[&str]) -> RhiIdentityHealthV1 { 42 RhiIdentityHealthV1::new(configured, available, reasons(reason)).expect("identity health") 43 } 44 45 fn observation( 46 phase: RhiServicePhase, 47 ready: bool, 48 uptime_ms: u64, 49 pending_jobs: u64, 50 ) -> RhiStatusObservationV1 { 51 let lifecycle_reasons = if phase == RhiServicePhase::Degraded { 52 reasons(&["source_unavailable"]) 53 } else { 54 RhiStatusReasonCodes::empty() 55 }; 56 let configuration = RhiStatusConfigurationIdentityV1::new( 57 "a".repeat(64), 58 RhiStatusConfigurationSource::ExplicitConfig, 59 ) 60 .expect("configuration"); 61 let persistence = RhiPersistenceStatusV1::new( 62 RhiPersistenceHealthV1::Ready, 63 10, 64 42, 65 RhiIntegrityStateV1::Verified, 66 RhiStatusReasonCodes::empty(), 67 ) 68 .expect("persistence"); 69 let provider = 70 RhiProviderStatusV1::new(identity(true, true, &[]), RhiStatusReasonCodes::empty()) 71 .expect("provider"); 72 let transport = RhiEvidenceTransportStatusV1::new( 73 if phase == RhiServicePhase::Degraded { 74 RhiTransportHealthV1::Degraded 75 } else { 76 RhiTransportHealthV1::Ready 77 }, 78 true, 79 true, 80 2, 81 if phase == RhiServicePhase::Degraded { 82 1 83 } else { 84 2 85 }, 86 if phase == RhiServicePhase::Degraded { 87 reasons(&["source_unavailable"]) 88 } else { 89 RhiStatusReasonCodes::empty() 90 }, 91 ) 92 .expect("transport"); 93 RhiStatusObservationV1::new( 94 RhiStatusCommonV1::new( 95 phase, 96 ready, 97 lifecycle_reasons, 98 uptime_ms, 99 build_info(), 100 configuration, 101 persistence, 102 ) 103 .expect("common status"), 104 provider, 105 transport, 106 RhiReconciliationStatusV1::new( 107 pending_jobs, 108 3, 109 1, 110 Some(RhiStatusUnixSeconds::new(1_723_456_700).expect("job time")), 111 ), 112 RhiPublicationStatusV1::new( 113 4, 114 1, 115 Some(RhiStatusUnixSeconds::new(1_723_456_789).expect("publication time")), 116 ), 117 RhiPresenceStatusV1::new(2, 1), 118 ) 119 } 120 121 #[test] 122 fn machine_contract_and_canonical_detailed_status_are_exact() { 123 let contract: serde_json::Value = serde_json::from_str(CONTRACT).expect("status contract"); 124 assert_eq!(contract["schema"], "radroots.rhi.status-cache.v1"); 125 assert_eq!( 126 contract["contract_version"], 127 RHI_STATUS_CACHE_CONTRACT_VERSION 128 ); 129 assert_eq!(contract["step"], 212); 130 assert_eq!( 131 contract["publication"]["capacity"], 132 "one_latest_immutable_arc" 133 ); 134 assert_eq!(contract["read"]["fresh_probe"], false); 135 assert_eq!( 136 contract["detailed_status"]["maximum_utf8_bytes"], 137 RHI_DETAILED_STATUS_MAX_UTF8_BYTES 138 ); 139 assert_eq!( 140 contract["detailed_status"]["reason_codes"] 141 .as_array() 142 .expect("reason inventory") 143 .iter() 144 .map(|value| value.as_str().expect("reason")) 145 .collect::<Vec<_>>(), 146 [ 147 RhiStatusReasonCode::IdentityUnavailable, 148 RhiStatusReasonCode::DatabaseSchemaMismatch, 149 RhiStatusReasonCode::DatabaseReadOnly, 150 RhiStatusReasonCode::DatabaseLowDisk, 151 RhiStatusReasonCode::SourceUnavailable, 152 RhiStatusReasonCode::SubscriptionInactive, 153 RhiStatusReasonCode::RecoveryIncomplete, 154 RhiStatusReasonCode::PublicationRecoveryIncomplete, 155 RhiStatusReasonCode::PresenceStateUnavailable, 156 RhiStatusReasonCode::ReconciliationBacklogExceeded, 157 RhiStatusReasonCode::AdminListenerFailed, 158 RhiStatusReasonCode::OperationsListenerFailed, 159 RhiStatusReasonCode::ShutdownInProgress, 160 ] 161 .map(RhiStatusReasonCode::as_str) 162 ); 163 164 let (_publisher, reader) = rhi_status_cache( 165 InstanceId::new("primary").expect("instance"), 166 observation(RhiServicePhase::Ready, true, 120_000, 5), 167 ) 168 .expect("status cache"); 169 let snapshot = reader.snapshot(); 170 let wire = std::str::from_utf8(snapshot.detailed_status_json()).expect("status UTF-8"); 171 assert_eq!( 172 wire, 173 r#"{"contract_version":1,"service":"rhi","instance":"primary","phase":"ready","ready":true,"uptime_millis":120000,"reason_codes":[],"build_info":{"version":"0.1.0","revision":"0123456789abcdef0123456789abcdef01234567","toolchain":"1.97.1","contract_versions":{"config":1,"state":11,"admin":1,"status":1,"provider":1}},"configuration":{"schema":"radroots.rhi.config","schema_version":1,"digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","source":"explicit_config"},"persistence":{"health":"ready","schema_version":10,"generation":42,"integrity":"verified","reason_codes":[]},"provider":{"health":"ready","identity":{"configured":true,"available":true,"reason_codes":[]},"reason_codes":[]},"transport":{"health":"ready","required_sources_ready":true,"subscriber_active":true,"configured_source_count":2,"reachable_source_count":2,"reason_codes":[]},"rhi":{"identity":{"configured":true,"available":true,"reason_codes":[]},"reconciliation":{"pending":5,"leased":3,"exhausted":1,"oldest_pending_at_utc":1723456700},"publication":{"pending":4,"unknown":1,"oldest_pending_at_utc":1723456789},"presence":{"pending":2,"unknown":1}}}"# 174 ); 175 assert!(wire.len() < RHI_DETAILED_STATUS_MAX_UTF8_BYTES); 176 for forbidden in [ 177 "secret", 178 "credential", 179 "private_key", 180 "password", 181 "filesystem_path", 182 "relay_url", 183 "raw_error", 184 ] { 185 assert!(!wire.contains(forbidden), "wire leaked `{forbidden}`"); 186 } 187 } 188 189 #[tokio::test] 190 async fn latest_publication_is_atomic_passive_and_retains_old_snapshots() { 191 let (mut publisher, reader) = rhi_status_cache( 192 InstanceId::new("primary").expect("instance"), 193 observation(RhiServicePhase::Starting, false, 0, 9), 194 ) 195 .expect("status cache"); 196 let old = reader.snapshot(); 197 let old_pointer = old.detailed_status_json().as_ptr(); 198 for _ in 0..1_000 { 199 let same = reader.snapshot(); 200 assert_eq!(same.detailed_status_json().as_ptr(), old_pointer); 201 assert_eq!(same.phase(), RhiServicePhase::Starting); 202 } 203 204 let mut changed = publisher.subscribe(); 205 publisher 206 .publish(observation(RhiServicePhase::Ready, true, 10, 2)) 207 .expect("ready publication"); 208 publisher 209 .publish(observation(RhiServicePhase::Degraded, true, 20, 1)) 210 .expect("degraded publication"); 211 212 let latest = changed.changed().await.expect("latest publication"); 213 assert_eq!(latest.phase(), RhiServicePhase::Degraded); 214 assert!(latest.is_ready()); 215 assert!( 216 std::str::from_utf8(latest.detailed_status_json()) 217 .expect("status UTF-8") 218 .contains("\"uptime_millis\":20") 219 ); 220 assert_eq!(old.phase(), RhiServicePhase::Starting); 221 assert!( 222 std::str::from_utf8(old.detailed_status_json()) 223 .expect("old UTF-8") 224 .contains("\"uptime_millis\":0") 225 ); 226 } 227 228 #[tokio::test] 229 async fn illegal_transition_and_publisher_drop_preserve_the_last_valid_value() { 230 let (mut publisher, reader) = rhi_status_cache( 231 InstanceId::new("primary").expect("instance"), 232 observation(RhiServicePhase::Starting, false, 1, 0), 233 ) 234 .expect("status cache"); 235 let before = reader.snapshot().detailed_status_json().to_vec(); 236 let error = publisher 237 .publish(observation(RhiServicePhase::Unready, false, 2, 0)) 238 .expect_err("illegal starting to unready transition"); 239 assert_eq!(error.kind(), RhiStatusErrorKind::InvalidTransition); 240 assert_eq!(reader.snapshot().detailed_status_json(), before); 241 assert!(Error::source(&error).is_none()); 242 243 let mut dropped = publisher.subscribe(); 244 drop(publisher); 245 let dropped_error = dropped.changed().await.expect_err("publisher dropped"); 246 assert_eq!(dropped_error.kind(), RhiStatusErrorKind::PublisherDropped); 247 assert_eq!(reader.snapshot().detailed_status_json(), before); 248 } 249 250 #[test] 251 fn closed_work_counts_time_and_safe_debug_bound_the_status_surface() { 252 let work = RhiReconciliationStatusV1::new(u64::MAX, 2, 3, None); 253 assert_eq!(work.pending(), u64::MAX); 254 assert_eq!(work.leased(), 2); 255 assert_eq!(work.exhausted(), 3); 256 assert_eq!(work.oldest_pending_at_utc(), None); 257 assert_eq!(RhiStatusUnixSeconds::new(0).expect("zero").get(), 0); 258 assert_eq!( 259 RhiStatusUnixSeconds::new(i64::MAX as u64) 260 .expect("maximum") 261 .get(), 262 i64::MAX as u64 263 ); 264 assert_eq!( 265 RhiStatusUnixSeconds::new(i64::MAX as u64 + 1) 266 .expect_err("over maximum") 267 .kind(), 268 RhiStatusErrorKind::InvalidTime 269 ); 270 271 let (publisher, reader) = rhi_status_cache( 272 InstanceId::new("primary").expect("instance"), 273 observation(RhiServicePhase::Ready, true, 5, 0), 274 ) 275 .expect("status cache"); 276 let rendered = format!("{publisher:?} {reader:?} {:?}", reader.snapshot()); 277 for forbidden in [ 278 SERVICE_REVISION, 279 LIB_REVISION, 280 "radroots.rhi.config", 281 "aaaaaaaaaaaaaaaa", 282 "oldest_pending_at_utc", 283 ] { 284 assert!(!rendered.contains(forbidden)); 285 } 286 } 287 288 #[test] 289 fn model_boundaries_fail_closed_before_publication() { 290 assert_eq!( 291 RhiStatusReasonCode::new("") 292 .expect_err("empty reason") 293 .kind(), 294 RhiStatusErrorKind::InvalidReasonCode 295 ); 296 assert_eq!( 297 RhiStatusReasonCode::new("secret_canary_value") 298 .expect_err("unknown reason") 299 .kind(), 300 RhiStatusErrorKind::InvalidReasonCode 301 ); 302 let maximum = [ 303 RhiStatusReasonCode::IdentityUnavailable, 304 RhiStatusReasonCode::DatabaseSchemaMismatch, 305 RhiStatusReasonCode::DatabaseReadOnly, 306 RhiStatusReasonCode::DatabaseLowDisk, 307 RhiStatusReasonCode::SourceUnavailable, 308 RhiStatusReasonCode::SubscriptionInactive, 309 RhiStatusReasonCode::RecoveryIncomplete, 310 RhiStatusReasonCode::PublicationRecoveryIncomplete, 311 RhiStatusReasonCode::PresenceStateUnavailable, 312 RhiStatusReasonCode::ReconciliationBacklogExceeded, 313 RhiStatusReasonCode::AdminListenerFailed, 314 RhiStatusReasonCode::OperationsListenerFailed, 315 RhiStatusReasonCode::ShutdownInProgress, 316 ]; 317 assert_eq!( 318 RhiStatusReasonCodes::new(maximum) 319 .expect("maximum reasons") 320 .as_slice() 321 .len(), 322 13 323 ); 324 let mut infinite = std::iter::repeat(RhiStatusReasonCode::IdentityUnavailable); 325 assert_eq!( 326 RhiStatusReasonCodes::new(&mut infinite) 327 .expect_err("bounded infinite iterator") 328 .kind(), 329 RhiStatusErrorKind::TooManyReasonCodes 330 ); 331 assert_eq!( 332 infinite.next().expect("iterator retained"), 333 RhiStatusReasonCode::IdentityUnavailable 334 ); 335 336 assert_eq!( 337 RhiStatusBuildInfoV1::new( 338 RhiStatusBuildMode::Release, 339 Some("0.1.0"), 340 None, 341 Some(LIB_REVISION), 342 Some("1.97.1"), 343 Some("x86_64-unknown-linux-gnu"), 344 Some("service-host"), 345 ) 346 .expect_err("release revision required") 347 .kind(), 348 RhiStatusErrorKind::InvalidBuildInfo 349 ); 350 assert_eq!( 351 RhiStatusConfigurationIdentityV1::new( 352 "A".repeat(64), 353 RhiStatusConfigurationSource::ExplicitConfig, 354 ) 355 .expect_err("lowercase digest required") 356 .kind(), 357 RhiStatusErrorKind::InvalidConfiguration 358 ); 359 assert_eq!( 360 RhiPersistenceStatusV1::new( 361 RhiPersistenceHealthV1::Ready, 362 0, 363 0, 364 RhiIntegrityStateV1::Verified, 365 RhiStatusReasonCodes::empty(), 366 ) 367 .expect_err("positive schema required") 368 .kind(), 369 RhiStatusErrorKind::InvalidPersistence 370 ); 371 372 let error = RhiStatusCommonV1::new( 373 RhiServicePhase::Ready, 374 false, 375 RhiStatusReasonCodes::empty(), 376 0, 377 build_info(), 378 RhiStatusConfigurationIdentityV1::new( 379 "a".repeat(64), 380 RhiStatusConfigurationSource::ExplicitConfig, 381 ) 382 .expect("configuration"), 383 RhiPersistenceStatusV1::new( 384 RhiPersistenceHealthV1::Ready, 385 10, 386 0, 387 RhiIntegrityStateV1::Verified, 388 RhiStatusReasonCodes::empty(), 389 ) 390 .expect("persistence"), 391 ) 392 .expect_err("ready phase requires readiness"); 393 assert_eq!(error.kind(), RhiStatusErrorKind::InvalidLifecycle); 394 395 let inconsistent = RhiStatusObservationV1::new( 396 RhiStatusCommonV1::new( 397 RhiServicePhase::Ready, 398 true, 399 RhiStatusReasonCodes::empty(), 400 1, 401 build_info(), 402 RhiStatusConfigurationIdentityV1::new( 403 "a".repeat(64), 404 RhiStatusConfigurationSource::ExplicitConfig, 405 ) 406 .expect("configuration"), 407 RhiPersistenceStatusV1::new( 408 RhiPersistenceHealthV1::Ready, 409 10, 410 1, 411 RhiIntegrityStateV1::Verified, 412 RhiStatusReasonCodes::empty(), 413 ) 414 .expect("persistence"), 415 ) 416 .expect("common"), 417 RhiProviderStatusV1::new( 418 identity(true, false, &["identity_unavailable"]), 419 reasons(&["identity_unavailable"]), 420 ) 421 .expect("provider"), 422 RhiEvidenceTransportStatusV1::new( 423 RhiTransportHealthV1::Ready, 424 true, 425 true, 426 1, 427 1, 428 RhiStatusReasonCodes::empty(), 429 ) 430 .expect("transport"), 431 RhiReconciliationStatusV1::default(), 432 RhiPublicationStatusV1::default(), 433 RhiPresenceStatusV1::default(), 434 ); 435 assert_eq!( 436 rhi_status_cache(InstanceId::new("primary").expect("instance"), inconsistent,) 437 .expect_err("ready status requires healthy critical dependencies") 438 .kind(), 439 RhiStatusErrorKind::InvalidLifecycle 440 ); 441 } 442 443 #[test] 444 fn provider_transport_and_optional_oldest_time_are_deterministic() { 445 let ready = RhiProviderStatusV1::new(identity(true, true, &[]), RhiStatusReasonCodes::empty()) 446 .expect("ready provider"); 447 assert_eq!(ready.health(), rhi::RhiProviderHealthV1::Ready); 448 449 let unavailable = RhiProviderStatusV1::new( 450 identity(true, false, &["identity_unavailable"]), 451 reasons(&["identity_unavailable"]), 452 ) 453 .expect("unavailable provider"); 454 assert_eq!(unavailable.health(), rhi::RhiProviderHealthV1::Unavailable); 455 456 let (_publisher, reader) = rhi_status_cache( 457 InstanceId::new("primary").expect("instance"), 458 RhiStatusObservationV1::new( 459 RhiStatusCommonV1::new( 460 RhiServicePhase::Ready, 461 true, 462 RhiStatusReasonCodes::empty(), 463 1, 464 build_info(), 465 RhiStatusConfigurationIdentityV1::new( 466 "a".repeat(64), 467 RhiStatusConfigurationSource::ExplicitConfig, 468 ) 469 .expect("configuration"), 470 RhiPersistenceStatusV1::new( 471 RhiPersistenceHealthV1::Ready, 472 10, 473 0, 474 RhiIntegrityStateV1::Verified, 475 RhiStatusReasonCodes::empty(), 476 ) 477 .expect("persistence"), 478 ) 479 .expect("common status"), 480 ready, 481 RhiEvidenceTransportStatusV1::new( 482 RhiTransportHealthV1::Ready, 483 true, 484 true, 485 1, 486 1, 487 RhiStatusReasonCodes::empty(), 488 ) 489 .expect("transport"), 490 RhiReconciliationStatusV1::default(), 491 RhiPublicationStatusV1::new(0, 0, None), 492 RhiPresenceStatusV1::default(), 493 ), 494 ) 495 .expect("cache"); 496 let wire = std::str::from_utf8(reader.snapshot().detailed_status_json()) 497 .expect("status UTF-8") 498 .to_owned(); 499 assert!(wire.contains("\"publication\":{\"pending\":0,\"unknown\":0}")); 500 assert!(!wire.contains("oldest_pending_at_utc")); 501 }