services_hardening_status.rs (16832B)
1 #![forbid(unsafe_code)] 2 3 use std::error::Error; 4 5 use myc::{ 6 InstanceId, MYC_DETAILED_STATUS_MAX_UTF8_BYTES, MYC_STATE_SCHEMA_VERSION, 7 MYC_STATUS_CACHE_CONTRACT_VERSION, MycConnectionCountsV1, MycIdentityHealthV1, 8 MycIntegrityStateV1, MycOutboxStatusV1, MycPersistenceHealthV1, MycPersistenceStatusV1, 9 MycProviderStatusV1, MycRelayTransportStatusV1, MycServicePhase, MycStatusBuildInfoV1, 10 MycStatusBuildMode, MycStatusCommonV1, MycStatusConfigurationIdentityV1, 11 MycStatusConfigurationSource, MycStatusErrorKind, MycStatusObservationV1, MycStatusReasonCode, 12 MycStatusReasonCodes, MycStatusUnixSeconds, MycTransportHealthV1, myc_status_cache, 13 }; 14 15 const CONTRACT: &str = include_str!("../contracts/services_hardening/status_cache.v1.json"); 16 const SERVICE_REVISION: &str = "0123456789abcdef0123456789abcdef01234567"; 17 const LIB_REVISION: &str = "89abcdef0123456789abcdef0123456789abcdef"; 18 19 fn reasons(values: &[&str]) -> MycStatusReasonCodes { 20 MycStatusReasonCodes::new( 21 values 22 .iter() 23 .map(|value| MycStatusReasonCode::new(value).expect("reason code")), 24 ) 25 .expect("reason codes") 26 } 27 28 fn build_info() -> MycStatusBuildInfoV1 { 29 MycStatusBuildInfoV1::new( 30 MycStatusBuildMode::Release, 31 Some("0.1.0"), 32 Some(SERVICE_REVISION), 33 Some(LIB_REVISION), 34 Some("1.97.1"), 35 Some("x86_64-unknown-linux-gnu"), 36 Some("service-host"), 37 ) 38 .expect("build info") 39 } 40 41 fn identity(configured: bool, available: bool, reason: &[&str]) -> MycIdentityHealthV1 { 42 MycIdentityHealthV1::new(configured, available, reasons(reason)).expect("identity health") 43 } 44 45 fn observation( 46 phase: MycServicePhase, 47 ready: bool, 48 uptime_ms: u64, 49 pending_connections: u64, 50 ) -> MycStatusObservationV1 { 51 let lifecycle_reasons = if phase == MycServicePhase::Degraded { 52 reasons(&["required_relay_unavailable"]) 53 } else { 54 MycStatusReasonCodes::empty() 55 }; 56 let configuration = MycStatusConfigurationIdentityV1::new( 57 "a".repeat(64), 58 MycStatusConfigurationSource::ExplicitConfig, 59 ) 60 .expect("configuration"); 61 let persistence = MycPersistenceStatusV1::new( 62 MycPersistenceHealthV1::Ready, 63 MYC_STATE_SCHEMA_VERSION, 64 42, 65 MycIntegrityStateV1::Verified, 66 MycStatusReasonCodes::empty(), 67 ) 68 .expect("persistence"); 69 let provider = MycProviderStatusV1::new( 70 identity(true, true, &[]), 71 identity(true, true, &[]), 72 identity(false, false, &[]), 73 MycStatusReasonCodes::empty(), 74 ) 75 .expect("provider"); 76 let transport = MycRelayTransportStatusV1::new( 77 if phase == MycServicePhase::Degraded { 78 MycTransportHealthV1::Degraded 79 } else { 80 MycTransportHealthV1::Ready 81 }, 82 phase != MycServicePhase::Degraded, 83 2, 84 if phase == MycServicePhase::Degraded { 85 reasons(&["required_relay_unavailable"]) 86 } else { 87 MycStatusReasonCodes::empty() 88 }, 89 ) 90 .expect("transport"); 91 MycStatusObservationV1::new( 92 MycStatusCommonV1::new( 93 phase, 94 ready, 95 lifecycle_reasons, 96 uptime_ms, 97 build_info(), 98 configuration, 99 persistence, 100 ) 101 .expect("common status"), 102 provider, 103 transport, 104 MycConnectionCountsV1::new(pending_connections, 3, 1, 2), 105 MycOutboxStatusV1::new( 106 4, 107 1, 108 Some(MycStatusUnixSeconds::new(1_723_456_789).expect("outbox time")), 109 ), 110 ) 111 } 112 113 #[test] 114 fn machine_contract_and_canonical_detailed_status_are_exact() { 115 let contract: serde_json::Value = serde_json::from_str(CONTRACT).expect("status contract"); 116 assert_eq!(contract["schema"], "radroots.myc.status-cache.v1"); 117 assert_eq!( 118 contract["contract_version"], 119 MYC_STATUS_CACHE_CONTRACT_VERSION 120 ); 121 assert_eq!(contract["step"], 154); 122 assert_eq!( 123 contract["publication"]["capacity"], 124 "one_latest_immutable_arc" 125 ); 126 assert_eq!(contract["read"]["fresh_probe"], false); 127 assert_eq!( 128 contract["detailed_status"]["maximum_utf8_bytes"], 129 MYC_DETAILED_STATUS_MAX_UTF8_BYTES 130 ); 131 assert_eq!( 132 contract["detailed_status"]["reason_codes"] 133 .as_array() 134 .expect("reason inventory") 135 .iter() 136 .map(|value| value.as_str().expect("reason")) 137 .collect::<Vec<_>>(), 138 [ 139 MycStatusReasonCode::IdentityUnavailable, 140 MycStatusReasonCode::DatabaseSchemaMismatch, 141 MycStatusReasonCode::DatabaseReadOnly, 142 MycStatusReasonCode::DatabaseLowDisk, 143 MycStatusReasonCode::RequiredRelayUnavailable, 144 MycStatusReasonCode::SubscriberNotActive, 145 MycStatusReasonCode::SignerProviderUnavailable, 146 MycStatusReasonCode::OutboxInvariantFailed, 147 MycStatusReasonCode::PublicationBacklogExceeded, 148 MycStatusReasonCode::AdminListenerFailed, 149 MycStatusReasonCode::OperationsListenerFailed, 150 MycStatusReasonCode::ShutdownInProgress, 151 ] 152 .map(MycStatusReasonCode::as_str) 153 ); 154 155 let (_publisher, reader) = myc_status_cache( 156 InstanceId::new("primary").expect("instance"), 157 observation(MycServicePhase::Ready, true, 120_000, 5), 158 ) 159 .expect("status cache"); 160 let snapshot = reader.snapshot(); 161 let wire = std::str::from_utf8(snapshot.detailed_status_json()).expect("status UTF-8"); 162 assert_eq!( 163 wire, 164 r#"{"contract_version":1,"service":"myc","instance":"primary","phase":"ready","ready":true,"uptime_millis":120000,"reason_codes":[],"build_info":{"version":"0.1.0","revision":"0123456789abcdef0123456789abcdef01234567","toolchain":"1.97.1","contract_versions":{"config":1,"state":12,"admin":1,"status":1,"provider":1}},"configuration":{"schema":"radroots.myc.config","schema_version":1,"digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","source":"explicit_config"},"persistence":{"health":"ready","schema_version":12,"generation":42,"integrity":"verified","reason_codes":[]},"provider":{"health":"ready","transport":{"configured":true,"available":true,"reason_codes":[]},"user":{"configured":true,"available":true,"reason_codes":[]},"discovery":{"configured":false,"available":false,"reason_codes":[]},"reason_codes":[]},"transport":{"health":"ready","required_relays_ready":true,"connected_relay_count":2,"reason_codes":[]},"myc":{"transport":{"configured":true,"available":true,"reason_codes":[]},"user":{"configured":true,"available":true,"reason_codes":[]},"discovery":{"configured":false,"available":false,"reason_codes":[]},"connection_counts":{"pending":5,"active":3,"denied":1,"expired":2},"outbox":{"pending":4,"unknown":1,"oldest_pending_at_utc":1723456789}}}"# 165 ); 166 assert!(wire.len() < MYC_DETAILED_STATUS_MAX_UTF8_BYTES); 167 for forbidden in [ 168 "secret", 169 "credential", 170 "private_key", 171 "password", 172 "filesystem_path", 173 "relay_url", 174 "raw_error", 175 ] { 176 assert!(!wire.contains(forbidden), "wire leaked `{forbidden}`"); 177 } 178 } 179 180 #[tokio::test] 181 async fn latest_publication_is_atomic_passive_and_retains_old_snapshots() { 182 let (mut publisher, reader) = myc_status_cache( 183 InstanceId::new("primary").expect("instance"), 184 observation(MycServicePhase::Starting, false, 0, 9), 185 ) 186 .expect("status cache"); 187 let old = reader.snapshot(); 188 let old_pointer = old.detailed_status_json().as_ptr(); 189 for _ in 0..1_000 { 190 let same = reader.snapshot(); 191 assert_eq!(same.detailed_status_json().as_ptr(), old_pointer); 192 assert_eq!(same.phase(), MycServicePhase::Starting); 193 } 194 195 let mut changed = publisher.subscribe(); 196 publisher 197 .publish(observation(MycServicePhase::Ready, true, 10, 2)) 198 .expect("ready publication"); 199 publisher 200 .publish(observation(MycServicePhase::Degraded, true, 20, 1)) 201 .expect("degraded publication"); 202 203 let latest = changed.changed().await.expect("latest publication"); 204 assert_eq!(latest.phase(), MycServicePhase::Degraded); 205 assert!(latest.is_ready()); 206 assert!( 207 std::str::from_utf8(latest.detailed_status_json()) 208 .expect("status UTF-8") 209 .contains("\"uptime_millis\":20") 210 ); 211 assert_eq!(old.phase(), MycServicePhase::Starting); 212 assert!( 213 std::str::from_utf8(old.detailed_status_json()) 214 .expect("old UTF-8") 215 .contains("\"uptime_millis\":0") 216 ); 217 } 218 219 #[tokio::test] 220 async fn illegal_transition_and_publisher_drop_preserve_the_last_valid_value() { 221 let (mut publisher, reader) = myc_status_cache( 222 InstanceId::new("primary").expect("instance"), 223 observation(MycServicePhase::Starting, false, 1, 0), 224 ) 225 .expect("status cache"); 226 let before = reader.snapshot().detailed_status_json().to_vec(); 227 let error = publisher 228 .publish(observation(MycServicePhase::Unready, false, 2, 0)) 229 .expect_err("illegal starting to unready transition"); 230 assert_eq!(error.kind(), MycStatusErrorKind::InvalidTransition); 231 assert_eq!(reader.snapshot().detailed_status_json(), before); 232 assert!(Error::source(&error).is_none()); 233 234 let mut dropped = publisher.subscribe(); 235 drop(publisher); 236 let dropped_error = dropped.changed().await.expect_err("publisher dropped"); 237 assert_eq!(dropped_error.kind(), MycStatusErrorKind::PublisherDropped); 238 assert_eq!(reader.snapshot().detailed_status_json(), before); 239 } 240 241 #[test] 242 fn closed_role_counts_time_and_safe_debug_bound_the_status_surface() { 243 let counts = MycConnectionCountsV1::new(u64::MAX, 2, 3, 4); 244 assert_eq!(counts.pending(), u64::MAX); 245 assert_eq!(counts.active(), 2); 246 assert_eq!(counts.denied(), 3); 247 assert_eq!(counts.expired(), 4); 248 assert_eq!(MycStatusUnixSeconds::new(0).expect("zero").get(), 0); 249 assert_eq!( 250 MycStatusUnixSeconds::new(i64::MAX as u64) 251 .expect("maximum") 252 .get(), 253 i64::MAX as u64 254 ); 255 assert_eq!( 256 MycStatusUnixSeconds::new(i64::MAX as u64 + 1) 257 .expect_err("over maximum") 258 .kind(), 259 MycStatusErrorKind::InvalidTime 260 ); 261 262 let (publisher, reader) = myc_status_cache( 263 InstanceId::new("primary").expect("instance"), 264 observation(MycServicePhase::Ready, true, 5, 0), 265 ) 266 .expect("status cache"); 267 let rendered = format!("{publisher:?} {reader:?} {:?}", reader.snapshot()); 268 for forbidden in [ 269 SERVICE_REVISION, 270 LIB_REVISION, 271 "radroots.myc.config", 272 "aaaaaaaaaaaaaaaa", 273 "oldest_pending_at_utc", 274 ] { 275 assert!(!rendered.contains(forbidden)); 276 } 277 } 278 279 #[test] 280 fn model_boundaries_fail_closed_before_publication() { 281 assert_eq!( 282 MycStatusReasonCode::new("") 283 .expect_err("empty reason") 284 .kind(), 285 MycStatusErrorKind::InvalidReasonCode 286 ); 287 assert_eq!( 288 MycStatusReasonCode::new("secret_canary_value") 289 .expect_err("unknown reason") 290 .kind(), 291 MycStatusErrorKind::InvalidReasonCode 292 ); 293 let maximum = [ 294 MycStatusReasonCode::IdentityUnavailable, 295 MycStatusReasonCode::DatabaseSchemaMismatch, 296 MycStatusReasonCode::DatabaseReadOnly, 297 MycStatusReasonCode::DatabaseLowDisk, 298 MycStatusReasonCode::RequiredRelayUnavailable, 299 MycStatusReasonCode::SubscriberNotActive, 300 MycStatusReasonCode::SignerProviderUnavailable, 301 MycStatusReasonCode::OutboxInvariantFailed, 302 MycStatusReasonCode::PublicationBacklogExceeded, 303 MycStatusReasonCode::AdminListenerFailed, 304 MycStatusReasonCode::OperationsListenerFailed, 305 MycStatusReasonCode::ShutdownInProgress, 306 ]; 307 assert_eq!( 308 MycStatusReasonCodes::new(maximum) 309 .expect("maximum reasons") 310 .as_slice() 311 .len(), 312 12 313 ); 314 let mut infinite = std::iter::repeat(MycStatusReasonCode::IdentityUnavailable); 315 assert_eq!( 316 MycStatusReasonCodes::new(&mut infinite) 317 .expect_err("bounded infinite iterator") 318 .kind(), 319 MycStatusErrorKind::TooManyReasonCodes 320 ); 321 assert_eq!( 322 infinite.next().expect("iterator retained"), 323 MycStatusReasonCode::IdentityUnavailable 324 ); 325 326 assert_eq!( 327 MycStatusBuildInfoV1::new( 328 MycStatusBuildMode::Release, 329 Some("0.1.0"), 330 None, 331 Some(LIB_REVISION), 332 Some("1.97.1"), 333 Some("x86_64-unknown-linux-gnu"), 334 Some("service-host"), 335 ) 336 .expect_err("release revision required") 337 .kind(), 338 MycStatusErrorKind::InvalidBuildInfo 339 ); 340 assert_eq!( 341 MycStatusConfigurationIdentityV1::new( 342 "A".repeat(64), 343 MycStatusConfigurationSource::ExplicitConfig, 344 ) 345 .expect_err("lowercase digest required") 346 .kind(), 347 MycStatusErrorKind::InvalidConfiguration 348 ); 349 assert_eq!( 350 MycPersistenceStatusV1::new( 351 MycPersistenceHealthV1::Ready, 352 0, 353 0, 354 MycIntegrityStateV1::Verified, 355 MycStatusReasonCodes::empty(), 356 ) 357 .expect_err("positive schema required") 358 .kind(), 359 MycStatusErrorKind::InvalidPersistence 360 ); 361 362 let error = MycStatusCommonV1::new( 363 MycServicePhase::Ready, 364 false, 365 MycStatusReasonCodes::empty(), 366 0, 367 build_info(), 368 MycStatusConfigurationIdentityV1::new( 369 "a".repeat(64), 370 MycStatusConfigurationSource::ExplicitConfig, 371 ) 372 .expect("configuration"), 373 MycPersistenceStatusV1::new( 374 MycPersistenceHealthV1::Ready, 375 9, 376 0, 377 MycIntegrityStateV1::Verified, 378 MycStatusReasonCodes::empty(), 379 ) 380 .expect("persistence"), 381 ) 382 .expect_err("ready phase requires readiness"); 383 assert_eq!(error.kind(), MycStatusErrorKind::InvalidLifecycle); 384 } 385 386 #[test] 387 fn provider_aggregate_and_optional_oldest_time_are_deterministic() { 388 let ready = MycProviderStatusV1::new( 389 identity(true, true, &[]), 390 identity(true, true, &[]), 391 identity(false, false, &[]), 392 MycStatusReasonCodes::empty(), 393 ) 394 .expect("ready provider"); 395 assert_eq!(ready.health(), myc::MycProviderHealthV1::Ready); 396 397 let degraded = MycProviderStatusV1::new( 398 identity(true, true, &[]), 399 identity(true, true, &[]), 400 identity(true, false, &["identity_unavailable"]), 401 reasons(&["identity_unavailable"]), 402 ) 403 .expect("degraded provider"); 404 assert_eq!(degraded.health(), myc::MycProviderHealthV1::Degraded); 405 406 let unavailable = MycProviderStatusV1::new( 407 identity(true, false, &["identity_unavailable"]), 408 identity(true, true, &[]), 409 identity(false, false, &[]), 410 reasons(&["identity_unavailable"]), 411 ) 412 .expect("unavailable provider"); 413 assert_eq!(unavailable.health(), myc::MycProviderHealthV1::Unavailable); 414 415 let (_publisher, reader) = myc_status_cache( 416 InstanceId::new("primary").expect("instance"), 417 MycStatusObservationV1::new( 418 MycStatusCommonV1::new( 419 MycServicePhase::Ready, 420 true, 421 MycStatusReasonCodes::empty(), 422 1, 423 build_info(), 424 MycStatusConfigurationIdentityV1::new( 425 "a".repeat(64), 426 MycStatusConfigurationSource::ExplicitConfig, 427 ) 428 .expect("configuration"), 429 MycPersistenceStatusV1::new( 430 MycPersistenceHealthV1::Ready, 431 9, 432 0, 433 MycIntegrityStateV1::Verified, 434 MycStatusReasonCodes::empty(), 435 ) 436 .expect("persistence"), 437 ) 438 .expect("common status"), 439 ready, 440 MycRelayTransportStatusV1::new( 441 MycTransportHealthV1::Ready, 442 true, 443 0, 444 MycStatusReasonCodes::empty(), 445 ) 446 .expect("transport"), 447 MycConnectionCountsV1::default(), 448 MycOutboxStatusV1::new(0, 0, None), 449 ), 450 ) 451 .expect("cache"); 452 let wire = std::str::from_utf8(reader.snapshot().detailed_status_json()) 453 .expect("status UTF-8") 454 .to_owned(); 455 assert!(wire.contains("\"outbox\":{\"pending\":0,\"unknown\":0}")); 456 assert!(!wire.contains("oldest_pending_at_utc")); 457 }