services_hardening_state_resilience.rs (15908B)
1 #![forbid(unsafe_code)] 2 #![cfg(any(target_os = "linux", target_os = "macos"))] 3 4 use std::{ 5 error::Error, 6 fs, 7 num::NonZeroU64, 8 os::unix::fs::{MetadataExt, PermissionsExt}, 9 path::{Path, PathBuf}, 10 time::Duration, 11 }; 12 13 use radroots_service_sqlite::{ 14 BackupCreatedAtUnixMs, IntegrityCheckOutcome, IntegrityCheckedAtUnixMs, 15 MigrationAppliedAtUnixSeconds, MigrationBuildIdentity, 16 }; 17 use radroots_storage::event::SourceGeneration; 18 use rhi::{ 19 RHI_STATE_SCHEMA_VERSION, RadrootsHostEnvironment, RadrootsPathResolver, RadrootsPlatform, 20 RhiConfigProfile, RhiStateHostErrorKind, RhiStateMaintenanceErrorKind, RhiStateMetadata, 21 RhiStateRepositoryKind, finalize_rhi_state_restore, initialize_rhi_state, 22 open_rhi_state_inspection, open_rhi_state_read_write, parse_rhi_cli_v1_from, 23 parse_rhi_config_v1, resolve_rhi_runtime_context, stage_rhi_state_restore, 24 verify_rhi_state_backup, 25 }; 26 use sqlx::{ConnectOptions, Connection, SqliteConnection, sqlite::SqliteConnectOptions}; 27 28 const CONFIG_EXAMPLE: &[u8] = 29 include_bytes!("../contracts/services_hardening/config.v1.example.toml"); 30 const LIB_SOURCE: &str = include_str!("../src/lib.rs"); 31 const HOST_SOURCE: &str = include_str!("../src/state_host.rs"); 32 const MAINTENANCE_SOURCE: &str = include_str!("../src/state_maintenance.rs"); 33 34 fn runtime(root: &Path, instance: &str) -> rhi::RhiRuntimeContext { 35 let invocation = parse_rhi_cli_v1_from([ 36 "rhi", 37 "--profile", 38 "repo-local", 39 "--instance", 40 instance, 41 "--repo-local-root", 42 root.to_str().expect("UTF-8 temporary root"), 43 "run", 44 ]) 45 .expect("valid invocation"); 46 resolve_rhi_runtime_context( 47 &RadrootsPathResolver::new(RadrootsPlatform::Linux, RadrootsHostEnvironment::default()), 48 &invocation, 49 ) 50 .expect("runtime context") 51 } 52 53 fn prepare_state_directory(runtime: &rhi::RhiRuntimeContext) { 54 let directory = runtime.context().paths().state(); 55 fs::create_dir_all(directory).expect("state directory"); 56 fs::set_permissions(directory, fs::Permissions::from_mode(0o700)).expect("state mode"); 57 } 58 59 fn metadata(runtime: &rhi::RhiRuntimeContext) -> RhiStateMetadata { 60 let configuration = 61 parse_rhi_config_v1(CONFIG_EXAMPLE, RhiConfigProfile::RepoLocal).expect("configuration"); 62 RhiStateMetadata::new( 63 runtime, 64 &configuration, 65 SourceGeneration::new([0x5a; 32]).expect("generation"), 66 1_725_000_000_000, 67 ) 68 .expect("metadata") 69 } 70 71 fn migration_evidence() -> (MigrationAppliedAtUnixSeconds, MigrationBuildIdentity) { 72 let applied_at = MigrationAppliedAtUnixSeconds::new(1_725_000_000).expect("migration time"); 73 let build = MigrationBuildIdentity::new( 74 env!("CARGO_PKG_VERSION"), 75 "1111111111111111111111111111111111111111", 76 "053d0c750bf9cd683c6ea37cefe7e79617ba629f", 77 "rustc-test", 78 "test-target", 79 "service-host", 80 1, 81 RHI_STATE_SCHEMA_VERSION, 82 1, 83 1, 84 1, 85 ) 86 .expect("build identity"); 87 (applied_at, build) 88 } 89 90 fn recovery_paths(runtime: &rhi::RhiRuntimeContext) -> [PathBuf; 4] { 91 let state = runtime.context().paths().state(); 92 [ 93 state.join("state.restore-staged.sqlite"), 94 state.join("state.restore-backup.sqlite"), 95 state.join("state.restore-marker.v1"), 96 state.join("state.restore-marker.v1.next"), 97 ] 98 } 99 100 fn directory_inventory(directory: &Path) -> Vec<String> { 101 let mut entries = fs::read_dir(directory) 102 .expect("state directory") 103 .map(|entry| { 104 entry 105 .expect("state entry") 106 .file_name() 107 .to_string_lossy() 108 .into_owned() 109 }) 110 .collect::<Vec<_>>(); 111 entries.sort(); 112 entries 113 } 114 115 #[tokio::test] 116 async fn backup_integrity_and_offline_restore_obey_one_exact_rhi_authority() { 117 let directory = tempfile::tempdir().expect("temporary root"); 118 let runtime = runtime(directory.path(), "primary"); 119 prepare_state_directory(&runtime); 120 let metadata = metadata(&runtime); 121 let (applied_at, build) = migration_evidence(); 122 initialize_rhi_state(&runtime, &metadata, applied_at, &build) 123 .await 124 .expect("initialization"); 125 126 let writer = open_rhi_state_read_write(&runtime, &metadata, applied_at, &build) 127 .await 128 .expect("writable host"); 129 let cancelled_bundle = directory.path().join("cancelled-backup"); 130 let cancelled = tokio::time::timeout( 131 Duration::from_nanos(1), 132 writer.capture_online_backup( 133 &cancelled_bundle, 134 BackupCreatedAtUnixMs::new(1_725_000_000_050).expect("capture time"), 135 ), 136 ) 137 .await; 138 assert!(cancelled.is_err(), "capture future must be cancellable"); 139 writer 140 .close() 141 .await 142 .expect("close drains cancelled capture cleanup"); 143 assert!(!cancelled_bundle.exists()); 144 145 let writer = open_rhi_state_read_write(&runtime, &metadata, applied_at, &build) 146 .await 147 .expect("writer reacquisition after cancelled capture"); 148 let cancelled_integrity = tokio::time::timeout( 149 Duration::from_nanos(1), 150 writer.inspect_integrity( 151 IntegrityCheckedAtUnixMs::new(1_725_000_000_099).expect("inspection time"), 152 ), 153 ) 154 .await; 155 assert!( 156 cancelled_integrity.is_err(), 157 "integrity future must be cancellable" 158 ); 159 let report = writer 160 .inspect_integrity( 161 IntegrityCheckedAtUnixMs::new(1_725_000_000_100).expect("inspection time"), 162 ) 163 .await 164 .expect("writable integrity inspection"); 165 assert_eq!(report.sqlite(), IntegrityCheckOutcome::Verified); 166 assert_eq!(report.foreign_keys(), IntegrityCheckOutcome::Verified); 167 assert!(report.diagnostics().is_empty()); 168 169 let bundle = directory.path().join("backup"); 170 let manifest = writer 171 .capture_online_backup( 172 &bundle, 173 BackupCreatedAtUnixMs::new(1_725_000_000_200).expect("capture time"), 174 ) 175 .await 176 .expect("online backup"); 177 assert_eq!(manifest.service().as_str(), "rhi"); 178 assert_eq!(manifest.instance().as_str(), "primary"); 179 assert_eq!( 180 manifest.state_schema_version().get(), 181 RHI_STATE_SCHEMA_VERSION 182 ); 183 assert!(!manifest.protected_material_included()); 184 assert_eq!(manifest.members().len(), 1); 185 assert_eq!(manifest.members()[0].name(), "state.sqlite"); 186 let entries = fs::read_dir(&bundle) 187 .expect("backup directory") 188 .map(|entry| entry.expect("entry").file_name()) 189 .collect::<Vec<_>>(); 190 assert_eq!(entries, ["state.sqlite"]); 191 let manifest_bytes = manifest.canonical_bytes().to_vec(); 192 let manifest_digest = manifest.digest(); 193 let maximum_state_bytes = 194 NonZeroU64::new(manifest.members()[0].byte_length()).expect("member length"); 195 writer.close().await.expect("writer close"); 196 197 let live_path = runtime.artifacts().state_database(); 198 let old_live_inode = fs::metadata(live_path).expect("live metadata").ino(); 199 let state_directory = runtime.context().paths().state(); 200 let live_bytes_before_inspection = fs::read(live_path).expect("live bytes"); 201 let live_modified_before_inspection = fs::metadata(live_path) 202 .expect("live metadata") 203 .modified() 204 .expect("live modified time"); 205 let inventory_before_inspection = directory_inventory(state_directory); 206 let inspection = open_rhi_state_inspection(&runtime, &metadata) 207 .await 208 .expect("read-only inspection"); 209 let inspection_report = inspection 210 .inspect_integrity( 211 IntegrityCheckedAtUnixMs::new(1_725_000_000_300).expect("inspection time"), 212 ) 213 .await 214 .expect("read-only integrity inspection"); 215 assert_eq!(inspection_report.sqlite(), IntegrityCheckOutcome::Verified); 216 assert_eq!( 217 inspection_report.foreign_keys(), 218 IntegrityCheckOutcome::Verified 219 ); 220 let forbidden_bundle = directory.path().join("inspection-backup"); 221 let error = inspection 222 .capture_online_backup( 223 &forbidden_bundle, 224 BackupCreatedAtUnixMs::new(1_725_000_000_400).expect("capture time"), 225 ) 226 .await 227 .expect_err("read-only capture"); 228 assert_eq!(error.kind(), RhiStateMaintenanceErrorKind::InvalidMode); 229 assert!(!forbidden_bundle.exists()); 230 231 let verified = verify_rhi_state_backup( 232 &manifest_bytes, 233 manifest_digest, 234 &bundle, 235 &metadata, 236 maximum_state_bytes, 237 ) 238 .expect("verified retained backup"); 239 let contended = stage_rhi_state_restore(&runtime, &metadata, verified) 240 .await 241 .expect_err("offline staging rejects a live inspection host"); 242 assert_eq!(contended.kind(), RhiStateMaintenanceErrorKind::Authority); 243 inspection.close().await.expect("inspection close"); 244 assert_eq!( 245 fs::read(live_path).expect("live bytes after inspection"), 246 live_bytes_before_inspection 247 ); 248 assert_eq!( 249 fs::metadata(live_path) 250 .expect("live metadata after inspection") 251 .modified() 252 .expect("live modified time after inspection"), 253 live_modified_before_inspection 254 ); 255 assert_eq!( 256 directory_inventory(state_directory), 257 inventory_before_inspection 258 ); 259 260 let verified = verify_rhi_state_backup( 261 &manifest_bytes, 262 manifest_digest, 263 &bundle, 264 &metadata, 265 maximum_state_bytes, 266 ) 267 .expect("reverified backup for runtime mismatch"); 268 let secondary = self::runtime(directory.path(), "secondary"); 269 let mismatch = stage_rhi_state_restore(&secondary, &metadata, verified) 270 .await 271 .expect_err("runtime and metadata remain cross-bound"); 272 assert_eq!( 273 mismatch.kind(), 274 RhiStateMaintenanceErrorKind::InvalidEvidence 275 ); 276 assert!(!secondary.artifacts().state_database().exists()); 277 assert!(recovery_paths(&secondary).iter().all(|path| !path.exists())); 278 279 let verified = verify_rhi_state_backup( 280 &manifest_bytes, 281 manifest_digest, 282 &bundle, 283 &metadata, 284 maximum_state_bytes, 285 ) 286 .expect("reverified backup"); 287 assert_eq!( 288 format!("{verified:?}"), 289 "RhiVerifiedStateBackup([redacted])" 290 ); 291 assert_eq!( 292 verified.database_metadata().state_schema_version().get(), 293 RHI_STATE_SCHEMA_VERSION 294 ); 295 let staged = stage_rhi_state_restore(&runtime, &metadata, verified) 296 .await 297 .expect("offline staging"); 298 assert_eq!(format!("{staged:?}"), "RhiStagedStateRestore([redacted])"); 299 finalize_rhi_state_restore(staged) 300 .await 301 .expect("atomic finalization"); 302 303 let unavailable = open_rhi_state_inspection(&runtime, &metadata) 304 .await 305 .expect_err("inspection never performs restore recovery"); 306 assert_eq!(unavailable.kind(), RhiStateHostErrorKind::InspectionOpen); 307 let recovered = open_rhi_state_read_write(&runtime, &metadata, applied_at, &build) 308 .await 309 .expect("writable open reconciles exact recovery evidence"); 310 assert_eq!( 311 recovered.repositories().sources().kind(), 312 RhiStateRepositoryKind::Source 313 ); 314 recovered.close().await.expect("recovered writer close"); 315 assert_ne!( 316 fs::metadata(live_path) 317 .expect("recovered live metadata") 318 .ino(), 319 old_live_inode 320 ); 321 for path in recovery_paths(&runtime) { 322 assert!(!path.exists(), "recovery evidence must be retired"); 323 } 324 } 325 326 #[tokio::test] 327 async fn exact_open_rejects_unexpected_migration_history_without_repair() { 328 let directory = tempfile::tempdir().expect("temporary root"); 329 let runtime = runtime(directory.path(), "primary"); 330 prepare_state_directory(&runtime); 331 let metadata = metadata(&runtime); 332 let (applied_at, build) = migration_evidence(); 333 initialize_rhi_state(&runtime, &metadata, applied_at, &build) 334 .await 335 .expect("initialization"); 336 337 let options = SqliteConnectOptions::new() 338 .filename(runtime.artifacts().state_database()) 339 .create_if_missing(false) 340 .disable_statement_logging(); 341 let mut connection = SqliteConnection::connect_with(&options) 342 .await 343 .expect("test-only offline connection"); 344 sqlx::query( 345 "INSERT INTO schema_migrations ( 346 version, name, checksum, applied_at_unix_s, 347 service_version, service_commit, lib_revision, rust_version, target, 348 feature_profile, config_contract_version, state_contract_version, 349 admin_contract_version, status_contract_version, provider_contract_version 350 ) VALUES (12, 'unexpected_schema', ?, 1725000000, '0.1.0', ?, ?, 351 'rustc-test', 'test-target', 'service-host', 1, 11, 1, 1, 1)", 352 ) 353 .bind([0x44_u8; 32].as_slice()) 354 .bind("1111111111111111111111111111111111111111") 355 .bind("053d0c750bf9cd683c6ea37cefe7e79617ba629f") 356 .execute(&mut connection) 357 .await 358 .expect("insert unexpected ledger row"); 359 connection.close().await.expect("test connection close"); 360 361 let error = open_rhi_state_read_write(&runtime, &metadata, applied_at, &build) 362 .await 363 .expect_err("migration drift must fail closed"); 364 assert_eq!(error.kind(), RhiStateHostErrorKind::ReadWriteOpen); 365 let inspection = open_rhi_state_inspection(&runtime, &metadata) 366 .await 367 .expect_err("inspection rejects migration drift"); 368 assert_eq!(inspection.kind(), RhiStateHostErrorKind::InspectionOpen); 369 } 370 371 #[test] 372 fn maintenance_boundary_is_sealed_source_free_and_sqlx_owned() { 373 assert!(LIB_SOURCE.contains("mod state_maintenance;")); 374 assert!(!LIB_SOURCE.contains("pub mod state_maintenance;")); 375 assert!(HOST_SOURCE.contains(".capture_online_backup(staging_directory, created_at)")); 376 assert!(HOST_SOURCE.contains(".inspect_integrity(checked_at)")); 377 assert!(MAINTENANCE_SOURCE.contains("verify_backup_bundle(")); 378 assert!(MAINTENANCE_SOURCE.contains("stage_verified_restore(")); 379 assert!(MAINTENANCE_SOURCE.contains("finalize_staged_restore(")); 380 for forbidden in [ 381 "sqlx::", 382 "SqliteConnection", 383 "SqlitePool", 384 "raw_sql", 385 "BEGIN ", 386 "COMMIT", 387 "ROLLBACK", 388 "std::fs", 389 "std::env", 390 "std::time", 391 "provider", 392 "relay", 393 "tokio::spawn", 394 "spawn_blocking", 395 ] { 396 assert!( 397 !MAINTENANCE_SOURCE.contains(forbidden), 398 "found forbidden maintenance authority `{forbidden}`" 399 ); 400 } 401 402 for kind in [ 403 RhiStateMaintenanceErrorKind::InvalidEvidence, 404 RhiStateMaintenanceErrorKind::InvalidMode, 405 RhiStateMaintenanceErrorKind::Catalog, 406 RhiStateMaintenanceErrorKind::Authority, 407 RhiStateMaintenanceErrorKind::Open, 408 RhiStateMaintenanceErrorKind::Metadata, 409 RhiStateMaintenanceErrorKind::Migration, 410 RhiStateMaintenanceErrorKind::Backup, 411 RhiStateMaintenanceErrorKind::Restore, 412 RhiStateMaintenanceErrorKind::Integrity, 413 RhiStateMaintenanceErrorKind::Recovery, 414 ] { 415 assert!(!kind.code().is_empty()); 416 } 417 418 let error = verify_rhi_state_backup( 419 b"/tmp/secret-state.sqlite", 420 radroots_service_sqlite::BackupManifestSha256::from_bytes([0x11; 32]), 421 Path::new("/tmp/secret-bundle"), 422 &metadata(&runtime(Path::new("/tmp/secret-root"), "primary")), 423 NonZeroU64::new(1).expect("limit"), 424 ) 425 .expect_err("invalid manifest"); 426 assert!(Error::source(&error).is_none()); 427 let rendered = format!("{error} {error:?}"); 428 assert!(!rendered.contains("secret")); 429 assert!(!rendered.contains("/tmp")); 430 assert!(!rendered.contains("sqlite")); 431 }