services_hardening_state_resilience.rs (15145B)
1 #![forbid(unsafe_code)] 2 #![cfg(any(target_os = "linux", target_os = "macos"))] 3 4 use std::{ 5 error::Error, 6 fs, 7 num::NonZeroU64, 8 os::unix::fs::PermissionsExt, 9 path::{Path, PathBuf}, 10 time::Duration, 11 }; 12 13 use myc::{ 14 MYC_STATE_SCHEMA_VERSION, MycConfigProfile, MycStateHostErrorKind, 15 MycStateMaintenanceErrorKind, MycStateMetadata, RadrootsHostEnvironment, RadrootsPathResolver, 16 RadrootsPlatform, finalize_myc_state_restore, initialize_myc_state, open_myc_state_inspection, 17 open_myc_state_read_write, parse_myc_cli_v1_from, parse_myc_config_v1, 18 resolve_myc_runtime_context, stage_myc_state_restore, verify_myc_state_backup, 19 }; 20 use radroots_service_sqlite::{ 21 BackupCreatedAtUnixMs, IntegrityCheckOutcome, IntegrityCheckedAtUnixMs, 22 MigrationAppliedAtUnixSeconds, MigrationBuildIdentity, 23 }; 24 use radroots_storage::event::SourceGeneration; 25 use sqlx::{ConnectOptions, Connection, SqliteConnection, sqlite::SqliteConnectOptions}; 26 27 const CONFIG_EXAMPLE: &[u8] = 28 include_bytes!("../contracts/services_hardening/config.v1.example.toml"); 29 const LIB_SOURCE: &str = include_str!("../src/lib.rs"); 30 const HOST_SOURCE: &str = include_str!("../src/state_host.rs"); 31 const MAINTENANCE_SOURCE: &str = include_str!("../src/state_maintenance.rs"); 32 33 fn runtime(root: &Path, instance: &str) -> myc::MycRuntimeContext { 34 let invocation = parse_myc_cli_v1_from([ 35 "myc", 36 "--profile", 37 "repo-local", 38 "--instance", 39 instance, 40 "--repo-local-root", 41 root.to_str().expect("UTF-8 temporary root"), 42 "run", 43 ]) 44 .expect("valid invocation"); 45 resolve_myc_runtime_context( 46 &RadrootsPathResolver::new(RadrootsPlatform::Linux, RadrootsHostEnvironment::default()), 47 &invocation, 48 ) 49 .expect("runtime context") 50 } 51 52 fn prepare_state_directory(runtime: &myc::MycRuntimeContext) { 53 let directory = runtime.context().paths().state(); 54 fs::create_dir_all(directory).expect("state directory"); 55 fs::set_permissions(directory, fs::Permissions::from_mode(0o700)).expect("state mode"); 56 } 57 58 fn metadata(runtime: &myc::MycRuntimeContext) -> MycStateMetadata { 59 let configuration = 60 parse_myc_config_v1(CONFIG_EXAMPLE, MycConfigProfile::RepoLocal).expect("configuration"); 61 MycStateMetadata::new( 62 runtime, 63 &configuration, 64 SourceGeneration::new([0x5a; 32]).expect("generation"), 65 1_725_000_000_000, 66 ) 67 .expect("metadata") 68 } 69 70 fn migration_evidence() -> (MigrationAppliedAtUnixSeconds, MigrationBuildIdentity) { 71 let applied_at = MigrationAppliedAtUnixSeconds::new(1_725_000_000).expect("migration time"); 72 let build = MigrationBuildIdentity::new( 73 env!("CARGO_PKG_VERSION"), 74 "1111111111111111111111111111111111111111", 75 "053d0c750bf9cd683c6ea37cefe7e79617ba629f", 76 "rustc-test", 77 "test-target", 78 "service-host", 79 1, 80 myc::MYC_STATE_SCHEMA_VERSION, 81 1, 82 1, 83 1, 84 ) 85 .expect("build identity"); 86 (applied_at, build) 87 } 88 89 fn recovery_paths(runtime: &myc::MycRuntimeContext) -> [PathBuf; 4] { 90 let state = runtime.context().paths().state(); 91 [ 92 state.join("state.restore-staged.sqlite"), 93 state.join("state.restore-backup.sqlite"), 94 state.join("state.restore-marker.v1"), 95 state.join("state.restore-marker.v1.next"), 96 ] 97 } 98 99 fn directory_inventory(directory: &Path) -> Vec<String> { 100 let mut entries = fs::read_dir(directory) 101 .expect("state directory") 102 .map(|entry| { 103 entry 104 .expect("state entry") 105 .file_name() 106 .to_string_lossy() 107 .into_owned() 108 }) 109 .collect::<Vec<_>>(); 110 entries.sort(); 111 entries 112 } 113 114 #[tokio::test] 115 async fn backup_integrity_and_offline_restore_obey_one_exact_myc_authority() { 116 let directory = tempfile::tempdir().expect("temporary root"); 117 let runtime = runtime(directory.path(), "primary"); 118 prepare_state_directory(&runtime); 119 let metadata = metadata(&runtime); 120 let (applied_at, build) = migration_evidence(); 121 initialize_myc_state(&runtime, &metadata, applied_at, &build) 122 .await 123 .expect("initialization"); 124 125 let writer = open_myc_state_read_write(&runtime, &metadata, applied_at, &build) 126 .await 127 .expect("writable host"); 128 let cancelled_bundle = directory.path().join("cancelled-backup"); 129 let cancelled = tokio::time::timeout( 130 Duration::from_nanos(1), 131 writer.capture_online_backup( 132 &cancelled_bundle, 133 BackupCreatedAtUnixMs::new(1_725_000_000_100).expect("capture time"), 134 ), 135 ) 136 .await; 137 assert!(cancelled.is_err(), "capture future must be cancellable"); 138 writer 139 .close() 140 .await 141 .expect("close drains cancelled capture cleanup"); 142 assert!(!cancelled_bundle.exists()); 143 144 let writer = open_myc_state_read_write(&runtime, &metadata, applied_at, &build) 145 .await 146 .expect("writer reacquisition after cancelled capture"); 147 let cancelled_integrity = tokio::time::timeout( 148 Duration::from_nanos(1), 149 writer.inspect_integrity( 150 IntegrityCheckedAtUnixMs::new(1_725_000_000_200).expect("inspection time"), 151 ), 152 ) 153 .await; 154 assert!( 155 cancelled_integrity.is_err(), 156 "integrity future must be cancellable" 157 ); 158 let report = writer 159 .inspect_integrity( 160 IntegrityCheckedAtUnixMs::new(1_725_000_000_201).expect("retry inspection time"), 161 ) 162 .await 163 .expect("integrity retry after cancellation"); 164 assert_eq!(report.sqlite(), IntegrityCheckOutcome::Verified); 165 assert_eq!(report.foreign_keys(), IntegrityCheckOutcome::Verified); 166 assert!(report.diagnostics().is_empty()); 167 168 let bundle = directory.path().join("backup"); 169 let manifest = writer 170 .capture_online_backup( 171 &bundle, 172 BackupCreatedAtUnixMs::new(1_725_000_000_300).expect("capture time"), 173 ) 174 .await 175 .expect("online backup"); 176 assert_eq!(manifest.service().as_str(), "myc"); 177 assert_eq!(manifest.instance().as_str(), "primary"); 178 assert_eq!( 179 manifest.state_schema_version().get(), 180 MYC_STATE_SCHEMA_VERSION 181 ); 182 assert!(!manifest.protected_material_included()); 183 let members = fs::read_dir(&bundle) 184 .expect("backup directory") 185 .map(|entry| entry.expect("entry").file_name()) 186 .collect::<Vec<_>>(); 187 assert_eq!(members, ["state.sqlite"]); 188 let manifest_bytes = manifest.canonical_bytes().to_vec(); 189 let manifest_digest = manifest.digest(); 190 let maximum_state_bytes = 191 NonZeroU64::new(manifest.members()[0].byte_length()).expect("nonzero captured member"); 192 writer.close().await.expect("writer close"); 193 194 let live_path = runtime.artifacts().state_database(); 195 let state_directory = runtime.context().paths().state(); 196 let live_bytes_before_inspection = fs::read(live_path).expect("live bytes"); 197 let live_modified_before_inspection = fs::metadata(live_path) 198 .expect("live metadata") 199 .modified() 200 .expect("live modified time"); 201 let inventory_before_inspection = directory_inventory(state_directory); 202 let inspection = open_myc_state_inspection(&runtime, &metadata) 203 .await 204 .expect("read-only inspection"); 205 let inspection_report = inspection 206 .inspect_integrity( 207 IntegrityCheckedAtUnixMs::new(1_725_000_000_350).expect("inspection time"), 208 ) 209 .await 210 .expect("read-only integrity inspection"); 211 assert_eq!(inspection_report.sqlite(), IntegrityCheckOutcome::Verified); 212 assert_eq!( 213 inspection_report.foreign_keys(), 214 IntegrityCheckOutcome::Verified 215 ); 216 let forbidden_bundle = directory.path().join("inspection-backup"); 217 let error = inspection 218 .capture_online_backup( 219 &forbidden_bundle, 220 BackupCreatedAtUnixMs::new(1_725_000_000_400).expect("capture time"), 221 ) 222 .await 223 .expect_err("read-only capture"); 224 assert_eq!(error.kind(), MycStateMaintenanceErrorKind::InvalidMode); 225 assert!(!forbidden_bundle.exists()); 226 227 let verified = verify_myc_state_backup( 228 &manifest_bytes, 229 manifest_digest, 230 &bundle, 231 &metadata, 232 maximum_state_bytes, 233 ) 234 .expect("verified retained backup"); 235 let contended = stage_myc_state_restore(&runtime, &metadata, verified) 236 .await 237 .expect_err("offline staging must reject a live inspection host"); 238 assert_eq!(contended.kind(), MycStateMaintenanceErrorKind::Authority); 239 inspection.close().await.expect("inspection close"); 240 assert_eq!( 241 fs::read(live_path).expect("live bytes after inspection"), 242 live_bytes_before_inspection 243 ); 244 assert_eq!( 245 fs::metadata(live_path) 246 .expect("live metadata after inspection") 247 .modified() 248 .expect("live modified time after inspection"), 249 live_modified_before_inspection 250 ); 251 assert_eq!( 252 directory_inventory(state_directory), 253 inventory_before_inspection 254 ); 255 256 let verified = verify_myc_state_backup( 257 &manifest_bytes, 258 manifest_digest, 259 &bundle, 260 &metadata, 261 maximum_state_bytes, 262 ) 263 .expect("reverified backup for runtime mismatch"); 264 let secondary = self::runtime(directory.path(), "secondary"); 265 let mismatch = stage_myc_state_restore(&secondary, &metadata, verified) 266 .await 267 .expect_err("runtime and metadata must remain cross-bound"); 268 assert_eq!( 269 mismatch.kind(), 270 MycStateMaintenanceErrorKind::InvalidEvidence 271 ); 272 assert!(!secondary.artifacts().state_database().exists()); 273 assert!(recovery_paths(&secondary).iter().all(|path| !path.exists())); 274 275 let verified = verify_myc_state_backup( 276 &manifest_bytes, 277 manifest_digest, 278 &bundle, 279 &metadata, 280 maximum_state_bytes, 281 ) 282 .expect("reverified backup"); 283 assert_eq!( 284 format!("{verified:?}"), 285 "MycVerifiedStateBackup([redacted])" 286 ); 287 assert_eq!( 288 verified.database_metadata().state_schema_version().get(), 289 MYC_STATE_SCHEMA_VERSION 290 ); 291 let staged = stage_myc_state_restore(&runtime, &metadata, verified) 292 .await 293 .expect("offline staging"); 294 assert_eq!(format!("{staged:?}"), "MycStagedStateRestore([redacted])"); 295 finalize_myc_state_restore(staged) 296 .await 297 .expect("atomic finalization"); 298 299 let unavailable = open_myc_state_inspection(&runtime, &metadata) 300 .await 301 .expect_err("inspection never performs restore recovery"); 302 assert_eq!(unavailable.kind(), MycStateHostErrorKind::InspectionOpen); 303 let recovered = open_myc_state_read_write(&runtime, &metadata, applied_at, &build) 304 .await 305 .expect("writable open reconciles exact recovery evidence"); 306 recovered 307 .repository() 308 .verify_binding() 309 .await 310 .expect("restored Myc binding"); 311 recovered.close().await.expect("recovered writer close"); 312 for path in recovery_paths(&runtime) { 313 assert!(!path.exists(), "recovery evidence must be retired"); 314 } 315 } 316 317 #[tokio::test] 318 async fn exact_open_rejects_migration_history_drift_without_repair() { 319 let directory = tempfile::tempdir().expect("temporary root"); 320 let runtime = runtime(directory.path(), "primary"); 321 prepare_state_directory(&runtime); 322 let metadata = metadata(&runtime); 323 let (applied_at, build) = migration_evidence(); 324 initialize_myc_state(&runtime, &metadata, applied_at, &build) 325 .await 326 .expect("initialization"); 327 328 let options = SqliteConnectOptions::new() 329 .filename(runtime.artifacts().state_database()) 330 .create_if_missing(false) 331 .disable_statement_logging(); 332 let mut connection = SqliteConnection::connect_with(&options) 333 .await 334 .expect("test-only offline connection"); 335 sqlx::query("DROP TRIGGER schema_migrations_no_delete") 336 .execute(&mut connection) 337 .await 338 .expect("remove immutable test guard"); 339 sqlx::query("DELETE FROM schema_migrations WHERE version = 2") 340 .execute(&mut connection) 341 .await 342 .expect("create invalid migration prefix"); 343 connection.close().await.expect("test connection close"); 344 345 let error = open_myc_state_read_write(&runtime, &metadata, applied_at, &build) 346 .await 347 .expect_err("migration drift must fail closed"); 348 assert_eq!(error.kind(), MycStateHostErrorKind::ReadWriteOpen); 349 let inspection = open_myc_state_inspection(&runtime, &metadata) 350 .await 351 .expect_err("inspection must reject migration drift"); 352 assert_eq!(inspection.kind(), MycStateHostErrorKind::InspectionOpen); 353 } 354 355 #[test] 356 fn maintenance_boundary_is_sealed_source_free_and_sqlx_owned() { 357 assert!(LIB_SOURCE.contains("mod state_maintenance;")); 358 assert!(!LIB_SOURCE.contains("pub mod state_maintenance;")); 359 assert!(HOST_SOURCE.contains(".capture_online_backup(staging_directory, created_at)")); 360 assert!(HOST_SOURCE.contains(".inspect_integrity(checked_at)")); 361 assert!(MAINTENANCE_SOURCE.contains("verify_backup_bundle(")); 362 assert!(MAINTENANCE_SOURCE.contains("stage_verified_restore(")); 363 assert!(MAINTENANCE_SOURCE.contains("finalize_staged_restore(")); 364 for forbidden in [ 365 "sqlx::", 366 "SqliteConnection", 367 "SqlitePool", 368 "raw_sql", 369 "BEGIN ", 370 "COMMIT", 371 "ROLLBACK", 372 "std::fs", 373 "std::env", 374 "std::time", 375 "provider", 376 "relay", 377 "tokio::spawn", 378 "spawn_blocking", 379 ] { 380 assert!( 381 !MAINTENANCE_SOURCE.contains(forbidden), 382 "found forbidden maintenance authority `{forbidden}`" 383 ); 384 } 385 386 for kind in [ 387 MycStateMaintenanceErrorKind::InvalidEvidence, 388 MycStateMaintenanceErrorKind::InvalidMode, 389 MycStateMaintenanceErrorKind::Catalog, 390 MycStateMaintenanceErrorKind::Authority, 391 MycStateMaintenanceErrorKind::Open, 392 MycStateMaintenanceErrorKind::Metadata, 393 MycStateMaintenanceErrorKind::Migration, 394 MycStateMaintenanceErrorKind::Backup, 395 MycStateMaintenanceErrorKind::Restore, 396 MycStateMaintenanceErrorKind::Integrity, 397 MycStateMaintenanceErrorKind::Recovery, 398 ] { 399 assert!(!kind.code().is_empty()); 400 } 401 402 let error = verify_myc_state_backup( 403 b"/tmp/secret-state.sqlite", 404 radroots_service_sqlite::BackupManifestSha256::from_bytes([0x11; 32]), 405 Path::new("/tmp/secret-bundle"), 406 &metadata(&runtime(Path::new("/tmp/secret-root"), "primary")), 407 NonZeroU64::new(1).expect("limit"), 408 ) 409 .expect_err("invalid manifest"); 410 assert!(Error::source(&error).is_none()); 411 let rendered = format!("{error} {error:?}"); 412 assert!(!rendered.contains("secret")); 413 assert!(!rendered.contains("/tmp")); 414 assert!(!rendered.contains("sqlite")); 415 }