services_hardening_process.rs (28261B)
1 #![forbid(unsafe_code)] 2 #![cfg(any(target_os = "linux", target_os = "macos"))] 3 4 use std::{ 5 collections::BTreeSet, 6 fs, 7 io::{Read as _, Write as _}, 8 net::{TcpListener, TcpStream}, 9 os::unix::fs::PermissionsExt as _, 10 path::PathBuf, 11 process::{Child, Command, ExitStatus, Output, Stdio}, 12 sync::{ 13 Arc, 14 atomic::{AtomicBool, Ordering}, 15 }, 16 thread, 17 time::{Duration, Instant}, 18 }; 19 20 use nostr::{Keys, SecretKey}; 21 use rhi::{ 22 RadrootsHostEnvironment, RadrootsPathResolver, RadrootsPlatform, parse_rhi_cli_v1_from, 23 resolve_rhi_runtime_context, 24 }; 25 use sha2::{Digest as _, Sha256}; 26 use tungstenite::{Error as WebSocketError, Message, accept}; 27 28 const CONFIG_EXAMPLE: &str = include_str!("../contracts/services_hardening/config.v1.example.toml"); 29 const PROCESS_QUALIFICATION_CONTRACT: &str = 30 include_str!("../contracts/services_hardening/process_qualification.v1.json"); 31 const FAILURE_QUALIFICATION_CONTRACT: &[u8] = 32 include_bytes!("../contracts/services_hardening/failure_qualification.v1.json"); 33 const SOURCE_LOCK: &str = include_str!("../radroots.service.source_lock.v4.toml"); 34 const PROCESS_DEADLINE: Duration = Duration::from_secs(30); 35 const POLL_INTERVAL: Duration = Duration::from_millis(2); 36 const CONNECT_DEADLINE_MILLISECONDS: u64 = 5_000; 37 const RELAY_HANDSHAKE_TIMEOUT: Duration = Duration::from_millis(5_000); 38 const RELAY_IO_TIMEOUT: Duration = Duration::from_millis(100); 39 const PARALLEL_INSPECTIONS: usize = 8; 40 const SOAK_ITERATIONS: usize = 32; 41 const MAXIMUM_STDOUT_BYTES: usize = 1_048_576; 42 const MAXIMUM_STDERR_BYTES: usize = 8_192; 43 #[cfg(target_os = "linux")] 44 const PROCESS_TEMPORARY_ROOT: &str = "/tmp"; 45 #[cfg(target_os = "macos")] 46 const PROCESS_TEMPORARY_ROOT: &str = "/private/tmp"; 47 48 struct RelayHarness { 49 address: std::net::SocketAddr, 50 stop: Arc<AtomicBool>, 51 thread: Option<thread::JoinHandle<()>>, 52 } 53 54 impl RelayHarness { 55 fn start() -> Self { 56 let listener = TcpListener::bind("127.0.0.1:0").expect("test relay listener"); 57 listener 58 .set_nonblocking(true) 59 .expect("nonblocking test relay"); 60 let address = listener.local_addr().expect("test relay address"); 61 let stop = Arc::new(AtomicBool::new(false)); 62 let thread_stop = Arc::clone(&stop); 63 let thread = thread::spawn(move || { 64 let mut sessions = Vec::new(); 65 while !thread_stop.load(Ordering::SeqCst) { 66 match listener.accept() { 67 Ok((stream, _)) => { 68 let session_stop = Arc::clone(&thread_stop); 69 sessions.push(thread::spawn(move || relay_session(stream, &session_stop))); 70 } 71 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { 72 thread::sleep(Duration::from_millis(2)); 73 } 74 Err(error) => panic!("test relay accept failed: {error}"), 75 } 76 } 77 for session in sessions { 78 session.join().expect("test relay session"); 79 } 80 }); 81 Self { 82 address, 83 stop, 84 thread: Some(thread), 85 } 86 } 87 88 fn url(&self, path: &str) -> String { 89 format!("ws://{}/{path}", self.address) 90 } 91 } 92 93 impl Drop for RelayHarness { 94 fn drop(&mut self) { 95 self.stop.store(true, Ordering::SeqCst); 96 let _ = TcpStream::connect(self.address); 97 if let Some(thread) = self.thread.take() { 98 thread.join().expect("test relay"); 99 } 100 } 101 } 102 103 fn relay_session(stream: TcpStream, stop: &AtomicBool) { 104 relay_session_when_ready(stream, stop, || true); 105 } 106 107 fn relay_session_when_ready( 108 stream: TcpStream, 109 stop: &AtomicBool, 110 handshake_ready: impl FnOnce() -> bool, 111 ) { 112 // Accepted sockets can inherit the listener's nonblocking mode on macOS. 113 // This fixture uses blocking tungstenite with explicit I/O deadlines. 114 stream 115 .set_nonblocking(false) 116 .expect("blocking accepted relay socket"); 117 stream 118 .set_read_timeout(Some(RELAY_HANDSHAKE_TIMEOUT)) 119 .expect("relay handshake read timeout"); 120 stream 121 .set_write_timeout(Some(RELAY_HANDSHAKE_TIMEOUT)) 122 .expect("relay handshake write timeout"); 123 if !handshake_ready() { 124 return; 125 } 126 let mut websocket = match accept(stream) { 127 Ok(websocket) => websocket, 128 Err(_) => return, 129 }; 130 websocket 131 .get_mut() 132 .set_read_timeout(Some(RELAY_IO_TIMEOUT)) 133 .expect("relay read timeout"); 134 websocket 135 .get_mut() 136 .set_write_timeout(Some(RELAY_IO_TIMEOUT)) 137 .expect("relay write timeout"); 138 while !stop.load(Ordering::SeqCst) { 139 let message = match websocket.read() { 140 Ok(message) => message, 141 Err(WebSocketError::Io(error)) 142 if matches!( 143 error.kind(), 144 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut 145 ) => 146 { 147 continue; 148 } 149 Err(WebSocketError::ConnectionClosed | WebSocketError::AlreadyClosed) => break, 150 Err(_) => break, 151 }; 152 match message { 153 Message::Text(text) => relay_text(&mut websocket, text.as_str()), 154 Message::Ping(bytes) => { 155 if websocket.send(Message::Pong(bytes)).is_err() { 156 break; 157 } 158 } 159 Message::Close(_) => break, 160 _ => {} 161 } 162 } 163 } 164 165 fn relay_text(websocket: &mut tungstenite::WebSocket<TcpStream>, text: &str) { 166 let Ok(message) = serde_json::from_str::<serde_json::Value>(text) else { 167 return; 168 }; 169 let Some(parts) = message.as_array() else { 170 return; 171 }; 172 match parts.first().and_then(serde_json::Value::as_str) { 173 Some("REQ") => { 174 let Some(subscription) = parts.get(1).and_then(serde_json::Value::as_str) else { 175 return; 176 }; 177 let response = serde_json::json!(["EOSE", subscription]).to_string(); 178 let _ = websocket.send(Message::Text(response.into())); 179 } 180 Some("EVENT") => { 181 let Some(event_id) = parts 182 .get(1) 183 .and_then(|event| event.get("id")) 184 .and_then(serde_json::Value::as_str) 185 else { 186 return; 187 }; 188 let response = serde_json::json!(["OK", event_id, true, ""]).to_string(); 189 let _ = websocket.send(Message::Text(response.into())); 190 } 191 _ => {} 192 } 193 } 194 195 #[test] 196 fn loopback_relay_waits_for_delayed_upgrade_on_an_inherited_nonblocking_socket() { 197 let listener = TcpListener::bind("127.0.0.1:0").expect("regression listener"); 198 let address = listener.local_addr().expect("regression address"); 199 let client = TcpStream::connect(address).expect("regression client"); 200 client 201 .set_read_timeout(Some(RELAY_HANDSHAKE_TIMEOUT)) 202 .expect("client read deadline"); 203 client 204 .set_write_timeout(Some(RELAY_HANDSHAKE_TIMEOUT)) 205 .expect("client write deadline"); 206 let (server, _) = listener.accept().expect("accepted regression socket"); 207 // Exercise inherited nonblocking mode explicitly on every platform. 208 server.set_nonblocking(true).expect("inherited socket mode"); 209 let observed_server = server.try_clone().expect("observe accepted socket mode"); 210 let stop = Arc::new(AtomicBool::new(false)); 211 let session_stop = Arc::clone(&stop); 212 let (ready, configured) = std::sync::mpsc::sync_channel(0); 213 let (resume, released) = std::sync::mpsc::sync_channel(0); 214 let session = thread::spawn(move || { 215 relay_session_when_ready(server, &session_stop, || { 216 ready.send(()).is_ok() && released.recv_timeout(PROCESS_DEADLINE).is_ok() 217 }); 218 }); 219 let response = (|| -> Result<Message, String> { 220 configured 221 .recv_timeout(PROCESS_DEADLINE) 222 .map_err(|error| error.to_string())?; 223 // Observe the actual shared socket flags before sending upgrade bytes. 224 // A timed absence of completion is not evidence under scheduler load. 225 let flags = rustix::fs::fcntl_getfl(&observed_server).map_err(|error| error.to_string())?; 226 if flags.contains(rustix::fs::OFlags::NONBLOCK) { 227 return Err("relay retained inherited nonblocking mode".to_owned()); 228 } 229 resume.send(()).map_err(|error| error.to_string())?; 230 let (mut socket, _) = tungstenite::client(format!("ws://{address}/delayed"), client) 231 .map_err(|error| error.to_string())?; 232 socket 233 .send(Message::Text(r#"["REQ","delayed",{}]"#.into())) 234 .map_err(|error| error.to_string())?; 235 socket.read().map_err(|error| error.to_string()) 236 })(); 237 stop.store(true, Ordering::SeqCst); 238 drop(resume); 239 drop(configured); 240 session.join().expect("joined regression relay"); 241 assert_eq!( 242 response.expect("delayed upgrade and request succeed"), 243 Message::Text(r#"["EOSE","delayed"]"#.into()) 244 ); 245 } 246 247 struct ProcessFixture { 248 root: tempfile::TempDir, 249 config: PathBuf, 250 runtime: rhi::RhiRuntimeContext, 251 } 252 253 impl ProcessFixture { 254 fn new() -> Self { 255 let root = tempfile::Builder::new() 256 .prefix("rhi-") 257 .tempdir_in(PROCESS_TEMPORARY_ROOT) 258 .expect("short repo-local root"); 259 fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700)) 260 .expect("secure repo-local root"); 261 let config = root.path().join("rhi.toml"); 262 let invocation = parse_rhi_cli_v1_from([ 263 "rhi", 264 "--profile", 265 "repo-local", 266 "--instance", 267 "primary", 268 "--repo-local-root", 269 root.path().to_str().expect("UTF-8 test root"), 270 "--config", 271 config.to_str().expect("UTF-8 config path"), 272 "run", 273 ]) 274 .expect("runtime invocation"); 275 let runtime = resolve_rhi_runtime_context( 276 &RadrootsPathResolver::new( 277 RadrootsPlatform::current(), 278 RadrootsHostEnvironment::default(), 279 ), 280 &invocation, 281 ) 282 .expect("runtime context"); 283 Self { 284 root, 285 config, 286 runtime, 287 } 288 } 289 290 fn command(&self, command: &[&str]) -> Command { 291 let mut process = Command::new(env!("CARGO_BIN_EXE_rhi")); 292 process 293 .args(["--profile", "repo-local", "--instance", "primary"]) 294 .arg("--repo-local-root") 295 .arg(self.root.path()) 296 .arg("--config") 297 .arg(&self.config) 298 .args(command) 299 .stdin(Stdio::null()) 300 .stdout(Stdio::piped()) 301 .stderr(Stdio::piped()); 302 process 303 } 304 305 fn run(&self, command: &[&str]) -> Output { 306 BoundedProcess::spawn(self.command(command)).wait() 307 } 308 309 fn run_with_stdin(&self, command: &[&str], bytes: &[u8]) -> Output { 310 let mut process = self.command(command); 311 process.stdin(Stdio::piped()); 312 let mut child = BoundedProcess::spawn(process); 313 child 314 .child 315 .stdin 316 .take() 317 .expect("process stdin") 318 .write_all(bytes) 319 .expect("bounded stdin"); 320 child.wait() 321 } 322 } 323 324 struct BoundedProcess { 325 child: Child, 326 stdout_reader: Option<thread::JoinHandle<Vec<u8>>>, 327 stderr_reader: Option<thread::JoinHandle<Vec<u8>>>, 328 completed: bool, 329 } 330 331 impl BoundedProcess { 332 fn spawn(mut command: Command) -> Self { 333 let child = command.spawn().expect("RHI process"); 334 Self::from_child(child) 335 } 336 337 fn from_child(mut child: Child) -> Self { 338 let stdout = child.stdout.take().expect("captured process stdout"); 339 let stderr = child.stderr.take().expect("captured process stderr"); 340 Self { 341 child, 342 stdout_reader: Some(read_bounded(stdout, MAXIMUM_STDOUT_BYTES)), 343 stderr_reader: Some(read_bounded(stderr, MAXIMUM_STDERR_BYTES)), 344 completed: false, 345 } 346 } 347 348 fn id(&self) -> u32 { 349 self.child.id() 350 } 351 352 fn try_wait(&mut self) -> Option<ExitStatus> { 353 self.child.try_wait().expect("poll RHI process") 354 } 355 356 fn collect(&mut self, status: ExitStatus) -> Output { 357 self.completed = true; 358 let stdout = self 359 .stdout_reader 360 .take() 361 .expect("stdout reader available") 362 .join() 363 .expect("join stdout reader"); 364 let stderr = self 365 .stderr_reader 366 .take() 367 .expect("stderr reader available") 368 .join() 369 .expect("join stderr reader"); 370 assert!(stdout.len() <= MAXIMUM_STDOUT_BYTES); 371 assert!(stderr.len() <= MAXIMUM_STDERR_BYTES); 372 Output { 373 status, 374 stdout, 375 stderr, 376 } 377 } 378 379 fn wait(mut self) -> Output { 380 let deadline = Instant::now() + PROCESS_DEADLINE; 381 loop { 382 if let Some(status) = self.try_wait() { 383 return self.collect(status); 384 } 385 if Instant::now() >= deadline { 386 let _ = self.child.kill(); 387 let status = self.child.wait().expect("reap RHI process"); 388 let output = self.collect(status); 389 panic!("RHI process exceeded deadline: {:?}", output.stderr); 390 } 391 thread::sleep(POLL_INTERVAL); 392 } 393 } 394 } 395 396 impl Drop for BoundedProcess { 397 fn drop(&mut self) { 398 if !self.completed { 399 let _ = self.child.kill(); 400 let _ = self.child.wait(); 401 } 402 if let Some(reader) = self.stdout_reader.take() { 403 let _ = reader.join(); 404 } 405 if let Some(reader) = self.stderr_reader.take() { 406 let _ = reader.join(); 407 } 408 } 409 } 410 411 fn read_bounded( 412 reader: impl std::io::Read + Send + 'static, 413 maximum_bytes: usize, 414 ) -> thread::JoinHandle<Vec<u8>> { 415 thread::spawn(move || { 416 let limit = u64::try_from(maximum_bytes) 417 .expect("output bound fits u64") 418 .checked_add(1) 419 .expect("output bound plus sentinel fits u64"); 420 let mut output = Vec::with_capacity(maximum_bytes.min(8_192)); 421 reader 422 .take(limit) 423 .read_to_end(&mut output) 424 .expect("read bounded process output"); 425 output 426 }) 427 } 428 429 fn identity_secret() -> [u8; 32] { 430 let mut candidate: [u8; 32] = 431 Sha256::digest(b"radroots.rhi.step-213.process-identity.v1").into(); 432 while SecretKey::from_slice(&candidate).is_err() { 433 candidate = Sha256::digest(candidate).into(); 434 } 435 candidate 436 } 437 438 fn provisioning_document(secret: [u8; 32]) -> [u8; 117] { 439 let mut document = [0_u8; 117]; 440 document[..4].copy_from_slice(b"RHIP"); 441 document[4] = 1; 442 document[5..37].copy_from_slice(&secret); 443 document[37..69].copy_from_slice(&Sha256::digest(b"rhi-step-213-data-key")); 444 document[69..93].copy_from_slice(&[3; 24]); 445 document[93..117].copy_from_slice(&[4; 24]); 446 document 447 } 448 449 fn configuration( 450 fixture: &ProcessFixture, 451 expected_public_key: &str, 452 primary_relay: &str, 453 secondary_relay: &str, 454 ) -> String { 455 CONFIG_EXAMPLE 456 .replace( 457 "/var/lib/radroots/services/rhi/default/secrets/service.identity.ncrypt", 458 fixture 459 .runtime 460 .identity_path() 461 .to_str() 462 .expect("UTF-8 identity path"), 463 ) 464 .replace(&"2".repeat(64), expected_public_key) 465 .replace("wss://relay.example.com/", primary_relay) 466 .replace("wss://relay-secondary.example.com/", secondary_relay) 467 .replace( 468 "connect_deadline_ms = 10000", 469 &format!("connect_deadline_ms = {CONNECT_DEADLINE_MILLISECONDS}"), 470 ) 471 .replace("request_deadline_ms = 15000", "request_deadline_ms = 1000") 472 } 473 474 fn diagnostic_code(output: &Output) -> String { 475 let value: serde_json::Value = serde_json::from_slice(&output.stderr).expect("diagnostic JSON"); 476 value["code"].as_str().expect("diagnostic code").to_owned() 477 } 478 479 fn assert_success(output: &Output) { 480 assert_eq!(output.status.code(), Some(0), "stderr: {:?}", output.stderr); 481 assert_eq!(diagnostic_code(output), "success"); 482 } 483 484 fn bootstrap(fixture: &ProcessFixture, configuration: &str, secret: [u8; 32]) -> String { 485 let expected_public_key = Keys::new(SecretKey::from_slice(&secret).expect("secret")) 486 .public_key() 487 .to_hex(); 488 let config = fixture.run_with_stdin(&["config", "init"], configuration.as_bytes()); 489 assert_success(&config); 490 assert_eq!(config.stdout, b"config_initialized\n"); 491 492 for directory in [ 493 fixture.runtime.context().paths().state(), 494 fixture.runtime.context().paths().secrets(), 495 fixture.runtime.context().paths().run(), 496 ] { 497 fs::create_dir_all(directory).expect("secure runtime directory"); 498 fs::set_permissions(directory, fs::Permissions::from_mode(0o700)) 499 .expect("secure runtime mode"); 500 } 501 let credential = fixture 502 .runtime 503 .context() 504 .paths() 505 .secrets() 506 .join("service_wrapping_key"); 507 fs::write(&credential, Sha256::digest(b"rhi-step-213-wrapping-key")) 508 .expect("wrapping credential"); 509 fs::set_permissions(&credential, fs::Permissions::from_mode(0o600)).expect("credential mode"); 510 511 let state = fixture.run(&["state", "init"]); 512 assert_success(&state); 513 assert_eq!(state.stdout, b"state_initialized\n"); 514 let identity = fixture.run_with_stdin( 515 &["--output", "json", "identity", "init"], 516 &provisioning_document(secret), 517 ); 518 assert_success(&identity); 519 let identity_value: serde_json::Value = 520 serde_json::from_slice(&identity.stdout).expect("identity result"); 521 assert_eq!(identity_value["public_key"], expected_public_key); 522 523 for command in [ 524 &["config", "validate"][..], 525 &["state", "verify"][..], 526 &["state", "migrate"][..], 527 ] { 528 assert_success(&fixture.run(command)); 529 } 530 expected_public_key 531 } 532 533 fn wait_for_live_status(fixture: &ProcessFixture, daemon: &mut BoundedProcess) -> Output { 534 let deadline = Instant::now() + PROCESS_DEADLINE; 535 let mut last_diagnostic = Vec::new(); 536 loop { 537 if let Some(status) = daemon.try_wait() { 538 let output = daemon.collect(status); 539 panic!( 540 "RHI daemon exited before admin became ready: {status}; stderr={:?}", 541 output.stderr 542 ); 543 } 544 if fixture.runtime.artifacts().admin_socket().exists() { 545 let status = fixture.run(&["status"]); 546 if status.status.success() { 547 return status; 548 } 549 last_diagnostic = status.stderr; 550 } 551 if Instant::now() >= deadline { 552 panic!( 553 "RHI admin did not become ready before deadline; socket={}; stderr={last_diagnostic:?}", 554 fixture.runtime.artifacts().admin_socket().exists() 555 ); 556 } 557 thread::sleep(POLL_INTERVAL); 558 } 559 } 560 561 fn interrupt_and_wait(daemon: BoundedProcess) -> Output { 562 let signal = Command::new("kill") 563 .arg("-INT") 564 .arg(daemon.id().to_string()) 565 .status() 566 .expect("send interrupt"); 567 assert!(signal.success(), "interrupt command failed: {signal}"); 568 daemon.wait() 569 } 570 571 #[test] 572 fn process_qualification_contract_freezes_the_exact_wave_closure() { 573 let contract: serde_json::Value = 574 serde_json::from_str(PROCESS_QUALIFICATION_CONTRACT).expect("process contract"); 575 assert_eq!( 576 contract 577 .as_object() 578 .expect("process contract object") 579 .keys() 580 .map(String::as_str) 581 .collect::<BTreeSet<_>>(), 582 BTreeSet::from([ 583 "actual_process_corpus", 584 "binary", 585 "bounds", 586 "component_qualification", 587 "contract_version", 588 "deferred", 589 "invariants", 590 "schema", 591 "service", 592 "source_lock", 593 "step", 594 ]) 595 ); 596 assert_eq!(contract["schema"], "radroots.rhi.process-qualification.v1"); 597 assert_eq!(contract["contract_version"], 1); 598 assert_eq!(contract["step"], 215); 599 assert_eq!(contract["service"], "rhi"); 600 assert_eq!(contract["binary"], "rhi"); 601 assert_eq!( 602 contract["source_lock"], 603 serde_json::json!({ 604 "schema": "radroots.service.source-lock.v2", 605 "lib_revision": "053d0c750bf9cd683c6ea37cefe7e79617ba629f" 606 }) 607 ); 608 assert_eq!( 609 contract["component_qualification"], 610 serde_json::json!({ 611 "schema": "radroots.rhi.failure-qualification.v1", 612 "step": 214, 613 "sha256": "f05da8e559f463f99c67c3c7e22eafa57935be91fb0094f2aa2f98a58544b8b9" 614 }) 615 ); 616 assert_eq!( 617 contract["bounds"], 618 serde_json::json!({ 619 "process_deadline_ms": 30_000, 620 "poll_interval_ms": 2, 621 "connect_deadline_ms": 5_000, 622 "relay_handshake_timeout_ms": 5_000, 623 "relay_io_timeout_ms": 100, 624 "parallel_inspection_processes": 8, 625 "soak_iterations": 32, 626 "maximum_stdout_bytes": 1_048_576, 627 "maximum_stderr_bytes": 8_192 628 }) 629 ); 630 assert_eq!( 631 PROCESS_DEADLINE, 632 Duration::from_millis( 633 contract["bounds"]["process_deadline_ms"] 634 .as_u64() 635 .expect("process deadline"), 636 ) 637 ); 638 assert_eq!( 639 POLL_INTERVAL, 640 Duration::from_millis( 641 contract["bounds"]["poll_interval_ms"] 642 .as_u64() 643 .expect("poll interval"), 644 ) 645 ); 646 assert_eq!( 647 RELAY_HANDSHAKE_TIMEOUT, 648 Duration::from_millis( 649 contract["bounds"]["relay_handshake_timeout_ms"] 650 .as_u64() 651 .expect("relay handshake timeout"), 652 ) 653 ); 654 assert_eq!( 655 RELAY_IO_TIMEOUT, 656 Duration::from_millis( 657 contract["bounds"]["relay_io_timeout_ms"] 658 .as_u64() 659 .expect("relay timeout"), 660 ) 661 ); 662 assert_eq!( 663 CONNECT_DEADLINE_MILLISECONDS, 664 contract["bounds"]["connect_deadline_ms"] 665 ); 666 assert_eq!( 667 u64::try_from(PARALLEL_INSPECTIONS).expect("inspection bound"), 668 contract["bounds"]["parallel_inspection_processes"] 669 ); 670 assert_eq!( 671 u64::try_from(SOAK_ITERATIONS).expect("soak bound"), 672 contract["bounds"]["soak_iterations"] 673 ); 674 assert_eq!( 675 u64::try_from(MAXIMUM_STDOUT_BYTES).expect("stdout bound"), 676 contract["bounds"]["maximum_stdout_bytes"] 677 ); 678 assert_eq!( 679 u64::try_from(MAXIMUM_STDERR_BYTES).expect("stderr bound"), 680 contract["bounds"]["maximum_stderr_bytes"] 681 ); 682 assert_eq!( 683 contract["actual_process_corpus"], 684 serde_json::json!([ 685 "actual_binary_executes_offline_bootstrap_and_reaches_real_runtime_dependency_boundary", 686 "actual_binary_runs_the_task_graph_serves_admin_and_shuts_down_on_interrupt", 687 "actual_binary_is_bounded_under_parallel_inspection_and_reopen_soak" 688 ]) 689 ); 690 assert_eq!( 691 contract["invariants"], 692 serde_json::json!({ 693 "actual_executable_required": true, 694 "loopback_relay_only": true, 695 "production_failpoint_surface": false, 696 "test_environment_selector": false, 697 "detached_test_worker": false, 698 "parallel_inspection_is_read_only": true, 699 "every_daemon_is_interrupted_joined_and_reaped": true, 700 "admin_socket_absent_after_shutdown": true, 701 "state_verifies_after_every_reopen": true, 702 "captured_output_bounded": true, 703 "diagnostics_path_secret_free": true 704 }) 705 ); 706 assert_eq!( 707 contract["deferred"], 708 serde_json::json!([ 709 "native_release_artifacts_step_216", 710 "rcld_promotion_step_217", 711 "parent_pin_alignment_step_217", 712 "nix", 713 "oci", 714 "signing", 715 "publication", 716 "deployment" 717 ]) 718 ); 719 assert_eq!( 720 lower_hex(&Sha256::digest(FAILURE_QUALIFICATION_CONTRACT)), 721 contract["component_qualification"]["sha256"] 722 ); 723 assert!(SOURCE_LOCK.contains("revision = \"e55c1749902811732167a2ce5c099e22e964624f\"")); 724 } 725 726 #[test] 727 fn actual_binary_executes_offline_bootstrap_and_reaches_real_runtime_dependency_boundary() { 728 let fixture = ProcessFixture::new(); 729 let secret = identity_secret(); 730 let expected_public_key = Keys::new(SecretKey::from_slice(&secret).expect("secret")) 731 .public_key() 732 .to_hex(); 733 let configuration = configuration( 734 &fixture, 735 &expected_public_key, 736 "ws://127.0.0.1:9/", 737 "ws://127.0.0.1:10/", 738 ); 739 bootstrap(&fixture, &configuration, secret); 740 741 let run = fixture.run(&["run"]); 742 assert_eq!(run.status.code(), Some(3)); 743 assert!(run.stdout.is_empty()); 744 assert_eq!(diagnostic_code(&run), "service_or_dependency_unavailable"); 745 let diagnostic = String::from_utf8(run.stderr).expect("diagnostic UTF-8"); 746 assert!(!diagnostic.contains(fixture.root.path().to_str().expect("UTF-8 root"))); 747 assert!(!diagnostic.contains("relay.example")); 748 assert!(!diagnostic.contains("service_wrapping_key")); 749 } 750 751 #[test] 752 fn actual_binary_runs_the_task_graph_serves_admin_and_shuts_down_on_interrupt() { 753 let relay = RelayHarness::start(); 754 let fixture = ProcessFixture::new(); 755 let secret = identity_secret(); 756 let expected_public_key = Keys::new(SecretKey::from_slice(&secret).expect("secret")) 757 .public_key() 758 .to_hex(); 759 let configuration = configuration( 760 &fixture, 761 &expected_public_key, 762 &relay.url("primary"), 763 &relay.url("secondary"), 764 ); 765 bootstrap(&fixture, &configuration, secret); 766 767 let mut daemon = BoundedProcess::spawn(fixture.command(&["run"])); 768 let status = wait_for_live_status(&fixture, &mut daemon); 769 assert_success(&status); 770 let status_value: serde_json::Value = 771 serde_json::from_slice(&status.stdout).expect("live status JSON"); 772 assert_eq!(status_value["service"], "rhi"); 773 assert_eq!(status_value["phase"], "ready"); 774 assert_eq!(status_value["ready"], true); 775 assert_eq!(status_value["persistence"]["schema_version"], 11); 776 777 let shutdown = interrupt_and_wait(daemon); 778 assert_success(&shutdown); 779 assert!(shutdown.stdout.is_empty()); 780 assert!(!fixture.runtime.artifacts().admin_socket().exists()); 781 } 782 783 #[test] 784 fn actual_binary_is_bounded_under_parallel_inspection_and_reopen_soak() { 785 let fixture = ProcessFixture::new(); 786 let secret = identity_secret(); 787 let expected_public_key = Keys::new(SecretKey::from_slice(&secret).expect("secret")) 788 .public_key() 789 .to_hex(); 790 let configuration = configuration( 791 &fixture, 792 &expected_public_key, 793 "ws://127.0.0.1:9/", 794 "ws://127.0.0.1:10/", 795 ); 796 bootstrap(&fixture, &configuration, secret); 797 798 let inspections = (0..PARALLEL_INSPECTIONS) 799 .map(|_| BoundedProcess::spawn(fixture.command(&["config", "validate"]))) 800 .collect::<Vec<_>>(); 801 for inspection in inspections { 802 assert_success(&inspection.wait()); 803 } 804 805 for _ in 0..SOAK_ITERATIONS { 806 assert_success(&fixture.run(&["state", "verify"])); 807 } 808 assert!(!fixture.runtime.artifacts().admin_socket().exists()); 809 } 810 811 fn lower_hex(bytes: &[u8]) -> String { 812 const DIGITS: &[u8; 16] = b"0123456789abcdef"; 813 let mut output = String::with_capacity(bytes.len() * 2); 814 for byte in bytes { 815 output.push(char::from(DIGITS[usize::from(byte >> 4)])); 816 output.push(char::from(DIGITS[usize::from(byte & 0x0f)])); 817 } 818 output 819 }