rhi

Coordinated trade for connected markets
git clone https://radroots.dev/git/rhi.git
Log | Files | Refs | README | LICENSE

services_hardening_process.rs (28261B)


      1 #![forbid(unsafe_code)]
      2 #![cfg(any(target_os = "linux", target_os = "macos"))]
      3 
      4 use std::{
      5     collections::BTreeSet,
      6     fs,
      7     io::{Read as _, Write as _},
      8     net::{TcpListener, TcpStream},
      9     os::unix::fs::PermissionsExt as _,
     10     path::PathBuf,
     11     process::{Child, Command, ExitStatus, Output, Stdio},
     12     sync::{
     13         Arc,
     14         atomic::{AtomicBool, Ordering},
     15     },
     16     thread,
     17     time::{Duration, Instant},
     18 };
     19 
     20 use nostr::{Keys, SecretKey};
     21 use rhi::{
     22     RadrootsHostEnvironment, RadrootsPathResolver, RadrootsPlatform, parse_rhi_cli_v1_from,
     23     resolve_rhi_runtime_context,
     24 };
     25 use sha2::{Digest as _, Sha256};
     26 use tungstenite::{Error as WebSocketError, Message, accept};
     27 
     28 const CONFIG_EXAMPLE: &str = include_str!("../contracts/services_hardening/config.v1.example.toml");
     29 const PROCESS_QUALIFICATION_CONTRACT: &str =
     30     include_str!("../contracts/services_hardening/process_qualification.v1.json");
     31 const FAILURE_QUALIFICATION_CONTRACT: &[u8] =
     32     include_bytes!("../contracts/services_hardening/failure_qualification.v1.json");
     33 const SOURCE_LOCK: &str = include_str!("../radroots.service.source_lock.v4.toml");
     34 const PROCESS_DEADLINE: Duration = Duration::from_secs(30);
     35 const POLL_INTERVAL: Duration = Duration::from_millis(2);
     36 const CONNECT_DEADLINE_MILLISECONDS: u64 = 5_000;
     37 const RELAY_HANDSHAKE_TIMEOUT: Duration = Duration::from_millis(5_000);
     38 const RELAY_IO_TIMEOUT: Duration = Duration::from_millis(100);
     39 const PARALLEL_INSPECTIONS: usize = 8;
     40 const SOAK_ITERATIONS: usize = 32;
     41 const MAXIMUM_STDOUT_BYTES: usize = 1_048_576;
     42 const MAXIMUM_STDERR_BYTES: usize = 8_192;
     43 #[cfg(target_os = "linux")]
     44 const PROCESS_TEMPORARY_ROOT: &str = "/tmp";
     45 #[cfg(target_os = "macos")]
     46 const PROCESS_TEMPORARY_ROOT: &str = "/private/tmp";
     47 
     48 struct RelayHarness {
     49     address: std::net::SocketAddr,
     50     stop: Arc<AtomicBool>,
     51     thread: Option<thread::JoinHandle<()>>,
     52 }
     53 
     54 impl RelayHarness {
     55     fn start() -> Self {
     56         let listener = TcpListener::bind("127.0.0.1:0").expect("test relay listener");
     57         listener
     58             .set_nonblocking(true)
     59             .expect("nonblocking test relay");
     60         let address = listener.local_addr().expect("test relay address");
     61         let stop = Arc::new(AtomicBool::new(false));
     62         let thread_stop = Arc::clone(&stop);
     63         let thread = thread::spawn(move || {
     64             let mut sessions = Vec::new();
     65             while !thread_stop.load(Ordering::SeqCst) {
     66                 match listener.accept() {
     67                     Ok((stream, _)) => {
     68                         let session_stop = Arc::clone(&thread_stop);
     69                         sessions.push(thread::spawn(move || relay_session(stream, &session_stop)));
     70                     }
     71                     Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
     72                         thread::sleep(Duration::from_millis(2));
     73                     }
     74                     Err(error) => panic!("test relay accept failed: {error}"),
     75                 }
     76             }
     77             for session in sessions {
     78                 session.join().expect("test relay session");
     79             }
     80         });
     81         Self {
     82             address,
     83             stop,
     84             thread: Some(thread),
     85         }
     86     }
     87 
     88     fn url(&self, path: &str) -> String {
     89         format!("ws://{}/{path}", self.address)
     90     }
     91 }
     92 
     93 impl Drop for RelayHarness {
     94     fn drop(&mut self) {
     95         self.stop.store(true, Ordering::SeqCst);
     96         let _ = TcpStream::connect(self.address);
     97         if let Some(thread) = self.thread.take() {
     98             thread.join().expect("test relay");
     99         }
    100     }
    101 }
    102 
    103 fn relay_session(stream: TcpStream, stop: &AtomicBool) {
    104     relay_session_when_ready(stream, stop, || true);
    105 }
    106 
    107 fn relay_session_when_ready(
    108     stream: TcpStream,
    109     stop: &AtomicBool,
    110     handshake_ready: impl FnOnce() -> bool,
    111 ) {
    112     // Accepted sockets can inherit the listener's nonblocking mode on macOS.
    113     // This fixture uses blocking tungstenite with explicit I/O deadlines.
    114     stream
    115         .set_nonblocking(false)
    116         .expect("blocking accepted relay socket");
    117     stream
    118         .set_read_timeout(Some(RELAY_HANDSHAKE_TIMEOUT))
    119         .expect("relay handshake read timeout");
    120     stream
    121         .set_write_timeout(Some(RELAY_HANDSHAKE_TIMEOUT))
    122         .expect("relay handshake write timeout");
    123     if !handshake_ready() {
    124         return;
    125     }
    126     let mut websocket = match accept(stream) {
    127         Ok(websocket) => websocket,
    128         Err(_) => return,
    129     };
    130     websocket
    131         .get_mut()
    132         .set_read_timeout(Some(RELAY_IO_TIMEOUT))
    133         .expect("relay read timeout");
    134     websocket
    135         .get_mut()
    136         .set_write_timeout(Some(RELAY_IO_TIMEOUT))
    137         .expect("relay write timeout");
    138     while !stop.load(Ordering::SeqCst) {
    139         let message = match websocket.read() {
    140             Ok(message) => message,
    141             Err(WebSocketError::Io(error))
    142                 if matches!(
    143                     error.kind(),
    144                     std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
    145                 ) =>
    146             {
    147                 continue;
    148             }
    149             Err(WebSocketError::ConnectionClosed | WebSocketError::AlreadyClosed) => break,
    150             Err(_) => break,
    151         };
    152         match message {
    153             Message::Text(text) => relay_text(&mut websocket, text.as_str()),
    154             Message::Ping(bytes) => {
    155                 if websocket.send(Message::Pong(bytes)).is_err() {
    156                     break;
    157                 }
    158             }
    159             Message::Close(_) => break,
    160             _ => {}
    161         }
    162     }
    163 }
    164 
    165 fn relay_text(websocket: &mut tungstenite::WebSocket<TcpStream>, text: &str) {
    166     let Ok(message) = serde_json::from_str::<serde_json::Value>(text) else {
    167         return;
    168     };
    169     let Some(parts) = message.as_array() else {
    170         return;
    171     };
    172     match parts.first().and_then(serde_json::Value::as_str) {
    173         Some("REQ") => {
    174             let Some(subscription) = parts.get(1).and_then(serde_json::Value::as_str) else {
    175                 return;
    176             };
    177             let response = serde_json::json!(["EOSE", subscription]).to_string();
    178             let _ = websocket.send(Message::Text(response.into()));
    179         }
    180         Some("EVENT") => {
    181             let Some(event_id) = parts
    182                 .get(1)
    183                 .and_then(|event| event.get("id"))
    184                 .and_then(serde_json::Value::as_str)
    185             else {
    186                 return;
    187             };
    188             let response = serde_json::json!(["OK", event_id, true, ""]).to_string();
    189             let _ = websocket.send(Message::Text(response.into()));
    190         }
    191         _ => {}
    192     }
    193 }
    194 
    195 #[test]
    196 fn loopback_relay_waits_for_delayed_upgrade_on_an_inherited_nonblocking_socket() {
    197     let listener = TcpListener::bind("127.0.0.1:0").expect("regression listener");
    198     let address = listener.local_addr().expect("regression address");
    199     let client = TcpStream::connect(address).expect("regression client");
    200     client
    201         .set_read_timeout(Some(RELAY_HANDSHAKE_TIMEOUT))
    202         .expect("client read deadline");
    203     client
    204         .set_write_timeout(Some(RELAY_HANDSHAKE_TIMEOUT))
    205         .expect("client write deadline");
    206     let (server, _) = listener.accept().expect("accepted regression socket");
    207     // Exercise inherited nonblocking mode explicitly on every platform.
    208     server.set_nonblocking(true).expect("inherited socket mode");
    209     let observed_server = server.try_clone().expect("observe accepted socket mode");
    210     let stop = Arc::new(AtomicBool::new(false));
    211     let session_stop = Arc::clone(&stop);
    212     let (ready, configured) = std::sync::mpsc::sync_channel(0);
    213     let (resume, released) = std::sync::mpsc::sync_channel(0);
    214     let session = thread::spawn(move || {
    215         relay_session_when_ready(server, &session_stop, || {
    216             ready.send(()).is_ok() && released.recv_timeout(PROCESS_DEADLINE).is_ok()
    217         });
    218     });
    219     let response = (|| -> Result<Message, String> {
    220         configured
    221             .recv_timeout(PROCESS_DEADLINE)
    222             .map_err(|error| error.to_string())?;
    223         // Observe the actual shared socket flags before sending upgrade bytes.
    224         // A timed absence of completion is not evidence under scheduler load.
    225         let flags = rustix::fs::fcntl_getfl(&observed_server).map_err(|error| error.to_string())?;
    226         if flags.contains(rustix::fs::OFlags::NONBLOCK) {
    227             return Err("relay retained inherited nonblocking mode".to_owned());
    228         }
    229         resume.send(()).map_err(|error| error.to_string())?;
    230         let (mut socket, _) = tungstenite::client(format!("ws://{address}/delayed"), client)
    231             .map_err(|error| error.to_string())?;
    232         socket
    233             .send(Message::Text(r#"["REQ","delayed",{}]"#.into()))
    234             .map_err(|error| error.to_string())?;
    235         socket.read().map_err(|error| error.to_string())
    236     })();
    237     stop.store(true, Ordering::SeqCst);
    238     drop(resume);
    239     drop(configured);
    240     session.join().expect("joined regression relay");
    241     assert_eq!(
    242         response.expect("delayed upgrade and request succeed"),
    243         Message::Text(r#"["EOSE","delayed"]"#.into())
    244     );
    245 }
    246 
    247 struct ProcessFixture {
    248     root: tempfile::TempDir,
    249     config: PathBuf,
    250     runtime: rhi::RhiRuntimeContext,
    251 }
    252 
    253 impl ProcessFixture {
    254     fn new() -> Self {
    255         let root = tempfile::Builder::new()
    256             .prefix("rhi-")
    257             .tempdir_in(PROCESS_TEMPORARY_ROOT)
    258             .expect("short repo-local root");
    259         fs::set_permissions(root.path(), fs::Permissions::from_mode(0o700))
    260             .expect("secure repo-local root");
    261         let config = root.path().join("rhi.toml");
    262         let invocation = parse_rhi_cli_v1_from([
    263             "rhi",
    264             "--profile",
    265             "repo-local",
    266             "--instance",
    267             "primary",
    268             "--repo-local-root",
    269             root.path().to_str().expect("UTF-8 test root"),
    270             "--config",
    271             config.to_str().expect("UTF-8 config path"),
    272             "run",
    273         ])
    274         .expect("runtime invocation");
    275         let runtime = resolve_rhi_runtime_context(
    276             &RadrootsPathResolver::new(
    277                 RadrootsPlatform::current(),
    278                 RadrootsHostEnvironment::default(),
    279             ),
    280             &invocation,
    281         )
    282         .expect("runtime context");
    283         Self {
    284             root,
    285             config,
    286             runtime,
    287         }
    288     }
    289 
    290     fn command(&self, command: &[&str]) -> Command {
    291         let mut process = Command::new(env!("CARGO_BIN_EXE_rhi"));
    292         process
    293             .args(["--profile", "repo-local", "--instance", "primary"])
    294             .arg("--repo-local-root")
    295             .arg(self.root.path())
    296             .arg("--config")
    297             .arg(&self.config)
    298             .args(command)
    299             .stdin(Stdio::null())
    300             .stdout(Stdio::piped())
    301             .stderr(Stdio::piped());
    302         process
    303     }
    304 
    305     fn run(&self, command: &[&str]) -> Output {
    306         BoundedProcess::spawn(self.command(command)).wait()
    307     }
    308 
    309     fn run_with_stdin(&self, command: &[&str], bytes: &[u8]) -> Output {
    310         let mut process = self.command(command);
    311         process.stdin(Stdio::piped());
    312         let mut child = BoundedProcess::spawn(process);
    313         child
    314             .child
    315             .stdin
    316             .take()
    317             .expect("process stdin")
    318             .write_all(bytes)
    319             .expect("bounded stdin");
    320         child.wait()
    321     }
    322 }
    323 
    324 struct BoundedProcess {
    325     child: Child,
    326     stdout_reader: Option<thread::JoinHandle<Vec<u8>>>,
    327     stderr_reader: Option<thread::JoinHandle<Vec<u8>>>,
    328     completed: bool,
    329 }
    330 
    331 impl BoundedProcess {
    332     fn spawn(mut command: Command) -> Self {
    333         let child = command.spawn().expect("RHI process");
    334         Self::from_child(child)
    335     }
    336 
    337     fn from_child(mut child: Child) -> Self {
    338         let stdout = child.stdout.take().expect("captured process stdout");
    339         let stderr = child.stderr.take().expect("captured process stderr");
    340         Self {
    341             child,
    342             stdout_reader: Some(read_bounded(stdout, MAXIMUM_STDOUT_BYTES)),
    343             stderr_reader: Some(read_bounded(stderr, MAXIMUM_STDERR_BYTES)),
    344             completed: false,
    345         }
    346     }
    347 
    348     fn id(&self) -> u32 {
    349         self.child.id()
    350     }
    351 
    352     fn try_wait(&mut self) -> Option<ExitStatus> {
    353         self.child.try_wait().expect("poll RHI process")
    354     }
    355 
    356     fn collect(&mut self, status: ExitStatus) -> Output {
    357         self.completed = true;
    358         let stdout = self
    359             .stdout_reader
    360             .take()
    361             .expect("stdout reader available")
    362             .join()
    363             .expect("join stdout reader");
    364         let stderr = self
    365             .stderr_reader
    366             .take()
    367             .expect("stderr reader available")
    368             .join()
    369             .expect("join stderr reader");
    370         assert!(stdout.len() <= MAXIMUM_STDOUT_BYTES);
    371         assert!(stderr.len() <= MAXIMUM_STDERR_BYTES);
    372         Output {
    373             status,
    374             stdout,
    375             stderr,
    376         }
    377     }
    378 
    379     fn wait(mut self) -> Output {
    380         let deadline = Instant::now() + PROCESS_DEADLINE;
    381         loop {
    382             if let Some(status) = self.try_wait() {
    383                 return self.collect(status);
    384             }
    385             if Instant::now() >= deadline {
    386                 let _ = self.child.kill();
    387                 let status = self.child.wait().expect("reap RHI process");
    388                 let output = self.collect(status);
    389                 panic!("RHI process exceeded deadline: {:?}", output.stderr);
    390             }
    391             thread::sleep(POLL_INTERVAL);
    392         }
    393     }
    394 }
    395 
    396 impl Drop for BoundedProcess {
    397     fn drop(&mut self) {
    398         if !self.completed {
    399             let _ = self.child.kill();
    400             let _ = self.child.wait();
    401         }
    402         if let Some(reader) = self.stdout_reader.take() {
    403             let _ = reader.join();
    404         }
    405         if let Some(reader) = self.stderr_reader.take() {
    406             let _ = reader.join();
    407         }
    408     }
    409 }
    410 
    411 fn read_bounded(
    412     reader: impl std::io::Read + Send + 'static,
    413     maximum_bytes: usize,
    414 ) -> thread::JoinHandle<Vec<u8>> {
    415     thread::spawn(move || {
    416         let limit = u64::try_from(maximum_bytes)
    417             .expect("output bound fits u64")
    418             .checked_add(1)
    419             .expect("output bound plus sentinel fits u64");
    420         let mut output = Vec::with_capacity(maximum_bytes.min(8_192));
    421         reader
    422             .take(limit)
    423             .read_to_end(&mut output)
    424             .expect("read bounded process output");
    425         output
    426     })
    427 }
    428 
    429 fn identity_secret() -> [u8; 32] {
    430     let mut candidate: [u8; 32] =
    431         Sha256::digest(b"radroots.rhi.step-213.process-identity.v1").into();
    432     while SecretKey::from_slice(&candidate).is_err() {
    433         candidate = Sha256::digest(candidate).into();
    434     }
    435     candidate
    436 }
    437 
    438 fn provisioning_document(secret: [u8; 32]) -> [u8; 117] {
    439     let mut document = [0_u8; 117];
    440     document[..4].copy_from_slice(b"RHIP");
    441     document[4] = 1;
    442     document[5..37].copy_from_slice(&secret);
    443     document[37..69].copy_from_slice(&Sha256::digest(b"rhi-step-213-data-key"));
    444     document[69..93].copy_from_slice(&[3; 24]);
    445     document[93..117].copy_from_slice(&[4; 24]);
    446     document
    447 }
    448 
    449 fn configuration(
    450     fixture: &ProcessFixture,
    451     expected_public_key: &str,
    452     primary_relay: &str,
    453     secondary_relay: &str,
    454 ) -> String {
    455     CONFIG_EXAMPLE
    456         .replace(
    457             "/var/lib/radroots/services/rhi/default/secrets/service.identity.ncrypt",
    458             fixture
    459                 .runtime
    460                 .identity_path()
    461                 .to_str()
    462                 .expect("UTF-8 identity path"),
    463         )
    464         .replace(&"2".repeat(64), expected_public_key)
    465         .replace("wss://relay.example.com/", primary_relay)
    466         .replace("wss://relay-secondary.example.com/", secondary_relay)
    467         .replace(
    468             "connect_deadline_ms = 10000",
    469             &format!("connect_deadline_ms = {CONNECT_DEADLINE_MILLISECONDS}"),
    470         )
    471         .replace("request_deadline_ms = 15000", "request_deadline_ms = 1000")
    472 }
    473 
    474 fn diagnostic_code(output: &Output) -> String {
    475     let value: serde_json::Value = serde_json::from_slice(&output.stderr).expect("diagnostic JSON");
    476     value["code"].as_str().expect("diagnostic code").to_owned()
    477 }
    478 
    479 fn assert_success(output: &Output) {
    480     assert_eq!(output.status.code(), Some(0), "stderr: {:?}", output.stderr);
    481     assert_eq!(diagnostic_code(output), "success");
    482 }
    483 
    484 fn bootstrap(fixture: &ProcessFixture, configuration: &str, secret: [u8; 32]) -> String {
    485     let expected_public_key = Keys::new(SecretKey::from_slice(&secret).expect("secret"))
    486         .public_key()
    487         .to_hex();
    488     let config = fixture.run_with_stdin(&["config", "init"], configuration.as_bytes());
    489     assert_success(&config);
    490     assert_eq!(config.stdout, b"config_initialized\n");
    491 
    492     for directory in [
    493         fixture.runtime.context().paths().state(),
    494         fixture.runtime.context().paths().secrets(),
    495         fixture.runtime.context().paths().run(),
    496     ] {
    497         fs::create_dir_all(directory).expect("secure runtime directory");
    498         fs::set_permissions(directory, fs::Permissions::from_mode(0o700))
    499             .expect("secure runtime mode");
    500     }
    501     let credential = fixture
    502         .runtime
    503         .context()
    504         .paths()
    505         .secrets()
    506         .join("service_wrapping_key");
    507     fs::write(&credential, Sha256::digest(b"rhi-step-213-wrapping-key"))
    508         .expect("wrapping credential");
    509     fs::set_permissions(&credential, fs::Permissions::from_mode(0o600)).expect("credential mode");
    510 
    511     let state = fixture.run(&["state", "init"]);
    512     assert_success(&state);
    513     assert_eq!(state.stdout, b"state_initialized\n");
    514     let identity = fixture.run_with_stdin(
    515         &["--output", "json", "identity", "init"],
    516         &provisioning_document(secret),
    517     );
    518     assert_success(&identity);
    519     let identity_value: serde_json::Value =
    520         serde_json::from_slice(&identity.stdout).expect("identity result");
    521     assert_eq!(identity_value["public_key"], expected_public_key);
    522 
    523     for command in [
    524         &["config", "validate"][..],
    525         &["state", "verify"][..],
    526         &["state", "migrate"][..],
    527     ] {
    528         assert_success(&fixture.run(command));
    529     }
    530     expected_public_key
    531 }
    532 
    533 fn wait_for_live_status(fixture: &ProcessFixture, daemon: &mut BoundedProcess) -> Output {
    534     let deadline = Instant::now() + PROCESS_DEADLINE;
    535     let mut last_diagnostic = Vec::new();
    536     loop {
    537         if let Some(status) = daemon.try_wait() {
    538             let output = daemon.collect(status);
    539             panic!(
    540                 "RHI daemon exited before admin became ready: {status}; stderr={:?}",
    541                 output.stderr
    542             );
    543         }
    544         if fixture.runtime.artifacts().admin_socket().exists() {
    545             let status = fixture.run(&["status"]);
    546             if status.status.success() {
    547                 return status;
    548             }
    549             last_diagnostic = status.stderr;
    550         }
    551         if Instant::now() >= deadline {
    552             panic!(
    553                 "RHI admin did not become ready before deadline; socket={}; stderr={last_diagnostic:?}",
    554                 fixture.runtime.artifacts().admin_socket().exists()
    555             );
    556         }
    557         thread::sleep(POLL_INTERVAL);
    558     }
    559 }
    560 
    561 fn interrupt_and_wait(daemon: BoundedProcess) -> Output {
    562     let signal = Command::new("kill")
    563         .arg("-INT")
    564         .arg(daemon.id().to_string())
    565         .status()
    566         .expect("send interrupt");
    567     assert!(signal.success(), "interrupt command failed: {signal}");
    568     daemon.wait()
    569 }
    570 
    571 #[test]
    572 fn process_qualification_contract_freezes_the_exact_wave_closure() {
    573     let contract: serde_json::Value =
    574         serde_json::from_str(PROCESS_QUALIFICATION_CONTRACT).expect("process contract");
    575     assert_eq!(
    576         contract
    577             .as_object()
    578             .expect("process contract object")
    579             .keys()
    580             .map(String::as_str)
    581             .collect::<BTreeSet<_>>(),
    582         BTreeSet::from([
    583             "actual_process_corpus",
    584             "binary",
    585             "bounds",
    586             "component_qualification",
    587             "contract_version",
    588             "deferred",
    589             "invariants",
    590             "schema",
    591             "service",
    592             "source_lock",
    593             "step",
    594         ])
    595     );
    596     assert_eq!(contract["schema"], "radroots.rhi.process-qualification.v1");
    597     assert_eq!(contract["contract_version"], 1);
    598     assert_eq!(contract["step"], 215);
    599     assert_eq!(contract["service"], "rhi");
    600     assert_eq!(contract["binary"], "rhi");
    601     assert_eq!(
    602         contract["source_lock"],
    603         serde_json::json!({
    604             "schema": "radroots.service.source-lock.v2",
    605             "lib_revision": "053d0c750bf9cd683c6ea37cefe7e79617ba629f"
    606         })
    607     );
    608     assert_eq!(
    609         contract["component_qualification"],
    610         serde_json::json!({
    611             "schema": "radroots.rhi.failure-qualification.v1",
    612             "step": 214,
    613             "sha256": "f05da8e559f463f99c67c3c7e22eafa57935be91fb0094f2aa2f98a58544b8b9"
    614         })
    615     );
    616     assert_eq!(
    617         contract["bounds"],
    618         serde_json::json!({
    619             "process_deadline_ms": 30_000,
    620             "poll_interval_ms": 2,
    621             "connect_deadline_ms": 5_000,
    622             "relay_handshake_timeout_ms": 5_000,
    623             "relay_io_timeout_ms": 100,
    624             "parallel_inspection_processes": 8,
    625             "soak_iterations": 32,
    626             "maximum_stdout_bytes": 1_048_576,
    627             "maximum_stderr_bytes": 8_192
    628         })
    629     );
    630     assert_eq!(
    631         PROCESS_DEADLINE,
    632         Duration::from_millis(
    633             contract["bounds"]["process_deadline_ms"]
    634                 .as_u64()
    635                 .expect("process deadline"),
    636         )
    637     );
    638     assert_eq!(
    639         POLL_INTERVAL,
    640         Duration::from_millis(
    641             contract["bounds"]["poll_interval_ms"]
    642                 .as_u64()
    643                 .expect("poll interval"),
    644         )
    645     );
    646     assert_eq!(
    647         RELAY_HANDSHAKE_TIMEOUT,
    648         Duration::from_millis(
    649             contract["bounds"]["relay_handshake_timeout_ms"]
    650                 .as_u64()
    651                 .expect("relay handshake timeout"),
    652         )
    653     );
    654     assert_eq!(
    655         RELAY_IO_TIMEOUT,
    656         Duration::from_millis(
    657             contract["bounds"]["relay_io_timeout_ms"]
    658                 .as_u64()
    659                 .expect("relay timeout"),
    660         )
    661     );
    662     assert_eq!(
    663         CONNECT_DEADLINE_MILLISECONDS,
    664         contract["bounds"]["connect_deadline_ms"]
    665     );
    666     assert_eq!(
    667         u64::try_from(PARALLEL_INSPECTIONS).expect("inspection bound"),
    668         contract["bounds"]["parallel_inspection_processes"]
    669     );
    670     assert_eq!(
    671         u64::try_from(SOAK_ITERATIONS).expect("soak bound"),
    672         contract["bounds"]["soak_iterations"]
    673     );
    674     assert_eq!(
    675         u64::try_from(MAXIMUM_STDOUT_BYTES).expect("stdout bound"),
    676         contract["bounds"]["maximum_stdout_bytes"]
    677     );
    678     assert_eq!(
    679         u64::try_from(MAXIMUM_STDERR_BYTES).expect("stderr bound"),
    680         contract["bounds"]["maximum_stderr_bytes"]
    681     );
    682     assert_eq!(
    683         contract["actual_process_corpus"],
    684         serde_json::json!([
    685             "actual_binary_executes_offline_bootstrap_and_reaches_real_runtime_dependency_boundary",
    686             "actual_binary_runs_the_task_graph_serves_admin_and_shuts_down_on_interrupt",
    687             "actual_binary_is_bounded_under_parallel_inspection_and_reopen_soak"
    688         ])
    689     );
    690     assert_eq!(
    691         contract["invariants"],
    692         serde_json::json!({
    693             "actual_executable_required": true,
    694             "loopback_relay_only": true,
    695             "production_failpoint_surface": false,
    696             "test_environment_selector": false,
    697             "detached_test_worker": false,
    698             "parallel_inspection_is_read_only": true,
    699             "every_daemon_is_interrupted_joined_and_reaped": true,
    700             "admin_socket_absent_after_shutdown": true,
    701             "state_verifies_after_every_reopen": true,
    702             "captured_output_bounded": true,
    703             "diagnostics_path_secret_free": true
    704         })
    705     );
    706     assert_eq!(
    707         contract["deferred"],
    708         serde_json::json!([
    709             "native_release_artifacts_step_216",
    710             "rcld_promotion_step_217",
    711             "parent_pin_alignment_step_217",
    712             "nix",
    713             "oci",
    714             "signing",
    715             "publication",
    716             "deployment"
    717         ])
    718     );
    719     assert_eq!(
    720         lower_hex(&Sha256::digest(FAILURE_QUALIFICATION_CONTRACT)),
    721         contract["component_qualification"]["sha256"]
    722     );
    723     assert!(SOURCE_LOCK.contains("revision = \"e55c1749902811732167a2ce5c099e22e964624f\""));
    724 }
    725 
    726 #[test]
    727 fn actual_binary_executes_offline_bootstrap_and_reaches_real_runtime_dependency_boundary() {
    728     let fixture = ProcessFixture::new();
    729     let secret = identity_secret();
    730     let expected_public_key = Keys::new(SecretKey::from_slice(&secret).expect("secret"))
    731         .public_key()
    732         .to_hex();
    733     let configuration = configuration(
    734         &fixture,
    735         &expected_public_key,
    736         "ws://127.0.0.1:9/",
    737         "ws://127.0.0.1:10/",
    738     );
    739     bootstrap(&fixture, &configuration, secret);
    740 
    741     let run = fixture.run(&["run"]);
    742     assert_eq!(run.status.code(), Some(3));
    743     assert!(run.stdout.is_empty());
    744     assert_eq!(diagnostic_code(&run), "service_or_dependency_unavailable");
    745     let diagnostic = String::from_utf8(run.stderr).expect("diagnostic UTF-8");
    746     assert!(!diagnostic.contains(fixture.root.path().to_str().expect("UTF-8 root")));
    747     assert!(!diagnostic.contains("relay.example"));
    748     assert!(!diagnostic.contains("service_wrapping_key"));
    749 }
    750 
    751 #[test]
    752 fn actual_binary_runs_the_task_graph_serves_admin_and_shuts_down_on_interrupt() {
    753     let relay = RelayHarness::start();
    754     let fixture = ProcessFixture::new();
    755     let secret = identity_secret();
    756     let expected_public_key = Keys::new(SecretKey::from_slice(&secret).expect("secret"))
    757         .public_key()
    758         .to_hex();
    759     let configuration = configuration(
    760         &fixture,
    761         &expected_public_key,
    762         &relay.url("primary"),
    763         &relay.url("secondary"),
    764     );
    765     bootstrap(&fixture, &configuration, secret);
    766 
    767     let mut daemon = BoundedProcess::spawn(fixture.command(&["run"]));
    768     let status = wait_for_live_status(&fixture, &mut daemon);
    769     assert_success(&status);
    770     let status_value: serde_json::Value =
    771         serde_json::from_slice(&status.stdout).expect("live status JSON");
    772     assert_eq!(status_value["service"], "rhi");
    773     assert_eq!(status_value["phase"], "ready");
    774     assert_eq!(status_value["ready"], true);
    775     assert_eq!(status_value["persistence"]["schema_version"], 11);
    776 
    777     let shutdown = interrupt_and_wait(daemon);
    778     assert_success(&shutdown);
    779     assert!(shutdown.stdout.is_empty());
    780     assert!(!fixture.runtime.artifacts().admin_socket().exists());
    781 }
    782 
    783 #[test]
    784 fn actual_binary_is_bounded_under_parallel_inspection_and_reopen_soak() {
    785     let fixture = ProcessFixture::new();
    786     let secret = identity_secret();
    787     let expected_public_key = Keys::new(SecretKey::from_slice(&secret).expect("secret"))
    788         .public_key()
    789         .to_hex();
    790     let configuration = configuration(
    791         &fixture,
    792         &expected_public_key,
    793         "ws://127.0.0.1:9/",
    794         "ws://127.0.0.1:10/",
    795     );
    796     bootstrap(&fixture, &configuration, secret);
    797 
    798     let inspections = (0..PARALLEL_INSPECTIONS)
    799         .map(|_| BoundedProcess::spawn(fixture.command(&["config", "validate"])))
    800         .collect::<Vec<_>>();
    801     for inspection in inspections {
    802         assert_success(&inspection.wait());
    803     }
    804 
    805     for _ in 0..SOAK_ITERATIONS {
    806         assert_success(&fixture.run(&["state", "verify"]));
    807     }
    808     assert!(!fixture.runtime.artifacts().admin_socket().exists());
    809 }
    810 
    811 fn lower_hex(bytes: &[u8]) -> String {
    812     const DIGITS: &[u8; 16] = b"0123456789abcdef";
    813     let mut output = String::with_capacity(bytes.len() * 2);
    814     for byte in bytes {
    815         output.push(char::from(DIGITS[usize::from(byte >> 4)]));
    816         output.push(char::from(DIGITS[usize::from(byte & 0x0f)]));
    817     }
    818     output
    819 }