commit 370dee1ac85d6ccb5d4a7bbf8e8363c8d6905341
parent 46d77b432290040e236273a01f3556c933c3732b
Author: triesap <tyson@radroots.org>
Date: Sun, 23 Aug 2026 22:12:42 +0000
state(rhi): integrate backup and recovery
Compose the shared SQLx-owned migration, integrity, backup, staged restore, finalization, and interrupted-recovery boundaries without adding a second database authority.
Diffstat:
7 files changed, 826 insertions(+), 30 deletions(-)
diff --git a/README b/README
@@ -151,6 +151,18 @@ pool, connection, transaction, SQL, path, or cloneable write authority.
Schema migration and verification, CRUD behavior, backup/restore, network I/O,
and task supervision remain owned by their later ordered checkpoints.
+RHI now composes the shared SQLx-owned resilience boundary without exposing a
+second database authority. A writable `RhiStateHost` can capture one governed
+online backup, and either host mode can run an explicit bounded integrity
+inspection. Offline verification returns a sealed exact-inode proof; staging
+retains exclusive writer authority; finalization uses the shared durable
+marker and atomic replacement protocol; and the next writable existing-state
+open reconciles interrupted restore evidence before exposing a host. Read-only
+inspection never performs recovery. Callers inject all times, manifest bytes
+and digest, and the positive backup-size limit. RHI adds no SQLite dependency,
+raw connection, background runtime, implicit deadline, or direct file-copy
+authority.
+
Validate the standalone crate through extbuild:
```text
diff --git a/src/lib.rs b/src/lib.rs
@@ -9,6 +9,7 @@ mod identity_envelope;
mod runtime_context;
mod state_catalog;
mod state_host;
+mod state_maintenance;
mod state_metadata;
mod state_repository;
@@ -56,6 +57,11 @@ pub use state_host::{
RhiStateHost, RhiStateHostError, RhiStateHostErrorKind, RhiStateHostMode, initialize_rhi_state,
open_rhi_state_inspection, open_rhi_state_read_write,
};
+pub use state_maintenance::{
+ RhiStagedStateRestore, RhiStateMaintenanceError, RhiStateMaintenanceErrorKind,
+ RhiVerifiedStateBackup, finalize_rhi_state_restore, stage_rhi_state_restore,
+ verify_rhi_state_backup,
+};
pub use state_metadata::{
RHI_ADMIN_CONTRACT_VERSION, RHI_PROVIDER_CONTRACT_VERSION, RHI_STATE_APPLICATION_ID,
RHI_STATUS_CONTRACT_VERSION, RhiEvidencePolicyDigest, RhiExpectedPublicIdentity,
diff --git a/src/state_host.rs b/src/state_host.rs
@@ -1,17 +1,22 @@
//! Sealed lifecycle boundary for the canonical RHI SQLite state catalog.
use core::fmt;
-use std::{error::Error, path::PathBuf};
+use std::{
+ error::Error,
+ path::{Path, PathBuf},
+};
use radroots_service_sqlite::{
- MigrationAppliedAtUnixSeconds, MigrationBuildIdentity, OpenMode,
- ServiceSqliteConnectionOptions, ServiceSqliteHost, ServiceSqlitePaths, initialize_database,
+ BackupCreatedAtUnixMs, IntegrityCheckedAtUnixMs, MigrationAppliedAtUnixSeconds,
+ MigrationBuildIdentity, OpenMode, ServiceBackupManifest, ServiceSqliteConnectionOptions,
+ ServiceSqliteHost, ServiceSqliteIntegrityReport, ServiceSqlitePaths, initialize_database,
};
use sqlx::{ConnectOptions, Connection, SqliteConnection, sqlite::SqliteConnectOptions};
use crate::{
- RHI_STATE_SCHEMA_VERSION, RhiRuntimeContext, RhiStateMetadata, RhiStateRepositories,
- rhi_migration_catalog, rhi_schema_catalog, validate_rhi_state_catalogs,
+ RHI_STATE_SCHEMA_VERSION, RhiRuntimeContext, RhiStateMaintenanceError,
+ RhiStateMaintenanceErrorKind, RhiStateMetadata, RhiStateRepositories, rhi_migration_catalog,
+ rhi_schema_catalog, validate_rhi_state_catalogs,
};
/// Stable lifecycle mode of one opened RHI state host.
@@ -145,6 +150,37 @@ impl RhiStateHost {
RhiStateRepositories::new(self)
}
+ /// Captures one governed point-in-time backup from a writable RHI host.
+ ///
+ /// The staging directory must be a new absolute path. The returned
+ /// manifest remains in memory and contains no protected identity material.
+ pub async fn capture_online_backup(
+ &self,
+ staging_directory: &Path,
+ created_at: BackupCreatedAtUnixMs,
+ ) -> Result<ServiceBackupManifest, RhiStateMaintenanceError> {
+ if self.mode != RhiStateHostMode::ReadWriteExisting {
+ return Err(RhiStateMaintenanceError::new(
+ RhiStateMaintenanceErrorKind::InvalidMode,
+ ));
+ }
+ self.host
+ .capture_online_backup(staging_directory, created_at)
+ .await
+ .map_err(RhiStateMaintenanceError::from_sqlite)
+ }
+
+ /// Runs one explicit bounded integrity inspection over this host.
+ pub async fn inspect_integrity(
+ &self,
+ checked_at: IntegrityCheckedAtUnixMs,
+ ) -> Result<ServiceSqliteIntegrityReport, RhiStateMaintenanceError> {
+ self.host
+ .inspect_integrity(checked_at)
+ .await
+ .map_err(RhiStateMaintenanceError::from_sqlite)
+ }
+
/// Drains the shared host and explicitly releases retained authority.
pub async fn close(&self) -> Result<(), RhiStateHostError> {
self.host
@@ -167,16 +203,20 @@ impl fmt::Debug for RhiStateHost {
/// Creates a missing RHI catalog exactly once and releases initialization authority.
///
/// This function never opens an existing database as initialization. The caller
-/// injects the shared metadata evidence; Step 171 owns its exact RHI application,
-/// configuration, evidence-policy, identity, and contract-version bindings.
+/// injects the shared metadata and migration evidence. Contract versions are
+/// cross-bound before database I/O, and all governed migrations are applied by
+/// the shared host before initialization authority is explicitly released.
pub async fn initialize_rhi_state(
runtime: &RhiRuntimeContext,
metadata: &RhiStateMetadata,
+ applied_at: MigrationAppliedAtUnixSeconds,
+ build: &MigrationBuildIdentity,
) -> Result<(), RhiStateHostError> {
let paths = state_paths(runtime)?;
require_metadata(runtime, metadata)?;
+ require_migration_build(metadata, build)?;
let (migrations, schema) = catalogs()?;
- let mut authority = initialize_database(
+ let authority = initialize_database(
&paths,
OpenMode::Initialize,
metadata.database(),
@@ -185,11 +225,26 @@ pub async fn initialize_rhi_state(
)
.await
.map_err(|_| RhiStateHostError::new(RhiStateHostErrorKind::Initialize))?;
- authority
- .release()
- .map_err(|_| RhiStateHostError::new(RhiStateHostErrorKind::Initialize))?;
- drop(migrations);
- Ok(())
+ let identity = metadata.database_identity();
+ let (host, outcome) = ServiceSqliteHost::open_initialized(
+ &paths,
+ &identity,
+ &migrations,
+ &schema,
+ ServiceSqliteConnectionOptions::reviewed(),
+ authority,
+ applied_at,
+ build,
+ &[],
+ )
+ .await
+ .map_err(|_| RhiStateHostError::new(RhiStateHostErrorKind::Initialize))?;
+ if !exact_migration_outcome(outcome) {
+ return Err(close_error(&host, RhiStateHostErrorKind::Catalog).await);
+ }
+ host.close()
+ .await
+ .map_err(|_| RhiStateHostError::new(RhiStateHostErrorKind::Initialize))
}
/// Opens an already initialized RHI catalog with exclusive writer authority.
@@ -205,6 +260,7 @@ pub async fn open_rhi_state_read_write(
) -> Result<RhiStateHost, RhiStateHostError> {
let paths = state_paths(runtime)?;
require_metadata(runtime, metadata)?;
+ require_migration_build(metadata, build)?;
let identity = metadata.database_identity();
let (migrations, schema) = catalogs()?;
let (host, outcome) = ServiceSqliteHost::open_read_write_existing(
@@ -219,12 +275,8 @@ pub async fn open_rhi_state_read_write(
)
.await
.map_err(|_| RhiStateHostError::new(RhiStateHostErrorKind::ReadWriteOpen))?;
- if outcome.initial_version() != RHI_STATE_SCHEMA_VERSION
- || outcome.final_version() != RHI_STATE_SCHEMA_VERSION
- || outcome.applied_count() != 0
- {
- let _ = host.close().await;
- return Err(RhiStateHostError::new(RhiStateHostErrorKind::Catalog));
+ if !exact_migration_outcome(outcome) {
+ return Err(close_error(&host, RhiStateHostErrorKind::Catalog).await);
}
Ok(RhiStateHost {
host,
@@ -258,12 +310,14 @@ pub async fn open_rhi_state_inspection(
})
}
-fn state_paths(runtime: &RhiRuntimeContext) -> Result<ServiceSqlitePaths, RhiStateHostError> {
+pub(crate) fn state_paths(
+ runtime: &RhiRuntimeContext,
+) -> Result<ServiceSqlitePaths, RhiStateHostError> {
ServiceSqlitePaths::from_runtime_context(runtime.context())
.map_err(|_| RhiStateHostError::new(RhiStateHostErrorKind::InvalidPaths))
}
-fn require_metadata(
+pub(crate) fn require_metadata(
runtime: &RhiRuntimeContext,
metadata: &RhiStateMetadata,
) -> Result<(), RhiStateHostError> {
@@ -277,7 +331,39 @@ fn require_metadata(
.ok_or_else(|| RhiStateHostError::new(RhiStateHostErrorKind::InvalidEvidence))
}
-fn catalogs() -> Result<
+fn require_migration_build(
+ metadata: &RhiStateMetadata,
+ build: &MigrationBuildIdentity,
+) -> Result<(), RhiStateHostError> {
+ let versions = metadata.policy_versions();
+ let matches = build.config_contract_version() == versions.configuration()
+ && build.state_contract_version() == versions.state()
+ && build.admin_contract_version() == versions.admin()
+ && build.status_contract_version() == versions.status()
+ && build.provider_contract_version() == versions.provider();
+ matches
+ .then_some(())
+ .ok_or_else(|| RhiStateHostError::new(RhiStateHostErrorKind::InvalidEvidence))
+}
+
+fn exact_migration_outcome(outcome: radroots_service_sqlite::MigrationApplicationOutcome) -> bool {
+ outcome.initial_version() == RHI_STATE_SCHEMA_VERSION
+ && outcome.final_version() == RHI_STATE_SCHEMA_VERSION
+ && outcome.applied_count() == 0
+}
+
+async fn close_error(
+ host: &ServiceSqliteHost,
+ fallback: RhiStateHostErrorKind,
+) -> RhiStateHostError {
+ if host.close().await.is_err() {
+ RhiStateHostError::new(RhiStateHostErrorKind::Close)
+ } else {
+ RhiStateHostError::new(fallback)
+ }
+}
+
+pub(crate) fn catalogs() -> Result<
(
radroots_service_sqlite::MigrationCatalog,
radroots_service_sqlite::SchemaCatalog,
diff --git a/src/state_maintenance.rs b/src/state_maintenance.rs
@@ -0,0 +1,307 @@
+//! RHI-bound integrity, backup, and offline restore integration.
+
+use core::{fmt, num::NonZeroU64};
+use std::{error::Error, path::Path};
+
+use radroots_service_sqlite::{
+ BackupManifestSha256, ServiceBackupManifest, ServiceDatabaseMetadata, ServiceSqliteError,
+ ServiceSqliteErrorKind, StagedServiceRestore, VerifiedServiceBackup, finalize_staged_restore,
+ stage_verified_restore, verify_backup_bundle,
+};
+
+use crate::{RhiRuntimeContext, RhiStateMetadata, state_host};
+
+/// Stable source-free class for an RHI state-maintenance failure.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum RhiStateMaintenanceErrorKind {
+ InvalidEvidence,
+ InvalidMode,
+ Catalog,
+ Authority,
+ Open,
+ Metadata,
+ Migration,
+ Backup,
+ Restore,
+ Integrity,
+ Recovery,
+}
+
+impl RhiStateMaintenanceErrorKind {
+ /// Returns the stable machine-readable failure code.
+ #[must_use]
+ pub const fn code(self) -> &'static str {
+ match self {
+ Self::InvalidEvidence => "state_maintenance_evidence_invalid",
+ Self::InvalidMode => "state_maintenance_mode_invalid",
+ Self::Catalog => "state_maintenance_catalog_invalid",
+ Self::Authority => "state_maintenance_authority_failed",
+ Self::Open => "state_maintenance_open_failed",
+ Self::Metadata => "state_maintenance_metadata_invalid",
+ Self::Migration => "state_maintenance_migration_invalid",
+ Self::Backup => "state_backup_failed",
+ Self::Restore => "state_restore_failed",
+ Self::Integrity => "state_integrity_failed",
+ Self::Recovery => "state_recovery_failed",
+ }
+ }
+}
+
+/// Redacted RHI state-maintenance failure.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub struct RhiStateMaintenanceError {
+ kind: RhiStateMaintenanceErrorKind,
+}
+
+impl RhiStateMaintenanceError {
+ pub(crate) const fn new(kind: RhiStateMaintenanceErrorKind) -> Self {
+ Self { kind }
+ }
+
+ pub(crate) fn from_sqlite(error: ServiceSqliteError) -> Self {
+ let kind = match error.kind() {
+ ServiceSqliteErrorKind::Authority => RhiStateMaintenanceErrorKind::Authority,
+ ServiceSqliteErrorKind::Open
+ | ServiceSqliteErrorKind::Create
+ | ServiceSqliteErrorKind::Pragma => RhiStateMaintenanceErrorKind::Open,
+ ServiceSqliteErrorKind::Metadata => RhiStateMaintenanceErrorKind::Metadata,
+ ServiceSqliteErrorKind::Migration => RhiStateMaintenanceErrorKind::Migration,
+ ServiceSqliteErrorKind::Backup => RhiStateMaintenanceErrorKind::Backup,
+ ServiceSqliteErrorKind::Restore => RhiStateMaintenanceErrorKind::Restore,
+ ServiceSqliteErrorKind::Integrity => RhiStateMaintenanceErrorKind::Integrity,
+ ServiceSqliteErrorKind::Recovery => RhiStateMaintenanceErrorKind::Recovery,
+ };
+ Self::new(kind)
+ }
+
+ /// Returns the stable failure class.
+ #[must_use]
+ pub const fn kind(self) -> RhiStateMaintenanceErrorKind {
+ self.kind
+ }
+
+ /// Returns the stable machine-readable failure code.
+ #[must_use]
+ pub const fn code(self) -> &'static str {
+ self.kind.code()
+ }
+}
+
+impl fmt::Display for RhiStateMaintenanceError {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter.write_str(match self.kind {
+ RhiStateMaintenanceErrorKind::InvalidEvidence => {
+ "RHI state maintenance evidence is invalid"
+ }
+ RhiStateMaintenanceErrorKind::InvalidMode => {
+ "RHI state maintenance is unavailable in this host mode"
+ }
+ RhiStateMaintenanceErrorKind::Catalog => "RHI state catalogs are invalid",
+ RhiStateMaintenanceErrorKind::Authority => {
+ "RHI state maintenance authority could not be established"
+ }
+ RhiStateMaintenanceErrorKind::Open => "RHI state maintenance could not open state",
+ RhiStateMaintenanceErrorKind::Metadata => "RHI state metadata is invalid",
+ RhiStateMaintenanceErrorKind::Migration => "RHI state migration history is invalid",
+ RhiStateMaintenanceErrorKind::Backup => "RHI state backup failed",
+ RhiStateMaintenanceErrorKind::Restore => "RHI state restore failed",
+ RhiStateMaintenanceErrorKind::Integrity => "RHI state integrity check failed",
+ RhiStateMaintenanceErrorKind::Recovery => "RHI state recovery failed",
+ })
+ }
+}
+
+impl fmt::Debug for RhiStateMaintenanceError {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter
+ .debug_struct("RhiStateMaintenanceError")
+ .field("kind", &self.kind)
+ .finish()
+ }
+}
+
+impl Error for RhiStateMaintenanceError {}
+
+/// Retained exact-inode proof of one verified RHI backup.
+///
+/// Construction is sealed to [`verify_rhi_state_backup`]. No raw descriptor or
+/// pathname is exposed.
+///
+/// ```compile_fail
+/// use rhi::RhiVerifiedStateBackup;
+/// let _ = RhiVerifiedStateBackup { inner: todo!() };
+/// ```
+pub struct RhiVerifiedStateBackup {
+ inner: VerifiedServiceBackup,
+}
+
+impl RhiVerifiedStateBackup {
+ /// Returns the admitted canonical manifest.
+ #[must_use]
+ pub const fn manifest(&self) -> &ServiceBackupManifest {
+ self.inner.manifest()
+ }
+
+ /// Returns the actual immutable database metadata read from the retained member.
+ #[must_use]
+ pub const fn database_metadata(&self) -> &ServiceDatabaseMetadata {
+ self.inner.database_metadata()
+ }
+}
+
+impl fmt::Debug for RhiVerifiedStateBackup {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter.write_str("RhiVerifiedStateBackup([redacted])")
+ }
+}
+
+/// Offline staged RHI replacement that retains exclusive writer authority.
+///
+/// Construction is sealed to [`stage_rhi_state_restore`]. Dropping this value
+/// preserves the shared exact-inode cleanup and fail-closed evidence contract.
+///
+/// ```compile_fail
+/// use rhi::RhiStagedStateRestore;
+/// let _ = RhiStagedStateRestore { inner: todo!() };
+/// ```
+pub struct RhiStagedStateRestore {
+ inner: StagedServiceRestore,
+}
+
+impl fmt::Debug for RhiStagedStateRestore {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter.write_str("RhiStagedStateRestore([redacted])")
+ }
+}
+
+/// Verifies an untrusted backup bundle against one sealed RHI state identity.
+pub fn verify_rhi_state_backup(
+ manifest_bytes: &[u8],
+ expected_manifest_digest: BackupManifestSha256,
+ bundle_directory: &Path,
+ expected: &RhiStateMetadata,
+ maximum_state_bytes: NonZeroU64,
+) -> Result<RhiVerifiedStateBackup, RhiStateMaintenanceError> {
+ verify_backup_bundle(
+ manifest_bytes,
+ expected_manifest_digest,
+ bundle_directory,
+ &expected.database_identity(),
+ maximum_state_bytes,
+ )
+ .map(|inner| RhiVerifiedStateBackup { inner })
+ .map_err(RhiStateMaintenanceError::from_sqlite)
+}
+
+/// Copies and completely reverifies a verified backup beside closed RHI state.
+///
+/// This operation acquires exclusive writer authority. It never creates a
+/// recovery marker or replaces the live database.
+pub async fn stage_rhi_state_restore(
+ runtime: &RhiRuntimeContext,
+ expected: &RhiStateMetadata,
+ verified: RhiVerifiedStateBackup,
+) -> Result<RhiStagedStateRestore, RhiStateMaintenanceError> {
+ state_host::require_metadata(runtime, expected).map_err(|_| {
+ RhiStateMaintenanceError::new(RhiStateMaintenanceErrorKind::InvalidEvidence)
+ })?;
+ let paths = state_host::state_paths(runtime).map_err(|_| {
+ RhiStateMaintenanceError::new(RhiStateMaintenanceErrorKind::InvalidEvidence)
+ })?;
+ let (migrations, schema) = state_host::catalogs()
+ .map_err(|_| RhiStateMaintenanceError::new(RhiStateMaintenanceErrorKind::Catalog))?;
+ stage_verified_restore(
+ &paths,
+ &expected.database_identity(),
+ &migrations,
+ &schema,
+ verified.inner,
+ )
+ .await
+ .map(|inner| RhiStagedStateRestore { inner })
+ .map_err(RhiStateMaintenanceError::from_sqlite)
+}
+
+/// Atomically installs a completely verified staged RHI restore.
+///
+/// Success intentionally returns no open host. The next writable open owns
+/// exact recovery-evidence reconciliation before SQLite is exposed again.
+pub async fn finalize_rhi_state_restore(
+ staged: RhiStagedStateRestore,
+) -> Result<(), RhiStateMaintenanceError> {
+ finalize_staged_restore(staged.inner)
+ .await
+ .map_err(RhiStateMaintenanceError::from_sqlite)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn shared_failures_map_to_the_closed_source_free_rhi_vocabulary() {
+ for (source, expected) in [
+ (
+ ServiceSqliteErrorKind::Authority,
+ RhiStateMaintenanceErrorKind::Authority,
+ ),
+ (
+ ServiceSqliteErrorKind::Open,
+ RhiStateMaintenanceErrorKind::Open,
+ ),
+ (
+ ServiceSqliteErrorKind::Create,
+ RhiStateMaintenanceErrorKind::Open,
+ ),
+ (
+ ServiceSqliteErrorKind::Pragma,
+ RhiStateMaintenanceErrorKind::Open,
+ ),
+ (
+ ServiceSqliteErrorKind::Metadata,
+ RhiStateMaintenanceErrorKind::Metadata,
+ ),
+ (
+ ServiceSqliteErrorKind::Migration,
+ RhiStateMaintenanceErrorKind::Migration,
+ ),
+ (
+ ServiceSqliteErrorKind::Backup,
+ RhiStateMaintenanceErrorKind::Backup,
+ ),
+ (
+ ServiceSqliteErrorKind::Restore,
+ RhiStateMaintenanceErrorKind::Restore,
+ ),
+ (
+ ServiceSqliteErrorKind::Integrity,
+ RhiStateMaintenanceErrorKind::Integrity,
+ ),
+ (
+ ServiceSqliteErrorKind::Recovery,
+ RhiStateMaintenanceErrorKind::Recovery,
+ ),
+ ] {
+ let mapped = RhiStateMaintenanceError::from_sqlite(ServiceSqliteError::with_source(
+ source,
+ SensitiveSource,
+ ));
+ assert_eq!(mapped.kind(), expected);
+ assert!(Error::source(&mapped).is_none());
+ let rendered = format!("{mapped} {mapped:?}");
+ assert!(!rendered.contains("sensitive"));
+ assert!(!mapped.code().is_empty());
+ }
+ }
+
+ #[derive(Debug)]
+ struct SensitiveSource;
+
+ impl fmt::Display for SensitiveSource {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter.write_str("sensitive /tmp/state.sqlite")
+ }
+ }
+
+ impl Error for SensitiveSource {}
+}
diff --git a/tests/services_hardening_state_host.rs b/tests/services_hardening_state_host.rs
@@ -84,7 +84,8 @@ async fn initialize_is_create_new_and_both_existing_open_modes_close_explicitly(
let lock = runtime.artifacts().state_lock();
assert!(!state.exists());
- initialize_rhi_state(&runtime, &metadata)
+ let (applied_at, build) = migration_evidence();
+ initialize_rhi_state(&runtime, &metadata, applied_at, &build)
.await
.expect("create-new initialization");
assert!(state.is_file());
@@ -98,12 +99,11 @@ async fn initialize_is_create_new_and_both_existing_open_modes_close_explicitly(
0o600
);
- let duplicate = initialize_rhi_state(&runtime, &metadata)
+ let duplicate = initialize_rhi_state(&runtime, &metadata, applied_at, &build)
.await
.expect_err("second initialization must fail");
assert_eq!(duplicate.kind(), RhiStateHostErrorKind::Initialize);
- let (applied_at, build) = migration_evidence();
let writer = open_rhi_state_read_write(&runtime, &metadata, applied_at, &build)
.await
.expect("existing writable state");
@@ -224,7 +224,27 @@ async fn missing_state_and_mismatched_evidence_fail_before_database_creation() {
assert_eq!(missing.kind(), RhiStateHostErrorKind::ReadWriteOpen);
assert!(!primary.artifacts().state_database().exists());
- let mismatch = initialize_rhi_state(&secondary, &primary_metadata)
+ let invalid_build = MigrationBuildIdentity::new(
+ env!("CARGO_PKG_VERSION"),
+ "1111111111111111111111111111111111111111",
+ "7d7b454b4c9ed86569671993bd03ca868b676665",
+ "rustc-test",
+ "test-target",
+ "service-host",
+ 2,
+ 1,
+ 1,
+ 1,
+ 1,
+ )
+ .expect("structurally valid mismatched build");
+ let invalid = initialize_rhi_state(&primary, &primary_metadata, applied_at, &invalid_build)
+ .await
+ .expect_err("migration build must match RHI policy before I/O");
+ assert_eq!(invalid.kind(), RhiStateHostErrorKind::InvalidEvidence);
+ assert!(!primary.artifacts().state_database().exists());
+
+ let mismatch = initialize_rhi_state(&secondary, &primary_metadata, applied_at, &build)
.await
.expect_err("cross-instance metadata");
assert_eq!(mismatch.kind(), RhiStateHostErrorKind::InvalidEvidence);
diff --git a/tests/services_hardening_state_resilience.rs b/tests/services_hardening_state_resilience.rs
@@ -0,0 +1,363 @@
+#![forbid(unsafe_code)]
+#![cfg(any(target_os = "linux", target_os = "macos"))]
+
+use std::{
+ error::Error,
+ fs,
+ num::NonZeroU64,
+ os::unix::fs::{MetadataExt, PermissionsExt},
+ path::{Path, PathBuf},
+};
+
+use radroots_service_sqlite::{
+ BackupCreatedAtUnixMs, IntegrityCheckOutcome, IntegrityCheckedAtUnixMs,
+ MigrationAppliedAtUnixSeconds, MigrationBuildIdentity,
+};
+use radroots_storage::event::SourceGeneration;
+use rhi::{
+ RHI_STATE_SCHEMA_VERSION, RadrootsHostEnvironment, RadrootsPathResolver, RadrootsPlatform,
+ RhiConfigProfile, RhiStateHostErrorKind, RhiStateMaintenanceErrorKind, RhiStateMetadata,
+ RhiStateRepositoryKind, finalize_rhi_state_restore, initialize_rhi_state,
+ open_rhi_state_inspection, open_rhi_state_read_write, parse_rhi_cli_v1_from,
+ parse_rhi_config_v1, resolve_rhi_runtime_context, stage_rhi_state_restore,
+ verify_rhi_state_backup,
+};
+use sqlx::{ConnectOptions, Connection, SqliteConnection, sqlite::SqliteConnectOptions};
+
+const CONFIG_EXAMPLE: &[u8] =
+ include_bytes!("../contracts/services_hardening/config.v1.example.toml");
+const LIB_SOURCE: &str = include_str!("../src/lib.rs");
+const HOST_SOURCE: &str = include_str!("../src/state_host.rs");
+const MAINTENANCE_SOURCE: &str = include_str!("../src/state_maintenance.rs");
+
+fn runtime(root: &Path, instance: &str) -> rhi::RhiRuntimeContext {
+ let invocation = parse_rhi_cli_v1_from([
+ "rhi",
+ "--profile",
+ "repo-local",
+ "--instance",
+ instance,
+ "--repo-local-root",
+ root.to_str().expect("UTF-8 temporary root"),
+ "run",
+ ])
+ .expect("valid invocation");
+ resolve_rhi_runtime_context(
+ &RadrootsPathResolver::new(RadrootsPlatform::Linux, RadrootsHostEnvironment::default()),
+ &invocation,
+ )
+ .expect("runtime context")
+}
+
+fn prepare_state_directory(runtime: &rhi::RhiRuntimeContext) {
+ let directory = runtime.context().paths().state();
+ fs::create_dir_all(directory).expect("state directory");
+ fs::set_permissions(directory, fs::Permissions::from_mode(0o700)).expect("state mode");
+}
+
+fn metadata(runtime: &rhi::RhiRuntimeContext) -> RhiStateMetadata {
+ let configuration =
+ parse_rhi_config_v1(CONFIG_EXAMPLE, RhiConfigProfile::RepoLocal).expect("configuration");
+ RhiStateMetadata::new(
+ runtime,
+ &configuration,
+ SourceGeneration::new([0x5a; 32]).expect("generation"),
+ 1_725_000_000_000,
+ )
+ .expect("metadata")
+}
+
+fn migration_evidence() -> (MigrationAppliedAtUnixSeconds, MigrationBuildIdentity) {
+ let applied_at = MigrationAppliedAtUnixSeconds::new(1_725_000_000).expect("migration time");
+ let build = MigrationBuildIdentity::new(
+ env!("CARGO_PKG_VERSION"),
+ "1111111111111111111111111111111111111111",
+ "7d7b454b4c9ed86569671993bd03ca868b676665",
+ "rustc-test",
+ "test-target",
+ "service-host",
+ 1,
+ RHI_STATE_SCHEMA_VERSION,
+ 1,
+ 1,
+ 1,
+ )
+ .expect("build identity");
+ (applied_at, build)
+}
+
+fn recovery_paths(runtime: &rhi::RhiRuntimeContext) -> [PathBuf; 4] {
+ let state = runtime.context().paths().state();
+ [
+ state.join("state.restore-staged.sqlite"),
+ state.join("state.restore-backup.sqlite"),
+ state.join("state.restore-marker.v1"),
+ state.join("state.restore-marker.v1.next"),
+ ]
+}
+
+#[tokio::test]
+async fn backup_integrity_and_offline_restore_obey_one_exact_rhi_authority() {
+ let directory = tempfile::tempdir().expect("temporary root");
+ let runtime = runtime(directory.path(), "primary");
+ prepare_state_directory(&runtime);
+ let metadata = metadata(&runtime);
+ let (applied_at, build) = migration_evidence();
+ initialize_rhi_state(&runtime, &metadata, applied_at, &build)
+ .await
+ .expect("initialization");
+
+ let writer = open_rhi_state_read_write(&runtime, &metadata, applied_at, &build)
+ .await
+ .expect("writable host");
+ let report = writer
+ .inspect_integrity(
+ IntegrityCheckedAtUnixMs::new(1_725_000_000_100).expect("inspection time"),
+ )
+ .await
+ .expect("writable integrity inspection");
+ assert_eq!(report.sqlite(), IntegrityCheckOutcome::Verified);
+ assert_eq!(report.foreign_keys(), IntegrityCheckOutcome::Verified);
+ assert!(report.diagnostics().is_empty());
+
+ let bundle = directory.path().join("backup");
+ let manifest = writer
+ .capture_online_backup(
+ &bundle,
+ BackupCreatedAtUnixMs::new(1_725_000_000_200).expect("capture time"),
+ )
+ .await
+ .expect("online backup");
+ assert_eq!(manifest.service().as_str(), "rhi");
+ assert_eq!(manifest.instance().as_str(), "primary");
+ assert_eq!(
+ manifest.state_schema_version().get(),
+ RHI_STATE_SCHEMA_VERSION
+ );
+ assert!(!manifest.protected_material_included());
+ assert_eq!(manifest.members().len(), 1);
+ assert_eq!(manifest.members()[0].name(), "state.sqlite");
+ let entries = fs::read_dir(&bundle)
+ .expect("backup directory")
+ .map(|entry| entry.expect("entry").file_name())
+ .collect::<Vec<_>>();
+ assert_eq!(entries, ["state.sqlite"]);
+ let manifest_bytes = manifest.canonical_bytes().to_vec();
+ let manifest_digest = manifest.digest();
+ let maximum_state_bytes =
+ NonZeroU64::new(manifest.members()[0].byte_length()).expect("member length");
+ writer.close().await.expect("writer close");
+
+ let live_path = runtime.artifacts().state_database();
+ let old_live_inode = fs::metadata(live_path).expect("live metadata").ino();
+ let inspection = open_rhi_state_inspection(&runtime, &metadata)
+ .await
+ .expect("read-only inspection");
+ let inspection_report = inspection
+ .inspect_integrity(
+ IntegrityCheckedAtUnixMs::new(1_725_000_000_300).expect("inspection time"),
+ )
+ .await
+ .expect("read-only integrity inspection");
+ assert_eq!(inspection_report.sqlite(), IntegrityCheckOutcome::Verified);
+ assert_eq!(
+ inspection_report.foreign_keys(),
+ IntegrityCheckOutcome::Verified
+ );
+ let forbidden_bundle = directory.path().join("inspection-backup");
+ let error = inspection
+ .capture_online_backup(
+ &forbidden_bundle,
+ BackupCreatedAtUnixMs::new(1_725_000_000_400).expect("capture time"),
+ )
+ .await
+ .expect_err("read-only capture");
+ assert_eq!(error.kind(), RhiStateMaintenanceErrorKind::InvalidMode);
+ assert!(!forbidden_bundle.exists());
+
+ let verified = verify_rhi_state_backup(
+ &manifest_bytes,
+ manifest_digest,
+ &bundle,
+ &metadata,
+ maximum_state_bytes,
+ )
+ .expect("verified retained backup");
+ let contended = stage_rhi_state_restore(&runtime, &metadata, verified)
+ .await
+ .expect_err("offline staging rejects a live inspection host");
+ assert_eq!(contended.kind(), RhiStateMaintenanceErrorKind::Authority);
+ inspection.close().await.expect("inspection close");
+
+ let verified = verify_rhi_state_backup(
+ &manifest_bytes,
+ manifest_digest,
+ &bundle,
+ &metadata,
+ maximum_state_bytes,
+ )
+ .expect("reverified backup for runtime mismatch");
+ let secondary = self::runtime(directory.path(), "secondary");
+ let mismatch = stage_rhi_state_restore(&secondary, &metadata, verified)
+ .await
+ .expect_err("runtime and metadata remain cross-bound");
+ assert_eq!(
+ mismatch.kind(),
+ RhiStateMaintenanceErrorKind::InvalidEvidence
+ );
+ assert!(!secondary.artifacts().state_database().exists());
+ assert!(recovery_paths(&secondary).iter().all(|path| !path.exists()));
+
+ let verified = verify_rhi_state_backup(
+ &manifest_bytes,
+ manifest_digest,
+ &bundle,
+ &metadata,
+ maximum_state_bytes,
+ )
+ .expect("reverified backup");
+ assert_eq!(
+ format!("{verified:?}"),
+ "RhiVerifiedStateBackup([redacted])"
+ );
+ assert_eq!(
+ verified.database_metadata().state_schema_version().get(),
+ RHI_STATE_SCHEMA_VERSION
+ );
+ let staged = stage_rhi_state_restore(&runtime, &metadata, verified)
+ .await
+ .expect("offline staging");
+ assert_eq!(format!("{staged:?}"), "RhiStagedStateRestore([redacted])");
+ finalize_rhi_state_restore(staged)
+ .await
+ .expect("atomic finalization");
+
+ let unavailable = open_rhi_state_inspection(&runtime, &metadata)
+ .await
+ .expect_err("inspection never performs restore recovery");
+ assert_eq!(unavailable.kind(), RhiStateHostErrorKind::InspectionOpen);
+ let recovered = open_rhi_state_read_write(&runtime, &metadata, applied_at, &build)
+ .await
+ .expect("writable open reconciles exact recovery evidence");
+ assert_eq!(
+ recovered.repositories().sources().kind(),
+ RhiStateRepositoryKind::Source
+ );
+ recovered.close().await.expect("recovered writer close");
+ assert_ne!(
+ fs::metadata(live_path)
+ .expect("recovered live metadata")
+ .ino(),
+ old_live_inode
+ );
+ for path in recovery_paths(&runtime) {
+ assert!(!path.exists(), "recovery evidence must be retired");
+ }
+}
+
+#[tokio::test]
+async fn exact_open_rejects_unexpected_migration_history_without_repair() {
+ let directory = tempfile::tempdir().expect("temporary root");
+ let runtime = runtime(directory.path(), "primary");
+ prepare_state_directory(&runtime);
+ let metadata = metadata(&runtime);
+ let (applied_at, build) = migration_evidence();
+ initialize_rhi_state(&runtime, &metadata, applied_at, &build)
+ .await
+ .expect("initialization");
+
+ let options = SqliteConnectOptions::new()
+ .filename(runtime.artifacts().state_database())
+ .create_if_missing(false)
+ .disable_statement_logging();
+ let mut connection = SqliteConnection::connect_with(&options)
+ .await
+ .expect("test-only offline connection");
+ sqlx::query(
+ "INSERT INTO schema_migrations (
+ version, name, checksum, applied_at_unix_s,
+ service_version, service_commit, lib_revision, rust_version, target,
+ feature_profile, config_contract_version, state_contract_version,
+ admin_contract_version, status_contract_version, provider_contract_version
+ ) VALUES (2, 'unexpected_schema', ?, 1725000000, '0.1.0', ?, ?,
+ 'rustc-test', 'test-target', 'service-host', 1, 1, 1, 1, 1)",
+ )
+ .bind([0x44_u8; 32].as_slice())
+ .bind("1111111111111111111111111111111111111111")
+ .bind("7d7b454b4c9ed86569671993bd03ca868b676665")
+ .execute(&mut connection)
+ .await
+ .expect("insert unexpected ledger row");
+ connection.close().await.expect("test connection close");
+
+ let error = open_rhi_state_read_write(&runtime, &metadata, applied_at, &build)
+ .await
+ .expect_err("migration drift must fail closed");
+ assert_eq!(error.kind(), RhiStateHostErrorKind::ReadWriteOpen);
+ let inspection = open_rhi_state_inspection(&runtime, &metadata)
+ .await
+ .expect_err("inspection rejects migration drift");
+ assert_eq!(inspection.kind(), RhiStateHostErrorKind::InspectionOpen);
+}
+
+#[test]
+fn maintenance_boundary_is_sealed_source_free_and_sqlx_owned() {
+ assert!(LIB_SOURCE.contains("mod state_maintenance;"));
+ assert!(!LIB_SOURCE.contains("pub mod state_maintenance;"));
+ assert!(HOST_SOURCE.contains(".capture_online_backup(staging_directory, created_at)"));
+ assert!(HOST_SOURCE.contains(".inspect_integrity(checked_at)"));
+ assert!(MAINTENANCE_SOURCE.contains("verify_backup_bundle("));
+ assert!(MAINTENANCE_SOURCE.contains("stage_verified_restore("));
+ assert!(MAINTENANCE_SOURCE.contains("finalize_staged_restore("));
+ for forbidden in [
+ "sqlx::",
+ "SqliteConnection",
+ "SqlitePool",
+ "raw_sql",
+ "BEGIN ",
+ "COMMIT",
+ "ROLLBACK",
+ "std::fs",
+ "std::env",
+ "std::time",
+ "provider",
+ "relay",
+ "tokio::spawn",
+ "spawn_blocking",
+ ] {
+ assert!(
+ !MAINTENANCE_SOURCE.contains(forbidden),
+ "found forbidden maintenance authority `{forbidden}`"
+ );
+ }
+
+ for kind in [
+ RhiStateMaintenanceErrorKind::InvalidEvidence,
+ RhiStateMaintenanceErrorKind::InvalidMode,
+ RhiStateMaintenanceErrorKind::Catalog,
+ RhiStateMaintenanceErrorKind::Authority,
+ RhiStateMaintenanceErrorKind::Open,
+ RhiStateMaintenanceErrorKind::Metadata,
+ RhiStateMaintenanceErrorKind::Migration,
+ RhiStateMaintenanceErrorKind::Backup,
+ RhiStateMaintenanceErrorKind::Restore,
+ RhiStateMaintenanceErrorKind::Integrity,
+ RhiStateMaintenanceErrorKind::Recovery,
+ ] {
+ assert!(!kind.code().is_empty());
+ }
+
+ let error = verify_rhi_state_backup(
+ b"/tmp/secret-state.sqlite",
+ radroots_service_sqlite::BackupManifestSha256::from_bytes([0x11; 32]),
+ Path::new("/tmp/secret-bundle"),
+ &metadata(&runtime(Path::new("/tmp/secret-root"), "primary")),
+ NonZeroU64::new(1).expect("limit"),
+ )
+ .expect_err("invalid manifest");
+ assert!(Error::source(&error).is_none());
+ let rendered = format!("{error} {error:?}");
+ assert!(!rendered.contains("secret"));
+ assert!(!rendered.contains("/tmp"));
+ assert!(!rendered.contains("sqlite"));
+}
diff --git a/tests/services_hardening_wave_100_b.rs b/tests/services_hardening_wave_100_b.rs
@@ -24,6 +24,7 @@ const CONFIG_SOURCE: &str = include_str!("../src/config_v1.rs");
const CREDENTIAL_SOURCE: &str = include_str!("../src/identity_credential.rs");
const ENVELOPE_SOURCE: &str = include_str!("../src/identity_envelope.rs");
const STATE_HOST_SOURCE: &str = include_str!("../src/state_host.rs");
+const STATE_MAINTENANCE_SOURCE: &str = include_str!("../src/state_maintenance.rs");
fn digest(label: &str) -> [u8; 32] {
Sha256::digest(label.as_bytes()).into()
@@ -152,10 +153,10 @@ async fn wave_two_composes_one_runtime_without_crossing_secret_or_state_authorit
assert_eq!(opened.public_identity().as_hex(), expected_identity);
prepare_secure_directory(runtime.context().paths().state());
- initialize_rhi_state(&runtime, &metadata)
+ let (applied_at, build) = migration_evidence();
+ initialize_rhi_state(&runtime, &metadata, applied_at, &build)
.await
.expect("create-new state initialization");
- let (applied_at, build) = migration_evidence();
let state = open_rhi_state_read_write(&runtime, &metadata, applied_at, &build)
.await
.expect("existing state open");
@@ -201,7 +202,7 @@ async fn wave_two_composes_one_runtime_without_crossing_secret_or_state_authorit
}
#[test]
-fn wave_two_contracts_freeze_backup_exclusion_without_claiming_backup_execution() {
+fn wave_two_contracts_freeze_backup_exclusion_at_the_sealed_maintenance_boundary() {
let envelope: serde_json::Value =
serde_json::from_str(ENVELOPE_CONTRACT).expect("envelope contract");
let credential: serde_json::Value =
@@ -216,8 +217,9 @@ fn wave_two_contracts_freeze_backup_exclusion_without_claiming_backup_execution(
false
);
assert_eq!(credential["backup_included"], false);
- assert!(!STATE_HOST_SOURCE.contains("capture_online_backup"));
+ assert!(STATE_HOST_SOURCE.contains("capture_online_backup"));
assert!(!STATE_HOST_SOURCE.contains("verify_backup_bundle"));
+ assert!(STATE_MAINTENANCE_SOURCE.contains("verify_backup_bundle"));
assert!(!STATE_HOST_SOURCE.contains("finalize_staged_restore"));
}
