commit fce00896ca3f87a14ea57f46e628c429167d4f2c
parent e1a59474f8770f5e46c6de1fd8ee351d45137011
Author: triesap <tyson@radroots.org>
Date: Tue, 25 Aug 2026 11:40:11 +0000
test(service-sqlite): qualify storage exhaustion
- Inject semantic storage-full failures at backup durability boundaries.
- Preserve exact cleanup, authority, and retry behavior after exhaustion.
- Exercise restore staging, marker, and finalization sync failures.
- Keep the qualification seam private with no public API change.
Diffstat:
6 files changed, 40 insertions(+), 16 deletions(-)
diff --git a/crates/service_sqlite/src/backup/capture.rs b/crates/service_sqlite/src/backup/capture.rs
@@ -144,7 +144,7 @@ struct FailingCaptureOperations {
impl CaptureOperations for FailingCaptureOperations {
fn sync_state(&self, state: &File) -> io::Result<()> {
if self.failure == TestCaptureSyncFailure::State {
- Err(io::Error::other("injected state sync failure"))
+ Err(crate::failpoint::storage_full_error())
} else {
state.sync_all()
}
@@ -152,7 +152,7 @@ impl CaptureOperations for FailingCaptureOperations {
fn sync_staging(&self, staging: &File) -> io::Result<()> {
if self.failure == TestCaptureSyncFailure::Staging {
- Err(io::Error::other("injected staging sync failure"))
+ Err(crate::failpoint::storage_full_error())
} else {
staging.sync_all()
}
@@ -161,7 +161,7 @@ impl CaptureOperations for FailingCaptureOperations {
fn sync_parent(&self, parent: &File) -> io::Result<()> {
let occurrence = self.parent_syncs.fetch_add(1, Ordering::AcqRel);
if self.failure == TestCaptureSyncFailure::FinalParent && occurrence == 1 {
- Err(io::Error::other("injected parent sync failure"))
+ Err(crate::failpoint::storage_full_error())
} else {
parent.sync_all()
}
diff --git a/crates/service_sqlite/src/connection.rs b/crates/service_sqlite/src/connection.rs
@@ -2595,7 +2595,7 @@ mod tests {
#[cfg(any(target_os = "linux", target_os = "macos"))]
#[tokio::test]
- async fn backup_sync_failures_cleanup_and_leave_host_recoverable() {
+ async fn backup_storage_full_sync_failures_cleanup_and_leave_host_recoverable() {
let _serial = CAPTURE_TEST_LOCK.lock().await;
crate::backup::test_capture_reset();
let (root, _paths, _identity, _migrations, _schema, host) = initialized_host().await;
@@ -2625,6 +2625,12 @@ mod tests {
.await
.expect_err("injected synchronization failure must reject capture");
assert_eq!(error.kind(), ServiceSqliteErrorKind::Backup);
+ let storage = error
+ .source()
+ .and_then(Error::source)
+ .and_then(|source| source.downcast_ref::<std::io::Error>())
+ .expect("storage-full cause");
+ assert_eq!(storage.kind(), std::io::ErrorKind::StorageFull);
assert!(!stage.exists());
assert!(!host.backup_active.load(Ordering::Acquire));
assert_eq!(row_count(&host).await, 0);
diff --git a/crates/service_sqlite/src/failpoint.rs b/crates/service_sqlite/src/failpoint.rs
@@ -11,6 +11,11 @@ use std::{
#[cfg(test)]
const PROCESS_BARRIER_READY: &[u8] = b"\nRSHR_STEP073_READY\n";
+#[cfg(test)]
+pub(crate) fn storage_full_error() -> io::Error {
+ io::Error::from(io::ErrorKind::StorageFull)
+}
+
/// Closed inventory of durability edges exercised by the crash-boundary harness.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum DurabilityFailpoint {
@@ -364,6 +369,11 @@ mod tests {
use std::thread;
#[test]
+ fn storage_full_injection_uses_the_semantic_io_kind() {
+ assert_eq!(storage_full_error().kind(), io::ErrorKind::StorageFull);
+ }
+
+ #[test]
fn every_closed_point_fires_once_on_its_owned_plan() {
for point in DurabilityFailpoint::ALL {
let plan = DurabilityFailpoints::armed(point);
diff --git a/crates/service_sqlite/src/restore/finalize.rs b/crates/service_sqlite/src/restore/finalize.rs
@@ -674,7 +674,7 @@ impl FinalizeOperations for FailingFinalizeOperations {
RenameStep::InstallStage => 7,
};
if failure == target {
- return Err(std::io::Error::other("injected directory sync failure"));
+ return Err(crate::failpoint::storage_full_error());
}
SystemFinalizeOperations.sync_directory(directory, step)
}
diff --git a/crates/service_sqlite/src/restore/marker.rs b/crates/service_sqlite/src/restore/marker.rs
@@ -1537,25 +1537,21 @@ mod store {
impl StoreOperations for FailingStoreOperations {
fn sync_file(&self, file: &File, directory: &File) -> std::io::Result<()> {
match self.failure {
- TestStoreFailure::ScratchSync => {
- Err(std::io::Error::other("injected scratch sync failure"))
- }
+ TestStoreFailure::ScratchSync => Err(crate::failpoint::storage_full_error()),
TestStoreFailure::AuthorityDriftAndScratchSync => {
fchmod(
directory,
Mode::RUSR | Mode::WUSR | Mode::XUSR | Mode::RGRP | Mode::WGRP | Mode::XGRP,
)
.map_err(std::io::Error::from)?;
- Err(std::io::Error::other(
- "injected authority drift and scratch sync failure",
- ))
+ Err(crate::failpoint::storage_full_error())
}
TestStoreFailure::ParentSyncAfterRename => file.sync_all(),
}
}
fn sync_directory(&self, _directory: &File) -> std::io::Result<()> {
- Err(std::io::Error::other("injected parent sync failure"))
+ Err(crate::failpoint::storage_full_error())
}
fn replace_marker(&self, directory: &File) -> std::io::Result<()> {
@@ -2168,7 +2164,7 @@ mod tests {
#[cfg(any(target_os = "linux", target_os = "macos"))]
#[test]
- fn atomic_advance_failures_leave_exact_old_or_new_valid_marker() {
+ fn atomic_advance_storage_full_failures_leave_exact_old_or_new_valid_marker() {
use super::store::TestStoreFailure;
use std::{fs, os::unix::fs::PermissionsExt};
diff --git a/crates/service_sqlite/src/restore/stage.rs b/crates/service_sqlite/src/restore/stage.rs
@@ -1096,7 +1096,10 @@ fn sync_staged_file(file: &File, occurrence: u8) -> Result<(), ServiceSqliteErro
.compare_exchange(occurrence, 0, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
- return Err(restore_error(RestoreFailureKind::SyncStaged));
+ return Err(restore_source(
+ RestoreFailureKind::SyncStaged,
+ crate::failpoint::storage_full_error(),
+ ));
}
#[cfg(not(test))]
let _ = occurrence;
@@ -1111,7 +1114,10 @@ fn sync_stage_directory(directory: &File) -> Result<(), ServiceSqliteError> {
.compare_exchange(3, 0, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
- return Err(restore_error(RestoreFailureKind::SyncDirectory));
+ return Err(restore_source(
+ RestoreFailureKind::SyncDirectory,
+ crate::failpoint::storage_full_error(),
+ ));
}
directory
.sync_all()
@@ -2291,7 +2297,7 @@ mod tests {
}
#[tokio::test(flavor = "current_thread")]
- async fn sync_and_join_failures_cleanup_and_allow_retry() {
+ async fn storage_full_sync_and_join_failures_cleanup_and_allow_retry() {
let _serial = STAGE_TEST_LOCK.lock().await;
for failure in [1, 2, 3] {
reset_test_controls();
@@ -2307,6 +2313,12 @@ mod tests {
.await
.expect_err("sync failure");
assert_eq!(error.kind(), ServiceSqliteErrorKind::Restore);
+ let storage = error
+ .source()
+ .and_then(Error::source)
+ .and_then(|source| source.downcast_ref::<std::io::Error>())
+ .expect("storage-full cause");
+ assert_eq!(storage.kind(), std::io::ErrorKind::StorageFull);
wait_for_cleanup(&fixture.paths, &fixture.staged_path()).await;
}
