commit 6a0c73088160a4aec68cb84036fbc2eb859dda52
parent c6c83d2f2000d9bc0b93dab71ebd6a941ee8f7df
Author: triesap <tyson@radroots.org>
Date: Sat, 10 Oct 2026 07:42:10 +0000
storage: bound projection cache and retire obsolete generations
- Account legacy, active and staged values under transactional limits
- Retire exact derived generations and collect at most 64 rows per pass
- Fence snapshot expiry with owner time and validate lengths before hydration
- Preserve canonical work and qualify Memory and SQLite provider behavior
Diffstat:
17 files changed, 3914 insertions(+), 161 deletions(-)
diff --git a/contracts/api_baselines/radroots_storage.txt b/contracts/api_baselines/radroots_storage.txt
@@ -1081,6 +1081,8 @@ pub fn radroots_storage::memory::MemoryStorage::status(&self) -> radroots_transp
pub fn radroots_storage::memory::MemoryStorage::tombstone(&self, radroots_storage::private_artifact::PrivateArtifactId, radroots_storage::private_artifact::PrivateArtifactRevision, u64, radroots_storage::private_artifact::DeletionReason) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::private_artifact::PrivateArtifactMetadata, radroots_storage::Error>>
impl radroots_storage::projection::ProjectionStore for radroots_storage::memory::MemoryStorage
pub fn radroots_storage::memory::MemoryStorage::checkpoint(&self, radroots_storage::projection::ProjectionCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::collect_projection_cache(&self, radroots_storage::projection::cache::ProjectionCacheScope) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheCollection, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::configure_projection_cache(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::cache::ProjectionCacheLimits) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheUsage, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::event_index_checkpoint(&self, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::EventIndexCheckpoint>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::event_index_manifest(&self, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::EventIndexManifest>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::invalidate(&self, radroots_storage::projection::ProjectionInvalidation) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
@@ -1097,6 +1099,7 @@ pub fn radroots_storage::memory::MemoryStorage::query_projection_documents(&self
pub fn radroots_storage::memory::MemoryStorage::query_projection_documents_at(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery, radroots_storage::event::EventSourceToken, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::rebuild(&self, radroots_storage::projection::RebuildTicketId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::RebuildTicket>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::request_rebuild(&self, radroots_storage::projection::RebuildTicket) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::retire_projection_generation(&self, radroots_storage::projection::cache::ProjectionCacheScope) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheUsage, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::status(&self, radroots_storage::projection::ProjectionId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::transition_rebuild(&self, radroots_storage::projection::RebuildTransition) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
impl radroots_storage::status::StorageStatusProvider for radroots_storage::memory::MemoryStorage
@@ -1474,6 +1477,31 @@ pub const fn radroots_storage::projection::batch::ProjectionWriteScope::projecti
pub const fn radroots_storage::projection::batch::ProjectionWriteScope::source(&self) -> radroots_storage::event::EventSourceToken
pub const radroots_storage::projection::batch::PROJECTION_BATCH_BYTES_MAX: usize
pub const radroots_storage::projection::batch::PROJECTION_BATCH_ROWS_MAX: usize
+pub mod radroots_storage::projection::cache
+pub struct radroots_storage::projection::cache::ProjectionCacheCollection
+pub radroots_storage::projection::cache::ProjectionCacheCollection::more: bool
+pub radroots_storage::projection::cache::ProjectionCacheCollection::removed_rows: u64
+pub radroots_storage::projection::cache::ProjectionCacheCollection::removed_value_bytes: u64
+pub radroots_storage::projection::cache::ProjectionCacheCollection::usage: radroots_storage::projection::cache::ProjectionCacheUsage
+pub struct radroots_storage::projection::cache::ProjectionCacheLimits
+impl radroots_storage::projection::cache::ProjectionCacheLimits
+pub const fn radroots_storage::projection::cache::ProjectionCacheLimits::legacy_age_ms(self) -> u64
+pub const fn radroots_storage::projection::cache::ProjectionCacheLimits::legacy_snapshots(self) -> u64
+pub fn radroots_storage::projection::cache::ProjectionCacheLimits::new(u64, u64, u64, u64) -> core::result::Result<Self, radroots_storage::Error>
+pub const fn radroots_storage::projection::cache::ProjectionCacheLimits::protected_generations(self) -> u64
+pub const fn radroots_storage::projection::cache::ProjectionCacheLimits::value_bytes(self) -> u64
+pub struct radroots_storage::projection::cache::ProjectionCacheScope
+impl radroots_storage::projection::cache::ProjectionCacheScope
+pub const fn radroots_storage::projection::cache::ProjectionCacheScope::expected_active(&self) -> core::option::Option<radroots_storage::projection::ProjectionGeneration>
+pub const fn radroots_storage::projection::cache::ProjectionCacheScope::generation(&self) -> radroots_storage::projection::ProjectionGeneration
+pub fn radroots_storage::projection::cache::ProjectionCacheScope::new(radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, radroots_storage::event::EventSourceToken, core::option::Option<radroots_storage::projection::ProjectionGeneration>) -> core::result::Result<Self, radroots_storage::Error>
+pub const fn radroots_storage::projection::cache::ProjectionCacheScope::projection_id(&self) -> &radroots_storage::projection::ProjectionId
+pub const fn radroots_storage::projection::cache::ProjectionCacheScope::source(&self) -> radroots_storage::event::EventSourceToken
+pub struct radroots_storage::projection::cache::ProjectionCacheUsage
+pub radroots_storage::projection::cache::ProjectionCacheUsage::legacy_snapshots: u64
+pub radroots_storage::projection::cache::ProjectionCacheUsage::protected_generations: u64
+pub radroots_storage::projection::cache::ProjectionCacheUsage::value_bytes: u64
+pub const radroots_storage::projection::cache::PROJECTION_CACHE_COLLECTION_ROWS_MAX: usize
pub mod radroots_storage::projection::document_query
pub enum radroots_storage::projection::document_query::ProjectionDocumentGenerations
pub radroots_storage::projection::document_query::ProjectionDocumentGenerations::All
@@ -1667,6 +1695,8 @@ pub const radroots_storage::projection::PROJECTION_DOCUMENT_VALUE_MAX_BYTES: usi
pub const radroots_storage::projection::PROJECTION_ID_MAX_BYTES: usize
pub trait radroots_storage::projection::ProjectionStore: core::marker::Send + core::marker::Sync
pub fn radroots_storage::projection::ProjectionStore::checkpoint(&self, radroots_storage::projection::ProjectionCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
+pub fn radroots_storage::projection::ProjectionStore::collect_projection_cache(&self, radroots_storage::projection::cache::ProjectionCacheScope) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheCollection, radroots_storage::Error>>
+pub fn radroots_storage::projection::ProjectionStore::configure_projection_cache(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::cache::ProjectionCacheLimits) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheUsage, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::event_index_checkpoint(&self, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::EventIndexCheckpoint>, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::event_index_manifest(&self, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::EventIndexManifest>, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::invalidate(&self, radroots_storage::projection::ProjectionInvalidation) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
@@ -1683,10 +1713,13 @@ pub fn radroots_storage::projection::ProjectionStore::query_projection_documents
pub fn radroots_storage::projection::ProjectionStore::query_projection_documents_at(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery, radroots_storage::event::EventSourceToken, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::rebuild(&self, radroots_storage::projection::RebuildTicketId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::RebuildTicket>, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::request_rebuild(&self, radroots_storage::projection::RebuildTicket) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
+pub fn radroots_storage::projection::ProjectionStore::retire_projection_generation(&self, radroots_storage::projection::cache::ProjectionCacheScope) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheUsage, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::status(&self, radroots_storage::projection::ProjectionId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::transition_rebuild(&self, radroots_storage::projection::RebuildTransition) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
impl radroots_storage::projection::ProjectionStore for radroots_storage::memory::MemoryStorage
pub fn radroots_storage::memory::MemoryStorage::checkpoint(&self, radroots_storage::projection::ProjectionCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::collect_projection_cache(&self, radroots_storage::projection::cache::ProjectionCacheScope) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheCollection, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::configure_projection_cache(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::cache::ProjectionCacheLimits) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheUsage, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::event_index_checkpoint(&self, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::EventIndexCheckpoint>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::event_index_manifest(&self, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::EventIndexManifest>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::invalidate(&self, radroots_storage::projection::ProjectionInvalidation) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
@@ -1703,6 +1736,7 @@ pub fn radroots_storage::memory::MemoryStorage::query_projection_documents(&self
pub fn radroots_storage::memory::MemoryStorage::query_projection_documents_at(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery, radroots_storage::event::EventSourceToken, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::rebuild(&self, radroots_storage::projection::RebuildTicketId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::RebuildTicket>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::request_rebuild(&self, radroots_storage::projection::RebuildTicket) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::retire_projection_generation(&self, radroots_storage::projection::cache::ProjectionCacheScope) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheUsage, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::status(&self, radroots_storage::projection::ProjectionId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::transition_rebuild(&self, radroots_storage::projection::RebuildTransition) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
pub mod radroots_storage::status
@@ -1851,6 +1885,7 @@ pub radroots_storage::Error::InvalidPrivateArtifactRevision
pub radroots_storage::Error::InvalidPrivateArtifactSchema
pub radroots_storage::Error::InvalidPrivateArtifactSecretReference
pub radroots_storage::Error::InvalidPrivateArtifactTimestamp
+pub radroots_storage::Error::InvalidProjectionCacheLimits
pub radroots_storage::Error::InvalidProjectionDocument
pub radroots_storage::Error::InvalidProjectionGeneration
pub radroots_storage::Error::InvalidProjectionId
@@ -1993,6 +2028,8 @@ pub fn radroots_storage::memory::MemoryStorage::release(&self, radroots_storage:
pub fn radroots_storage::memory::MemoryStorage::status(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::outbox::OutboxStatus, radroots_storage::Error>>
pub trait radroots_storage::ProjectionStore: core::marker::Send + core::marker::Sync
pub fn radroots_storage::ProjectionStore::checkpoint(&self, radroots_storage::projection::ProjectionCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
+pub fn radroots_storage::ProjectionStore::collect_projection_cache(&self, radroots_storage::projection::cache::ProjectionCacheScope) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheCollection, radroots_storage::Error>>
+pub fn radroots_storage::ProjectionStore::configure_projection_cache(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::cache::ProjectionCacheLimits) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheUsage, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::event_index_checkpoint(&self, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::EventIndexCheckpoint>, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::event_index_manifest(&self, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::EventIndexManifest>, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::invalidate(&self, radroots_storage::projection::ProjectionInvalidation) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
@@ -2009,10 +2046,13 @@ pub fn radroots_storage::ProjectionStore::query_projection_documents(&self, radr
pub fn radroots_storage::ProjectionStore::query_projection_documents_at(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery, radroots_storage::event::EventSourceToken, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::rebuild(&self, radroots_storage::projection::RebuildTicketId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::RebuildTicket>, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::request_rebuild(&self, radroots_storage::projection::RebuildTicket) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
+pub fn radroots_storage::ProjectionStore::retire_projection_generation(&self, radroots_storage::projection::cache::ProjectionCacheScope) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheUsage, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::status(&self, radroots_storage::projection::ProjectionId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::transition_rebuild(&self, radroots_storage::projection::RebuildTransition) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
impl radroots_storage::projection::ProjectionStore for radroots_storage::memory::MemoryStorage
pub fn radroots_storage::memory::MemoryStorage::checkpoint(&self, radroots_storage::projection::ProjectionCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::collect_projection_cache(&self, radroots_storage::projection::cache::ProjectionCacheScope) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheCollection, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::configure_projection_cache(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::cache::ProjectionCacheLimits) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheUsage, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::event_index_checkpoint(&self, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::EventIndexCheckpoint>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::event_index_manifest(&self, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::EventIndexManifest>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::invalidate(&self, radroots_storage::projection::ProjectionInvalidation) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
@@ -2029,6 +2069,7 @@ pub fn radroots_storage::memory::MemoryStorage::query_projection_documents(&self
pub fn radroots_storage::memory::MemoryStorage::query_projection_documents_at(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery, radroots_storage::event::EventSourceToken, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::rebuild(&self, radroots_storage::projection::RebuildTicketId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::RebuildTicket>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::request_rebuild(&self, radroots_storage::projection::RebuildTicket) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::retire_projection_generation(&self, radroots_storage::projection::cache::ProjectionCacheScope) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheUsage, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::status(&self, radroots_storage::projection::ProjectionId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::transition_rebuild(&self, radroots_storage::projection::RebuildTransition) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
pub trait radroots_storage::Storage: radroots_storage::event::EventStore + radroots_storage::journal::Journal + radroots_storage::outbox::Outbox + radroots_storage::projection::ProjectionStore + radroots_storage::private_artifact::PrivateArtifactStore + radroots_storage::backup::StorageReliability + radroots_storage::atomic::AtomicStorage + radroots_storage::authored_atomic::AuthoredAtomicStorage + radroots_storage::authored_draft::AuthoredDraftStore
diff --git a/contracts/api_baselines/radroots_storage_sqlite.txt b/contracts/api_baselines/radroots_storage_sqlite.txt
@@ -613,6 +613,8 @@ pub fn radroots_storage_sqlite::SqliteStorage::status(&self) -> radroots_transpo
pub fn radroots_storage_sqlite::SqliteStorage::tombstone(&self, radroots_storage::private_artifact::PrivateArtifactId, radroots_storage::private_artifact::PrivateArtifactRevision, u64, radroots_storage::private_artifact::DeletionReason) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::private_artifact::PrivateArtifactMetadata, radroots_storage::error::Error>>
impl radroots_storage::projection::ProjectionStore for radroots_storage_sqlite::SqliteStorage
pub fn radroots_storage_sqlite::SqliteStorage::checkpoint(&self, radroots_storage::projection::ProjectionCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::error::Error>>
+pub fn radroots_storage_sqlite::SqliteStorage::collect_projection_cache(&self, radroots_storage::projection::cache::ProjectionCacheScope) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheCollection, radroots_storage::error::Error>>
+pub fn radroots_storage_sqlite::SqliteStorage::configure_projection_cache(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::cache::ProjectionCacheLimits) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheUsage, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::event_index_checkpoint(&self, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::EventIndexCheckpoint>, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::event_index_manifest(&self, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::EventIndexManifest>, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::invalidate(&self, radroots_storage::projection::ProjectionInvalidation) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::error::Error>>
@@ -629,6 +631,7 @@ pub fn radroots_storage_sqlite::SqliteStorage::query_projection_documents(&self,
pub fn radroots_storage_sqlite::SqliteStorage::query_projection_documents_at(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery, radroots_storage::event::EventSourceToken, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::rebuild(&self, radroots_storage::projection::RebuildTicketId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::RebuildTicket>, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::request_rebuild(&self, radroots_storage::projection::RebuildTicket) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::error::Error>>
+pub fn radroots_storage_sqlite::SqliteStorage::retire_projection_generation(&self, radroots_storage::projection::cache::ProjectionCacheScope) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::cache::ProjectionCacheUsage, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::status(&self, radroots_storage::projection::ProjectionId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::transition_rebuild(&self, radroots_storage::projection::RebuildTransition) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::error::Error>>
impl radroots_storage::status::StorageStatusProvider for radroots_storage_sqlite::SqliteStorage
diff --git a/contracts/architecture/decisions/projection_cache_retirement.v1.json b/contracts/architecture/decisions/projection_cache_retirement.v1.json
@@ -0,0 +1,25 @@
+{
+ "schema": "radroots.projection-cache-retirement.v1",
+ "status": "implemented_candidate",
+ "owners": [
+ "radroots_storage",
+ "radroots_storage_sqlite"
+ ],
+ "authority": "Explicit cache identity registration and immutable positive caller-owned limits. Application policy remains outside Lib. Unsupported providers refuse the new methods.",
+ "accounting": "One store-wide logical value-byte, protected-generation and legacy-snapshot budget includes legacy, active and staging projection rows. Scalar migration initialization is explicit; subsequent declared writers update cached counters under their owner lock or transaction. Growing admission refuses capacity without partial rows or status. Collection may reduce pre-existing excess.",
+ "retirement": "Exact registered cache owner, canonical source token and expected active generation are compared before effects. A current Ready active generation is protected. Obsolete, staged or explicitly invalidated generations can be retired. Stale active retirement clears its derived status atomically; canonical events, authored work, intents, receipts and media custody are untouched.",
+ "collection": "At most64 total exact-generation document, legacy snapshot and registration rows per pass. Active/staging documents are protected. Unrelated projection identities cannot be collected unless explicitly registered as derived cache by their owner. Continuation reflects remaining eligible work. Metadata selection and deletion occur in one transaction or lock, without payload hydration.",
+ "legacy_snapshots": "Configured cache identities create no new whole-view snapshot sessions. Existing exact duplicate replay cannot extend fixed expiry. Owner wall time and a durable high-water govern legacy expiry; caller as-of is never a clock input. Configured legacy snapshot reads on a read-only provider refuse continuation. Exact unconfigured lookups validate scalar lengths and fixed digests before hydration.",
+ "compatibility": "Historical migrations1-21 are immutable. Forward22 owns only derived-cache accounting, registration and indexes. Unconfigured inventory semantics remain supported. Physical database/WAL pages are separate from logical live value-byte quota and are not promised to shrink after collection.",
+ "non_goals": [
+ "leases",
+ "new per-read snapshots",
+ "application policy",
+ "canonical data deletion",
+ "new database",
+ "worker",
+ "vacuum",
+ "physical-device qualification"
+ ],
+ "batch_admission": "One final aggregate policy independent of input order. Up to64 scalar old lengths are preflighted in BEGIN IMMEDIATE. True aggregate growth beyond immutable configured limits refuses before payload effects; non-growing replacement can reduce pre-existing excess. A private final-byte reservation exists only inside that transaction, shrinking rows precede growth, and exact counters/reservation are checked and cleared before commit. Failure, cancellation and crash roll back the transaction. Direct writers retain ordinary per-write guards. Historical migrations1-21 remain unchanged."
+}
diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs
@@ -94,6 +94,7 @@ pub enum Error {
InvalidEventIndexCheckpoint,
DuplicateEventIndexShard,
CorruptProjectionRecord,
+ InvalidProjectionCacheLimits,
InvalidProjectionDocument,
InvalidProjectionSnapshot,
CorruptProjectionDocument,
@@ -236,6 +237,7 @@ impl fmt::Display for Error {
Self::InvalidEventIndexCheckpoint => "storage event-index checkpoint is invalid",
Self::DuplicateEventIndexShard => "storage event-index shard is duplicated",
Self::CorruptProjectionRecord => "storage projection record is corrupt",
+ Self::InvalidProjectionCacheLimits => "storage projection cache limits are invalid",
Self::InvalidProjectionDocument => "storage projection document is invalid",
Self::InvalidProjectionSnapshot => "storage projection snapshot is invalid",
Self::CorruptProjectionDocument => "storage projection document is corrupt",
@@ -394,6 +396,7 @@ mod tests {
InvalidEventIndexCheckpoint,
DuplicateEventIndexShard,
CorruptProjectionRecord,
+ InvalidProjectionCacheLimits,
InvalidProjectionDocument,
InvalidProjectionSnapshot,
CorruptProjectionDocument,
diff --git a/crates/storage/src/memory.rs b/crates/storage/src/memory.rs
@@ -70,6 +70,8 @@ use crate::{
#[path = "memory_projection_batch.rs"]
mod projection_batch;
+#[path = "memory_projection_cache.rs"]
+mod projection_cache;
#[derive(Clone)]
struct EventEntry {
@@ -97,7 +99,8 @@ struct State {
(ProjectionId, ProjectionGeneration),
projection_batch::Generation,
>,
- projection_snapshots: Vec<ProjectionSnapshot>,
+ projection_snapshots: projection_cache::Snapshots,
+ projection_cache: projection_cache::Cache,
private_artifacts: Vec<PrivateArtifactMetadata>,
private_artifact_reseals: Vec<PrivateArtifactResealReceipt>,
backups: Vec<BackupOperation>,
@@ -139,7 +142,8 @@ impl MemoryStorage {
event_index_checkpoints: Vec::new(),
projection_documents: std::collections::BTreeMap::new(),
projection_generations: std::collections::BTreeMap::new(),
- projection_snapshots: Vec::new(),
+ projection_snapshots: projection_cache::Snapshots::new(),
+ projection_cache: projection_cache::Cache::new(),
private_artifacts: Vec::new(),
private_artifact_reseals: Vec::new(),
backups: Vec::new(),
@@ -481,6 +485,8 @@ impl MemoryStorage {
{
return Err(Error::ProjectionRevisionConflict);
}
+ let identity = (checkpoint.projection_id().clone(), checkpoint.generation());
+ state.projection_cache.check_protected(&identity)?;
if let Some(status) = state
.projections
.iter_mut()
@@ -512,6 +518,7 @@ impl MemoryStorage {
Some(checkpoint),
None,
)?;
+ state.projection_cache.protected.insert(identity);
state.projections.push(status.clone());
Ok(status)
}
@@ -891,6 +898,36 @@ impl Outbox for MemoryStorage {
}
impl ProjectionStore for MemoryStorage {
+ fn configure_projection_cache(
+ &self,
+ id: ProjectionId,
+ limits: crate::projection::cache::ProjectionCacheLimits,
+ ) -> BoxFuture<'_, Result<crate::projection::cache::ProjectionCacheUsage, Error>> {
+ Box::pin(async move {
+ projection_cache::configure(&mut *self.state()?, id, limits, projection_cache::now()?)
+ })
+ }
+ fn retire_projection_generation(
+ &self,
+ scope: crate::projection::cache::ProjectionCacheScope,
+ ) -> BoxFuture<'_, Result<crate::projection::cache::ProjectionCacheUsage, Error>> {
+ Box::pin(
+ async move { projection_cache::retire(&mut *self.state()?, self.generation, scope) },
+ )
+ }
+ fn collect_projection_cache(
+ &self,
+ scope: crate::projection::cache::ProjectionCacheScope,
+ ) -> BoxFuture<'_, Result<crate::projection::cache::ProjectionCacheCollection, Error>> {
+ Box::pin(async move {
+ projection_cache::collect(
+ &mut *self.state()?,
+ self.generation,
+ scope,
+ projection_cache::now()?,
+ )
+ })
+ }
fn status(
&self,
projection_id: ProjectionId,
@@ -1064,6 +1101,22 @@ impl ProjectionStore for MemoryStorage {
return Err(Error::SourceGenerationChanged);
}
let projection_id = next.invalidation().projection_id();
+ let old_identity = (
+ projection_id.clone(),
+ next.invalidation().invalid_generation(),
+ );
+ let new_identity = (
+ projection_id.clone(),
+ next.invalidation().replacement_generation(),
+ );
+ let old_registered = state.projection_generations.contains_key(&old_identity);
+ if next.stage() == RebuildStage::Completed
+ && (old_registered || !state.projection_cache.protected.contains(&old_identity))
+ {
+ // A live registration retains its reservation until explicit retirement.
+ // Only an unregistered legacy checkpoint transfers its existing slot.
+ state.projection_cache.check_protected(&new_identity)?;
+ }
let status = state
.projections
.iter_mut()
@@ -1102,6 +1155,12 @@ impl ProjectionStore for MemoryStorage {
checkpoint,
active_rebuild,
)?;
+ if next.stage() == RebuildStage::Completed {
+ if !old_registered {
+ state.projection_cache.protected.remove(&old_identity);
+ }
+ state.projection_cache.protected.insert(new_identity);
+ }
state.rebuilds[index] = next.clone();
Ok(next)
})
@@ -1232,16 +1291,25 @@ impl ProjectionStore for MemoryStorage {
) -> BoxFuture<'_, Result<(), Error>> {
Box::pin(async move {
let mut state = self.state()?;
+ if state.projection_cache.owners.contains(&projection_id) {
+ return Err(Error::ProjectionRevisionConflict);
+ }
if state
.projection_generations
.contains_key(&(projection_id.clone(), generation))
{
return Err(Error::ProjectionRevisionConflict);
}
- state.projection_documents.insert(
- (projection_id, generation, document.key().to_owned()),
- document,
- );
+ let key = (projection_id, generation, document.key().to_owned());
+ let bytes = state.projection_cache.check_bytes(
+ state
+ .projection_documents
+ .get(&key)
+ .map_or(0, |value| value.value().len() as u64),
+ document.value().len() as u64,
+ )?;
+ state.projection_cache.value_bytes = bytes;
+ state.projection_documents.insert(key, document);
Ok(())
})
}
@@ -1277,36 +1345,62 @@ impl ProjectionStore for MemoryStorage {
) -> BoxFuture<'_, Result<(), Error>> {
Box::pin(async move {
let mut state = self.state()?;
- if let Some(existing) = state.projection_snapshots.iter().find(|existing| {
- existing.projection_id() == snapshot.projection_id()
- && existing.snapshot_id() == snapshot.snapshot_id()
- }) {
+ let key = (snapshot.projection_id().clone(), *snapshot.snapshot_id());
+ if let Some(existing) = state.projection_snapshots.get(&key) {
return if existing == &snapshot {
Ok(())
} else {
Err(Error::CorruptProjectionDocument)
};
}
- state.projection_snapshots.push(snapshot);
+ if state
+ .projection_cache
+ .owners
+ .contains(snapshot.projection_id())
+ {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ if state.projection_cache.limits.is_some_and(|limit| {
+ state.projection_snapshots.len() as u64 >= limit.legacy_snapshots()
+ }) {
+ return Err(Error::SpaceInsufficient);
+ }
+ let bytes = state
+ .projection_cache
+ .check_bytes(0, snapshot.value().len() as u64)?;
+ state.projection_cache.value_bytes = bytes;
+ state.projection_cache.snapshot_order.insert((
+ key.0.clone(),
+ snapshot.generation(),
+ snapshot.created_at_unix_ms(),
+ key.1,
+ ));
+ state.projection_snapshots.insert(key, snapshot);
Ok(())
})
}
-
fn projection_snapshot(
&self,
- projection_id: ProjectionId,
- snapshot_id: [u8; 32],
+ id: ProjectionId,
+ snapshot: [u8; 32],
) -> BoxFuture<'_, Result<Option<ProjectionSnapshot>, Error>> {
Box::pin(async move {
- Ok(self
- .state()?
- .projection_snapshots
- .iter()
- .find(|snapshot| {
- snapshot.projection_id() == &projection_id
- && snapshot.snapshot_id() == &snapshot_id
- })
- .cloned())
+ let mut state = self.state()?;
+ if state.projection_cache.owners.contains(&id) {
+ let now = state.projection_cache.time(projection_cache::now()?);
+ if state
+ .projection_snapshots
+ .get(&(id.clone(), snapshot))
+ .is_some_and(|value| {
+ state
+ .projection_cache
+ .expired(value.created_at_unix_ms(), now)
+ })
+ {
+ return Ok(None);
+ }
+ }
+ Ok(state.projection_snapshots.get(&(id, snapshot)).cloned())
})
}
}
diff --git a/crates/storage/src/memory_projection_batch.rs b/crates/storage/src/memory_projection_batch.rs
@@ -14,7 +14,7 @@ use crate::{
use std::ops::Bound::{Excluded, Included, Unbounded};
#[derive(Clone, Copy, Eq, PartialEq)]
-enum Phase {
+pub(super) enum Phase {
Staging,
Promoted,
Retired,
@@ -22,10 +22,10 @@ enum Phase {
#[derive(Clone)]
pub(super) struct Generation {
- scope: ProjectionWriteScope,
- phase: Phase,
- rows: u64,
- bytes: u64,
+ pub(super) scope: ProjectionWriteScope,
+ pub(super) phase: Phase,
+ pub(super) rows: u64,
+ pub(super) bytes: u64,
}
pub(super) fn fence(
@@ -143,6 +143,10 @@ pub(super) fn write(
)
})
.transpose()?;
+ let cache_bytes = state
+ .projection_cache
+ .check_bytes(prior.map_or(0, |value| value.bytes), next.bytes)?;
+ state.projection_cache.check_protected(&identity)?;
// Every fallible condition is checked before publishing any in-memory changes.
let documents: Vec<_> = batch
.documents()
@@ -154,11 +158,19 @@ pub(super) fn write(
)
})
.collect();
+ state.projection_cache.value_bytes = cache_bytes;
+ state.projection_cache.protected.insert(identity.clone());
for (key, document) in documents {
state.projection_documents.insert(key, document);
}
if let Some(status) = promoted.as_ref() {
next.phase = Phase::Promoted;
+ if let Some(active) = scope.expected_active() {
+ state
+ .projection_cache
+ .protected
+ .remove(&(identity.0.clone(), active));
+ }
if let Some(active) = scope.expected_active()
&& let Some(previous) = state
.projection_generations
diff --git a/crates/storage/src/memory_projection_cache.rs b/crates/storage/src/memory_projection_cache.rs
@@ -0,0 +1,379 @@
+use super::{Error, State, projection_batch};
+use crate::{
+ event::{EventSourceToken, SourceGeneration},
+ projection::{
+ ProjectionGeneration, ProjectionHealth, ProjectionId, ProjectionSnapshot,
+ batch::ProjectionWriteScope,
+ cache::{
+ PROJECTION_CACHE_COLLECTION_ROWS_MAX, ProjectionCacheCollection, ProjectionCacheLimits,
+ ProjectionCacheScope, ProjectionCacheUsage,
+ },
+ },
+};
+use std::{
+ collections::{BTreeMap, BTreeSet},
+ ops::Bound::{Excluded, Included, Unbounded},
+ time::{SystemTime, UNIX_EPOCH},
+};
+
+type Identity = (ProjectionId, ProjectionGeneration);
+pub(super) type Snapshots = BTreeMap<(ProjectionId, [u8; 32]), ProjectionSnapshot>;
+#[derive(Clone, Default)]
+pub(super) struct Cache {
+ pub(super) limits: Option<ProjectionCacheLimits>,
+ pub(super) owners: BTreeSet<ProjectionId>,
+ pub(super) protected: BTreeSet<Identity>,
+ pub(super) snapshot_order: BTreeSet<(ProjectionId, ProjectionGeneration, u64, [u8; 32])>,
+ pub(super) value_bytes: u64,
+ pub(super) owner_time_ms: u64,
+}
+impl Cache {
+ pub(super) const fn new() -> Self {
+ Self {
+ limits: None,
+ owners: BTreeSet::new(),
+ protected: BTreeSet::new(),
+ snapshot_order: BTreeSet::new(),
+ value_bytes: 0,
+ owner_time_ms: 0,
+ }
+ }
+ pub(super) fn usage(&self) -> ProjectionCacheUsage {
+ ProjectionCacheUsage {
+ value_bytes: self.value_bytes,
+ protected_generations: self.protected.len() as u64,
+ legacy_snapshots: self.snapshot_order.len() as u64,
+ }
+ }
+ pub(super) fn check_bytes(&self, previous: u64, next: u64) -> Result<u64, Error> {
+ let value = self
+ .value_bytes
+ .checked_sub(previous)
+ .and_then(|value| value.checked_add(next))
+ .ok_or(Error::SpaceInsufficient)?;
+ if next > previous && self.limits.is_some_and(|limit| value > limit.value_bytes()) {
+ return Err(Error::SpaceInsufficient);
+ }
+ Ok(value)
+ }
+ pub(super) fn check_protected(&self, identity: &Identity) -> Result<(), Error> {
+ if !self.protected.contains(identity)
+ && self
+ .limits
+ .is_some_and(|limit| self.protected.len() as u64 >= limit.protected_generations())
+ {
+ return Err(Error::SpaceInsufficient);
+ }
+ Ok(())
+ }
+ pub(super) fn time(&mut self, now: u64) -> u64 {
+ self.owner_time_ms = self.owner_time_ms.max(now);
+ self.owner_time_ms
+ }
+ pub(super) fn expired(&self, created: u64, now: u64) -> bool {
+ self.limits
+ .is_some_and(|limit| created > now || now - created >= limit.legacy_age_ms())
+ }
+}
+pub(super) fn now() -> Result<u64, Error> {
+ u64::try_from(
+ SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map_err(|_| Error::InvalidProjectionTimestamp)?
+ .as_millis(),
+ )
+ .map_err(|_| Error::InvalidProjectionTimestamp)
+}
+pub(super) fn configure(
+ state: &mut State,
+ id: ProjectionId,
+ limits: ProjectionCacheLimits,
+ now: u64,
+) -> Result<ProjectionCacheUsage, Error> {
+ ProjectionId::parse(id.as_str())?;
+ if state
+ .projection_cache
+ .limits
+ .is_some_and(|existing| existing != limits)
+ {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ state.projection_cache.limits = Some(limits);
+ state.projection_cache.owners.insert(id);
+ state.projection_cache.time(now);
+ Ok(state.projection_cache.usage())
+}
+fn fence(
+ state: &State,
+ generation: SourceGeneration,
+ scope: &ProjectionCacheScope,
+) -> Result<(), Error> {
+ if scope.source() != EventSourceToken::new(generation, state.event_mutation_epoch) {
+ return Err(Error::SourceEpochChanged);
+ }
+ if !state
+ .projection_cache
+ .owners
+ .contains(scope.projection_id())
+ || state
+ .projections
+ .iter()
+ .find(|value| value.projection_id() == scope.projection_id())
+ .map(|value| value.generation())
+ != scope.expected_active()
+ {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ Ok(())
+}
+pub(super) fn retire(
+ state: &mut State,
+ source: SourceGeneration,
+ scope: ProjectionCacheScope,
+) -> Result<ProjectionCacheUsage, Error> {
+ fence(state, source, &scope)?;
+ let identity = (scope.projection_id().clone(), scope.generation());
+ let prior = state.projection_generations.get(&identity);
+ if scope.expected_active() == Some(scope.generation()) {
+ let current = state
+ .projections
+ .iter()
+ .find(|value| value.projection_id() == scope.projection_id())
+ .ok_or(Error::ProjectionRevisionConflict)?;
+ if current.health() == ProjectionHealth::Ready
+ && prior.is_none_or(|value| value.scope.source() == scope.source())
+ {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ }
+ let retired = prior.cloned().unwrap_or(projection_batch::Generation {
+ scope: ProjectionWriteScope::new(
+ scope.projection_id().clone(),
+ scope.generation(),
+ scope.source(),
+ None,
+ )?,
+ phase: projection_batch::Phase::Retired,
+ rows: 0,
+ bytes: 0,
+ });
+ state.projection_generations.insert(
+ identity.clone(),
+ projection_batch::Generation {
+ phase: projection_batch::Phase::Retired,
+ ..retired
+ },
+ );
+ state.projection_cache.protected.remove(&identity);
+ if scope.expected_active() == Some(scope.generation()) {
+ state
+ .projections
+ .retain(|value| value.projection_id() != scope.projection_id());
+ }
+ Ok(state.projection_cache.usage())
+}
+pub(super) fn collect(
+ state: &mut State,
+ source: SourceGeneration,
+ scope: ProjectionCacheScope,
+ now: u64,
+) -> Result<ProjectionCacheCollection, Error> {
+ fence(state, source, &scope)?;
+ let identity = (scope.projection_id().clone(), scope.generation());
+ let reserved = state
+ .projection_generations
+ .get(&identity)
+ .is_some_and(|value| value.phase != projection_batch::Phase::Retired);
+ let protected = scope.expected_active() == Some(scope.generation()) || reserved;
+ let now = state.projection_cache.owner_time_ms.max(now);
+ let documents: Vec<_> = if protected {
+ Vec::new()
+ } else {
+ state
+ .projection_documents
+ .range((
+ Included((identity.0.clone(), identity.1, String::new())),
+ Unbounded,
+ ))
+ .take_while(|((id, generation, _), _)| id == &identity.0 && generation == &identity.1)
+ .take(PROJECTION_CACHE_COLLECTION_ROWS_MAX)
+ .map(|(key, value)| (key.clone(), value.value().len() as u64))
+ .collect()
+ };
+ let mut snapshots = Vec::new();
+ let remaining = PROJECTION_CACHE_COLLECTION_ROWS_MAX - documents.len();
+ for key in state
+ .projection_cache
+ .snapshot_order
+ .range((
+ Included((identity.0.clone(), identity.1, 0, [0; 32])),
+ Unbounded,
+ ))
+ .take_while(|(id, generation, _, _)| id == &identity.0 && generation == &identity.1)
+ .take(remaining)
+ {
+ let pressure = state.projection_cache.limits.is_some_and(|limit| {
+ state.projection_snapshots.len() as u64 - snapshots.len() as u64
+ > limit.legacy_snapshots()
+ });
+ if !protected || pressure || state.projection_cache.expired(key.2, now) {
+ let value = state
+ .projection_snapshots
+ .get(&(key.0.clone(), key.3))
+ .ok_or(Error::CorruptProjectionRecord)?;
+ snapshots.push((key.clone(), value.value().len() as u64));
+ } else {
+ break;
+ }
+ }
+ if protected {
+ let start = snapshots.last().filter(|(key, _)| key.2 > now).map_or(
+ (identity.0.clone(), identity.1, now, [255; 32]),
+ |(key, _)| key.clone(),
+ );
+ for key in state
+ .projection_cache
+ .snapshot_order
+ .range((Excluded(start), Unbounded))
+ .take_while(|(id, generation, _, _)| id == &identity.0 && generation == &identity.1)
+ .take(remaining - snapshots.len())
+ {
+ let value = state
+ .projection_snapshots
+ .get(&(key.0.clone(), key.3))
+ .ok_or(Error::CorruptProjectionRecord)?;
+ snapshots.push((key.clone(), value.value().len() as u64));
+ }
+ }
+ let freed = documents
+ .iter()
+ .map(|(_, bytes)| *bytes)
+ .chain(snapshots.iter().map(|(_, bytes)| *bytes))
+ .try_fold(0u64, |sum, bytes| {
+ sum.checked_add(bytes).ok_or(Error::CorruptProjectionRecord)
+ })?;
+ let value_bytes = state
+ .projection_cache
+ .value_bytes
+ .checked_sub(freed)
+ .ok_or(Error::CorruptProjectionRecord)?;
+ let mut removed = (documents.len() + snapshots.len()) as u64;
+ for (key, _) in documents {
+ state.projection_documents.remove(&key);
+ }
+ for (key, _) in snapshots {
+ state.projection_cache.snapshot_order.remove(&key);
+ state.projection_snapshots.remove(&(key.0, key.3));
+ }
+ state.projection_cache.value_bytes = value_bytes;
+ state.projection_cache.time(now);
+ let document_more = !protected
+ && state
+ .projection_documents
+ .range((
+ Included((identity.0.clone(), identity.1, String::new())),
+ Unbounded,
+ ))
+ .next()
+ .is_some_and(|((id, generation, _), _)| id == &identity.0 && generation == &identity.1);
+ let snapshot_more = state
+ .projection_cache
+ .snapshot_order
+ .range((
+ Included((identity.0.clone(), identity.1, 0, [0; 32])),
+ Unbounded,
+ ))
+ .next()
+ .is_some_and(|(id, generation, created, _)| {
+ id == &identity.0
+ && generation == &identity.1
+ && (!protected
+ || state.projection_cache.expired(*created, now)
+ || state.projection_cache.limits.is_some_and(|limit| {
+ state.projection_snapshots.len() as u64 > limit.legacy_snapshots()
+ }))
+ })
+ || state
+ .projection_cache
+ .snapshot_order
+ .range((
+ Excluded((identity.0.clone(), identity.1, now, [255; 32])),
+ Unbounded,
+ ))
+ .next()
+ .is_some_and(|(id, generation, _, _)| id == &identity.0 && generation == &identity.1);
+ if !protected
+ && !document_more
+ && !snapshot_more
+ && state.projection_generations.contains_key(&identity)
+ && removed < PROJECTION_CACHE_COLLECTION_ROWS_MAX as u64
+ {
+ state.projection_generations.remove(&identity);
+ removed += 1;
+ }
+ let registration_more = !protected && state.projection_generations.contains_key(&identity);
+ Ok(ProjectionCacheCollection {
+ removed_rows: removed,
+ removed_value_bytes: freed,
+ more: document_more || snapshot_more || registration_more,
+ usage: state.projection_cache.usage(),
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::{EventStore, ProjectionStore, memory::MemoryStorage};
+ use futures_executor::block_on;
+ #[test]
+ fn projection_cache_memory_owner_time_age_boundary_rollback_and_future_do_not_extend_lifetime()
+ {
+ let generation = SourceGeneration::new([9; 32]).unwrap();
+ let store = MemoryStorage::new(generation);
+ let id = ProjectionId::parse("clock_cache").unwrap();
+ let projection_generation = ProjectionGeneration::new([1; 32]).unwrap();
+ let created = now().unwrap();
+ let snapshot =
+ ProjectionSnapshot::new(id.clone(), [1; 32], projection_generation, created, vec![7])
+ .unwrap();
+ block_on(store.put_projection_snapshot(snapshot.clone())).unwrap();
+ let limits = ProjectionCacheLimits::new(67_108_864, 2, 32, 1_800_000).unwrap();
+ block_on(store.configure_projection_cache(id.clone(), limits)).unwrap();
+ {
+ let mut state = store.state().unwrap();
+ assert!(!state.projection_cache.expired(created, created + 1_799_999));
+ assert!(state.projection_cache.expired(created, created + 1_800_000));
+ assert!(state.projection_cache.expired(created + 1, created));
+ assert_eq!(
+ state.projection_cache.time(created + 1_800_001),
+ created + 1_800_001
+ );
+ assert_eq!(
+ state.projection_cache.time(created - 1),
+ created + 1_800_001
+ );
+ }
+ assert_eq!(
+ block_on(store.projection_snapshot(id.clone(), [1; 32])).unwrap(),
+ None
+ );
+ block_on(store.put_projection_snapshot(snapshot.clone())).unwrap();
+ assert_eq!(
+ block_on(store.projection_snapshot(id.clone(), [1; 32])).unwrap(),
+ None
+ );
+ let source = block_on(store.source_token()).unwrap();
+ let exact =
+ ProjectionCacheScope::new(id.clone(), projection_generation, source, None).unwrap();
+ assert_eq!(
+ block_on(store.collect_projection_cache(exact))
+ .unwrap()
+ .removed_rows,
+ 1
+ );
+ assert_eq!(
+ block_on(store.put_projection_snapshot(snapshot)),
+ Err(Error::ProjectionRevisionConflict)
+ );
+ }
+}
diff --git a/crates/storage/src/projection.rs b/crates/storage/src/projection.rs
@@ -4,6 +4,7 @@
//! row representations remain in their domain packages.
pub mod batch;
+pub mod cache;
pub mod document_query;
pub use radroots_event::EventId;
@@ -1028,6 +1029,29 @@ impl ProjectionStatus {
/// Backend-neutral projection coordination SPI.
pub trait ProjectionStore: Send + Sync {
+ /// Registers one exact derived-cache identity under an immutable store budget.
+ /// Configured identities cannot create new legacy whole-view snapshots.
+ fn configure_projection_cache(
+ &self,
+ _id: ProjectionId,
+ _limits: cache::ProjectionCacheLimits,
+ ) -> BoxFuture<'_, Result<cache::ProjectionCacheUsage, Error>> {
+ Box::pin(async { Err(Error::BackendUnavailable) })
+ }
+ /// Retires an exact obsolete generation without touching canonical or authored data.
+ fn retire_projection_generation(
+ &self,
+ _scope: cache::ProjectionCacheScope,
+ ) -> BoxFuture<'_, Result<cache::ProjectionCacheUsage, Error>> {
+ Box::pin(async { Err(Error::BackendUnavailable) })
+ }
+ /// Collects at most64 exact-generation derived rows under a source/active fence.
+ fn collect_projection_cache(
+ &self,
+ _scope: cache::ProjectionCacheScope,
+ ) -> BoxFuture<'_, Result<cache::ProjectionCacheCollection, Error>> {
+ Box::pin(async { Err(Error::BackendUnavailable) })
+ }
/// Atomically stages a bounded batch under a durable source and generation fence.
fn put_projection_batch(
&self,
diff --git a/crates/storage/src/projection/cache.rs b/crates/storage/src/projection/cache.rs
@@ -0,0 +1,111 @@
+//! Explicit derived-cache budgets and exact, bounded retirement requests.
+use super::{Error, ProjectionGeneration, ProjectionId};
+use crate::event::EventSourceToken;
+
+pub const PROJECTION_CACHE_COLLECTION_ROWS_MAX: usize = 64;
+
+/// Caller-owned positive limits. Configuring a cache never silently raises them.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct ProjectionCacheLimits {
+ value_bytes: u64,
+ protected_generations: u64,
+ legacy_snapshots: u64,
+ legacy_age_ms: u64,
+}
+impl ProjectionCacheLimits {
+ pub fn new(
+ value_bytes: u64,
+ protected_generations: u64,
+ legacy_snapshots: u64,
+ legacy_age_ms: u64,
+ ) -> Result<Self, Error> {
+ if [
+ value_bytes,
+ protected_generations,
+ legacy_snapshots,
+ legacy_age_ms,
+ ]
+ .iter()
+ .any(|value| *value == 0 || *value > i64::MAX as u64)
+ {
+ return Err(Error::InvalidProjectionCacheLimits);
+ }
+ Ok(Self {
+ value_bytes,
+ protected_generations,
+ legacy_snapshots,
+ legacy_age_ms,
+ })
+ }
+ pub const fn value_bytes(self) -> u64 {
+ self.value_bytes
+ }
+ pub const fn protected_generations(self) -> u64 {
+ self.protected_generations
+ }
+ pub const fn legacy_snapshots(self) -> u64 {
+ self.legacy_snapshots
+ }
+ pub const fn legacy_age_ms(self) -> u64 {
+ self.legacy_age_ms
+ }
+}
+
+/// An exact cache owner/generation and the source/active identity observed by its caller.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct ProjectionCacheScope {
+ id: ProjectionId,
+ generation: ProjectionGeneration,
+ source: EventSourceToken,
+ active: Option<ProjectionGeneration>,
+}
+impl ProjectionCacheScope {
+ pub fn new(
+ id: ProjectionId,
+ generation: ProjectionGeneration,
+ source: EventSourceToken,
+ active: Option<ProjectionGeneration>,
+ ) -> Result<Self, Error> {
+ ProjectionId::parse(id.as_str())?;
+ ProjectionGeneration::new(*generation.as_bytes())?;
+ crate::event::SourceGeneration::new(*source.generation().as_bytes())?;
+ if let Some(value) = active {
+ ProjectionGeneration::new(*value.as_bytes())?;
+ }
+ Ok(Self {
+ id,
+ generation,
+ source,
+ active,
+ })
+ }
+ pub const fn projection_id(&self) -> &ProjectionId {
+ &self.id
+ }
+ pub const fn generation(&self) -> ProjectionGeneration {
+ self.generation
+ }
+ pub const fn source(&self) -> EventSourceToken {
+ self.source
+ }
+ pub const fn expected_active(&self) -> Option<ProjectionGeneration> {
+ self.active
+ }
+}
+
+/// Cached logical usage; physical database and WAL allocation are separate.
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+pub struct ProjectionCacheUsage {
+ pub value_bytes: u64,
+ pub protected_generations: u64,
+ pub legacy_snapshots: u64,
+}
+
+/// Actual collection progress. A pass removes at most64 total derived rows.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct ProjectionCacheCollection {
+ pub removed_rows: u64,
+ pub removed_value_bytes: u64,
+ pub more: bool,
+ pub usage: ProjectionCacheUsage,
+}
diff --git a/crates/storage/tests/projection_cache.rs b/crates/storage/tests/projection_cache.rs
@@ -0,0 +1,851 @@
+use futures_executor::block_on;
+use radroots_storage::{
+ Error, EventStore, ProjectionStore,
+ event::{EventSourceToken, SourceGeneration},
+ memory::MemoryStorage,
+ projection::{
+ ProjectionDocument, ProjectionGeneration, ProjectionId, ProjectionSnapshot,
+ cache::{ProjectionCacheLimits, ProjectionCacheScope},
+ },
+};
+fn id(value: &str) -> ProjectionId {
+ ProjectionId::parse(value).unwrap()
+}
+fn generation(value: u8) -> ProjectionGeneration {
+ ProjectionGeneration::new([value; 32]).unwrap()
+}
+
+#[test]
+fn projection_cache_limits_and_exact_scope_validate_before_effects() {
+ for values in [
+ [0, 2, 32, 1],
+ [1, 0, 32, 1],
+ [1, 2, 0, 1],
+ [1, 2, 32, 0],
+ [i64::MAX as u64 + 1, 2, 32, 1],
+ ] {
+ assert_eq!(
+ ProjectionCacheLimits::new(values[0], values[1], values[2], values[3]),
+ Err(Error::InvalidProjectionCacheLimits)
+ );
+ }
+ let maximum = ProjectionCacheLimits::new(
+ i64::MAX as u64,
+ i64::MAX as u64,
+ i64::MAX as u64,
+ i64::MAX as u64,
+ )
+ .unwrap();
+ assert_eq!(
+ [
+ maximum.value_bytes(),
+ maximum.protected_generations(),
+ maximum.legacy_snapshots(),
+ maximum.legacy_age_ms()
+ ],
+ [i64::MAX as u64; 4]
+ );
+ let source = EventSourceToken::new(SourceGeneration::new([9; 32]).unwrap(), 0);
+ let scope =
+ ProjectionCacheScope::new(id("fixture"), generation(1), source, Some(generation(1)))
+ .unwrap();
+ assert_eq!(scope.projection_id(), &id("fixture"));
+ assert_eq!(scope.generation(), generation(1));
+ assert_eq!(scope.source(), source);
+ assert_eq!(scope.expected_active(), Some(generation(1)));
+ assert!(!Error::InvalidProjectionCacheLimits.to_string().is_empty());
+}
+
+#[test]
+fn projection_cache_preserves_other_projection_state_and_expires_legacy_by_owner_time() {
+ let store = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ block_on(store.put_projection_document(
+ id("settings"),
+ generation(1),
+ ProjectionDocument::new("owned_work".into(), vec![7]).unwrap(),
+ ))
+ .unwrap();
+ block_on(store.put_projection_snapshot(
+ ProjectionSnapshot::new(id("cache"), [1; 32], generation(1), 1, vec![8]).unwrap(),
+ ))
+ .unwrap();
+ let limits = ProjectionCacheLimits::new(67_108_864, 2, 32, 1_800_000).unwrap();
+ let usage = block_on(store.configure_projection_cache(id("cache"), limits)).unwrap();
+ assert_eq!(usage.value_bytes, 2);
+ assert_eq!(usage.legacy_snapshots, 1);
+ assert_eq!(
+ block_on(store.projection_snapshot(id("cache"), [1; 32])).unwrap(),
+ None
+ );
+ let source = block_on(store.source_token()).unwrap();
+ let foreign = ProjectionCacheScope::new(id("settings"), generation(1), source, None).unwrap();
+ assert_eq!(
+ block_on(store.collect_projection_cache(foreign)),
+ Err(Error::ProjectionRevisionConflict)
+ );
+ let exact = ProjectionCacheScope::new(id("cache"), generation(1), source, None).unwrap();
+ let result = block_on(store.collect_projection_cache(exact)).unwrap();
+ assert_eq!(result.removed_rows, 1);
+ assert_eq!(result.removed_value_bytes, 1);
+ assert_eq!(result.usage.value_bytes, 1);
+ assert_eq!(
+ block_on(store.projection_document(id("settings"), generation(1), "owned_work".into()))
+ .unwrap()
+ .unwrap()
+ .value(),
+ &[7]
+ );
+ assert_eq!(
+ block_on(store.put_projection_snapshot(
+ ProjectionSnapshot::new(id("cache"), [1; 32], generation(1), 1, vec![8]).unwrap()
+ )),
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(block_on(store.source_token()).unwrap(), source);
+ assert_eq!(
+ block_on(store.configure_projection_cache(
+ id("cache"),
+ ProjectionCacheLimits::new(1, 2, 32, 1_800_000).unwrap()
+ )),
+ Err(Error::ProjectionRevisionConflict)
+ );
+}
+
+#[test]
+fn projection_cache_memory_third_protected_and_foreign_bytes_refuse_without_partial_rows() {
+ use radroots_storage::projection::{
+ ProjectionCheckpoint,
+ batch::{ProjectionDocumentBatch, ProjectionWriteScope},
+ document_query::{ProjectionDocumentGenerations, ProjectionDocumentQuery},
+ };
+ let store = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ let source = block_on(store.source_token()).unwrap();
+ let limits = ProjectionCacheLimits::new(3, 2, 32, 1_800_000).unwrap();
+ block_on(store.configure_projection_cache(id("cache"), limits)).unwrap();
+ let first = ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(id("cache"), generation(1), source, None).unwrap(),
+ vec![ProjectionDocument::new("a".into(), vec![7]).unwrap()],
+ )
+ .unwrap();
+ block_on(store.promote_projection_batch(
+ first,
+ ProjectionCheckpoint::new(id("cache"), generation(1), None, 1, 1).unwrap(),
+ ))
+ .unwrap();
+ block_on(store.configure_projection_cache(id("second"), limits)).unwrap();
+ let staged = ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(id("second"), generation(2), source, None).unwrap(),
+ vec![ProjectionDocument::new("b".into(), vec![7]).unwrap()],
+ )
+ .unwrap();
+ block_on(store.put_projection_batch(staged)).unwrap();
+ let third = ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(id("cache"), generation(3), source, Some(generation(1))).unwrap(),
+ vec![ProjectionDocument::new("c".into(), vec![7]).unwrap()],
+ )
+ .unwrap();
+ assert_eq!(
+ block_on(store.put_projection_batch(third)),
+ Err(Error::SpaceInsufficient)
+ );
+ let query = ProjectionDocumentQuery::new(
+ id("cache"),
+ ProjectionDocumentGenerations::Exact(generation(3)),
+ 1,
+ )
+ .unwrap();
+ assert!(
+ block_on(store.query_projection_documents(query))
+ .unwrap()
+ .records()
+ .is_empty()
+ );
+ assert_eq!(
+ block_on(store.configure_projection_cache(id("cache"), limits))
+ .unwrap()
+ .value_bytes,
+ 2
+ );
+ assert_eq!(
+ block_on(store.put_projection_document(
+ id("foreign"),
+ generation(1),
+ ProjectionDocument::new("too_big".into(), vec![7; 2]).unwrap()
+ )),
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(
+ block_on(store.projection_document(id("foreign"), generation(1), "too_big".into()))
+ .unwrap(),
+ None
+ );
+ assert_eq!(
+ block_on(store.configure_projection_cache(id("cache"), limits))
+ .unwrap()
+ .protected_generations,
+ 2
+ );
+}
+
+fn cache_batch(
+ source: EventSourceToken,
+ active: Option<ProjectionGeneration>,
+ target: u8,
+ keys: &[String],
+) -> radroots_storage::projection::batch::ProjectionDocumentBatch {
+ use radroots_storage::projection::batch::{ProjectionDocumentBatch, ProjectionWriteScope};
+ ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(id("cache"), generation(target), source, active).unwrap(),
+ keys.iter()
+ .map(|key| ProjectionDocument::new(key.clone(), vec![7]).unwrap())
+ .collect(),
+ )
+ .unwrap()
+}
+fn cache_scope(
+ source: EventSourceToken,
+ active: Option<ProjectionGeneration>,
+ target: u8,
+) -> ProjectionCacheScope {
+ ProjectionCacheScope::new(id("cache"), generation(target), source, active).unwrap()
+}
+fn cache_checkpoint(target: u8, count: u64) -> radroots_storage::projection::ProjectionCheckpoint {
+ radroots_storage::projection::ProjectionCheckpoint::new(
+ id("cache"),
+ generation(target),
+ None,
+ count,
+ 1,
+ )
+ .unwrap()
+}
+fn cache_limits() -> ProjectionCacheLimits {
+ ProjectionCacheLimits::new(67_108_864, 2, 32, 1_800_000).unwrap()
+}
+#[test]
+fn projection_cache_memory_live_staging_and_63_64_65_retirement_preserve_exact_continuation() {
+ use radroots_storage::projection::{InvalidationReason, ProjectionInvalidation};
+ let store = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ let source = block_on(store.source_token()).unwrap();
+ block_on(store.put_projection_document(
+ id("z_foreign"),
+ generation(9),
+ ProjectionDocument::new("keep".into(), vec![8]).unwrap(),
+ ))
+ .unwrap();
+ block_on(store.configure_projection_cache(id("cache"), cache_limits())).unwrap();
+ block_on(store.promote_projection_batch(
+ cache_batch(source, None, 1, &["active".into()]),
+ cache_checkpoint(1, 1),
+ ))
+ .unwrap();
+ assert_eq!(
+ block_on(store.retire_projection_generation(cache_scope(source, Some(generation(1)), 1))),
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ block_on(store.collect_projection_cache(cache_scope(source, Some(generation(1)), 1)))
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ assert_eq!(
+ block_on(store.collect_projection_cache(cache_scope(
+ EventSourceToken::new(source.generation(), 1),
+ Some(generation(1)),
+ 1
+ ))),
+ Err(Error::SourceEpochChanged)
+ );
+ assert_eq!(
+ block_on(store.collect_projection_cache(cache_scope(source, None, 1))),
+ Err(Error::ProjectionRevisionConflict)
+ );
+ for (target, count) in [(2, 63), (3, 64), (4, 65), (5, 0)] {
+ let keys: Vec<_> = (0..count.min(64))
+ .map(|number| format!("key.{number:02}"))
+ .collect();
+ block_on(store.put_projection_batch(cache_batch(
+ source,
+ Some(generation(1)),
+ target,
+ &keys,
+ )))
+ .unwrap();
+ if count == 65 {
+ block_on(store.put_projection_batch(cache_batch(
+ source,
+ Some(generation(1)),
+ target,
+ &["key.64".into()],
+ )))
+ .unwrap();
+ }
+ assert_eq!(
+ block_on(store.collect_projection_cache(cache_scope(
+ source,
+ Some(generation(1)),
+ target
+ )))
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ block_on(store.retire_projection_generation(cache_scope(
+ source,
+ Some(generation(1)),
+ target,
+ )))
+ .unwrap();
+ let first = block_on(store.collect_projection_cache(cache_scope(
+ source,
+ Some(generation(1)),
+ target,
+ )))
+ .unwrap();
+ assert_eq!(first.removed_rows, (count + 1).min(64));
+ assert_eq!(first.more, count >= 64);
+ if first.more {
+ let second = block_on(store.collect_projection_cache(cache_scope(
+ source,
+ Some(generation(1)),
+ target,
+ )))
+ .unwrap();
+ assert_eq!(second.removed_rows, count - 63);
+ assert!(!second.more);
+ }
+ assert_eq!(
+ block_on(store.configure_projection_cache(id("cache"), cache_limits()))
+ .unwrap()
+ .value_bytes,
+ 2
+ );
+ }
+ block_on(
+ store.invalidate(
+ ProjectionInvalidation::new(
+ id("cache"),
+ generation(1),
+ generation(6),
+ InvalidationReason::OperatorRequested,
+ 1,
+ )
+ .unwrap(),
+ ),
+ )
+ .unwrap();
+ block_on(store.retire_projection_generation(cache_scope(source, Some(generation(1)), 1)))
+ .unwrap();
+ assert_eq!(
+ block_on(ProjectionStore::status(&store, id("cache"))).unwrap(),
+ None
+ );
+ block_on(store.promote_projection_batch(
+ cache_batch(source, None, 6, &["successor".into()]),
+ cache_checkpoint(6, 1),
+ ))
+ .unwrap();
+ block_on(store.collect_projection_cache(cache_scope(source, Some(generation(6)), 1))).unwrap();
+ assert_eq!(
+ block_on(store.projection_document(id("z_foreign"), generation(9), "keep".into()))
+ .unwrap()
+ .unwrap()
+ .value(),
+ &[8]
+ );
+}
+
+#[test]
+fn projection_cache_memory_protected_legacy_pressure_future_and_expiry_collect_without_replay() {
+ let store = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ let source = block_on(store.source_token()).unwrap();
+ let now = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_millis() as u64;
+ block_on(store.checkpoint(cache_checkpoint(1, 0))).unwrap();
+ for number in 1..=33 {
+ block_on(
+ store.put_projection_snapshot(
+ ProjectionSnapshot::new(id("cache"), [number; 32], generation(1), now, vec![8])
+ .unwrap(),
+ ),
+ )
+ .unwrap();
+ }
+ block_on(store.put_projection_snapshot(
+ ProjectionSnapshot::new(id("z_foreign"), [1; 32], generation(9), now, vec![8]).unwrap(),
+ ))
+ .unwrap();
+ block_on(store.configure_projection_cache(id("cache"), cache_limits())).unwrap();
+ let first =
+ block_on(store.collect_projection_cache(cache_scope(source, Some(generation(1)), 1)))
+ .unwrap();
+ assert_eq!(first.removed_rows, 2);
+ assert!(!first.more);
+ assert_eq!(first.usage.legacy_snapshots, 32);
+ assert_eq!(
+ block_on(store.projection_snapshot(id("cache"), [1; 32])).unwrap(),
+ None
+ );
+ assert!(
+ block_on(store.projection_snapshot(id("cache"), [3; 32]))
+ .unwrap()
+ .is_some()
+ );
+ let future_store = MemoryStorage::new(source.generation());
+ block_on(future_store.checkpoint(cache_checkpoint(1, 0))).unwrap();
+ for number in 1..=65 {
+ block_on(
+ future_store.put_projection_snapshot(
+ ProjectionSnapshot::new(
+ id("cache"),
+ [number; 32],
+ generation(1),
+ now + 3_600_000,
+ vec![8],
+ )
+ .unwrap(),
+ ),
+ )
+ .unwrap();
+ }
+ block_on(
+ future_store.put_projection_snapshot(
+ ProjectionSnapshot::new(
+ id("z_foreign"),
+ [1; 32],
+ generation(9),
+ now + 3_600_000,
+ vec![8],
+ )
+ .unwrap(),
+ ),
+ )
+ .unwrap();
+ block_on(future_store.configure_projection_cache(id("cache"), cache_limits())).unwrap();
+ let first = block_on(future_store.collect_projection_cache(cache_scope(
+ source,
+ Some(generation(1)),
+ 1,
+ )))
+ .unwrap();
+ assert_eq!(first.removed_rows, 64);
+ assert!(first.more);
+ let second = block_on(future_store.collect_projection_cache(cache_scope(
+ source,
+ Some(generation(1)),
+ 1,
+ )))
+ .unwrap();
+ assert_eq!(second.removed_rows, 1);
+ assert!(!second.more);
+ assert_eq!(second.usage.legacy_snapshots, 1);
+ assert_eq!(
+ block_on(future_store.put_projection_snapshot(
+ ProjectionSnapshot::new(id("cache"), [1; 32], generation(1), now, vec![8]).unwrap()
+ )),
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert!(
+ block_on(future_store.projection_snapshot(id("z_foreign"), [1; 32]))
+ .unwrap()
+ .is_some()
+ );
+}
+
+async fn legacy_cache_completion(backend: &dyn ProjectionStore, registered: bool, capacity: u64) {
+ use radroots_storage::projection::{
+ InvalidationReason, ProjectionCheckpoint, ProjectionHealth, ProjectionInvalidation,
+ RawSourceDigest, RebuildTicket, RebuildTicketId, RebuildTransition,
+ batch::{ProjectionDocumentBatch, ProjectionWriteScope},
+ };
+ let id = ProjectionId::parse("legacy_cache_completion").unwrap();
+ let source = EventSourceToken::new(SourceGeneration::new([9; 32]).unwrap(), 0);
+ let limits = ProjectionCacheLimits::new(67_108_864, capacity, 32, 1_800_000).unwrap();
+ let checkpoint =
+ |target| ProjectionCheckpoint::new(id.clone(), generation(target), None, 1, 1).unwrap();
+ let batch = |target, active| {
+ ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(id.clone(), generation(target), source, active).unwrap(),
+ vec![ProjectionDocument::new("kept".into(), vec![7]).unwrap()],
+ )
+ .unwrap()
+ };
+ let scope = |target, active| {
+ ProjectionCacheScope::new(
+ id.clone(),
+ generation(target),
+ source,
+ Some(generation(active)),
+ )
+ .unwrap()
+ };
+ backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap();
+ if registered {
+ backend
+ .promote_projection_batch(batch(1, None), checkpoint(1))
+ .await
+ .unwrap();
+ } else {
+ backend.checkpoint(checkpoint(1)).await.unwrap();
+ }
+ backend
+ .put_projection_batch(batch(2, Some(generation(1))))
+ .await
+ .unwrap();
+ let invalidation = ProjectionInvalidation::new(
+ id.clone(),
+ generation(1),
+ generation(3),
+ InvalidationReason::OperatorRequested,
+ 10,
+ )
+ .unwrap();
+ backend.invalidate(invalidation.clone()).await.unwrap();
+ let ticket = RebuildTicket::requested(
+ RebuildTicketId::new([3; 16]).unwrap(),
+ invalidation,
+ source.generation(),
+ None,
+ RawSourceDigest::new([1; 32]),
+ )
+ .unwrap();
+ backend.request_rebuild(ticket.clone()).await.unwrap();
+ let running = backend
+ .transition_rebuild(RebuildTransition::start(
+ ticket.ticket_id(),
+ ticket.revision(),
+ 11,
+ ))
+ .await
+ .unwrap();
+ let before_status = backend.status(id.clone()).await.unwrap();
+ let before_usage = backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap();
+ assert_eq!(before_usage.protected_generations, 2);
+ let complete =
+ RebuildTransition::complete(running.ticket_id(), running.revision(), 12, checkpoint(3));
+ if registered && capacity == 2 {
+ assert_eq!(
+ backend.transition_rebuild(complete.clone()).await,
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(backend.status(id.clone()).await.unwrap(), before_status);
+ assert_eq!(
+ backend.rebuild(ticket.ticket_id()).await.unwrap(),
+ Some(running.clone())
+ );
+ assert_eq!(
+ backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap(),
+ before_usage
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(1, 1))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(2, 1))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ assert_eq!(
+ backend
+ .retire_projection_generation(scope(2, 1))
+ .await
+ .unwrap()
+ .protected_generations,
+ 1
+ );
+ }
+ let completed = backend.transition_rebuild(complete).await.unwrap();
+ assert_eq!(
+ backend.rebuild(ticket.ticket_id()).await.unwrap(),
+ Some(completed)
+ );
+ let status = backend.status(id.clone()).await.unwrap().unwrap();
+ assert_eq!(status.generation(), generation(3));
+ assert_eq!(status.health(), ProjectionHealth::Ready);
+ assert_eq!(status.active_rebuild(), None);
+ assert_eq!(status.checkpoint(), Some(&checkpoint(3)));
+ let expected = if registered && capacity == 3 { 3 } else { 2 };
+ assert_eq!(
+ backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap()
+ .protected_generations,
+ expected
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(3, 3))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ if registered {
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(1, 3))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ assert_eq!(
+ backend
+ .retire_projection_generation(scope(1, 3))
+ .await
+ .unwrap()
+ .protected_generations,
+ expected - 1
+ );
+ let collected = backend.collect_projection_cache(scope(1, 3)).await.unwrap();
+ assert_eq!(collected.removed_rows, 2);
+ assert_eq!(collected.removed_value_bytes, 1);
+ assert!(!collected.more);
+ assert_eq!(collected.usage.protected_generations, expected - 1);
+ }
+ if !registered || capacity == 3 {
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(2, 3))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ backend
+ .retire_projection_generation(scope(2, 3))
+ .await
+ .unwrap();
+ }
+ let collected = backend.collect_projection_cache(scope(2, 3)).await.unwrap();
+ assert_eq!(collected.removed_rows, 2);
+ assert_eq!(collected.removed_value_bytes, 1);
+ assert!(!collected.more);
+ assert_eq!(collected.usage.protected_generations, 1);
+ assert_eq!(backend.status(id.clone()).await.unwrap(), Some(status));
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(3, 3))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+}
+
+#[test]
+fn projection_cache_memory_legacy_completion_preserves_registry_capacity_and_atomic_refusal() {
+ for (registered, capacity) in [(true, 2), (true, 3), (false, 2)] {
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ block_on(legacy_cache_completion(&memory, registered, capacity));
+ }
+}
+
+async fn cache_batch_replacement(
+ backend: &dyn ProjectionStore,
+ quota: u64,
+ initial: [usize; 2],
+ replacement: [usize; 2],
+ reverse: bool,
+) {
+ use radroots_storage::projection::batch::{ProjectionDocumentBatch, ProjectionWriteScope};
+ let id = ProjectionId::parse("batch_replacement").unwrap();
+ let source = EventSourceToken::new(SourceGeneration::new([9; 32]).unwrap(), 0);
+ let batch = |sizes: [usize; 2], target, reverse| {
+ let mut documents = vec![
+ ProjectionDocument::new("a".into(), vec![1; sizes[0]]).unwrap(),
+ ProjectionDocument::new("b".into(), vec![2; sizes[1]]).unwrap(),
+ ];
+ if reverse {
+ documents.reverse();
+ }
+ ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(id.clone(), generation(target), source, None).unwrap(),
+ documents,
+ )
+ .unwrap()
+ };
+ backend
+ .put_projection_batch(batch(initial, 1, false))
+ .await
+ .unwrap();
+ let limits = ProjectionCacheLimits::new(quota, 2, 32, 1_800_000).unwrap();
+ let before = backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap();
+ let status = backend.status(id.clone()).await.unwrap();
+ let previous_bytes = initial.iter().sum::<usize>() as u64;
+ let next_bytes = replacement.iter().sum::<usize>() as u64;
+ let result = backend
+ .put_projection_batch(batch(replacement, 1, reverse))
+ .await;
+ let refused = next_bytes > previous_bytes && next_bytes > quota;
+ let expected = if refused {
+ assert_eq!(result, Err(Error::SpaceInsufficient));
+ assert_eq!(
+ backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap(),
+ before
+ );
+ initial
+ } else {
+ result.unwrap();
+ let current = backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap();
+ assert_eq!(current.value_bytes, next_bytes);
+ assert_eq!(current.protected_generations, 1);
+ replacement
+ };
+ assert_eq!(backend.status(id.clone()).await.unwrap(), status);
+ for (key, size, value) in [("a", expected[0], 1), ("b", expected[1], 2)] {
+ assert_eq!(
+ backend
+ .projection_document(id.clone(), generation(1), key.into())
+ .await
+ .unwrap()
+ .unwrap()
+ .value(),
+ vec![value; size]
+ );
+ }
+ let current = backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap();
+ let over = quota.saturating_sub(current.value_bytes) + 1;
+ let foreign = ProjectionId::parse("direct_writer").unwrap();
+ assert_eq!(
+ backend
+ .put_projection_document(
+ foreign.clone(),
+ generation(9),
+ ProjectionDocument::new("overflow".into(), vec![9; over as usize]).unwrap()
+ )
+ .await,
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(
+ backend
+ .projection_document(foreign, generation(9), "overflow".into())
+ .await
+ .unwrap(),
+ None
+ );
+ assert_eq!(
+ backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap(),
+ current
+ );
+ assert_eq!(
+ backend
+ .put_projection_batch(
+ ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(
+ ProjectionId::parse("new_admission").unwrap(),
+ generation(2),
+ source,
+ None
+ )
+ .unwrap(),
+ vec![ProjectionDocument::new("a".into(), vec![7; over as usize]).unwrap()]
+ )
+ .unwrap()
+ )
+ .await,
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(
+ backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap(),
+ current
+ );
+ assert_eq!(
+ backend
+ .projection_document(
+ ProjectionId::parse("new_admission").unwrap(),
+ generation(2),
+ "a".into()
+ )
+ .await
+ .unwrap(),
+ None
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(
+ ProjectionCacheScope::new(id.clone(), generation(1), source, None).unwrap()
+ )
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ backend
+ .retire_projection_generation(
+ ProjectionCacheScope::new(id.clone(), generation(1), source, None).unwrap(),
+ )
+ .await
+ .unwrap();
+ let collected = backend
+ .collect_projection_cache(
+ ProjectionCacheScope::new(id, generation(1), source, None).unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(collected.removed_rows, 3);
+ assert!(!collected.more);
+ assert_eq!(collected.usage.value_bytes, 0);
+ assert_eq!(collected.usage.protected_generations, 0);
+}
+
+#[test]
+fn projection_cache_memory_batch_replacement_has_one_aggregate_policy_in_both_orders() {
+ for (quota, initial, replacement) in [
+ (4, [2, 2], [3, 1]),
+ (4, [2, 2], [2, 1]),
+ (6, [4, 4], [5, 2]),
+ (6, [4, 4], [5, 4]),
+ (6, [1, 1], [2, 2]),
+ ] {
+ for reverse in [false, true] {
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ block_on(cache_batch_replacement(
+ &memory,
+ quota,
+ initial,
+ replacement,
+ reverse,
+ ));
+ }
+ }
+}
diff --git a/crates/storage_sqlite/src/migration.rs b/crates/storage_sqlite/src/migration.rs
@@ -469,7 +469,7 @@ fn schema_metadata_error(source: &sqlx::Error, database: &'static str) -> Error
fn validate_plan(plan: &MigrationPlan) -> Result<(), Error> {
let valid = plan.minimum_version > 0
&& plan.minimum_version <= plan.current_version
- && plan.current_version <= 21
+ && plan.current_version <= 22
&& plan.steps.len() == usize::try_from(plan.current_version).unwrap_or(usize::MAX)
&& plan
.steps
@@ -597,6 +597,7 @@ const fn set_user_version_sql(version: u32) -> Option<&'static str> {
19 => Some("PRAGMA user_version = 19"),
20 => Some("PRAGMA user_version = 20"),
21 => Some("PRAGMA user_version = 21"),
+ 22 => Some("PRAGMA user_version = 22"),
_ => None,
}
}
@@ -881,8 +882,8 @@ mod tests {
.await
.expect("application id");
let newer_version = runtime::CURRENT_VERSION + 1;
- assert_eq!(newer_version, 22);
- sqlx::raw_sql("PRAGMA user_version = 22")
+ assert_eq!(newer_version, 23);
+ sqlx::raw_sql("PRAGMA user_version = 23")
.execute(&mut newer)
.await
.expect("newer version");
diff --git a/crates/storage_sqlite/src/migration/runtime/0022_projection_cache_budget.up.sql b/crates/storage_sqlite/src/migration/runtime/0022_projection_cache_budget.up.sql
@@ -0,0 +1,74 @@
+CREATE TABLE radroots_runtime_projection_cache_budget (
+ singleton INTEGER PRIMARY KEY CHECK(singleton=1),
+ value_bytes INTEGER NOT NULL CHECK(value_bytes>=0),
+ protected_generations INTEGER NOT NULL CHECK(protected_generations>=0),
+ legacy_snapshots INTEGER NOT NULL CHECK(legacy_snapshots>=0),
+ max_value_bytes INTEGER CHECK(max_value_bytes>0),
+ max_protected_generations INTEGER CHECK(max_protected_generations>0),
+ max_legacy_snapshots INTEGER CHECK(max_legacy_snapshots>0),
+ legacy_age_ms INTEGER CHECK(legacy_age_ms>0),
+ batch_value_ceiling INTEGER CHECK(batch_value_ceiling>=0),
+ owner_time_ms INTEGER NOT NULL DEFAULT 0 CHECK(owner_time_ms>=0)
+) STRICT;
+CREATE TABLE radroots_runtime_projection_cache_owners (
+ projection_id TEXT PRIMARY KEY CHECK(length(projection_id) BETWEEN 1 AND 128)
+) STRICT, WITHOUT ROWID;
+INSERT INTO radroots_runtime_projection_cache_budget(singleton,value_bytes,protected_generations,legacy_snapshots)
+SELECT 1,
+ (SELECT coalesce(sum(length(value)),0) FROM radroots_runtime_projection_documents)
+ +(SELECT coalesce(sum(length(value)),0) FROM radroots_runtime_projection_snapshots),
+ (SELECT count(*) FROM radroots_runtime_projection_generations WHERE phase!='retired')
+ +(SELECT count(*) FROM radroots_runtime_projection_checkpoints AS c WHERE NOT EXISTS
+ (SELECT 1 FROM radroots_runtime_projection_generations AS g WHERE g.projection_id=c.projection_id AND g.generation=c.projection_generation)),
+ (SELECT count(*) FROM radroots_runtime_projection_snapshots);
+CREATE INDEX radroots_runtime_projection_snapshots_generation_created_idx
+ON radroots_runtime_projection_snapshots(projection_id,generation,created_at_unix_ms,snapshot_id);
+CREATE TRIGGER radroots_runtime_projection_cache_document_insert AFTER INSERT ON radroots_runtime_projection_documents
+BEGIN
+ UPDATE radroots_runtime_projection_cache_budget SET value_bytes=value_bytes+(length(NEW.value)) WHERE singleton=1;
+END;
+CREATE TRIGGER radroots_runtime_projection_cache_document_delete AFTER DELETE ON radroots_runtime_projection_documents
+BEGIN
+ UPDATE radroots_runtime_projection_cache_budget SET value_bytes=value_bytes+(-length(OLD.value)) WHERE singleton=1;
+END;
+CREATE TRIGGER radroots_runtime_projection_cache_document_update AFTER UPDATE OF value ON radroots_runtime_projection_documents
+BEGIN
+ UPDATE radroots_runtime_projection_cache_budget SET value_bytes=value_bytes+(length(NEW.value)-length(OLD.value)) WHERE singleton=1;
+END;
+CREATE TRIGGER radroots_runtime_projection_cache_snapshot_insert AFTER INSERT ON radroots_runtime_projection_snapshots
+BEGIN
+ UPDATE radroots_runtime_projection_cache_budget SET value_bytes=value_bytes+(length(NEW.value)),legacy_snapshots=legacy_snapshots+(1) WHERE singleton=1;
+END;
+CREATE TRIGGER radroots_runtime_projection_cache_snapshot_delete AFTER DELETE ON radroots_runtime_projection_snapshots
+BEGIN
+ UPDATE radroots_runtime_projection_cache_budget SET value_bytes=value_bytes+(-length(OLD.value)),legacy_snapshots=legacy_snapshots+(-1) WHERE singleton=1;
+END;
+CREATE TRIGGER radroots_runtime_projection_cache_generation_insert AFTER INSERT ON radroots_runtime_projection_generations
+BEGIN
+ UPDATE radroots_runtime_projection_cache_budget SET protected_generations=protected_generations+(NEW.phase!='retired')-(EXISTS(SELECT 1 FROM radroots_runtime_projection_checkpoints WHERE projection_id=NEW.projection_id AND projection_generation=NEW.generation)) WHERE singleton=1;
+END;
+CREATE TRIGGER radroots_runtime_projection_cache_generation_delete AFTER DELETE ON radroots_runtime_projection_generations
+BEGIN
+ UPDATE radroots_runtime_projection_cache_budget SET protected_generations=protected_generations-(OLD.phase!='retired')+(EXISTS(SELECT 1 FROM radroots_runtime_projection_checkpoints WHERE projection_id=OLD.projection_id AND projection_generation=OLD.generation)) WHERE singleton=1;
+END;
+CREATE TRIGGER radroots_runtime_projection_cache_generation_update AFTER UPDATE OF phase ON radroots_runtime_projection_generations
+BEGIN
+ UPDATE radroots_runtime_projection_cache_budget SET protected_generations=protected_generations+(NEW.phase!='retired')-(OLD.phase!='retired') WHERE singleton=1;
+END;
+CREATE TRIGGER radroots_runtime_projection_cache_checkpoint_insert AFTER INSERT ON radroots_runtime_projection_checkpoints
+BEGIN
+ UPDATE radroots_runtime_projection_cache_budget SET protected_generations=protected_generations+NOT(EXISTS(SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id=NEW.projection_id AND generation=NEW.projection_generation)) WHERE singleton=1;
+END;
+CREATE TRIGGER radroots_runtime_projection_cache_checkpoint_delete AFTER DELETE ON radroots_runtime_projection_checkpoints
+BEGIN
+ UPDATE radroots_runtime_projection_cache_budget SET protected_generations=protected_generations-NOT(EXISTS(SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id=OLD.projection_id AND generation=OLD.projection_generation)) WHERE singleton=1;
+END;
+CREATE TRIGGER radroots_runtime_projection_cache_checkpoint_update AFTER UPDATE OF projection_generation ON radroots_runtime_projection_checkpoints
+BEGIN
+ UPDATE radroots_runtime_projection_cache_budget SET protected_generations=protected_generations+NOT(EXISTS(SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id=NEW.projection_id AND generation=NEW.projection_generation))-NOT(EXISTS(SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id=OLD.projection_id AND generation=OLD.projection_generation)) WHERE singleton=1;
+END;
+CREATE TRIGGER radroots_runtime_projection_cache_capacity_guard BEFORE UPDATE ON radroots_runtime_projection_cache_budget
+WHEN (NEW.value_bytes>OLD.value_bytes AND NEW.value_bytes>coalesce(NEW.batch_value_ceiling,NEW.max_value_bytes))
+ OR (NEW.protected_generations>OLD.protected_generations AND NEW.protected_generations>NEW.max_protected_generations)
+ OR (NEW.legacy_snapshots>OLD.legacy_snapshots AND NEW.legacy_snapshots>NEW.max_legacy_snapshots)
+BEGIN SELECT RAISE(ABORT,'projection cache capacity exceeded'); END;
diff --git a/crates/storage_sqlite/src/migration/runtime/mod.rs b/crates/storage_sqlite/src/migration/runtime/mod.rs
@@ -6,7 +6,7 @@
/// Lowest runtime schema version this package can recognize.
pub const MINIMUM_VERSION: u32 = 1;
/// Current runtime schema version created by this package.
-pub const CURRENT_VERSION: u32 = 21;
+pub const CURRENT_VERSION: u32 = 22;
const RUNTIME_V1_SQL: &str = include_str!("0001_runtime.up.sql");
const CANONICAL_EVENT_STORAGE_V2_SQL: &str = include_str!("0002_canonical_event_storage.up.sql");
@@ -38,6 +38,8 @@ const EVENT_MUTATION_EPOCH_V18_SQL: &str = include_str!("0018_event_mutation_epo
const CANONICAL_VISIBILITY_INDEX_V19_SQL: &str =
include_str!("0019_canonical_visibility_index.up.sql");
+const PROJECTION_CACHE_BUDGET_V22_SQL: &str = include_str!("0022_projection_cache_budget.up.sql");
+
const PROJECTION_ATOMIC_GENERATIONS_V21_SQL: &str =
include_str!("0021_projection_atomic_generations.up.sql");
@@ -1322,6 +1324,133 @@ const RUNTIME_V21_OBJECTS: &[&str] = &[
"radroots_runtime_visibility_progress",
];
+const RUNTIME_V22_OBJECTS: &[&str] = &[
+ "radroots_runtime_atomic_commits",
+ "radroots_runtime_authored_artifacts",
+ "radroots_runtime_authored_artifacts_admission_ready_idx",
+ "radroots_runtime_authored_artifacts_signed_fact_guard",
+ "radroots_runtime_authored_artifacts_signing_ready_idx",
+ "radroots_runtime_authored_atomic_commits",
+ "radroots_runtime_authored_atomic_commits_delete_guard",
+ "radroots_runtime_authored_atomic_commits_update_guard",
+ "radroots_runtime_authored_atomic_target_phase_idx",
+ "radroots_runtime_authored_delivery_attempts",
+ "radroots_runtime_authored_delivery_claims",
+ "radroots_runtime_authored_delivery_claims_delete_guard",
+ "radroots_runtime_authored_delivery_claims_update_guard",
+ "radroots_runtime_authored_delivery_facts",
+ "radroots_runtime_authored_delivery_facts_delete_guard",
+ "radroots_runtime_authored_delivery_facts_update_guard",
+ "radroots_runtime_authored_delivery_plans",
+ "radroots_runtime_authored_delivery_ready_idx",
+ "radroots_runtime_authored_delivery_reconciliations",
+ "radroots_runtime_authored_delivery_reconciliations_delete_guard",
+ "radroots_runtime_authored_delivery_reconciliations_update_guard",
+ "radroots_runtime_authored_delivery_stop_guard",
+ "radroots_runtime_authored_delivery_targets",
+ "radroots_runtime_authored_draft_author_head_idx",
+ "radroots_runtime_authored_draft_revisions",
+ "radroots_runtime_authored_draft_revisions_delete_guard",
+ "radroots_runtime_authored_draft_revisions_update_guard",
+ "radroots_runtime_authored_draft_scope_head_idx",
+ "radroots_runtime_authored_migration_evidence",
+ "radroots_runtime_authored_migration_evidence_delete_guard",
+ "radroots_runtime_authored_migration_evidence_update_guard",
+ "radroots_runtime_authored_operations",
+ "radroots_runtime_delivery_evidence",
+ "radroots_runtime_delivery_evidence_item_idx",
+ "radroots_runtime_event_index_checkpoints",
+ "radroots_runtime_event_index_manifests",
+ "radroots_runtime_event_index_shards",
+ "radroots_runtime_event_provenance",
+ "radroots_runtime_event_provenance_observed_idx",
+ "radroots_runtime_events",
+ "radroots_runtime_events_admission_idx",
+ "radroots_runtime_events_contract_metadata_guard",
+ "radroots_runtime_events_contract_metadata_insert_guard",
+ "radroots_runtime_events_delete_guard",
+ "radroots_runtime_events_event_id_idx",
+ "radroots_runtime_events_mutation_idx",
+ "radroots_runtime_events_mutation_insert",
+ "radroots_runtime_events_mutation_update",
+ "radroots_runtime_events_raw_update_guard",
+ "radroots_runtime_journal_idempotency_idx",
+ "radroots_runtime_journal_operations",
+ "radroots_runtime_journal_recovery_idx",
+ "radroots_runtime_legacy_event_staging",
+ "radroots_runtime_legacy_event_staging_delete_guard",
+ "radroots_runtime_legacy_event_staging_insert_guard",
+ "radroots_runtime_legacy_event_staging_update_guard",
+ "radroots_runtime_legacy_import_commit_delete_guard",
+ "radroots_runtime_legacy_import_commit_update_guard",
+ "radroots_runtime_legacy_import_commits",
+ "radroots_runtime_legacy_import_delete_guard",
+ "radroots_runtime_legacy_import_identity_guard",
+ "radroots_runtime_legacy_import_member_delete_guard",
+ "radroots_runtime_legacy_import_member_identity_guard",
+ "radroots_runtime_legacy_import_member_state_guard",
+ "radroots_runtime_legacy_import_members",
+ "radroots_runtime_legacy_import_state_guard",
+ "radroots_runtime_legacy_import_state_idx",
+ "radroots_runtime_legacy_imports",
+ "radroots_runtime_legacy_outbox_staging",
+ "radroots_runtime_legacy_outbox_staging_delete_guard",
+ "radroots_runtime_legacy_outbox_staging_insert_guard",
+ "radroots_runtime_legacy_outbox_staging_parent_idx",
+ "radroots_runtime_legacy_outbox_staging_update_guard",
+ "radroots_runtime_outbox_items",
+ "radroots_runtime_outbox_operation_idx",
+ "radroots_runtime_outbox_ready_idx",
+ "radroots_runtime_outbox_targets",
+ "radroots_runtime_projection_cache_budget",
+ "radroots_runtime_projection_cache_capacity_guard",
+ "radroots_runtime_projection_cache_checkpoint_delete",
+ "radroots_runtime_projection_cache_checkpoint_insert",
+ "radroots_runtime_projection_cache_checkpoint_update",
+ "radroots_runtime_projection_cache_document_delete",
+ "radroots_runtime_projection_cache_document_insert",
+ "radroots_runtime_projection_cache_document_update",
+ "radroots_runtime_projection_cache_generation_delete",
+ "radroots_runtime_projection_cache_generation_insert",
+ "radroots_runtime_projection_cache_generation_update",
+ "radroots_runtime_projection_cache_owners",
+ "radroots_runtime_projection_cache_snapshot_delete",
+ "radroots_runtime_projection_cache_snapshot_insert",
+ "radroots_runtime_projection_checkpoints",
+ "radroots_runtime_projection_documents",
+ "radroots_runtime_projection_generations",
+ "radroots_runtime_projection_generations_staging_idx",
+ "radroots_runtime_projection_invalidations",
+ "radroots_runtime_projection_rebuilds",
+ "radroots_runtime_projection_rebuilds_stage_idx",
+ "radroots_runtime_projection_snapshots",
+ "radroots_runtime_projection_snapshots_created_idx",
+ "radroots_runtime_projection_snapshots_generation_created_idx",
+ "radroots_runtime_projection_snapshots_update_guard",
+ "radroots_runtime_source_generations",
+ "radroots_runtime_source_generations_active_idx",
+ "radroots_runtime_source_generations_delete_guard",
+ "radroots_runtime_source_generations_identity_guard",
+ "radroots_runtime_source_generations_mutation_guard",
+ "radroots_runtime_source_generations_sequence_guard",
+ "radroots_runtime_visibility_deletion_addresses",
+ "radroots_runtime_visibility_deletion_addresses_request_idx",
+ "radroots_runtime_visibility_deletion_events",
+ "radroots_runtime_visibility_deletion_events_request_idx",
+ "radroots_runtime_visibility_empty_source",
+ "radroots_runtime_visibility_events",
+ "radroots_runtime_visibility_events_head_idx",
+ "radroots_runtime_visibility_events_stage_idx",
+ "radroots_runtime_visibility_heads",
+ "radroots_runtime_visibility_members",
+ "radroots_runtime_visibility_members_count_insert",
+ "radroots_runtime_visibility_members_count_update",
+ "radroots_runtime_visibility_members_event_idx",
+ "radroots_runtime_visibility_members_page_idx",
+ "radroots_runtime_visibility_pending",
+ "radroots_runtime_visibility_progress",
+];
+
pub const MIGRATIONS: &[MigrationDescriptor] = &[
MigrationDescriptor {
version: 1,
@@ -1449,6 +1578,12 @@ pub const MIGRATIONS: &[MigrationDescriptor] = &[
up_sha256: "e50dc6896a40547febf7df26f00ad66814e59d05c7bc9b09e474843195f3b79e",
owned_objects: RUNTIME_V21_OBJECTS,
},
+ MigrationDescriptor {
+ version: 22,
+ name: "projection_cache_budget",
+ up_sha256: "fa0b0c26e0ac782b77abc072f67f79c135549c8970dc9ced093220460451531d",
+ owned_objects: RUNTIME_V22_OBJECTS,
+ },
];
pub(crate) const fn migration_sql(version: u32) -> Option<&'static str> {
@@ -1474,6 +1609,7 @@ pub(crate) const fn migration_sql(version: u32) -> Option<&'static str> {
19 => Some(CANONICAL_VISIBILITY_INDEX_V19_SQL),
20 => Some(INDEXED_VISIBILITY_READS_V20_SQL),
21 => Some(PROJECTION_ATOMIC_GENERATIONS_V21_SQL),
+ 22 => Some(PROJECTION_CACHE_BUDGET_V22_SQL),
_ => None,
}
}
@@ -1517,8 +1653,8 @@ mod tests {
fn migration_plan_matches_governed_snapshot() {
let snapshot = toml::from_str::<PlanSnapshot>(PLAN_SNAPSHOT).expect("valid snapshot");
assert_eq!(MINIMUM_VERSION, 1);
- assert_eq!(CURRENT_VERSION, 21);
- assert_eq!(MIGRATIONS.len(), 21);
+ assert_eq!(CURRENT_VERSION, 22);
+ assert_eq!(MIGRATIONS.len(), 22);
let migration = MIGRATIONS[8];
assert_eq!(snapshot.schema_version, 1);
assert_eq!(snapshot.database, "runtime.sqlite");
diff --git a/crates/storage_sqlite/src/projection/batch.rs b/crates/storage_sqlite/src/projection/batch.rs
@@ -1,4 +1,4 @@
-use super::{SqliteStorage, decode_status, map_backend, put_status_transaction};
+use super::{SqliteStorage, decode_status, put_status_transaction};
use radroots_storage::{
Error,
event::EventSourceToken,
@@ -20,7 +20,7 @@ pub(super) async fn fence(
return Err(Error::SourceEpochChanged);
}
let row = sqlx::query("SELECT mutation_epoch, sequence_head FROM radroots_runtime_source_generations WHERE generation = ? AND state = 'active'")
- .bind(source.generation().as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(map_backend)?
+ .bind(source.generation().as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(super::cache::map)?
.ok_or(Error::SourceEpochChanged)?;
let epoch = super::u64_from_i64(row.try_get("mutation_epoch").map_err(super::map_corrupt)?)?;
if epoch != source.epoch() {
@@ -32,7 +32,7 @@ pub(super) async fn fence(
.bind(id.as_str())
.fetch_optional(&mut **transaction)
.await
- .map_err(map_backend)?
+ .map_err(super::cache::map)?
.as_ref()
.map(decode_status)
.transpose()?;
@@ -55,7 +55,7 @@ pub(super) async fn read_fence(
) -> Result<(), Error> {
fence(transaction, store, source, id, Some(active)).await?;
let row = sqlx::query("SELECT source_generation, source_epoch, phase FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?")
- .bind(id.as_str()).bind(active.as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(map_backend)?
+ .bind(id.as_str()).bind(active.as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(super::cache::map)?
.ok_or(Error::ProjectionRevisionConflict)?;
let phase: String = row.try_get("phase").map_err(super::map_corrupt)?;
if phase != "promoted" {
@@ -77,7 +77,7 @@ pub(super) async fn refuse_registered_legacy(
generation: ProjectionGeneration,
) -> Result<(), Error> {
let registered: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?")
- .bind(id.as_str()).bind(generation.as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(map_backend)?;
+ .bind(id.as_str()).bind(generation.as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(super::cache::map)?;
if registered.is_some() {
return Err(Error::ProjectionRevisionConflict);
}
@@ -94,7 +94,7 @@ pub(super) async fn write(
.pool()
.begin_with("BEGIN IMMEDIATE")
.await
- .map_err(map_backend)?;
+ .map_err(super::cache::map)?;
let scope = batch.scope();
let sequence = fence(
&mut transaction,
@@ -106,7 +106,7 @@ pub(super) async fn write(
.await?;
let prior = sqlx::query("SELECT source_generation, source_epoch, expected_active, phase, document_rows, value_bytes FROM radroots_runtime_projection_generations WHERE projection_id = ? AND generation = ?")
.bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice())
- .fetch_optional(&mut *transaction).await.map_err(map_backend)?;
+ .fetch_optional(&mut *transaction).await.map_err(super::cache::map)?;
let (mut count, mut bytes) = if let Some(row) = prior {
let generation: Vec<u8> = row
.try_get("source_generation")
@@ -131,31 +131,70 @@ pub(super) async fn write(
)
} else {
let staging: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id = ? AND phase = 'staging' LIMIT 1")
- .bind(scope.projection_id().as_str()).fetch_optional(&mut *transaction).await.map_err(map_backend)?;
+ .bind(scope.projection_id().as_str()).fetch_optional(&mut *transaction).await.map_err(super::cache::map)?;
let existing: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_documents WHERE projection_id = ? AND generation = ? LIMIT 1")
.bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice())
- .fetch_optional(&mut *transaction).await.map_err(map_backend)?;
+ .fetch_optional(&mut *transaction).await.map_err(super::cache::map)?;
if staging.is_some() || existing.is_some() {
return Err(Error::ProjectionRevisionConflict);
}
sqlx::query("INSERT INTO radroots_runtime_projection_generations (projection_id,generation,source_generation,source_epoch,expected_active,phase) VALUES (?,?,?,?,?,'staging')")
.bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice())
.bind(scope.source().generation().as_bytes().as_slice()).bind(super::i64_from_u64(scope.source().epoch())?)
- .bind(scope.expected_active().map(|value| value.as_bytes().to_vec())).execute(&mut *transaction).await.map_err(map_backend)?;
+ .bind(scope.expected_active().map(|value| value.as_bytes().to_vec())).execute(&mut *transaction).await.map_err(super::cache::map)?;
(0, 0)
};
+ let mut writes = Vec::with_capacity(batch.documents().len());
+ let mut replaced_bytes = 0u64;
+ let mut replacement_bytes = 0u64;
for document in batch.documents() {
let previous: Option<i64> = sqlx::query_scalar("SELECT length(value) FROM radroots_runtime_projection_documents WHERE projection_id = ? AND generation = ? AND document_key = ?")
.bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind(document.key())
- .fetch_optional(&mut *transaction).await.map_err(map_backend)?;
+ .fetch_optional(&mut *transaction).await.map_err(super::cache::map)?;
count = count
.checked_add(u64::from(previous.is_none()))
.ok_or(Error::SpaceInsufficient)?;
let previous = previous.map(super::u64_from_i64).transpose()?.unwrap_or(0);
- bytes = bytes
- .checked_sub(previous)
- .and_then(|size| size.checked_add(document.value().len() as u64))
+ replaced_bytes = replaced_bytes
+ .checked_add(previous)
.ok_or(Error::CorruptProjectionRecord)?;
+ replacement_bytes = replacement_bytes
+ .checked_add(document.value().len() as u64)
+ .ok_or(Error::SpaceInsufficient)?;
+ writes.push((document, previous));
+ }
+ bytes = bytes
+ .checked_sub(replaced_bytes)
+ .and_then(|size| size.checked_add(replacement_bytes))
+ .ok_or(Error::CorruptProjectionRecord)?;
+ let budget = sqlx::query("SELECT value_bytes,max_value_bytes,batch_value_ceiling FROM radroots_runtime_projection_cache_budget WHERE singleton=1")
+ .fetch_one(&mut *transaction).await.map_err(super::cache::map)?;
+ let current_bytes =
+ super::u64_from_i64(budget.try_get("value_bytes").map_err(super::map_corrupt)?)?;
+ let limit = budget
+ .try_get::<Option<i64>, _>("max_value_bytes")
+ .map_err(super::map_corrupt)?;
+ if budget
+ .try_get::<Option<i64>, _>("batch_value_ceiling")
+ .map_err(super::map_corrupt)?
+ .is_some()
+ {
+ return Err(Error::CorruptProjectionRecord);
+ }
+ let final_bytes = current_bytes
+ .checked_sub(replaced_bytes)
+ .and_then(|value| value.checked_add(replacement_bytes))
+ .ok_or(Error::CorruptProjectionRecord)?;
+ if final_bytes > current_bytes && limit.is_some_and(|limit| final_bytes > limit as u64) {
+ return Err(Error::SpaceInsufficient);
+ }
+ // The immutable policy is unchanged. This reservation exists only inside
+ // this transaction, admits its final aggregate and rolls back on failure.
+ sqlx::query("UPDATE radroots_runtime_projection_cache_budget SET batch_value_ceiling=? WHERE singleton=1")
+ .bind(super::i64_from_u64(final_bytes)?).execute(&mut *transaction).await.map_err(super::cache::map)?;
+ // Shrink before growth, so no intermediate increase exceeds the admitted total.
+ writes.sort_by_key(|(document, previous)| document.value().len() as u64 > *previous);
+ for (document, _) in writes {
put(
&mut transaction,
scope.projection_id(),
@@ -186,7 +225,7 @@ pub(super) async fn write(
None,
)?;
sqlx::query("UPDATE radroots_runtime_projection_generations SET phase = 'retired' WHERE projection_id = ? AND phase = 'promoted'")
- .bind(scope.projection_id().as_str()).execute(&mut *transaction).await.map_err(map_backend)?;
+ .bind(scope.projection_id().as_str()).execute(&mut *transaction).await.map_err(super::cache::map)?;
put_status_transaction(&mut transaction, &status).await?;
Some(status)
} else {
@@ -196,8 +235,13 @@ pub(super) async fn write(
.bind(super::i64_from_u64(count)?).bind(super::i64_from_u64(bytes)?)
.bind(if promoted.is_some() { "promoted" } else { "staging" })
.bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice())
- .execute(&mut *transaction).await.map_err(map_backend)?;
- transaction.commit().await.map_err(map_backend)?;
+ .execute(&mut *transaction).await.map_err(super::cache::map)?;
+ let cleared = sqlx::query("UPDATE radroots_runtime_projection_cache_budget SET batch_value_ceiling=NULL WHERE singleton=1 AND batch_value_ceiling=? AND value_bytes=?")
+ .bind(super::i64_from_u64(final_bytes)?).bind(super::i64_from_u64(final_bytes)?).execute(&mut *transaction).await.map_err(super::cache::map)?;
+ if cleared.rows_affected() != 1 {
+ return Err(Error::CorruptProjectionRecord);
+ }
+ transaction.commit().await.map_err(super::cache::map)?;
Ok(promoted)
}
@@ -209,7 +253,7 @@ pub(super) async fn put(
) -> Result<(), Error> {
sqlx::query("INSERT INTO radroots_runtime_projection_documents (projection_id,generation,document_key,value,value_sha256) VALUES (?,?,?,?,?) ON CONFLICT(projection_id,generation,document_key) DO UPDATE SET value=excluded.value,value_sha256=excluded.value_sha256")
.bind(id.as_str()).bind(generation.as_bytes().as_slice()).bind(document.key()).bind(document.value())
- .bind(document.value_sha256().as_slice()).execute(&mut **transaction).await.map_err(map_backend)?;
+ .bind(document.value_sha256().as_slice()).execute(&mut **transaction).await.map_err(super::cache::map)?;
Ok(())
}
@@ -224,14 +268,17 @@ pub(super) async fn legacy_put(
.pool()
.begin_with("BEGIN IMMEDIATE")
.await
- .map_err(map_backend)?;
+ .map_err(super::cache::map)?;
+ if super::cache::owner(&mut transaction, &id).await? {
+ return Err(Error::ProjectionRevisionConflict);
+ }
let registered: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?")
- .bind(id.as_str()).bind(generation.as_bytes().as_slice()).fetch_optional(&mut *transaction).await.map_err(map_backend)?;
+ .bind(id.as_str()).bind(generation.as_bytes().as_slice()).fetch_optional(&mut *transaction).await.map_err(super::cache::map)?;
if registered.is_some() {
return Err(Error::ProjectionRevisionConflict);
}
put(&mut transaction, &id, generation, &document).await?;
- transaction.commit().await.map_err(map_backend)
+ transaction.commit().await.map_err(super::cache::map)
}
pub(super) async fn load_document(
@@ -245,10 +292,10 @@ pub(super) async fn load_document(
key.clone(),
)
.map_err(|_| Error::InvalidProjectionDocument)?;
- let mut transaction = store.pool().begin().await.map_err(map_backend)?;
+ let mut transaction = store.pool().begin().await.map_err(super::cache::map)?;
let metadata = sqlx::query("SELECT length(value) AS value_bytes, CASE WHEN length(value_sha256)=32 THEN value_sha256 END AS value_sha256 FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? AND document_key=?")
.bind(id.as_str()).bind(generation.as_bytes().as_slice()).bind(&key)
- .fetch_optional(&mut *transaction).await.map_err(map_backend)?;
+ .fetch_optional(&mut *transaction).await.map_err(super::cache::map)?;
let document = if let Some(row) = metadata {
let length: i64 = row.try_get("value_bytes").map_err(super::map_corrupt)?;
if length <= 0
@@ -264,12 +311,12 @@ pub(super) async fn load_document(
.map_err(|_| Error::CorruptProjectionDocument)?;
let value: Vec<u8> = sqlx::query_scalar("SELECT value FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? AND document_key=?")
.bind(id.as_str()).bind(generation.as_bytes().as_slice()).bind(&key)
- .fetch_one(&mut *transaction).await.map_err(map_backend)?;
+ .fetch_one(&mut *transaction).await.map_err(super::cache::map)?;
Some(ProjectionDocument::from_stored_parts(key, value, digest)?)
} else {
None
};
- transaction.rollback().await.map_err(map_backend)?;
+ transaction.rollback().await.map_err(super::cache::map)?;
Ok(document)
}
diff --git a/crates/storage_sqlite/src/projection/cache.rs b/crates/storage_sqlite/src/projection/cache.rs
@@ -0,0 +1,398 @@
+use super::{SqliteStorage, decode_status, i64_from_u64, map_backend, map_corrupt, u64_from_i64};
+use radroots_storage::{
+ Error,
+ projection::{
+ ProjectionHealth, ProjectionId, ProjectionSnapshot,
+ cache::{
+ PROJECTION_CACHE_COLLECTION_ROWS_MAX, ProjectionCacheCollection, ProjectionCacheLimits,
+ ProjectionCacheScope, ProjectionCacheUsage,
+ },
+ },
+};
+use sqlx::{Row, Sqlite, Transaction};
+
+pub(super) fn map(error: sqlx::Error) -> Error {
+ if error
+ .as_database_error()
+ .is_some_and(|value| value.message() == "projection cache capacity exceeded")
+ {
+ Error::SpaceInsufficient
+ } else {
+ map_backend(error)
+ }
+}
+fn now() -> Result<u64, Error> {
+ u64::try_from(
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .map_err(|_| Error::InvalidProjectionTimestamp)?
+ .as_millis(),
+ )
+ .map_err(|_| Error::InvalidProjectionTimestamp)
+}
+pub(super) async fn owner(
+ transaction: &mut Transaction<'_, Sqlite>,
+ id: &ProjectionId,
+) -> Result<bool, Error> {
+ let row: Option<i64> = sqlx::query_scalar(
+ "SELECT 1 FROM radroots_runtime_projection_cache_owners WHERE projection_id=?",
+ )
+ .bind(id.as_str())
+ .fetch_optional(&mut **transaction)
+ .await
+ .map_err(map)?;
+ Ok(row.is_some())
+}
+async fn usage(transaction: &mut Transaction<'_, Sqlite>) -> Result<ProjectionCacheUsage, Error> {
+ let row = sqlx::query("SELECT value_bytes,protected_generations,legacy_snapshots FROM radroots_runtime_projection_cache_budget WHERE singleton=1").fetch_one(&mut **transaction).await.map_err(map)?;
+ Ok(ProjectionCacheUsage {
+ value_bytes: u64_from_i64(row.try_get("value_bytes").map_err(map_corrupt)?)?,
+ protected_generations: u64_from_i64(
+ row.try_get("protected_generations").map_err(map_corrupt)?,
+ )?,
+ legacy_snapshots: u64_from_i64(row.try_get("legacy_snapshots").map_err(map_corrupt)?)?,
+ })
+}
+async fn limits(
+ transaction: &mut Transaction<'_, Sqlite>,
+) -> Result<Option<ProjectionCacheLimits>, Error> {
+ let row = sqlx::query("SELECT max_value_bytes,max_protected_generations,max_legacy_snapshots,legacy_age_ms FROM radroots_runtime_projection_cache_budget WHERE singleton=1").fetch_one(&mut **transaction).await.map_err(map)?;
+ let bytes: Option<i64> = row.try_get("max_value_bytes").map_err(map_corrupt)?;
+ bytes
+ .map(|bytes| {
+ ProjectionCacheLimits::new(
+ u64_from_i64(bytes)?,
+ u64_from_i64(
+ row.try_get("max_protected_generations")
+ .map_err(map_corrupt)?,
+ )?,
+ u64_from_i64(row.try_get("max_legacy_snapshots").map_err(map_corrupt)?)?,
+ u64_from_i64(row.try_get("legacy_age_ms").map_err(map_corrupt)?)?,
+ )
+ .map_err(|_| Error::CorruptProjectionRecord)
+ })
+ .transpose()
+}
+async fn time(transaction: &mut Transaction<'_, Sqlite>, now: u64) -> Result<u64, Error> {
+ let clock: i64 = sqlx::query_scalar("UPDATE radroots_runtime_projection_cache_budget SET owner_time_ms=max(owner_time_ms,?) WHERE singleton=1 RETURNING owner_time_ms").bind(i64_from_u64(now)?).fetch_one(&mut **transaction).await.map_err(map)?;
+ u64_from_i64(clock)
+}
+pub(super) async fn configure(
+ store: &SqliteStorage,
+ id: ProjectionId,
+ requested: ProjectionCacheLimits,
+) -> Result<ProjectionCacheUsage, Error> {
+ store.require_projection_writer()?;
+ ProjectionId::parse(id.as_str())?;
+ let mut transaction = store
+ .pool()
+ .begin_with("BEGIN IMMEDIATE")
+ .await
+ .map_err(map)?;
+ if limits(&mut transaction)
+ .await?
+ .is_some_and(|current| current != requested)
+ {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ sqlx::query("UPDATE radroots_runtime_projection_cache_budget SET max_value_bytes=?,max_protected_generations=?,max_legacy_snapshots=?,legacy_age_ms=? WHERE singleton=1")
+ .bind(i64_from_u64(requested.value_bytes())?).bind(i64_from_u64(requested.protected_generations())?).bind(i64_from_u64(requested.legacy_snapshots())?).bind(i64_from_u64(requested.legacy_age_ms())?).execute(&mut *transaction).await.map_err(map)?;
+ sqlx::query("INSERT INTO radroots_runtime_projection_cache_owners(projection_id) VALUES(?) ON CONFLICT DO NOTHING").bind(id.as_str()).execute(&mut *transaction).await.map_err(map)?;
+ time(&mut transaction, now()?).await?;
+ let current = usage(&mut transaction).await?;
+ transaction.commit().await.map_err(map)?;
+ Ok(current)
+}
+async fn fence(
+ transaction: &mut Transaction<'_, Sqlite>,
+ store: &SqliteStorage,
+ scope: &ProjectionCacheScope,
+) -> Result<Option<radroots_storage::projection::ProjectionStatus>, Error> {
+ if scope.source().generation() != store.generation {
+ return Err(Error::SourceEpochChanged);
+ }
+ let epoch: Option<i64> = sqlx::query_scalar("SELECT mutation_epoch FROM radroots_runtime_source_generations WHERE generation=? AND state='active'").bind(scope.source().generation().as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(map)?;
+ if epoch.map(u64_from_i64).transpose()? != Some(scope.source().epoch()) {
+ return Err(Error::SourceEpochChanged);
+ }
+ let status =
+ sqlx::query("SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id=?")
+ .bind(scope.projection_id().as_str())
+ .fetch_optional(&mut **transaction)
+ .await
+ .map_err(map)?
+ .as_ref()
+ .map(decode_status)
+ .transpose()?;
+ if !owner(transaction, scope.projection_id()).await?
+ || status.as_ref().map(|value| value.generation()) != scope.expected_active()
+ {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ Ok(status)
+}
+pub(super) async fn retire(
+ store: &SqliteStorage,
+ scope: ProjectionCacheScope,
+) -> Result<ProjectionCacheUsage, Error> {
+ store.require_projection_writer()?;
+ let mut transaction = store
+ .pool()
+ .begin_with("BEGIN IMMEDIATE")
+ .await
+ .map_err(map)?;
+ let status = fence(&mut transaction, store, &scope).await?;
+ let row = sqlx::query("SELECT source_generation,source_epoch FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).fetch_optional(&mut *transaction).await.map_err(map)?;
+ if scope.expected_active() == Some(scope.generation())
+ && status
+ .as_ref()
+ .is_some_and(|value| value.health() == ProjectionHealth::Ready)
+ {
+ let stale = if let Some(row) = row.as_ref() {
+ let generation: Vec<u8> = row.try_get("source_generation").map_err(map_corrupt)?;
+ generation != scope.source().generation().as_bytes()
+ || u64_from_i64(row.try_get("source_epoch").map_err(map_corrupt)?)?
+ != scope.source().epoch()
+ } else {
+ false
+ };
+ if !stale {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ }
+ sqlx::query("INSERT INTO radroots_runtime_projection_generations(projection_id,generation,source_generation,source_epoch,expected_active,phase) VALUES(?,?,?,?,NULL,'retired') ON CONFLICT(projection_id,generation) DO UPDATE SET phase='retired'")
+ .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind(scope.source().generation().as_bytes().as_slice()).bind(i64_from_u64(scope.source().epoch())?).execute(&mut *transaction).await.map_err(map)?;
+ if scope.expected_active() == Some(scope.generation()) {
+ sqlx::query("DELETE FROM radroots_runtime_projection_checkpoints WHERE projection_id=? AND projection_generation=?").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).execute(&mut *transaction).await.map_err(map)?;
+ }
+ let current = usage(&mut transaction).await?;
+ transaction.commit().await.map_err(map)?;
+ Ok(current)
+}
+pub(super) async fn collect(
+ store: &SqliteStorage,
+ scope: ProjectionCacheScope,
+) -> Result<ProjectionCacheCollection, Error> {
+ store.require_projection_writer()?;
+ let mut transaction = store
+ .pool()
+ .begin_with("BEGIN IMMEDIATE")
+ .await
+ .map_err(map)?;
+ fence(&mut transaction, store, &scope).await?;
+ let phase: Option<String> = sqlx::query_scalar("SELECT phase FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).fetch_optional(&mut *transaction).await.map_err(map)?;
+ let protected = scope.expected_active() == Some(scope.generation())
+ || phase.as_deref().is_some_and(|value| value != "retired");
+ let now = time(&mut transaction, now()?).await?;
+ let limit = limits(&mut transaction)
+ .await?
+ .ok_or(Error::ProjectionRevisionConflict)?;
+ let mut removed = 0u64;
+ let mut freed = 0u64;
+ if !protected {
+ let rows = sqlx::query("SELECT document_key,length(value) AS bytes FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? ORDER BY document_key LIMIT 64")
+ .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).fetch_all(&mut *transaction).await.map_err(map)?;
+ for row in rows {
+ let key: String = row.try_get("document_key").map_err(map_corrupt)?;
+ let bytes = u64_from_i64(row.try_get("bytes").map_err(map_corrupt)?)?;
+ sqlx::query("DELETE FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? AND document_key=?").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind(key).execute(&mut *transaction).await.map_err(map)?;
+ removed += 1;
+ freed = freed
+ .checked_add(bytes)
+ .ok_or(Error::CorruptProjectionRecord)?;
+ }
+ }
+ let mut current = usage(&mut transaction).await?;
+ let rows = sqlx::query("SELECT snapshot_id,created_at_unix_ms,length(value) AS bytes FROM radroots_runtime_projection_snapshots INDEXED BY radroots_runtime_projection_snapshots_generation_created_idx WHERE projection_id=? AND generation=? ORDER BY created_at_unix_ms,snapshot_id LIMIT ?")
+ .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind((PROJECTION_CACHE_COLLECTION_ROWS_MAX as u64-removed) as i64).fetch_all(&mut *transaction).await.map_err(map)?;
+ let mut future_after = now;
+ let mut future_id = vec![255; 32];
+ for row in rows {
+ let created = u64_from_i64(row.try_get("created_at_unix_ms").map_err(map_corrupt)?)?;
+ if protected
+ && created <= now
+ && now - created < limit.legacy_age_ms()
+ && current.legacy_snapshots <= limit.legacy_snapshots()
+ {
+ break;
+ }
+ let id: Vec<u8> = row.try_get("snapshot_id").map_err(map_corrupt)?;
+ let bytes = u64_from_i64(row.try_get("bytes").map_err(map_corrupt)?)?;
+ delete_snapshot(&mut transaction, scope.projection_id(), &id).await?;
+ if created > now {
+ future_after = created;
+ future_id = id;
+ }
+ removed += 1;
+ freed = freed
+ .checked_add(bytes)
+ .ok_or(Error::CorruptProjectionRecord)?;
+ current.legacy_snapshots -= 1;
+ }
+ if protected && removed < PROJECTION_CACHE_COLLECTION_ROWS_MAX as u64 {
+ let rows = sqlx::query("SELECT snapshot_id,length(value) AS bytes FROM radroots_runtime_projection_snapshots INDEXED BY radroots_runtime_projection_snapshots_generation_created_idx WHERE projection_id=? AND generation=? AND (created_at_unix_ms,snapshot_id)>(?,?) ORDER BY created_at_unix_ms,snapshot_id LIMIT ?")
+ .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind(i64_from_u64(future_after)?).bind(future_id).bind((PROJECTION_CACHE_COLLECTION_ROWS_MAX as u64-removed) as i64).fetch_all(&mut *transaction).await.map_err(map)?;
+ for row in rows {
+ let id: Vec<u8> = row.try_get("snapshot_id").map_err(map_corrupt)?;
+ delete_snapshot(&mut transaction, scope.projection_id(), &id).await?;
+ removed += 1;
+ freed = freed
+ .checked_add(u64_from_i64(row.try_get("bytes").map_err(map_corrupt)?)?)
+ .ok_or(Error::CorruptProjectionRecord)?;
+ }
+ }
+ let document_more: bool = !protected && sqlx::query_scalar::<_,i64>("SELECT EXISTS(SELECT 1 FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=?)").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).fetch_one(&mut *transaction).await.map_err(map)? != 0;
+ current = usage(&mut transaction).await?;
+ let snapshot_sql = "SELECT EXISTS(SELECT 1 FROM radroots_runtime_projection_snapshots INDEXED BY radroots_runtime_projection_snapshots_generation_created_idx WHERE projection_id=? AND generation=? AND created_at_unix_ms<=?)";
+ let cutoff = if !protected || current.legacy_snapshots > limit.legacy_snapshots() {
+ i64::MAX as u64
+ } else {
+ now.saturating_sub(limit.legacy_age_ms())
+ };
+ let expired_more: i64 = sqlx::query_scalar(snapshot_sql)
+ .bind(scope.projection_id().as_str())
+ .bind(scope.generation().as_bytes().as_slice())
+ .bind(i64_from_u64(cutoff)?)
+ .fetch_one(&mut *transaction)
+ .await
+ .map_err(map)?;
+ let future_more: i64 = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM radroots_runtime_projection_snapshots INDEXED BY radroots_runtime_projection_snapshots_generation_created_idx WHERE projection_id=? AND generation=? AND created_at_unix_ms>?)").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind(i64_from_u64(now)?).fetch_one(&mut *transaction).await.map_err(map)?;
+ let snapshot_more = expired_more | future_more;
+ let mut registration_more = !protected && phase.is_some();
+ if !protected
+ && !document_more
+ && snapshot_more == 0
+ && phase.is_some()
+ && removed < PROJECTION_CACHE_COLLECTION_ROWS_MAX as u64
+ {
+ sqlx::query("DELETE FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).execute(&mut *transaction).await.map_err(map)?;
+ removed += 1;
+ registration_more = false;
+ }
+ let result = ProjectionCacheCollection {
+ removed_rows: removed,
+ removed_value_bytes: freed,
+ more: document_more || snapshot_more != 0 || registration_more,
+ usage: usage(&mut transaction).await?,
+ };
+ transaction.commit().await.map_err(map)?;
+ Ok(result)
+}
+async fn delete_snapshot(
+ transaction: &mut Transaction<'_, Sqlite>,
+ id: &ProjectionId,
+ snapshot: &[u8],
+) -> Result<(), Error> {
+ sqlx::query(
+ "DELETE FROM radroots_runtime_projection_snapshots WHERE projection_id=? AND snapshot_id=?",
+ )
+ .bind(id.as_str())
+ .bind(snapshot)
+ .execute(&mut **transaction)
+ .await
+ .map_err(map)?;
+ Ok(())
+}
+async fn load_snapshot(
+ transaction: &mut Transaction<'_, Sqlite>,
+ id: &ProjectionId,
+ snapshot: &[u8; 32],
+ expiry: Option<(ProjectionCacheLimits, u64)>,
+) -> Result<Option<ProjectionSnapshot>, Error> {
+ let row = sqlx::query("SELECT generation,created_at_unix_ms,length(value) AS bytes,CASE WHEN length(value_sha256)=32 THEN value_sha256 END AS digest FROM radroots_runtime_projection_snapshots WHERE projection_id=? AND snapshot_id=?").bind(id.as_str()).bind(snapshot.as_slice()).fetch_optional(&mut **transaction).await.map_err(map)?;
+ let Some(row) = row else {
+ return Ok(None);
+ };
+ let created = u64_from_i64(row.try_get("created_at_unix_ms").map_err(map_corrupt)?)?;
+ if expiry.is_some_and(|(limit, now)| created > now || now - created >= limit.legacy_age_ms()) {
+ return Ok(None);
+ }
+ let bytes: i64 = row.try_get("bytes").map_err(map_corrupt)?;
+ if bytes <= 0
+ || bytes > radroots_storage::projection::PROJECTION_DOCUMENT_VALUE_MAX_BYTES as i64
+ {
+ return Err(Error::CorruptProjectionDocument);
+ }
+ let digest: [u8; 32] = row
+ .try_get::<Vec<u8>, _>("digest")
+ .map_err(|_| Error::CorruptProjectionDocument)?
+ .try_into()
+ .map_err(|_| Error::CorruptProjectionDocument)?;
+ let generation = radroots_storage::projection::ProjectionGeneration::new(
+ row.try_get::<Vec<u8>, _>("generation")
+ .map_err(map_corrupt)?
+ .try_into()
+ .map_err(|_| Error::CorruptProjectionDocument)?,
+ )?;
+ let value: Vec<u8> = sqlx::query_scalar("SELECT value FROM radroots_runtime_projection_snapshots WHERE projection_id=? AND snapshot_id=?").bind(id.as_str()).bind(snapshot.as_slice()).fetch_one(&mut **transaction).await.map_err(map)?;
+ ProjectionSnapshot::from_stored_parts(id.clone(), *snapshot, generation, created, value, digest)
+ .map(Some)
+}
+pub(super) async fn snapshot(
+ store: &SqliteStorage,
+ id: ProjectionId,
+ snapshot: [u8; 32],
+) -> Result<Option<ProjectionSnapshot>, Error> {
+ let readonly = store.event_mode() == radroots_storage::status::EventStoreMode::ReadOnly;
+ let mut transaction = if readonly {
+ store.pool().begin().await
+ } else {
+ store.pool().begin_with("BEGIN IMMEDIATE").await
+ }
+ .map_err(map)?;
+ let expiry = if owner(&mut transaction, &id).await? {
+ if readonly {
+ return Ok(None);
+ }
+ Some((
+ limits(&mut transaction)
+ .await?
+ .ok_or(Error::CorruptProjectionRecord)?,
+ time(&mut transaction, now()?).await?,
+ ))
+ } else {
+ None
+ };
+ let result = load_snapshot(&mut transaction, &id, &snapshot, expiry).await?;
+ if readonly {
+ transaction.rollback().await.map_err(map)?;
+ } else {
+ transaction.commit().await.map_err(map)?;
+ }
+ Ok(result)
+}
+pub(super) async fn put_snapshot(
+ store: &SqliteStorage,
+ snapshot: ProjectionSnapshot,
+) -> Result<(), Error> {
+ store.require_projection_writer()?;
+ let mut transaction = store
+ .pool()
+ .begin_with("BEGIN IMMEDIATE")
+ .await
+ .map_err(map)?;
+ if let Some(existing) = load_snapshot(
+ &mut transaction,
+ snapshot.projection_id(),
+ snapshot.snapshot_id(),
+ None,
+ )
+ .await?
+ {
+ return if existing == snapshot {
+ Ok(())
+ } else {
+ Err(Error::CorruptProjectionDocument)
+ };
+ }
+ if owner(&mut transaction, snapshot.projection_id()).await? {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ sqlx::query("INSERT INTO radroots_runtime_projection_snapshots(projection_id,snapshot_id,generation,created_at_unix_ms,value,value_sha256) VALUES(?,?,?,?,?,?)")
+ .bind(snapshot.projection_id().as_str()).bind(snapshot.snapshot_id().as_slice()).bind(snapshot.generation().as_bytes().as_slice()).bind(i64_from_u64(snapshot.created_at_unix_ms())?).bind(snapshot.value()).bind(snapshot.value_sha256().as_slice()).execute(&mut *transaction).await.map_err(map)?;
+ transaction.commit().await.map_err(map)
+}
+
+#[cfg(test)]
+#[path = "cache_tests.rs"]
+mod tests;
diff --git a/crates/storage_sqlite/src/projection/cache_tests.rs b/crates/storage_sqlite/src/projection/cache_tests.rs
@@ -0,0 +1,1591 @@
+use super::*;
+use crate::{OpenMode, OpenOptions, Paths};
+use radroots_storage::{
+ EventStore, ProjectionStore,
+ event::{EventSourceToken, SourceGeneration},
+ memory::MemoryStorage,
+ projection::{
+ ProjectionCheckpoint, ProjectionDocument, ProjectionGeneration, ProjectionSnapshot,
+ batch::{ProjectionDocumentBatch, ProjectionWriteScope},
+ },
+};
+use tempfile::TempDir;
+fn id() -> ProjectionId {
+ ProjectionId::parse("cache_fixture").unwrap()
+}
+fn generation(value: u8) -> ProjectionGeneration {
+ ProjectionGeneration::new([value; 32]).unwrap()
+}
+fn limits() -> ProjectionCacheLimits {
+ ProjectionCacheLimits::new(67_108_864, 2, 32, 1_800_000).unwrap()
+}
+fn batch(
+ source: EventSourceToken,
+ active: Option<ProjectionGeneration>,
+ target: u8,
+ keys: &[&str],
+) -> ProjectionDocumentBatch {
+ ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(id(), generation(target), source, active).unwrap(),
+ keys.iter()
+ .map(|key| ProjectionDocument::new((*key).into(), vec![7]).unwrap())
+ .collect(),
+ )
+ .unwrap()
+}
+fn scope(
+ source: EventSourceToken,
+ active: Option<ProjectionGeneration>,
+ target: u8,
+) -> ProjectionCacheScope {
+ ProjectionCacheScope::new(id(), generation(target), source, active).unwrap()
+}
+fn checkpoint(target: u8, count: u64) -> ProjectionCheckpoint {
+ ProjectionCheckpoint::new(id(), generation(target), None, count, 1).unwrap()
+}
+async fn open(temp: &TempDir) -> SqliteStorage {
+ SqliteStorage::open(
+ OpenOptions::new(
+ Paths::from_directory(temp.path()).unwrap(),
+ OpenMode::Create,
+ )
+ .with_source_generation(SourceGeneration::new([9; 32]).unwrap(), 9)
+ .unwrap(),
+ )
+ .await
+ .unwrap()
+}
+
+#[tokio::test]
+async fn projection_cache_protection_and_retirement_match_memory_with_bounded_collection() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ let source = store.source_token().await.unwrap();
+ for backend in [&memory as &dyn ProjectionStore, &store] {
+ assert_eq!(
+ backend
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap(),
+ ProjectionCacheUsage::default()
+ );
+ backend
+ .promote_projection_batch(batch(source, None, 1, &["active"]), checkpoint(1, 1))
+ .await
+ .unwrap();
+ backend
+ .put_projection_batch(batch(source, Some(generation(1)), 2, &["staging"]))
+ .await
+ .unwrap();
+ assert_eq!(
+ backend
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap(),
+ ProjectionCacheUsage {
+ value_bytes: 2,
+ protected_generations: 2,
+ legacy_snapshots: 0
+ }
+ );
+ assert_eq!(
+ backend
+ .retire_projection_generation(scope(source, Some(generation(1)), 1))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(source, Some(generation(1)), 1))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(source, Some(generation(1)), 2))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ backend
+ .retire_projection_generation(scope(source, Some(generation(1)), 2))
+ .await
+ .unwrap();
+ let collected = backend
+ .collect_projection_cache(scope(source, Some(generation(1)), 2))
+ .await
+ .unwrap();
+ assert_eq!(collected.removed_rows, 2);
+ assert_eq!(collected.removed_value_bytes, 1);
+ assert!(!collected.more);
+ assert_eq!(
+ collected.usage,
+ ProjectionCacheUsage {
+ value_bytes: 1,
+ protected_generations: 1,
+ legacy_snapshots: 0
+ }
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(
+ EventSourceToken::new(source.generation(), 1),
+ Some(generation(1)),
+ 2
+ ))
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(source, None, 2))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ backend
+ .put_projection_document(
+ id(),
+ generation(9),
+ ProjectionDocument::new("legacy".into(), vec![7]).unwrap()
+ )
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ let keys: Vec<_> = (0..64).map(|number| format!("key.{number:02}")).collect();
+ let keys: Vec<_> = keys.iter().map(String::as_str).collect();
+ backend
+ .put_projection_batch(batch(source, Some(generation(1)), 3, &keys))
+ .await
+ .unwrap();
+ backend
+ .put_projection_batch(batch(source, Some(generation(1)), 3, &["key.64"]))
+ .await
+ .unwrap();
+ backend
+ .retire_projection_generation(scope(source, Some(generation(1)), 3))
+ .await
+ .unwrap();
+ let first = backend
+ .collect_projection_cache(scope(source, Some(generation(1)), 3))
+ .await
+ .unwrap();
+ assert_eq!(first.removed_rows, 64);
+ assert!(first.more);
+ let last = backend
+ .collect_projection_cache(scope(source, Some(generation(1)), 3))
+ .await
+ .unwrap();
+ assert_eq!(last.removed_rows, 2);
+ assert!(!last.more);
+ assert_eq!(last.usage.value_bytes, 1);
+ }
+ store.close().await.unwrap();
+}
+
+fn value_batch(
+ source: EventSourceToken,
+ active: Option<ProjectionGeneration>,
+ target: u8,
+ key: &str,
+ size: usize,
+) -> ProjectionDocumentBatch {
+ ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(id(), generation(target), source, active).unwrap(),
+ vec![ProjectionDocument::new(key.into(), vec![7; size]).unwrap()],
+ )
+ .unwrap()
+}
+fn query(target: u8) -> radroots_storage::projection::document_query::ProjectionDocumentQuery {
+ use radroots_storage::projection::document_query::{
+ ProjectionDocumentGenerations, ProjectionDocumentQuery,
+ };
+ ProjectionDocumentQuery::new(
+ id(),
+ ProjectionDocumentGenerations::Exact(generation(target)),
+ 64,
+ )
+ .unwrap()
+}
+fn snapshot(number: u8, target: u8, created: u64, size: usize) -> ProjectionSnapshot {
+ ProjectionSnapshot::new(
+ id(),
+ [number; 32],
+ generation(target),
+ created,
+ vec![8; size],
+ )
+ .unwrap()
+}
+
+#[tokio::test]
+async fn projection_cache_actual_64mib_includes_legacy_active_and_staging_and_rolls_back_overflow()
+{
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ let source = store.source_token().await.unwrap();
+ for backend in [&memory as &dyn ProjectionStore, &store] {
+ backend
+ .put_projection_snapshot(snapshot(1, 9, 1, 16_777_216))
+ .await
+ .unwrap();
+ backend
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap();
+ for number in 0..4 {
+ backend
+ .put_projection_batch(value_batch(
+ source,
+ None,
+ 1,
+ &format!("active.{number}"),
+ 4_194_304,
+ ))
+ .await
+ .unwrap();
+ }
+ backend
+ .promote_projection_batch(batch(source, None, 1, &[]), checkpoint(1, 4))
+ .await
+ .unwrap();
+ for number in 0..8 {
+ backend
+ .put_projection_batch(value_batch(
+ source,
+ Some(generation(1)),
+ 2,
+ &format!("staging.{number}"),
+ 4_194_304,
+ ))
+ .await
+ .unwrap();
+ }
+ let full = backend
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap();
+ assert_eq!(
+ full,
+ ProjectionCacheUsage {
+ value_bytes: 67_108_864,
+ protected_generations: 2,
+ legacy_snapshots: 1
+ }
+ );
+ assert_eq!(
+ backend
+ .put_projection_batch(value_batch(source, Some(generation(1)), 2, "one_over", 1))
+ .await,
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(
+ backend
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap(),
+ full
+ );
+ assert_eq!(
+ backend
+ .query_projection_documents(query(2))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 4
+ ); // existing page ceiling selects four4MiB values
+ backend
+ .collect_projection_cache(scope(source, Some(generation(1)), 9))
+ .await
+ .unwrap();
+ backend
+ .promote_projection_batch(batch(source, Some(generation(1)), 2, &[]), checkpoint(2, 8))
+ .await
+ .unwrap();
+ let collected = backend
+ .collect_projection_cache(scope(source, Some(generation(2)), 1))
+ .await
+ .unwrap();
+ assert_eq!(collected.removed_value_bytes, 16_777_216);
+ assert_eq!(
+ collected.usage,
+ ProjectionCacheUsage {
+ value_bytes: 33_554_432,
+ protected_generations: 1,
+ legacy_snapshots: 0
+ }
+ );
+ }
+ let mut transaction = store.pool().begin().await.unwrap();
+ let logical = usage(&mut transaction).await.unwrap();
+ transaction.rollback().await.unwrap();
+ let page_count: i64 = sqlx::query_scalar("PRAGMA page_count")
+ .fetch_one(store.pool())
+ .await
+ .unwrap();
+ let page_size: i64 = sqlx::query_scalar("PRAGMA page_size")
+ .fetch_one(store.pool())
+ .await
+ .unwrap();
+ let wal = temp.path().join("runtime.sqlite-wal");
+ assert!(page_count * page_size > logical.value_bytes as i64);
+ assert!(std::fs::metadata(wal).unwrap().len() > 0);
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_cache_quota_and_third_protected_wal_races_preserve_transactional_counts() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let source = store.source_token().await.unwrap();
+ let small = ProjectionCacheLimits::new(3, 2, 32, 1_800_000).unwrap();
+ store.configure_projection_cache(id(), small).await.unwrap();
+ store
+ .promote_projection_batch(batch(source, None, 1, &["active"]), checkpoint(1, 1))
+ .await
+ .unwrap();
+ let (left, right) = tokio::join!(
+ store.put_projection_batch(value_batch(source, Some(generation(1)), 2, "left", 2)),
+ store.put_projection_batch(value_batch(source, Some(generation(1)), 2, "right", 2))
+ );
+ assert!(matches!(
+ (&left, &right),
+ (Ok(()), Err(Error::SpaceInsufficient)) | (Err(Error::SpaceInsufficient), Ok(()))
+ ));
+ assert_eq!(
+ store.configure_projection_cache(id(), small).await.unwrap(),
+ ProjectionCacheUsage {
+ value_bytes: 3,
+ protected_generations: 2,
+ legacy_snapshots: 0
+ }
+ );
+ assert_eq!(
+ store
+ .query_projection_documents(query(2))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ store
+ .retire_projection_generation(scope(source, Some(generation(1)), 2))
+ .await
+ .unwrap();
+ store
+ .collect_projection_cache(scope(source, Some(generation(1)), 2))
+ .await
+ .unwrap();
+ let other = ProjectionId::parse("other_cache").unwrap();
+ store
+ .configure_projection_cache(other.clone(), small)
+ .await
+ .unwrap();
+ let second = ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(other.clone(), generation(3), source, None).unwrap(),
+ vec![ProjectionDocument::new("other".into(), vec![7]).unwrap()],
+ )
+ .unwrap();
+ let (left, right) = tokio::join!(
+ store.put_projection_batch(batch(source, Some(generation(1)), 2, &["new"])),
+ store.put_projection_batch(second)
+ );
+ assert!(matches!(
+ (&left, &right),
+ (Ok(()), Err(Error::SpaceInsufficient)) | (Err(Error::SpaceInsufficient), Ok(()))
+ ));
+ assert_eq!(
+ store
+ .configure_projection_cache(id(), small)
+ .await
+ .unwrap()
+ .protected_generations,
+ 2
+ );
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_cache_legacy_32_33_pressure_no_new_snapshots_and_no_sliding_replay() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ let source = store.source_token().await.unwrap();
+ let created = now().unwrap();
+ for backend in [&memory as &dyn ProjectionStore, &store] {
+ backend.checkpoint(checkpoint(1, 0)).await.unwrap();
+ for number in 1..=33 {
+ backend
+ .put_projection_snapshot(snapshot(number, 1, created, 1))
+ .await
+ .unwrap();
+ }
+ assert_eq!(
+ backend
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap()
+ .legacy_snapshots,
+ 33
+ );
+ let replay = snapshot(2, 1, created, 1);
+ backend
+ .put_projection_snapshot(replay.clone())
+ .await
+ .unwrap();
+ assert_eq!(
+ backend.projection_snapshot(id(), [2; 32]).await.unwrap(),
+ Some(replay)
+ );
+ assert_eq!(
+ backend
+ .put_projection_snapshot(snapshot(34, 1, created, 1))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ backend
+ .put_projection_snapshot(snapshot(2, 1, created + 1, 1))
+ .await,
+ Err(Error::CorruptProjectionDocument)
+ );
+ let result = backend
+ .collect_projection_cache(scope(source, Some(generation(1)), 1))
+ .await
+ .unwrap();
+ assert_eq!(result.removed_rows, 1);
+ assert!(!result.more);
+ assert_eq!(result.usage.legacy_snapshots, 32);
+ let foreign = ProjectionId::parse("legacy_other").unwrap();
+ let extra =
+ ProjectionSnapshot::new(foreign, [1; 32], generation(1), created, vec![8]).unwrap();
+ assert_eq!(
+ backend.put_projection_snapshot(extra).await,
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(
+ backend
+ .put_projection_snapshot(snapshot(1, 1, created, 1))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ backend
+ .configure_projection_cache(
+ id(),
+ ProjectionCacheLimits::new(67_108_865, 2, 32, 1_800_000).unwrap()
+ )
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ }
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_cache_stale_and_invalidated_active_retirement_allows_successor_without_canonical_loss()
+ {
+ use radroots_storage::projection::{InvalidationReason, ProjectionInvalidation};
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ for backend in [&memory as &dyn ProjectionStore, &store] {
+ let source = EventSourceToken::new(SourceGeneration::new([9; 32]).unwrap(), 0);
+ backend
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap();
+ backend
+ .promote_projection_batch(batch(source, None, 1, &["active"]), checkpoint(1, 1))
+ .await
+ .unwrap();
+ backend
+ .invalidate(
+ ProjectionInvalidation::new(
+ id(),
+ generation(1),
+ generation(2),
+ InvalidationReason::OperatorRequested,
+ 1,
+ )
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ let retired = backend
+ .retire_projection_generation(scope(source, Some(generation(1)), 1))
+ .await
+ .unwrap();
+ assert_eq!(retired.protected_generations, 0);
+ assert_eq!(backend.status(id()).await.unwrap(), None);
+ backend
+ .promote_projection_batch(batch(source, None, 2, &["successor"]), checkpoint(2, 1))
+ .await
+ .unwrap();
+ backend
+ .collect_projection_cache(scope(source, Some(generation(2)), 1))
+ .await
+ .unwrap();
+ assert_eq!(
+ backend
+ .query_projection_documents_at(query(2), source, generation(2))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ }
+ sqlx::query("UPDATE radroots_runtime_source_generations SET mutation_epoch=mutation_epoch+1 WHERE state='active'").execute(store.pool()).await.unwrap();
+ let current = store.source_token().await.unwrap();
+ assert_eq!(
+ store
+ .retire_projection_generation(scope(
+ EventSourceToken::new(current.generation(), 0),
+ Some(generation(2)),
+ 2
+ ))
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ store
+ .retire_projection_generation(scope(current, Some(generation(2)), 2))
+ .await
+ .unwrap();
+ store
+ .promote_projection_batch(batch(current, None, 3, &["fresh"]), checkpoint(3, 1))
+ .await
+ .unwrap();
+ store
+ .collect_projection_cache(scope(current, Some(generation(3)), 2))
+ .await
+ .unwrap();
+ assert_eq!(store.source_token().await.unwrap(), current);
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_cache_gc_cancellation_rollback_read_race_reopen_and_readonly_preserve_active() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let source = store.source_token().await.unwrap();
+ store
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap();
+ store
+ .promote_projection_batch(batch(source, None, 1, &["active"]), checkpoint(1, 1))
+ .await
+ .unwrap();
+ store
+ .put_projection_batch(batch(
+ source,
+ Some(generation(1)),
+ 2,
+ &["obsolete.a", "obsolete.b"],
+ ))
+ .await
+ .unwrap();
+ store
+ .retire_projection_generation(scope(source, Some(generation(1)), 2))
+ .await
+ .unwrap();
+ let mut connections = Vec::new();
+ for _ in 0..4 {
+ connections.push(store.pool().acquire().await.unwrap());
+ }
+ let mut pending = store.collect_projection_cache(scope(source, Some(generation(1)), 2));
+ assert!(matches!(
+ pending
+ .as_mut()
+ .poll(&mut std::task::Context::from_waker(std::task::Waker::noop())),
+ std::task::Poll::Pending
+ ));
+ drop(pending);
+ drop(connections);
+ assert_eq!(
+ store
+ .query_projection_documents(query(2))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 2
+ );
+ sqlx::raw_sql("CREATE TRIGGER fixture_cache_delete BEFORE DELETE ON radroots_runtime_projection_documents WHEN OLD.document_key='obsolete.b' BEGIN SELECT RAISE(ABORT,'fixture'); END;").execute(store.pool()).await.unwrap();
+ let before = store
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap();
+ assert!(
+ store
+ .collect_projection_cache(scope(source, Some(generation(1)), 2))
+ .await
+ .is_err()
+ );
+ assert_eq!(
+ store
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap(),
+ before
+ );
+ assert_eq!(
+ store
+ .query_projection_documents(query(2))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 2
+ );
+ sqlx::raw_sql("DROP TRIGGER fixture_cache_delete")
+ .execute(store.pool())
+ .await
+ .unwrap();
+ let (read, collected) = tokio::join!(
+ store.query_projection_documents_at(query(1), source, generation(1)),
+ store.collect_projection_cache(scope(source, Some(generation(1)), 2))
+ );
+ assert_eq!(read.unwrap().records().len(), 1);
+ assert_eq!(collected.unwrap().removed_rows, 3);
+ store.close().await.unwrap();
+ let reopened = SqliteStorage::open(OpenOptions::new(
+ Paths::from_directory(temp.path()).unwrap(),
+ OpenMode::ReadWriteExisting,
+ ))
+ .await
+ .unwrap();
+ assert_eq!(
+ reopened
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap(),
+ ProjectionCacheUsage {
+ value_bytes: 1,
+ protected_generations: 1,
+ legacy_snapshots: 0
+ }
+ );
+ reopened.close().await.unwrap();
+ let readonly = SqliteStorage::open(OpenOptions::new(
+ Paths::from_directory(temp.path()).unwrap(),
+ OpenMode::ReadOnly,
+ ))
+ .await
+ .unwrap();
+ assert_eq!(
+ readonly
+ .collect_projection_cache(scope(source, Some(generation(1)), 2))
+ .await,
+ Err(Error::BackendUnavailable)
+ );
+ assert_eq!(
+ readonly
+ .retire_projection_generation(scope(source, Some(generation(1)), 1))
+ .await,
+ Err(Error::BackendUnavailable)
+ );
+ assert_eq!(
+ readonly.configure_projection_cache(id(), limits()).await,
+ Err(Error::BackendUnavailable)
+ );
+ assert_eq!(
+ readonly
+ .query_projection_documents_at(query(1), source, generation(1))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ readonly.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_cache_owner_time_highwater_expires_future_and_rollback_without_resurrection() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let created = now().unwrap();
+ let source = store.source_token().await.unwrap();
+ for (number, time) in [(1, 1), (2, created), (3, created + 3_600_000)] {
+ store
+ .put_projection_snapshot(snapshot(number, 1, time, 1))
+ .await
+ .unwrap();
+ }
+ store.checkpoint(checkpoint(1, 0)).await.unwrap();
+ store
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap();
+ assert!(
+ store
+ .projection_snapshot(id(), [2; 32])
+ .await
+ .unwrap()
+ .is_some()
+ );
+ assert_eq!(
+ store.projection_snapshot(id(), [3; 32]).await.unwrap(),
+ None
+ );
+ let first = store
+ .collect_projection_cache(scope(source, Some(generation(1)), 1))
+ .await
+ .unwrap();
+ assert_eq!(first.removed_rows, 2);
+ assert!(!first.more);
+ let mut transaction = store.pool().begin_with("BEGIN IMMEDIATE").await.unwrap();
+ assert_eq!(
+ time(&mut transaction, created + 1_800_000).await.unwrap(),
+ created + 1_800_000
+ );
+ assert_eq!(
+ time(&mut transaction, created - 1).await.unwrap(),
+ created + 1_800_000
+ );
+ transaction.commit().await.unwrap();
+ assert_eq!(
+ store.projection_snapshot(id(), [2; 32]).await.unwrap(),
+ None
+ );
+ store
+ .put_projection_snapshot(snapshot(2, 1, created, 1))
+ .await
+ .unwrap(); // exact replay cannot renew time
+ assert_eq!(
+ store.projection_snapshot(id(), [2; 32]).await.unwrap(),
+ None
+ );
+ store.close().await.unwrap();
+ let reopened = SqliteStorage::open(OpenOptions::new(
+ Paths::from_directory(temp.path()).unwrap(),
+ OpenMode::ReadWriteExisting,
+ ))
+ .await
+ .unwrap();
+ assert_eq!(
+ reopened.projection_snapshot(id(), [2; 32]).await.unwrap(),
+ None
+ );
+ assert_eq!(
+ reopened
+ .collect_projection_cache(scope(source, Some(generation(1)), 1))
+ .await
+ .unwrap()
+ .removed_rows,
+ 1
+ );
+ assert_eq!(
+ reopened
+ .put_projection_snapshot(snapshot(2, 1, created, 1))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ reopened.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_cache_snapshot_scalar_length_and_digest_admission_precedes_payload_hydration() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ for (number, size) in [(1, 0), (2, 16_777_217)] {
+ let mut connection = store.pool().acquire().await.unwrap();
+ sqlx::raw_sql("PRAGMA ignore_check_constraints=ON")
+ .execute(&mut *connection)
+ .await
+ .unwrap();
+ sqlx::query("INSERT INTO radroots_runtime_projection_snapshots(projection_id,snapshot_id,generation,created_at_unix_ms,value,value_sha256) VALUES(?,?,?,1,zeroblob(?),zeroblob(32))").bind(id().as_str()).bind([number;32].as_slice()).bind(generation(1).as_bytes().as_slice()).bind(size).execute(&mut *connection).await.unwrap();
+ sqlx::raw_sql("PRAGMA ignore_check_constraints=OFF")
+ .execute(&mut *connection)
+ .await
+ .unwrap();
+ drop(connection);
+ assert_eq!(
+ store.projection_snapshot(id(), [number; 32]).await,
+ Err(Error::CorruptProjectionDocument)
+ );
+ }
+ sqlx::query("INSERT INTO radroots_runtime_projection_snapshots(projection_id,snapshot_id,generation,created_at_unix_ms,value,value_sha256) VALUES(?,?,?,1,x'01',zeroblob(32))").bind(id().as_str()).bind([3;32].as_slice()).bind(generation(1).as_bytes().as_slice()).execute(store.pool()).await.unwrap();
+ assert_eq!(
+ store.projection_snapshot(id(), [3; 32]).await,
+ Err(Error::CorruptProjectionDocument)
+ );
+ store
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap();
+ assert_eq!(
+ store.projection_snapshot(id(), [2; 32]).await.unwrap(),
+ None
+ ); // expired oversized row is rejected before value fetch
+ store.close().await.unwrap();
+}
+
+fn review_admission(verified: bool) -> radroots_storage::event::EventAdmission {
+ use radroots_event::{SignedEvent, admission::RawEvent, wire::Nip01EventWire};
+ use radroots_storage::event::EventAdmission;
+ use radroots_transport::{
+ Target, TransportId,
+ source::{EventProvenance, ObservedEvent},
+ };
+ struct FixtureVerifier;
+ impl radroots_event::admission::SignatureVerifier for FixtureVerifier {
+ fn verify_signature(
+ &self,
+ _event: &radroots_event::Event,
+ ) -> Result<(), radroots_event::Error> {
+ Ok(())
+ }
+ }
+ let mut wire = Nip01EventWire {
+ id: "0".repeat(64),
+ pubkey: "585591529da0bab31b3b1b1f986611cf5f435dca84f978c89ee8a40cca7103df".into(),
+ created_at: 17,
+ kind: 0,
+ tags: vec![],
+ content: r#"{"name":"review"}"#.into(),
+ sig: "42".repeat(64),
+ extra: Default::default(),
+ };
+ wire.id = wire.computed_event_id().unwrap().to_hex();
+ let raw = serde_json::to_string(&wire).unwrap();
+ let event = SignedEvent::from_wire_verified_id(wire, raw).unwrap();
+ let target = Target::new(TransportId::NOSTR, "wss://fixture.example").unwrap();
+ let observed = ObservedEvent::new(
+ event.clone(),
+ EventProvenance::new(TransportId::NOSTR, target.fingerprint().clone(), 1).unwrap(),
+ );
+ if verified {
+ let verified = RawEvent::new(event.envelope().clone())
+ .verify_id()
+ .unwrap()
+ .verify_signature(&FixtureVerifier)
+ .unwrap();
+ EventAdmission::verified(observed, verified).unwrap()
+ } else {
+ EventAdmission::raw(observed)
+ }
+}
+
+#[tokio::test]
+async fn projection_cache_collection_preserves_canonical_authored_intent_receipt_custody_and_foreign_projection()
+ {
+ use radroots_storage::journal::OperationId;
+ use radroots_storage::{
+ Storage,
+ atomic::{
+ AtomicCommit, AtomicCommitDigest, AtomicCommitId, AtomicWorkflow, CommitIngested,
+ },
+ authored_draft::{AuthoredDraft, AuthoredDraftId, AuthoredDraftStage},
+ journal::{IdempotencyDigest, IdempotencyKey, OperationInstanceId, PrepareOperation},
+ private_artifact::{
+ ArtifactCommitment, ArtifactKind, ArtifactSchemaId, DurableSecretReference,
+ PrivateArtifactId, PrivateArtifactMetadata, RetentionPolicy,
+ },
+ };
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ for backend in [&memory as &dyn Storage, &store] {
+ let request = AtomicCommit::new(
+ AtomicCommitId::new([5; 16]).unwrap(),
+ AtomicCommitDigest::new([5; 32]),
+ 1,
+ AtomicWorkflow::Ingested(Box::new(CommitIngested::new(review_admission(false), None))),
+ )
+ .unwrap();
+ let receipt = backend.commit(request.clone()).await.unwrap();
+ let source = backend.source_token().await.unwrap();
+ let draft = AuthoredDraft::initial(
+ AuthoredDraftId::new([1; 16]).unwrap(),
+ [3; 32],
+ "fixture.preserved.v1",
+ b"authored intent".to_vec(),
+ AuthoredDraftStage::Draft,
+ None,
+ 1,
+ )
+ .unwrap();
+ backend
+ .append_authored_draft(draft.clone(), None)
+ .await
+ .unwrap();
+ let intent = PrepareOperation::new(
+ OperationInstanceId::new([1; 16]).unwrap(),
+ OperationId::SyncPush,
+ IdempotencyKey::parse("cache-preserve").unwrap(),
+ IdempotencyDigest::new([1; 32]),
+ 1,
+ )
+ .unwrap();
+ backend.prepare(intent.clone()).await.unwrap();
+ let custody = PrivateArtifactMetadata::new(
+ PrivateArtifactId::new([1; 16]).unwrap(),
+ ArtifactKind::parse("fixture.custody").unwrap(),
+ ArtifactSchemaId::parse("fixture.custody.v1").unwrap(),
+ ArtifactCommitment::new([1; 32]),
+ 1,
+ DurableSecretReference::new("memory", "fixture", 1).unwrap(),
+ RetentionPolicy::indefinite(),
+ 1,
+ )
+ .unwrap();
+ backend.put_metadata(custody.clone()).await.unwrap();
+ let foreign = ProjectionId::parse("settings_and_recovery").unwrap();
+ backend
+ .put_projection_document(
+ foreign.clone(),
+ generation(1),
+ ProjectionDocument::new("keep".into(), vec![7]).unwrap(),
+ )
+ .await
+ .unwrap();
+ backend
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap();
+ backend
+ .put_projection_batch(batch(source, None, 1, &["retire"]))
+ .await
+ .unwrap();
+ backend
+ .retire_projection_generation(scope(source, None, 1))
+ .await
+ .unwrap();
+ assert_eq!(
+ backend
+ .collect_projection_cache(
+ ProjectionCacheScope::new(foreign.clone(), generation(1), source, None)
+ .unwrap()
+ )
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(source, None, 1))
+ .await
+ .unwrap()
+ .removed_rows,
+ 2
+ );
+ assert_eq!(backend.source_token().await.unwrap(), source);
+ assert_eq!(
+ backend.receipt(request.commit_id()).await.unwrap(),
+ Some(receipt)
+ );
+ assert_eq!(
+ backend.authored_draft_head(draft.draft_id()).await.unwrap(),
+ Some(draft)
+ );
+ assert!(
+ backend
+ .by_idempotency_key(OperationId::SyncPush, intent.idempotency_key().clone())
+ .await
+ .unwrap()
+ .is_some()
+ );
+ assert_eq!(
+ backend.metadata(custody.artifact_id()).await.unwrap(),
+ Some(custody)
+ );
+ assert_eq!(
+ backend
+ .projection_document(foreign, generation(1), "keep".into())
+ .await
+ .unwrap()
+ .unwrap()
+ .value(),
+ &[7]
+ );
+ }
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_cache_63_64_registration_boundary_reports_only_actual_remaining_work() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ let source = store.source_token().await.unwrap();
+ for backend in [&memory as &dyn ProjectionStore, &store] {
+ backend
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap();
+ for (target, count) in [(1, 63), (2, 64), (3, 0)] {
+ let keys: Vec<_> = (0..count)
+ .map(|number| format!("key.{number:02}"))
+ .collect();
+ let keys: Vec<_> = keys.iter().map(String::as_str).collect();
+ backend
+ .put_projection_batch(batch(source, None, target, &keys))
+ .await
+ .unwrap();
+ backend
+ .retire_projection_generation(scope(source, None, target))
+ .await
+ .unwrap();
+ let first = backend
+ .collect_projection_cache(scope(source, None, target))
+ .await
+ .unwrap();
+ assert_eq!(first.removed_rows, (count + 1).min(64));
+ assert_eq!(first.more, count == 64);
+ if first.more {
+ let second = backend
+ .collect_projection_cache(scope(source, None, target))
+ .await
+ .unwrap();
+ assert_eq!(second.removed_rows, 1);
+ assert!(!second.more);
+ }
+ assert_eq!(
+ backend
+ .configure_projection_cache(id(), limits())
+ .await
+ .unwrap(),
+ ProjectionCacheUsage::default()
+ );
+ }
+ }
+ store.close().await.unwrap();
+}
+
+async fn legacy_cache_completion(backend: &dyn ProjectionStore, registered: bool, capacity: u64) {
+ use radroots_storage::projection::{
+ InvalidationReason, ProjectionCheckpoint, ProjectionHealth, ProjectionInvalidation,
+ RawSourceDigest, RebuildTicket, RebuildTicketId, RebuildTransition,
+ batch::{ProjectionDocumentBatch, ProjectionWriteScope},
+ };
+ let id = ProjectionId::parse("legacy_cache_completion").unwrap();
+ let source = EventSourceToken::new(SourceGeneration::new([9; 32]).unwrap(), 0);
+ let limits = ProjectionCacheLimits::new(67_108_864, capacity, 32, 1_800_000).unwrap();
+ let checkpoint =
+ |target| ProjectionCheckpoint::new(id.clone(), generation(target), None, 1, 1).unwrap();
+ let batch = |target, active| {
+ ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(id.clone(), generation(target), source, active).unwrap(),
+ vec![ProjectionDocument::new("kept".into(), vec![7]).unwrap()],
+ )
+ .unwrap()
+ };
+ let scope = |target, active| {
+ ProjectionCacheScope::new(
+ id.clone(),
+ generation(target),
+ source,
+ Some(generation(active)),
+ )
+ .unwrap()
+ };
+ backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap();
+ if registered {
+ backend
+ .promote_projection_batch(batch(1, None), checkpoint(1))
+ .await
+ .unwrap();
+ } else {
+ backend.checkpoint(checkpoint(1)).await.unwrap();
+ }
+ backend
+ .put_projection_batch(batch(2, Some(generation(1))))
+ .await
+ .unwrap();
+ let invalidation = ProjectionInvalidation::new(
+ id.clone(),
+ generation(1),
+ generation(3),
+ InvalidationReason::OperatorRequested,
+ 10,
+ )
+ .unwrap();
+ backend.invalidate(invalidation.clone()).await.unwrap();
+ let ticket = RebuildTicket::requested(
+ RebuildTicketId::new([3; 16]).unwrap(),
+ invalidation,
+ source.generation(),
+ None,
+ RawSourceDigest::new([1; 32]),
+ )
+ .unwrap();
+ backend.request_rebuild(ticket.clone()).await.unwrap();
+ let running = backend
+ .transition_rebuild(RebuildTransition::start(
+ ticket.ticket_id(),
+ ticket.revision(),
+ 11,
+ ))
+ .await
+ .unwrap();
+ let before_status = backend.status(id.clone()).await.unwrap();
+ let before_usage = backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap();
+ assert_eq!(before_usage.protected_generations, 2);
+ let complete =
+ RebuildTransition::complete(running.ticket_id(), running.revision(), 12, checkpoint(3));
+ if registered && capacity == 2 {
+ assert_eq!(
+ backend.transition_rebuild(complete.clone()).await,
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(backend.status(id.clone()).await.unwrap(), before_status);
+ assert_eq!(
+ backend.rebuild(ticket.ticket_id()).await.unwrap(),
+ Some(running.clone())
+ );
+ assert_eq!(
+ backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap(),
+ before_usage
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(1, 1))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(2, 1))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ assert_eq!(
+ backend
+ .retire_projection_generation(scope(2, 1))
+ .await
+ .unwrap()
+ .protected_generations,
+ 1
+ );
+ }
+ let completed = backend.transition_rebuild(complete).await.unwrap();
+ assert_eq!(
+ backend.rebuild(ticket.ticket_id()).await.unwrap(),
+ Some(completed)
+ );
+ let status = backend.status(id.clone()).await.unwrap().unwrap();
+ assert_eq!(status.generation(), generation(3));
+ assert_eq!(status.health(), ProjectionHealth::Ready);
+ assert_eq!(status.active_rebuild(), None);
+ assert_eq!(status.checkpoint(), Some(&checkpoint(3)));
+ let expected = if registered && capacity == 3 { 3 } else { 2 };
+ assert_eq!(
+ backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap()
+ .protected_generations,
+ expected
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(3, 3))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ if registered {
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(1, 3))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ assert_eq!(
+ backend
+ .retire_projection_generation(scope(1, 3))
+ .await
+ .unwrap()
+ .protected_generations,
+ expected - 1
+ );
+ let collected = backend.collect_projection_cache(scope(1, 3)).await.unwrap();
+ assert_eq!(collected.removed_rows, 2);
+ assert_eq!(collected.removed_value_bytes, 1);
+ assert!(!collected.more);
+ assert_eq!(collected.usage.protected_generations, expected - 1);
+ }
+ if !registered || capacity == 3 {
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(2, 3))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ backend
+ .retire_projection_generation(scope(2, 3))
+ .await
+ .unwrap();
+ }
+ let collected = backend.collect_projection_cache(scope(2, 3)).await.unwrap();
+ assert_eq!(collected.removed_rows, 2);
+ assert_eq!(collected.removed_value_bytes, 1);
+ assert!(!collected.more);
+ assert_eq!(collected.usage.protected_generations, 1);
+ assert_eq!(backend.status(id.clone()).await.unwrap(), Some(status));
+ assert_eq!(
+ backend
+ .collect_projection_cache(scope(3, 3))
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+}
+
+#[tokio::test]
+async fn projection_cache_legacy_completion_preserves_registry_capacity_and_atomic_refusal_in_both_providers()
+ {
+ for (registered, capacity) in [(true, 2), (true, 3), (false, 2)] {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ for backend in [&memory as &dyn ProjectionStore, &store] {
+ legacy_cache_completion(backend, registered, capacity).await;
+ }
+ store.close().await.unwrap();
+ }
+}
+
+async fn cache_batch_replacement(
+ backend: &dyn ProjectionStore,
+ quota: u64,
+ initial: [usize; 2],
+ replacement: [usize; 2],
+ reverse: bool,
+) {
+ use radroots_storage::projection::batch::{ProjectionDocumentBatch, ProjectionWriteScope};
+ let id = ProjectionId::parse("batch_replacement").unwrap();
+ let source = EventSourceToken::new(SourceGeneration::new([9; 32]).unwrap(), 0);
+ let batch = |sizes: [usize; 2], target, reverse| {
+ let mut documents = vec![
+ ProjectionDocument::new("a".into(), vec![1; sizes[0]]).unwrap(),
+ ProjectionDocument::new("b".into(), vec![2; sizes[1]]).unwrap(),
+ ];
+ if reverse {
+ documents.reverse();
+ }
+ ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(id.clone(), generation(target), source, None).unwrap(),
+ documents,
+ )
+ .unwrap()
+ };
+ backend
+ .put_projection_batch(batch(initial, 1, false))
+ .await
+ .unwrap();
+ let limits = ProjectionCacheLimits::new(quota, 2, 32, 1_800_000).unwrap();
+ let before = backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap();
+ let status = backend.status(id.clone()).await.unwrap();
+ let previous_bytes = initial.iter().sum::<usize>() as u64;
+ let next_bytes = replacement.iter().sum::<usize>() as u64;
+ let result = backend
+ .put_projection_batch(batch(replacement, 1, reverse))
+ .await;
+ let refused = next_bytes > previous_bytes && next_bytes > quota;
+ let expected = if refused {
+ assert_eq!(result, Err(Error::SpaceInsufficient));
+ assert_eq!(
+ backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap(),
+ before
+ );
+ initial
+ } else {
+ result.unwrap();
+ let current = backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap();
+ assert_eq!(current.value_bytes, next_bytes);
+ assert_eq!(current.protected_generations, 1);
+ replacement
+ };
+ assert_eq!(backend.status(id.clone()).await.unwrap(), status);
+ for (key, size, value) in [("a", expected[0], 1), ("b", expected[1], 2)] {
+ assert_eq!(
+ backend
+ .projection_document(id.clone(), generation(1), key.into())
+ .await
+ .unwrap()
+ .unwrap()
+ .value(),
+ vec![value; size]
+ );
+ }
+ let current = backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap();
+ let over = quota.saturating_sub(current.value_bytes) + 1;
+ let foreign = ProjectionId::parse("direct_writer").unwrap();
+ assert_eq!(
+ backend
+ .put_projection_document(
+ foreign.clone(),
+ generation(9),
+ ProjectionDocument::new("overflow".into(), vec![9; over as usize]).unwrap()
+ )
+ .await,
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(
+ backend
+ .projection_document(foreign, generation(9), "overflow".into())
+ .await
+ .unwrap(),
+ None
+ );
+ assert_eq!(
+ backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap(),
+ current
+ );
+ assert_eq!(
+ backend
+ .put_projection_batch(
+ ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(
+ ProjectionId::parse("new_admission").unwrap(),
+ generation(2),
+ source,
+ None
+ )
+ .unwrap(),
+ vec![ProjectionDocument::new("a".into(), vec![7; over as usize]).unwrap()]
+ )
+ .unwrap()
+ )
+ .await,
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(
+ backend
+ .configure_projection_cache(id.clone(), limits)
+ .await
+ .unwrap(),
+ current
+ );
+ assert_eq!(
+ backend
+ .projection_document(
+ ProjectionId::parse("new_admission").unwrap(),
+ generation(2),
+ "a".into()
+ )
+ .await
+ .unwrap(),
+ None
+ );
+ assert_eq!(
+ backend
+ .collect_projection_cache(
+ ProjectionCacheScope::new(id.clone(), generation(1), source, None).unwrap()
+ )
+ .await
+ .unwrap()
+ .removed_rows,
+ 0
+ );
+ backend
+ .retire_projection_generation(
+ ProjectionCacheScope::new(id.clone(), generation(1), source, None).unwrap(),
+ )
+ .await
+ .unwrap();
+ let collected = backend
+ .collect_projection_cache(
+ ProjectionCacheScope::new(id, generation(1), source, None).unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(collected.removed_rows, 3);
+ assert!(!collected.more);
+ assert_eq!(collected.usage.value_bytes, 0);
+ assert_eq!(collected.usage.protected_generations, 0);
+}
+
+#[tokio::test]
+async fn projection_cache_batch_replacement_has_one_aggregate_policy_in_both_orders_and_providers()
+{
+ for (quota, initial, replacement) in [
+ (4, [2, 2], [3, 1]),
+ (4, [2, 2], [2, 1]),
+ (6, [4, 4], [5, 2]),
+ (6, [4, 4], [5, 4]),
+ (6, [1, 1], [2, 2]),
+ ] {
+ for reverse in [false, true] {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ for backend in [&memory as &dyn ProjectionStore, &store] {
+ cache_batch_replacement(backend, quota, initial, replacement, reverse).await;
+ }
+ store.close().await.unwrap();
+ }
+ }
+}
+
+#[tokio::test]
+async fn projection_cache_batch_reservation_faults_roll_back_rows_counters_registration_and_status()
+{
+ for corrupt_reservation in [false, true] {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let source = store.source_token().await.unwrap();
+ let batch = |sizes: [usize; 2]| {
+ ProjectionDocumentBatch::new(
+ ProjectionWriteScope::new(id(), generation(1), source, None).unwrap(),
+ vec![
+ ProjectionDocument::new("a".into(), vec![1; sizes[0]]).unwrap(),
+ ProjectionDocument::new("b".into(), vec![2; sizes[1]]).unwrap(),
+ ],
+ )
+ .unwrap()
+ };
+ store.put_projection_batch(batch([2, 2])).await.unwrap();
+ let limits = ProjectionCacheLimits::new(4, 2, 32, 1_800_000).unwrap();
+ let before = store
+ .configure_projection_cache(id(), limits)
+ .await
+ .unwrap();
+ let trigger = if corrupt_reservation {
+ "CREATE TRIGGER fixture_batch_fault AFTER UPDATE OF value ON radroots_runtime_projection_documents WHEN NEW.document_key='a' BEGIN UPDATE radroots_runtime_projection_cache_budget SET batch_value_ceiling=batch_value_ceiling+1 WHERE singleton=1; END;"
+ } else {
+ "CREATE TRIGGER fixture_batch_fault BEFORE UPDATE OF value ON radroots_runtime_projection_documents WHEN NEW.document_key='a' BEGIN SELECT RAISE(ABORT,'fixture batch abort'); END;"
+ };
+ sqlx::raw_sql(trigger).execute(store.pool()).await.unwrap();
+ let result = store
+ .promote_projection_batch(batch([3, 1]), checkpoint(1, 2))
+ .await;
+ if corrupt_reservation {
+ assert_eq!(result, Err(Error::CorruptProjectionRecord));
+ } else {
+ assert!(result.is_err());
+ }
+ assert_eq!(ProjectionStore::status(&store, id()).await.unwrap(), None);
+ assert_eq!(
+ store
+ .configure_projection_cache(id(), limits)
+ .await
+ .unwrap(),
+ before
+ );
+ for (key, value) in [("a", 1), ("b", 2)] {
+ assert_eq!(
+ store
+ .projection_document(id(), generation(1), key.into())
+ .await
+ .unwrap()
+ .unwrap()
+ .value(),
+ vec![value; 2]
+ );
+ }
+ let row = sqlx::query("SELECT document_rows,value_bytes,phase FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?")
+ .bind(id().as_str()).bind(generation(1).as_bytes().as_slice()).fetch_one(store.pool()).await.unwrap();
+ assert_eq!(row.get::<i64, _>("document_rows"), 2);
+ assert_eq!(row.get::<i64, _>("value_bytes"), 4);
+ assert_eq!(row.get::<String, _>("phase"), "staging");
+ let ceiling: Option<i64> = sqlx::query_scalar("SELECT batch_value_ceiling FROM radroots_runtime_projection_cache_budget WHERE singleton=1").fetch_one(store.pool()).await.unwrap();
+ assert_eq!(ceiling, None);
+ sqlx::raw_sql("DROP TRIGGER fixture_batch_fault")
+ .execute(store.pool())
+ .await
+ .unwrap();
+ sqlx::query("UPDATE radroots_runtime_projection_cache_budget SET batch_value_ceiling=4 WHERE singleton=1").execute(store.pool()).await.unwrap();
+ assert_eq!(
+ store
+ .promote_projection_batch(batch([3, 1]), checkpoint(1, 2))
+ .await,
+ Err(Error::CorruptProjectionRecord)
+ );
+ assert_eq!(ProjectionStore::status(&store, id()).await.unwrap(), None);
+ assert_eq!(
+ store
+ .configure_projection_cache(id(), limits)
+ .await
+ .unwrap(),
+ before
+ );
+ sqlx::query("UPDATE radroots_runtime_projection_cache_budget SET batch_value_ceiling=NULL WHERE singleton=1").execute(store.pool()).await.unwrap();
+ store
+ .promote_projection_batch(batch([3, 1]), checkpoint(1, 2))
+ .await
+ .unwrap();
+ let ceiling: Option<i64> = sqlx::query_scalar("SELECT batch_value_ceiling FROM radroots_runtime_projection_cache_budget WHERE singleton=1").fetch_one(store.pool()).await.unwrap();
+ assert_eq!(ceiling, None);
+ assert_eq!(
+ store
+ .configure_projection_cache(id(), limits)
+ .await
+ .unwrap(),
+ before
+ );
+ store.close().await.unwrap();
+ let reopened = open(&temp).await;
+ assert_eq!(
+ reopened
+ .configure_projection_cache(id(), limits)
+ .await
+ .unwrap(),
+ before
+ );
+ assert_eq!(
+ reopened
+ .projection_document(id(), generation(1), "a".into())
+ .await
+ .unwrap()
+ .unwrap()
+ .value(),
+ vec![1; 3]
+ );
+ assert_eq!(
+ ProjectionStore::status(&reopened, id())
+ .await
+ .unwrap()
+ .unwrap()
+ .generation(),
+ generation(1)
+ );
+ let ceiling: Option<i64> = sqlx::query_scalar("SELECT batch_value_ceiling FROM radroots_runtime_projection_cache_budget WHERE singleton=1").fetch_one(reopened.pool()).await.unwrap();
+ assert_eq!(ceiling, None);
+ reopened.close().await.unwrap();
+ }
+}
diff --git a/crates/storage_sqlite/src/projection/mod.rs b/crates/storage_sqlite/src/projection/mod.rs
@@ -15,10 +15,33 @@ use radroots_storage::{
use sqlx::{Row, Sqlite, SqliteConnection};
mod batch;
+mod cache;
mod document_query;
#[cfg_attr(coverage_nightly, coverage(off))]
impl ProjectionStore for SqliteStorage {
+ fn configure_projection_cache(
+ &self,
+ id: ProjectionId,
+ limits: radroots_storage::projection::cache::ProjectionCacheLimits,
+ ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheUsage, Error>>
+ {
+ Box::pin(cache::configure(self, id, limits))
+ }
+ fn retire_projection_generation(
+ &self,
+ scope: radroots_storage::projection::cache::ProjectionCacheScope,
+ ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheUsage, Error>>
+ {
+ Box::pin(cache::retire(self, scope))
+ }
+ fn collect_projection_cache(
+ &self,
+ scope: radroots_storage::projection::cache::ProjectionCacheScope,
+ ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheCollection, Error>>
+ {
+ Box::pin(cache::collect(self, scope))
+ }
fn status(
&self,
projection_id: ProjectionId,
@@ -30,7 +53,7 @@ impl ProjectionStore for SqliteStorage {
.bind(projection_id.as_str())
.fetch_optional(self.pool())
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
.as_ref()
.map(decode_status)
.transpose()
@@ -47,9 +70,9 @@ impl ProjectionStore for SqliteStorage {
.pool()
.begin_with("BEGIN IMMEDIATE")
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
let status = checkpoint_transaction(&mut transaction, checkpoint).await?;
- transaction.commit().await.map_err(map_backend)?;
+ transaction.commit().await.map_err(cache::map)?;
Ok(status)
})
}
@@ -64,14 +87,14 @@ impl ProjectionStore for SqliteStorage {
.pool()
.begin_with("BEGIN IMMEDIATE")
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
let row = sqlx::query(
"SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?",
)
.bind(invalidation.projection_id().as_str())
.fetch_optional(&mut *transaction)
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
.ok_or(Error::ProjectionCheckpointMismatch)?;
let current = decode_status(&row)?;
if current.generation() != invalidation.invalid_generation() {
@@ -101,7 +124,7 @@ impl ProjectionStore for SqliteStorage {
.bind(i64_from_u64(invalidation.invalidated_at_unix_ms())?)
.execute(&mut *transaction)
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
}
let next = ProjectionStatus::new(
invalidation.projection_id().clone(),
@@ -111,7 +134,7 @@ impl ProjectionStore for SqliteStorage {
None,
)?;
put_status_transaction(&mut transaction, &next).await?;
- transaction.commit().await.map_err(map_backend)?;
+ transaction.commit().await.map_err(cache::map)?;
Ok(next)
})
}
@@ -126,18 +149,18 @@ impl ProjectionStore for SqliteStorage {
.pool()
.begin_with("BEGIN IMMEDIATE")
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
if let Some(row) = sqlx::query(
"SELECT * FROM radroots_runtime_projection_rebuilds WHERE ticket_id = ?",
)
.bind(ticket.ticket_id().as_bytes().as_slice())
.fetch_optional(&mut *transaction)
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
{
let existing = decode_ticket(&mut transaction, &row).await?;
return if existing == ticket {
- transaction.commit().await.map_err(map_backend)?;
+ transaction.commit().await.map_err(cache::map)?;
Ok(existing)
} else {
Err(Error::ProjectionRevisionConflict)
@@ -149,7 +172,7 @@ impl ProjectionStore for SqliteStorage {
.bind(ticket.invalidation().projection_id().as_str())
.fetch_optional(&mut *transaction)
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
.ok_or(Error::ProjectionCheckpointMismatch)?;
let status = decode_status(&status_row)?;
if status.generation() != ticket.invalidation().invalid_generation()
@@ -174,7 +197,7 @@ impl ProjectionStore for SqliteStorage {
Some(ticket.ticket_id()),
)?;
put_status_transaction(&mut transaction, &next).await?;
- transaction.commit().await.map_err(map_backend)?;
+ transaction.commit().await.map_err(cache::map)?;
Ok(ticket)
})
}
@@ -194,7 +217,7 @@ impl ProjectionStore for SqliteStorage {
.bind(replacement_generation.as_bytes().as_slice())
.fetch_optional(self.pool())
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
.as_ref()
.map(decode_invalidation)
.transpose()
@@ -206,14 +229,14 @@ impl ProjectionStore for SqliteStorage {
ticket_id: RebuildTicketId,
) -> BoxFuture<'_, Result<Option<RebuildTicket>, Error>> {
Box::pin(async move {
- let mut connection = self.pool().acquire().await.map_err(map_backend)?;
+ let mut connection = self.pool().acquire().await.map_err(cache::map)?;
let row = sqlx::query(
"SELECT * FROM radroots_runtime_projection_rebuilds WHERE ticket_id = ?",
)
.bind(ticket_id.as_bytes().as_slice())
.fetch_optional(&mut *connection)
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
match row {
Some(row) => decode_ticket(&mut connection, &row).await.map(Some),
None => Ok(None),
@@ -231,14 +254,14 @@ impl ProjectionStore for SqliteStorage {
.pool()
.begin_with("BEGIN IMMEDIATE")
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
let row = sqlx::query(
"SELECT * FROM radroots_runtime_projection_rebuilds WHERE ticket_id = ?",
)
.bind(transition.ticket_id().as_bytes().as_slice())
.fetch_optional(&mut *transaction)
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
.ok_or(Error::ProjectionRevisionConflict)?;
let current = decode_ticket(&mut transaction, &row).await?;
let status_row = sqlx::query(
@@ -247,7 +270,7 @@ impl ProjectionStore for SqliteStorage {
.bind(current.invalidation().projection_id().as_str())
.fetch_optional(&mut *transaction)
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
.ok_or(Error::CorruptProjectionRecord)?;
let current_status = decode_status(&status_row)?;
if current_status.generation() != current.invalidation().invalid_generation()
@@ -270,7 +293,7 @@ impl ProjectionStore for SqliteStorage {
)
.fetch_one(&mut *transaction)
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
let generation = SourceGeneration::new(array(
source
.try_get::<Vec<u8>, _>("generation")
@@ -320,7 +343,7 @@ impl ProjectionStore for SqliteStorage {
active_rebuild,
)?;
put_status_transaction(&mut transaction, &status).await?;
- transaction.commit().await.map_err(map_backend)?;
+ transaction.commit().await.map_err(cache::map)?;
Ok(next)
})
}
@@ -330,7 +353,7 @@ impl ProjectionStore for SqliteStorage {
generation: ProjectionGeneration,
) -> BoxFuture<'_, Result<Option<EventIndexManifest>, Error>> {
Box::pin(async move {
- let mut connection = self.pool().acquire().await.map_err(map_backend)?;
+ let mut connection = self.pool().acquire().await.map_err(cache::map)?;
load_manifest(&mut connection, generation).await
})
}
@@ -345,10 +368,10 @@ impl ProjectionStore for SqliteStorage {
.pool()
.begin_with("BEGIN IMMEDIATE")
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
if let Some(existing) = load_manifest(&mut transaction, manifest.generation()).await? {
return if existing == manifest {
- transaction.commit().await.map_err(map_backend)?;
+ transaction.commit().await.map_err(cache::map)?;
Ok(())
} else {
Err(Error::CorruptProjectionRecord)
@@ -367,7 +390,7 @@ impl ProjectionStore for SqliteStorage {
.bind(i64_from_u64(manifest.last_published_at_unix_s())?)
.execute(&mut *transaction)
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
for (ordinal, shard) in manifest.shards().iter().enumerate() {
sqlx::query(
"INSERT INTO radroots_runtime_event_index_shards (
@@ -388,9 +411,9 @@ impl ProjectionStore for SqliteStorage {
.bind(shard.sha256().as_bytes().as_slice())
.execute(&mut *transaction)
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
}
- transaction.commit().await.map_err(map_backend)
+ transaction.commit().await.map_err(cache::map)
})
}
@@ -407,7 +430,7 @@ impl ProjectionStore for SqliteStorage {
.bind(generation.as_bytes().as_slice())
.fetch_optional(self.pool())
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
.as_ref()
.map(|row| decode_index_checkpoint(row, generation))
.transpose()
@@ -424,7 +447,7 @@ impl ProjectionStore for SqliteStorage {
.pool()
.begin_with("BEGIN IMMEDIATE")
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
if let Some(row) = sqlx::query(
"SELECT generated_at_unix_ms, checkpoint
FROM radroots_runtime_event_index_checkpoints
@@ -433,7 +456,7 @@ impl ProjectionStore for SqliteStorage {
.bind(checkpoint.generation().as_bytes().as_slice())
.fetch_optional(&mut *transaction)
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
&& checkpoint.generated_at_unix_ms()
< decode_index_checkpoint(&row, checkpoint.generation())?.generated_at_unix_ms()
{
@@ -452,8 +475,8 @@ impl ProjectionStore for SqliteStorage {
.bind(encode_index_checkpoint(&checkpoint)?)
.execute(&mut *transaction)
.await
- .map_err(map_backend)?;
- transaction.commit().await.map_err(map_backend)?;
+ .map_err(cache::map)?;
+ transaction.commit().await.map_err(cache::map)?;
Ok(())
})
}
@@ -518,74 +541,14 @@ impl ProjectionStore for SqliteStorage {
&self,
snapshot: ProjectionSnapshot,
) -> BoxFuture<'_, Result<(), Error>> {
- Box::pin(async move {
- self.require_projection_writer()?;
- let result = sqlx::query(
- "INSERT INTO radroots_runtime_projection_snapshots (
- projection_id, snapshot_id, generation, created_at_unix_ms,
- value, value_sha256
- ) VALUES (?, ?, ?, ?, ?, ?)
- ON CONFLICT(projection_id, snapshot_id) DO NOTHING",
- )
- .bind(snapshot.projection_id().as_str())
- .bind(snapshot.snapshot_id().as_slice())
- .bind(snapshot.generation().as_bytes().as_slice())
- .bind(i64_from_u64(snapshot.created_at_unix_ms())?)
- .bind(snapshot.value())
- .bind(snapshot.value_sha256().as_slice())
- .execute(self.pool())
- .await
- .map_err(map_backend)?;
- if result.rows_affected() == 1 {
- return Ok(());
- }
- match self
- .projection_snapshot(snapshot.projection_id().clone(), *snapshot.snapshot_id())
- .await?
- {
- Some(existing) if existing == snapshot => Ok(()),
- Some(_) => Err(Error::CorruptProjectionDocument),
- None => Err(Error::CorruptProjectionDocument),
- }
- })
+ Box::pin(cache::put_snapshot(self, snapshot))
}
-
fn projection_snapshot(
&self,
- projection_id: ProjectionId,
- snapshot_id: [u8; 32],
+ id: ProjectionId,
+ snapshot: [u8; 32],
) -> BoxFuture<'_, Result<Option<ProjectionSnapshot>, Error>> {
- Box::pin(async move {
- sqlx::query(
- "SELECT projection_id, snapshot_id, generation, created_at_unix_ms,
- value, value_sha256
- FROM radroots_runtime_projection_snapshots
- WHERE projection_id = ? AND snapshot_id = ?",
- )
- .bind(projection_id.as_str())
- .bind(snapshot_id.as_slice())
- .fetch_optional(self.pool())
- .await
- .map_err(map_backend)?
- .map(|row| {
- ProjectionSnapshot::from_stored_parts(
- ProjectionId::parse(
- row.try_get::<String, _>("projection_id")
- .map_err(map_corrupt)?,
- )
- .map_err(|_| Error::CorruptProjectionDocument)?,
- array(row.try_get("snapshot_id").map_err(map_corrupt)?)?,
- ProjectionGeneration::new(array(
- row.try_get("generation").map_err(map_corrupt)?,
- )?)
- .map_err(|_| Error::CorruptProjectionDocument)?,
- u64_from_i64(row.try_get("created_at_unix_ms").map_err(map_corrupt)?)?,
- row.try_get("value").map_err(map_corrupt)?,
- array(row.try_get("value_sha256").map_err(map_corrupt)?)?,
- )
- })
- .transpose()
- })
+ Box::pin(cache::snapshot(self, id, snapshot))
}
}
@@ -606,7 +569,7 @@ pub(crate) async fn checkpoint_transaction(
.bind(checkpoint.projection_id().as_str())
.fetch_optional(&mut **transaction)
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
.as_ref()
.map(decode_status)
.transpose()?;
@@ -662,7 +625,7 @@ async fn put_status_transaction(
)
.execute(&mut **transaction)
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
Ok(())
}
@@ -789,7 +752,7 @@ async fn load_invalidation(
.bind(generation.as_bytes().as_slice())
.fetch_optional(&mut *connection)
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
.as_ref()
.map(decode_invalidation)
.transpose()
@@ -863,7 +826,7 @@ async fn insert_ticket(
.bind(i64_from_u64(ticket.updated_at_unix_ms())?)
.execute(&mut **transaction)
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
Ok(())
}
@@ -893,7 +856,7 @@ async fn update_ticket(
.bind(i64_from_u64(prior.get())?)
.execute(&mut **transaction)
.await
- .map_err(map_backend)?;
+ .map_err(cache::map)?;
if result.rows_affected() != 1 {
return Err(Error::ProjectionRevisionConflict);
}
@@ -985,7 +948,7 @@ async fn load_manifest(
.bind(generation.as_bytes().as_slice())
.fetch_optional(&mut *connection)
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
else {
return Ok(None);
};
@@ -996,7 +959,7 @@ async fn load_manifest(
.bind(generation.as_bytes().as_slice())
.fetch_all(&mut *connection)
.await
- .map_err(map_backend)?
+ .map_err(cache::map)?
.iter()
.enumerate()
.map(|(ordinal, row)| {