commit c6c83d2f2000d9bc0b93dab71ebd6a941ee8f7df
parent 68c5b835474dca92f1647f9449947b60efd43a4e
Author: triesap <tyson@radroots.org>
Date: Sat, 10 Oct 2026 05:53:41 +0000
storage: provide fenced atomic projection generations
- Stage bounded opaque document batches and promote complete generations atomically.
- Bind current projection reads to their persisted promoted source token.
- Prevent legacy checkpoint and rebuild paths from bypassing registered generation controls.
- Verify 3963 test executions, unchanged coverage gates and fresh independent acceptance.
Diffstat:
15 files changed, 2480 insertions(+), 113 deletions(-)
diff --git a/contracts/api_baselines/radroots_storage.txt b/contracts/api_baselines/radroots_storage.txt
@@ -1087,11 +1087,14 @@ pub fn radroots_storage::memory::MemoryStorage::invalidate(&self, radroots_stora
pub fn radroots_storage::memory::MemoryStorage::invalidation(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionInvalidation>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::projection_document(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, alloc::string::String) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionDocument>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::projection_snapshot(&self, radroots_storage::projection::ProjectionId, [u8; 32]) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionSnapshot>, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::promote_projection_batch(&self, radroots_storage::projection::batch::ProjectionDocumentBatch, radroots_storage::projection::ProjectionCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::put_event_index_checkpoint(&self, radroots_storage::projection::EventIndexCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::put_event_index_manifest(&self, radroots_storage::projection::EventIndexManifest) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::put_projection_batch(&self, radroots_storage::projection::batch::ProjectionDocumentBatch) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::put_projection_document(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, radroots_storage::projection::ProjectionDocument) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::put_projection_snapshot(&self, radroots_storage::projection::ProjectionSnapshot) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_projection_documents(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::query_projection_documents_at(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery, radroots_storage::event::EventSourceToken, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::rebuild(&self, radroots_storage::projection::RebuildTicketId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::RebuildTicket>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::request_rebuild(&self, radroots_storage::projection::RebuildTicket) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::status(&self, radroots_storage::projection::ProjectionId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::Error>>
@@ -1453,6 +1456,24 @@ pub fn radroots_storage::memory::MemoryStorage::tombstone(&self, radroots_storag
pub mod radroots_storage::projection
pub use radroots_storage::projection::BoxFuture
pub use radroots_storage::projection::EventId
+pub mod radroots_storage::projection::batch
+pub struct radroots_storage::projection::batch::ProjectionDocumentBatch
+impl radroots_storage::projection::batch::ProjectionDocumentBatch
+pub fn radroots_storage::projection::batch::ProjectionDocumentBatch::documents(&self) -> &[radroots_storage::projection::ProjectionDocument]
+pub const fn radroots_storage::projection::batch::ProjectionDocumentBatch::logical_bytes(&self) -> usize
+pub fn radroots_storage::projection::batch::ProjectionDocumentBatch::new(radroots_storage::projection::batch::ProjectionWriteScope, alloc::vec::Vec<radroots_storage::projection::ProjectionDocument>) -> core::result::Result<Self, radroots_storage::Error>
+pub const fn radroots_storage::projection::batch::ProjectionDocumentBatch::scope(&self) -> &radroots_storage::projection::batch::ProjectionWriteScope
+impl core::fmt::Debug for radroots_storage::projection::batch::ProjectionDocumentBatch
+pub fn radroots_storage::projection::batch::ProjectionDocumentBatch::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result
+pub struct radroots_storage::projection::batch::ProjectionWriteScope
+impl radroots_storage::projection::batch::ProjectionWriteScope
+pub const fn radroots_storage::projection::batch::ProjectionWriteScope::expected_active(&self) -> core::option::Option<radroots_storage::projection::ProjectionGeneration>
+pub const fn radroots_storage::projection::batch::ProjectionWriteScope::generation(&self) -> radroots_storage::projection::ProjectionGeneration
+pub fn radroots_storage::projection::batch::ProjectionWriteScope::new(radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, radroots_storage::event::EventSourceToken, core::option::Option<radroots_storage::projection::ProjectionGeneration>) -> core::result::Result<Self, radroots_storage::Error>
+pub const fn radroots_storage::projection::batch::ProjectionWriteScope::projection_id(&self) -> &radroots_storage::projection::ProjectionId
+pub const fn radroots_storage::projection::batch::ProjectionWriteScope::source(&self) -> radroots_storage::event::EventSourceToken
+pub const radroots_storage::projection::batch::PROJECTION_BATCH_BYTES_MAX: usize
+pub const radroots_storage::projection::batch::PROJECTION_BATCH_ROWS_MAX: usize
pub mod radroots_storage::projection::document_query
pub enum radroots_storage::projection::document_query::ProjectionDocumentGenerations
pub radroots_storage::projection::document_query::ProjectionDocumentGenerations::All
@@ -1652,11 +1673,14 @@ pub fn radroots_storage::projection::ProjectionStore::invalidate(&self, radroots
pub fn radroots_storage::projection::ProjectionStore::invalidation(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionInvalidation>, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::projection_document(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, alloc::string::String) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionDocument>, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::projection_snapshot(&self, radroots_storage::projection::ProjectionId, [u8; 32]) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionSnapshot>, radroots_storage::Error>>
+pub fn radroots_storage::projection::ProjectionStore::promote_projection_batch(&self, radroots_storage::projection::batch::ProjectionDocumentBatch, radroots_storage::projection::ProjectionCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::put_event_index_checkpoint(&self, radroots_storage::projection::EventIndexCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::put_event_index_manifest(&self, radroots_storage::projection::EventIndexManifest) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
+pub fn radroots_storage::projection::ProjectionStore::put_projection_batch(&self, radroots_storage::projection::batch::ProjectionDocumentBatch) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::put_projection_document(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, radroots_storage::projection::ProjectionDocument) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::put_projection_snapshot(&self, radroots_storage::projection::ProjectionSnapshot) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::query_projection_documents(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
+pub fn radroots_storage::projection::ProjectionStore::query_projection_documents_at(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery, radroots_storage::event::EventSourceToken, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::rebuild(&self, radroots_storage::projection::RebuildTicketId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::RebuildTicket>, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::request_rebuild(&self, radroots_storage::projection::RebuildTicket) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
pub fn radroots_storage::projection::ProjectionStore::status(&self, radroots_storage::projection::ProjectionId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::Error>>
@@ -1669,11 +1693,14 @@ pub fn radroots_storage::memory::MemoryStorage::invalidate(&self, radroots_stora
pub fn radroots_storage::memory::MemoryStorage::invalidation(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionInvalidation>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::projection_document(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, alloc::string::String) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionDocument>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::projection_snapshot(&self, radroots_storage::projection::ProjectionId, [u8; 32]) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionSnapshot>, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::promote_projection_batch(&self, radroots_storage::projection::batch::ProjectionDocumentBatch, radroots_storage::projection::ProjectionCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::put_event_index_checkpoint(&self, radroots_storage::projection::EventIndexCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::put_event_index_manifest(&self, radroots_storage::projection::EventIndexManifest) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::put_projection_batch(&self, radroots_storage::projection::batch::ProjectionDocumentBatch) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::put_projection_document(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, radroots_storage::projection::ProjectionDocument) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::put_projection_snapshot(&self, radroots_storage::projection::ProjectionSnapshot) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_projection_documents(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::query_projection_documents_at(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery, radroots_storage::event::EventSourceToken, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::rebuild(&self, radroots_storage::projection::RebuildTicketId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::RebuildTicket>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::request_rebuild(&self, radroots_storage::projection::RebuildTicket) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::status(&self, radroots_storage::projection::ProjectionId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::Error>>
@@ -1972,11 +1999,14 @@ pub fn radroots_storage::ProjectionStore::invalidate(&self, radroots_storage::pr
pub fn radroots_storage::ProjectionStore::invalidation(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionInvalidation>, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::projection_document(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, alloc::string::String) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionDocument>, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::projection_snapshot(&self, radroots_storage::projection::ProjectionId, [u8; 32]) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionSnapshot>, radroots_storage::Error>>
+pub fn radroots_storage::ProjectionStore::promote_projection_batch(&self, radroots_storage::projection::batch::ProjectionDocumentBatch, radroots_storage::projection::ProjectionCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::put_event_index_checkpoint(&self, radroots_storage::projection::EventIndexCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::put_event_index_manifest(&self, radroots_storage::projection::EventIndexManifest) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
+pub fn radroots_storage::ProjectionStore::put_projection_batch(&self, radroots_storage::projection::batch::ProjectionDocumentBatch) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::put_projection_document(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, radroots_storage::projection::ProjectionDocument) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::put_projection_snapshot(&self, radroots_storage::projection::ProjectionSnapshot) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::query_projection_documents(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
+pub fn radroots_storage::ProjectionStore::query_projection_documents_at(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery, radroots_storage::event::EventSourceToken, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::rebuild(&self, radroots_storage::projection::RebuildTicketId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::RebuildTicket>, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::request_rebuild(&self, radroots_storage::projection::RebuildTicket) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
pub fn radroots_storage::ProjectionStore::status(&self, radroots_storage::projection::ProjectionId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::Error>>
@@ -1989,11 +2019,14 @@ pub fn radroots_storage::memory::MemoryStorage::invalidate(&self, radroots_stora
pub fn radroots_storage::memory::MemoryStorage::invalidation(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionInvalidation>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::projection_document(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, alloc::string::String) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionDocument>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::projection_snapshot(&self, radroots_storage::projection::ProjectionId, [u8; 32]) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionSnapshot>, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::promote_projection_batch(&self, radroots_storage::projection::batch::ProjectionDocumentBatch, radroots_storage::projection::ProjectionCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::put_event_index_checkpoint(&self, radroots_storage::projection::EventIndexCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::put_event_index_manifest(&self, radroots_storage::projection::EventIndexManifest) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::put_projection_batch(&self, radroots_storage::projection::batch::ProjectionDocumentBatch) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::put_projection_document(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, radroots_storage::projection::ProjectionDocument) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::put_projection_snapshot(&self, radroots_storage::projection::ProjectionSnapshot) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_projection_documents(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::query_projection_documents_at(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery, radroots_storage::event::EventSourceToken, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::rebuild(&self, radroots_storage::projection::RebuildTicketId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::RebuildTicket>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::request_rebuild(&self, radroots_storage::projection::RebuildTicket) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::status(&self, radroots_storage::projection::ProjectionId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::Error>>
diff --git a/contracts/api_baselines/radroots_storage_sqlite.txt b/contracts/api_baselines/radroots_storage_sqlite.txt
@@ -619,11 +619,14 @@ pub fn radroots_storage_sqlite::SqliteStorage::invalidate(&self, radroots_storag
pub fn radroots_storage_sqlite::SqliteStorage::invalidation(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionInvalidation>, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::projection_document(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, alloc::string::String) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionDocument>, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::projection_snapshot(&self, radroots_storage::projection::ProjectionId, [u8; 32]) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionSnapshot>, radroots_storage::error::Error>>
+pub fn radroots_storage_sqlite::SqliteStorage::promote_projection_batch(&self, radroots_storage::projection::batch::ProjectionDocumentBatch, radroots_storage::projection::ProjectionCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::ProjectionStatus, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::put_event_index_checkpoint(&self, radroots_storage::projection::EventIndexCheckpoint) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::put_event_index_manifest(&self, radroots_storage::projection::EventIndexManifest) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::error::Error>>
+pub fn radroots_storage_sqlite::SqliteStorage::put_projection_batch(&self, radroots_storage::projection::batch::ProjectionDocumentBatch) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::put_projection_document(&self, radroots_storage::projection::ProjectionId, radroots_storage::projection::ProjectionGeneration, radroots_storage::projection::ProjectionDocument) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::put_projection_snapshot(&self, radroots_storage::projection::ProjectionSnapshot) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::query_projection_documents(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::error::Error>>
+pub fn radroots_storage_sqlite::SqliteStorage::query_projection_documents_at(&self, radroots_storage::projection::document_query::ProjectionDocumentQuery, radroots_storage::event::EventSourceToken, radroots_storage::projection::ProjectionGeneration) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::document_query::ProjectionDocumentPage, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::rebuild(&self, radroots_storage::projection::RebuildTicketId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::RebuildTicket>, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::request_rebuild(&self, radroots_storage::projection::RebuildTicket) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::projection::RebuildTicket, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::status(&self, radroots_storage::projection::ProjectionId) -> radroots_transport::source::BoxFuture<'_, core::result::Result<core::option::Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::error::Error>>
diff --git a/contracts/architecture/decisions/projection_atomic_generations.v1.json b/contracts/architecture/decisions/projection_atomic_generations.v1.json
@@ -0,0 +1,27 @@
+{
+ "schema": "radroots.projection-atomic-generations.v1",
+ "status": "implemented",
+ "owners": [
+ "radroots_storage",
+ "radroots_storage_sqlite"
+ ],
+ "bounds": {
+ "batch_rows": 64,
+ "logical_batch_bytes": 16777216,
+ "key_bytes": 512
+ },
+ "charge": "4096 fixed bytes plus twice every opaque value and key plus256 bytes per row, checked before provider work. Callers already own input allocations.",
+ "scope": "ProjectionWriteScope binds caller-owned content-derived generation (including policy/schema), expected active generation and exact canonical source token. The first staging write persists that entire fence. Every later batch and promotion must agree. One staging generation per projection identity.",
+ "promotion": "One owner transaction or memory lock checks semantic source epoch and active-generation CAS, writes the bounded final batch, validates exact staged document count and source checkpoint, seals the target and publishes Ready atomically. Legacy APIs cannot rewrite registered generations. Failed checks publish neither rows nor status. Shared legacy checkpoint helpers (including atomic admission workflows) and legacy rebuild completion refuse registered generations before changes. Sealed checkpoint metadata cannot be rewritten through legacy operations.",
+ "reads": "query_projection_documents_at requires exact expected active generation, Ready health and registered promoted state whose persisted source token equals both caller and canonical source. All checks and page hydration share one read transaction or lock. A new current token cannot relabel an older projection. Write-side CAS still permits building a successor from current source while the old active generation is stale. Existing inventory remains a live bounded inventory.",
+ "compatibility": "Historical migrations remain unchanged. Unsupported providers refuse new operations with BackendUnavailable. No public SQL, pool, transaction or application policy.",
+ "later_qualification": "Aggregate protected-cache reservation and cache-only retirement require their separate owner implementation and qualification.",
+ "non_goals": [
+ "application row schema",
+ "policy interpretation",
+ "new per-read snapshots",
+ "lease framework",
+ "background tasks",
+ "second database"
+ ]
+}
diff --git a/crates/storage/src/memory.rs b/crates/storage/src/memory.rs
@@ -68,6 +68,9 @@ use crate::{
},
};
+#[path = "memory_projection_batch.rs"]
+mod projection_batch;
+
#[derive(Clone)]
struct EventEntry {
position: EventPosition,
@@ -89,7 +92,11 @@ struct State {
rebuilds: Vec<RebuildTicket>,
event_index_manifests: Vec<EventIndexManifest>,
event_index_checkpoints: Vec<EventIndexCheckpoint>,
- projection_documents: Vec<(ProjectionId, ProjectionGeneration, ProjectionDocument)>,
+ projection_documents: projection_batch::Documents,
+ projection_generations: std::collections::BTreeMap<
+ (ProjectionId, ProjectionGeneration),
+ projection_batch::Generation,
+ >,
projection_snapshots: Vec<ProjectionSnapshot>,
private_artifacts: Vec<PrivateArtifactMetadata>,
private_artifact_reseals: Vec<PrivateArtifactResealReceipt>,
@@ -130,7 +137,8 @@ impl MemoryStorage {
rebuilds: Vec::new(),
event_index_manifests: Vec::new(),
event_index_checkpoints: Vec::new(),
- projection_documents: Vec::new(),
+ projection_documents: std::collections::BTreeMap::new(),
+ projection_generations: std::collections::BTreeMap::new(),
projection_snapshots: Vec::new(),
private_artifacts: Vec::new(),
private_artifact_reseals: Vec::new(),
@@ -467,6 +475,12 @@ impl MemoryStorage {
state: &mut State,
checkpoint: ProjectionCheckpoint,
) -> Result<ProjectionStatus, Error> {
+ if state
+ .projection_generations
+ .contains_key(&(checkpoint.projection_id().clone(), checkpoint.generation()))
+ {
+ return Err(Error::ProjectionRevisionConflict);
+ }
if let Some(status) = state
.projections
.iter_mut()
@@ -1032,6 +1046,14 @@ impl ProjectionStore for MemoryStorage {
.ok_or(Error::ProjectionRevisionConflict)?;
let next = state.rebuilds[index].transition(transition)?;
if next.stage() == RebuildStage::Completed
+ && state.projection_generations.contains_key(&(
+ next.invalidation().projection_id().clone(),
+ next.invalidation().replacement_generation(),
+ ))
+ {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ if next.stage() == RebuildStage::Completed
&& (next.source_generation() != self.generation
|| next
.source_high_water()
@@ -1157,6 +1179,51 @@ impl ProjectionStore for MemoryStorage {
})
}
+ fn put_projection_batch(
+ &self,
+ batch: crate::projection::batch::ProjectionDocumentBatch,
+ ) -> BoxFuture<'_, Result<(), Error>> {
+ Box::pin(async move {
+ let mut state = self.state()?;
+ projection_batch::write(&mut state, self.generation, &batch, None)?;
+ Ok(())
+ })
+ }
+ fn promote_projection_batch(
+ &self,
+ batch: crate::projection::batch::ProjectionDocumentBatch,
+ checkpoint: ProjectionCheckpoint,
+ ) -> BoxFuture<'_, Result<ProjectionStatus, Error>> {
+ Box::pin(async move {
+ let mut state = self.state()?;
+ projection_batch::write(&mut state, self.generation, &batch, Some(checkpoint))?
+ .ok_or(Error::CorruptProjectionRecord)
+ })
+ }
+ fn query_projection_documents_at(
+ &self,
+ query: crate::projection::document_query::ProjectionDocumentQuery,
+ source: EventSourceToken,
+ active: ProjectionGeneration,
+ ) -> BoxFuture<'_, Result<crate::projection::document_query::ProjectionDocumentPage, Error>>
+ {
+ Box::pin(async move {
+ if query.generations()
+ != crate::projection::document_query::ProjectionDocumentGenerations::Exact(active)
+ {
+ return Err(Error::InvalidProjectionDocument);
+ }
+ let state = self.state()?;
+ projection_batch::read_fence(
+ &state,
+ self.generation,
+ source,
+ query.projection_id(),
+ active,
+ )?;
+ projection_batch::page(&state, &query)
+ })
+ }
fn put_projection_document(
&self,
projection_id: ProjectionId,
@@ -1165,59 +1232,29 @@ impl ProjectionStore for MemoryStorage {
) -> BoxFuture<'_, Result<(), Error>> {
Box::pin(async move {
let mut state = self.state()?;
- if let Some((_, _, existing)) = state.projection_documents.iter_mut().find(
- |(existing_id, existing_generation, existing)| {
- existing_id == &projection_id
- && *existing_generation == generation
- && existing.key() == document.key()
- },
- ) {
- *existing = document;
- } else {
- state
- .projection_documents
- .push((projection_id, generation, document));
+ if state
+ .projection_generations
+ .contains_key(&(projection_id.clone(), generation))
+ {
+ return Err(Error::ProjectionRevisionConflict);
}
+ state.projection_documents.insert(
+ (projection_id, generation, document.key().to_owned()),
+ document,
+ );
Ok(())
})
}
-
fn query_projection_documents(
&self,
query: crate::projection::document_query::ProjectionDocumentQuery,
) -> BoxFuture<'_, Result<crate::projection::document_query::ProjectionDocumentPage, Error>>
{
- use crate::projection::document_query::{
- PROJECTION_DOCUMENT_PAGE_BYTES_MAX, ProjectionDocumentPage, ProjectionDocumentRecord,
- };
Box::pin(async move {
let state = self.state()?;
- let mut selected = std::collections::BTreeMap::new();
- let maximum = usize::from(query.limit()) + 1;
- for (id, generation, document) in &state.projection_documents {
- if id == query.projection_id() && query.matches(*generation, document.key()) {
- selected.insert((*generation, document.key()), document);
- if selected.len() > maximum {
- selected.pop_last();
- }
- }
- }
- let mut has_more = selected.len() > usize::from(query.limit());
- let mut records = Vec::new();
- let mut bytes = 0;
- for ((generation, _), document) in selected.into_iter().take(usize::from(query.limit()))
- {
- if bytes + document.value().len() > PROJECTION_DOCUMENT_PAGE_BYTES_MAX {
- has_more = true;
- break;
- }
- bytes += document.value().len();
- records.push(ProjectionDocumentRecord::new(generation, document.clone())?);
- }
- ProjectionDocumentPage::new(&query, records, has_more)
+ projection_batch::page(&state, &query)
})
}
-
fn projection_document(
&self,
projection_id: ProjectionId,
@@ -1225,16 +1262,12 @@ impl ProjectionStore for MemoryStorage {
key: String,
) -> BoxFuture<'_, Result<Option<ProjectionDocument>, Error>> {
Box::pin(async move {
+ projection_batch::validate_key(&key)?;
Ok(self
.state()?
.projection_documents
- .iter()
- .find(|(existing_id, existing_generation, existing)| {
- existing_id == &projection_id
- && *existing_generation == generation
- && existing.key() == key
- })
- .map(|(_, _, document)| document.clone()))
+ .get(&(projection_id, generation, key))
+ .cloned())
})
}
diff --git a/crates/storage/src/memory_projection_batch.rs b/crates/storage/src/memory_projection_batch.rs
@@ -0,0 +1,237 @@
+use super::{Error, State};
+use crate::{
+ event::{EventSourceToken, SourceGeneration},
+ projection::{
+ ProjectionCheckpoint, ProjectionDocument, ProjectionGeneration, ProjectionHealth,
+ ProjectionId, ProjectionStatus,
+ batch::{ProjectionDocumentBatch, ProjectionWriteScope},
+ document_query::{
+ PROJECTION_DOCUMENT_PAGE_BYTES_MAX, ProjectionDocumentGenerations,
+ ProjectionDocumentPage, ProjectionDocumentQuery, ProjectionDocumentRecord,
+ },
+ },
+};
+use std::ops::Bound::{Excluded, Included, Unbounded};
+
+#[derive(Clone, Copy, Eq, PartialEq)]
+enum Phase {
+ Staging,
+ Promoted,
+ Retired,
+}
+
+#[derive(Clone)]
+pub(super) struct Generation {
+ scope: ProjectionWriteScope,
+ phase: Phase,
+ rows: u64,
+ bytes: u64,
+}
+
+pub(super) fn fence(
+ state: &State,
+ generation: SourceGeneration,
+ source: EventSourceToken,
+ id: &ProjectionId,
+ active: Option<ProjectionGeneration>,
+) -> Result<(), Error> {
+ if source != EventSourceToken::new(generation, state.event_mutation_epoch) {
+ return Err(Error::SourceEpochChanged);
+ }
+ let status = state
+ .projections
+ .iter()
+ .find(|value| value.projection_id() == id);
+ if status.map(ProjectionStatus::generation) != active
+ || status.is_some_and(|value| value.health() != ProjectionHealth::Ready)
+ {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ Ok(())
+}
+
+pub(super) fn read_fence(
+ state: &State,
+ generation: SourceGeneration,
+ source: EventSourceToken,
+ id: &ProjectionId,
+ active: ProjectionGeneration,
+) -> Result<(), Error> {
+ fence(state, generation, source, id, Some(active))?;
+ let registered = state
+ .projection_generations
+ .get(&(id.clone(), active))
+ .filter(|value| value.phase == Phase::Promoted)
+ .ok_or(Error::ProjectionRevisionConflict)?;
+ if registered.scope.source() != source {
+ return Err(Error::SourceEpochChanged);
+ }
+ Ok(())
+}
+
+pub(super) fn write(
+ state: &mut State,
+ source_generation: SourceGeneration,
+ batch: &ProjectionDocumentBatch,
+ checkpoint: Option<ProjectionCheckpoint>,
+) -> Result<Option<ProjectionStatus>, Error> {
+ let scope = batch.scope();
+ fence(
+ state,
+ source_generation,
+ scope.source(),
+ scope.projection_id(),
+ scope.expected_active(),
+ )?;
+ let identity = (scope.projection_id().clone(), scope.generation());
+ let prior = state.projection_generations.get(&identity);
+ if prior.is_some_and(|value| value.phase != Phase::Staging || value.scope != *scope) {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ if prior.is_none()
+ && (state
+ .projection_generations
+ .iter()
+ .any(|((id, _), value)| id == scope.projection_id() && value.phase == Phase::Staging)
+ || state
+ .projection_documents
+ .range((
+ Included((identity.0.clone(), identity.1, String::new())),
+ Unbounded,
+ ))
+ .next()
+ .is_some_and(|((id, generation, _), _)| {
+ id == &identity.0 && generation == &identity.1
+ }))
+ {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ let mut next = prior.cloned().unwrap_or(Generation {
+ scope: scope.clone(),
+ phase: Phase::Staging,
+ rows: 0,
+ bytes: 0,
+ });
+ for document in batch.documents() {
+ let key = (identity.0.clone(), identity.1, document.key().to_owned());
+ let previous = state.projection_documents.get(&key);
+ next.rows = next
+ .rows
+ .checked_add(u64::from(previous.is_none()))
+ .ok_or(Error::SpaceInsufficient)?;
+ next.bytes = next
+ .bytes
+ .checked_sub(previous.map_or(0, |doc| doc.value().len() as u64))
+ .and_then(|bytes| bytes.checked_add(document.value().len() as u64))
+ .ok_or(Error::SpaceInsufficient)?;
+ }
+ let promoted = checkpoint
+ .map(|checkpoint| {
+ if checkpoint.projection_id() != scope.projection_id()
+ || checkpoint.generation() != scope.generation()
+ || checkpoint.projected_rows() != next.rows
+ || checkpoint.source_position() != state.events.last().map(|entry| entry.position)
+ {
+ return Err(Error::ProjectionCheckpointMismatch);
+ }
+ ProjectionStatus::new(
+ identity.0.clone(),
+ identity.1,
+ ProjectionHealth::Ready,
+ Some(checkpoint),
+ None,
+ )
+ })
+ .transpose()?;
+ // Every fallible condition is checked before publishing any in-memory changes.
+ let documents: Vec<_> = batch
+ .documents()
+ .iter()
+ .map(|document| {
+ (
+ (identity.0.clone(), identity.1, document.key().to_owned()),
+ document.clone(),
+ )
+ })
+ .collect();
+ for (key, document) in documents {
+ state.projection_documents.insert(key, document);
+ }
+ if let Some(status) = promoted.as_ref() {
+ next.phase = Phase::Promoted;
+ if let Some(active) = scope.expected_active()
+ && let Some(previous) = state
+ .projection_generations
+ .get_mut(&(identity.0.clone(), active))
+ {
+ previous.phase = Phase::Retired;
+ }
+ if let Some(prior) = state
+ .projections
+ .iter_mut()
+ .find(|value| value.projection_id() == &identity.0)
+ {
+ *prior = status.clone();
+ } else {
+ state.projections.push(status.clone());
+ }
+ }
+ state.projection_generations.insert(identity, next);
+ Ok(promoted)
+}
+
+pub(super) fn page(
+ state: &State,
+ query: &ProjectionDocumentQuery,
+) -> Result<ProjectionDocumentPage, Error> {
+ let id = query.projection_id().clone();
+ let generation = match query.generations() {
+ ProjectionDocumentGenerations::Exact(generation) => generation,
+ ProjectionDocumentGenerations::All => ProjectionGeneration::new([
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 1,
+ ])?,
+ };
+ // The smallest valid generation is [0,..,1]; zero is never admitted.
+ let start = query.after().map_or_else(
+ || Included((id.clone(), generation, String::new())),
+ |(generation, key)| Excluded((id.clone(), generation, key.to_owned())),
+ );
+ let selected: Vec<_> = state
+ .projection_documents
+ .range((start, Unbounded))
+ .take_while(|((row_id, generation, _), _)| {
+ row_id == &id
+ && match query.generations() {
+ ProjectionDocumentGenerations::Exact(expected) => *generation == expected,
+ ProjectionDocumentGenerations::All => true,
+ }
+ })
+ .take(usize::from(query.limit()) + 1)
+ .collect();
+ let mut more = selected.len() > usize::from(query.limit());
+ let mut records = Vec::new();
+ let mut bytes = 0usize;
+ for ((_, generation, _), document) in selected.into_iter().take(usize::from(query.limit())) {
+ if bytes + document.value().len() > PROJECTION_DOCUMENT_PAGE_BYTES_MAX {
+ more = true;
+ break;
+ }
+ bytes += document.value().len();
+ records.push(ProjectionDocumentRecord::new(
+ *generation,
+ document.clone(),
+ )?);
+ }
+ ProjectionDocumentPage::new(query, records, more)
+}
+
+pub(super) fn validate_key(key: &str) -> Result<(), Error> {
+ if !crate::projection::valid_document_key(key) {
+ return Err(Error::InvalidProjectionDocument);
+ }
+ Ok(())
+}
+
+pub(super) type Documents =
+ std::collections::BTreeMap<(ProjectionId, ProjectionGeneration, String), ProjectionDocument>;
diff --git a/crates/storage/src/projection.rs b/crates/storage/src/projection.rs
@@ -3,6 +3,7 @@
//! Storage owns durable coordination metadata. Domain reducers and projected
//! row representations remain in their domain packages.
+pub mod batch;
pub mod document_query;
pub use radroots_event::EventId;
@@ -1027,6 +1028,30 @@ impl ProjectionStatus {
/// Backend-neutral projection coordination SPI.
pub trait ProjectionStore: Send + Sync {
+ /// Atomically stages a bounded batch under a durable source and generation fence.
+ fn put_projection_batch(
+ &self,
+ _batch: batch::ProjectionDocumentBatch,
+ ) -> BoxFuture<'_, Result<(), Error>> {
+ Box::pin(async { Err(Error::BackendUnavailable) })
+ }
+ /// Writes the final bounded batch and publishes its complete generation atomically.
+ fn promote_projection_batch(
+ &self,
+ _batch: batch::ProjectionDocumentBatch,
+ _checkpoint: ProjectionCheckpoint,
+ ) -> BoxFuture<'_, Result<ProjectionStatus, Error>> {
+ Box::pin(async { Err(Error::BackendUnavailable) })
+ }
+ /// Reads an exact active generation under one canonical source fence.
+ fn query_projection_documents_at(
+ &self,
+ _query: document_query::ProjectionDocumentQuery,
+ _source: crate::event::EventSourceToken,
+ _active: ProjectionGeneration,
+ ) -> BoxFuture<'_, Result<document_query::ProjectionDocumentPage, Error>> {
+ Box::pin(async { Err(Error::BackendUnavailable) })
+ }
fn status(
&self,
projection_id: ProjectionId,
@@ -1115,7 +1140,7 @@ fn valid_label(value: &str, max: usize) -> bool {
})
}
-fn valid_document_key(value: &str) -> bool {
+pub(crate) fn valid_document_key(value: &str) -> bool {
!value.is_empty()
&& value.len() <= PROJECTION_DOCUMENT_KEY_MAX_BYTES
&& value == value.trim()
diff --git a/crates/storage/src/projection/batch.rs b/crates/storage/src/projection/batch.rs
@@ -0,0 +1,114 @@
+//! Bounded opaque writes; callers retain policy and serialization ownership.
+use super::{Error, ProjectionDocument, ProjectionGeneration, ProjectionId};
+use crate::event::EventSourceToken;
+use std::collections::BTreeSet;
+
+pub const PROJECTION_BATCH_ROWS_MAX: usize = 64;
+pub const PROJECTION_BATCH_BYTES_MAX: usize = 16_777_216;
+
+/// A content-derived generation binds caller policy/schema and the canonical source.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct ProjectionWriteScope {
+ projection_id: ProjectionId,
+ generation: ProjectionGeneration,
+ source: EventSourceToken,
+ expected_active: Option<ProjectionGeneration>,
+}
+impl ProjectionWriteScope {
+ pub fn new(
+ projection_id: ProjectionId,
+ generation: ProjectionGeneration,
+ source: EventSourceToken,
+ expected_active: Option<ProjectionGeneration>,
+ ) -> Result<Self, Error> {
+ ProjectionId::parse(projection_id.as_str())?;
+ ProjectionGeneration::new(*generation.as_bytes())?;
+ crate::event::SourceGeneration::new(*source.generation().as_bytes())?;
+ if let Some(active) = expected_active {
+ ProjectionGeneration::new(*active.as_bytes())?;
+ if active == generation {
+ return Err(Error::InvalidProjectionGeneration);
+ }
+ }
+ Ok(Self {
+ projection_id,
+ generation,
+ source,
+ expected_active,
+ })
+ }
+ pub const fn projection_id(&self) -> &ProjectionId {
+ &self.projection_id
+ }
+ pub const fn generation(&self) -> ProjectionGeneration {
+ self.generation
+ }
+ pub const fn source(&self) -> EventSourceToken {
+ self.source
+ }
+ pub const fn expected_active(&self) -> Option<ProjectionGeneration> {
+ self.expected_active
+ }
+}
+
+#[derive(Clone, Eq, PartialEq)]
+pub struct ProjectionDocumentBatch {
+ scope: ProjectionWriteScope,
+ documents: Vec<ProjectionDocument>,
+ logical_bytes: usize,
+}
+impl ProjectionDocumentBatch {
+ /// Empty batches allow publishing a genuinely empty or already staged generation.
+ pub fn new(
+ scope: ProjectionWriteScope,
+ documents: Vec<ProjectionDocument>,
+ ) -> Result<Self, Error> {
+ if documents.len() > PROJECTION_BATCH_ROWS_MAX {
+ return Err(Error::InvalidProjectionDocument);
+ }
+ let mut keys = BTreeSet::new();
+ let mut charge = 4096usize;
+ for document in &documents {
+ if !keys.insert(document.key()) {
+ return Err(Error::InvalidProjectionDocument);
+ }
+ charge = charge
+ .checked_add(
+ document
+ .value()
+ .len()
+ .checked_mul(2)
+ .and_then(|size| size.checked_add(document.key().len() * 2 + 256))
+ .ok_or(Error::SpaceInsufficient)?,
+ )
+ .ok_or(Error::SpaceInsufficient)?;
+ if charge > PROJECTION_BATCH_BYTES_MAX {
+ return Err(Error::SpaceInsufficient);
+ }
+ }
+ Ok(Self {
+ scope,
+ documents,
+ logical_bytes: charge,
+ })
+ }
+ pub const fn scope(&self) -> &ProjectionWriteScope {
+ &self.scope
+ }
+ pub fn documents(&self) -> &[ProjectionDocument] {
+ &self.documents
+ }
+ pub const fn logical_bytes(&self) -> usize {
+ self.logical_bytes
+ }
+}
+impl std::fmt::Debug for ProjectionDocumentBatch {
+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ formatter
+ .debug_struct("ProjectionDocumentBatch")
+ .field("scope", &self.scope)
+ .field("rows", &self.documents.len())
+ .field("logical_bytes", &self.logical_bytes)
+ .finish()
+ }
+}
diff --git a/crates/storage/tests/projection_batch.rs b/crates/storage/tests/projection_batch.rs
@@ -0,0 +1,262 @@
+use futures_executor::block_on;
+use radroots_storage::{
+ Error, EventStore, ProjectionStore,
+ event::{EventSourceToken, SourceGeneration},
+ memory::MemoryStorage,
+ projection::{
+ ProjectionCheckpoint, ProjectionDocument, ProjectionGeneration, ProjectionId,
+ batch::{PROJECTION_BATCH_BYTES_MAX, ProjectionDocumentBatch, ProjectionWriteScope},
+ document_query::{ProjectionDocumentGenerations, ProjectionDocumentQuery},
+ },
+};
+fn generation(byte: u8) -> ProjectionGeneration {
+ ProjectionGeneration::new([byte; 32]).unwrap()
+}
+fn id() -> ProjectionId {
+ ProjectionId::parse("fixture").unwrap()
+}
+fn document(key: &str, size: usize) -> ProjectionDocument {
+ ProjectionDocument::new(key.into(), vec![7; size]).unwrap()
+}
+fn scope(
+ source: EventSourceToken,
+ active: Option<ProjectionGeneration>,
+ target: u8,
+) -> ProjectionWriteScope {
+ ProjectionWriteScope::new(id(), generation(target), source, active).unwrap()
+}
+fn query(target: u8) -> ProjectionDocumentQuery {
+ ProjectionDocumentQuery::new(
+ id(),
+ ProjectionDocumentGenerations::Exact(generation(target)),
+ 64,
+ )
+ .unwrap()
+}
+
+#[test]
+fn projection_batch_bounds_duplicates_redaction_and_scope_fail_closed() {
+ let source = EventSourceToken::new(SourceGeneration::new([9; 32]).unwrap(), 0);
+ let scope = scope(source, None, 1);
+ let docs: Vec<_> = (0..64)
+ .map(|i| document(&format!("key.{i:02}"), 1))
+ .collect();
+ let batch = ProjectionDocumentBatch::new(scope.clone(), docs).unwrap();
+ assert_eq!(batch.documents().len(), 64);
+ assert!(!format!("{batch:?}").contains("key.00"));
+ assert_eq!(
+ ProjectionDocumentBatch::new(
+ scope.clone(),
+ (0..65).map(|i| document(&format!("k{i}"), 1)).collect()
+ ),
+ Err(Error::InvalidProjectionDocument)
+ );
+ assert_eq!(
+ ProjectionDocumentBatch::new(
+ scope.clone(),
+ vec![document("same", 1), document("same", 2)]
+ ),
+ Err(Error::InvalidProjectionDocument)
+ );
+ let value_max = (PROJECTION_BATCH_BYTES_MAX - 4096 - 256 - 2) / 2;
+ let exact =
+ ProjectionDocumentBatch::new(scope.clone(), vec![document("k", value_max)]).unwrap();
+ assert_eq!(exact.logical_bytes(), PROJECTION_BATCH_BYTES_MAX);
+ assert_eq!(
+ ProjectionDocumentBatch::new(scope, vec![document("k", value_max + 1)]),
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(
+ ProjectionWriteScope::new(id(), generation(1), source, Some(generation(1))),
+ Err(Error::InvalidProjectionGeneration)
+ );
+ assert!(ProjectionDocument::new("\n".into(), vec![1]).is_err());
+ assert!(ProjectionId::parse("wrong scope").is_err());
+}
+
+#[test]
+fn projection_memory_atomic_generation_counts_fences_and_sealing_are_coherent() {
+ let store = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ let source = block_on(store.source_token()).unwrap();
+ let first = scope(source, None, 1);
+ block_on(
+ store.put_projection_batch(
+ ProjectionDocumentBatch::new(first.clone(), vec![document("a", 1), document("b", 2)])
+ .unwrap(),
+ ),
+ )
+ .unwrap();
+ assert_eq!(
+ block_on(ProjectionStore::status(&store, id())).unwrap(),
+ None
+ );
+ let bad = ProjectionCheckpoint::new(id(), generation(1), None, 1, 1).unwrap();
+ assert_eq!(
+ block_on(store.promote_projection_batch(
+ ProjectionDocumentBatch::new(first.clone(), vec![document("c", 3)]).unwrap(),
+ bad
+ )),
+ Err(Error::ProjectionCheckpointMismatch)
+ );
+ assert_eq!(
+ block_on(store.query_projection_documents(query(1)))
+ .unwrap()
+ .records()
+ .len(),
+ 2
+ );
+ let good = ProjectionCheckpoint::new(id(), generation(1), None, 3, 1).unwrap();
+ block_on(store.promote_projection_batch(
+ ProjectionDocumentBatch::new(first, vec![document("c", 3)]).unwrap(),
+ good,
+ ))
+ .unwrap();
+ assert_eq!(
+ block_on(store.query_projection_documents_at(query(1), source, generation(1)))
+ .unwrap()
+ .records()
+ .len(),
+ 3
+ );
+ assert_eq!(
+ block_on(store.put_projection_document(id(), generation(1), document("a", 9))),
+ Err(Error::ProjectionRevisionConflict)
+ );
+ let second = scope(source, Some(generation(1)), 2);
+ block_on(store.put_projection_batch(
+ ProjectionDocumentBatch::new(second.clone(), vec![document("x", 1)]).unwrap(),
+ ))
+ .unwrap();
+ assert_eq!(
+ block_on(store.put_projection_batch(
+ ProjectionDocumentBatch::new(scope(source, Some(generation(1)), 3), vec![]).unwrap()
+ )),
+ Err(Error::ProjectionRevisionConflict)
+ );
+ let stale = EventSourceToken::new(source.generation(), source.epoch() + 1);
+ assert_eq!(
+ block_on(store.query_projection_documents_at(query(1), stale, generation(1))),
+ Err(Error::SourceEpochChanged)
+ );
+ assert_eq!(
+ block_on(store.query_projection_documents_at(query(2), source, generation(1))),
+ Err(Error::InvalidProjectionDocument)
+ );
+ block_on(store.promote_projection_batch(
+ ProjectionDocumentBatch::new(second, vec![document("y", 2)]).unwrap(),
+ ProjectionCheckpoint::new(id(), generation(2), None, 2, 2).unwrap(),
+ ))
+ .unwrap();
+ assert_eq!(
+ block_on(store.query_projection_documents_at(query(1), source, generation(1))),
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ block_on(store.query_projection_documents_at(query(2), source, generation(2)))
+ .unwrap()
+ .records()
+ .len(),
+ 2
+ );
+ assert_eq!(
+ block_on(store.put_projection_batch(
+ ProjectionDocumentBatch::new(scope(source, None, 4), vec![]).unwrap()
+ )),
+ Err(Error::ProjectionRevisionConflict)
+ );
+}
+
+#[test]
+fn projection_memory_updates_empty_generations_and_checkpoint_scopes_are_atomic() {
+ let store = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ let source = block_on(store.source_token()).unwrap();
+ block_on(store.put_projection_document(id(), generation(1), document("legacy", 1))).unwrap();
+ assert_eq!(
+ block_on(store.put_projection_batch(
+ ProjectionDocumentBatch::new(scope(source, None, 1), vec![]).unwrap()
+ )),
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ block_on(store.projection_document(id(), generation(1), "bad\nkey".into())),
+ Err(Error::InvalidProjectionDocument)
+ );
+ let next = scope(source, None, 2);
+ let rows: Vec<_> = (0..64).map(|i| document(&format!("k{i:02}"), 1)).collect();
+ block_on(store.put_projection_batch(ProjectionDocumentBatch::new(next.clone(), rows).unwrap()))
+ .unwrap();
+ block_on(store.put_projection_batch(
+ ProjectionDocumentBatch::new(next.clone(), vec![document("k00", 3)]).unwrap(),
+ ))
+ .unwrap();
+ assert_eq!(
+ block_on(store.projection_document(id(), generation(2), "k00".into()))
+ .unwrap()
+ .unwrap()
+ .value()
+ .len(),
+ 3
+ );
+ for checkpoint in [
+ ProjectionCheckpoint::new(
+ ProjectionId::parse("foreign").unwrap(),
+ generation(2),
+ None,
+ 64,
+ 1,
+ )
+ .unwrap(),
+ ProjectionCheckpoint::new(id(), generation(3), None, 64, 1).unwrap(),
+ ProjectionCheckpoint::new(
+ id(),
+ generation(2),
+ Some(radroots_storage::event::EventPosition::new(
+ source.generation(),
+ radroots_storage::event::EventSequence::new(1).unwrap(),
+ )),
+ 64,
+ 1,
+ )
+ .unwrap(),
+ ] {
+ assert_eq!(
+ block_on(store.promote_projection_batch(
+ ProjectionDocumentBatch::new(next.clone(), vec![document("k00", 4)]).unwrap(),
+ checkpoint
+ )),
+ Err(Error::ProjectionCheckpointMismatch)
+ );
+ assert_eq!(
+ block_on(store.projection_document(id(), generation(2), "k00".into()))
+ .unwrap()
+ .unwrap()
+ .value()
+ .len(),
+ 3
+ );
+ }
+ block_on(store.promote_projection_batch(
+ ProjectionDocumentBatch::new(next, vec![]).unwrap(),
+ ProjectionCheckpoint::new(id(), generation(2), None, 64, 1).unwrap(),
+ ))
+ .unwrap();
+ let empty = scope(source, Some(generation(2)), 3);
+ block_on(store.promote_projection_batch(
+ ProjectionDocumentBatch::new(empty, vec![]).unwrap(),
+ ProjectionCheckpoint::new(id(), generation(3), None, 0, 2).unwrap(),
+ ))
+ .unwrap();
+ assert!(
+ block_on(store.query_projection_documents_at(query(3), source, generation(3)))
+ .unwrap()
+ .records()
+ .is_empty()
+ );
+ let foreign = EventSourceToken::new(SourceGeneration::new([8; 32]).unwrap(), 0);
+ assert_eq!(
+ block_on(store.put_projection_batch(
+ ProjectionDocumentBatch::new(scope(foreign, Some(generation(3)), 4), vec![]).unwrap()
+ )),
+ Err(Error::SourceEpochChanged)
+ );
+}
diff --git a/crates/storage_sqlite/src/migration.rs b/crates/storage_sqlite/src/migration.rs
@@ -469,7 +469,7 @@ fn schema_metadata_error(source: &sqlx::Error, database: &'static str) -> Error
fn validate_plan(plan: &MigrationPlan) -> Result<(), Error> {
let valid = plan.minimum_version > 0
&& plan.minimum_version <= plan.current_version
- && plan.current_version <= 20
+ && plan.current_version <= 21
&& plan.steps.len() == usize::try_from(plan.current_version).unwrap_or(usize::MAX)
&& plan
.steps
@@ -596,6 +596,7 @@ const fn set_user_version_sql(version: u32) -> Option<&'static str> {
18 => Some("PRAGMA user_version = 18"),
19 => Some("PRAGMA user_version = 19"),
20 => Some("PRAGMA user_version = 20"),
+ 21 => Some("PRAGMA user_version = 21"),
_ => None,
}
}
@@ -880,8 +881,8 @@ mod tests {
.await
.expect("application id");
let newer_version = runtime::CURRENT_VERSION + 1;
- assert_eq!(newer_version, 21);
- sqlx::raw_sql("PRAGMA user_version = 21")
+ assert_eq!(newer_version, 22);
+ sqlx::raw_sql("PRAGMA user_version = 22")
.execute(&mut newer)
.await
.expect("newer version");
diff --git a/crates/storage_sqlite/src/migration/runtime/0021_projection_atomic_generations.up.sql b/crates/storage_sqlite/src/migration/runtime/0021_projection_atomic_generations.up.sql
@@ -0,0 +1,14 @@
+CREATE TABLE radroots_runtime_projection_generations (
+ projection_id TEXT NOT NULL CHECK(length(projection_id) BETWEEN 1 AND 128),
+ generation BLOB NOT NULL CHECK(length(generation) = 32),
+ source_generation BLOB NOT NULL CHECK(length(source_generation) = 32),
+ source_epoch INTEGER NOT NULL CHECK(source_epoch >= 0),
+ expected_active BLOB CHECK(expected_active IS NULL OR length(expected_active) = 32),
+ phase TEXT NOT NULL CHECK(phase IN ('staging', 'promoted', 'retired')),
+ document_rows INTEGER NOT NULL DEFAULT 0 CHECK(document_rows >= 0),
+ value_bytes INTEGER NOT NULL DEFAULT 0 CHECK(value_bytes >= 0),
+ PRIMARY KEY(projection_id, generation)
+) STRICT, WITHOUT ROWID;
+
+CREATE UNIQUE INDEX radroots_runtime_projection_generations_staging_idx
+ON radroots_runtime_projection_generations(projection_id) WHERE phase = 'staging';
diff --git a/crates/storage_sqlite/src/migration/runtime/mod.rs b/crates/storage_sqlite/src/migration/runtime/mod.rs
@@ -6,7 +6,7 @@
/// Lowest runtime schema version this package can recognize.
pub const MINIMUM_VERSION: u32 = 1;
/// Current runtime schema version created by this package.
-pub const CURRENT_VERSION: u32 = 20;
+pub const CURRENT_VERSION: u32 = 21;
const RUNTIME_V1_SQL: &str = include_str!("0001_runtime.up.sql");
const CANONICAL_EVENT_STORAGE_V2_SQL: &str = include_str!("0002_canonical_event_storage.up.sql");
@@ -38,6 +38,9 @@ const EVENT_MUTATION_EPOCH_V18_SQL: &str = include_str!("0018_event_mutation_epo
const CANONICAL_VISIBILITY_INDEX_V19_SQL: &str =
include_str!("0019_canonical_visibility_index.up.sql");
+const PROJECTION_ATOMIC_GENERATIONS_V21_SQL: &str =
+ include_str!("0021_projection_atomic_generations.up.sql");
+
const INDEXED_VISIBILITY_READS_V20_SQL: &str = include_str!("0020_indexed_visibility_reads.up.sql");
/// Stable, non-SQL description of one forward runtime migration.
@@ -1207,6 +1210,118 @@ const RUNTIME_V20_OBJECTS: &[&str] = &[
];
/// Ordered, immutable runtime migration plan.
+const RUNTIME_V21_OBJECTS: &[&str] = &[
+ "radroots_runtime_atomic_commits",
+ "radroots_runtime_authored_artifacts",
+ "radroots_runtime_authored_artifacts_admission_ready_idx",
+ "radroots_runtime_authored_artifacts_signed_fact_guard",
+ "radroots_runtime_authored_artifacts_signing_ready_idx",
+ "radroots_runtime_authored_atomic_commits",
+ "radroots_runtime_authored_atomic_commits_delete_guard",
+ "radroots_runtime_authored_atomic_commits_update_guard",
+ "radroots_runtime_authored_atomic_target_phase_idx",
+ "radroots_runtime_authored_delivery_attempts",
+ "radroots_runtime_authored_delivery_claims",
+ "radroots_runtime_authored_delivery_claims_delete_guard",
+ "radroots_runtime_authored_delivery_claims_update_guard",
+ "radroots_runtime_authored_delivery_facts",
+ "radroots_runtime_authored_delivery_facts_delete_guard",
+ "radroots_runtime_authored_delivery_facts_update_guard",
+ "radroots_runtime_authored_delivery_plans",
+ "radroots_runtime_authored_delivery_ready_idx",
+ "radroots_runtime_authored_delivery_reconciliations",
+ "radroots_runtime_authored_delivery_reconciliations_delete_guard",
+ "radroots_runtime_authored_delivery_reconciliations_update_guard",
+ "radroots_runtime_authored_delivery_stop_guard",
+ "radroots_runtime_authored_delivery_targets",
+ "radroots_runtime_authored_draft_author_head_idx",
+ "radroots_runtime_authored_draft_revisions",
+ "radroots_runtime_authored_draft_revisions_delete_guard",
+ "radroots_runtime_authored_draft_revisions_update_guard",
+ "radroots_runtime_authored_draft_scope_head_idx",
+ "radroots_runtime_authored_migration_evidence",
+ "radroots_runtime_authored_migration_evidence_delete_guard",
+ "radroots_runtime_authored_migration_evidence_update_guard",
+ "radroots_runtime_authored_operations",
+ "radroots_runtime_delivery_evidence",
+ "radroots_runtime_delivery_evidence_item_idx",
+ "radroots_runtime_event_index_checkpoints",
+ "radroots_runtime_event_index_manifests",
+ "radroots_runtime_event_index_shards",
+ "radroots_runtime_event_provenance",
+ "radroots_runtime_event_provenance_observed_idx",
+ "radroots_runtime_events",
+ "radroots_runtime_events_admission_idx",
+ "radroots_runtime_events_contract_metadata_guard",
+ "radroots_runtime_events_contract_metadata_insert_guard",
+ "radroots_runtime_events_delete_guard",
+ "radroots_runtime_events_event_id_idx",
+ "radroots_runtime_events_mutation_idx",
+ "radroots_runtime_events_mutation_insert",
+ "radroots_runtime_events_mutation_update",
+ "radroots_runtime_events_raw_update_guard",
+ "radroots_runtime_journal_idempotency_idx",
+ "radroots_runtime_journal_operations",
+ "radroots_runtime_journal_recovery_idx",
+ "radroots_runtime_legacy_event_staging",
+ "radroots_runtime_legacy_event_staging_delete_guard",
+ "radroots_runtime_legacy_event_staging_insert_guard",
+ "radroots_runtime_legacy_event_staging_update_guard",
+ "radroots_runtime_legacy_import_commit_delete_guard",
+ "radroots_runtime_legacy_import_commit_update_guard",
+ "radroots_runtime_legacy_import_commits",
+ "radroots_runtime_legacy_import_delete_guard",
+ "radroots_runtime_legacy_import_identity_guard",
+ "radroots_runtime_legacy_import_member_delete_guard",
+ "radroots_runtime_legacy_import_member_identity_guard",
+ "radroots_runtime_legacy_import_member_state_guard",
+ "radroots_runtime_legacy_import_members",
+ "radroots_runtime_legacy_import_state_guard",
+ "radroots_runtime_legacy_import_state_idx",
+ "radroots_runtime_legacy_imports",
+ "radroots_runtime_legacy_outbox_staging",
+ "radroots_runtime_legacy_outbox_staging_delete_guard",
+ "radroots_runtime_legacy_outbox_staging_insert_guard",
+ "radroots_runtime_legacy_outbox_staging_parent_idx",
+ "radroots_runtime_legacy_outbox_staging_update_guard",
+ "radroots_runtime_outbox_items",
+ "radroots_runtime_outbox_operation_idx",
+ "radroots_runtime_outbox_ready_idx",
+ "radroots_runtime_outbox_targets",
+ "radroots_runtime_projection_checkpoints",
+ "radroots_runtime_projection_documents",
+ "radroots_runtime_projection_generations",
+ "radroots_runtime_projection_generations_staging_idx",
+ "radroots_runtime_projection_invalidations",
+ "radroots_runtime_projection_rebuilds",
+ "radroots_runtime_projection_rebuilds_stage_idx",
+ "radroots_runtime_projection_snapshots",
+ "radroots_runtime_projection_snapshots_created_idx",
+ "radroots_runtime_projection_snapshots_update_guard",
+ "radroots_runtime_source_generations",
+ "radroots_runtime_source_generations_active_idx",
+ "radroots_runtime_source_generations_delete_guard",
+ "radroots_runtime_source_generations_identity_guard",
+ "radroots_runtime_source_generations_mutation_guard",
+ "radroots_runtime_source_generations_sequence_guard",
+ "radroots_runtime_visibility_deletion_addresses",
+ "radroots_runtime_visibility_deletion_addresses_request_idx",
+ "radroots_runtime_visibility_deletion_events",
+ "radroots_runtime_visibility_deletion_events_request_idx",
+ "radroots_runtime_visibility_empty_source",
+ "radroots_runtime_visibility_events",
+ "radroots_runtime_visibility_events_head_idx",
+ "radroots_runtime_visibility_events_stage_idx",
+ "radroots_runtime_visibility_heads",
+ "radroots_runtime_visibility_members",
+ "radroots_runtime_visibility_members_count_insert",
+ "radroots_runtime_visibility_members_count_update",
+ "radroots_runtime_visibility_members_event_idx",
+ "radroots_runtime_visibility_members_page_idx",
+ "radroots_runtime_visibility_pending",
+ "radroots_runtime_visibility_progress",
+];
+
pub const MIGRATIONS: &[MigrationDescriptor] = &[
MigrationDescriptor {
version: 1,
@@ -1328,6 +1443,12 @@ pub const MIGRATIONS: &[MigrationDescriptor] = &[
up_sha256: "914ed79cfb5073f3031413ebc21d64c11f3644e46cb38ef32cd85126b00a95cc",
owned_objects: RUNTIME_V20_OBJECTS,
},
+ MigrationDescriptor {
+ version: 21,
+ name: "projection_atomic_generations",
+ up_sha256: "e50dc6896a40547febf7df26f00ad66814e59d05c7bc9b09e474843195f3b79e",
+ owned_objects: RUNTIME_V21_OBJECTS,
+ },
];
pub(crate) const fn migration_sql(version: u32) -> Option<&'static str> {
@@ -1352,6 +1473,7 @@ pub(crate) const fn migration_sql(version: u32) -> Option<&'static str> {
18 => Some(EVENT_MUTATION_EPOCH_V18_SQL),
19 => Some(CANONICAL_VISIBILITY_INDEX_V19_SQL),
20 => Some(INDEXED_VISIBILITY_READS_V20_SQL),
+ 21 => Some(PROJECTION_ATOMIC_GENERATIONS_V21_SQL),
_ => None,
}
}
@@ -1395,8 +1517,8 @@ mod tests {
fn migration_plan_matches_governed_snapshot() {
let snapshot = toml::from_str::<PlanSnapshot>(PLAN_SNAPSHOT).expect("valid snapshot");
assert_eq!(MINIMUM_VERSION, 1);
- assert_eq!(CURRENT_VERSION, 20);
- assert_eq!(MIGRATIONS.len(), 20);
+ assert_eq!(CURRENT_VERSION, 21);
+ assert_eq!(MIGRATIONS.len(), 21);
let migration = MIGRATIONS[8];
assert_eq!(snapshot.schema_version, 1);
assert_eq!(snapshot.database, "runtime.sqlite");
diff --git a/crates/storage_sqlite/src/projection/batch.rs b/crates/storage_sqlite/src/projection/batch.rs
@@ -0,0 +1,278 @@
+use super::{SqliteStorage, decode_status, map_backend, put_status_transaction};
+use radroots_storage::{
+ Error,
+ event::EventSourceToken,
+ projection::{
+ ProjectionCheckpoint, ProjectionDocument, ProjectionGeneration, ProjectionHealth,
+ ProjectionId, ProjectionStatus, batch::ProjectionDocumentBatch,
+ },
+};
+use sqlx::{Row, Sqlite, Transaction};
+
+pub(super) async fn fence(
+ transaction: &mut Transaction<'_, Sqlite>,
+ store: &SqliteStorage,
+ source: EventSourceToken,
+ id: &ProjectionId,
+ expected: Option<ProjectionGeneration>,
+) -> Result<u64, Error> {
+ if source.generation() != store.generation {
+ return Err(Error::SourceEpochChanged);
+ }
+ let row = sqlx::query("SELECT mutation_epoch, sequence_head FROM radroots_runtime_source_generations WHERE generation = ? AND state = 'active'")
+ .bind(source.generation().as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(map_backend)?
+ .ok_or(Error::SourceEpochChanged)?;
+ let epoch = super::u64_from_i64(row.try_get("mutation_epoch").map_err(super::map_corrupt)?)?;
+ if epoch != source.epoch() {
+ return Err(Error::SourceEpochChanged);
+ }
+ let status = sqlx::query(
+ "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?",
+ )
+ .bind(id.as_str())
+ .fetch_optional(&mut **transaction)
+ .await
+ .map_err(map_backend)?
+ .as_ref()
+ .map(decode_status)
+ .transpose()?;
+ if status.as_ref().map(ProjectionStatus::generation) != expected
+ || status
+ .as_ref()
+ .is_some_and(|value| value.health() != ProjectionHealth::Ready)
+ {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ super::u64_from_i64(row.try_get("sequence_head").map_err(super::map_corrupt)?)
+}
+
+pub(super) async fn read_fence(
+ transaction: &mut Transaction<'_, Sqlite>,
+ store: &SqliteStorage,
+ source: EventSourceToken,
+ id: &ProjectionId,
+ active: ProjectionGeneration,
+) -> Result<(), Error> {
+ fence(transaction, store, source, id, Some(active)).await?;
+ let row = sqlx::query("SELECT source_generation, source_epoch, phase FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?")
+ .bind(id.as_str()).bind(active.as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(map_backend)?
+ .ok_or(Error::ProjectionRevisionConflict)?;
+ let phase: String = row.try_get("phase").map_err(super::map_corrupt)?;
+ if phase != "promoted" {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ let generation: Vec<u8> = row
+ .try_get("source_generation")
+ .map_err(super::map_corrupt)?;
+ let epoch = super::u64_from_i64(row.try_get("source_epoch").map_err(super::map_corrupt)?)?;
+ if generation != source.generation().as_bytes() || epoch != source.epoch() {
+ return Err(Error::SourceEpochChanged);
+ }
+ Ok(())
+}
+
+pub(super) async fn refuse_registered_legacy(
+ transaction: &mut Transaction<'_, Sqlite>,
+ id: &ProjectionId,
+ generation: ProjectionGeneration,
+) -> Result<(), Error> {
+ let registered: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?")
+ .bind(id.as_str()).bind(generation.as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(map_backend)?;
+ if registered.is_some() {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ Ok(())
+}
+
+pub(super) async fn write(
+ store: &SqliteStorage,
+ batch: ProjectionDocumentBatch,
+ checkpoint: Option<ProjectionCheckpoint>,
+) -> Result<Option<ProjectionStatus>, Error> {
+ store.require_projection_writer()?;
+ let mut transaction = store
+ .pool()
+ .begin_with("BEGIN IMMEDIATE")
+ .await
+ .map_err(map_backend)?;
+ let scope = batch.scope();
+ let sequence = fence(
+ &mut transaction,
+ store,
+ scope.source(),
+ scope.projection_id(),
+ scope.expected_active(),
+ )
+ .await?;
+ let prior = sqlx::query("SELECT source_generation, source_epoch, expected_active, phase, document_rows, value_bytes FROM radroots_runtime_projection_generations WHERE projection_id = ? AND generation = ?")
+ .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice())
+ .fetch_optional(&mut *transaction).await.map_err(map_backend)?;
+ let (mut count, mut bytes) = if let Some(row) = prior {
+ let generation: Vec<u8> = row
+ .try_get("source_generation")
+ .map_err(super::map_corrupt)?;
+ let epoch = super::u64_from_i64(row.try_get("source_epoch").map_err(super::map_corrupt)?)?;
+ let active: Option<Vec<u8>> = row.try_get("expected_active").map_err(super::map_corrupt)?;
+ let phase: String = row.try_get("phase").map_err(super::map_corrupt)?;
+ if generation != scope.source().generation().as_bytes()
+ || epoch != scope.source().epoch()
+ || active.as_deref()
+ != scope
+ .expected_active()
+ .as_ref()
+ .map(|value| value.as_bytes().as_slice())
+ || phase != "staging"
+ {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ (
+ super::u64_from_i64(row.try_get("document_rows").map_err(super::map_corrupt)?)?,
+ super::u64_from_i64(row.try_get("value_bytes").map_err(super::map_corrupt)?)?,
+ )
+ } else {
+ let staging: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id = ? AND phase = 'staging' LIMIT 1")
+ .bind(scope.projection_id().as_str()).fetch_optional(&mut *transaction).await.map_err(map_backend)?;
+ let existing: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_documents WHERE projection_id = ? AND generation = ? LIMIT 1")
+ .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice())
+ .fetch_optional(&mut *transaction).await.map_err(map_backend)?;
+ if staging.is_some() || existing.is_some() {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ sqlx::query("INSERT INTO radroots_runtime_projection_generations (projection_id,generation,source_generation,source_epoch,expected_active,phase) VALUES (?,?,?,?,?,'staging')")
+ .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice())
+ .bind(scope.source().generation().as_bytes().as_slice()).bind(super::i64_from_u64(scope.source().epoch())?)
+ .bind(scope.expected_active().map(|value| value.as_bytes().to_vec())).execute(&mut *transaction).await.map_err(map_backend)?;
+ (0, 0)
+ };
+ for document in batch.documents() {
+ let previous: Option<i64> = sqlx::query_scalar("SELECT length(value) FROM radroots_runtime_projection_documents WHERE projection_id = ? AND generation = ? AND document_key = ?")
+ .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind(document.key())
+ .fetch_optional(&mut *transaction).await.map_err(map_backend)?;
+ count = count
+ .checked_add(u64::from(previous.is_none()))
+ .ok_or(Error::SpaceInsufficient)?;
+ let previous = previous.map(super::u64_from_i64).transpose()?.unwrap_or(0);
+ bytes = bytes
+ .checked_sub(previous)
+ .and_then(|size| size.checked_add(document.value().len() as u64))
+ .ok_or(Error::CorruptProjectionRecord)?;
+ put(
+ &mut transaction,
+ scope.projection_id(),
+ scope.generation(),
+ document,
+ )
+ .await?;
+ }
+ let promoted = if let Some(checkpoint) = checkpoint {
+ if checkpoint.projection_id() != scope.projection_id()
+ || checkpoint.generation() != scope.generation()
+ || checkpoint.projected_rows() != count
+ || match checkpoint.source_position() {
+ Some(position) => {
+ position.generation() != scope.source().generation()
+ || position.sequence().get() != sequence
+ }
+ None => sequence != 0,
+ }
+ {
+ return Err(Error::ProjectionCheckpointMismatch);
+ }
+ let status = ProjectionStatus::new(
+ scope.projection_id().clone(),
+ scope.generation(),
+ ProjectionHealth::Ready,
+ Some(checkpoint),
+ None,
+ )?;
+ sqlx::query("UPDATE radroots_runtime_projection_generations SET phase = 'retired' WHERE projection_id = ? AND phase = 'promoted'")
+ .bind(scope.projection_id().as_str()).execute(&mut *transaction).await.map_err(map_backend)?;
+ put_status_transaction(&mut transaction, &status).await?;
+ Some(status)
+ } else {
+ None
+ };
+ sqlx::query("UPDATE radroots_runtime_projection_generations SET document_rows = ?, value_bytes = ?, phase = ? WHERE projection_id = ? AND generation = ?")
+ .bind(super::i64_from_u64(count)?).bind(super::i64_from_u64(bytes)?)
+ .bind(if promoted.is_some() { "promoted" } else { "staging" })
+ .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice())
+ .execute(&mut *transaction).await.map_err(map_backend)?;
+ transaction.commit().await.map_err(map_backend)?;
+ Ok(promoted)
+}
+
+pub(super) async fn put(
+ transaction: &mut Transaction<'_, Sqlite>,
+ id: &ProjectionId,
+ generation: ProjectionGeneration,
+ document: &ProjectionDocument,
+) -> Result<(), Error> {
+ sqlx::query("INSERT INTO radroots_runtime_projection_documents (projection_id,generation,document_key,value,value_sha256) VALUES (?,?,?,?,?) ON CONFLICT(projection_id,generation,document_key) DO UPDATE SET value=excluded.value,value_sha256=excluded.value_sha256")
+ .bind(id.as_str()).bind(generation.as_bytes().as_slice()).bind(document.key()).bind(document.value())
+ .bind(document.value_sha256().as_slice()).execute(&mut **transaction).await.map_err(map_backend)?;
+ Ok(())
+}
+
+pub(super) async fn legacy_put(
+ store: &SqliteStorage,
+ id: ProjectionId,
+ generation: ProjectionGeneration,
+ document: ProjectionDocument,
+) -> Result<(), Error> {
+ store.require_projection_writer()?;
+ let mut transaction = store
+ .pool()
+ .begin_with("BEGIN IMMEDIATE")
+ .await
+ .map_err(map_backend)?;
+ let registered: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?")
+ .bind(id.as_str()).bind(generation.as_bytes().as_slice()).fetch_optional(&mut *transaction).await.map_err(map_backend)?;
+ if registered.is_some() {
+ return Err(Error::ProjectionRevisionConflict);
+ }
+ put(&mut transaction, &id, generation, &document).await?;
+ transaction.commit().await.map_err(map_backend)
+}
+
+pub(super) async fn load_document(
+ store: &SqliteStorage,
+ id: ProjectionId,
+ generation: ProjectionGeneration,
+ key: String,
+) -> Result<Option<ProjectionDocument>, Error> {
+ radroots_storage::projection::document_query::ProjectionDocumentRecord::corrupt(
+ generation,
+ key.clone(),
+ )
+ .map_err(|_| Error::InvalidProjectionDocument)?;
+ let mut transaction = store.pool().begin().await.map_err(map_backend)?;
+ let metadata = sqlx::query("SELECT length(value) AS value_bytes, CASE WHEN length(value_sha256)=32 THEN value_sha256 END AS value_sha256 FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? AND document_key=?")
+ .bind(id.as_str()).bind(generation.as_bytes().as_slice()).bind(&key)
+ .fetch_optional(&mut *transaction).await.map_err(map_backend)?;
+ let document = if let Some(row) = metadata {
+ let length: i64 = row.try_get("value_bytes").map_err(super::map_corrupt)?;
+ if length <= 0
+ || length > radroots_storage::projection::PROJECTION_DOCUMENT_VALUE_MAX_BYTES as i64
+ {
+ return Err(Error::CorruptProjectionDocument);
+ }
+ let digest: Vec<u8> = row
+ .try_get("value_sha256")
+ .map_err(|_| Error::CorruptProjectionDocument)?;
+ let digest: [u8; 32] = digest
+ .try_into()
+ .map_err(|_| Error::CorruptProjectionDocument)?;
+ let value: Vec<u8> = sqlx::query_scalar("SELECT value FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? AND document_key=?")
+ .bind(id.as_str()).bind(generation.as_bytes().as_slice()).bind(&key)
+ .fetch_one(&mut *transaction).await.map_err(map_backend)?;
+ Some(ProjectionDocument::from_stored_parts(key, value, digest)?)
+ } else {
+ None
+ };
+ transaction.rollback().await.map_err(map_backend)?;
+ Ok(document)
+}
+
+#[cfg(test)]
+#[path = "batch_tests.rs"]
+mod tests;
diff --git a/crates/storage_sqlite/src/projection/batch_tests.rs b/crates/storage_sqlite/src/projection/batch_tests.rs
@@ -0,0 +1,1178 @@
+use super::*;
+use crate::{OpenMode, OpenOptions, Paths};
+use radroots_storage::{
+ EventStore, ProjectionStore,
+ event::SourceGeneration,
+ memory::MemoryStorage,
+ projection::{
+ batch::ProjectionWriteScope,
+ document_query::{ProjectionDocumentGenerations, ProjectionDocumentQuery},
+ },
+};
+use tempfile::TempDir;
+
+fn id() -> ProjectionId {
+ ProjectionId::parse("fixture").unwrap()
+}
+fn generation(byte: u8) -> ProjectionGeneration {
+ ProjectionGeneration::new([byte; 32]).unwrap()
+}
+fn document(key: &str) -> ProjectionDocument {
+ ProjectionDocument::new(key.into(), key.as_bytes().to_vec()).unwrap()
+}
+fn scope(
+ source: EventSourceToken,
+ active: Option<ProjectionGeneration>,
+ target: u8,
+) -> ProjectionWriteScope {
+ ProjectionWriteScope::new(id(), generation(target), source, active).unwrap()
+}
+fn batch(scope: ProjectionWriteScope, keys: &[&str]) -> ProjectionDocumentBatch {
+ ProjectionDocumentBatch::new(scope, keys.iter().map(|key| document(key)).collect()).unwrap()
+}
+fn checkpoint(target: u8, count: u64) -> ProjectionCheckpoint {
+ ProjectionCheckpoint::new(id(), generation(target), None, count, target as u64).unwrap()
+}
+fn query(target: u8) -> ProjectionDocumentQuery {
+ ProjectionDocumentQuery::new(
+ id(),
+ ProjectionDocumentGenerations::Exact(generation(target)),
+ 64,
+ )
+ .unwrap()
+}
+async fn open(temp: &TempDir) -> SqliteStorage {
+ SqliteStorage::open(
+ OpenOptions::new(
+ Paths::from_directory(temp.path()).unwrap(),
+ OpenMode::Create,
+ )
+ .with_source_generation(SourceGeneration::new([9; 32]).unwrap(), 9)
+ .unwrap(),
+ )
+ .await
+ .unwrap()
+}
+
+#[tokio::test]
+async fn projection_atomic_batch_and_promotion_match_memory_and_survive_reopen() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ let source = store.source_token().await.unwrap();
+ for backend in [&store as &dyn ProjectionStore, &memory] {
+ let first = scope(source, None, 1);
+ backend
+ .put_projection_batch(batch(first.clone(), &["a", "b"]))
+ .await
+ .unwrap();
+ assert_eq!(backend.status(id()).await.unwrap(), None);
+ assert_eq!(
+ backend
+ .promote_projection_batch(batch(first.clone(), &["c"]), checkpoint(1, 2))
+ .await,
+ Err(Error::ProjectionCheckpointMismatch)
+ );
+ assert_eq!(
+ backend
+ .query_projection_documents(query(1))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 2
+ );
+ backend
+ .promote_projection_batch(batch(first, &["c"]), checkpoint(1, 3))
+ .await
+ .unwrap();
+ assert_eq!(
+ backend
+ .put_projection_document(id(), generation(1), document("a"))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ let second = scope(source, Some(generation(1)), 2);
+ backend
+ .put_projection_batch(batch(second.clone(), &["x"]))
+ .await
+ .unwrap();
+ assert_eq!(
+ backend
+ .put_projection_batch(batch(scope(source, Some(generation(1)), 3), &[]))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ backend
+ .query_projection_documents_at(query(1), source, generation(1))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 3
+ );
+ }
+ store.close().await.unwrap();
+ let store = open(&temp).await;
+ assert_eq!(
+ store
+ .query_projection_documents_at(query(1), source, generation(1))
+ .await
+ .unwrap(),
+ memory
+ .query_projection_documents_at(query(1), source, generation(1))
+ .await
+ .unwrap()
+ );
+ for backend in [&store as &dyn ProjectionStore, &memory] {
+ backend
+ .promote_projection_batch(
+ batch(scope(source, Some(generation(1)), 2), &["y"]),
+ checkpoint(2, 2),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ backend
+ .query_projection_documents_at(query(1), source, generation(1))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ backend
+ .query_projection_documents_at(query(2), source, generation(2))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 2
+ );
+ }
+ assert_eq!(
+ store
+ .query_projection_documents_at(query(2), source, generation(2))
+ .await
+ .unwrap(),
+ memory
+ .query_projection_documents_at(query(2), source, generation(2))
+ .await
+ .unwrap()
+ );
+ store.close().await.unwrap();
+ let readonly = SqliteStorage::open(OpenOptions::new(
+ Paths::from_directory(temp.path()).unwrap(),
+ OpenMode::ReadOnly,
+ ))
+ .await
+ .unwrap();
+ assert_eq!(
+ readonly
+ .query_projection_documents_at(query(2), source, generation(2))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 2
+ );
+ assert_eq!(
+ readonly
+ .put_projection_batch(batch(scope(source, Some(generation(2)), 3), &["z"]))
+ .await,
+ Err(Error::BackendUnavailable)
+ );
+ readonly.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_final_batch_fault_rolls_back_every_row_fence_and_status() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let source = store.source_token().await.unwrap();
+ store
+ .promote_projection_batch(batch(scope(source, None, 1), &["old"]), checkpoint(1, 1))
+ .await
+ .unwrap();
+ sqlx::raw_sql("CREATE TRIGGER fixture_projection_abort BEFORE INSERT ON radroots_runtime_projection_documents WHEN NEW.document_key = 'fail' BEGIN SELECT RAISE(ABORT,'fixture'); END;")
+ .execute(store.pool()).await.unwrap();
+ let next = scope(source, Some(generation(1)), 2);
+ assert!(
+ store
+ .promote_projection_batch(batch(next.clone(), &["new", "fail"]), checkpoint(2, 2))
+ .await
+ .is_err()
+ );
+ assert!(
+ store
+ .query_projection_documents(query(2))
+ .await
+ .unwrap()
+ .records()
+ .is_empty()
+ );
+ assert_eq!(
+ ProjectionStore::status(&store, id())
+ .await
+ .unwrap()
+ .unwrap()
+ .generation(),
+ generation(1)
+ );
+ let count: i64 = sqlx::query_scalar(
+ "SELECT count(*) FROM radroots_runtime_projection_generations WHERE phase='staging'",
+ )
+ .fetch_one(store.pool())
+ .await
+ .unwrap();
+ assert_eq!(count, 0);
+ sqlx::raw_sql("DROP TRIGGER fixture_projection_abort")
+ .execute(store.pool())
+ .await
+ .unwrap();
+ store
+ .promote_projection_batch(batch(next, &["new", "fail"]), checkpoint(2, 2))
+ .await
+ .unwrap();
+ assert_eq!(
+ store
+ .query_projection_documents_at(query(2), source, generation(2))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 2
+ );
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_stale_source_epoch_and_cancelled_wait_cannot_publish() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let source = store.source_token().await.unwrap();
+ let mut all = Vec::new();
+ for _ in 0..4 {
+ all.push(store.pool().acquire().await.unwrap());
+ }
+ let next = scope(source, None, 1);
+ let mut future = store.promote_projection_batch(batch(next.clone(), &["a"]), checkpoint(1, 1));
+ assert!(matches!(
+ future
+ .as_mut()
+ .poll(&mut std::task::Context::from_waker(std::task::Waker::noop())),
+ std::task::Poll::Pending
+ ));
+ drop(future);
+ drop(all);
+ assert_eq!(ProjectionStore::status(&store, id()).await.unwrap(), None);
+ assert!(
+ store
+ .query_projection_documents(query(1))
+ .await
+ .unwrap()
+ .records()
+ .is_empty()
+ );
+ store
+ .put_projection_batch(batch(next.clone(), &["a"]))
+ .await
+ .unwrap();
+ sqlx::query("UPDATE radroots_runtime_source_generations SET mutation_epoch = mutation_epoch + 1 WHERE generation = ?")
+ .bind(source.generation().as_bytes().as_slice()).execute(store.pool()).await.unwrap();
+ assert_eq!(
+ store
+ .promote_projection_batch(batch(next, &["b"]), checkpoint(1, 2))
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ let changed = store.source_token().await.unwrap();
+ assert_eq!(
+ store
+ .put_projection_batch(batch(scope(changed, None, 1), &["b"]))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ store
+ .query_projection_documents(query(1))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ assert_eq!(ProjectionStore::status(&store, id()).await.unwrap(), None);
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_wal_readers_observe_complete_old_new_or_typed_stale() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let source = store.source_token().await.unwrap();
+ store
+ .promote_projection_batch(
+ batch(scope(source, None, 1), &["old.a", "old.b"]),
+ checkpoint(1, 2),
+ )
+ .await
+ .unwrap();
+ for next in 2..18u8 {
+ let old = next - 1;
+ let (read, promoted) = tokio::join!(
+ store.query_projection_documents_at(query(old), source, generation(old)),
+ store.promote_projection_batch(
+ batch(
+ scope(source, Some(generation(old)), next),
+ &["new.a", "new.b"]
+ ),
+ checkpoint(next, 2)
+ )
+ );
+ promoted.unwrap();
+ match read {
+ Ok(page) => assert_eq!(page.records().len(), 2),
+ Err(Error::ProjectionRevisionConflict) => {}
+ other => panic!("unexpected read {other:?}"),
+ }
+ assert_eq!(
+ store
+ .query_projection_documents_at(query(next), source, generation(next))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 2
+ );
+ }
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_real_same_count_admission_change_invalidates_durable_staging() {
+ use radroots_event::{SignedEvent, admission::RawEvent, wire::Nip01EventWire};
+ use radroots_storage::{Storage, event::EventAdmission};
+ use radroots_transport::{
+ Target, TransportId,
+ source::{EventProvenance, ObservedEvent},
+ };
+ struct FixtureVerifier;
+ impl radroots_event::admission::SignatureVerifier for FixtureVerifier {
+ fn verify_signature(
+ &self,
+ _event: &radroots_event::Event,
+ ) -> Result<(), radroots_event::Error> {
+ Ok(())
+ }
+ }
+ let mut wire = Nip01EventWire {
+ id: "0".repeat(64),
+ pubkey: "585591529da0bab31b3b1b1f986611cf5f435dca84f978c89ee8a40cca7103df".into(),
+ created_at: 1,
+ kind: 0,
+ tags: vec![],
+ content: r#"{"name":"fixture"}"#.into(),
+ sig: "42".repeat(64),
+ extra: Default::default(),
+ };
+ wire.id = wire.computed_event_id().unwrap().to_hex();
+ let raw = serde_json::to_string(&wire).unwrap();
+ let event = SignedEvent::from_wire_verified_id(wire, raw).unwrap();
+ let target = Target::new(TransportId::NOSTR, "wss://fixture.example").unwrap();
+ let observed = ObservedEvent::new(
+ event.clone(),
+ EventProvenance::new(TransportId::NOSTR, target.fingerprint().clone(), 1).unwrap(),
+ );
+ let verified = RawEvent::new(event.envelope().clone())
+ .verify_id()
+ .unwrap()
+ .verify_signature(&FixtureVerifier)
+ .unwrap();
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ for backend in [&store as &dyn Storage, &memory] {
+ let empty = backend.source_token().await.unwrap();
+ let receipt = backend
+ .admit(EventAdmission::raw(observed.clone()))
+ .await
+ .unwrap();
+ assert_eq!(
+ backend
+ .put_projection_batch(batch(scope(empty, None, 1), &["a"]))
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ let source = backend.source_token().await.unwrap();
+ let next = scope(source, None, 1);
+ backend
+ .put_projection_batch(batch(next.clone(), &["a"]))
+ .await
+ .unwrap();
+ backend
+ .admit(EventAdmission::raw(observed.clone()))
+ .await
+ .unwrap();
+ assert_eq!(backend.source_token().await.unwrap(), source);
+ backend
+ .admit(EventAdmission::verified(observed.clone(), verified.clone()).unwrap())
+ .await
+ .unwrap();
+ let changed = backend.source_token().await.unwrap();
+ assert_eq!(changed.epoch(), source.epoch() + 1);
+ let checkpoint =
+ ProjectionCheckpoint::new(id(), generation(1), Some(receipt.position()), 2, 1).unwrap();
+ assert_eq!(
+ backend
+ .promote_projection_batch(batch(next, &["b"]), checkpoint)
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ assert_eq!(ProjectionStore::status(backend, id()).await.unwrap(), None);
+ assert_eq!(
+ backend
+ .query_projection_documents(query(1))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ }
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_max_batch_updates_and_exact_lookup_validate_before_hydration() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let source = store.source_token().await.unwrap();
+ let next = scope(source, None, 1);
+ let documents: Vec<_> = (0..64).map(|i| document(&format!("k{i:02}"))).collect();
+ store
+ .put_projection_batch(ProjectionDocumentBatch::new(next.clone(), documents).unwrap())
+ .await
+ .unwrap();
+ store
+ .promote_projection_batch(batch(next, &["k00"]), checkpoint(1, 64))
+ .await
+ .unwrap();
+ assert_eq!(
+ store
+ .query_projection_documents_at(query(1), source, generation(1))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 64
+ );
+ assert_eq!(
+ store
+ .projection_document(id(), generation(1), "k00".into())
+ .await
+ .unwrap()
+ .unwrap()
+ .value(),
+ b"k00"
+ );
+ assert_eq!(
+ store
+ .projection_document(id(), generation(1), "missing".into())
+ .await
+ .unwrap(),
+ None
+ );
+ assert_eq!(
+ store
+ .projection_document(id(), generation(1), "bad\nkey".into())
+ .await,
+ Err(Error::InvalidProjectionDocument)
+ );
+ sqlx::query("UPDATE radroots_runtime_projection_documents SET value_sha256 = zeroblob(32) WHERE document_key='k00'").execute(store.pool()).await.unwrap();
+ assert_eq!(
+ store
+ .projection_document(id(), generation(1), "k00".into())
+ .await,
+ Err(Error::CorruptProjectionDocument)
+ );
+ let plan=sqlx::query("EXPLAIN QUERY PLAN SELECT document_key FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? AND document_key>? ORDER BY document_key LIMIT 65")
+ .bind(id().as_str()).bind(generation(1).as_bytes().as_slice()).bind("k00").fetch_all(store.pool()).await.unwrap();
+ let details: Vec<String> = plan.iter().map(|row| row.get("detail")).collect();
+ assert!(
+ details
+ .iter()
+ .any(|line| line.contains("PRIMARY KEY") && line.contains("document_key>"))
+ );
+ assert!(!details.iter().any(|line| line.contains("TEMP B-TREE")));
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_foreign_sources_legacy_rows_and_invalidated_headers_refuse_without_writes() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let source = store.source_token().await.unwrap();
+ let foreign = EventSourceToken::new(SourceGeneration::new([8; 32]).unwrap(), 0);
+ assert_eq!(
+ store
+ .put_projection_batch(batch(scope(foreign, None, 1), &["foreign"]))
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ store
+ .put_projection_document(id(), generation(1), document("legacy"))
+ .await
+ .unwrap();
+ assert_eq!(
+ store
+ .put_projection_batch(batch(scope(source, None, 1), &[]))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ store
+ .promote_projection_batch(batch(scope(source, None, 2), &["active"]), checkpoint(2, 1))
+ .await
+ .unwrap();
+ assert_eq!(
+ store
+ .query_projection_documents_at(query(1), source, generation(2))
+ .await,
+ Err(Error::InvalidProjectionDocument)
+ );
+ store
+ .invalidate(
+ radroots_storage::projection::ProjectionInvalidation::new(
+ id(),
+ generation(2),
+ generation(3),
+ radroots_storage::projection::InvalidationReason::OperatorRequested,
+ 3,
+ )
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ store
+ .put_projection_batch(batch(scope(source, Some(generation(2)), 3), &["refused"]))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ store
+ .query_projection_documents_at(query(2), source, generation(2))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert!(
+ store
+ .query_projection_documents(query(3))
+ .await
+ .unwrap()
+ .records()
+ .is_empty()
+ );
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_retired_or_corrupt_staging_fences_cannot_be_rebound() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let source = store.source_token().await.unwrap();
+ let next = scope(source, None, 1);
+ store
+ .put_projection_batch(batch(next.clone(), &["retained"]))
+ .await
+ .unwrap();
+ for statement in [
+ "UPDATE radroots_runtime_projection_generations SET source_generation=zeroblob(32)",
+ "UPDATE radroots_runtime_projection_generations SET source_epoch=1",
+ "UPDATE radroots_runtime_projection_generations SET expected_active=zeroblob(32)",
+ "UPDATE radroots_runtime_projection_generations SET phase='retired'",
+ ] {
+ sqlx::query(statement).execute(store.pool()).await.unwrap();
+ assert_eq!(
+ store
+ .put_projection_batch(batch(next.clone(), &["refused"]))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ store
+ .query_projection_documents(query(1))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ sqlx::query("UPDATE radroots_runtime_projection_generations SET source_generation=?,source_epoch=0,expected_active=NULL,phase='staging'")
+ .bind(source.generation().as_bytes().as_slice()).execute(store.pool()).await.unwrap();
+ }
+ store
+ .promote_projection_batch(batch(next, &[]), checkpoint(1, 1))
+ .await
+ .unwrap();
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_nonempty_checkpoints_bind_exact_identity_generation_and_source_position() {
+ use radroots_event::{SignedEvent, wire::Nip01EventWire};
+ use radroots_storage::event::{EventAdmission, EventPosition, EventSequence};
+ use radroots_transport::{
+ Target, TransportId,
+ source::{EventProvenance, ObservedEvent},
+ };
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let mut wire = Nip01EventWire {
+ id: "0".repeat(64),
+ pubkey: "585591529da0bab31b3b1b1f986611cf5f435dca84f978c89ee8a40cca7103df".into(),
+ created_at: 1,
+ kind: 0,
+ tags: vec![],
+ content: r#"{"name":"fixture"}"#.into(),
+ sig: "42".repeat(64),
+ extra: Default::default(),
+ };
+ wire.id = wire.computed_event_id().unwrap().to_hex();
+ let raw = serde_json::to_string(&wire).unwrap();
+ let event = SignedEvent::from_wire_verified_id(wire, raw).unwrap();
+ let target = Target::new(TransportId::NOSTR, "wss://fixture.example").unwrap();
+ let observed = ObservedEvent::new(
+ event,
+ EventProvenance::new(TransportId::NOSTR, target.fingerprint().clone(), 1).unwrap(),
+ );
+ let receipt = store.admit(EventAdmission::raw(observed)).await.unwrap();
+ let source = store.source_token().await.unwrap();
+ let next = scope(source, None, 1);
+ store
+ .put_projection_batch(batch(next.clone(), &["a"]))
+ .await
+ .unwrap();
+ for checkpoint in [
+ ProjectionCheckpoint::new(
+ ProjectionId::parse("foreign").unwrap(),
+ generation(1),
+ Some(receipt.position()),
+ 2,
+ 1,
+ )
+ .unwrap(),
+ ProjectionCheckpoint::new(id(), generation(2), Some(receipt.position()), 2, 1).unwrap(),
+ ProjectionCheckpoint::new(id(), generation(1), None, 2, 1).unwrap(),
+ ProjectionCheckpoint::new(
+ id(),
+ generation(1),
+ Some(EventPosition::new(
+ SourceGeneration::new([8; 32]).unwrap(),
+ EventSequence::new(1).unwrap(),
+ )),
+ 2,
+ 1,
+ )
+ .unwrap(),
+ ProjectionCheckpoint::new(
+ id(),
+ generation(1),
+ Some(EventPosition::new(
+ source.generation(),
+ EventSequence::new(2).unwrap(),
+ )),
+ 2,
+ 1,
+ )
+ .unwrap(),
+ ] {
+ assert_eq!(
+ store
+ .promote_projection_batch(batch(next.clone(), &["b"]), checkpoint)
+ .await,
+ Err(Error::ProjectionCheckpointMismatch)
+ );
+ assert_eq!(
+ store
+ .query_projection_documents(query(1))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ assert_eq!(ProjectionStore::status(&store, id()).await.unwrap(), None);
+ }
+ store
+ .promote_projection_batch(
+ batch(next, &["b"]),
+ ProjectionCheckpoint::new(id(), generation(1), Some(receipt.position()), 2, 1).unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ store
+ .query_projection_documents_at(query(1), source, generation(1))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 2
+ );
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_exact_lookup_refuses_zero_and_oversized_stored_values_before_hydration() {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let source = store.source_token().await.unwrap();
+ store
+ .put_projection_document(id(), generation(1), document("retained"))
+ .await
+ .unwrap();
+ for size in [0, 16_777_217] {
+ let mut connection = store.pool().acquire().await.unwrap();
+ sqlx::raw_sql("PRAGMA ignore_check_constraints=ON")
+ .execute(&mut *connection)
+ .await
+ .unwrap();
+ sqlx::query("UPDATE radroots_runtime_projection_documents SET value=zeroblob(?) WHERE document_key='retained'")
+ .bind(size).execute(&mut *connection).await.unwrap();
+ sqlx::raw_sql("PRAGMA ignore_check_constraints=OFF")
+ .execute(&mut *connection)
+ .await
+ .unwrap();
+ drop(connection);
+ assert_eq!(
+ store
+ .projection_document(id(), generation(1), "retained".into())
+ .await,
+ Err(Error::CorruptProjectionDocument)
+ );
+ assert_eq!(store.source_token().await.unwrap(), source);
+ }
+ store.close().await.unwrap();
+}
+
+fn review_admission(verified: bool) -> radroots_storage::event::EventAdmission {
+ use radroots_event::{SignedEvent, admission::RawEvent, wire::Nip01EventWire};
+ use radroots_storage::event::EventAdmission;
+ use radroots_transport::{
+ Target, TransportId,
+ source::{EventProvenance, ObservedEvent},
+ };
+ struct FixtureVerifier;
+ impl radroots_event::admission::SignatureVerifier for FixtureVerifier {
+ fn verify_signature(
+ &self,
+ _event: &radroots_event::Event,
+ ) -> Result<(), radroots_event::Error> {
+ Ok(())
+ }
+ }
+ let mut wire = Nip01EventWire {
+ id: "0".repeat(64),
+ pubkey: "585591529da0bab31b3b1b1f986611cf5f435dca84f978c89ee8a40cca7103df".into(),
+ created_at: 17,
+ kind: 0,
+ tags: vec![],
+ content: r#"{"name":"review"}"#.into(),
+ sig: "42".repeat(64),
+ extra: Default::default(),
+ };
+ wire.id = wire.computed_event_id().unwrap().to_hex();
+ let raw = serde_json::to_string(&wire).unwrap();
+ let event = SignedEvent::from_wire_verified_id(wire, raw).unwrap();
+ let target = Target::new(TransportId::NOSTR, "wss://fixture.example").unwrap();
+ let observed = ObservedEvent::new(
+ event.clone(),
+ EventProvenance::new(TransportId::NOSTR, target.fingerprint().clone(), 1).unwrap(),
+ );
+ if verified {
+ let verified = RawEvent::new(event.envelope().clone())
+ .verify_id()
+ .unwrap()
+ .verify_signature(&FixtureVerifier)
+ .unwrap();
+ EventAdmission::verified(observed, verified).unwrap()
+ } else {
+ EventAdmission::raw(observed)
+ }
+}
+
+#[tokio::test]
+async fn projection_promoted_source_binding_survives_real_insert_and_same_count_change() {
+ use radroots_storage::Storage;
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ for backend in [&memory as &dyn Storage, &store] {
+ let empty = backend.source_token().await.unwrap();
+ backend
+ .promote_projection_batch(batch(scope(empty, None, 1), &["empty"]), checkpoint(1, 1))
+ .await
+ .unwrap();
+ let receipt = backend.admit(review_admission(false)).await.unwrap();
+ let raw = backend.source_token().await.unwrap();
+ assert_eq!(
+ backend
+ .query_projection_documents_at(query(1), empty, generation(1))
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ assert_eq!(
+ backend
+ .query_projection_documents_at(query(1), raw, generation(1))
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ backend
+ .promote_projection_batch(
+ batch(scope(raw, Some(generation(1)), 2), &["raw"]),
+ ProjectionCheckpoint::new(id(), generation(2), Some(receipt.position()), 1, 2)
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ backend.admit(review_admission(true)).await.unwrap();
+ let verified = backend.source_token().await.unwrap();
+ assert_eq!(verified.epoch(), raw.epoch() + 1);
+ assert_eq!(
+ backend
+ .query_projection_documents_at(query(2), raw, generation(2))
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ assert_eq!(
+ backend
+ .query_projection_documents_at(query(2), verified, generation(2))
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ assert_eq!(
+ backend
+ .query_projection_documents(query(2))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ backend
+ .promote_projection_batch(
+ batch(scope(verified, Some(generation(2)), 3), &["verified"]),
+ ProjectionCheckpoint::new(id(), generation(3), Some(receipt.position()), 1, 3)
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ backend
+ .query_projection_documents_at(query(3), verified, generation(3))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ }
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_registered_checkpoint_and_atomic_admission_cannot_bypass_promotion() {
+ use radroots_storage::{
+ Storage,
+ atomic::{
+ AtomicCommit, AtomicCommitDigest, AtomicCommitId, AtomicWorkflow, CommitIngested,
+ },
+ };
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ for backend in [&memory as &dyn Storage, &store] {
+ let source = backend.source_token().await.unwrap();
+ let staged = scope(source, None, 1);
+ backend
+ .put_projection_batch(batch(staged.clone(), &["partial"]))
+ .await
+ .unwrap();
+ assert_eq!(
+ backend.checkpoint(checkpoint(1, 999)).await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(ProjectionStore::status(backend, id()).await.unwrap(), None);
+ let request = AtomicCommit::new(
+ AtomicCommitId::new([1; 16]).unwrap(),
+ AtomicCommitDigest::new([1; 32]),
+ 1,
+ AtomicWorkflow::Ingested(Box::new(CommitIngested::new(
+ review_admission(false),
+ Some(checkpoint(1, 999)),
+ ))),
+ )
+ .unwrap();
+ assert_eq!(
+ backend.commit(request.clone()).await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(backend.receipt(request.commit_id()).await.unwrap(), None);
+ assert_eq!(backend.source_token().await.unwrap(), source);
+ assert_eq!(ProjectionStore::status(backend, id()).await.unwrap(), None);
+ backend
+ .promote_projection_batch(batch(staged, &[]), checkpoint(1, 1))
+ .await
+ .unwrap();
+ let prior = ProjectionStore::status(backend, id()).await.unwrap();
+ assert_eq!(
+ backend.checkpoint(checkpoint(1, 999)).await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(ProjectionStore::status(backend, id()).await.unwrap(), prior);
+ assert_eq!(
+ backend
+ .query_projection_documents_at(query(1), source, generation(1))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ backend
+ .promote_projection_batch(
+ batch(scope(source, Some(generation(1)), 2), &["new"]),
+ checkpoint(2, 1),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ backend.checkpoint(checkpoint(1, 999)).await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ let legacy = ProjectionId::parse("legacy").unwrap();
+ backend
+ .put_projection_document(legacy.clone(), generation(9), document("legacy"))
+ .await
+ .unwrap();
+ backend
+ .checkpoint(
+ ProjectionCheckpoint::new(legacy.clone(), generation(9), None, 7, 9).unwrap(),
+ )
+ .await
+ .unwrap();
+ let legacy_query = ProjectionDocumentQuery::new(
+ legacy,
+ ProjectionDocumentGenerations::Exact(generation(9)),
+ 1,
+ )
+ .unwrap();
+ assert_eq!(
+ backend
+ .query_projection_documents_at(legacy_query.clone(), source, generation(9))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(
+ backend
+ .query_projection_documents(legacy_query)
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ }
+ store.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn projection_legacy_rebuild_cannot_publish_registered_staging_or_retired_generation() {
+ use radroots_storage::projection::{
+ InvalidationReason, ProjectionInvalidation, RawSourceDigest, RebuildTicket,
+ RebuildTicketId, RebuildTransition,
+ };
+ for retired in [false, true] {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
+ for backend in [&memory as &dyn ProjectionStore, &store] {
+ let source = store.source_token().await.unwrap();
+ backend
+ .promote_projection_batch(
+ batch(scope(source, None, 1), &["first"]),
+ checkpoint(1, 1),
+ )
+ .await
+ .unwrap();
+ let second = batch(scope(source, Some(generation(1)), 2), &["second"]);
+ let (active, target) = if retired {
+ backend
+ .promote_projection_batch(second, checkpoint(2, 1))
+ .await
+ .unwrap();
+ (2, 1)
+ } else {
+ backend.put_projection_batch(second).await.unwrap();
+ (1, 2)
+ };
+ let invalidation = ProjectionInvalidation::new(
+ id(),
+ generation(active),
+ generation(target),
+ InvalidationReason::OperatorRequested,
+ 10,
+ )
+ .unwrap();
+ backend.invalidate(invalidation.clone()).await.unwrap();
+ let ticket = RebuildTicket::requested(
+ RebuildTicketId::new([target; 16]).unwrap(),
+ invalidation,
+ source.generation(),
+ None,
+ RawSourceDigest::new([1; 32]),
+ )
+ .unwrap();
+ backend.request_rebuild(ticket.clone()).await.unwrap();
+ let running = backend
+ .transition_rebuild(RebuildTransition::start(
+ ticket.ticket_id(),
+ ticket.revision(),
+ 11,
+ ))
+ .await
+ .unwrap();
+ let status = backend.status(id()).await.unwrap();
+ assert_eq!(
+ backend
+ .transition_rebuild(RebuildTransition::complete(
+ running.ticket_id(),
+ running.revision(),
+ 12,
+ ProjectionCheckpoint::new(id(), generation(target), None, 999, 12).unwrap()
+ ))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ assert_eq!(backend.status(id()).await.unwrap(), status);
+ assert_eq!(
+ backend.rebuild(ticket.ticket_id()).await.unwrap(),
+ Some(running)
+ );
+ assert_eq!(
+ backend
+ .query_projection_documents(query(target))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ assert_eq!(
+ backend
+ .query_projection_documents_at(query(target), source, generation(target))
+ .await,
+ Err(Error::ProjectionRevisionConflict)
+ );
+ }
+ store.close().await.unwrap();
+ }
+}
+
+#[tokio::test]
+async fn projection_source_mutation_wal_race_reopen_and_read_only_never_relabel_old_rows() {
+ for _ in 0..8 {
+ let temp = TempDir::new().unwrap();
+ let store = open(&temp).await;
+ let old = store.source_token().await.unwrap();
+ store
+ .promote_projection_batch(batch(scope(old, None, 1), &["old"]), checkpoint(1, 1))
+ .await
+ .unwrap();
+ let (mutation, read) = tokio::join!(
+ store.admit(review_admission(false)),
+ store.query_projection_documents_at(query(1), old, generation(1))
+ );
+ let receipt = mutation.unwrap();
+ match read {
+ Ok(page) => assert_eq!(page.records().len(), 1),
+ Err(error) => assert_eq!(error, Error::SourceEpochChanged),
+ }
+ let current = store.source_token().await.unwrap();
+ assert_ne!(current, old);
+ for token in [old, current] {
+ assert_eq!(
+ store
+ .query_projection_documents_at(query(1), token, generation(1))
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ }
+ store.close().await.unwrap();
+ let readonly = SqliteStorage::open(OpenOptions::new(
+ Paths::from_directory(temp.path()).unwrap(),
+ OpenMode::ReadOnly,
+ ))
+ .await
+ .unwrap();
+ for token in [old, current] {
+ assert_eq!(
+ readonly
+ .query_projection_documents_at(query(1), token, generation(1))
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ }
+ assert_eq!(
+ readonly
+ .query_projection_documents(query(1))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ readonly.close().await.unwrap();
+ let reopened = open(&temp).await;
+ assert_eq!(
+ reopened
+ .query_projection_documents_at(query(1), current, generation(1))
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+ reopened
+ .promote_projection_batch(
+ batch(scope(current, Some(generation(1)), 2), &["new"]),
+ ProjectionCheckpoint::new(id(), generation(2), Some(receipt.position()), 1, 2)
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ reopened
+ .query_projection_documents_at(query(2), current, generation(2))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ reopened.close().await.unwrap();
+ let readonly = SqliteStorage::open(OpenOptions::new(
+ Paths::from_directory(temp.path()).unwrap(),
+ OpenMode::ReadOnly,
+ ))
+ .await
+ .unwrap();
+ assert_eq!(
+ readonly
+ .query_projection_documents_at(query(2), current, generation(2))
+ .await
+ .unwrap()
+ .records()
+ .len(),
+ 1
+ );
+ readonly.close().await.unwrap();
+ }
+}
diff --git a/crates/storage_sqlite/src/projection/document_query.rs b/crates/storage_sqlite/src/projection/document_query.rs
@@ -27,33 +27,71 @@ pub(super) async fn page(
}
}
+pub(super) async fn page_at(
+ store: &SqliteStorage,
+ query: ProjectionDocumentQuery,
+ source: radroots_storage::event::EventSourceToken,
+ active: ProjectionGeneration,
+) -> Result<ProjectionDocumentPage, Error> {
+ if query.generations() != ProjectionDocumentGenerations::Exact(active) {
+ return Err(Error::InvalidProjectionDocument);
+ }
+ let mut transaction = store.pool().begin().await.map_err(map_backend)?;
+ super::batch::read_fence(
+ &mut transaction,
+ store,
+ source,
+ query.projection_id(),
+ active,
+ )
+ .await?;
+ let result = read_page(&mut transaction, &query).await;
+ let rollback = transaction.rollback().await.map_err(map_backend);
+ match result {
+ Ok(page) => {
+ rollback?;
+ Ok(page)
+ }
+ Err(error) => Err(error),
+ }
+}
+
async fn read_page(
transaction: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
query: &ProjectionDocumentQuery,
) -> Result<ProjectionDocumentPage, Error> {
- let generation = match query.generations() {
- ProjectionDocumentGenerations::All => None,
- ProjectionDocumentGenerations::Exact(value) => Some(value.as_bytes().to_vec()),
- };
- let after_generation = query
- .after()
- .map(|(generation, _)| generation.as_bytes().to_vec());
- let after_key = query.after().map(|(_, key)| key);
// CASE bounds every variable-width metadata value before SQLx allocates it.
// Payloads are loaded separately, only after charging this page's byte budget.
- let rows = sqlx::query(
+ let mut selection = sqlx::QueryBuilder::new(
"SELECT CASE WHEN length(generation) = 32 THEN generation END AS generation,
CASE WHEN length(CAST(document_key AS BLOB)) BETWEEN 1 AND 512 THEN document_key END AS document_key,
CASE WHEN length(value_sha256) = 32 THEN value_sha256 END AS value_sha256,
length(value) AS value_bytes
FROM radroots_runtime_projection_documents AS documents
- WHERE projection_id = ? AND (? IS NULL OR generation = ?)
- AND (? IS NULL OR (generation, document_key) > (?, ?))
- ORDER BY documents.generation, documents.document_key LIMIT ?"
- ).bind(query.projection_id().as_str()).bind(&generation).bind(generation)
- .bind(&after_generation).bind(&after_generation).bind(after_key)
- .bind(i64::from(query.limit()) + 1)
- .fetch_all(&mut **transaction).await.map_err(map_backend)?;
+ WHERE projection_id = "
+ );
+ selection.push_bind(query.projection_id().as_str());
+ if let ProjectionDocumentGenerations::Exact(generation) = query.generations() {
+ selection
+ .push(" AND generation = ")
+ .push_bind(generation.as_bytes().to_vec());
+ }
+ if let Some((generation, key)) = query.after() {
+ selection
+ .push(" AND (generation, document_key) > (")
+ .push_bind(generation.as_bytes().to_vec())
+ .push(", ")
+ .push_bind(key)
+ .push(")");
+ }
+ selection
+ .push(" ORDER BY documents.generation, documents.document_key LIMIT ")
+ .push_bind(i64::from(query.limit()) + 1);
+ let rows = selection
+ .build()
+ .fetch_all(&mut **transaction)
+ .await
+ .map_err(map_backend)?;
let mut has_more = rows.len() > usize::from(query.limit());
let mut records = Vec::new();
let mut bytes = 0;
diff --git a/crates/storage_sqlite/src/projection/mod.rs b/crates/storage_sqlite/src/projection/mod.rs
@@ -14,6 +14,7 @@ use radroots_storage::{
};
use sqlx::{Row, Sqlite, SqliteConnection};
+mod batch;
mod document_query;
#[cfg_attr(coverage_nightly, coverage(off))]
@@ -257,6 +258,12 @@ impl ProjectionStore for SqliteStorage {
}
let next = current.transition(transition)?;
if next.stage() == RebuildStage::Completed {
+ batch::refuse_registered_legacy(
+ &mut transaction,
+ next.invalidation().projection_id(),
+ next.invalidation().replacement_generation(),
+ )
+ .await?;
let source = sqlx::query(
"SELECT generation, sequence_head
FROM radroots_runtime_source_generations WHERE state = 'active'",
@@ -451,32 +458,41 @@ impl ProjectionStore for SqliteStorage {
})
}
+ fn put_projection_batch(
+ &self,
+ value: radroots_storage::projection::batch::ProjectionDocumentBatch,
+ ) -> BoxFuture<'_, Result<(), Error>> {
+ Box::pin(async move { batch::write(self, value, None).await.map(|_| ()) })
+ }
+ fn promote_projection_batch(
+ &self,
+ value: radroots_storage::projection::batch::ProjectionDocumentBatch,
+ checkpoint: ProjectionCheckpoint,
+ ) -> BoxFuture<'_, Result<ProjectionStatus, Error>> {
+ Box::pin(async move {
+ batch::write(self, value, Some(checkpoint))
+ .await?
+ .ok_or(Error::CorruptProjectionRecord)
+ })
+ }
+ fn query_projection_documents_at(
+ &self,
+ query: radroots_storage::projection::document_query::ProjectionDocumentQuery,
+ source: radroots_storage::event::EventSourceToken,
+ active: ProjectionGeneration,
+ ) -> BoxFuture<
+ '_,
+ Result<radroots_storage::projection::document_query::ProjectionDocumentPage, Error>,
+ > {
+ Box::pin(document_query::page_at(self, query, source, active))
+ }
fn put_projection_document(
&self,
projection_id: ProjectionId,
generation: ProjectionGeneration,
document: ProjectionDocument,
) -> BoxFuture<'_, Result<(), Error>> {
- Box::pin(async move {
- self.require_projection_writer()?;
- sqlx::query(
- "INSERT INTO radroots_runtime_projection_documents (
- projection_id, generation, document_key, value, value_sha256
- ) VALUES (?, ?, ?, ?, ?)
- ON CONFLICT(projection_id, generation, document_key) DO UPDATE SET
- value = excluded.value,
- value_sha256 = excluded.value_sha256",
- )
- .bind(projection_id.as_str())
- .bind(generation.as_bytes().as_slice())
- .bind(document.key())
- .bind(document.value())
- .bind(document.value_sha256().as_slice())
- .execute(self.pool())
- .await
- .map_err(map_backend)?;
- Ok(())
- })
+ Box::pin(batch::legacy_put(self, projection_id, generation, document))
}
fn projection_document(
@@ -485,27 +501,7 @@ impl ProjectionStore for SqliteStorage {
generation: ProjectionGeneration,
key: String,
) -> BoxFuture<'_, Result<Option<ProjectionDocument>, Error>> {
- Box::pin(async move {
- sqlx::query(
- "SELECT document_key, value, value_sha256
- FROM radroots_runtime_projection_documents
- WHERE projection_id = ? AND generation = ? AND document_key = ?",
- )
- .bind(projection_id.as_str())
- .bind(generation.as_bytes().as_slice())
- .bind(key)
- .fetch_optional(self.pool())
- .await
- .map_err(map_backend)?
- .map(|row| {
- ProjectionDocument::from_stored_parts(
- row.try_get("document_key").map_err(map_corrupt)?,
- row.try_get("value").map_err(map_corrupt)?,
- array(row.try_get("value_sha256").map_err(map_corrupt)?)?,
- )
- })
- .transpose()
- })
+ Box::pin(batch::load_document(self, projection_id, generation, key))
}
fn query_projection_documents(
@@ -598,6 +594,12 @@ pub(crate) async fn checkpoint_transaction(
transaction: &mut sqlx::Transaction<'_, Sqlite>,
checkpoint: ProjectionCheckpoint,
) -> Result<ProjectionStatus, Error> {
+ batch::refuse_registered_legacy(
+ transaction,
+ checkpoint.projection_id(),
+ checkpoint.generation(),
+ )
+ .await?;
let prior = sqlx::query(
"SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?",
)