commit 68c5b835474dca92f1647f9449947b60efd43a4e
parent 690b6b17f2040a879fb33ba643c6a4d7e44ebe76
Author: triesap <tyson@radroots.org>
Date: Sat, 10 Oct 2026 04:56:10 +0000
storage: provide bounded indexed visibility reads
- Maintain visible membership and cached counts in canonical admission transactions.
- Fence indexed sequence and event-id pages to the current source epoch.
- Preserve resumable deletion work and historical migration integrity.
- Verify workspace checks, 3946 test executions, coverage and independent acceptance.
Diffstat:
20 files changed, 1828 insertions(+), 140 deletions(-)
diff --git a/contracts/api_baselines/radroots_storage.txt b/contracts/api_baselines/radroots_storage.txt
@@ -856,6 +856,7 @@ pub struct radroots_storage::event::VisibilityIndexProgress
impl radroots_storage::event::VisibilityIndexProgress
pub const fn radroots_storage::event::VisibilityIndexProgress::charged_bytes(self) -> u64
pub const fn radroots_storage::event::VisibilityIndexProgress::is_complete(self) -> bool
+pub const fn radroots_storage::event::VisibilityIndexProgress::processed_keys(self) -> u16
pub const fn radroots_storage::event::VisibilityIndexProgress::processed_rows(self) -> u16
pub const fn radroots_storage::event::VisibilityIndexProgress::replay_epoch(self) -> u64
pub const fn radroots_storage::event::VisibilityIndexProgress::source_sequence(self) -> u64
@@ -878,6 +879,7 @@ pub fn radroots_storage::event::EventStore::query_provenance(&self, radroots_eve
pub fn radroots_storage::event::EventStore::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>>
pub fn radroots_storage::event::EventStore::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>>
pub fn radroots_storage::event::EventStore::query_visible(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, radroots_storage::Error>>
+pub fn radroots_storage::event::EventStore::query_visible_at(&self, radroots_storage::event::EventQuery, radroots_storage::event::EventSourceToken) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, radroots_storage::Error>>
pub fn radroots_storage::event::EventStore::rebuild_visibility(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilitySnapshot, radroots_storage::Error>>
pub fn radroots_storage::event::EventStore::source_token(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventSourceToken, radroots_storage::Error>>
pub fn radroots_storage::event::EventStore::status(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::status::EventStoreStatus, radroots_storage::Error>>
@@ -888,6 +890,7 @@ pub fn radroots_storage::memory::MemoryStorage::query_provenance(&self, radroots
pub fn radroots_storage::memory::MemoryStorage::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_visible(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::query_visible_at(&self, radroots_storage::event::EventQuery, radroots_storage::event::EventSourceToken) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::rebuild_visibility(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilitySnapshot, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::source_token(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventSourceToken, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::status(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::status::EventStoreStatus, radroots_storage::Error>>
@@ -1051,6 +1054,7 @@ pub fn radroots_storage::memory::MemoryStorage::query_provenance(&self, radroots
pub fn radroots_storage::memory::MemoryStorage::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_visible(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::query_visible_at(&self, radroots_storage::event::EventQuery, radroots_storage::event::EventSourceToken) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::rebuild_visibility(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilitySnapshot, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::source_token(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventSourceToken, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::status(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::status::EventStoreStatus, radroots_storage::Error>>
@@ -1866,9 +1870,11 @@ pub radroots_storage::Error::RebuildTicketTerminal
pub radroots_storage::Error::ReliabilityOperationTerminal
pub radroots_storage::Error::ReliabilityRevisionConflict
pub radroots_storage::Error::RestoreMemberVerificationFailed
+pub radroots_storage::Error::SourceEpochChanged
pub radroots_storage::Error::SourceGenerationChanged
pub radroots_storage::Error::SpaceInsufficient
pub radroots_storage::Error::TooManyEventQueryIds
+pub radroots_storage::Error::VisibilityRebuilding
impl core::error::Error for radroots_storage::Error
impl core::fmt::Display for radroots_storage::Error
pub fn radroots_storage::Error::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result
@@ -1917,6 +1923,7 @@ pub fn radroots_storage::EventStore::query_provenance(&self, radroots_event::id:
pub fn radroots_storage::EventStore::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>>
pub fn radroots_storage::EventStore::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>>
pub fn radroots_storage::EventStore::query_visible(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, radroots_storage::Error>>
+pub fn radroots_storage::EventStore::query_visible_at(&self, radroots_storage::event::EventQuery, radroots_storage::event::EventSourceToken) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, radroots_storage::Error>>
pub fn radroots_storage::EventStore::rebuild_visibility(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilitySnapshot, radroots_storage::Error>>
pub fn radroots_storage::EventStore::source_token(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventSourceToken, radroots_storage::Error>>
pub fn radroots_storage::EventStore::status(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::status::EventStoreStatus, radroots_storage::Error>>
@@ -1927,6 +1934,7 @@ pub fn radroots_storage::memory::MemoryStorage::query_provenance(&self, radroots
pub fn radroots_storage::memory::MemoryStorage::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_visible(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::query_visible_at(&self, radroots_storage::event::EventQuery, radroots_storage::event::EventSourceToken) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::rebuild_visibility(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilitySnapshot, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::source_token(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventSourceToken, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::status(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::status::EventStoreStatus, radroots_storage::Error>>
diff --git a/contracts/api_baselines/radroots_storage_sqlite.txt b/contracts/api_baselines/radroots_storage_sqlite.txt
@@ -586,6 +586,7 @@ pub fn radroots_storage_sqlite::SqliteStorage::query_provenance(&self, radroots_
pub fn radroots_storage_sqlite::SqliteStorage::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::query_visible(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, radroots_storage::error::Error>>
+pub fn radroots_storage_sqlite::SqliteStorage::query_visible_at(&self, radroots_storage::event::EventQuery, radroots_storage::event::EventSourceToken) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::rebuild_visibility(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilitySnapshot, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::source_token(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventSourceToken, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::status(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::status::EventStoreStatus, radroots_storage::error::Error>>
diff --git a/contracts/architecture/decisions/indexed_visibility_reads.v1.json b/contracts/architecture/decisions/indexed_visibility_reads.v1.json
@@ -0,0 +1,43 @@
+{
+ "schema_version": 1,
+ "decision_id": "radroots.storage.indexed_visibility_reads.v1",
+ "status": "implemented_not_published",
+ "owners": ["radroots_storage", "radroots_storage_sqlite"],
+ "runtime_version": 20,
+ "immutable_runtime_prefix": "versions 1 through 19",
+ "authority": "immutable canonical signed events and the existing visibility reducer",
+ "current_state": {
+ "head_membership": "highest unsigned created_at, then lowest event_id; verified unsupported heads retain authority",
+ "head_update": "refresh the previous and new selected member only",
+ "deletion": "retain authorized immutable targets and refresh affected event or selected address keys through a durable bounded queue",
+ "status": "one cached header read; never evaluate or decode source history",
+ "ordinary_admission": "a ready index advances inside the canonical source transaction; unfinished derived work remains dirty",
+ "empty_source": "atomic empty ready header on fresh source-generation insertion",
+ "legacy": "forward20 resets derived preparation once, then uses existing bounded source backfill and changed-epoch replay",
+ "readiness": "source backfill, changed replay and affected deletion keys all complete at one source fence",
+ "incomplete": "VisibilityRebuilding; no stale state advertised current"
+ },
+ "bounded_reads": {
+ "maximum_generic_page_rows": 1000,
+ "selected_keys": "limit plus one length-only indexed keys",
+ "all_events": "forced source-generation/visible/sequence index",
+ "event_ids": "forced source-generation/event-id index; existing maximum 256 ids",
+ "maximum_logical_bytes": 16777216,
+ "charge": "4096 + (limit + 1) * 128 + sum(signed_bytes * 16 + 4096)",
+ "hydration": "only charged returned rows; byte exhaustion returns an honest continuation or SpaceInsufficient",
+ "consistency": "header, lengths and hydrated events share one read transaction or memory lock",
+ "fenced_api": "EventStore::query_visible_at checks expected EventSourceToken in that same read",
+ "stale": "SourceEpochChanged",
+ "unsupported_fenced_adapter": "BackendUnavailable"
+ },
+ "maintenance": {
+ "maximum_total_source_and_affected_key_units": 64,
+ "maximum_logical_bytes": 16777216,
+ "affected_key_charge": "4096 + target_key_bytes * 16; inspect lengths before payload allocation",
+ "priority": "retained affected targets drain before new source work",
+ "progress": "processed_rows reports materialized source rows; processed_keys reports refreshed deletion keys",
+ "worker": "explicit caller-owned bounded calls; no hidden task"
+ },
+ "diagnostics": "rebuild_visibility remains the explicit full semantic oracle",
+ "forbidden": ["private application policy", "second database", "full-source read fallback", "hidden worker", "protocol registry changes", "new host configuration"]
+}
diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs
@@ -34,6 +34,10 @@ pub enum Error {
EventPageLimitExceeded,
CursorGenerationMismatch,
SourceGenerationChanged,
+ /// The requested semantic fence no longer matches canonical truth.
+ SourceEpochChanged,
+ /// Canonical visibility requires explicit bounded maintenance.
+ VisibilityRebuilding,
EventNotFound,
CorruptStoredEvent,
BackendUnavailable,
@@ -169,6 +173,8 @@ impl fmt::Display for Error {
Self::EventPageLimitExceeded => "storage event page exceeds its requested limit",
Self::CursorGenerationMismatch => "storage cursor belongs to another source generation",
Self::SourceGenerationChanged => "storage source generation changed",
+ Self::SourceEpochChanged => "storage source epoch changed",
+ Self::VisibilityRebuilding => "storage visibility is rebuilding",
Self::EventNotFound => "storage event was not found",
Self::CorruptStoredEvent => "storage event data is corrupt",
Self::BackendUnavailable => "storage backend is unavailable",
@@ -331,6 +337,8 @@ mod tests {
EventPageLimitExceeded,
CursorGenerationMismatch,
SourceGenerationChanged,
+ SourceEpochChanged,
+ VisibilityRebuilding,
EventNotFound,
CorruptStoredEvent,
BackendUnavailable,
diff --git a/crates/storage/src/event.rs b/crates/storage/src/event.rs
@@ -673,9 +673,21 @@ pub trait EventStore: Send + Sync {
query: EventQuery,
) -> BoxFuture<'_, Result<EventPage<StoredVisibleEvent>, Error>>;
+ /// Queries indexed visibility at exactly the expected canonical source fence.
+ ///
+ /// Compare the fence and hydrate the page in one consistent read. A mismatch
+ /// returns [`Error::SourceEpochChanged`]; unsupported providers fail closed.
+ fn query_visible_at(
+ &self,
+ _query: EventQuery,
+ _expected: EventSourceToken,
+ ) -> BoxFuture<'_, Result<EventPage<StoredVisibleEvent>, Error>> {
+ Box::pin(async { Err(Error::BackendUnavailable) })
+ }
+
/// Rebuilds current visibility from immutable retained event truth.
///
- /// Implementations must use the same reducer as [`Self::query_visible`].
+ /// This explicit full diagnostic is the semantic oracle for indexed reads.
fn rebuild_visibility(&self) -> BoxFuture<'_, Result<VisibilitySnapshot, Error>>;
/// Queries bounded provenance for one event.
diff --git a/crates/storage/src/event/visibility_index.rs b/crates/storage/src/event/visibility_index.rs
@@ -48,6 +48,7 @@ pub struct VisibilityIndexProgress {
source_sequence: u64,
replay_epoch: u64,
processed_rows: u16,
+ processed_keys: u16,
charged_bytes: u64,
complete: bool,
}
@@ -67,6 +68,7 @@ impl VisibilityIndexProgress {
source_sequence,
replay_epoch,
processed_rows,
+ processed_keys: 0,
charged_bytes,
complete,
}
@@ -88,6 +90,16 @@ impl VisibilityIndexProgress {
pub const fn processed_rows(self) -> u16 {
self.processed_rows
}
+ /// Affected deletion keys refreshed within the same total row-unit budget.
+ pub const fn processed_keys(self) -> u16 {
+ self.processed_keys
+ }
+
+ #[doc(hidden)]
+ pub const fn with_processed_keys(mut self, keys: u16) -> Self {
+ self.processed_keys = keys;
+ self
+ }
/// Conservative logical allocation charge of this call.
pub const fn charged_bytes(self) -> u64 {
self.charged_bytes
diff --git a/crates/storage/src/memory.rs b/crates/storage/src/memory.rs
@@ -201,6 +201,87 @@ impl MemoryStorage {
Ok((selected, next))
}
+ fn visible_page_locked(
+ &self,
+ state: &State,
+ query: EventQuery,
+ expected: Option<EventSourceToken>,
+ ) -> Result<EventPage<StoredVisibleEvent>, Error> {
+ if query
+ .bounds()
+ .cursor()
+ .is_some_and(|cursor| cursor.generation() != self.generation)
+ {
+ return Err(Error::SourceGenerationChanged);
+ }
+ if expected.is_some_and(|token| {
+ token != EventSourceToken::new(self.generation, state.event_mutation_epoch)
+ }) {
+ return Err(Error::SourceEpochChanged);
+ }
+ if !state.visibility_index.ready {
+ return Err(Error::VisibilityRebuilding);
+ }
+ let after = query
+ .bounds()
+ .cursor()
+ .map_or(0, |cursor| cursor.sequence().get());
+ let maximum = usize::from(query.bounds().limit());
+ let positions = if query.event_ids().is_empty() {
+ state
+ .visibility_index
+ .visible
+ .range((std::ops::Bound::Excluded(after), std::ops::Bound::Unbounded))
+ .take(maximum + 1)
+ .copied()
+ .collect::<Vec<_>>()
+ } else {
+ let positions = query
+ .event_ids()
+ .iter()
+ .filter_map(|id| state.visibility_index.ids.get(id).copied())
+ .filter(|position| {
+ *position > after && state.visibility_index.visible.contains(position)
+ })
+ .collect::<std::collections::BTreeSet<_>>();
+ positions.into_iter().take(maximum + 1).collect()
+ };
+ let mut charged = 4096 + (u64::from(query.bounds().limit()) + 1) * 128;
+ let mut items = Vec::new();
+ let mut more = false;
+ for position in positions {
+ if items.len() == maximum {
+ more = true;
+ break;
+ }
+ let entry = state
+ .events
+ .get(usize::try_from(position - 1).map_err(|_| Error::CorruptStoredEvent)?)
+ .ok_or(Error::CorruptStoredEvent)?;
+ let charge = charged
+ .checked_add(crate::event::VisibilityIndexBounds::charge(
+ entry.admission.event().raw_json().len() as u64,
+ )?)
+ .ok_or(Error::SpaceInsufficient)?;
+ if charge > 16 * 1024 * 1024 {
+ if items.is_empty() {
+ return Err(Error::SpaceInsufficient);
+ }
+ more = true;
+ break;
+ }
+ items.push(StoredVisibleEvent::new(
+ entry.position,
+ entry.admission.event().clone(),
+ ));
+ charged = charge;
+ }
+ let next = more
+ .then(|| items.last().map(StoredVisibleEvent::position))
+ .flatten();
+ EventPage::new(self.generation, items, next, query.bounds())
+ }
+
fn visibility_locked(&self, state: &State) -> Result<VisibilityEvaluation, Error> {
evaluate_visibility(
self.generation,
@@ -219,6 +300,28 @@ impl MemoryStorage {
state: &mut State,
admission: EventAdmission,
) -> Result<AdmissionReceipt, Error> {
+ let was_ready = state.visibility_index.ready;
+ crate::event::VisibilityEventMetadata::from_event(admission.event(), admission.stage())?;
+ let bytes = u64::try_from(admission.event().raw_json().len())
+ .map_err(|_| Error::SpaceInsufficient)?;
+ let fits = crate::event::VisibilityIndexBounds::charge(bytes)?
+ <= 16 * 1024 * 1024 - 4096 - 64 * 128;
+ let receipt = self.admit_source_locked(state, admission)?;
+ if was_ready && receipt.disposition() != AdmissionDisposition::Duplicate && fits {
+ visibility_index::prepare(
+ self.generation,
+ state,
+ crate::event::VisibilityIndexBounds::new(64, 16 * 1024 * 1024)?,
+ )?;
+ }
+ Ok(receipt)
+ }
+
+ fn admit_source_locked(
+ &self,
+ state: &mut State,
+ admission: EventAdmission,
+ ) -> Result<AdmissionReceipt, Error> {
if let Some(entry) = state
.events
.iter_mut()
@@ -463,29 +566,17 @@ impl EventStore for MemoryStorage {
fn status(&self) -> BoxFuture<'_, Result<EventStoreStatus, Error>> {
Box::pin(async move {
let state = self.state()?;
- let raw = u64::try_from(state.events.len()).map_err(|_| Error::CorruptStoredEvent)?;
- let verified = u64::try_from(
- state
- .events
- .iter()
- .filter(|entry| entry.admission.stage() >= AdmissionStage::Verified)
- .count(),
- )
- .map_err(|_| Error::CorruptStoredEvent)?;
- let visible = u64::try_from(
- self.visibility_locked(&state)?
- .snapshot()
- .visible_event_ids()
- .len(),
- )
- .map_err(|_| Error::CorruptStoredEvent)?;
+ if !state.visibility_index.ready {
+ return Err(Error::VisibilityRebuilding);
+ }
EventStoreStatus::new(
self.generation,
EventStoreMode::ReadWrite,
EventStoreHealth::Available,
- raw,
- verified,
- visible,
+ u64::try_from(state.events.len()).map_err(|_| Error::CorruptStoredEvent)?,
+ state.visibility_index.verified,
+ u64::try_from(state.visibility_index.visible.len())
+ .map_err(|_| Error::CorruptStoredEvent)?,
)
})
}
@@ -543,17 +634,18 @@ impl EventStore for MemoryStorage {
) -> BoxFuture<'_, Result<EventPage<StoredVisibleEvent>, Error>> {
Box::pin(async move {
let state = self.state()?;
- let visibility = self.visibility_locked(&state)?;
- let (entries, next) = self.selected(&state, &query, |entry| {
- visibility.is_visible(entry.admission.event_id())
- })?;
- let items = entries
- .into_iter()
- .map(|entry| {
- StoredVisibleEvent::new(entry.position, entry.admission.event().clone())
- })
- .collect();
- EventPage::new(self.generation, items, next, query.bounds())
+ self.visible_page_locked(&state, query, None)
+ })
+ }
+
+ fn query_visible_at(
+ &self,
+ query: EventQuery,
+ expected: EventSourceToken,
+ ) -> BoxFuture<'_, Result<EventPage<StoredVisibleEvent>, Error>> {
+ Box::pin(async move {
+ let state = self.state()?;
+ self.visible_page_locked(&state, query, Some(expected))
})
}
diff --git a/crates/storage/src/memory_visibility_index.rs b/crates/storage/src/memory_visibility_index.rs
@@ -1,4 +1,5 @@
use super::State;
+use crate::event::{AdmissionStage, EventId};
use crate::{
Error,
event::{
@@ -6,13 +7,21 @@ use crate::{
VisibilityIndexProgress,
},
};
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
#[derive(Clone, Default)]
pub(super) struct Index {
cursor: Option<Cursor>,
pub(super) ready: bool,
pub(super) events: BTreeMap<u64, VisibilityEventMetadata>,
+ pub(super) visible: BTreeSet<u64>,
+ pub(super) ids: BTreeMap<EventId, u64>,
+ pub(super) verified: u64,
+ heads: BTreeMap<Vec<u8>, u64>,
+ address_heads: BTreeMap<String, u64>,
+ deletion_events: BTreeSet<(EventId, [u8; 32])>,
+ deletion_addresses: BTreeMap<(String, [u8; 32]), u64>,
+ pending: BTreeSet<(u8, Vec<u8>)>,
}
#[derive(Clone, Copy)]
@@ -25,11 +34,134 @@ struct Cursor {
impl Index {
pub(super) const fn new() -> Self {
Self {
- cursor: None,
- ready: false,
+ cursor: Some(Cursor {
+ target: 0,
+ backfill: 0,
+ replay_epoch: 0,
+ }),
+ ready: true,
events: BTreeMap::new(),
+ visible: BTreeSet::new(),
+ ids: BTreeMap::new(),
+ verified: 0,
+ heads: BTreeMap::new(),
+ address_heads: BTreeMap::new(),
+ deletion_events: BTreeSet::new(),
+ deletion_addresses: BTreeMap::new(),
+ pending: BTreeSet::new(),
}
}
+
+ fn refresh(&mut self, position: u64) {
+ let Some(event) = self.events.get(&position) else {
+ return;
+ };
+ let head = event
+ .head_coordinate
+ .as_ref()
+ .is_none_or(|key| self.heads.get(key) == Some(&position));
+ let deleted = !event.deletion_request
+ && (self
+ .deletion_events
+ .contains(&(event.event_id, event.author))
+ || event
+ .deletion_coordinate
+ .as_ref()
+ .is_some_and(|coordinate| {
+ self.deletion_addresses
+ .get(&(coordinate.clone(), event.author))
+ .is_some_and(|time| *time >= event.created_at)
+ }));
+ if event.stage == AdmissionStage::Visible && !event.ephemeral && head && !deleted {
+ self.visible.insert(position);
+ } else {
+ self.visible.remove(&position);
+ }
+ }
+
+ fn put(&mut self, position: u64, event: VisibilityEventMetadata) {
+ let previously_verified = self
+ .events
+ .get(&position)
+ .is_some_and(|old| old.stage != AdmissionStage::Raw);
+ if !previously_verified && event.stage != AdmissionStage::Raw {
+ self.verified += 1;
+ }
+ self.ids.insert(event.event_id, position);
+ for target in &event.event_targets {
+ self.deletion_events.insert((*target, event.author));
+ self.pending.insert((0, target.as_bytes().to_vec()));
+ }
+ for coordinate in &event.address_targets {
+ self.deletion_addresses
+ .entry((coordinate.clone(), event.author))
+ .and_modify(|time| *time = (*time).max(event.created_at))
+ .or_insert(event.created_at);
+ self.pending.insert((1, coordinate.as_bytes().to_vec()));
+ }
+ let old_head = event
+ .head_coordinate
+ .as_ref()
+ .and_then(|key| self.heads.get(key))
+ .copied();
+ let replace = old_head.is_none_or(|old| {
+ self.events.get(&old).is_some_and(|current| {
+ event.created_at > current.created_at
+ || (event.created_at == current.created_at && event.event_id < current.event_id)
+ })
+ });
+ if let Some(key) = &event.head_coordinate
+ && replace
+ {
+ self.heads.insert(key.clone(), position);
+ if let Some(old) = old_head.and_then(|old| self.events.get(&old))
+ && let Some(address) = &old.deletion_coordinate
+ {
+ self.address_heads.remove(address);
+ }
+ if let Some(address) = &event.deletion_coordinate {
+ self.address_heads.insert(address.clone(), position);
+ }
+ }
+ self.events.insert(position, event);
+ if replace && let Some(old) = old_head {
+ self.refresh(old);
+ }
+ self.refresh(position);
+ }
+
+ fn drain(&mut self, maximum: u16, charged: &mut u64, byte_limit: u64) -> Result<u16, Error> {
+ let mut processed = 0;
+ while processed < maximum {
+ let Some((kind, key)) = self.pending.first() else {
+ break;
+ };
+ let charge = charged
+ .checked_add(4096 + 16 * key.len() as u64)
+ .ok_or(Error::SpaceInsufficient)?;
+ if charge > byte_limit {
+ break;
+ }
+ let position = if *kind == 0 {
+ let id = EventId::from_bytes(
+ key.as_slice()
+ .try_into()
+ .map_err(|_| Error::CorruptStoredEvent)?,
+ );
+ self.ids.get(&id).copied()
+ } else {
+ let address = std::str::from_utf8(key).map_err(|_| Error::CorruptStoredEvent)?;
+ self.address_heads.get(address).copied()
+ };
+ self.pending.pop_first();
+ if let Some(position) = position {
+ self.refresh(position);
+ }
+ *charged = charge;
+ processed += 1;
+ }
+ Ok(processed)
+ }
}
pub(super) fn prepare(
@@ -48,7 +180,9 @@ pub(super) fn prepare(
replay_epoch: state.event_mutation_epoch,
});
let building = cursor.backfill < cursor.target;
- let positions = if building {
+ let positions = if !state.visibility_index.pending.is_empty() {
+ Vec::new()
+ } else if building {
((cursor.backfill + 1)..=cursor.target)
.take(usize::from(bounds.rows()))
.collect::<Vec<_>>()
@@ -98,12 +232,25 @@ pub(super) fn prepare(
))
.next()
.is_some();
- let complete = cursor.backfill == cursor.target && !pending;
- if complete {
+ let source_complete = cursor.backfill == cursor.target && !pending;
+ if source_complete {
cursor.replay_epoch = state.event_mutation_epoch;
}
let processed = u16::try_from(prepared.len()).map_err(|_| Error::CorruptStoredEvent)?;
- state.visibility_index.events.extend(prepared);
+ for (position, event) in prepared {
+ state.visibility_index.put(position, event);
+ }
+ let keys =
+ state
+ .visibility_index
+ .drain(bounds.rows() - processed, &mut charged, bounds.bytes())?;
+ if processed == 0 && keys == 0 && !state.visibility_index.pending.is_empty() {
+ return Err(Error::SpaceInsufficient);
+ }
+ let complete = source_complete && state.visibility_index.pending.is_empty();
+ if complete && state.visibility_index.events.len() != state.events.len() {
+ return Err(Error::CorruptStoredEvent);
+ }
state.visibility_index.cursor = Some(cursor);
state.visibility_index.ready = complete;
Ok(VisibilityIndexProgress::new(
@@ -113,7 +260,8 @@ pub(super) fn prepare(
processed,
charged,
complete,
- ))
+ )
+ .with_processed_keys(keys))
}
#[cfg(test)]
@@ -152,6 +300,7 @@ mod tests {
fn memory_visibility_index_retains_backfill_and_refuses_oversized_rows_without_partial_state() {
let generation = SourceGeneration::new([82; 32]).unwrap();
let storage = MemoryStorage::new(generation);
+ storage.state().unwrap().visibility_index = Index::default();
let first = admission("first", 10);
block_on(storage.admit(first.clone())).unwrap();
block_on(storage.admit(admission("second", 20))).unwrap();
@@ -176,6 +325,9 @@ mod tests {
.processed_rows(),
0
);
+ // Exercise explicit maintenance of an already dirty index. Ordinary
+ // ready-index admission is separately covered by indexed-read parity.
+ storage.state().unwrap().visibility_index.ready = false;
block_on(storage.admit(admission(&"x".repeat(1000), 40))).unwrap();
assert_eq!(
block_on(storage.maintain_visibility(VisibilityIndexBounds::new(1, 10000).unwrap())),
diff --git a/crates/storage/tests/event_store.rs b/crates/storage/tests/event_store.rs
@@ -71,6 +71,16 @@ fn unsupported_adapter_cannot_invent_a_source_fence() {
Err(Error::BackendUnavailable)
);
assert_eq!(
+ block_on(MemoryEventStore::new().query_visible_at(
+ EventQuery::all(EventQueryBounds::first(5).unwrap()),
+ radroots_storage::event::EventSourceToken::new(
+ SourceGeneration::new([1; 32]).unwrap(),
+ 0
+ ),
+ )),
+ Err(Error::BackendUnavailable)
+ );
+ assert_eq!(
block_on(MemoryEventStore::new().maintain_visibility(
radroots_storage::event::VisibilityIndexBounds::new(64, 16 * 1024 * 1024).unwrap()
)),
@@ -339,15 +349,210 @@ fn verified(event: &SignedEvent) -> VerifiedEvent {
}
fn visible(event: &SignedEvent) -> VisibleEvent {
- verified(event)
- .validate_contract()
- .expect("contract")
+ let verified = verified(event);
+ let validated = if event.envelope().kind_u32() == 5 {
+ verified
+ .validate_contract_for_admission("radroots.social.deletion_request.v1")
+ .expect("deletion contract")
+ } else {
+ verified.validate_contract().expect("contract")
+ };
+ validated
.admit_with(&Allow)
.expect("admission")
.make_visible_with(&Allow)
.expect("visibility")
}
+fn indexed_event(
+ author: &str,
+ time: u64,
+ kind: u32,
+ tags: Vec<Vec<String>>,
+ content: &str,
+) -> SignedEvent {
+ let mut wire = Nip01EventWire {
+ id: "0".repeat(64),
+ pubkey: author.to_owned(),
+ created_at: time,
+ kind,
+ tags,
+ content: content.to_owned(),
+ sig: "42".repeat(64),
+ extra: Default::default(),
+ };
+ wire.id = wire.computed_event_id().unwrap().to_hex();
+ let raw = serde_json::to_string(&wire).unwrap();
+ SignedEvent::from_wire_verified_id(wire, raw).unwrap()
+}
+
+#[test]
+fn memory_indexed_address_deletion_head_ties_and_unsupported_authority_match_reference() {
+ use radroots_storage::memory::MemoryStorage;
+ const AUTHOR: &str = "585591529da0bab31b3b1b1f986611cf5f435dca84f978c89ee8a40cca7103df";
+ const OTHER: &str = "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798";
+ let storage = MemoryStorage::new(SourceGeneration::new([101; 32]).unwrap());
+ let address = format!("0:{AUTHOR}:");
+ let addressable = format!("30023:{AUTHOR}:");
+ let cases = [
+ (
+ indexed_event(AUTHOR, 10, 0, vec![], r#"{"name":"old"}"#),
+ true,
+ ),
+ (
+ indexed_event(AUTHOR, 20, 0, vec![], r#"{"name":"first tie"}"#),
+ true,
+ ),
+ (
+ indexed_event(AUTHOR, 20, 0, vec![], r#"{"name":"second tie"}"#),
+ true,
+ ),
+ (
+ indexed_event(
+ OTHER,
+ 25,
+ 5,
+ vec![vec!["a".into(), address.clone()]],
+ "wrong author",
+ ),
+ true,
+ ),
+ (
+ indexed_event(AUTHOR, 25, 5, vec![vec!["a".into(), address]], "cutoff"),
+ true,
+ ),
+ (
+ indexed_event(AUTHOR, 30, 0, vec![], r#"{"name":"later"}"#),
+ true,
+ ),
+ (
+ indexed_event(AUTHOR, 5, 0, vec![], r#"{"name":"older arrives late"}"#),
+ true,
+ ),
+ (
+ indexed_event(AUTHOR, 40, 0, vec![], "unsupported profile content"),
+ false,
+ ),
+ (
+ indexed_event(AUTHOR, 50, 30023, vec![], "missing identifier"),
+ false,
+ ),
+ (
+ indexed_event(
+ AUTHOR,
+ 51,
+ 30023,
+ vec![vec!["d".into(), "".into()]],
+ "explicit empty",
+ ),
+ false,
+ ),
+ (
+ indexed_event(AUTHOR, 52, 30023, vec![], "missing again"),
+ false,
+ ),
+ (
+ indexed_event(
+ AUTHOR,
+ 60,
+ 5,
+ vec![vec!["a".into(), addressable]],
+ "address cutoff",
+ ),
+ true,
+ ),
+ ];
+ for (event, make_visible) in cases {
+ let admission = if make_visible {
+ EventAdmission::visible(observed(event.clone(), 100), visible(&event)).unwrap()
+ } else {
+ EventAdmission::verified(observed(event.clone(), 100), verified(&event)).unwrap()
+ };
+ block_on(storage.admit(admission.clone())).unwrap();
+ block_on(storage.admit(admission)).unwrap();
+ let oracle = block_on(storage.rebuild_visibility()).unwrap();
+ let mut actual =
+ block_on(storage.query_visible(EventQuery::all(EventQueryBounds::first(100).unwrap())))
+ .unwrap()
+ .items()
+ .iter()
+ .map(|item| *item.event().id())
+ .collect::<Vec<_>>();
+ actual.sort();
+ assert_eq!(actual, oracle.visible_event_ids());
+ assert_eq!(
+ block_on(storage.status()).unwrap().visible_events(),
+ actual.len() as u64
+ );
+ }
+}
+
+#[test]
+fn memory_indexed_scoped_byte_pages_and_fences_preserve_continuation() {
+ use radroots_storage::{event::EventSourceToken, memory::MemoryStorage};
+ let generation = SourceGeneration::new([102; 32]).unwrap();
+ let storage = MemoryStorage::new(generation);
+ let author = "585591529da0bab31b3b1b1f986611cf5f435dca84f978c89ee8a40cca7103df";
+ let mut ids = Vec::new();
+ for time in 0..12 {
+ let event = indexed_event(author, time, 1, vec![], &"x".repeat(120_000));
+ ids.push(*event.id());
+ block_on(storage.admit(
+ EventAdmission::visible(observed(event.clone(), 100), visible(&event)).unwrap(),
+ ))
+ .unwrap();
+ }
+ let token = block_on(storage.source_token()).unwrap();
+ let first = block_on(
+ storage.query_visible_at(EventQuery::all(EventQueryBounds::first(12).unwrap()), token),
+ )
+ .unwrap();
+ assert!(first.items().len() < 12);
+ let next = block_on(
+ storage.query_visible_at(
+ EventQuery::all(
+ EventQueryBounds::first(12)
+ .unwrap()
+ .after(first.next_cursor().unwrap()),
+ ),
+ token,
+ ),
+ )
+ .unwrap();
+ assert_eq!(first.items().len() + next.items().len(), 12);
+ assert!(next.next_cursor().is_none());
+ let selected = EventQuery::for_ids(
+ EventQueryBounds::first(5).unwrap(),
+ vec![ids[11], EventId::from_bytes([255; 32]), ids[0]],
+ )
+ .unwrap();
+ let page = block_on(storage.query_visible_at(selected, token)).unwrap();
+ assert_eq!(
+ page.items()
+ .iter()
+ .map(|item| *item.event().id())
+ .collect::<Vec<_>>(),
+ vec![ids[0], ids[11]]
+ );
+ assert_eq!(
+ block_on(storage.query_visible_at(
+ EventQuery::all(EventQueryBounds::first(5).unwrap()),
+ EventSourceToken::new(generation, token.epoch() - 1)
+ )),
+ Err(Error::SourceEpochChanged)
+ );
+ let cursor = EventPosition::new(
+ SourceGeneration::new([103; 32]).unwrap(),
+ EventSequence::new(1).unwrap(),
+ );
+ assert_eq!(
+ block_on(storage.query_visible(EventQuery::all(
+ EventQueryBounds::first(5).unwrap().after(cursor)
+ ))),
+ Err(Error::SourceGenerationChanged)
+ );
+}
+
#[test]
fn event_store_is_dyn_compatible_and_enforces_monotonic_admission() {
let store = MemoryEventStore::new();
diff --git a/crates/storage_sqlite/src/event/mod.rs b/crates/storage_sqlite/src/event/mod.rs
@@ -2,6 +2,9 @@ use crate::backend::map_backend;
mod visibility_index;
#[cfg(test)]
mod visibility_index_tests;
+mod visibility_reads;
+#[cfg(test)]
+mod visibility_reads_tests;
use radroots_event::SignedEvent;
use radroots_event_codec::Codec;
use radroots_storage::{
@@ -300,6 +303,14 @@ impl SqliteStorage {
transaction: &mut sqlx::Transaction<'_, Sqlite>,
admission: EventAdmission,
) -> Result<AdmissionReceipt, Error> {
+ let index_was_ready = sqlx::query_scalar::<_, bool>(
+ "SELECT ready FROM radroots_runtime_visibility_progress WHERE source_generation = ?",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .fetch_optional(&mut **transaction)
+ .await
+ .map_err(map_backend)?
+ .unwrap_or(false);
let existing = sqlx::query(
"SELECT source_generation, source_sequence, signed_event, admission_stage,
admitted_contract_id, admitted_registry_version
@@ -380,6 +391,15 @@ impl SqliteStorage {
)
};
Self::store_provenance(transaction, &admission).await?;
+ let bounds = radroots_storage::event::VisibilityIndexBounds::new(64, 16 * 1024 * 1024)?;
+ let bytes = u64::try_from(admission.event().raw_json().len())
+ .map_err(|_| Error::SpaceInsufficient)?;
+ let fits = radroots_storage::event::VisibilityIndexBounds::charge(bytes)?
+ <= bounds.bytes() - 4096 - u64::from(bounds.rows()) * 128;
+ if index_was_ready && disposition != AdmissionDisposition::Duplicate && fits {
+ self.prepare_visibility_transaction(transaction, bounds)
+ .await?;
+ }
Ok(AdmissionReceipt::new(
*admission.event_id(),
position,
@@ -427,36 +447,10 @@ impl EventStore for SqliteStorage {
fn status(&self) -> BoxFuture<'_, Result<EventStoreStatus, Error>> {
Box::pin(async move {
- let sql = "SELECT
- COUNT(*) AS raw_events,
- COALESCE(SUM(CASE WHEN admission_stage IN ('verified', 'visible') THEN 1 ELSE 0 END), 0)
- AS verified_events
- FROM radroots_runtime_events WHERE source_generation = ?";
- #[cfg(test)]
- self.work.request(work::Read::Aggregate, sql);
- let row = sqlx::query(sql)
- .bind(self.generation.as_bytes().as_slice())
- .fetch_one(&self.pool)
- .await
- .map_err(map_backend)?;
- #[cfg(test)]
- self.work.aggregate_returned();
- let visibility_rows = self.current_event_rows().await?;
- let visible_events = u64::try_from(
- self.visibility_for_rows(&visibility_rows)?
- .snapshot()
- .visible_event_ids()
- .len(),
- )
- .map_err(|_| Error::CorruptStoredEvent)?;
- EventStoreStatus::new(
- self.generation,
- self.mode,
- EventStoreHealth::Available,
- u64_from_i64(row.try_get("raw_events").map_err(map_corrupt)?)?,
- u64_from_i64(row.try_get("verified_events").map_err(map_corrupt)?)?,
- visible_events,
- )
+ let mut transaction = self.pool.begin().await.map_err(map_backend)?;
+ let status = self.visibility_header(&mut transaction, None).await?;
+ transaction.commit().await.map_err(map_backend)?;
+ Ok(status)
})
}
@@ -508,35 +502,15 @@ impl EventStore for SqliteStorage {
&self,
query: EventQuery,
) -> BoxFuture<'_, Result<EventPage<StoredVisibleEvent>, Error>> {
- Box::pin(async move {
- self.validate_cursor(&query)?;
- let after = query
- .bounds()
- .cursor()
- .map_or(0, |cursor| cursor.sequence().get());
- let rows = self.current_event_rows().await?;
- let visibility = self.visibility_for_rows(&rows)?;
- let mut selected = rows
- .into_iter()
- .filter(|row| {
- row.position.sequence().get() > after
- && query.selects(row.event.id())
- && visibility.is_visible(row.event.id())
- })
- .take(usize::from(query.bounds().limit()) + 1)
- .collect::<Vec<_>>();
- let next = if selected.len() > usize::from(query.bounds().limit()) {
- selected.truncate(usize::from(query.bounds().limit()));
- selected.last().map(|row| row.position)
- } else {
- None
- };
- let items = selected
- .into_iter()
- .map(|row| StoredVisibleEvent::new(row.position, row.event))
- .collect();
- EventPage::new(self.generation, items, next, query.bounds())
- })
+ Box::pin(async move { self.indexed_visible_page(query, None).await })
+ }
+
+ fn query_visible_at(
+ &self,
+ query: EventQuery,
+ expected: EventSourceToken,
+ ) -> BoxFuture<'_, Result<EventPage<StoredVisibleEvent>, Error>> {
+ Box::pin(async move { self.indexed_visible_page(query, Some(expected)).await })
}
fn rebuild_visibility(&self) -> BoxFuture<'_, Result<VisibilitySnapshot, Error>> {
@@ -745,7 +719,24 @@ mod tests {
.execute(&pool)
.await
.expect("source generation");
- SqliteStorage::new(pool, generation, EventStoreMode::ReadWrite)
+ let storage = SqliteStorage::new(pool, generation, EventStoreMode::ReadWrite);
+ storage
+ .maintain_visibility(
+ radroots_storage::event::VisibilityIndexBounds::new(64, 16 * 1024 * 1024).unwrap(),
+ )
+ .await
+ .unwrap();
+ storage
+ }
+
+ pub(super) async fn unprepared_store(generation: SourceGeneration) -> SqliteStorage {
+ let storage = store(generation).await;
+ sqlx::query("DELETE FROM radroots_runtime_visibility_progress WHERE source_generation = ?")
+ .bind(generation.as_bytes().as_slice())
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ storage
}
fn signed_event(content: &str, pretty: bool) -> SignedEvent {
diff --git a/crates/storage_sqlite/src/event/visibility_index.rs b/crates/storage_sqlite/src/event/visibility_index.rs
@@ -11,21 +11,33 @@ impl SqliteStorage {
if self.mode == EventStoreMode::ReadOnly {
return Err(Error::BackendUnavailable);
}
- let mut charged = 4096 + u64::from(bounds.rows()) * 128;
- if charged > bounds.bytes() {
- return Err(Error::SpaceInsufficient);
- }
let mut transaction = self
.pool
.begin_with("BEGIN IMMEDIATE")
.await
.map_err(map_backend)?;
+ let progress = self
+ .prepare_visibility_transaction(&mut transaction, bounds)
+ .await?;
+ transaction.commit().await.map_err(map_backend)?;
+ Ok(progress)
+ }
+
+ pub(super) async fn prepare_visibility_transaction(
+ &self,
+ transaction: &mut sqlx::Transaction<'_, Sqlite>,
+ bounds: VisibilityIndexBounds,
+ ) -> Result<VisibilityIndexProgress, Error> {
+ let mut charged = 4096 + u64::from(bounds.rows()) * 128;
+ if charged > bounds.bytes() {
+ return Err(Error::SpaceInsufficient);
+ }
let source = sqlx::query(
"SELECT mutation_epoch, sequence_head FROM radroots_runtime_source_generations
WHERE generation = ? AND state = 'active'",
)
.bind(self.generation.as_bytes().as_slice())
- .fetch_optional(&mut *transaction)
+ .fetch_optional(&mut **transaction)
.await
.map_err(map_backend)?
.ok_or(Error::SourceGenerationChanged)?;
@@ -39,7 +51,7 @@ impl SqliteStorage {
.bind(self.generation.as_bytes().as_slice())
.bind(sequence)
.bind(epoch)
- .execute(&mut *transaction)
+ .execute(&mut **transaction)
.await
.map_err(map_backend)?;
let progress = sqlx::query(
@@ -47,7 +59,7 @@ impl SqliteStorage {
FROM radroots_runtime_visibility_progress WHERE source_generation = ?",
)
.bind(self.generation.as_bytes().as_slice())
- .fetch_one(&mut *transaction).await.map_err(map_backend)?;
+ .fetch_one(&mut **transaction).await.map_err(map_backend)?;
let version: i64 = progress.try_get("schema_version").map_err(map_corrupt)?;
let target: i64 = progress.try_get("backfill_target").map_err(map_corrupt)?;
let mut backfill: i64 = progress.try_get("backfill_sequence").map_err(map_corrupt)?;
@@ -56,6 +68,16 @@ impl SqliteStorage {
if version != 1 || target > sequence || backfill > target || replay_epoch > epoch {
return Err(Error::CorruptStoredEvent);
}
+ let had_pending: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM radroots_runtime_visibility_pending WHERE source_generation = ?)")
+ .bind(self.generation.as_bytes().as_slice()).fetch_one(&mut **transaction).await.map_err(map_backend)?;
+ let mut processed_keys = self
+ .drain_visibility_keys(transaction, bounds.rows(), &mut charged, bounds.bytes())
+ .await?;
+ let remaining = if had_pending {
+ 0
+ } else {
+ bounds.rows() - processed_keys
+ };
let building = backfill < target;
let mut query = QueryBuilder::<Sqlite>::new(
"SELECT source_sequence, last_mutation_epoch, length(signed_event) AS signed_bytes
@@ -67,7 +89,7 @@ impl SqliteStorage {
query.push(" AND source_sequence <= ").push_bind(target);
query
.push(" ORDER BY source_sequence LIMIT ")
- .push_bind(i64::from(bounds.rows()));
+ .push_bind(i64::from(remaining));
} else {
query.push(" AND (last_mutation_epoch, source_sequence) > (");
query
@@ -77,14 +99,14 @@ impl SqliteStorage {
.push(")");
query
.push(" ORDER BY last_mutation_epoch, source_sequence LIMIT ")
- .push_bind(i64::from(bounds.rows()));
+ .push_bind(i64::from(remaining));
}
let lengths = query
.build()
- .fetch_all(&mut *transaction)
+ .fetch_all(&mut **transaction)
.await
.map_err(map_backend)?;
- if building && lengths.is_empty() {
+ if building && remaining > 0 && lengths.is_empty() {
return Err(Error::CorruptStoredEvent);
}
let mut processed = 0u16;
@@ -109,7 +131,7 @@ impl SqliteStorage {
)
.bind(self.generation.as_bytes().as_slice())
.bind(position)
- .fetch_one(&mut *transaction)
+ .fetch_one(&mut **transaction)
.await
.map_err(map_backend)?;
let decoded = self.decode_event_row(&row)?;
@@ -117,7 +139,7 @@ impl SqliteStorage {
return Err(Error::CorruptStoredEvent);
}
let metadata = VisibilityEventMetadata::from_event(&decoded.event, decoded.stage)?;
- self.put_visibility_metadata(&mut transaction, position, changed, metadata)
+ self.put_visibility_metadata(transaction, position, changed, metadata)
.await?;
charged = next_charge;
processed += 1;
@@ -135,11 +157,30 @@ impl SqliteStorage {
.bind(self.generation.as_bytes().as_slice())
.bind(replay_epoch)
.bind(replay_sequence)
- .fetch_one(&mut *transaction)
+ .fetch_one(&mut **transaction)
.await
.map_err(map_backend)?;
- let complete = backfill == target && !pending;
+ processed_keys += self
+ .drain_visibility_keys(
+ transaction,
+ bounds.rows() - processed - processed_keys,
+ &mut charged,
+ bounds.bytes(),
+ )
+ .await?;
+ let pending_keys: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM radroots_runtime_visibility_pending WHERE source_generation = ?)")
+ .bind(self.generation.as_bytes().as_slice()).fetch_one(&mut **transaction).await.map_err(map_backend)?;
+ if processed == 0 && processed_keys == 0 && pending_keys {
+ return Err(Error::SpaceInsufficient);
+ }
+ let complete = backfill == target && !pending && !pending_keys;
if complete {
+ let retained: i64 = sqlx::query_scalar(
+ "SELECT raw_events FROM radroots_runtime_visibility_progress WHERE source_generation = ?")
+ .bind(self.generation.as_bytes().as_slice()).fetch_one(&mut **transaction).await.map_err(map_backend)?;
+ if retained != sequence {
+ return Err(Error::CorruptStoredEvent);
+ }
replay_epoch = epoch;
replay_sequence = i64::MAX;
}
@@ -152,10 +193,9 @@ impl SqliteStorage {
.bind(replay_sequence)
.bind(complete)
.bind(self.generation.as_bytes().as_slice())
- .execute(&mut *transaction)
+ .execute(&mut **transaction)
.await
.map_err(map_backend)?;
- transaction.commit().await.map_err(map_backend)?;
Ok(VisibilityIndexProgress::new(
EventSourceToken::new(self.generation, u64_from_i64(epoch)?),
u64_from_i64(backfill)?,
@@ -163,7 +203,8 @@ impl SqliteStorage {
processed,
charged,
complete,
- ))
+ )
+ .with_processed_keys(processed_keys))
}
async fn put_visibility_metadata(
@@ -220,6 +261,8 @@ impl SqliteStorage {
.await
.map_err(map_backend)?;
}
+ self.update_visibility_membership(transaction, position)
+ .await?;
Ok(())
}
}
diff --git a/crates/storage_sqlite/src/event/visibility_index_tests.rs b/crates/storage_sqlite/src/event/visibility_index_tests.rs
@@ -1,4 +1,4 @@
-use super::tests::{observed, signed_event_with, store, verified, visible};
+use super::tests::{observed, signed_event_with, unprepared_store as store, verified, visible};
use super::*;
use radroots_storage::event::VisibilityIndexBounds;
@@ -52,10 +52,21 @@ async fn indexed_visible(storage: &SqliteStorage) -> Vec<EventId> {
AND deletion.author = candidate.author AND deletion.created_at >= candidate.created_at)))
ORDER BY candidate.event_id",
).bind(storage.generation.as_bytes().as_slice()).fetch_all(&storage.pool).await.unwrap();
- values
+ let ids = values
.into_iter()
.map(|value| EventId::from_bytes(value.try_into().unwrap()))
- .collect()
+ .collect::<Vec<_>>();
+ let mut public_ids = storage
+ .query_visible(EventQuery::all(EventQueryBounds::first(1000).unwrap()))
+ .await
+ .unwrap()
+ .items()
+ .iter()
+ .map(|item| *item.event().id())
+ .collect::<Vec<_>>();
+ public_ids.sort();
+ assert_eq!(public_ids, ids);
+ ids
}
#[tokio::test]
@@ -133,7 +144,7 @@ async fn visibility_index_matches_reference_winners_deletion_before_target_and_u
.await
.unwrap();
assert!(
- !sqlx::query_scalar::<_, bool>("SELECT ready FROM radroots_runtime_visibility_progress")
+ sqlx::query_scalar::<_, bool>("SELECT ready FROM radroots_runtime_visibility_progress")
.fetch_one(&storage.pool)
.await
.unwrap()
@@ -397,6 +408,13 @@ async fn visibility_index_real_database_close_and_reopen_preserves_bounded_progr
)
.await
.unwrap();
+ // Retain a real file-backed unprepared-source fixture; fresh empty sources
+ // otherwise initialize a ready index under forward20.
+ sqlx::query("DELETE FROM radroots_runtime_visibility_progress WHERE source_generation = ?")
+ .bind(generation().as_bytes().as_slice())
+ .execute(&storage.pool)
+ .await
+ .unwrap();
for event in [profile("first", 10), profile("second", 20)] {
storage.admit(admitted(event)).await.unwrap();
}
@@ -450,7 +468,19 @@ async fn visibility_index_forward_migration_retains_legacy_truth_and_refuses_cor
let storage = SqliteStorage::new(pool.clone(), generation(), EventStoreMode::ReadWrite);
let event = profile("legacy", 10);
if state != 0 {
- storage.admit(raw(event.clone())).await.unwrap();
+ sqlx::query("UPDATE radroots_runtime_source_generations SET sequence_head = 1 WHERE generation = ?")
+ .bind(generation().as_bytes().as_slice()).execute(&pool).await.unwrap();
+ sqlx::query(
+ "INSERT INTO radroots_runtime_events (source_generation, source_sequence, event_id,
+ admission_stage, signed_event, admitted_at_unix_ms, updated_at_unix_ms)
+ VALUES (?, 1, ?, 'raw', ?, 100, 100)",
+ )
+ .bind(generation().as_bytes().as_slice())
+ .bind(event.id().as_bytes().as_slice())
+ .bind(event.raw_json().as_bytes())
+ .execute(&pool)
+ .await
+ .unwrap();
if state == 2 {
sqlx::query("DROP TRIGGER radroots_runtime_events_raw_update_guard")
.execute(&pool)
@@ -472,6 +502,10 @@ async fn visibility_index_forward_migration_retains_legacy_truth_and_refuses_cor
.execute(&pool)
.await
.unwrap();
+ sqlx::raw_sql(migration_sql(20).unwrap())
+ .execute(&pool)
+ .await
+ .unwrap();
assert_eq!(storage.source_token().await.unwrap(), token);
assert_eq!(storage.work.take().codec_inputs, 0);
if state == 2 {
@@ -548,6 +582,21 @@ async fn visibility_index_malformed_progress_or_missing_source_tail_cannot_claim
.execute(&storage.pool)
.await
.unwrap();
+ // Clear only derived membership before deliberately restarting this fixture.
+ for table in [
+ "radroots_runtime_visibility_heads",
+ "radroots_runtime_visibility_members",
+ "radroots_runtime_visibility_pending",
+ ] {
+ let sql = match table {
+ "radroots_runtime_visibility_heads" => "DELETE FROM radroots_runtime_visibility_heads",
+ "radroots_runtime_visibility_members" => {
+ "DELETE FROM radroots_runtime_visibility_members"
+ }
+ _ => "DELETE FROM radroots_runtime_visibility_pending",
+ };
+ sqlx::query(sql).execute(&storage.pool).await.unwrap();
+ }
sqlx::query("DELETE FROM radroots_runtime_visibility_progress")
.execute(&storage.pool)
.await
@@ -571,7 +620,7 @@ async fn visibility_index_malformed_progress_or_missing_source_tail_cannot_claim
#[tokio::test]
async fn visibility_index_memory_and_sqlite_progress_agree_across_same_count_promotion() {
- let storage = store(generation()).await;
+ let storage = super::tests::store(generation()).await;
let memory = radroots_storage::memory::MemoryStorage::new(generation());
let first = profile("first", 10);
let initial = [raw(first.clone()), raw(profile("large", 20))];
diff --git a/crates/storage_sqlite/src/event/visibility_reads.rs b/crates/storage_sqlite/src/event/visibility_reads.rs
@@ -0,0 +1,345 @@
+use super::*;
+use radroots_event::id::Nip01Coordinate;
+use radroots_storage::event::VisibilityIndexBounds;
+
+type Transaction<'a> = sqlx::Transaction<'a, Sqlite>;
+
+impl SqliteStorage {
+ pub(super) async fn visibility_header(
+ &self,
+ transaction: &mut Transaction<'_>,
+ expected: Option<EventSourceToken>,
+ ) -> Result<EventStoreStatus, Error> {
+ let sql = "SELECT source.mutation_epoch, source.sequence_head, progress.ready,
+ progress.replay_epoch, progress.raw_events, progress.verified_events,
+ progress.visible_events
+ FROM radroots_runtime_source_generations AS source
+ LEFT JOIN radroots_runtime_visibility_progress AS progress
+ ON progress.source_generation = source.generation
+ WHERE source.generation = ? AND source.state = 'active'";
+ #[cfg(test)]
+ self.work.request(work::Read::Aggregate, sql);
+ let row = sqlx::query(sql)
+ .bind(self.generation.as_bytes().as_slice())
+ .fetch_optional(&mut **transaction)
+ .await
+ .map_err(map_backend)?
+ .ok_or(Error::SourceGenerationChanged)?;
+ #[cfg(test)]
+ self.work.aggregate_returned();
+ let epoch = u64_from_i64(row.try_get("mutation_epoch").map_err(map_corrupt)?)?;
+ if expected.is_some_and(|token| token != EventSourceToken::new(self.generation, epoch)) {
+ return Err(Error::SourceEpochChanged);
+ }
+ let ready: Option<bool> = row.try_get("ready").map_err(map_corrupt)?;
+ if ready != Some(true) {
+ return Err(Error::VisibilityRebuilding);
+ }
+ let replay = u64_from_i64(row.try_get("replay_epoch").map_err(map_corrupt)?)?;
+ let raw = u64_from_i64(row.try_get("raw_events").map_err(map_corrupt)?)?;
+ let sequence = u64_from_i64(row.try_get("sequence_head").map_err(map_corrupt)?)?;
+ if replay != epoch || raw != sequence {
+ return Err(Error::CorruptStoredEvent);
+ }
+ EventStoreStatus::new(
+ self.generation,
+ self.mode,
+ EventStoreHealth::Available,
+ raw,
+ u64_from_i64(row.try_get("verified_events").map_err(map_corrupt)?)?,
+ u64_from_i64(row.try_get("visible_events").map_err(map_corrupt)?)?,
+ )
+ }
+
+ pub(super) async fn indexed_visible_page(
+ &self,
+ query: EventQuery,
+ expected: Option<EventSourceToken>,
+ ) -> Result<EventPage<StoredVisibleEvent>, Error> {
+ self.validate_cursor(&query)?;
+ let mut transaction = self.pool.begin().await.map_err(map_backend)?;
+ self.visibility_header(&mut transaction, expected).await?;
+ let after = query
+ .bounds()
+ .cursor()
+ .map_or(0, |cursor| cursor.sequence().get());
+ let mut builder = QueryBuilder::<Sqlite>::new(
+ "SELECT member.source_sequence, length(event.signed_event) AS signed_bytes
+ FROM radroots_runtime_visibility_members AS member INDEXED BY ",
+ );
+ if query.event_ids().is_empty() {
+ builder.push("radroots_runtime_visibility_members_page_idx");
+ } else {
+ builder.push("radroots_runtime_visibility_members_event_idx");
+ }
+ builder.push(
+ " JOIN radroots_runtime_events AS event ON event.event_id = member.event_id
+ WHERE member.source_generation = ",
+ );
+ builder.push_bind(self.generation.as_bytes().as_slice());
+ builder
+ .push(" AND member.visible = 1 AND member.source_sequence > ")
+ .push_bind(i64_from_u64(after)?);
+ if !query.event_ids().is_empty() {
+ builder.push(" AND member.event_id IN (");
+ let mut ids = builder.separated(", ");
+ for id in query.event_ids() {
+ ids.push_bind(id.as_bytes().to_vec());
+ }
+ ids.push_unseparated(")");
+ }
+ builder
+ .push(" ORDER BY member.source_sequence LIMIT ")
+ .push_bind(i64::from(query.bounds().limit()) + 1);
+ #[cfg(test)]
+ self.work.request(work::Read::Keys, builder.sql().as_str());
+ let lengths = builder
+ .build()
+ .fetch_all(&mut *transaction)
+ .await
+ .map_err(map_backend)?;
+ #[cfg(test)]
+ self.work.keys_returned(lengths.len());
+ let mut charged = 4096 + (u64::from(query.bounds().limit()) + 1) * 128;
+ let mut items = Vec::new();
+ let mut more = false;
+ for row in lengths {
+ if items.len() == usize::from(query.bounds().limit()) {
+ more = true;
+ break;
+ }
+ let sequence: i64 = row.try_get("source_sequence").map_err(map_corrupt)?;
+ let bytes = u64_from_i64(row.try_get("signed_bytes").map_err(map_corrupt)?)?;
+ let next_charge = charged
+ .checked_add(VisibilityIndexBounds::charge(bytes)?)
+ .ok_or(Error::SpaceInsufficient)?;
+ if next_charge > 16 * 1024 * 1024 {
+ if items.is_empty() {
+ return Err(Error::SpaceInsufficient);
+ }
+ more = true;
+ break;
+ }
+ let sql = "SELECT source_generation, source_sequence, signed_event, admission_stage,
+ admitted_contract_id, admitted_registry_version
+ FROM radroots_runtime_events WHERE source_generation = ? AND source_sequence = ?";
+ #[cfg(test)]
+ self.work.request(work::Read::Bounded, sql);
+ let payload = sqlx::query(sql)
+ .bind(self.generation.as_bytes().as_slice())
+ .bind(sequence)
+ .fetch_one(&mut *transaction)
+ .await
+ .map_err(map_backend)?;
+ #[cfg(test)]
+ self.work.returned(std::slice::from_ref(&payload));
+ let decoded = self.decode_event_row(&payload)?;
+ if decoded.raw_json.len() as u64 != bytes || decoded.stage != AdmissionStage::Visible {
+ return Err(Error::CorruptStoredEvent);
+ }
+ items.push(StoredVisibleEvent::new(decoded.position, decoded.event));
+ charged = next_charge;
+ }
+ let next = more
+ .then(|| items.last().map(StoredVisibleEvent::position))
+ .flatten();
+ transaction.commit().await.map_err(map_backend)?;
+ EventPage::new(self.generation, items, next, query.bounds())
+ }
+
+ pub(super) async fn update_visibility_membership(
+ &self,
+ transaction: &mut Transaction<'_>,
+ position: i64,
+ ) -> Result<(), Error> {
+ let row = sqlx::query(
+ "SELECT event_id, admission_stage, head_coordinate, created_at
+ FROM radroots_runtime_visibility_events WHERE source_generation = ? AND source_sequence = ?")
+ .bind(self.generation.as_bytes().as_slice()).bind(position)
+ .fetch_one(&mut **transaction).await.map_err(map_backend)?;
+ let id: Vec<u8> = row.try_get("event_id").map_err(map_corrupt)?;
+ let stage: String = row.try_get("admission_stage").map_err(map_corrupt)?;
+ sqlx::query(
+ "INSERT INTO radroots_runtime_visibility_members
+ (source_generation, source_sequence, event_id, admission_stage, visible)
+ VALUES (?, ?, ?, ?, 0) ON CONFLICT(source_generation, source_sequence)
+ DO UPDATE SET admission_stage = excluded.admission_stage",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .bind(position)
+ .bind(&id)
+ .bind(stage)
+ .execute(&mut **transaction)
+ .await
+ .map_err(map_backend)?;
+ let coordinate: Option<Vec<u8>> = row.try_get("head_coordinate").map_err(map_corrupt)?;
+ if let Some(coordinate) = coordinate {
+ let old = sqlx::query(
+ "SELECT head.source_sequence, metadata.created_at, metadata.event_id
+ FROM radroots_runtime_visibility_heads AS head
+ JOIN radroots_runtime_visibility_events AS metadata ON metadata.event_id = head.event_id
+ WHERE head.source_generation = ? AND head.head_coordinate = ?")
+ .bind(self.generation.as_bytes().as_slice()).bind(&coordinate)
+ .fetch_optional(&mut **transaction).await.map_err(map_backend)?;
+ let time: Vec<u8> = row.try_get("created_at").map_err(map_corrupt)?;
+ let replace = match &old {
+ None => true,
+ Some(old) => {
+ let old_time: Vec<u8> = old.try_get("created_at").map_err(map_corrupt)?;
+ let old_id: Vec<u8> = old.try_get("event_id").map_err(map_corrupt)?;
+ time > old_time || (time == old_time && id < old_id)
+ }
+ };
+ if replace {
+ sqlx::query(
+ "INSERT INTO radroots_runtime_visibility_heads
+ (source_generation, head_coordinate, source_sequence, event_id) VALUES (?, ?, ?, ?)
+ ON CONFLICT(source_generation, head_coordinate) DO UPDATE SET
+ source_sequence = excluded.source_sequence, event_id = excluded.event_id")
+ .bind(self.generation.as_bytes().as_slice()).bind(coordinate).bind(position).bind(id)
+ .execute(&mut **transaction).await.map_err(map_backend)?;
+ if let Some(old) = old {
+ let previous = old.try_get("source_sequence").map_err(map_corrupt)?;
+ self.refresh_visibility_member(transaction, previous)
+ .await?;
+ }
+ }
+ }
+ self.refresh_visibility_member(transaction, position)
+ .await?;
+ // Immutable targets persist; this queue changes only membership, not source truth.
+ sqlx::query(
+ "INSERT OR IGNORE INTO radroots_runtime_visibility_pending
+ SELECT source_generation, 0, target_event FROM radroots_runtime_visibility_deletion_events
+ WHERE request_id = ?")
+ .bind(row.try_get::<Vec<u8>, _>("event_id").map_err(map_corrupt)?)
+ .execute(&mut **transaction).await.map_err(map_backend)?;
+ sqlx::query(
+ "INSERT OR IGNORE INTO radroots_runtime_visibility_pending
+ SELECT source_generation, 1, CAST(target_coordinate AS BLOB)
+ FROM radroots_runtime_visibility_deletion_addresses WHERE request_id = ?",
+ )
+ .bind(row.try_get::<Vec<u8>, _>("event_id").map_err(map_corrupt)?)
+ .execute(&mut **transaction)
+ .await
+ .map_err(map_backend)?;
+ Ok(())
+ }
+
+ async fn refresh_visibility_member(
+ &self,
+ transaction: &mut Transaction<'_>,
+ position: i64,
+ ) -> Result<(), Error> {
+ sqlx::query(
+ "UPDATE radroots_runtime_visibility_members SET visible = (
+ SELECT candidate.admission_stage = 'visible' AND candidate.ephemeral = 0
+ AND (candidate.head_coordinate IS NULL OR candidate.event_id = (
+ SELECT head.event_id FROM radroots_runtime_visibility_heads AS head
+ WHERE head.source_generation = candidate.source_generation
+ AND head.head_coordinate = candidate.head_coordinate))
+ AND (candidate.deletion_request = 1 OR (
+ NOT EXISTS (SELECT 1 FROM radroots_runtime_visibility_deletion_events AS deletion
+ WHERE deletion.source_generation = candidate.source_generation
+ AND deletion.target_event = candidate.event_id AND deletion.author = candidate.author)
+ AND NOT EXISTS (SELECT 1 FROM radroots_runtime_visibility_deletion_addresses AS deletion
+ WHERE deletion.source_generation = candidate.source_generation
+ AND deletion.target_coordinate = candidate.deletion_coordinate
+ AND deletion.author = candidate.author AND deletion.created_at >= candidate.created_at)))
+ FROM radroots_runtime_visibility_events AS candidate
+ WHERE candidate.source_generation = ? AND candidate.source_sequence = ?)
+ WHERE source_generation = ? AND source_sequence = ?")
+ .bind(self.generation.as_bytes().as_slice()).bind(position)
+ .bind(self.generation.as_bytes().as_slice()).bind(position)
+ .execute(&mut **transaction).await.map_err(map_backend)?;
+ Ok(())
+ }
+
+ pub(super) async fn drain_visibility_keys(
+ &self,
+ transaction: &mut Transaction<'_>,
+ remaining: u16,
+ charged: &mut u64,
+ byte_limit: u64,
+ ) -> Result<u16, Error> {
+ let keys = sqlx::query(
+ "SELECT target_type, length(target_key) AS key_bytes FROM radroots_runtime_visibility_pending
+ WHERE source_generation = ? ORDER BY target_type, target_key LIMIT ?",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .bind(i64::from(remaining))
+ .fetch_all(&mut **transaction)
+ .await
+ .map_err(map_backend)?;
+ let mut processed = 0;
+ for row in keys {
+ let kind: i64 = row.try_get("target_type").map_err(map_corrupt)?;
+ let key_bytes = u64_from_i64(row.try_get("key_bytes").map_err(map_corrupt)?)?;
+ let charge = charged
+ .checked_add(4096 + 16 * key_bytes)
+ .ok_or(Error::SpaceInsufficient)?;
+ if charge > byte_limit {
+ break;
+ }
+ let key: Vec<u8> = sqlx::query_scalar(
+ "SELECT target_key FROM radroots_runtime_visibility_pending
+ WHERE source_generation = ? ORDER BY target_type, target_key LIMIT 1",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .fetch_one(&mut **transaction)
+ .await
+ .map_err(map_backend)?;
+ if key.len() as u64 != key_bytes {
+ return Err(Error::CorruptStoredEvent);
+ }
+ let position = if kind == 0 {
+ sqlx::query_scalar::<_, i64>(
+ "SELECT source_sequence FROM radroots_runtime_visibility_members
+ WHERE source_generation = ? AND event_id = ?",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .bind(&key)
+ .fetch_optional(&mut **transaction)
+ .await
+ .map_err(map_backend)?
+ } else {
+ let coordinate =
+ std::str::from_utf8(&key).map_err(|_| Error::CorruptStoredEvent)?;
+ let coordinate =
+ Nip01Coordinate::parse(coordinate).map_err(|_| Error::CorruptStoredEvent)?;
+ let mut head = vec![u8::from(coordinate.kind() >= 30_000)];
+ head.extend_from_slice(&coordinate.kind().to_be_bytes());
+ head.extend_from_slice(coordinate.pubkey().as_bytes());
+ if coordinate.kind() >= 30_000 {
+ head.extend_from_slice(coordinate.identifier().as_bytes());
+ }
+ sqlx::query_scalar::<_, i64>(
+ "SELECT source_sequence FROM radroots_runtime_visibility_heads
+ WHERE source_generation = ? AND head_coordinate = ?",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .bind(head)
+ .fetch_optional(&mut **transaction)
+ .await
+ .map_err(map_backend)?
+ };
+ if let Some(position) = position {
+ self.refresh_visibility_member(transaction, position)
+ .await?;
+ }
+ sqlx::query(
+ "DELETE FROM radroots_runtime_visibility_pending
+ WHERE source_generation = ? AND target_type = ? AND target_key = ?",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .bind(kind)
+ .bind(key)
+ .execute(&mut **transaction)
+ .await
+ .map_err(map_backend)?;
+ *charged = charge;
+ processed += 1;
+ }
+ Ok(processed)
+ }
+}
diff --git a/crates/storage_sqlite/src/event/visibility_reads_tests.rs b/crates/storage_sqlite/src/event/visibility_reads_tests.rs
@@ -0,0 +1,495 @@
+use super::*;
+use radroots_storage::{event::VisibilityIndexBounds, memory::MemoryStorage};
+
+fn query(limit: u16) -> EventQuery {
+ EventQuery::all(EventQueryBounds::first(limit).unwrap())
+}
+
+fn admission(event: SignedEvent) -> EventAdmission {
+ EventAdmission::visible(
+ tests::observed(event.clone(), 10, None),
+ tests::visible(&event),
+ )
+ .unwrap()
+}
+
+async fn finish(storage: &SqliteStorage) {
+ for _ in 0..1000 {
+ let progress = storage
+ .maintain_visibility(VisibilityIndexBounds::new(64, 16 * 1024 * 1024).unwrap())
+ .await
+ .unwrap();
+ assert!(progress.processed_rows() + progress.processed_keys() <= 64);
+ assert!(progress.charged_bytes() <= 16 * 1024 * 1024);
+ if progress.is_complete() {
+ return;
+ }
+ }
+ panic!("bounded fixture maintenance failed to complete");
+}
+
+#[tokio::test]
+async fn indexed_visibility_refuses_stale_fences_without_full_source_decode() {
+ let generation = SourceGeneration::new([91; 32]).unwrap();
+ let storage = tests::store(generation).await;
+ let token = storage.source_token().await.unwrap();
+ let query = EventQuery::all(EventQueryBounds::first(5).unwrap());
+ assert!(
+ storage
+ .query_visible_at(query.clone(), token)
+ .await
+ .unwrap()
+ .items()
+ .is_empty()
+ );
+ let event = tests::signed_event_with("retained", false, 10, 1, vec![]);
+ let admission = EventAdmission::visible(
+ tests::observed(event.clone(), 10, None),
+ tests::visible(&event),
+ )
+ .unwrap();
+ storage.admit(admission).await.unwrap();
+ storage.work.take();
+ assert_eq!(
+ storage.query_visible_at(query.clone(), token).await,
+ Err(Error::SourceEpochChanged)
+ );
+ let refused = storage.work.take();
+ assert_eq!(refused.codec_inputs, 0);
+ assert_eq!(refused.materialization_requests, 0);
+ let current = storage.source_token().await.unwrap();
+ assert_eq!(
+ storage
+ .query_visible_at(query, current)
+ .await
+ .unwrap()
+ .items()
+ .len(),
+ 1
+ );
+ let page = storage.work.take();
+ assert_eq!(page.codec_inputs, 1);
+ assert_eq!(page.visibility_inputs, 0);
+ assert_eq!(page.materialization_requests, 0);
+ assert_eq!(storage.status().await.unwrap().visible_events(), 1);
+ assert_eq!(
+ storage
+ .query_visible_at(
+ EventQuery::all(EventQueryBounds::first(5).unwrap()),
+ EventSourceToken::new(SourceGeneration::new([90; 32]).unwrap(), 1)
+ )
+ .await,
+ Err(Error::SourceEpochChanged)
+ );
+}
+
+#[tokio::test]
+async fn indexed_visibility_tiny_key_budget_preserves_pending_work_and_cannot_claim_false_ready() {
+ let generation = SourceGeneration::new([98; 32]).unwrap();
+ let storage = tests::store(generation).await;
+ let memory = MemoryStorage::new(generation);
+ let mut tags = Vec::new();
+ for index in 0..65 {
+ let event = tests::signed_event_with(&format!("target {index}"), false, index, 1, vec![]);
+ tags.push(vec!["e".to_owned(), event.id().to_hex()]);
+ let value = admission(event);
+ storage.admit(value.clone()).await.unwrap();
+ memory.admit(value).await.unwrap();
+ }
+ let deletion = admission(tests::signed_event_with("delete", false, 100, 5, tags));
+ storage.admit(deletion.clone()).await.unwrap();
+ memory.admit(deletion).await.unwrap();
+ let tiny = VisibilityIndexBounds::new(1, 4096 + 128).unwrap();
+ assert_eq!(
+ storage.maintain_visibility(tiny).await,
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(
+ memory.maintain_visibility(tiny).await,
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(
+ storage.query_visible(query(5)).await,
+ Err(Error::VisibilityRebuilding)
+ );
+ assert_eq!(memory.status().await, Err(Error::VisibilityRebuilding));
+ let enough = VisibilityIndexBounds::new(1, 4096 + 128 + 4096 + 16 * 32).unwrap();
+ let first = storage.maintain_visibility(enough).await.unwrap();
+ assert_eq!(first, memory.maintain_visibility(enough).await.unwrap());
+ assert_eq!(first.processed_keys(), 1);
+ assert!(!first.is_complete());
+ let done = storage.maintain_visibility(enough).await.unwrap();
+ assert_eq!(done, memory.maintain_visibility(enough).await.unwrap());
+ assert!(done.is_complete());
+ assert_eq!(storage.status().await.unwrap().visible_events(), 1);
+ sqlx::query("UPDATE radroots_runtime_source_generations SET sequence_head = sequence_head + 1 WHERE generation = ?")
+ .bind(generation.as_bytes().as_slice()).execute(&storage.pool).await.unwrap();
+ assert_eq!(
+ storage
+ .maintain_visibility(VisibilityIndexBounds::new(64, 16 * 1024 * 1024).unwrap())
+ .await,
+ Err(Error::CorruptStoredEvent)
+ );
+ assert_eq!(storage.status().await, Err(Error::CorruptStoredEvent));
+}
+
+#[tokio::test]
+async fn indexed_visibility_large_mixed_histories_have_bounded_next_pages_and_truthful_cached_counts()
+ {
+ for size in [10_000_i64, 16_384_i64] {
+ let generation = SourceGeneration::new([92; 32]).unwrap();
+ let storage = tests::store(generation).await;
+ let mut transaction = storage.pool.begin_with("BEGIN IMMEDIATE").await.unwrap();
+ sqlx::query(
+ "UPDATE radroots_runtime_source_generations SET sequence_head = ? WHERE generation = ?",
+ )
+ .bind(size)
+ .bind(generation.as_bytes().as_slice())
+ .execute(&mut *transaction)
+ .await
+ .unwrap();
+ for index in 0..size {
+ let profile = index % 4 == 0;
+ let unsupported = index % 4 == 3;
+ let content = if profile {
+ format!(r#"{{"name":"Farm {index}"}}"#)
+ } else {
+ format!("retained {index}")
+ };
+ let event = tests::signed_event_with(
+ &content,
+ false,
+ index as u64 + 100,
+ if profile || unsupported { 0 } else { 1 },
+ vec![],
+ );
+ let stage = match index % 4 {
+ 2 => "raw",
+ 3 => "verified",
+ _ => "visible",
+ };
+ let contract = if stage == "visible" {
+ Some(
+ tests::visible(&event)
+ .admitted_event()
+ .validated_event()
+ .contract_id(),
+ )
+ } else {
+ None
+ };
+ sqlx::query("INSERT INTO radroots_runtime_events
+ (source_generation, source_sequence, event_id, admission_stage, signed_event,
+ admitted_at_unix_ms, updated_at_unix_ms, admitted_contract_id, admitted_registry_version)
+ VALUES (?, ?, ?, ?, ?, 10, 10, ?, ?)")
+ .bind(generation.as_bytes().as_slice()).bind(index + 1).bind(event.id().as_bytes().as_slice())
+ .bind(stage).bind(event.raw_json().as_bytes()).bind(contract)
+ .bind(contract.map(|_| i64::from(radroots_event::contract::RegistryVersion::CURRENT.get())))
+ .execute(&mut *transaction).await.unwrap();
+ }
+ transaction.commit().await.unwrap();
+ assert_eq!(storage.status().await, Err(Error::VisibilityRebuilding));
+ assert_eq!(
+ storage.query_visible(query(5)).await,
+ Err(Error::VisibilityRebuilding)
+ );
+ finish(&storage).await;
+ let oracle = storage.rebuild_visibility().await.unwrap();
+ let status = storage.status().await.unwrap();
+ assert_eq!(status.raw_events(), size as u64);
+ assert_eq!(status.verified_events(), (size * 3 / 4) as u64);
+ assert_eq!(
+ status.visible_events(),
+ oracle.visible_event_ids().len() as u64
+ );
+ storage.work.take();
+ let token = storage.source_token().await.unwrap();
+ let mut bounds = EventQueryBounds::first(1000).unwrap();
+ let mut returned = std::collections::BTreeSet::new();
+ loop {
+ let page = storage
+ .query_visible_at(EventQuery::all(bounds), token)
+ .await
+ .unwrap();
+ let counts = storage.work.take();
+ assert_eq!(counts.materialization_requests, 0);
+ assert_eq!(counts.visibility_inputs, 0);
+ assert!(counts.key_rows_returned <= 1001);
+ assert_eq!(counts.codec_inputs, page.items().len() as u64);
+ for item in page.items() {
+ assert!(returned.insert(*item.event().id()));
+ }
+ match page.next_cursor() {
+ Some(cursor) => bounds = EventQueryBounds::first(1000).unwrap().after(cursor),
+ None => break,
+ }
+ }
+ assert_eq!(
+ returned,
+ oracle.visible_event_ids().iter().copied().collect()
+ );
+ let first = storage.query_visible(query(5)).await.unwrap();
+ storage.work.take();
+ let next = storage
+ .query_visible(EventQuery::all(
+ EventQueryBounds::first(5)
+ .unwrap()
+ .after(first.next_cursor().unwrap()),
+ ))
+ .await
+ .unwrap();
+ let counts = storage.work.take();
+ assert_eq!(next.items().len(), 5);
+ assert_eq!(counts.key_rows_returned, 6);
+ assert_eq!(counts.codec_inputs, 5);
+ assert_eq!(counts.materialization_requests, 0);
+ let sql = storage.work.template(work::Read::Keys);
+ let mut builder = QueryBuilder::<Sqlite>::new("EXPLAIN QUERY PLAN ");
+ builder.push(sql);
+ let plans = builder
+ .build()
+ .bind(generation.as_bytes().as_slice())
+ .bind(i64_from_u64(first.next_cursor().unwrap().sequence().get()).unwrap())
+ .bind(6_i64)
+ .fetch_all(&storage.pool)
+ .await
+ .unwrap();
+ assert!(plans.iter().any(|row| {
+ row.get::<String, _>("detail")
+ .contains("visibility_members_page_idx")
+ }));
+ assert!(
+ !plans
+ .iter()
+ .any(|row| row.get::<String, _>("detail").contains("USE TEMP B-TREE"))
+ );
+ storage.work.take();
+ assert_eq!(storage.status().await.unwrap(), status);
+ let cached = storage.work.take();
+ assert_eq!(cached.aggregate_rows_returned, 1);
+ assert_eq!(cached.codec_inputs, 0);
+ storage.close().await.unwrap();
+ }
+}
+
+#[tokio::test]
+async fn indexed_visibility_deletion_queue_is_bounded_restartable_and_matches_memory() {
+ let generation = SourceGeneration::new([93; 32]).unwrap();
+ let storage = tests::store(generation).await;
+ let memory = MemoryStorage::new(generation);
+ let mut tags = Vec::new();
+ for index in 0..133 {
+ let event = tests::signed_event_with(&format!("target {index}"), false, index, 1, vec![]);
+ tags.push(vec!["e".to_owned(), event.id().to_hex()]);
+ let value = admission(event);
+ storage.admit(value.clone()).await.unwrap();
+ memory.admit(value).await.unwrap();
+ }
+ let deletion = admission(tests::signed_event_with("delete", false, 200, 5, tags));
+ storage.admit(deletion.clone()).await.unwrap();
+ memory.admit(deletion).await.unwrap();
+ assert_eq!(storage.status().await, Err(Error::VisibilityRebuilding));
+ assert_eq!(
+ memory.query_visible(query(5)).await,
+ Err(Error::VisibilityRebuilding)
+ );
+ let reopened = SqliteStorage::new(storage.pool.clone(), generation, EventStoreMode::ReadWrite);
+ let bounds = VisibilityIndexBounds::new(64, 16 * 1024 * 1024).unwrap();
+ let first = reopened.maintain_visibility(bounds).await.unwrap();
+ assert_eq!(first, memory.maintain_visibility(bounds).await.unwrap());
+ assert_eq!(first.processed_rows(), 0);
+ assert_eq!(first.processed_keys(), 64);
+ assert!(!first.is_complete());
+ // Admission while target work is unfinished must retain both cursors.
+ let later = admission(tests::signed_event_with("later", false, 300, 1, vec![]));
+ reopened.admit(later.clone()).await.unwrap();
+ memory.admit(later).await.unwrap();
+ for _ in 0..4 {
+ let progress = reopened.maintain_visibility(bounds).await.unwrap();
+ assert_eq!(progress, memory.maintain_visibility(bounds).await.unwrap());
+ assert!(progress.processed_rows() + progress.processed_keys() <= 64);
+ if progress.is_complete() {
+ break;
+ }
+ }
+ assert_eq!(
+ reopened.status().await.unwrap(),
+ memory.status().await.unwrap()
+ );
+ assert_eq!(reopened.status().await.unwrap().visible_events(), 2);
+ assert_eq!(
+ reopened.query_visible(query(5)).await.unwrap(),
+ memory.query_visible(query(5)).await.unwrap()
+ );
+ assert_eq!(
+ reopened.rebuild_visibility().await.unwrap(),
+ memory.rebuild_visibility().await.unwrap()
+ );
+}
+
+#[tokio::test]
+async fn indexed_visibility_partial_byte_pages_preserve_continuation_and_id_navigation() {
+ let generation = SourceGeneration::new([94; 32]).unwrap();
+ let storage = tests::store(generation).await;
+ let memory = MemoryStorage::new(generation);
+ let mut ids = Vec::new();
+ for index in 0..12 {
+ let event = tests::signed_event_with(
+ &format!("{index}{}", "x".repeat(120_000)),
+ false,
+ index,
+ 1,
+ vec![],
+ );
+ ids.push(*event.id());
+ let value = admission(event);
+ storage.admit(value.clone()).await.unwrap();
+ memory.admit(value).await.unwrap();
+ }
+ let first = storage.query_visible(query(12)).await.unwrap();
+ assert_eq!(first, memory.query_visible(query(12)).await.unwrap());
+ assert!(first.items().len() < 12 && !first.items().is_empty());
+ let next_bounds = EventQueryBounds::first(12)
+ .unwrap()
+ .after(first.next_cursor().unwrap());
+ let next = storage
+ .query_visible(EventQuery::all(next_bounds))
+ .await
+ .unwrap();
+ assert_eq!(first.items().len() + next.items().len(), 12);
+ assert!(next.next_cursor().is_none());
+ storage.work.take();
+ let selected =
+ EventQuery::for_ids(EventQueryBounds::first(5).unwrap(), vec![ids[0], ids[11]]).unwrap();
+ let page = storage.query_visible(selected.clone()).await.unwrap();
+ assert_eq!(page, memory.query_visible(selected).await.unwrap());
+ let counts = storage.work.take();
+ assert_eq!(counts.key_rows_returned, 2);
+ assert_eq!(counts.codec_inputs, 2);
+ assert!(
+ storage
+ .work
+ .template(work::Read::Keys)
+ .contains("INDEXED BY radroots_runtime_visibility_members_event_idx")
+ );
+}
+
+#[tokio::test]
+async fn indexed_visibility_fault_after_membership_rolls_back_source_counts_and_readiness() {
+ let storage = tests::store(SourceGeneration::new([95; 32]).unwrap()).await;
+ let token = storage.source_token().await.unwrap();
+ let before = storage.status().await.unwrap();
+ sqlx::raw_sql(
+ "CREATE TEMP TRIGGER fail_membership AFTER INSERT ON radroots_runtime_visibility_members
+ BEGIN SELECT RAISE(ABORT, 'fixture membership fault'); END",
+ )
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ let event = tests::signed_event_with("failure", false, 10, 1, vec![]);
+ assert_eq!(
+ storage.admit(admission(event.clone())).await,
+ Err(Error::BackendUnavailable)
+ );
+ assert_eq!(storage.source_token().await.unwrap(), token);
+ assert_eq!(storage.status().await.unwrap(), before);
+ assert!(
+ storage
+ .query_raw(query(5))
+ .await
+ .unwrap()
+ .items()
+ .is_empty()
+ );
+ sqlx::query("DROP TRIGGER fail_membership")
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ storage.admit(admission(event)).await.unwrap();
+ assert_eq!(storage.status().await.unwrap().visible_events(), 1);
+}
+
+#[tokio::test]
+async fn indexed_visibility_cancelled_pool_wait_has_no_payload_or_maintenance_effects() {
+ use std::task::{Context, Poll, Waker};
+ let storage = tests::store(SourceGeneration::new([96; 32]).unwrap()).await;
+ storage
+ .admit(admission(tests::signed_event_with(
+ "retained",
+ false,
+ 10,
+ 1,
+ vec![],
+ )))
+ .await
+ .unwrap();
+ let token = storage.source_token().await.unwrap();
+ storage.work.take();
+ let held = storage.pool.acquire().await.unwrap();
+ let mut future = storage.query_visible_at(query(5), token);
+ assert!(matches!(
+ future
+ .as_mut()
+ .poll(&mut Context::from_waker(Waker::noop())),
+ Poll::Pending
+ ));
+ drop(future);
+ drop(held);
+ assert_eq!(storage.work.take(), work::Counts::default());
+ assert_eq!(storage.source_token().await.unwrap(), token);
+ assert_eq!(
+ storage
+ .query_visible_at(query(5), token)
+ .await
+ .unwrap()
+ .items()
+ .len(),
+ 1
+ );
+}
+
+#[tokio::test]
+async fn indexed_visibility_concurrent_wal_admission_returns_one_fenced_head_or_typed_stale() {
+ let directory = tempfile::tempdir().unwrap();
+ let paths = crate::Paths::from_directory(directory.path()).unwrap();
+ let storage = SqliteStorage::open(
+ crate::OpenOptions::new(paths, crate::OpenMode::Create)
+ .with_source_generation(SourceGeneration::new([97; 32]).unwrap(), 100)
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ let mut current = tests::signed_event_with(r#"{"name":"initial"}"#, false, 1, 0, vec![]);
+ storage.admit(admission(current.clone())).await.unwrap();
+ for index in 2..18 {
+ let next = tests::signed_event_with(
+ &format!(r#"{{"name":"Farm {index}"}}"#),
+ false,
+ index,
+ 0,
+ vec![],
+ );
+ let token = storage.source_token().await.unwrap();
+ let (read, write) = tokio::join!(
+ storage.query_visible_at(query(5), token),
+ storage.admit(admission(next.clone()))
+ );
+ write.unwrap();
+ match read {
+ Ok(page) => {
+ assert_eq!(page.items().len(), 1);
+ assert_eq!(page.items()[0].event().id(), current.id());
+ }
+ Err(Error::SourceEpochChanged) => {}
+ Err(error) => panic!("unexpected fenced read failure: {error}"),
+ }
+ let page = storage
+ .query_visible_at(query(5), storage.source_token().await.unwrap())
+ .await
+ .unwrap();
+ assert_eq!(page.items().len(), 1);
+ assert_eq!(page.items()[0].event().id(), next.id());
+ current = next;
+ }
+ storage.close().await.unwrap();
+}
diff --git a/crates/storage_sqlite/src/event/work.rs b/crates/storage_sqlite/src/event/work.rs
@@ -5,6 +5,8 @@ use std::sync::Mutex;
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, serde::Serialize)]
pub(super) struct Counts {
pub bounded_requests: u64,
+ pub key_requests: u64,
+ pub key_rows_returned: u64,
pub materialization_requests: u64,
pub aggregate_requests: u64,
pub successful_fetches: u64,
@@ -22,13 +24,14 @@ pub(super) enum Read {
Bounded,
Materialize,
Aggregate,
+ Keys,
}
#[derive(Default)]
pub(super) struct Observer {
counts: Mutex<Counts>,
- // Three placeholder-only shapes; bind values and payloads are never retained.
- templates: Mutex<[Option<String>; 3]>,
+ // Placeholder-only shapes; bind values and payloads are never retained.
+ templates: Mutex<[Option<String>; 4]>,
}
impl Observer {
@@ -38,10 +41,17 @@ impl Observer {
Read::Bounded => counts.bounded_requests += 1,
Read::Materialize => counts.materialization_requests += 1,
Read::Aggregate => counts.aggregate_requests += 1,
+ Read::Keys => counts.key_requests += 1,
}
self.templates.lock().expect("test query templates")[kind as usize] = Some(sql.to_owned());
}
+ pub(super) fn keys_returned(&self, rows: usize) {
+ let mut counts = self.counts.lock().expect("test observer");
+ counts.successful_fetches += 1;
+ counts.key_rows_returned += rows as u64;
+ }
+
pub(super) fn returned(&self, rows: &[SqliteRow]) {
let bytes = rows
.iter()
diff --git a/crates/storage_sqlite/src/event/work_tests.rs b/crates/storage_sqlite/src/event/work_tests.rs
@@ -54,7 +54,7 @@ async fn plan(storage: &SqliteStorage, kind: Read) -> Vec<String> {
let mut query = builder
.build()
.bind(storage.generation.as_bytes().as_slice());
- if matches!(kind, Read::Bounded) {
+ if matches!(kind, Read::Bounded | Read::Keys) {
query = query.bind(0_i64).bind(6_i64);
}
query
@@ -86,10 +86,20 @@ async fn actual_returned_and_decoded_work_distinguishes_bounded_and_full_reads()
let page = storage.query_visible(first()).await.unwrap();
assert_eq!(page.items().len(), 5);
let full = storage.work.take();
- assert_materialized(full, retained, bytes);
- assert_eq!(full.successful_fetches, 1);
- let full_plan = plan(&storage, Read::Materialize).await;
+ assert_eq!(full.materialization_requests, 0);
+ assert_eq!(full.visibility_inputs, 0);
+ assert_eq!(full.key_requests, 1);
+ assert_eq!(full.key_rows_returned, 6);
+ assert_eq!(full.codec_inputs, 5);
+ assert_eq!(full.event_rows_returned, 5);
+ assert_eq!(full.aggregate_requests, 1);
+ let full_plan = plan(&storage, Read::Keys).await;
assert!(!bounded_plan.is_empty() && !full_plan.is_empty());
+ assert!(
+ full_plan
+ .iter()
+ .any(|line| line.contains("visibility_members_page_idx"))
+ );
println!(
"{}",
serde_json::json!({"scenario":"bounded_vs_materialized", "retained_events":retained,
@@ -109,10 +119,13 @@ async fn status_and_rebuild_measure_actual_full_source_materialization() {
assert_eq!(status.raw_events(), 32);
assert_eq!(status.verified_events(), 24);
let counts = storage.work.take();
- assert_materialized(counts, 32, bytes);
+ assert_eq!(counts.materialization_requests, 0);
+ assert_eq!(counts.visibility_inputs, 0);
+ assert_eq!(counts.codec_inputs, 0);
+ assert_eq!(counts.event_rows_returned, 0);
assert_eq!(counts.aggregate_requests, 1);
assert_eq!(counts.aggregate_rows_returned, 1);
- assert_eq!(counts.successful_fetches, 2);
+ assert_eq!(counts.successful_fetches, 1);
let aggregate_plan = plan(&storage, Read::Aggregate).await;
let visibility = storage.rebuild_visibility().await.unwrap();
let rebuilt = storage.work.take();
diff --git a/crates/storage_sqlite/src/migration.rs b/crates/storage_sqlite/src/migration.rs
@@ -469,7 +469,7 @@ fn schema_metadata_error(source: &sqlx::Error, database: &'static str) -> Error
fn validate_plan(plan: &MigrationPlan) -> Result<(), Error> {
let valid = plan.minimum_version > 0
&& plan.minimum_version <= plan.current_version
- && plan.current_version <= 19
+ && plan.current_version <= 20
&& plan.steps.len() == usize::try_from(plan.current_version).unwrap_or(usize::MAX)
&& plan
.steps
@@ -595,6 +595,7 @@ const fn set_user_version_sql(version: u32) -> Option<&'static str> {
17 => Some("PRAGMA user_version = 17"),
18 => Some("PRAGMA user_version = 18"),
19 => Some("PRAGMA user_version = 19"),
+ 20 => Some("PRAGMA user_version = 20"),
_ => None,
}
}
@@ -879,8 +880,8 @@ mod tests {
.await
.expect("application id");
let newer_version = runtime::CURRENT_VERSION + 1;
- assert_eq!(newer_version, 20);
- sqlx::raw_sql("PRAGMA user_version = 20")
+ assert_eq!(newer_version, 21);
+ sqlx::raw_sql("PRAGMA user_version = 21")
.execute(&mut newer)
.await
.expect("newer version");
diff --git a/crates/storage_sqlite/src/migration/runtime/0020_indexed_visibility_reads.up.sql b/crates/storage_sqlite/src/migration/runtime/0020_indexed_visibility_reads.up.sql
@@ -0,0 +1,74 @@
+ALTER TABLE radroots_runtime_visibility_progress
+ADD COLUMN raw_events INTEGER NOT NULL DEFAULT 0 CHECK (raw_events >= 0);
+ALTER TABLE radroots_runtime_visibility_progress
+ADD COLUMN verified_events INTEGER NOT NULL DEFAULT 0 CHECK (verified_events >= 0);
+ALTER TABLE radroots_runtime_visibility_progress
+ADD COLUMN visible_events INTEGER NOT NULL DEFAULT 0 CHECK (visible_events >= 0);
+
+-- A one-time forward upgrade rebuilds membership through bounded source work.
+-- No event payload is decoded or updated by migration.
+UPDATE radroots_runtime_visibility_progress
+SET backfill_target = (SELECT sequence_head FROM radroots_runtime_source_generations
+ WHERE generation = source_generation),
+ backfill_sequence = 0, replay_epoch = 0,
+ replay_sequence = 9223372036854775807, ready = 0;
+
+CREATE TABLE radroots_runtime_visibility_members (
+ source_generation BLOB NOT NULL REFERENCES radroots_runtime_visibility_progress(source_generation),
+ source_sequence INTEGER NOT NULL CHECK (source_sequence > 0),
+ event_id BLOB NOT NULL UNIQUE REFERENCES radroots_runtime_visibility_events(event_id),
+ admission_stage TEXT NOT NULL CHECK (admission_stage IN ('raw', 'verified', 'visible')),
+ visible INTEGER NOT NULL CHECK (visible IN (0, 1)),
+ PRIMARY KEY (source_generation, source_sequence)
+) STRICT, WITHOUT ROWID;
+
+CREATE INDEX radroots_runtime_visibility_members_page_idx
+ON radroots_runtime_visibility_members(source_generation, visible, source_sequence);
+
+CREATE UNIQUE INDEX radroots_runtime_visibility_members_event_idx
+ON radroots_runtime_visibility_members(source_generation, event_id);
+
+CREATE TABLE radroots_runtime_visibility_heads (
+ source_generation BLOB NOT NULL REFERENCES radroots_runtime_visibility_progress(source_generation),
+ head_coordinate BLOB NOT NULL,
+ source_sequence INTEGER NOT NULL CHECK (source_sequence > 0),
+ event_id BLOB NOT NULL REFERENCES radroots_runtime_visibility_events(event_id),
+ PRIMARY KEY (source_generation, head_coordinate)
+) STRICT, WITHOUT ROWID;
+
+CREATE TABLE radroots_runtime_visibility_pending (
+ source_generation BLOB NOT NULL REFERENCES radroots_runtime_visibility_progress(source_generation),
+ target_type INTEGER NOT NULL CHECK (target_type IN (0, 1)),
+ target_key BLOB NOT NULL CHECK (
+ (target_type = 0 AND length(target_key) = 32) OR
+ (target_type = 1 AND length(target_key) BETWEEN 1 AND 4096)),
+ PRIMARY KEY (source_generation, target_type, target_key)
+) STRICT, WITHOUT ROWID;
+
+CREATE TRIGGER radroots_runtime_visibility_members_count_insert
+AFTER INSERT ON radroots_runtime_visibility_members
+BEGIN
+ UPDATE radroots_runtime_visibility_progress
+ SET raw_events = raw_events + 1,
+ verified_events = verified_events + (NEW.admission_stage != 'raw'),
+ visible_events = visible_events + NEW.visible
+ WHERE source_generation = NEW.source_generation;
+END;
+
+CREATE TRIGGER radroots_runtime_visibility_members_count_update
+AFTER UPDATE OF admission_stage, visible ON radroots_runtime_visibility_members
+BEGIN
+ UPDATE radroots_runtime_visibility_progress
+ SET verified_events = verified_events + (NEW.admission_stage != 'raw') - (OLD.admission_stage != 'raw'),
+ visible_events = visible_events + NEW.visible - OLD.visible
+ WHERE source_generation = NEW.source_generation;
+END;
+
+CREATE TRIGGER radroots_runtime_visibility_empty_source
+AFTER INSERT ON radroots_runtime_source_generations
+WHEN NEW.sequence_head = 0 AND NEW.mutation_epoch = 0
+BEGIN
+ INSERT INTO radroots_runtime_visibility_progress
+ (source_generation, backfill_target, backfill_sequence, replay_epoch, replay_sequence, ready)
+ VALUES (NEW.generation, 0, 0, 0, 9223372036854775807, 1);
+END;
diff --git a/crates/storage_sqlite/src/migration/runtime/mod.rs b/crates/storage_sqlite/src/migration/runtime/mod.rs
@@ -6,7 +6,7 @@
/// Lowest runtime schema version this package can recognize.
pub const MINIMUM_VERSION: u32 = 1;
/// Current runtime schema version created by this package.
-pub const CURRENT_VERSION: u32 = 19;
+pub const CURRENT_VERSION: u32 = 20;
const RUNTIME_V1_SQL: &str = include_str!("0001_runtime.up.sql");
const CANONICAL_EVENT_STORAGE_V2_SQL: &str = include_str!("0002_canonical_event_storage.up.sql");
@@ -38,6 +38,8 @@ const EVENT_MUTATION_EPOCH_V18_SQL: &str = include_str!("0018_event_mutation_epo
const CANONICAL_VISIBILITY_INDEX_V19_SQL: &str =
include_str!("0019_canonical_visibility_index.up.sql");
+const INDEXED_VISIBILITY_READS_V20_SQL: &str = include_str!("0020_indexed_visibility_reads.up.sql");
+
/// Stable, non-SQL description of one forward runtime migration.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct MigrationDescriptor {
@@ -1094,6 +1096,116 @@ const RUNTIME_V19_OBJECTS: &[&str] = &[
"radroots_runtime_visibility_progress",
];
+const RUNTIME_V20_OBJECTS: &[&str] = &[
+ "radroots_runtime_atomic_commits",
+ "radroots_runtime_authored_artifacts",
+ "radroots_runtime_authored_artifacts_admission_ready_idx",
+ "radroots_runtime_authored_artifacts_signed_fact_guard",
+ "radroots_runtime_authored_artifacts_signing_ready_idx",
+ "radroots_runtime_authored_atomic_commits",
+ "radroots_runtime_authored_atomic_commits_delete_guard",
+ "radroots_runtime_authored_atomic_commits_update_guard",
+ "radroots_runtime_authored_atomic_target_phase_idx",
+ "radroots_runtime_authored_delivery_attempts",
+ "radroots_runtime_authored_delivery_claims",
+ "radroots_runtime_authored_delivery_claims_delete_guard",
+ "radroots_runtime_authored_delivery_claims_update_guard",
+ "radroots_runtime_authored_delivery_facts",
+ "radroots_runtime_authored_delivery_facts_delete_guard",
+ "radroots_runtime_authored_delivery_facts_update_guard",
+ "radroots_runtime_authored_delivery_plans",
+ "radroots_runtime_authored_delivery_ready_idx",
+ "radroots_runtime_authored_delivery_reconciliations",
+ "radroots_runtime_authored_delivery_reconciliations_delete_guard",
+ "radroots_runtime_authored_delivery_reconciliations_update_guard",
+ "radroots_runtime_authored_delivery_stop_guard",
+ "radroots_runtime_authored_delivery_targets",
+ "radroots_runtime_authored_draft_author_head_idx",
+ "radroots_runtime_authored_draft_revisions",
+ "radroots_runtime_authored_draft_revisions_delete_guard",
+ "radroots_runtime_authored_draft_revisions_update_guard",
+ "radroots_runtime_authored_draft_scope_head_idx",
+ "radroots_runtime_authored_migration_evidence",
+ "radroots_runtime_authored_migration_evidence_delete_guard",
+ "radroots_runtime_authored_migration_evidence_update_guard",
+ "radroots_runtime_authored_operations",
+ "radroots_runtime_delivery_evidence",
+ "radroots_runtime_delivery_evidence_item_idx",
+ "radroots_runtime_event_index_checkpoints",
+ "radroots_runtime_event_index_manifests",
+ "radroots_runtime_event_index_shards",
+ "radroots_runtime_event_provenance",
+ "radroots_runtime_event_provenance_observed_idx",
+ "radroots_runtime_events",
+ "radroots_runtime_events_admission_idx",
+ "radroots_runtime_events_contract_metadata_guard",
+ "radroots_runtime_events_contract_metadata_insert_guard",
+ "radroots_runtime_events_delete_guard",
+ "radroots_runtime_events_event_id_idx",
+ "radroots_runtime_events_mutation_idx",
+ "radroots_runtime_events_mutation_insert",
+ "radroots_runtime_events_mutation_update",
+ "radroots_runtime_events_raw_update_guard",
+ "radroots_runtime_journal_idempotency_idx",
+ "radroots_runtime_journal_operations",
+ "radroots_runtime_journal_recovery_idx",
+ "radroots_runtime_legacy_event_staging",
+ "radroots_runtime_legacy_event_staging_delete_guard",
+ "radroots_runtime_legacy_event_staging_insert_guard",
+ "radroots_runtime_legacy_event_staging_update_guard",
+ "radroots_runtime_legacy_import_commit_delete_guard",
+ "radroots_runtime_legacy_import_commit_update_guard",
+ "radroots_runtime_legacy_import_commits",
+ "radroots_runtime_legacy_import_delete_guard",
+ "radroots_runtime_legacy_import_identity_guard",
+ "radroots_runtime_legacy_import_member_delete_guard",
+ "radroots_runtime_legacy_import_member_identity_guard",
+ "radroots_runtime_legacy_import_member_state_guard",
+ "radroots_runtime_legacy_import_members",
+ "radroots_runtime_legacy_import_state_guard",
+ "radroots_runtime_legacy_import_state_idx",
+ "radroots_runtime_legacy_imports",
+ "radroots_runtime_legacy_outbox_staging",
+ "radroots_runtime_legacy_outbox_staging_delete_guard",
+ "radroots_runtime_legacy_outbox_staging_insert_guard",
+ "radroots_runtime_legacy_outbox_staging_parent_idx",
+ "radroots_runtime_legacy_outbox_staging_update_guard",
+ "radroots_runtime_outbox_items",
+ "radroots_runtime_outbox_operation_idx",
+ "radroots_runtime_outbox_ready_idx",
+ "radroots_runtime_outbox_targets",
+ "radroots_runtime_projection_checkpoints",
+ "radroots_runtime_projection_documents",
+ "radroots_runtime_projection_invalidations",
+ "radroots_runtime_projection_rebuilds",
+ "radroots_runtime_projection_rebuilds_stage_idx",
+ "radroots_runtime_projection_snapshots",
+ "radroots_runtime_projection_snapshots_created_idx",
+ "radroots_runtime_projection_snapshots_update_guard",
+ "radroots_runtime_source_generations",
+ "radroots_runtime_source_generations_active_idx",
+ "radroots_runtime_source_generations_delete_guard",
+ "radroots_runtime_source_generations_identity_guard",
+ "radroots_runtime_source_generations_mutation_guard",
+ "radroots_runtime_source_generations_sequence_guard",
+ "radroots_runtime_visibility_deletion_addresses",
+ "radroots_runtime_visibility_deletion_addresses_request_idx",
+ "radroots_runtime_visibility_deletion_events",
+ "radroots_runtime_visibility_deletion_events_request_idx",
+ "radroots_runtime_visibility_empty_source",
+ "radroots_runtime_visibility_events",
+ "radroots_runtime_visibility_events_head_idx",
+ "radroots_runtime_visibility_events_stage_idx",
+ "radroots_runtime_visibility_heads",
+ "radroots_runtime_visibility_members",
+ "radroots_runtime_visibility_members_count_insert",
+ "radroots_runtime_visibility_members_count_update",
+ "radroots_runtime_visibility_members_event_idx",
+ "radroots_runtime_visibility_members_page_idx",
+ "radroots_runtime_visibility_pending",
+ "radroots_runtime_visibility_progress",
+];
+
/// Ordered, immutable runtime migration plan.
pub const MIGRATIONS: &[MigrationDescriptor] = &[
MigrationDescriptor {
@@ -1210,6 +1322,12 @@ pub const MIGRATIONS: &[MigrationDescriptor] = &[
up_sha256: "fd574ba0459de5a8c36033d5332dbd647888ef35e0213bc4ee88656ac2ef49bb",
owned_objects: RUNTIME_V19_OBJECTS,
},
+ MigrationDescriptor {
+ version: 20,
+ name: "indexed_visibility_reads",
+ up_sha256: "914ed79cfb5073f3031413ebc21d64c11f3644e46cb38ef32cd85126b00a95cc",
+ owned_objects: RUNTIME_V20_OBJECTS,
+ },
];
pub(crate) const fn migration_sql(version: u32) -> Option<&'static str> {
@@ -1233,6 +1351,7 @@ pub(crate) const fn migration_sql(version: u32) -> Option<&'static str> {
17 => Some(AUTHORED_DELIVERY_RECONCILIATION_V17_SQL),
18 => Some(EVENT_MUTATION_EPOCH_V18_SQL),
19 => Some(CANONICAL_VISIBILITY_INDEX_V19_SQL),
+ 20 => Some(INDEXED_VISIBILITY_READS_V20_SQL),
_ => None,
}
}
@@ -1276,8 +1395,8 @@ mod tests {
fn migration_plan_matches_governed_snapshot() {
let snapshot = toml::from_str::<PlanSnapshot>(PLAN_SNAPSHOT).expect("valid snapshot");
assert_eq!(MINIMUM_VERSION, 1);
- assert_eq!(CURRENT_VERSION, 19);
- assert_eq!(MIGRATIONS.len(), 19);
+ assert_eq!(CURRENT_VERSION, 20);
+ assert_eq!(MIGRATIONS.len(), 20);
let migration = MIGRATIONS[8];
assert_eq!(snapshot.schema_version, 1);
assert_eq!(snapshot.database, "runtime.sqlite");
diff --git a/crates/storage_sqlite/src/migration_preflight_tests.rs b/crates/storage_sqlite/src/migration_preflight_tests.rs
@@ -152,6 +152,21 @@ async fn compatible_pending_pair_migrates_and_reopens_with_same_generation() {
SqliteStorage::open(OpenOptions::new(paths.clone(), OpenMode::ReadWriteExisting))
.await
.expect("supported pair");
+ if matches!(
+ store.status().await,
+ Err(radroots_storage::Error::VisibilityRebuilding)
+ ) {
+ assert!(
+ store
+ .maintain_visibility(
+ radroots_storage::event::VisibilityIndexBounds::new(64, 16 * 1024 * 1024)
+ .unwrap()
+ )
+ .await
+ .expect("explicit empty-index initialization")
+ .is_complete()
+ );
+ }
assert_eq!(
store
.status()