lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

commit 690b6b17f2040a879fb33ba643c6a4d7e44ebe76
parent e6561a0cc7a8f6fde416d37dd15c9612ce312770
Author: triesap <tyson@radroots.org>
Date:   Sat, 10 Oct 2026 04:07:43 +0000

storage: prepare canonical visibility indexes in resumable chunks

- Add validated work bounds and passive preparation progress to the event SPI.
- Track canonical mutation obligations within a forward SQLite migration.
- Retain backfill progress and replay later changes within row and byte bounds.
- Verify provider parity, rollback, durable reopen and API and coverage gates.

Diffstat:
Mcontracts/api_baselines/radroots_storage.txt | 19+++++++++++++++++++
Mcontracts/api_baselines/radroots_storage_sqlite.txt | 1+
Acontracts/architecture/decisions/canonical_visibility_index.v1.json | 42++++++++++++++++++++++++++++++++++++++++++
Mcrates/storage/src/error.rs | 2++
Mcrates/storage/src/event.rs | 15+++++++++++++++
Mcrates/storage/src/event/visibility.rs | 80+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acrates/storage/src/event/visibility_index.rs | 120+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mcrates/storage/src/memory.rs | 26++++++++++++++++++++++++++
Acrates/storage/src/memory_visibility_index.rs | 206+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mcrates/storage/tests/event_store.rs | 6++++++
Mcrates/storage_sqlite/src/event/mod.rs | 10++++++++++
Acrates/storage_sqlite/src/event/visibility_index.rs | 225+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acrates/storage_sqlite/src/event/visibility_index_tests.rs | 626+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mcrates/storage_sqlite/src/migration.rs | 7++++---
Acrates/storage_sqlite/src/migration/runtime/0019_canonical_visibility_index.up.sql | 111+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mcrates/storage_sqlite/src/migration/runtime/mod.rs | 118+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
16 files changed, 1608 insertions(+), 6 deletions(-)

diff --git a/contracts/api_baselines/radroots_storage.txt b/contracts/api_baselines/radroots_storage.txt @@ -847,6 +847,19 @@ pub const fn radroots_storage::event::StoredVisibleEvent::position(&self) -> rad pub struct radroots_storage::event::VisibilityDigest(_) impl radroots_storage::event::VisibilityDigest pub const fn radroots_storage::event::VisibilityDigest::as_bytes(&self) -> &[u8; 32] +pub struct radroots_storage::event::VisibilityIndexBounds +impl radroots_storage::event::VisibilityIndexBounds +pub const fn radroots_storage::event::VisibilityIndexBounds::bytes(self) -> u64 +pub fn radroots_storage::event::VisibilityIndexBounds::new(u16, u64) -> core::result::Result<Self, radroots_storage::Error> +pub const fn radroots_storage::event::VisibilityIndexBounds::rows(self) -> u16 +pub struct radroots_storage::event::VisibilityIndexProgress +impl radroots_storage::event::VisibilityIndexProgress +pub const fn radroots_storage::event::VisibilityIndexProgress::charged_bytes(self) -> u64 +pub const fn radroots_storage::event::VisibilityIndexProgress::is_complete(self) -> bool +pub const fn radroots_storage::event::VisibilityIndexProgress::processed_rows(self) -> u16 +pub const fn radroots_storage::event::VisibilityIndexProgress::replay_epoch(self) -> u64 +pub const fn radroots_storage::event::VisibilityIndexProgress::source_sequence(self) -> u64 +pub const fn radroots_storage::event::VisibilityIndexProgress::token(self) -> radroots_storage::event::EventSourceToken pub struct radroots_storage::event::VisibilitySnapshot impl radroots_storage::event::VisibilitySnapshot pub fn radroots_storage::event::VisibilitySnapshot::current_heads(&self) -> &[radroots_event::envelope::event_head::v1::CurrentEventHead] @@ -860,6 +873,7 @@ pub const radroots_storage::event::EVENT_QUERY_ID_MAX: usize pub const radroots_storage::event::EVENT_QUERY_LIMIT_MAX: u16 pub trait radroots_storage::event::EventStore: core::marker::Send + core::marker::Sync pub fn radroots_storage::event::EventStore::admit(&self, radroots_storage::event::EventAdmission) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::AdmissionReceipt, radroots_storage::Error>> +pub fn radroots_storage::event::EventStore::maintain_visibility(&self, radroots_storage::event::VisibilityIndexBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilityIndexProgress, radroots_storage::Error>> pub fn radroots_storage::event::EventStore::query_provenance(&self, radroots_event::id::EventId, radroots_storage::event::EventQueryBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, radroots_storage::Error>> pub fn radroots_storage::event::EventStore::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>> pub fn radroots_storage::event::EventStore::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>> @@ -869,6 +883,7 @@ pub fn radroots_storage::event::EventStore::source_token(&self) -> radroots_tran pub fn radroots_storage::event::EventStore::status(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::status::EventStoreStatus, radroots_storage::Error>> impl radroots_storage::event::EventStore for radroots_storage::memory::MemoryStorage pub fn radroots_storage::memory::MemoryStorage::admit(&self, radroots_storage::event::EventAdmission) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::AdmissionReceipt, radroots_storage::Error>> +pub fn radroots_storage::memory::MemoryStorage::maintain_visibility(&self, radroots_storage::event::VisibilityIndexBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilityIndexProgress, radroots_storage::Error>> pub fn radroots_storage::memory::MemoryStorage::query_provenance(&self, radroots_event::id::EventId, radroots_storage::event::EventQueryBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, radroots_storage::Error>> pub fn radroots_storage::memory::MemoryStorage::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>> pub fn radroots_storage::memory::MemoryStorage::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>> @@ -1031,6 +1046,7 @@ pub fn radroots_storage::memory::MemoryStorage::transition_restore(&self, radroo pub fn radroots_storage::memory::MemoryStorage::verify_backup(&self, radroots_storage::backup::BackupPlan, radroots_storage::backup::BackupManifest) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::backup::BackupCapabilityError>> impl radroots_storage::event::EventStore for radroots_storage::memory::MemoryStorage pub fn radroots_storage::memory::MemoryStorage::admit(&self, radroots_storage::event::EventAdmission) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::AdmissionReceipt, radroots_storage::Error>> +pub fn radroots_storage::memory::MemoryStorage::maintain_visibility(&self, radroots_storage::event::VisibilityIndexBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilityIndexProgress, radroots_storage::Error>> pub fn radroots_storage::memory::MemoryStorage::query_provenance(&self, radroots_event::id::EventId, radroots_storage::event::EventQueryBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, radroots_storage::Error>> pub fn radroots_storage::memory::MemoryStorage::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>> pub fn radroots_storage::memory::MemoryStorage::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>> @@ -1821,6 +1837,7 @@ pub radroots_storage::Error::InvalidRestoreTransition pub radroots_storage::Error::InvalidRetrySchedule pub radroots_storage::Error::InvalidSourceGeneration pub radroots_storage::Error::InvalidStorageStatus +pub radroots_storage::Error::InvalidVisibilityIndexBounds pub radroots_storage::Error::InvalidWorkClaim pub radroots_storage::Error::InvalidWorkFailure pub radroots_storage::Error::JournalOperationCommitted @@ -1895,6 +1912,7 @@ pub fn radroots_storage::memory::MemoryStorage::transition_restore(&self, radroo pub fn radroots_storage::memory::MemoryStorage::verify_backup(&self, radroots_storage::backup::BackupPlan, radroots_storage::backup::BackupManifest) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::backup::BackupCapabilityError>> pub trait radroots_storage::EventStore: core::marker::Send + core::marker::Sync pub fn radroots_storage::EventStore::admit(&self, radroots_storage::event::EventAdmission) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::AdmissionReceipt, radroots_storage::Error>> +pub fn radroots_storage::EventStore::maintain_visibility(&self, radroots_storage::event::VisibilityIndexBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilityIndexProgress, radroots_storage::Error>> pub fn radroots_storage::EventStore::query_provenance(&self, radroots_event::id::EventId, radroots_storage::event::EventQueryBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, radroots_storage::Error>> pub fn radroots_storage::EventStore::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>> pub fn radroots_storage::EventStore::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>> @@ -1904,6 +1922,7 @@ pub fn radroots_storage::EventStore::source_token(&self) -> radroots_transport:: pub fn radroots_storage::EventStore::status(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::status::EventStoreStatus, radroots_storage::Error>> impl radroots_storage::event::EventStore for radroots_storage::memory::MemoryStorage pub fn radroots_storage::memory::MemoryStorage::admit(&self, radroots_storage::event::EventAdmission) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::AdmissionReceipt, radroots_storage::Error>> +pub fn radroots_storage::memory::MemoryStorage::maintain_visibility(&self, radroots_storage::event::VisibilityIndexBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilityIndexProgress, radroots_storage::Error>> pub fn radroots_storage::memory::MemoryStorage::query_provenance(&self, radroots_event::id::EventId, radroots_storage::event::EventQueryBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, radroots_storage::Error>> pub fn radroots_storage::memory::MemoryStorage::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>> pub fn radroots_storage::memory::MemoryStorage::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>> diff --git a/contracts/api_baselines/radroots_storage_sqlite.txt b/contracts/api_baselines/radroots_storage_sqlite.txt @@ -581,6 +581,7 @@ pub fn radroots_storage_sqlite::SqliteStorage::transition_restore(&self, radroot pub fn radroots_storage_sqlite::SqliteStorage::verify_backup(&self, radroots_storage::backup::BackupPlan, radroots_storage::backup::BackupManifest) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::backup::capability::BackupCapabilityError>> impl radroots_storage::event::EventStore for radroots_storage_sqlite::SqliteStorage pub fn radroots_storage_sqlite::SqliteStorage::admit(&self, radroots_storage::event::EventAdmission) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::AdmissionReceipt, radroots_storage::error::Error>> +pub fn radroots_storage_sqlite::SqliteStorage::maintain_visibility(&self, radroots_storage::event::visibility_index::VisibilityIndexBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::visibility_index::VisibilityIndexProgress, radroots_storage::error::Error>> pub fn radroots_storage_sqlite::SqliteStorage::query_provenance(&self, radroots_event::id::EventId, radroots_storage::event::EventQueryBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, radroots_storage::error::Error>> pub fn radroots_storage_sqlite::SqliteStorage::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::error::Error>> pub fn radroots_storage_sqlite::SqliteStorage::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::error::Error>> diff --git a/contracts/architecture/decisions/canonical_visibility_index.v1.json b/contracts/architecture/decisions/canonical_visibility_index.v1.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "decision_id": "radroots.storage.canonical_visibility_index.v1", + "status": "implemented_not_published", + "owners": ["radroots_storage", "radroots_storage_sqlite"], + "public_boundary": { + "maintenance": "EventStore::maintain_visibility", + "bounds": "VisibilityIndexBounds", + "progress": "VisibilityIndexProgress", + "unsupported_provider": "BackendUnavailable", + "invalid_bounds": "InvalidVisibilityIndexBounds", + "capacity": "SpaceInsufficient", + "maximum_source_rows": 64, + "maximum_logical_bytes": 16777216, + "charge": "4096 + maximum_source_rows * 128 + sum(signed_bytes * 16 + 4096)", + "hidden_metadata": "VisibilityEventMetadata reuses existing head and deletion parsing; it carries no application policy" + }, + "durability": { + "forward_runtime_version": 19, + "immutable_prefix": "versions 1 through 18", + "canonical_truth": "signed event rows remain authoritative and immutable", + "mutation": "combined SQL triggers atomically advance generation epoch, event last-change epoch and dirty visibility obligation", + "backfill": "fixed initial sequence target with durable sequence cursor", + "legacy_change_epoch": "zero marks source rows predating this index; the fixed initial backfill prepares them without a whole-source UPDATE or payload decode in migration", + "replay": "ordered last-change epoch and source-sequence cursor; updates to previously scanned rows are replayed without resetting the initial cursor", + "completion": "all initial rows and subsequent changed rows prepared at the transaction's active source token", + "transaction": "one bounded immediate transaction; cancellation or failure before commit rolls back metadata and progress", + "retired_source": "SourceGenerationChanged", + "restore": "verified whole-file replacement preserves metadata and source together, closes old handles, and requires consumer runtime invalidation separately" + }, + "semantics": [ + "verified unsupported heads retain suppression authority", + "raw events have no head or deletion authority", + "only visible authorized deletion requests contribute targets", + "same-author event and address deletion semantics reuse the reference parser", + "created_at is stored as ordered unsigned 8-byte big-endian metadata", + "missing address identifier behavior matches the reference evaluator", + "no partially prepared index is advertised complete" + ], + "separate_followup": "Indexed visible/status providers and bounded repeated-read qualification remain a separate checkpoint", + "forbidden": ["raw SQL public API", "application ranking", "second database", "hidden worker", "unbounded fallback", "protocol registry revision"] +} diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs @@ -22,6 +22,7 @@ pub enum Error { InvalidSourceGeneration, InvalidEventSequence, InvalidEventQueryLimit, + InvalidVisibilityIndexBounds, EmptyEventQueryIds, TooManyEventQueryIds, DuplicateEventQueryId, @@ -158,6 +159,7 @@ impl fmt::Display for Error { Self::InvalidEventSequence => "storage event sequence is invalid", Self::EventMutationOverflow => "storage event mutation epoch overflowed", Self::InvalidEventQueryLimit => "storage event query limit is invalid", + Self::InvalidVisibilityIndexBounds => "storage visibility index bounds are invalid", Self::EmptyEventQueryIds => "storage event id query is empty", Self::TooManyEventQueryIds => "storage event id query exceeds its limit", Self::DuplicateEventQueryId => "storage event id query contains a duplicate", diff --git a/crates/storage/src/event.rs b/crates/storage/src/event.rs @@ -13,8 +13,12 @@ use std::collections::BTreeSet; use crate::{Error, status::EventStoreStatus}; mod visibility; +mod visibility_index; +#[doc(hidden)] +pub use visibility::VisibilityEventMetadata; #[doc(hidden)] pub use visibility::{VisibilityEvaluation, VisibilityInput, evaluate_visibility}; +pub use visibility_index::{VisibilityIndexBounds, VisibilityIndexProgress}; /// Maximum events returned by one storage query. pub const EVENT_QUERY_LIMIT_MAX: u16 = 1_000; @@ -626,6 +630,17 @@ impl StoredEventProvenance { /// idempotent for an identical event and provenance observation; a stage may /// advance but never regress. Query cursors are bound to one source generation. pub trait EventStore: Send + Sync { + /// Prepares one bounded, transactionally resumable canonical index chunk. + /// + /// Completion refers to canonical metadata at the returned source token, + /// not to any application projection. It never initiates a hidden task. + fn maintain_visibility( + &self, + _bounds: VisibilityIndexBounds, + ) -> BoxFuture<'_, Result<VisibilityIndexProgress, Error>> { + Box::pin(async { Err(Error::BackendUnavailable) }) + } + /// Reads the current source token without decoding retained event history. /// /// Providers that cannot supply this fence fail closed. SQLite epochs have diff --git a/crates/storage/src/event/visibility.rs b/crates/storage/src/event/visibility.rs @@ -79,6 +79,86 @@ struct DeletionRequest { address_targets: BTreeSet<Nip01Coordinate>, } +/// Canonical provider metadata extracted with the reference reducer's parsing. +#[doc(hidden)] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct VisibilityEventMetadata { + pub event_id: EventId, + pub author: [u8; 32], + pub created_at: u64, + pub stage: AdmissionStage, + pub head_coordinate: Option<Vec<u8>>, + pub deletion_coordinate: Option<String>, + pub ephemeral: bool, + pub deletion_request: bool, + pub event_targets: BTreeSet<EventId>, + pub address_targets: BTreeSet<String>, +} + +impl VisibilityEventMetadata { + #[doc(hidden)] + pub fn from_event(event: &SignedEvent, stage: AdmissionStage) -> Result<Self, Error> { + let envelope = event.envelope(); + let (head_coordinate, ephemeral) = if stage == AdmissionStage::Raw { + (None, false) + } else { + match event_head_candidate_for_nip01_event(envelope) { + EventHeadCandidateResult::Candidate(candidate) => { + let mut key = Vec::new(); + match candidate.coordinate { + EventHeadCoordinate::Replaceable { kind, pubkey } => { + key.push(0); + key.extend_from_slice(&kind.to_be_bytes()); + key.extend_from_slice(pubkey.as_bytes()); + } + EventHeadCoordinate::Addressable { + kind, + pubkey, + d_tag, + } => { + key.push(1); + key.extend_from_slice(&kind.to_be_bytes()); + key.extend_from_slice(pubkey.as_bytes()); + key.extend_from_slice(d_tag.as_bytes()); + } + } + (Some(key), false) + } + EventHeadCandidateResult::NotHeadSelected => (None, false), + EventHeadCandidateResult::NotPersisted => (None, true), + EventHeadCandidateResult::Malformed(_) => return Err(Error::CorruptStoredEvent), + } + }; + let deletion_request = + stage == AdmissionStage::Visible && envelope.kind_u32() == KIND_DELETION_REQUEST; + let (event_targets, address_targets) = if deletion_request { + let request = parse_deletion_request(event)?; + ( + request.event_targets, + request + .address_targets + .into_iter() + .map(|coordinate| coordinate.as_str().to_owned()) + .collect(), + ) + } else { + (BTreeSet::new(), BTreeSet::new()) + }; + Ok(Self { + event_id: *event.id(), + author: *envelope.author().as_bytes(), + created_at: envelope.created_at_u64(), + stage, + head_coordinate, + deletion_coordinate: event_coordinate(envelope).map(|value| value.as_str().to_owned()), + ephemeral, + deletion_request, + event_targets, + address_targets, + }) + } +} + #[doc(hidden)] pub fn evaluate_visibility<'a>( generation: SourceGeneration, diff --git a/crates/storage/src/event/visibility_index.rs b/crates/storage/src/event/visibility_index.rs @@ -0,0 +1,120 @@ +//! Bounded canonical visibility-index preparation contracts. + +use super::EventSourceToken; +use crate::Error; + +/// Validated logical work envelope for one canonical index preparation call. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct VisibilityIndexBounds { + rows: u16, + bytes: u64, +} + +impl VisibilityIndexBounds { + /// Validates 1..=64 source rows and 1..=16 MiB of logical allocation charge. + pub fn new(rows: u16, bytes: u64) -> Result<Self, Error> { + if rows == 0 || rows > 64 || bytes == 0 || bytes > 16 * 1024 * 1024 { + return Err(Error::InvalidVisibilityIndexBounds); + } + Ok(Self { rows, bytes }) + } + + /// Maximum source rows materialized in this call. + pub const fn rows(self) -> u16 { + self.rows + } + + /// Maximum conservative logical allocation charge in this call. + pub const fn bytes(self) -> u64 { + self.bytes + } + + /// Charges retained bytes, decoded variable data and derived metadata. + /// + /// This conservative envelope is not a measurement of allocator RSS. + #[doc(hidden)] + pub fn charge(signed_bytes: u64) -> Result<u64, Error> { + signed_bytes + .checked_mul(16) + .and_then(|value| value.checked_add(4096)) + .ok_or(Error::SpaceInsufficient) + } +} + +/// Durable progress observed after one canonical index preparation transaction. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct VisibilityIndexProgress { + token: EventSourceToken, + source_sequence: u64, + replay_epoch: u64, + processed_rows: u16, + charged_bytes: u64, + complete: bool, +} + +impl VisibilityIndexProgress { + #[doc(hidden)] + pub const fn new( + token: EventSourceToken, + source_sequence: u64, + replay_epoch: u64, + processed_rows: u16, + charged_bytes: u64, + complete: bool, + ) -> Self { + Self { + token, + source_sequence, + replay_epoch, + processed_rows, + charged_bytes, + complete, + } + } + + /// Source fence read inside the completed preparation transaction. + pub const fn token(self) -> EventSourceToken { + self.token + } + /// Last initial-backfill sequence durably prepared; it never restarts on admission. + pub const fn source_sequence(self) -> u64 { + self.source_sequence + } + /// Last durable changed-event replay epoch. + pub const fn replay_epoch(self) -> u64 { + self.replay_epoch + } + /// Actual source rows materialized, including repeated changed rows. + pub const fn processed_rows(self) -> u16 { + self.processed_rows + } + /// Conservative logical allocation charge of this call. + pub const fn charged_bytes(self) -> u64 { + self.charged_bytes + } + /// Whether canonical metadata matches `token`; it is not a consumer-content claim. + pub const fn is_complete(self) -> bool { + self.complete + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn work_envelope_rejects_zero_and_one_over_without_relaxing_caps() { + assert!(VisibilityIndexBounds::new(64, 16 * 1024 * 1024).is_ok()); + for (rows, bytes) in [(0, 1), (65, 1), (1, 0), (1, 16 * 1024 * 1024 + 1)] { + assert_eq!( + VisibilityIndexBounds::new(rows, bytes), + Err(Error::InvalidVisibilityIndexBounds) + ); + } + assert_eq!(VisibilityIndexBounds::charge(10).unwrap(), 4256); + assert_eq!( + VisibilityIndexBounds::charge(u64::MAX), + Err(Error::SpaceInsufficient) + ); + } +} diff --git a/crates/storage/src/memory.rs b/crates/storage/src/memory.rs @@ -2,6 +2,8 @@ #[path = "memory_delivery_history.rs"] mod delivery_history; +#[path = "memory_visibility_index.rs"] +mod visibility_index; #[cfg(test)] #[path = "memory_mutation_tests.rs"] @@ -71,12 +73,15 @@ struct EventEntry { position: EventPosition, admission: EventAdmission, provenance: Vec<EventProvenance>, + last_mutation_epoch: u64, } #[derive(Clone, Default)] struct State { events: Vec<EventEntry>, event_mutation_epoch: u64, + event_changes: std::collections::BTreeMap<u64, u64>, + visibility_index: visibility_index::Index, journal: Vec<OperationRecord>, outbox: Vec<OutboxRecord>, projections: Vec<ProjectionStatus>, @@ -116,6 +121,8 @@ impl MemoryStorage { state: Mutex::new(State { events: Vec::new(), event_mutation_epoch: 0, + event_changes: std::collections::BTreeMap::new(), + visibility_index: visibility_index::Index::new(), journal: Vec::new(), outbox: Vec::new(), projections: Vec::new(), @@ -230,6 +237,12 @@ impl MemoryStorage { .event_mutation_epoch .checked_add(1) .ok_or(Error::EventMutationOverflow)?; + state.event_changes.remove(&entry.last_mutation_epoch); + entry.last_mutation_epoch = state.event_mutation_epoch; + state + .event_changes + .insert(entry.last_mutation_epoch, entry.position.sequence().get()); + state.visibility_index.ready = false; AdmissionDisposition::Advanced }; if !entry.provenance.contains(admission.provenance()) { @@ -263,8 +276,11 @@ impl MemoryStorage { position, admission, provenance, + last_mutation_epoch: epoch, }); state.event_mutation_epoch = epoch; + state.event_changes.insert(epoch, next); + state.visibility_index.ready = false; Ok(receipt) } @@ -424,6 +440,16 @@ impl Default for MemoryStorage { } impl EventStore for MemoryStorage { + fn maintain_visibility( + &self, + bounds: crate::event::VisibilityIndexBounds, + ) -> BoxFuture<'_, Result<crate::event::VisibilityIndexProgress, Error>> { + Box::pin(async move { + let mut state = self.state()?; + visibility_index::prepare(self.generation, &mut state, bounds) + }) + } + fn source_token(&self) -> BoxFuture<'_, Result<EventSourceToken, Error>> { Box::pin(async move { let state = self.state()?; diff --git a/crates/storage/src/memory_visibility_index.rs b/crates/storage/src/memory_visibility_index.rs @@ -0,0 +1,206 @@ +use super::State; +use crate::{ + Error, + event::{ + EventSourceToken, SourceGeneration, VisibilityEventMetadata, VisibilityIndexBounds, + VisibilityIndexProgress, + }, +}; +use std::collections::BTreeMap; + +#[derive(Clone, Default)] +pub(super) struct Index { + cursor: Option<Cursor>, + pub(super) ready: bool, + pub(super) events: BTreeMap<u64, VisibilityEventMetadata>, +} + +#[derive(Clone, Copy)] +struct Cursor { + target: u64, + backfill: u64, + replay_epoch: u64, +} + +impl Index { + pub(super) const fn new() -> Self { + Self { + cursor: None, + ready: false, + events: BTreeMap::new(), + } + } +} + +pub(super) fn prepare( + generation: SourceGeneration, + state: &mut State, + bounds: VisibilityIndexBounds, +) -> Result<VisibilityIndexProgress, Error> { + let mut charged = 4096 + u64::from(bounds.rows()) * 128; + if charged > bounds.bytes() { + return Err(Error::SpaceInsufficient); + } + let sequence = u64::try_from(state.events.len()).map_err(|_| Error::CorruptStoredEvent)?; + let mut cursor = state.visibility_index.cursor.unwrap_or(Cursor { + target: sequence, + backfill: 0, + replay_epoch: state.event_mutation_epoch, + }); + let building = cursor.backfill < cursor.target; + let positions = if building { + ((cursor.backfill + 1)..=cursor.target) + .take(usize::from(bounds.rows())) + .collect::<Vec<_>>() + } else { + state + .event_changes + .range(( + std::ops::Bound::Excluded(cursor.replay_epoch), + std::ops::Bound::Unbounded, + )) + .take(usize::from(bounds.rows())) + .map(|(_, sequence)| *sequence) + .collect() + }; + let mut prepared = Vec::new(); + for position in positions { + let entry = state + .events + .get(usize::try_from(position - 1).map_err(|_| Error::CorruptStoredEvent)?) + .ok_or(Error::CorruptStoredEvent)?; + let bytes = u64::try_from(entry.admission.event().raw_json().len()) + .map_err(|_| Error::SpaceInsufficient)?; + let next_charge = charged + .checked_add(VisibilityIndexBounds::charge(bytes)?) + .ok_or(Error::SpaceInsufficient)?; + if next_charge > bounds.bytes() { + if prepared.is_empty() { + return Err(Error::SpaceInsufficient); + } + break; + } + let metadata = + VisibilityEventMetadata::from_event(entry.admission.event(), entry.admission.stage())?; + prepared.push((position, metadata)); + charged = next_charge; + if building { + cursor.backfill = position; + } else { + cursor.replay_epoch = entry.last_mutation_epoch; + } + } + let pending = state + .event_changes + .range(( + std::ops::Bound::Excluded(cursor.replay_epoch), + std::ops::Bound::Unbounded, + )) + .next() + .is_some(); + let complete = cursor.backfill == cursor.target && !pending; + if complete { + cursor.replay_epoch = state.event_mutation_epoch; + } + let processed = u16::try_from(prepared.len()).map_err(|_| Error::CorruptStoredEvent)?; + state.visibility_index.events.extend(prepared); + state.visibility_index.cursor = Some(cursor); + state.visibility_index.ready = complete; + Ok(VisibilityIndexProgress::new( + EventSourceToken::new(generation, state.event_mutation_epoch), + cursor.backfill, + cursor.replay_epoch, + processed, + charged, + complete, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EventStore, memory::MemoryStorage}; + use futures_executor::block_on; + use radroots_event::{SignedEvent, wire::Nip01EventWire}; + use radroots_transport::{ + Target, TransportId, + source::{EventProvenance, ObservedEvent}, + }; + + fn admission(content: &str, time: u64) -> crate::event::EventAdmission { + let mut wire = Nip01EventWire { + id: "0".repeat(64), + pubkey: "585591529da0bab31b3b1b1f986611cf5f435dca84f978c89ee8a40cca7103df".to_owned(), + created_at: time, + kind: 0, + tags: vec![], + content: content.to_owned(), + sig: "42".repeat(64), + extra: Default::default(), + }; + wire.id = wire.computed_event_id().unwrap().to_hex(); + let raw = serde_json::to_string(&wire).unwrap(); + let event = SignedEvent::from_wire_verified_id(wire, raw).unwrap(); + let target = Target::nostr_relay("wss://relay.example").unwrap(); + crate::event::EventAdmission::raw(ObservedEvent::new( + event, + EventProvenance::new(TransportId::NOSTR, target.fingerprint().clone(), 100).unwrap(), + )) + } + + #[test] + fn memory_visibility_index_retains_backfill_and_refuses_oversized_rows_without_partial_state() { + let generation = SourceGeneration::new([82; 32]).unwrap(); + let storage = MemoryStorage::new(generation); + let first = admission("first", 10); + block_on(storage.admit(first.clone())).unwrap(); + block_on(storage.admit(admission("second", 20))).unwrap(); + let bounds = VisibilityIndexBounds::new(1, 16 * 1024 * 1024).unwrap(); + let part = block_on(storage.maintain_visibility(bounds)).unwrap(); + assert_eq!(part.source_sequence(), 1); + assert!(!part.is_complete()); + block_on(storage.admit(admission("third", 30))).unwrap(); + let next = block_on(storage.maintain_visibility(bounds)).unwrap(); + assert_eq!(next.source_sequence(), 2); + assert!(!next.is_complete()); + assert!( + block_on(storage.maintain_visibility(bounds)) + .unwrap() + .is_complete() + ); + assert_eq!(storage.state().unwrap().visibility_index.events.len(), 3); + block_on(storage.admit(first)).unwrap(); + assert_eq!( + block_on(storage.maintain_visibility(bounds)) + .unwrap() + .processed_rows(), + 0 + ); + block_on(storage.admit(admission(&"x".repeat(1000), 40))).unwrap(); + assert_eq!( + block_on(storage.maintain_visibility(VisibilityIndexBounds::new(1, 10000).unwrap())), + Err(Error::SpaceInsufficient) + ); + assert_eq!(storage.state().unwrap().visibility_index.events.len(), 3); + assert!(!storage.state().unwrap().visibility_index.ready); + assert!( + block_on(storage.maintain_visibility(bounds)) + .unwrap() + .is_complete() + ); + } + + #[test] + fn memory_visibility_index_empty_source_has_a_real_zero_epoch_fence() { + let generation = SourceGeneration::new([83; 32]).unwrap(); + let storage = MemoryStorage::new(generation); + let progress = block_on( + storage.maintain_visibility(VisibilityIndexBounds::new(64, 16 * 1024 * 1024).unwrap()), + ) + .unwrap(); + assert!(progress.is_complete()); + assert_eq!(progress.token(), EventSourceToken::new(generation, 0)); + assert_eq!(progress.processed_rows(), 0); + assert!(storage.state().unwrap().visibility_index.events.is_empty()); + } +} diff --git a/crates/storage/tests/event_store.rs b/crates/storage/tests/event_store.rs @@ -70,6 +70,12 @@ fn unsupported_adapter_cannot_invent_a_source_fence() { block_on(MemoryEventStore::new().source_token()), Err(Error::BackendUnavailable) ); + assert_eq!( + block_on(MemoryEventStore::new().maintain_visibility( + radroots_storage::event::VisibilityIndexBounds::new(64, 16 * 1024 * 1024).unwrap() + )), + Err(Error::BackendUnavailable) + ); } impl EventStore for MemoryEventStore { diff --git a/crates/storage_sqlite/src/event/mod.rs b/crates/storage_sqlite/src/event/mod.rs @@ -1,4 +1,7 @@ use crate::backend::map_backend; +mod visibility_index; +#[cfg(test)] +mod visibility_index_tests; use radroots_event::SignedEvent; use radroots_event_codec::Codec; use radroots_storage::{ @@ -397,6 +400,13 @@ fn admission_contract_metadata(admission: &EventAdmission) -> Option<(&'static s #[cfg_attr(coverage_nightly, coverage(off))] impl EventStore for SqliteStorage { + fn maintain_visibility( + &self, + bounds: radroots_storage::event::VisibilityIndexBounds, + ) -> BoxFuture<'_, Result<radroots_storage::event::VisibilityIndexProgress, Error>> { + Box::pin(async move { self.prepare_visibility_index(bounds).await }) + } + fn source_token(&self) -> BoxFuture<'_, Result<EventSourceToken, Error>> { Box::pin(async move { let epoch = sqlx::query_scalar::<_, i64>( diff --git a/crates/storage_sqlite/src/event/visibility_index.rs b/crates/storage_sqlite/src/event/visibility_index.rs @@ -0,0 +1,225 @@ +use super::*; +use radroots_storage::event::{ + VisibilityEventMetadata, VisibilityIndexBounds, VisibilityIndexProgress, +}; + +impl SqliteStorage { + pub(super) async fn prepare_visibility_index( + &self, + bounds: VisibilityIndexBounds, + ) -> Result<VisibilityIndexProgress, Error> { + if self.mode == EventStoreMode::ReadOnly { + return Err(Error::BackendUnavailable); + } + let mut charged = 4096 + u64::from(bounds.rows()) * 128; + if charged > bounds.bytes() { + return Err(Error::SpaceInsufficient); + } + let mut transaction = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(map_backend)?; + let source = sqlx::query( + "SELECT mutation_epoch, sequence_head FROM radroots_runtime_source_generations + WHERE generation = ? AND state = 'active'", + ) + .bind(self.generation.as_bytes().as_slice()) + .fetch_optional(&mut *transaction) + .await + .map_err(map_backend)? + .ok_or(Error::SourceGenerationChanged)?; + let epoch: i64 = source.try_get("mutation_epoch").map_err(map_corrupt)?; + let sequence: i64 = source.try_get("sequence_head").map_err(map_corrupt)?; + sqlx::query( + "INSERT OR IGNORE INTO radroots_runtime_visibility_progress + (source_generation, backfill_target, replay_epoch, replay_sequence) + VALUES (?, ?, ?, 9223372036854775807)", + ) + .bind(self.generation.as_bytes().as_slice()) + .bind(sequence) + .bind(epoch) + .execute(&mut *transaction) + .await + .map_err(map_backend)?; + let progress = sqlx::query( + "SELECT schema_version, backfill_target, backfill_sequence, replay_epoch, replay_sequence + FROM radroots_runtime_visibility_progress WHERE source_generation = ?", + ) + .bind(self.generation.as_bytes().as_slice()) + .fetch_one(&mut *transaction).await.map_err(map_backend)?; + let version: i64 = progress.try_get("schema_version").map_err(map_corrupt)?; + let target: i64 = progress.try_get("backfill_target").map_err(map_corrupt)?; + let mut backfill: i64 = progress.try_get("backfill_sequence").map_err(map_corrupt)?; + let mut replay_epoch: i64 = progress.try_get("replay_epoch").map_err(map_corrupt)?; + let mut replay_sequence: i64 = progress.try_get("replay_sequence").map_err(map_corrupt)?; + if version != 1 || target > sequence || backfill > target || replay_epoch > epoch { + return Err(Error::CorruptStoredEvent); + } + let building = backfill < target; + let mut query = QueryBuilder::<Sqlite>::new( + "SELECT source_sequence, last_mutation_epoch, length(signed_event) AS signed_bytes + FROM radroots_runtime_events WHERE source_generation = ", + ); + query.push_bind(self.generation.as_bytes().as_slice()); + if building { + query.push(" AND source_sequence > ").push_bind(backfill); + query.push(" AND source_sequence <= ").push_bind(target); + query + .push(" ORDER BY source_sequence LIMIT ") + .push_bind(i64::from(bounds.rows())); + } else { + query.push(" AND (last_mutation_epoch, source_sequence) > ("); + query + .push_bind(replay_epoch) + .push(", ") + .push_bind(replay_sequence) + .push(")"); + query + .push(" ORDER BY last_mutation_epoch, source_sequence LIMIT ") + .push_bind(i64::from(bounds.rows())); + } + let lengths = query + .build() + .fetch_all(&mut *transaction) + .await + .map_err(map_backend)?; + if building && lengths.is_empty() { + return Err(Error::CorruptStoredEvent); + } + let mut processed = 0u16; + for length in lengths { + let position: i64 = length.try_get("source_sequence").map_err(map_corrupt)?; + let changed: i64 = length.try_get("last_mutation_epoch").map_err(map_corrupt)?; + let size = u64_from_i64(length.try_get("signed_bytes").map_err(map_corrupt)?)?; + let charge = VisibilityIndexBounds::charge(size)?; + let next_charge = charged + .checked_add(charge) + .ok_or(Error::SpaceInsufficient)?; + if next_charge > bounds.bytes() { + if processed == 0 { + return Err(Error::SpaceInsufficient); + } + break; + } + let row = sqlx::query( + "SELECT source_generation, source_sequence, signed_event, admission_stage, + admitted_contract_id, admitted_registry_version FROM radroots_runtime_events + WHERE source_generation = ? AND source_sequence = ?", + ) + .bind(self.generation.as_bytes().as_slice()) + .bind(position) + .fetch_one(&mut *transaction) + .await + .map_err(map_backend)?; + let decoded = self.decode_event_row(&row)?; + if decoded.raw_json.len() as u64 != size || changed > epoch { + return Err(Error::CorruptStoredEvent); + } + let metadata = VisibilityEventMetadata::from_event(&decoded.event, decoded.stage)?; + self.put_visibility_metadata(&mut transaction, position, changed, metadata) + .await?; + charged = next_charge; + processed += 1; + if building { + backfill = position; + } else { + replay_epoch = changed; + replay_sequence = position; + } + } + let pending: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM radroots_runtime_events + WHERE source_generation = ? AND (last_mutation_epoch, source_sequence) > (?, ?))", + ) + .bind(self.generation.as_bytes().as_slice()) + .bind(replay_epoch) + .bind(replay_sequence) + .fetch_one(&mut *transaction) + .await + .map_err(map_backend)?; + let complete = backfill == target && !pending; + if complete { + replay_epoch = epoch; + replay_sequence = i64::MAX; + } + sqlx::query( + "UPDATE radroots_runtime_visibility_progress SET backfill_sequence = ?, + replay_epoch = ?, replay_sequence = ?, ready = ? WHERE source_generation = ?", + ) + .bind(backfill) + .bind(replay_epoch) + .bind(replay_sequence) + .bind(complete) + .bind(self.generation.as_bytes().as_slice()) + .execute(&mut *transaction) + .await + .map_err(map_backend)?; + transaction.commit().await.map_err(map_backend)?; + Ok(VisibilityIndexProgress::new( + EventSourceToken::new(self.generation, u64_from_i64(epoch)?), + u64_from_i64(backfill)?, + u64_from_i64(replay_epoch)?, + processed, + charged, + complete, + )) + } + + async fn put_visibility_metadata( + &self, + transaction: &mut sqlx::Transaction<'_, Sqlite>, + position: i64, + changed: i64, + metadata: VisibilityEventMetadata, + ) -> Result<(), Error> { + sqlx::query( + "INSERT INTO radroots_runtime_visibility_events + (source_generation, source_sequence, event_id, last_mutation_epoch, author, created_at, + admission_stage, head_coordinate, deletion_coordinate, ephemeral, deletion_request) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(source_generation, source_sequence) DO UPDATE SET + last_mutation_epoch = excluded.last_mutation_epoch, + admission_stage = excluded.admission_stage, head_coordinate = excluded.head_coordinate, + deletion_coordinate = excluded.deletion_coordinate, + ephemeral = excluded.ephemeral, deletion_request = excluded.deletion_request", + ) + .bind(self.generation.as_bytes().as_slice()).bind(position) + .bind(metadata.event_id.as_bytes().as_slice()).bind(changed) + .bind(metadata.author.as_slice()).bind(metadata.created_at.to_be_bytes().as_slice()) + .bind(stage_name(metadata.stage)).bind(metadata.head_coordinate.as_deref()) + .bind(metadata.deletion_coordinate.as_deref()).bind(metadata.ephemeral) + .bind(metadata.deletion_request) + .execute(&mut **transaction).await.map_err(map_backend)?; + // Signed event bytes and deletion tags are immutable. Replays only add identical targets. + for target in metadata.event_targets { + sqlx::query( + "INSERT OR IGNORE INTO radroots_runtime_visibility_deletion_events + (source_generation, target_event, author, request_id) VALUES (?, ?, ?, ?)", + ) + .bind(self.generation.as_bytes().as_slice()) + .bind(target.as_bytes().as_slice()) + .bind(metadata.author.as_slice()) + .bind(metadata.event_id.as_bytes().as_slice()) + .execute(&mut **transaction) + .await + .map_err(map_backend)?; + } + for coordinate in metadata.address_targets { + sqlx::query( + "INSERT OR IGNORE INTO radroots_runtime_visibility_deletion_addresses + (source_generation, target_coordinate, author, created_at, request_id) + VALUES (?, ?, ?, ?, ?)", + ) + .bind(self.generation.as_bytes().as_slice()) + .bind(coordinate) + .bind(metadata.author.as_slice()) + .bind(metadata.created_at.to_be_bytes().as_slice()) + .bind(metadata.event_id.as_bytes().as_slice()) + .execute(&mut **transaction) + .await + .map_err(map_backend)?; + } + Ok(()) + } +} diff --git a/crates/storage_sqlite/src/event/visibility_index_tests.rs b/crates/storage_sqlite/src/event/visibility_index_tests.rs @@ -0,0 +1,626 @@ +use super::tests::{observed, signed_event_with, store, verified, visible}; +use super::*; +use radroots_storage::event::VisibilityIndexBounds; + +fn generation() -> SourceGeneration { + SourceGeneration::new([81; 32]).unwrap() +} +fn bounds(rows: u16) -> VisibilityIndexBounds { + VisibilityIndexBounds::new(rows, 16 * 1024 * 1024).unwrap() +} +fn profile(name: &str, time: u64) -> SignedEvent { + signed_event_with(&format!(r#"{{"name":"{name}"}}"#), false, time, 0, vec![]) +} +fn raw(event: SignedEvent) -> EventAdmission { + EventAdmission::raw(observed(event, 100, None)) +} +fn admitted(event: SignedEvent) -> EventAdmission { + EventAdmission::visible(observed(event.clone(), 100, None), visible(&event)).unwrap() +} + +async fn finish(storage: &SqliteStorage) { + for _ in 0..100 { + if storage + .maintain_visibility(bounds(2)) + .await + .unwrap() + .is_complete() + { + return; + } + } + panic!("bounded fixture index did not finish"); +} + +async fn indexed_visible(storage: &SqliteStorage) -> Vec<EventId> { + let values: Vec<Vec<u8>> = sqlx::query_scalar( + "SELECT candidate.event_id FROM radroots_runtime_visibility_events AS candidate + WHERE candidate.source_generation = ? AND candidate.admission_stage = 'visible' + AND candidate.ephemeral = 0 + AND (candidate.head_coordinate IS NULL OR candidate.event_id = ( + SELECT head.event_id FROM radroots_runtime_visibility_events AS head + WHERE head.source_generation = candidate.source_generation + AND head.head_coordinate = candidate.head_coordinate + ORDER BY head.created_at DESC, head.event_id ASC LIMIT 1)) + AND (candidate.deletion_request = 1 OR ( + NOT EXISTS (SELECT 1 FROM radroots_runtime_visibility_deletion_events AS deletion + WHERE deletion.source_generation = candidate.source_generation + AND deletion.target_event = candidate.event_id AND deletion.author = candidate.author) + AND NOT EXISTS (SELECT 1 FROM radroots_runtime_visibility_deletion_addresses AS deletion + WHERE deletion.source_generation = candidate.source_generation + AND deletion.target_coordinate = candidate.deletion_coordinate + AND deletion.author = candidate.author AND deletion.created_at >= candidate.created_at))) + ORDER BY candidate.event_id", + ).bind(storage.generation.as_bytes().as_slice()).fetch_all(&storage.pool).await.unwrap(); + values + .into_iter() + .map(|value| EventId::from_bytes(value.try_into().unwrap())) + .collect() +} + +#[tokio::test] +async fn visibility_index_backfill_and_changed_replay_survive_provider_reopen_without_restart() { + let storage = store(generation()).await; + let first = profile("first", 10); + for event in [first.clone(), profile("second", 20), profile("third", 30)] { + storage.admit(raw(event)).await.unwrap(); + } + let before = storage.maintain_visibility(bounds(1)).await.unwrap(); + assert_eq!(before.source_sequence(), 1); + assert_eq!(before.processed_rows(), 1); + assert!(!before.is_complete()); + storage.admit(admitted(first.clone())).await.unwrap(); + storage.admit(raw(profile("fourth", 40))).await.unwrap(); + let reopened = SqliteStorage::new( + storage.pool.clone(), + generation(), + EventStoreMode::ReadWrite, + ); + let after = reopened.maintain_visibility(bounds(1)).await.unwrap(); + assert_eq!(after.source_sequence(), 2); + assert!(!after.is_complete()); + let third = reopened.maintain_visibility(bounds(1)).await.unwrap(); + assert_eq!(third.source_sequence(), 3); + assert!(!third.is_complete()); + finish(&reopened).await; + let token = reopened.source_token().await.unwrap(); + let current = reopened.maintain_visibility(bounds(1)).await.unwrap(); + assert!(current.is_complete()); + assert_eq!(current.processed_rows(), 0); + assert_eq!(current.source_sequence(), 3); + assert_eq!(current.token(), token); + let row = sqlx::query("SELECT admission_stage, last_mutation_epoch FROM radroots_runtime_visibility_events WHERE event_id = ?") + .bind(first.id().as_bytes().as_slice()).fetch_one(&reopened.pool).await.unwrap(); + assert_eq!(row.get::<String, _>("admission_stage"), "visible"); + assert_eq!(row.get::<i64, _>("last_mutation_epoch"), 4); + assert_eq!(indexed_visible(&reopened).await, vec![*first.id()]); + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM radroots_runtime_visibility_events") + .fetch_one(&reopened.pool) + .await + .unwrap(); + assert_eq!(count, 4); +} + +#[tokio::test] +async fn visibility_index_matches_reference_winners_deletion_before_target_and_unsupported_head() { + let storage = store(generation()).await; + let old = profile("old", 10); + let current = profile("current", 20); + let deletion = signed_event_with( + "delete", + false, + 30, + 5, + vec![vec!["e".to_owned(), current.id().to_hex()]], + ); + for event in [deletion, old, current] { + storage.admit(admitted(event)).await.unwrap(); + } + finish(&storage).await; + assert_eq!( + indexed_visible(&storage).await, + storage + .rebuild_visibility() + .await + .unwrap() + .visible_event_ids() + ); + let newer = profile("unsupported", 40); + storage + .admit( + EventAdmission::verified(observed(newer.clone(), 100, None), verified(&newer)).unwrap(), + ) + .await + .unwrap(); + assert!( + !sqlx::query_scalar::<_, bool>("SELECT ready FROM radroots_runtime_visibility_progress") + .fetch_one(&storage.pool) + .await + .unwrap() + ); + finish(&storage).await; + assert_eq!( + indexed_visible(&storage).await, + storage + .rebuild_visibility() + .await + .unwrap() + .visible_event_ids() + ); + let same_a = profile("same-a", 50); + let same_b = profile("same-b", 50); + storage.admit(admitted(same_a)).await.unwrap(); + storage.admit(admitted(same_b)).await.unwrap(); + storage + .admit(raw(profile("forged raw later", 60))) + .await + .unwrap(); + let address_deletion = signed_event_with( + "address delete", + false, + 55, + 5, + vec![vec![ + "a".to_owned(), + "0:585591529da0bab31b3b1b1f986611cf5f435dca84f978c89ee8a40cca7103df:".to_owned(), + ]], + ); + storage.admit(admitted(address_deletion)).await.unwrap(); + finish(&storage).await; + assert_eq!( + indexed_visible(&storage).await, + storage + .rebuild_visibility() + .await + .unwrap() + .visible_event_ids() + ); +} + +#[tokio::test] +async fn visibility_index_bounds_charge_before_payload_decode_and_commit_partial_progress() { + let storage = store(generation()).await; + let first = profile("first", 10); + let size = first.raw_json().len() as u64; + storage.admit(raw(first)).await.unwrap(); + storage + .admit(raw(profile(&"x".repeat(1000), 20))) + .await + .unwrap(); + let one = 4096 + 2 * 128 + VisibilityIndexBounds::charge(size).unwrap(); + storage.work.take(); + let progress = storage + .maintain_visibility(VisibilityIndexBounds::new(2, one).unwrap()) + .await + .unwrap(); + assert_eq!(progress.charged_bytes(), one); + assert_eq!(progress.processed_rows(), 1); + assert_eq!(storage.work.take().codec_inputs, 1); + assert!(!progress.is_complete()); + assert_eq!( + storage + .maintain_visibility(VisibilityIndexBounds::new(2, one).unwrap()) + .await, + Err(Error::SpaceInsufficient) + ); + assert_eq!(storage.work.take().codec_inputs, 0); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT backfill_sequence FROM radroots_runtime_visibility_progress" + ) + .fetch_one(&storage.pool) + .await + .unwrap(), + 1 + ); + finish(&storage).await; + storage.work.take(); + assert!( + storage + .maintain_visibility(bounds(64)) + .await + .unwrap() + .is_complete() + ); + assert_eq!(storage.work.take().codec_inputs, 0); +} + +#[tokio::test] +async fn visibility_index_fault_rolls_back_all_metadata_and_progress_then_resumes() { + let storage = store(generation()).await; + storage.admit(raw(profile("first", 10))).await.unwrap(); + sqlx::raw_sql("CREATE TEMP TRIGGER visibility_test_fault AFTER INSERT ON radroots_runtime_visibility_events BEGIN SELECT RAISE(ABORT, 'index test fault'); END;") + .execute(&storage.pool).await.unwrap(); + assert_eq!( + storage.maintain_visibility(bounds(1)).await, + Err(Error::BackendUnavailable) + ); + for query in [ + "SELECT COUNT(*) FROM radroots_runtime_visibility_progress", + "SELECT COUNT(*) FROM radroots_runtime_visibility_events", + ] { + assert_eq!( + sqlx::query_scalar::<_, i64>(query) + .fetch_one(&storage.pool) + .await + .unwrap(), + 0 + ); + } + sqlx::query("DROP TRIGGER visibility_test_fault") + .execute(&storage.pool) + .await + .unwrap(); + assert!( + storage + .maintain_visibility(bounds(1)) + .await + .unwrap() + .is_complete() + ); +} + +#[tokio::test] +async fn visibility_index_direct_writer_invalidates_atomically_and_rollback_preserves_ready_state() +{ + let storage = store(generation()).await; + let event = profile("first", 10); + storage.admit(raw(event)).await.unwrap(); + finish(&storage).await; + let mut transaction = storage.pool.begin_with("BEGIN IMMEDIATE").await.unwrap(); + sqlx::query("UPDATE radroots_runtime_events SET admission_stage = 'verified'") + .execute(&mut *transaction) + .await + .unwrap(); + let dirty: bool = sqlx::query_scalar("SELECT ready FROM radroots_runtime_visibility_progress") + .fetch_one(&mut *transaction) + .await + .unwrap(); + assert!(!dirty); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT last_mutation_epoch FROM radroots_runtime_events") + .fetch_one(&mut *transaction) + .await + .unwrap(), + 2 + ); + transaction.rollback().await.unwrap(); + assert!( + sqlx::query_scalar::<_, bool>("SELECT ready FROM radroots_runtime_visibility_progress") + .fetch_one(&storage.pool) + .await + .unwrap() + ); + sqlx::query("UPDATE radroots_runtime_events SET admission_stage = 'verified'") + .execute(&storage.pool) + .await + .unwrap(); + assert!( + !sqlx::query_scalar::<_, bool>("SELECT ready FROM radroots_runtime_visibility_progress") + .fetch_one(&storage.pool) + .await + .unwrap() + ); + let current = storage.maintain_visibility(bounds(1)).await.unwrap(); + assert!(current.is_complete()); + assert_eq!(current.processed_rows(), 1); + sqlx::query("UPDATE radroots_runtime_events SET admission_stage = admission_stage") + .execute(&storage.pool) + .await + .unwrap(); + assert_eq!( + storage + .maintain_visibility(bounds(1)) + .await + .unwrap() + .processed_rows(), + 0 + ); +} + +#[tokio::test] +async fn visibility_index_corruption_and_retired_or_readonly_sources_refuse_completion() { + let storage = store(generation()).await; + storage.admit(raw(profile("first", 10))).await.unwrap(); + sqlx::query("DROP TRIGGER radroots_runtime_events_raw_update_guard") + .execute(&storage.pool) + .await + .unwrap(); + sqlx::query("UPDATE radroots_runtime_events SET signed_event = X'7b'") + .execute(&storage.pool) + .await + .unwrap(); + assert_eq!( + storage.maintain_visibility(bounds(1)).await, + Err(Error::CorruptStoredEvent) + ); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM radroots_runtime_visibility_progress") + .fetch_one(&storage.pool) + .await + .unwrap(), + 0 + ); + let readonly = SqliteStorage::new(storage.pool.clone(), generation(), EventStoreMode::ReadOnly); + assert_eq!( + readonly.maintain_visibility(bounds(1)).await, + Err(Error::BackendUnavailable) + ); + sqlx::query( + "UPDATE radroots_runtime_source_generations SET state = 'retired', retired_at_unix_ms = 2", + ) + .execute(&storage.pool) + .await + .unwrap(); + assert_eq!( + storage.maintain_visibility(bounds(1)).await, + Err(Error::SourceGenerationChanged) + ); +} + +#[tokio::test] +async fn visibility_index_empty_source_is_complete_and_last_change_replay_uses_index_navigation() { + let storage = store(generation()).await; + let empty = storage.maintain_visibility(bounds(1)).await.unwrap(); + assert!(empty.is_complete()); + assert_eq!(empty.token().epoch(), 0); + storage + .admit(raw(profile("first", u64::MAX))) + .await + .unwrap(); + let progress = storage.maintain_visibility(bounds(1)).await.unwrap(); + assert!(progress.is_complete()); + assert_eq!(progress.source_sequence(), 0); + let time: Vec<u8> = + sqlx::query_scalar("SELECT created_at FROM radroots_runtime_visibility_events") + .fetch_one(&storage.pool) + .await + .unwrap(); + assert_eq!(time, u64::MAX.to_be_bytes()); + let plan = sqlx::query("EXPLAIN QUERY PLAN SELECT source_sequence, last_mutation_epoch, length(signed_event) FROM radroots_runtime_events WHERE source_generation = ? AND (last_mutation_epoch, source_sequence) > (?, ?) ORDER BY last_mutation_epoch, source_sequence LIMIT ?") + .bind(generation().as_bytes().as_slice()).bind(0i64).bind(0i64).bind(1i64) + .fetch_all(&storage.pool).await.unwrap(); + assert!(plan.iter().any(|row| { + row.get::<String, _>("detail") + .contains("radroots_runtime_events_mutation_idx") + })); +} + +#[tokio::test] +async fn visibility_index_real_database_close_and_reopen_preserves_bounded_progress() { + let directory = tempfile::tempdir().unwrap(); + let paths = crate::Paths::from_directory(directory.path()).unwrap(); + let storage = SqliteStorage::open( + crate::OpenOptions::new(paths.clone(), crate::OpenMode::Create) + .with_source_generation(generation(), 100) + .unwrap(), + ) + .await + .unwrap(); + for event in [profile("first", 10), profile("second", 20)] { + storage.admit(admitted(event)).await.unwrap(); + } + let first = storage.maintain_visibility(bounds(1)).await.unwrap(); + assert_eq!(first.source_sequence(), 1); + assert!(!first.is_complete()); + storage.close().await.unwrap(); + assert_eq!( + storage.maintain_visibility(bounds(1)).await, + Err(Error::BackendUnavailable) + ); + let reopened = SqliteStorage::open(crate::OpenOptions::new( + paths, + crate::OpenMode::ReadWriteExisting, + )) + .await + .unwrap(); + let next = reopened.maintain_visibility(bounds(1)).await.unwrap(); + assert_eq!(next.source_sequence(), 2); + assert_eq!(next.processed_rows(), 1); + assert!(next.is_complete()); + assert_eq!( + indexed_visible(&reopened).await, + reopened + .rebuild_visibility() + .await + .unwrap() + .visible_event_ids() + ); + reopened.close().await.unwrap(); +} + +#[tokio::test] +async fn visibility_index_forward_migration_retains_legacy_truth_and_refuses_corrupt_payloads() { + use crate::migration::runtime::{MIGRATIONS, migration_sql}; + use sqlx::sqlite::SqlitePoolOptions; + for state in [0, 1, 2] { + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect("sqlite::memory:") + .await + .unwrap(); + for migration in &MIGRATIONS[..18] { + sqlx::raw_sql(migration_sql(migration.version()).unwrap()) + .execute(&pool) + .await + .unwrap(); + } + sqlx::query("INSERT INTO radroots_runtime_source_generations (generation, sequence_head, state, created_at_unix_ms) VALUES (?, 0, 'active', 1)") + .bind(generation().as_bytes().as_slice()).execute(&pool).await.unwrap(); + let storage = SqliteStorage::new(pool.clone(), generation(), EventStoreMode::ReadWrite); + let event = profile("legacy", 10); + if state != 0 { + storage.admit(raw(event.clone())).await.unwrap(); + if state == 2 { + sqlx::query("DROP TRIGGER radroots_runtime_events_raw_update_guard") + .execute(&pool) + .await + .unwrap(); + sqlx::query("UPDATE radroots_runtime_events SET signed_event = X'7b'") + .execute(&pool) + .await + .unwrap(); + } + } + let token = storage.source_token().await.unwrap(); + assert_eq!( + storage.maintain_visibility(bounds(1)).await, + Err(Error::BackendUnavailable) + ); + storage.work.take(); + sqlx::raw_sql(migration_sql(19).unwrap()) + .execute(&pool) + .await + .unwrap(); + assert_eq!(storage.source_token().await.unwrap(), token); + assert_eq!(storage.work.take().codec_inputs, 0); + if state == 2 { + assert_eq!( + storage.maintain_visibility(bounds(1)).await, + Err(Error::CorruptStoredEvent) + ); + } else { + let result = storage.maintain_visibility(bounds(1)).await.unwrap(); + assert!(result.is_complete()); + assert_eq!(result.processed_rows(), u16::from(state == 1)); + if state == 1 { + assert_eq!( + sqlx::query_scalar::<_, Vec<u8>>( + "SELECT signed_event FROM radroots_runtime_events" + ) + .fetch_one(&pool) + .await + .unwrap(), + event.raw_json().as_bytes() + ); + } + } + } +} + +#[tokio::test] +async fn visibility_index_cancellation_before_admission_leaves_no_progress_and_allows_retry() { + let storage = store(generation()).await; + storage.admit(raw(profile("first", 10))).await.unwrap(); + let held = storage.pool.acquire().await.unwrap(); + let mut future = storage.maintain_visibility(bounds(1)); + std::future::poll_fn(|context| { + assert!(future.as_mut().poll(context).is_pending()); + std::task::Poll::Ready(()) + }) + .await; + drop(future); + drop(held); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM radroots_runtime_visibility_progress") + .fetch_one(&storage.pool) + .await + .unwrap(), + 0 + ); + assert!( + storage + .maintain_visibility(bounds(1)) + .await + .unwrap() + .is_complete() + ); +} + +#[tokio::test] +async fn visibility_index_malformed_progress_or_missing_source_tail_cannot_claim_complete() { + let storage = store(generation()).await; + storage.admit(raw(profile("first", 10))).await.unwrap(); + finish(&storage).await; + sqlx::query("PRAGMA ignore_check_constraints = ON") + .execute(&storage.pool) + .await + .unwrap(); + sqlx::query("UPDATE radroots_runtime_visibility_progress SET schema_version = 2") + .execute(&storage.pool) + .await + .unwrap(); + assert_eq!( + storage.maintain_visibility(bounds(1)).await, + Err(Error::CorruptStoredEvent) + ); + sqlx::query("UPDATE radroots_runtime_visibility_progress SET schema_version = 1") + .execute(&storage.pool) + .await + .unwrap(); + sqlx::query("DELETE FROM radroots_runtime_visibility_progress") + .execute(&storage.pool) + .await + .unwrap(); + sqlx::query("UPDATE radroots_runtime_source_generations SET sequence_head = 2") + .execute(&storage.pool) + .await + .unwrap(); + assert!( + !storage + .maintain_visibility(bounds(1)) + .await + .unwrap() + .is_complete() + ); + assert_eq!( + storage.maintain_visibility(bounds(1)).await, + Err(Error::CorruptStoredEvent) + ); +} + +#[tokio::test] +async fn visibility_index_memory_and_sqlite_progress_agree_across_same_count_promotion() { + let storage = store(generation()).await; + let memory = radroots_storage::memory::MemoryStorage::new(generation()); + let first = profile("first", 10); + let initial = [raw(first.clone()), raw(profile("large", 20))]; + for admission in initial { + storage.admit(admission.clone()).await.unwrap(); + memory.admit(admission).await.unwrap(); + } + for budget in [ + VisibilityIndexBounds::new( + 2, + 4096 + 2 * 128 + VisibilityIndexBounds::charge(first.raw_json().len() as u64).unwrap(), + ) + .unwrap(), + bounds(1), + ] { + assert_eq!( + storage.maintain_visibility(budget).await.unwrap(), + memory.maintain_visibility(budget).await.unwrap() + ); + } + for admission in [admitted(first.clone()), admitted(profile("later", 30))] { + storage.admit(admission.clone()).await.unwrap(); + memory.admit(admission).await.unwrap(); + } + for _ in 0..3 { + let sql = storage.maintain_visibility(bounds(1)).await.unwrap(); + let reference = memory.maintain_visibility(bounds(1)).await.unwrap(); + assert_eq!(sql, reference); + if sql.is_complete() { + break; + } + } + assert!( + storage + .maintain_visibility(bounds(1)) + .await + .unwrap() + .is_complete() + ); + assert_eq!( + storage.rebuild_visibility().await.unwrap(), + memory.rebuild_visibility().await.unwrap() + ); + assert_eq!( + indexed_visible(&storage).await, + memory + .rebuild_visibility() + .await + .unwrap() + .visible_event_ids() + ); +} diff --git a/crates/storage_sqlite/src/migration.rs b/crates/storage_sqlite/src/migration.rs @@ -469,7 +469,7 @@ fn schema_metadata_error(source: &sqlx::Error, database: &'static str) -> Error fn validate_plan(plan: &MigrationPlan) -> Result<(), Error> { let valid = plan.minimum_version > 0 && plan.minimum_version <= plan.current_version - && plan.current_version <= 18 + && plan.current_version <= 19 && plan.steps.len() == usize::try_from(plan.current_version).unwrap_or(usize::MAX) && plan .steps @@ -594,6 +594,7 @@ const fn set_user_version_sql(version: u32) -> Option<&'static str> { 16 => Some("PRAGMA user_version = 16"), 17 => Some("PRAGMA user_version = 17"), 18 => Some("PRAGMA user_version = 18"), + 19 => Some("PRAGMA user_version = 19"), _ => None, } } @@ -878,8 +879,8 @@ mod tests { .await .expect("application id"); let newer_version = runtime::CURRENT_VERSION + 1; - assert_eq!(newer_version, 19); - sqlx::raw_sql("PRAGMA user_version = 19") + assert_eq!(newer_version, 20); + sqlx::raw_sql("PRAGMA user_version = 20") .execute(&mut newer) .await .expect("newer version"); diff --git a/crates/storage_sqlite/src/migration/runtime/0019_canonical_visibility_index.up.sql b/crates/storage_sqlite/src/migration/runtime/0019_canonical_visibility_index.up.sql @@ -0,0 +1,111 @@ +ALTER TABLE radroots_runtime_events +ADD COLUMN last_mutation_epoch INTEGER NOT NULL DEFAULT 0 CHECK (last_mutation_epoch >= 0); + +CREATE INDEX radroots_runtime_events_mutation_idx +ON radroots_runtime_events(source_generation, last_mutation_epoch, source_sequence); + +CREATE TABLE radroots_runtime_visibility_progress ( + source_generation BLOB PRIMARY KEY NOT NULL + REFERENCES radroots_runtime_source_generations(generation), + schema_version INTEGER NOT NULL DEFAULT 1 CHECK (schema_version = 1), + backfill_target INTEGER NOT NULL CHECK (backfill_target >= 0), + backfill_sequence INTEGER NOT NULL DEFAULT 0 CHECK (backfill_sequence >= 0), + replay_epoch INTEGER NOT NULL CHECK (replay_epoch >= 0), + replay_sequence INTEGER NOT NULL CHECK (replay_sequence >= 0), + ready INTEGER NOT NULL DEFAULT 0 CHECK (ready IN (0, 1)), + CHECK (backfill_sequence <= backfill_target) +) STRICT, WITHOUT ROWID; + +CREATE TABLE radroots_runtime_visibility_events ( + source_generation BLOB NOT NULL + REFERENCES radroots_runtime_source_generations(generation), + source_sequence INTEGER NOT NULL CHECK (source_sequence > 0), + event_id BLOB NOT NULL UNIQUE REFERENCES radroots_runtime_events(event_id), + last_mutation_epoch INTEGER NOT NULL CHECK (last_mutation_epoch >= 0), + author BLOB NOT NULL CHECK (length(author) = 32), + created_at BLOB NOT NULL CHECK (length(created_at) = 8), + admission_stage TEXT NOT NULL CHECK (admission_stage IN ('raw', 'verified', 'visible')), + head_coordinate BLOB, + deletion_coordinate TEXT, + ephemeral INTEGER NOT NULL CHECK (ephemeral IN (0, 1)), + deletion_request INTEGER NOT NULL CHECK (deletion_request IN (0, 1)), + PRIMARY KEY (source_generation, source_sequence) +) STRICT, WITHOUT ROWID; + +CREATE INDEX radroots_runtime_visibility_events_head_idx +ON radroots_runtime_visibility_events( + source_generation, head_coordinate, created_at DESC, event_id ASC +); + +CREATE INDEX radroots_runtime_visibility_events_stage_idx +ON radroots_runtime_visibility_events(source_generation, admission_stage, source_sequence); + +CREATE TABLE radroots_runtime_visibility_deletion_events ( + source_generation BLOB NOT NULL, + target_event BLOB NOT NULL CHECK (length(target_event) = 32), + author BLOB NOT NULL CHECK (length(author) = 32), + request_id BLOB NOT NULL + REFERENCES radroots_runtime_visibility_events(event_id) ON DELETE CASCADE, + PRIMARY KEY (source_generation, target_event, author, request_id) +) STRICT, WITHOUT ROWID; + +CREATE INDEX radroots_runtime_visibility_deletion_events_request_idx +ON radroots_runtime_visibility_deletion_events(request_id); + +CREATE TABLE radroots_runtime_visibility_deletion_addresses ( + source_generation BLOB NOT NULL, + target_coordinate TEXT NOT NULL, + author BLOB NOT NULL CHECK (length(author) = 32), + created_at BLOB NOT NULL CHECK (length(created_at) = 8), + request_id BLOB NOT NULL + REFERENCES radroots_runtime_visibility_events(event_id) ON DELETE CASCADE, + PRIMARY KEY (source_generation, target_coordinate, author, created_at, request_id) +) STRICT, WITHOUT ROWID; + +CREATE INDEX radroots_runtime_visibility_deletion_addresses_request_idx +ON radroots_runtime_visibility_deletion_addresses(request_id); + +DROP TRIGGER radroots_runtime_events_mutation_insert; +DROP TRIGGER radroots_runtime_events_mutation_update; + +CREATE TRIGGER radroots_runtime_events_mutation_insert +AFTER INSERT ON radroots_runtime_events +BEGIN + SELECT CASE WHEN ( + SELECT mutation_epoch FROM radroots_runtime_source_generations + WHERE generation = NEW.source_generation + ) = 9223372036854775807 + THEN RAISE(ABORT, 'runtime event mutation epoch overflow') END; + UPDATE radroots_runtime_source_generations + SET mutation_epoch = mutation_epoch + 1 + WHERE generation = NEW.source_generation; + UPDATE radroots_runtime_events SET last_mutation_epoch = ( + SELECT mutation_epoch FROM radroots_runtime_source_generations + WHERE generation = NEW.source_generation + ) WHERE event_id = NEW.event_id; + UPDATE radroots_runtime_visibility_progress SET ready = 0 + WHERE source_generation = NEW.source_generation; +END; + +CREATE TRIGGER radroots_runtime_events_mutation_update +AFTER UPDATE OF admission_stage, admitted_contract_id, admitted_registry_version +ON radroots_runtime_events +WHEN NEW.admission_stage IS NOT OLD.admission_stage + OR NEW.admitted_contract_id IS NOT OLD.admitted_contract_id + OR NEW.admitted_registry_version IS NOT OLD.admitted_registry_version +BEGIN + SELECT CASE WHEN ( + SELECT mutation_epoch FROM radroots_runtime_source_generations + WHERE generation = NEW.source_generation + ) = 9223372036854775807 + THEN RAISE(ABORT, 'runtime event mutation epoch overflow') END; + UPDATE radroots_runtime_source_generations + SET mutation_epoch = mutation_epoch + 1 + WHERE generation = NEW.source_generation; + UPDATE radroots_runtime_events SET last_mutation_epoch = ( + SELECT mutation_epoch FROM radroots_runtime_source_generations + WHERE generation = NEW.source_generation + ) WHERE event_id = NEW.event_id; + UPDATE radroots_runtime_visibility_progress SET ready = 0 + WHERE source_generation = NEW.source_generation; +END; diff --git a/crates/storage_sqlite/src/migration/runtime/mod.rs b/crates/storage_sqlite/src/migration/runtime/mod.rs @@ -6,7 +6,7 @@ /// Lowest runtime schema version this package can recognize. pub const MINIMUM_VERSION: u32 = 1; /// Current runtime schema version created by this package. -pub const CURRENT_VERSION: u32 = 18; +pub const CURRENT_VERSION: u32 = 19; const RUNTIME_V1_SQL: &str = include_str!("0001_runtime.up.sql"); const CANONICAL_EVENT_STORAGE_V2_SQL: &str = include_str!("0002_canonical_event_storage.up.sql"); @@ -35,6 +35,9 @@ const AUTHORED_DELIVERY_RECONCILIATION_V17_SQL: &str = const EVENT_MUTATION_EPOCH_V18_SQL: &str = include_str!("0018_event_mutation_epoch.up.sql"); +const CANONICAL_VISIBILITY_INDEX_V19_SQL: &str = + include_str!("0019_canonical_visibility_index.up.sql"); + /// Stable, non-SQL description of one forward runtime migration. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct MigrationDescriptor { @@ -989,6 +992,108 @@ const RUNTIME_V18_OBJECTS: &[&str] = &[ "radroots_runtime_source_generations_sequence_guard", ]; +const RUNTIME_V19_OBJECTS: &[&str] = &[ + "radroots_runtime_atomic_commits", + "radroots_runtime_authored_artifacts", + "radroots_runtime_authored_artifacts_admission_ready_idx", + "radroots_runtime_authored_artifacts_signed_fact_guard", + "radroots_runtime_authored_artifacts_signing_ready_idx", + "radroots_runtime_authored_atomic_commits", + "radroots_runtime_authored_atomic_commits_delete_guard", + "radroots_runtime_authored_atomic_commits_update_guard", + "radroots_runtime_authored_atomic_target_phase_idx", + "radroots_runtime_authored_delivery_attempts", + "radroots_runtime_authored_delivery_claims", + "radroots_runtime_authored_delivery_claims_delete_guard", + "radroots_runtime_authored_delivery_claims_update_guard", + "radroots_runtime_authored_delivery_facts", + "radroots_runtime_authored_delivery_facts_delete_guard", + "radroots_runtime_authored_delivery_facts_update_guard", + "radroots_runtime_authored_delivery_plans", + "radroots_runtime_authored_delivery_ready_idx", + "radroots_runtime_authored_delivery_reconciliations", + "radroots_runtime_authored_delivery_reconciliations_delete_guard", + "radroots_runtime_authored_delivery_reconciliations_update_guard", + "radroots_runtime_authored_delivery_stop_guard", + "radroots_runtime_authored_delivery_targets", + "radroots_runtime_authored_draft_author_head_idx", + "radroots_runtime_authored_draft_revisions", + "radroots_runtime_authored_draft_revisions_delete_guard", + "radroots_runtime_authored_draft_revisions_update_guard", + "radroots_runtime_authored_draft_scope_head_idx", + "radroots_runtime_authored_migration_evidence", + "radroots_runtime_authored_migration_evidence_delete_guard", + "radroots_runtime_authored_migration_evidence_update_guard", + "radroots_runtime_authored_operations", + "radroots_runtime_delivery_evidence", + "radroots_runtime_delivery_evidence_item_idx", + "radroots_runtime_event_index_checkpoints", + "radroots_runtime_event_index_manifests", + "radroots_runtime_event_index_shards", + "radroots_runtime_event_provenance", + "radroots_runtime_event_provenance_observed_idx", + "radroots_runtime_events", + "radroots_runtime_events_admission_idx", + "radroots_runtime_events_contract_metadata_guard", + "radroots_runtime_events_contract_metadata_insert_guard", + "radroots_runtime_events_delete_guard", + "radroots_runtime_events_event_id_idx", + "radroots_runtime_events_mutation_idx", + "radroots_runtime_events_mutation_insert", + "radroots_runtime_events_mutation_update", + "radroots_runtime_events_raw_update_guard", + "radroots_runtime_journal_idempotency_idx", + "radroots_runtime_journal_operations", + "radroots_runtime_journal_recovery_idx", + "radroots_runtime_legacy_event_staging", + "radroots_runtime_legacy_event_staging_delete_guard", + "radroots_runtime_legacy_event_staging_insert_guard", + "radroots_runtime_legacy_event_staging_update_guard", + "radroots_runtime_legacy_import_commit_delete_guard", + "radroots_runtime_legacy_import_commit_update_guard", + "radroots_runtime_legacy_import_commits", + "radroots_runtime_legacy_import_delete_guard", + "radroots_runtime_legacy_import_identity_guard", + "radroots_runtime_legacy_import_member_delete_guard", + "radroots_runtime_legacy_import_member_identity_guard", + "radroots_runtime_legacy_import_member_state_guard", + "radroots_runtime_legacy_import_members", + "radroots_runtime_legacy_import_state_guard", + "radroots_runtime_legacy_import_state_idx", + "radroots_runtime_legacy_imports", + "radroots_runtime_legacy_outbox_staging", + "radroots_runtime_legacy_outbox_staging_delete_guard", + "radroots_runtime_legacy_outbox_staging_insert_guard", + "radroots_runtime_legacy_outbox_staging_parent_idx", + "radroots_runtime_legacy_outbox_staging_update_guard", + "radroots_runtime_outbox_items", + "radroots_runtime_outbox_operation_idx", + "radroots_runtime_outbox_ready_idx", + "radroots_runtime_outbox_targets", + "radroots_runtime_projection_checkpoints", + "radroots_runtime_projection_documents", + "radroots_runtime_projection_invalidations", + "radroots_runtime_projection_rebuilds", + "radroots_runtime_projection_rebuilds_stage_idx", + "radroots_runtime_projection_snapshots", + "radroots_runtime_projection_snapshots_created_idx", + "radroots_runtime_projection_snapshots_update_guard", + "radroots_runtime_source_generations", + "radroots_runtime_source_generations_active_idx", + "radroots_runtime_source_generations_delete_guard", + "radroots_runtime_source_generations_identity_guard", + "radroots_runtime_source_generations_mutation_guard", + "radroots_runtime_source_generations_sequence_guard", + "radroots_runtime_visibility_deletion_addresses", + "radroots_runtime_visibility_deletion_addresses_request_idx", + "radroots_runtime_visibility_deletion_events", + "radroots_runtime_visibility_deletion_events_request_idx", + "radroots_runtime_visibility_events", + "radroots_runtime_visibility_events_head_idx", + "radroots_runtime_visibility_events_stage_idx", + "radroots_runtime_visibility_progress", +]; + /// Ordered, immutable runtime migration plan. pub const MIGRATIONS: &[MigrationDescriptor] = &[ MigrationDescriptor { @@ -1099,6 +1204,12 @@ pub const MIGRATIONS: &[MigrationDescriptor] = &[ up_sha256: "44e8f43722bb869f824dbfa28197d17c1e8f58c6d62b8219cdd19d452eb47c3c", owned_objects: RUNTIME_V18_OBJECTS, }, + MigrationDescriptor { + version: 19, + name: "canonical_visibility_index", + up_sha256: "fd574ba0459de5a8c36033d5332dbd647888ef35e0213bc4ee88656ac2ef49bb", + owned_objects: RUNTIME_V19_OBJECTS, + }, ]; pub(crate) const fn migration_sql(version: u32) -> Option<&'static str> { @@ -1121,6 +1232,7 @@ pub(crate) const fn migration_sql(version: u32) -> Option<&'static str> { 16 => Some(AUTHORED_DELIVERY_FACTS_V16_SQL), 17 => Some(AUTHORED_DELIVERY_RECONCILIATION_V17_SQL), 18 => Some(EVENT_MUTATION_EPOCH_V18_SQL), + 19 => Some(CANONICAL_VISIBILITY_INDEX_V19_SQL), _ => None, } } @@ -1164,8 +1276,8 @@ mod tests { fn migration_plan_matches_governed_snapshot() { let snapshot = toml::from_str::<PlanSnapshot>(PLAN_SNAPSHOT).expect("valid snapshot"); assert_eq!(MINIMUM_VERSION, 1); - assert_eq!(CURRENT_VERSION, 18); - assert_eq!(MIGRATIONS.len(), 18); + assert_eq!(CURRENT_VERSION, 19); + assert_eq!(MIGRATIONS.len(), 19); let migration = MIGRATIONS[8]; assert_eq!(snapshot.schema_version, 1); assert_eq!(snapshot.database, "runtime.sqlite");