commit 690b6b17f2040a879fb33ba643c6a4d7e44ebe76
parent e6561a0cc7a8f6fde416d37dd15c9612ce312770
Author: triesap <tyson@radroots.org>
Date: Sat, 10 Oct 2026 04:07:43 +0000
storage: prepare canonical visibility indexes in resumable chunks
- Add validated work bounds and passive preparation progress to the event SPI.
- Track canonical mutation obligations within a forward SQLite migration.
- Retain backfill progress and replay later changes within row and byte bounds.
- Verify provider parity, rollback, durable reopen and API and coverage gates.
Diffstat:
16 files changed, 1608 insertions(+), 6 deletions(-)
diff --git a/contracts/api_baselines/radroots_storage.txt b/contracts/api_baselines/radroots_storage.txt
@@ -847,6 +847,19 @@ pub const fn radroots_storage::event::StoredVisibleEvent::position(&self) -> rad
pub struct radroots_storage::event::VisibilityDigest(_)
impl radroots_storage::event::VisibilityDigest
pub const fn radroots_storage::event::VisibilityDigest::as_bytes(&self) -> &[u8; 32]
+pub struct radroots_storage::event::VisibilityIndexBounds
+impl radroots_storage::event::VisibilityIndexBounds
+pub const fn radroots_storage::event::VisibilityIndexBounds::bytes(self) -> u64
+pub fn radroots_storage::event::VisibilityIndexBounds::new(u16, u64) -> core::result::Result<Self, radroots_storage::Error>
+pub const fn radroots_storage::event::VisibilityIndexBounds::rows(self) -> u16
+pub struct radroots_storage::event::VisibilityIndexProgress
+impl radroots_storage::event::VisibilityIndexProgress
+pub const fn radroots_storage::event::VisibilityIndexProgress::charged_bytes(self) -> u64
+pub const fn radroots_storage::event::VisibilityIndexProgress::is_complete(self) -> bool
+pub const fn radroots_storage::event::VisibilityIndexProgress::processed_rows(self) -> u16
+pub const fn radroots_storage::event::VisibilityIndexProgress::replay_epoch(self) -> u64
+pub const fn radroots_storage::event::VisibilityIndexProgress::source_sequence(self) -> u64
+pub const fn radroots_storage::event::VisibilityIndexProgress::token(self) -> radroots_storage::event::EventSourceToken
pub struct radroots_storage::event::VisibilitySnapshot
impl radroots_storage::event::VisibilitySnapshot
pub fn radroots_storage::event::VisibilitySnapshot::current_heads(&self) -> &[radroots_event::envelope::event_head::v1::CurrentEventHead]
@@ -860,6 +873,7 @@ pub const radroots_storage::event::EVENT_QUERY_ID_MAX: usize
pub const radroots_storage::event::EVENT_QUERY_LIMIT_MAX: u16
pub trait radroots_storage::event::EventStore: core::marker::Send + core::marker::Sync
pub fn radroots_storage::event::EventStore::admit(&self, radroots_storage::event::EventAdmission) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::AdmissionReceipt, radroots_storage::Error>>
+pub fn radroots_storage::event::EventStore::maintain_visibility(&self, radroots_storage::event::VisibilityIndexBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilityIndexProgress, radroots_storage::Error>>
pub fn radroots_storage::event::EventStore::query_provenance(&self, radroots_event::id::EventId, radroots_storage::event::EventQueryBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, radroots_storage::Error>>
pub fn radroots_storage::event::EventStore::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>>
pub fn radroots_storage::event::EventStore::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>>
@@ -869,6 +883,7 @@ pub fn radroots_storage::event::EventStore::source_token(&self) -> radroots_tran
pub fn radroots_storage::event::EventStore::status(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::status::EventStoreStatus, radroots_storage::Error>>
impl radroots_storage::event::EventStore for radroots_storage::memory::MemoryStorage
pub fn radroots_storage::memory::MemoryStorage::admit(&self, radroots_storage::event::EventAdmission) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::AdmissionReceipt, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::maintain_visibility(&self, radroots_storage::event::VisibilityIndexBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilityIndexProgress, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_provenance(&self, radroots_event::id::EventId, radroots_storage::event::EventQueryBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>>
@@ -1031,6 +1046,7 @@ pub fn radroots_storage::memory::MemoryStorage::transition_restore(&self, radroo
pub fn radroots_storage::memory::MemoryStorage::verify_backup(&self, radroots_storage::backup::BackupPlan, radroots_storage::backup::BackupManifest) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::backup::BackupCapabilityError>>
impl radroots_storage::event::EventStore for radroots_storage::memory::MemoryStorage
pub fn radroots_storage::memory::MemoryStorage::admit(&self, radroots_storage::event::EventAdmission) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::AdmissionReceipt, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::maintain_visibility(&self, radroots_storage::event::VisibilityIndexBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilityIndexProgress, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_provenance(&self, radroots_event::id::EventId, radroots_storage::event::EventQueryBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>>
@@ -1821,6 +1837,7 @@ pub radroots_storage::Error::InvalidRestoreTransition
pub radroots_storage::Error::InvalidRetrySchedule
pub radroots_storage::Error::InvalidSourceGeneration
pub radroots_storage::Error::InvalidStorageStatus
+pub radroots_storage::Error::InvalidVisibilityIndexBounds
pub radroots_storage::Error::InvalidWorkClaim
pub radroots_storage::Error::InvalidWorkFailure
pub radroots_storage::Error::JournalOperationCommitted
@@ -1895,6 +1912,7 @@ pub fn radroots_storage::memory::MemoryStorage::transition_restore(&self, radroo
pub fn radroots_storage::memory::MemoryStorage::verify_backup(&self, radroots_storage::backup::BackupPlan, radroots_storage::backup::BackupManifest) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::backup::BackupCapabilityError>>
pub trait radroots_storage::EventStore: core::marker::Send + core::marker::Sync
pub fn radroots_storage::EventStore::admit(&self, radroots_storage::event::EventAdmission) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::AdmissionReceipt, radroots_storage::Error>>
+pub fn radroots_storage::EventStore::maintain_visibility(&self, radroots_storage::event::VisibilityIndexBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilityIndexProgress, radroots_storage::Error>>
pub fn radroots_storage::EventStore::query_provenance(&self, radroots_event::id::EventId, radroots_storage::event::EventQueryBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, radroots_storage::Error>>
pub fn radroots_storage::EventStore::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>>
pub fn radroots_storage::EventStore::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>>
@@ -1904,6 +1922,7 @@ pub fn radroots_storage::EventStore::source_token(&self) -> radroots_transport::
pub fn radroots_storage::EventStore::status(&self) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::status::EventStoreStatus, radroots_storage::Error>>
impl radroots_storage::event::EventStore for radroots_storage::memory::MemoryStorage
pub fn radroots_storage::memory::MemoryStorage::admit(&self, radroots_storage::event::EventAdmission) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::AdmissionReceipt, radroots_storage::Error>>
+pub fn radroots_storage::memory::MemoryStorage::maintain_visibility(&self, radroots_storage::event::VisibilityIndexBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::VisibilityIndexProgress, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_provenance(&self, radroots_event::id::EventId, radroots_storage::event::EventQueryBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::Error>>
pub fn radroots_storage::memory::MemoryStorage::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::Error>>
diff --git a/contracts/api_baselines/radroots_storage_sqlite.txt b/contracts/api_baselines/radroots_storage_sqlite.txt
@@ -581,6 +581,7 @@ pub fn radroots_storage_sqlite::SqliteStorage::transition_restore(&self, radroot
pub fn radroots_storage_sqlite::SqliteStorage::verify_backup(&self, radroots_storage::backup::BackupPlan, radroots_storage::backup::BackupManifest) -> radroots_transport::source::BoxFuture<'_, core::result::Result<(), radroots_storage::backup::capability::BackupCapabilityError>>
impl radroots_storage::event::EventStore for radroots_storage_sqlite::SqliteStorage
pub fn radroots_storage_sqlite::SqliteStorage::admit(&self, radroots_storage::event::EventAdmission) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::AdmissionReceipt, radroots_storage::error::Error>>
+pub fn radroots_storage_sqlite::SqliteStorage::maintain_visibility(&self, radroots_storage::event::visibility_index::VisibilityIndexBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::visibility_index::VisibilityIndexProgress, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::query_provenance(&self, radroots_event::id::EventId, radroots_storage::event::EventQueryBounds) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::query_raw(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, radroots_storage::error::Error>>
pub fn radroots_storage_sqlite::SqliteStorage::query_verified(&self, radroots_storage::event::EventQuery) -> radroots_transport::source::BoxFuture<'_, core::result::Result<radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, radroots_storage::error::Error>>
diff --git a/contracts/architecture/decisions/canonical_visibility_index.v1.json b/contracts/architecture/decisions/canonical_visibility_index.v1.json
@@ -0,0 +1,42 @@
+{
+ "schema_version": 1,
+ "decision_id": "radroots.storage.canonical_visibility_index.v1",
+ "status": "implemented_not_published",
+ "owners": ["radroots_storage", "radroots_storage_sqlite"],
+ "public_boundary": {
+ "maintenance": "EventStore::maintain_visibility",
+ "bounds": "VisibilityIndexBounds",
+ "progress": "VisibilityIndexProgress",
+ "unsupported_provider": "BackendUnavailable",
+ "invalid_bounds": "InvalidVisibilityIndexBounds",
+ "capacity": "SpaceInsufficient",
+ "maximum_source_rows": 64,
+ "maximum_logical_bytes": 16777216,
+ "charge": "4096 + maximum_source_rows * 128 + sum(signed_bytes * 16 + 4096)",
+ "hidden_metadata": "VisibilityEventMetadata reuses existing head and deletion parsing; it carries no application policy"
+ },
+ "durability": {
+ "forward_runtime_version": 19,
+ "immutable_prefix": "versions 1 through 18",
+ "canonical_truth": "signed event rows remain authoritative and immutable",
+ "mutation": "combined SQL triggers atomically advance generation epoch, event last-change epoch and dirty visibility obligation",
+ "backfill": "fixed initial sequence target with durable sequence cursor",
+ "legacy_change_epoch": "zero marks source rows predating this index; the fixed initial backfill prepares them without a whole-source UPDATE or payload decode in migration",
+ "replay": "ordered last-change epoch and source-sequence cursor; updates to previously scanned rows are replayed without resetting the initial cursor",
+ "completion": "all initial rows and subsequent changed rows prepared at the transaction's active source token",
+ "transaction": "one bounded immediate transaction; cancellation or failure before commit rolls back metadata and progress",
+ "retired_source": "SourceGenerationChanged",
+ "restore": "verified whole-file replacement preserves metadata and source together, closes old handles, and requires consumer runtime invalidation separately"
+ },
+ "semantics": [
+ "verified unsupported heads retain suppression authority",
+ "raw events have no head or deletion authority",
+ "only visible authorized deletion requests contribute targets",
+ "same-author event and address deletion semantics reuse the reference parser",
+ "created_at is stored as ordered unsigned 8-byte big-endian metadata",
+ "missing address identifier behavior matches the reference evaluator",
+ "no partially prepared index is advertised complete"
+ ],
+ "separate_followup": "Indexed visible/status providers and bounded repeated-read qualification remain a separate checkpoint",
+ "forbidden": ["raw SQL public API", "application ranking", "second database", "hidden worker", "unbounded fallback", "protocol registry revision"]
+}
diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs
@@ -22,6 +22,7 @@ pub enum Error {
InvalidSourceGeneration,
InvalidEventSequence,
InvalidEventQueryLimit,
+ InvalidVisibilityIndexBounds,
EmptyEventQueryIds,
TooManyEventQueryIds,
DuplicateEventQueryId,
@@ -158,6 +159,7 @@ impl fmt::Display for Error {
Self::InvalidEventSequence => "storage event sequence is invalid",
Self::EventMutationOverflow => "storage event mutation epoch overflowed",
Self::InvalidEventQueryLimit => "storage event query limit is invalid",
+ Self::InvalidVisibilityIndexBounds => "storage visibility index bounds are invalid",
Self::EmptyEventQueryIds => "storage event id query is empty",
Self::TooManyEventQueryIds => "storage event id query exceeds its limit",
Self::DuplicateEventQueryId => "storage event id query contains a duplicate",
diff --git a/crates/storage/src/event.rs b/crates/storage/src/event.rs
@@ -13,8 +13,12 @@ use std::collections::BTreeSet;
use crate::{Error, status::EventStoreStatus};
mod visibility;
+mod visibility_index;
+#[doc(hidden)]
+pub use visibility::VisibilityEventMetadata;
#[doc(hidden)]
pub use visibility::{VisibilityEvaluation, VisibilityInput, evaluate_visibility};
+pub use visibility_index::{VisibilityIndexBounds, VisibilityIndexProgress};
/// Maximum events returned by one storage query.
pub const EVENT_QUERY_LIMIT_MAX: u16 = 1_000;
@@ -626,6 +630,17 @@ impl StoredEventProvenance {
/// idempotent for an identical event and provenance observation; a stage may
/// advance but never regress. Query cursors are bound to one source generation.
pub trait EventStore: Send + Sync {
+ /// Prepares one bounded, transactionally resumable canonical index chunk.
+ ///
+ /// Completion refers to canonical metadata at the returned source token,
+ /// not to any application projection. It never initiates a hidden task.
+ fn maintain_visibility(
+ &self,
+ _bounds: VisibilityIndexBounds,
+ ) -> BoxFuture<'_, Result<VisibilityIndexProgress, Error>> {
+ Box::pin(async { Err(Error::BackendUnavailable) })
+ }
+
/// Reads the current source token without decoding retained event history.
///
/// Providers that cannot supply this fence fail closed. SQLite epochs have
diff --git a/crates/storage/src/event/visibility.rs b/crates/storage/src/event/visibility.rs
@@ -79,6 +79,86 @@ struct DeletionRequest {
address_targets: BTreeSet<Nip01Coordinate>,
}
+/// Canonical provider metadata extracted with the reference reducer's parsing.
+#[doc(hidden)]
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct VisibilityEventMetadata {
+ pub event_id: EventId,
+ pub author: [u8; 32],
+ pub created_at: u64,
+ pub stage: AdmissionStage,
+ pub head_coordinate: Option<Vec<u8>>,
+ pub deletion_coordinate: Option<String>,
+ pub ephemeral: bool,
+ pub deletion_request: bool,
+ pub event_targets: BTreeSet<EventId>,
+ pub address_targets: BTreeSet<String>,
+}
+
+impl VisibilityEventMetadata {
+ #[doc(hidden)]
+ pub fn from_event(event: &SignedEvent, stage: AdmissionStage) -> Result<Self, Error> {
+ let envelope = event.envelope();
+ let (head_coordinate, ephemeral) = if stage == AdmissionStage::Raw {
+ (None, false)
+ } else {
+ match event_head_candidate_for_nip01_event(envelope) {
+ EventHeadCandidateResult::Candidate(candidate) => {
+ let mut key = Vec::new();
+ match candidate.coordinate {
+ EventHeadCoordinate::Replaceable { kind, pubkey } => {
+ key.push(0);
+ key.extend_from_slice(&kind.to_be_bytes());
+ key.extend_from_slice(pubkey.as_bytes());
+ }
+ EventHeadCoordinate::Addressable {
+ kind,
+ pubkey,
+ d_tag,
+ } => {
+ key.push(1);
+ key.extend_from_slice(&kind.to_be_bytes());
+ key.extend_from_slice(pubkey.as_bytes());
+ key.extend_from_slice(d_tag.as_bytes());
+ }
+ }
+ (Some(key), false)
+ }
+ EventHeadCandidateResult::NotHeadSelected => (None, false),
+ EventHeadCandidateResult::NotPersisted => (None, true),
+ EventHeadCandidateResult::Malformed(_) => return Err(Error::CorruptStoredEvent),
+ }
+ };
+ let deletion_request =
+ stage == AdmissionStage::Visible && envelope.kind_u32() == KIND_DELETION_REQUEST;
+ let (event_targets, address_targets) = if deletion_request {
+ let request = parse_deletion_request(event)?;
+ (
+ request.event_targets,
+ request
+ .address_targets
+ .into_iter()
+ .map(|coordinate| coordinate.as_str().to_owned())
+ .collect(),
+ )
+ } else {
+ (BTreeSet::new(), BTreeSet::new())
+ };
+ Ok(Self {
+ event_id: *event.id(),
+ author: *envelope.author().as_bytes(),
+ created_at: envelope.created_at_u64(),
+ stage,
+ head_coordinate,
+ deletion_coordinate: event_coordinate(envelope).map(|value| value.as_str().to_owned()),
+ ephemeral,
+ deletion_request,
+ event_targets,
+ address_targets,
+ })
+ }
+}
+
#[doc(hidden)]
pub fn evaluate_visibility<'a>(
generation: SourceGeneration,
diff --git a/crates/storage/src/event/visibility_index.rs b/crates/storage/src/event/visibility_index.rs
@@ -0,0 +1,120 @@
+//! Bounded canonical visibility-index preparation contracts.
+
+use super::EventSourceToken;
+use crate::Error;
+
+/// Validated logical work envelope for one canonical index preparation call.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct VisibilityIndexBounds {
+ rows: u16,
+ bytes: u64,
+}
+
+impl VisibilityIndexBounds {
+ /// Validates 1..=64 source rows and 1..=16 MiB of logical allocation charge.
+ pub fn new(rows: u16, bytes: u64) -> Result<Self, Error> {
+ if rows == 0 || rows > 64 || bytes == 0 || bytes > 16 * 1024 * 1024 {
+ return Err(Error::InvalidVisibilityIndexBounds);
+ }
+ Ok(Self { rows, bytes })
+ }
+
+ /// Maximum source rows materialized in this call.
+ pub const fn rows(self) -> u16 {
+ self.rows
+ }
+
+ /// Maximum conservative logical allocation charge in this call.
+ pub const fn bytes(self) -> u64 {
+ self.bytes
+ }
+
+ /// Charges retained bytes, decoded variable data and derived metadata.
+ ///
+ /// This conservative envelope is not a measurement of allocator RSS.
+ #[doc(hidden)]
+ pub fn charge(signed_bytes: u64) -> Result<u64, Error> {
+ signed_bytes
+ .checked_mul(16)
+ .and_then(|value| value.checked_add(4096))
+ .ok_or(Error::SpaceInsufficient)
+ }
+}
+
+/// Durable progress observed after one canonical index preparation transaction.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct VisibilityIndexProgress {
+ token: EventSourceToken,
+ source_sequence: u64,
+ replay_epoch: u64,
+ processed_rows: u16,
+ charged_bytes: u64,
+ complete: bool,
+}
+
+impl VisibilityIndexProgress {
+ #[doc(hidden)]
+ pub const fn new(
+ token: EventSourceToken,
+ source_sequence: u64,
+ replay_epoch: u64,
+ processed_rows: u16,
+ charged_bytes: u64,
+ complete: bool,
+ ) -> Self {
+ Self {
+ token,
+ source_sequence,
+ replay_epoch,
+ processed_rows,
+ charged_bytes,
+ complete,
+ }
+ }
+
+ /// Source fence read inside the completed preparation transaction.
+ pub const fn token(self) -> EventSourceToken {
+ self.token
+ }
+ /// Last initial-backfill sequence durably prepared; it never restarts on admission.
+ pub const fn source_sequence(self) -> u64 {
+ self.source_sequence
+ }
+ /// Last durable changed-event replay epoch.
+ pub const fn replay_epoch(self) -> u64 {
+ self.replay_epoch
+ }
+ /// Actual source rows materialized, including repeated changed rows.
+ pub const fn processed_rows(self) -> u16 {
+ self.processed_rows
+ }
+ /// Conservative logical allocation charge of this call.
+ pub const fn charged_bytes(self) -> u64 {
+ self.charged_bytes
+ }
+ /// Whether canonical metadata matches `token`; it is not a consumer-content claim.
+ pub const fn is_complete(self) -> bool {
+ self.complete
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn work_envelope_rejects_zero_and_one_over_without_relaxing_caps() {
+ assert!(VisibilityIndexBounds::new(64, 16 * 1024 * 1024).is_ok());
+ for (rows, bytes) in [(0, 1), (65, 1), (1, 0), (1, 16 * 1024 * 1024 + 1)] {
+ assert_eq!(
+ VisibilityIndexBounds::new(rows, bytes),
+ Err(Error::InvalidVisibilityIndexBounds)
+ );
+ }
+ assert_eq!(VisibilityIndexBounds::charge(10).unwrap(), 4256);
+ assert_eq!(
+ VisibilityIndexBounds::charge(u64::MAX),
+ Err(Error::SpaceInsufficient)
+ );
+ }
+}
diff --git a/crates/storage/src/memory.rs b/crates/storage/src/memory.rs
@@ -2,6 +2,8 @@
#[path = "memory_delivery_history.rs"]
mod delivery_history;
+#[path = "memory_visibility_index.rs"]
+mod visibility_index;
#[cfg(test)]
#[path = "memory_mutation_tests.rs"]
@@ -71,12 +73,15 @@ struct EventEntry {
position: EventPosition,
admission: EventAdmission,
provenance: Vec<EventProvenance>,
+ last_mutation_epoch: u64,
}
#[derive(Clone, Default)]
struct State {
events: Vec<EventEntry>,
event_mutation_epoch: u64,
+ event_changes: std::collections::BTreeMap<u64, u64>,
+ visibility_index: visibility_index::Index,
journal: Vec<OperationRecord>,
outbox: Vec<OutboxRecord>,
projections: Vec<ProjectionStatus>,
@@ -116,6 +121,8 @@ impl MemoryStorage {
state: Mutex::new(State {
events: Vec::new(),
event_mutation_epoch: 0,
+ event_changes: std::collections::BTreeMap::new(),
+ visibility_index: visibility_index::Index::new(),
journal: Vec::new(),
outbox: Vec::new(),
projections: Vec::new(),
@@ -230,6 +237,12 @@ impl MemoryStorage {
.event_mutation_epoch
.checked_add(1)
.ok_or(Error::EventMutationOverflow)?;
+ state.event_changes.remove(&entry.last_mutation_epoch);
+ entry.last_mutation_epoch = state.event_mutation_epoch;
+ state
+ .event_changes
+ .insert(entry.last_mutation_epoch, entry.position.sequence().get());
+ state.visibility_index.ready = false;
AdmissionDisposition::Advanced
};
if !entry.provenance.contains(admission.provenance()) {
@@ -263,8 +276,11 @@ impl MemoryStorage {
position,
admission,
provenance,
+ last_mutation_epoch: epoch,
});
state.event_mutation_epoch = epoch;
+ state.event_changes.insert(epoch, next);
+ state.visibility_index.ready = false;
Ok(receipt)
}
@@ -424,6 +440,16 @@ impl Default for MemoryStorage {
}
impl EventStore for MemoryStorage {
+ fn maintain_visibility(
+ &self,
+ bounds: crate::event::VisibilityIndexBounds,
+ ) -> BoxFuture<'_, Result<crate::event::VisibilityIndexProgress, Error>> {
+ Box::pin(async move {
+ let mut state = self.state()?;
+ visibility_index::prepare(self.generation, &mut state, bounds)
+ })
+ }
+
fn source_token(&self) -> BoxFuture<'_, Result<EventSourceToken, Error>> {
Box::pin(async move {
let state = self.state()?;
diff --git a/crates/storage/src/memory_visibility_index.rs b/crates/storage/src/memory_visibility_index.rs
@@ -0,0 +1,206 @@
+use super::State;
+use crate::{
+ Error,
+ event::{
+ EventSourceToken, SourceGeneration, VisibilityEventMetadata, VisibilityIndexBounds,
+ VisibilityIndexProgress,
+ },
+};
+use std::collections::BTreeMap;
+
+#[derive(Clone, Default)]
+pub(super) struct Index {
+ cursor: Option<Cursor>,
+ pub(super) ready: bool,
+ pub(super) events: BTreeMap<u64, VisibilityEventMetadata>,
+}
+
+#[derive(Clone, Copy)]
+struct Cursor {
+ target: u64,
+ backfill: u64,
+ replay_epoch: u64,
+}
+
+impl Index {
+ pub(super) const fn new() -> Self {
+ Self {
+ cursor: None,
+ ready: false,
+ events: BTreeMap::new(),
+ }
+ }
+}
+
+pub(super) fn prepare(
+ generation: SourceGeneration,
+ state: &mut State,
+ bounds: VisibilityIndexBounds,
+) -> Result<VisibilityIndexProgress, Error> {
+ let mut charged = 4096 + u64::from(bounds.rows()) * 128;
+ if charged > bounds.bytes() {
+ return Err(Error::SpaceInsufficient);
+ }
+ let sequence = u64::try_from(state.events.len()).map_err(|_| Error::CorruptStoredEvent)?;
+ let mut cursor = state.visibility_index.cursor.unwrap_or(Cursor {
+ target: sequence,
+ backfill: 0,
+ replay_epoch: state.event_mutation_epoch,
+ });
+ let building = cursor.backfill < cursor.target;
+ let positions = if building {
+ ((cursor.backfill + 1)..=cursor.target)
+ .take(usize::from(bounds.rows()))
+ .collect::<Vec<_>>()
+ } else {
+ state
+ .event_changes
+ .range((
+ std::ops::Bound::Excluded(cursor.replay_epoch),
+ std::ops::Bound::Unbounded,
+ ))
+ .take(usize::from(bounds.rows()))
+ .map(|(_, sequence)| *sequence)
+ .collect()
+ };
+ let mut prepared = Vec::new();
+ for position in positions {
+ let entry = state
+ .events
+ .get(usize::try_from(position - 1).map_err(|_| Error::CorruptStoredEvent)?)
+ .ok_or(Error::CorruptStoredEvent)?;
+ let bytes = u64::try_from(entry.admission.event().raw_json().len())
+ .map_err(|_| Error::SpaceInsufficient)?;
+ let next_charge = charged
+ .checked_add(VisibilityIndexBounds::charge(bytes)?)
+ .ok_or(Error::SpaceInsufficient)?;
+ if next_charge > bounds.bytes() {
+ if prepared.is_empty() {
+ return Err(Error::SpaceInsufficient);
+ }
+ break;
+ }
+ let metadata =
+ VisibilityEventMetadata::from_event(entry.admission.event(), entry.admission.stage())?;
+ prepared.push((position, metadata));
+ charged = next_charge;
+ if building {
+ cursor.backfill = position;
+ } else {
+ cursor.replay_epoch = entry.last_mutation_epoch;
+ }
+ }
+ let pending = state
+ .event_changes
+ .range((
+ std::ops::Bound::Excluded(cursor.replay_epoch),
+ std::ops::Bound::Unbounded,
+ ))
+ .next()
+ .is_some();
+ let complete = cursor.backfill == cursor.target && !pending;
+ if complete {
+ cursor.replay_epoch = state.event_mutation_epoch;
+ }
+ let processed = u16::try_from(prepared.len()).map_err(|_| Error::CorruptStoredEvent)?;
+ state.visibility_index.events.extend(prepared);
+ state.visibility_index.cursor = Some(cursor);
+ state.visibility_index.ready = complete;
+ Ok(VisibilityIndexProgress::new(
+ EventSourceToken::new(generation, state.event_mutation_epoch),
+ cursor.backfill,
+ cursor.replay_epoch,
+ processed,
+ charged,
+ complete,
+ ))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::{EventStore, memory::MemoryStorage};
+ use futures_executor::block_on;
+ use radroots_event::{SignedEvent, wire::Nip01EventWire};
+ use radroots_transport::{
+ Target, TransportId,
+ source::{EventProvenance, ObservedEvent},
+ };
+
+ fn admission(content: &str, time: u64) -> crate::event::EventAdmission {
+ let mut wire = Nip01EventWire {
+ id: "0".repeat(64),
+ pubkey: "585591529da0bab31b3b1b1f986611cf5f435dca84f978c89ee8a40cca7103df".to_owned(),
+ created_at: time,
+ kind: 0,
+ tags: vec![],
+ content: content.to_owned(),
+ sig: "42".repeat(64),
+ extra: Default::default(),
+ };
+ wire.id = wire.computed_event_id().unwrap().to_hex();
+ let raw = serde_json::to_string(&wire).unwrap();
+ let event = SignedEvent::from_wire_verified_id(wire, raw).unwrap();
+ let target = Target::nostr_relay("wss://relay.example").unwrap();
+ crate::event::EventAdmission::raw(ObservedEvent::new(
+ event,
+ EventProvenance::new(TransportId::NOSTR, target.fingerprint().clone(), 100).unwrap(),
+ ))
+ }
+
+ #[test]
+ fn memory_visibility_index_retains_backfill_and_refuses_oversized_rows_without_partial_state() {
+ let generation = SourceGeneration::new([82; 32]).unwrap();
+ let storage = MemoryStorage::new(generation);
+ let first = admission("first", 10);
+ block_on(storage.admit(first.clone())).unwrap();
+ block_on(storage.admit(admission("second", 20))).unwrap();
+ let bounds = VisibilityIndexBounds::new(1, 16 * 1024 * 1024).unwrap();
+ let part = block_on(storage.maintain_visibility(bounds)).unwrap();
+ assert_eq!(part.source_sequence(), 1);
+ assert!(!part.is_complete());
+ block_on(storage.admit(admission("third", 30))).unwrap();
+ let next = block_on(storage.maintain_visibility(bounds)).unwrap();
+ assert_eq!(next.source_sequence(), 2);
+ assert!(!next.is_complete());
+ assert!(
+ block_on(storage.maintain_visibility(bounds))
+ .unwrap()
+ .is_complete()
+ );
+ assert_eq!(storage.state().unwrap().visibility_index.events.len(), 3);
+ block_on(storage.admit(first)).unwrap();
+ assert_eq!(
+ block_on(storage.maintain_visibility(bounds))
+ .unwrap()
+ .processed_rows(),
+ 0
+ );
+ block_on(storage.admit(admission(&"x".repeat(1000), 40))).unwrap();
+ assert_eq!(
+ block_on(storage.maintain_visibility(VisibilityIndexBounds::new(1, 10000).unwrap())),
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(storage.state().unwrap().visibility_index.events.len(), 3);
+ assert!(!storage.state().unwrap().visibility_index.ready);
+ assert!(
+ block_on(storage.maintain_visibility(bounds))
+ .unwrap()
+ .is_complete()
+ );
+ }
+
+ #[test]
+ fn memory_visibility_index_empty_source_has_a_real_zero_epoch_fence() {
+ let generation = SourceGeneration::new([83; 32]).unwrap();
+ let storage = MemoryStorage::new(generation);
+ let progress = block_on(
+ storage.maintain_visibility(VisibilityIndexBounds::new(64, 16 * 1024 * 1024).unwrap()),
+ )
+ .unwrap();
+ assert!(progress.is_complete());
+ assert_eq!(progress.token(), EventSourceToken::new(generation, 0));
+ assert_eq!(progress.processed_rows(), 0);
+ assert!(storage.state().unwrap().visibility_index.events.is_empty());
+ }
+}
diff --git a/crates/storage/tests/event_store.rs b/crates/storage/tests/event_store.rs
@@ -70,6 +70,12 @@ fn unsupported_adapter_cannot_invent_a_source_fence() {
block_on(MemoryEventStore::new().source_token()),
Err(Error::BackendUnavailable)
);
+ assert_eq!(
+ block_on(MemoryEventStore::new().maintain_visibility(
+ radroots_storage::event::VisibilityIndexBounds::new(64, 16 * 1024 * 1024).unwrap()
+ )),
+ Err(Error::BackendUnavailable)
+ );
}
impl EventStore for MemoryEventStore {
diff --git a/crates/storage_sqlite/src/event/mod.rs b/crates/storage_sqlite/src/event/mod.rs
@@ -1,4 +1,7 @@
use crate::backend::map_backend;
+mod visibility_index;
+#[cfg(test)]
+mod visibility_index_tests;
use radroots_event::SignedEvent;
use radroots_event_codec::Codec;
use radroots_storage::{
@@ -397,6 +400,13 @@ fn admission_contract_metadata(admission: &EventAdmission) -> Option<(&'static s
#[cfg_attr(coverage_nightly, coverage(off))]
impl EventStore for SqliteStorage {
+ fn maintain_visibility(
+ &self,
+ bounds: radroots_storage::event::VisibilityIndexBounds,
+ ) -> BoxFuture<'_, Result<radroots_storage::event::VisibilityIndexProgress, Error>> {
+ Box::pin(async move { self.prepare_visibility_index(bounds).await })
+ }
+
fn source_token(&self) -> BoxFuture<'_, Result<EventSourceToken, Error>> {
Box::pin(async move {
let epoch = sqlx::query_scalar::<_, i64>(
diff --git a/crates/storage_sqlite/src/event/visibility_index.rs b/crates/storage_sqlite/src/event/visibility_index.rs
@@ -0,0 +1,225 @@
+use super::*;
+use radroots_storage::event::{
+ VisibilityEventMetadata, VisibilityIndexBounds, VisibilityIndexProgress,
+};
+
+impl SqliteStorage {
+ pub(super) async fn prepare_visibility_index(
+ &self,
+ bounds: VisibilityIndexBounds,
+ ) -> Result<VisibilityIndexProgress, Error> {
+ if self.mode == EventStoreMode::ReadOnly {
+ return Err(Error::BackendUnavailable);
+ }
+ let mut charged = 4096 + u64::from(bounds.rows()) * 128;
+ if charged > bounds.bytes() {
+ return Err(Error::SpaceInsufficient);
+ }
+ let mut transaction = self
+ .pool
+ .begin_with("BEGIN IMMEDIATE")
+ .await
+ .map_err(map_backend)?;
+ let source = sqlx::query(
+ "SELECT mutation_epoch, sequence_head FROM radroots_runtime_source_generations
+ WHERE generation = ? AND state = 'active'",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .fetch_optional(&mut *transaction)
+ .await
+ .map_err(map_backend)?
+ .ok_or(Error::SourceGenerationChanged)?;
+ let epoch: i64 = source.try_get("mutation_epoch").map_err(map_corrupt)?;
+ let sequence: i64 = source.try_get("sequence_head").map_err(map_corrupt)?;
+ sqlx::query(
+ "INSERT OR IGNORE INTO radroots_runtime_visibility_progress
+ (source_generation, backfill_target, replay_epoch, replay_sequence)
+ VALUES (?, ?, ?, 9223372036854775807)",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .bind(sequence)
+ .bind(epoch)
+ .execute(&mut *transaction)
+ .await
+ .map_err(map_backend)?;
+ let progress = sqlx::query(
+ "SELECT schema_version, backfill_target, backfill_sequence, replay_epoch, replay_sequence
+ FROM radroots_runtime_visibility_progress WHERE source_generation = ?",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .fetch_one(&mut *transaction).await.map_err(map_backend)?;
+ let version: i64 = progress.try_get("schema_version").map_err(map_corrupt)?;
+ let target: i64 = progress.try_get("backfill_target").map_err(map_corrupt)?;
+ let mut backfill: i64 = progress.try_get("backfill_sequence").map_err(map_corrupt)?;
+ let mut replay_epoch: i64 = progress.try_get("replay_epoch").map_err(map_corrupt)?;
+ let mut replay_sequence: i64 = progress.try_get("replay_sequence").map_err(map_corrupt)?;
+ if version != 1 || target > sequence || backfill > target || replay_epoch > epoch {
+ return Err(Error::CorruptStoredEvent);
+ }
+ let building = backfill < target;
+ let mut query = QueryBuilder::<Sqlite>::new(
+ "SELECT source_sequence, last_mutation_epoch, length(signed_event) AS signed_bytes
+ FROM radroots_runtime_events WHERE source_generation = ",
+ );
+ query.push_bind(self.generation.as_bytes().as_slice());
+ if building {
+ query.push(" AND source_sequence > ").push_bind(backfill);
+ query.push(" AND source_sequence <= ").push_bind(target);
+ query
+ .push(" ORDER BY source_sequence LIMIT ")
+ .push_bind(i64::from(bounds.rows()));
+ } else {
+ query.push(" AND (last_mutation_epoch, source_sequence) > (");
+ query
+ .push_bind(replay_epoch)
+ .push(", ")
+ .push_bind(replay_sequence)
+ .push(")");
+ query
+ .push(" ORDER BY last_mutation_epoch, source_sequence LIMIT ")
+ .push_bind(i64::from(bounds.rows()));
+ }
+ let lengths = query
+ .build()
+ .fetch_all(&mut *transaction)
+ .await
+ .map_err(map_backend)?;
+ if building && lengths.is_empty() {
+ return Err(Error::CorruptStoredEvent);
+ }
+ let mut processed = 0u16;
+ for length in lengths {
+ let position: i64 = length.try_get("source_sequence").map_err(map_corrupt)?;
+ let changed: i64 = length.try_get("last_mutation_epoch").map_err(map_corrupt)?;
+ let size = u64_from_i64(length.try_get("signed_bytes").map_err(map_corrupt)?)?;
+ let charge = VisibilityIndexBounds::charge(size)?;
+ let next_charge = charged
+ .checked_add(charge)
+ .ok_or(Error::SpaceInsufficient)?;
+ if next_charge > bounds.bytes() {
+ if processed == 0 {
+ return Err(Error::SpaceInsufficient);
+ }
+ break;
+ }
+ let row = sqlx::query(
+ "SELECT source_generation, source_sequence, signed_event, admission_stage,
+ admitted_contract_id, admitted_registry_version FROM radroots_runtime_events
+ WHERE source_generation = ? AND source_sequence = ?",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .bind(position)
+ .fetch_one(&mut *transaction)
+ .await
+ .map_err(map_backend)?;
+ let decoded = self.decode_event_row(&row)?;
+ if decoded.raw_json.len() as u64 != size || changed > epoch {
+ return Err(Error::CorruptStoredEvent);
+ }
+ let metadata = VisibilityEventMetadata::from_event(&decoded.event, decoded.stage)?;
+ self.put_visibility_metadata(&mut transaction, position, changed, metadata)
+ .await?;
+ charged = next_charge;
+ processed += 1;
+ if building {
+ backfill = position;
+ } else {
+ replay_epoch = changed;
+ replay_sequence = position;
+ }
+ }
+ let pending: bool = sqlx::query_scalar(
+ "SELECT EXISTS (SELECT 1 FROM radroots_runtime_events
+ WHERE source_generation = ? AND (last_mutation_epoch, source_sequence) > (?, ?))",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .bind(replay_epoch)
+ .bind(replay_sequence)
+ .fetch_one(&mut *transaction)
+ .await
+ .map_err(map_backend)?;
+ let complete = backfill == target && !pending;
+ if complete {
+ replay_epoch = epoch;
+ replay_sequence = i64::MAX;
+ }
+ sqlx::query(
+ "UPDATE radroots_runtime_visibility_progress SET backfill_sequence = ?,
+ replay_epoch = ?, replay_sequence = ?, ready = ? WHERE source_generation = ?",
+ )
+ .bind(backfill)
+ .bind(replay_epoch)
+ .bind(replay_sequence)
+ .bind(complete)
+ .bind(self.generation.as_bytes().as_slice())
+ .execute(&mut *transaction)
+ .await
+ .map_err(map_backend)?;
+ transaction.commit().await.map_err(map_backend)?;
+ Ok(VisibilityIndexProgress::new(
+ EventSourceToken::new(self.generation, u64_from_i64(epoch)?),
+ u64_from_i64(backfill)?,
+ u64_from_i64(replay_epoch)?,
+ processed,
+ charged,
+ complete,
+ ))
+ }
+
+ async fn put_visibility_metadata(
+ &self,
+ transaction: &mut sqlx::Transaction<'_, Sqlite>,
+ position: i64,
+ changed: i64,
+ metadata: VisibilityEventMetadata,
+ ) -> Result<(), Error> {
+ sqlx::query(
+ "INSERT INTO radroots_runtime_visibility_events
+ (source_generation, source_sequence, event_id, last_mutation_epoch, author, created_at,
+ admission_stage, head_coordinate, deletion_coordinate, ephemeral, deletion_request)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(source_generation, source_sequence) DO UPDATE SET
+ last_mutation_epoch = excluded.last_mutation_epoch,
+ admission_stage = excluded.admission_stage, head_coordinate = excluded.head_coordinate,
+ deletion_coordinate = excluded.deletion_coordinate,
+ ephemeral = excluded.ephemeral, deletion_request = excluded.deletion_request",
+ )
+ .bind(self.generation.as_bytes().as_slice()).bind(position)
+ .bind(metadata.event_id.as_bytes().as_slice()).bind(changed)
+ .bind(metadata.author.as_slice()).bind(metadata.created_at.to_be_bytes().as_slice())
+ .bind(stage_name(metadata.stage)).bind(metadata.head_coordinate.as_deref())
+ .bind(metadata.deletion_coordinate.as_deref()).bind(metadata.ephemeral)
+ .bind(metadata.deletion_request)
+ .execute(&mut **transaction).await.map_err(map_backend)?;
+ // Signed event bytes and deletion tags are immutable. Replays only add identical targets.
+ for target in metadata.event_targets {
+ sqlx::query(
+ "INSERT OR IGNORE INTO radroots_runtime_visibility_deletion_events
+ (source_generation, target_event, author, request_id) VALUES (?, ?, ?, ?)",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .bind(target.as_bytes().as_slice())
+ .bind(metadata.author.as_slice())
+ .bind(metadata.event_id.as_bytes().as_slice())
+ .execute(&mut **transaction)
+ .await
+ .map_err(map_backend)?;
+ }
+ for coordinate in metadata.address_targets {
+ sqlx::query(
+ "INSERT OR IGNORE INTO radroots_runtime_visibility_deletion_addresses
+ (source_generation, target_coordinate, author, created_at, request_id)
+ VALUES (?, ?, ?, ?, ?)",
+ )
+ .bind(self.generation.as_bytes().as_slice())
+ .bind(coordinate)
+ .bind(metadata.author.as_slice())
+ .bind(metadata.created_at.to_be_bytes().as_slice())
+ .bind(metadata.event_id.as_bytes().as_slice())
+ .execute(&mut **transaction)
+ .await
+ .map_err(map_backend)?;
+ }
+ Ok(())
+ }
+}
diff --git a/crates/storage_sqlite/src/event/visibility_index_tests.rs b/crates/storage_sqlite/src/event/visibility_index_tests.rs
@@ -0,0 +1,626 @@
+use super::tests::{observed, signed_event_with, store, verified, visible};
+use super::*;
+use radroots_storage::event::VisibilityIndexBounds;
+
+fn generation() -> SourceGeneration {
+ SourceGeneration::new([81; 32]).unwrap()
+}
+fn bounds(rows: u16) -> VisibilityIndexBounds {
+ VisibilityIndexBounds::new(rows, 16 * 1024 * 1024).unwrap()
+}
+fn profile(name: &str, time: u64) -> SignedEvent {
+ signed_event_with(&format!(r#"{{"name":"{name}"}}"#), false, time, 0, vec![])
+}
+fn raw(event: SignedEvent) -> EventAdmission {
+ EventAdmission::raw(observed(event, 100, None))
+}
+fn admitted(event: SignedEvent) -> EventAdmission {
+ EventAdmission::visible(observed(event.clone(), 100, None), visible(&event)).unwrap()
+}
+
+async fn finish(storage: &SqliteStorage) {
+ for _ in 0..100 {
+ if storage
+ .maintain_visibility(bounds(2))
+ .await
+ .unwrap()
+ .is_complete()
+ {
+ return;
+ }
+ }
+ panic!("bounded fixture index did not finish");
+}
+
+async fn indexed_visible(storage: &SqliteStorage) -> Vec<EventId> {
+ let values: Vec<Vec<u8>> = sqlx::query_scalar(
+ "SELECT candidate.event_id FROM radroots_runtime_visibility_events AS candidate
+ WHERE candidate.source_generation = ? AND candidate.admission_stage = 'visible'
+ AND candidate.ephemeral = 0
+ AND (candidate.head_coordinate IS NULL OR candidate.event_id = (
+ SELECT head.event_id FROM radroots_runtime_visibility_events AS head
+ WHERE head.source_generation = candidate.source_generation
+ AND head.head_coordinate = candidate.head_coordinate
+ ORDER BY head.created_at DESC, head.event_id ASC LIMIT 1))
+ AND (candidate.deletion_request = 1 OR (
+ NOT EXISTS (SELECT 1 FROM radroots_runtime_visibility_deletion_events AS deletion
+ WHERE deletion.source_generation = candidate.source_generation
+ AND deletion.target_event = candidate.event_id AND deletion.author = candidate.author)
+ AND NOT EXISTS (SELECT 1 FROM radroots_runtime_visibility_deletion_addresses AS deletion
+ WHERE deletion.source_generation = candidate.source_generation
+ AND deletion.target_coordinate = candidate.deletion_coordinate
+ AND deletion.author = candidate.author AND deletion.created_at >= candidate.created_at)))
+ ORDER BY candidate.event_id",
+ ).bind(storage.generation.as_bytes().as_slice()).fetch_all(&storage.pool).await.unwrap();
+ values
+ .into_iter()
+ .map(|value| EventId::from_bytes(value.try_into().unwrap()))
+ .collect()
+}
+
+#[tokio::test]
+async fn visibility_index_backfill_and_changed_replay_survive_provider_reopen_without_restart() {
+ let storage = store(generation()).await;
+ let first = profile("first", 10);
+ for event in [first.clone(), profile("second", 20), profile("third", 30)] {
+ storage.admit(raw(event)).await.unwrap();
+ }
+ let before = storage.maintain_visibility(bounds(1)).await.unwrap();
+ assert_eq!(before.source_sequence(), 1);
+ assert_eq!(before.processed_rows(), 1);
+ assert!(!before.is_complete());
+ storage.admit(admitted(first.clone())).await.unwrap();
+ storage.admit(raw(profile("fourth", 40))).await.unwrap();
+ let reopened = SqliteStorage::new(
+ storage.pool.clone(),
+ generation(),
+ EventStoreMode::ReadWrite,
+ );
+ let after = reopened.maintain_visibility(bounds(1)).await.unwrap();
+ assert_eq!(after.source_sequence(), 2);
+ assert!(!after.is_complete());
+ let third = reopened.maintain_visibility(bounds(1)).await.unwrap();
+ assert_eq!(third.source_sequence(), 3);
+ assert!(!third.is_complete());
+ finish(&reopened).await;
+ let token = reopened.source_token().await.unwrap();
+ let current = reopened.maintain_visibility(bounds(1)).await.unwrap();
+ assert!(current.is_complete());
+ assert_eq!(current.processed_rows(), 0);
+ assert_eq!(current.source_sequence(), 3);
+ assert_eq!(current.token(), token);
+ let row = sqlx::query("SELECT admission_stage, last_mutation_epoch FROM radroots_runtime_visibility_events WHERE event_id = ?")
+ .bind(first.id().as_bytes().as_slice()).fetch_one(&reopened.pool).await.unwrap();
+ assert_eq!(row.get::<String, _>("admission_stage"), "visible");
+ assert_eq!(row.get::<i64, _>("last_mutation_epoch"), 4);
+ assert_eq!(indexed_visible(&reopened).await, vec![*first.id()]);
+ let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM radroots_runtime_visibility_events")
+ .fetch_one(&reopened.pool)
+ .await
+ .unwrap();
+ assert_eq!(count, 4);
+}
+
+#[tokio::test]
+async fn visibility_index_matches_reference_winners_deletion_before_target_and_unsupported_head() {
+ let storage = store(generation()).await;
+ let old = profile("old", 10);
+ let current = profile("current", 20);
+ let deletion = signed_event_with(
+ "delete",
+ false,
+ 30,
+ 5,
+ vec![vec!["e".to_owned(), current.id().to_hex()]],
+ );
+ for event in [deletion, old, current] {
+ storage.admit(admitted(event)).await.unwrap();
+ }
+ finish(&storage).await;
+ assert_eq!(
+ indexed_visible(&storage).await,
+ storage
+ .rebuild_visibility()
+ .await
+ .unwrap()
+ .visible_event_ids()
+ );
+ let newer = profile("unsupported", 40);
+ storage
+ .admit(
+ EventAdmission::verified(observed(newer.clone(), 100, None), verified(&newer)).unwrap(),
+ )
+ .await
+ .unwrap();
+ assert!(
+ !sqlx::query_scalar::<_, bool>("SELECT ready FROM radroots_runtime_visibility_progress")
+ .fetch_one(&storage.pool)
+ .await
+ .unwrap()
+ );
+ finish(&storage).await;
+ assert_eq!(
+ indexed_visible(&storage).await,
+ storage
+ .rebuild_visibility()
+ .await
+ .unwrap()
+ .visible_event_ids()
+ );
+ let same_a = profile("same-a", 50);
+ let same_b = profile("same-b", 50);
+ storage.admit(admitted(same_a)).await.unwrap();
+ storage.admit(admitted(same_b)).await.unwrap();
+ storage
+ .admit(raw(profile("forged raw later", 60)))
+ .await
+ .unwrap();
+ let address_deletion = signed_event_with(
+ "address delete",
+ false,
+ 55,
+ 5,
+ vec![vec![
+ "a".to_owned(),
+ "0:585591529da0bab31b3b1b1f986611cf5f435dca84f978c89ee8a40cca7103df:".to_owned(),
+ ]],
+ );
+ storage.admit(admitted(address_deletion)).await.unwrap();
+ finish(&storage).await;
+ assert_eq!(
+ indexed_visible(&storage).await,
+ storage
+ .rebuild_visibility()
+ .await
+ .unwrap()
+ .visible_event_ids()
+ );
+}
+
+#[tokio::test]
+async fn visibility_index_bounds_charge_before_payload_decode_and_commit_partial_progress() {
+ let storage = store(generation()).await;
+ let first = profile("first", 10);
+ let size = first.raw_json().len() as u64;
+ storage.admit(raw(first)).await.unwrap();
+ storage
+ .admit(raw(profile(&"x".repeat(1000), 20)))
+ .await
+ .unwrap();
+ let one = 4096 + 2 * 128 + VisibilityIndexBounds::charge(size).unwrap();
+ storage.work.take();
+ let progress = storage
+ .maintain_visibility(VisibilityIndexBounds::new(2, one).unwrap())
+ .await
+ .unwrap();
+ assert_eq!(progress.charged_bytes(), one);
+ assert_eq!(progress.processed_rows(), 1);
+ assert_eq!(storage.work.take().codec_inputs, 1);
+ assert!(!progress.is_complete());
+ assert_eq!(
+ storage
+ .maintain_visibility(VisibilityIndexBounds::new(2, one).unwrap())
+ .await,
+ Err(Error::SpaceInsufficient)
+ );
+ assert_eq!(storage.work.take().codec_inputs, 0);
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(
+ "SELECT backfill_sequence FROM radroots_runtime_visibility_progress"
+ )
+ .fetch_one(&storage.pool)
+ .await
+ .unwrap(),
+ 1
+ );
+ finish(&storage).await;
+ storage.work.take();
+ assert!(
+ storage
+ .maintain_visibility(bounds(64))
+ .await
+ .unwrap()
+ .is_complete()
+ );
+ assert_eq!(storage.work.take().codec_inputs, 0);
+}
+
+#[tokio::test]
+async fn visibility_index_fault_rolls_back_all_metadata_and_progress_then_resumes() {
+ let storage = store(generation()).await;
+ storage.admit(raw(profile("first", 10))).await.unwrap();
+ sqlx::raw_sql("CREATE TEMP TRIGGER visibility_test_fault AFTER INSERT ON radroots_runtime_visibility_events BEGIN SELECT RAISE(ABORT, 'index test fault'); END;")
+ .execute(&storage.pool).await.unwrap();
+ assert_eq!(
+ storage.maintain_visibility(bounds(1)).await,
+ Err(Error::BackendUnavailable)
+ );
+ for query in [
+ "SELECT COUNT(*) FROM radroots_runtime_visibility_progress",
+ "SELECT COUNT(*) FROM radroots_runtime_visibility_events",
+ ] {
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>(query)
+ .fetch_one(&storage.pool)
+ .await
+ .unwrap(),
+ 0
+ );
+ }
+ sqlx::query("DROP TRIGGER visibility_test_fault")
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ assert!(
+ storage
+ .maintain_visibility(bounds(1))
+ .await
+ .unwrap()
+ .is_complete()
+ );
+}
+
+#[tokio::test]
+async fn visibility_index_direct_writer_invalidates_atomically_and_rollback_preserves_ready_state()
+{
+ let storage = store(generation()).await;
+ let event = profile("first", 10);
+ storage.admit(raw(event)).await.unwrap();
+ finish(&storage).await;
+ let mut transaction = storage.pool.begin_with("BEGIN IMMEDIATE").await.unwrap();
+ sqlx::query("UPDATE radroots_runtime_events SET admission_stage = 'verified'")
+ .execute(&mut *transaction)
+ .await
+ .unwrap();
+ let dirty: bool = sqlx::query_scalar("SELECT ready FROM radroots_runtime_visibility_progress")
+ .fetch_one(&mut *transaction)
+ .await
+ .unwrap();
+ assert!(!dirty);
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT last_mutation_epoch FROM radroots_runtime_events")
+ .fetch_one(&mut *transaction)
+ .await
+ .unwrap(),
+ 2
+ );
+ transaction.rollback().await.unwrap();
+ assert!(
+ sqlx::query_scalar::<_, bool>("SELECT ready FROM radroots_runtime_visibility_progress")
+ .fetch_one(&storage.pool)
+ .await
+ .unwrap()
+ );
+ sqlx::query("UPDATE radroots_runtime_events SET admission_stage = 'verified'")
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ assert!(
+ !sqlx::query_scalar::<_, bool>("SELECT ready FROM radroots_runtime_visibility_progress")
+ .fetch_one(&storage.pool)
+ .await
+ .unwrap()
+ );
+ let current = storage.maintain_visibility(bounds(1)).await.unwrap();
+ assert!(current.is_complete());
+ assert_eq!(current.processed_rows(), 1);
+ sqlx::query("UPDATE radroots_runtime_events SET admission_stage = admission_stage")
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ assert_eq!(
+ storage
+ .maintain_visibility(bounds(1))
+ .await
+ .unwrap()
+ .processed_rows(),
+ 0
+ );
+}
+
+#[tokio::test]
+async fn visibility_index_corruption_and_retired_or_readonly_sources_refuse_completion() {
+ let storage = store(generation()).await;
+ storage.admit(raw(profile("first", 10))).await.unwrap();
+ sqlx::query("DROP TRIGGER radroots_runtime_events_raw_update_guard")
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ sqlx::query("UPDATE radroots_runtime_events SET signed_event = X'7b'")
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ assert_eq!(
+ storage.maintain_visibility(bounds(1)).await,
+ Err(Error::CorruptStoredEvent)
+ );
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM radroots_runtime_visibility_progress")
+ .fetch_one(&storage.pool)
+ .await
+ .unwrap(),
+ 0
+ );
+ let readonly = SqliteStorage::new(storage.pool.clone(), generation(), EventStoreMode::ReadOnly);
+ assert_eq!(
+ readonly.maintain_visibility(bounds(1)).await,
+ Err(Error::BackendUnavailable)
+ );
+ sqlx::query(
+ "UPDATE radroots_runtime_source_generations SET state = 'retired', retired_at_unix_ms = 2",
+ )
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ assert_eq!(
+ storage.maintain_visibility(bounds(1)).await,
+ Err(Error::SourceGenerationChanged)
+ );
+}
+
+#[tokio::test]
+async fn visibility_index_empty_source_is_complete_and_last_change_replay_uses_index_navigation() {
+ let storage = store(generation()).await;
+ let empty = storage.maintain_visibility(bounds(1)).await.unwrap();
+ assert!(empty.is_complete());
+ assert_eq!(empty.token().epoch(), 0);
+ storage
+ .admit(raw(profile("first", u64::MAX)))
+ .await
+ .unwrap();
+ let progress = storage.maintain_visibility(bounds(1)).await.unwrap();
+ assert!(progress.is_complete());
+ assert_eq!(progress.source_sequence(), 0);
+ let time: Vec<u8> =
+ sqlx::query_scalar("SELECT created_at FROM radroots_runtime_visibility_events")
+ .fetch_one(&storage.pool)
+ .await
+ .unwrap();
+ assert_eq!(time, u64::MAX.to_be_bytes());
+ let plan = sqlx::query("EXPLAIN QUERY PLAN SELECT source_sequence, last_mutation_epoch, length(signed_event) FROM radroots_runtime_events WHERE source_generation = ? AND (last_mutation_epoch, source_sequence) > (?, ?) ORDER BY last_mutation_epoch, source_sequence LIMIT ?")
+ .bind(generation().as_bytes().as_slice()).bind(0i64).bind(0i64).bind(1i64)
+ .fetch_all(&storage.pool).await.unwrap();
+ assert!(plan.iter().any(|row| {
+ row.get::<String, _>("detail")
+ .contains("radroots_runtime_events_mutation_idx")
+ }));
+}
+
+#[tokio::test]
+async fn visibility_index_real_database_close_and_reopen_preserves_bounded_progress() {
+ let directory = tempfile::tempdir().unwrap();
+ let paths = crate::Paths::from_directory(directory.path()).unwrap();
+ let storage = SqliteStorage::open(
+ crate::OpenOptions::new(paths.clone(), crate::OpenMode::Create)
+ .with_source_generation(generation(), 100)
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ for event in [profile("first", 10), profile("second", 20)] {
+ storage.admit(admitted(event)).await.unwrap();
+ }
+ let first = storage.maintain_visibility(bounds(1)).await.unwrap();
+ assert_eq!(first.source_sequence(), 1);
+ assert!(!first.is_complete());
+ storage.close().await.unwrap();
+ assert_eq!(
+ storage.maintain_visibility(bounds(1)).await,
+ Err(Error::BackendUnavailable)
+ );
+ let reopened = SqliteStorage::open(crate::OpenOptions::new(
+ paths,
+ crate::OpenMode::ReadWriteExisting,
+ ))
+ .await
+ .unwrap();
+ let next = reopened.maintain_visibility(bounds(1)).await.unwrap();
+ assert_eq!(next.source_sequence(), 2);
+ assert_eq!(next.processed_rows(), 1);
+ assert!(next.is_complete());
+ assert_eq!(
+ indexed_visible(&reopened).await,
+ reopened
+ .rebuild_visibility()
+ .await
+ .unwrap()
+ .visible_event_ids()
+ );
+ reopened.close().await.unwrap();
+}
+
+#[tokio::test]
+async fn visibility_index_forward_migration_retains_legacy_truth_and_refuses_corrupt_payloads() {
+ use crate::migration::runtime::{MIGRATIONS, migration_sql};
+ use sqlx::sqlite::SqlitePoolOptions;
+ for state in [0, 1, 2] {
+ let pool = SqlitePoolOptions::new()
+ .max_connections(1)
+ .connect("sqlite::memory:")
+ .await
+ .unwrap();
+ for migration in &MIGRATIONS[..18] {
+ sqlx::raw_sql(migration_sql(migration.version()).unwrap())
+ .execute(&pool)
+ .await
+ .unwrap();
+ }
+ sqlx::query("INSERT INTO radroots_runtime_source_generations (generation, sequence_head, state, created_at_unix_ms) VALUES (?, 0, 'active', 1)")
+ .bind(generation().as_bytes().as_slice()).execute(&pool).await.unwrap();
+ let storage = SqliteStorage::new(pool.clone(), generation(), EventStoreMode::ReadWrite);
+ let event = profile("legacy", 10);
+ if state != 0 {
+ storage.admit(raw(event.clone())).await.unwrap();
+ if state == 2 {
+ sqlx::query("DROP TRIGGER radroots_runtime_events_raw_update_guard")
+ .execute(&pool)
+ .await
+ .unwrap();
+ sqlx::query("UPDATE radroots_runtime_events SET signed_event = X'7b'")
+ .execute(&pool)
+ .await
+ .unwrap();
+ }
+ }
+ let token = storage.source_token().await.unwrap();
+ assert_eq!(
+ storage.maintain_visibility(bounds(1)).await,
+ Err(Error::BackendUnavailable)
+ );
+ storage.work.take();
+ sqlx::raw_sql(migration_sql(19).unwrap())
+ .execute(&pool)
+ .await
+ .unwrap();
+ assert_eq!(storage.source_token().await.unwrap(), token);
+ assert_eq!(storage.work.take().codec_inputs, 0);
+ if state == 2 {
+ assert_eq!(
+ storage.maintain_visibility(bounds(1)).await,
+ Err(Error::CorruptStoredEvent)
+ );
+ } else {
+ let result = storage.maintain_visibility(bounds(1)).await.unwrap();
+ assert!(result.is_complete());
+ assert_eq!(result.processed_rows(), u16::from(state == 1));
+ if state == 1 {
+ assert_eq!(
+ sqlx::query_scalar::<_, Vec<u8>>(
+ "SELECT signed_event FROM radroots_runtime_events"
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap(),
+ event.raw_json().as_bytes()
+ );
+ }
+ }
+ }
+}
+
+#[tokio::test]
+async fn visibility_index_cancellation_before_admission_leaves_no_progress_and_allows_retry() {
+ let storage = store(generation()).await;
+ storage.admit(raw(profile("first", 10))).await.unwrap();
+ let held = storage.pool.acquire().await.unwrap();
+ let mut future = storage.maintain_visibility(bounds(1));
+ std::future::poll_fn(|context| {
+ assert!(future.as_mut().poll(context).is_pending());
+ std::task::Poll::Ready(())
+ })
+ .await;
+ drop(future);
+ drop(held);
+ assert_eq!(
+ sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM radroots_runtime_visibility_progress")
+ .fetch_one(&storage.pool)
+ .await
+ .unwrap(),
+ 0
+ );
+ assert!(
+ storage
+ .maintain_visibility(bounds(1))
+ .await
+ .unwrap()
+ .is_complete()
+ );
+}
+
+#[tokio::test]
+async fn visibility_index_malformed_progress_or_missing_source_tail_cannot_claim_complete() {
+ let storage = store(generation()).await;
+ storage.admit(raw(profile("first", 10))).await.unwrap();
+ finish(&storage).await;
+ sqlx::query("PRAGMA ignore_check_constraints = ON")
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ sqlx::query("UPDATE radroots_runtime_visibility_progress SET schema_version = 2")
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ assert_eq!(
+ storage.maintain_visibility(bounds(1)).await,
+ Err(Error::CorruptStoredEvent)
+ );
+ sqlx::query("UPDATE radroots_runtime_visibility_progress SET schema_version = 1")
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ sqlx::query("DELETE FROM radroots_runtime_visibility_progress")
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ sqlx::query("UPDATE radroots_runtime_source_generations SET sequence_head = 2")
+ .execute(&storage.pool)
+ .await
+ .unwrap();
+ assert!(
+ !storage
+ .maintain_visibility(bounds(1))
+ .await
+ .unwrap()
+ .is_complete()
+ );
+ assert_eq!(
+ storage.maintain_visibility(bounds(1)).await,
+ Err(Error::CorruptStoredEvent)
+ );
+}
+
+#[tokio::test]
+async fn visibility_index_memory_and_sqlite_progress_agree_across_same_count_promotion() {
+ let storage = store(generation()).await;
+ let memory = radroots_storage::memory::MemoryStorage::new(generation());
+ let first = profile("first", 10);
+ let initial = [raw(first.clone()), raw(profile("large", 20))];
+ for admission in initial {
+ storage.admit(admission.clone()).await.unwrap();
+ memory.admit(admission).await.unwrap();
+ }
+ for budget in [
+ VisibilityIndexBounds::new(
+ 2,
+ 4096 + 2 * 128 + VisibilityIndexBounds::charge(first.raw_json().len() as u64).unwrap(),
+ )
+ .unwrap(),
+ bounds(1),
+ ] {
+ assert_eq!(
+ storage.maintain_visibility(budget).await.unwrap(),
+ memory.maintain_visibility(budget).await.unwrap()
+ );
+ }
+ for admission in [admitted(first.clone()), admitted(profile("later", 30))] {
+ storage.admit(admission.clone()).await.unwrap();
+ memory.admit(admission).await.unwrap();
+ }
+ for _ in 0..3 {
+ let sql = storage.maintain_visibility(bounds(1)).await.unwrap();
+ let reference = memory.maintain_visibility(bounds(1)).await.unwrap();
+ assert_eq!(sql, reference);
+ if sql.is_complete() {
+ break;
+ }
+ }
+ assert!(
+ storage
+ .maintain_visibility(bounds(1))
+ .await
+ .unwrap()
+ .is_complete()
+ );
+ assert_eq!(
+ storage.rebuild_visibility().await.unwrap(),
+ memory.rebuild_visibility().await.unwrap()
+ );
+ assert_eq!(
+ indexed_visible(&storage).await,
+ memory
+ .rebuild_visibility()
+ .await
+ .unwrap()
+ .visible_event_ids()
+ );
+}
diff --git a/crates/storage_sqlite/src/migration.rs b/crates/storage_sqlite/src/migration.rs
@@ -469,7 +469,7 @@ fn schema_metadata_error(source: &sqlx::Error, database: &'static str) -> Error
fn validate_plan(plan: &MigrationPlan) -> Result<(), Error> {
let valid = plan.minimum_version > 0
&& plan.minimum_version <= plan.current_version
- && plan.current_version <= 18
+ && plan.current_version <= 19
&& plan.steps.len() == usize::try_from(plan.current_version).unwrap_or(usize::MAX)
&& plan
.steps
@@ -594,6 +594,7 @@ const fn set_user_version_sql(version: u32) -> Option<&'static str> {
16 => Some("PRAGMA user_version = 16"),
17 => Some("PRAGMA user_version = 17"),
18 => Some("PRAGMA user_version = 18"),
+ 19 => Some("PRAGMA user_version = 19"),
_ => None,
}
}
@@ -878,8 +879,8 @@ mod tests {
.await
.expect("application id");
let newer_version = runtime::CURRENT_VERSION + 1;
- assert_eq!(newer_version, 19);
- sqlx::raw_sql("PRAGMA user_version = 19")
+ assert_eq!(newer_version, 20);
+ sqlx::raw_sql("PRAGMA user_version = 20")
.execute(&mut newer)
.await
.expect("newer version");
diff --git a/crates/storage_sqlite/src/migration/runtime/0019_canonical_visibility_index.up.sql b/crates/storage_sqlite/src/migration/runtime/0019_canonical_visibility_index.up.sql
@@ -0,0 +1,111 @@
+ALTER TABLE radroots_runtime_events
+ADD COLUMN last_mutation_epoch INTEGER NOT NULL DEFAULT 0 CHECK (last_mutation_epoch >= 0);
+
+CREATE INDEX radroots_runtime_events_mutation_idx
+ON radroots_runtime_events(source_generation, last_mutation_epoch, source_sequence);
+
+CREATE TABLE radroots_runtime_visibility_progress (
+ source_generation BLOB PRIMARY KEY NOT NULL
+ REFERENCES radroots_runtime_source_generations(generation),
+ schema_version INTEGER NOT NULL DEFAULT 1 CHECK (schema_version = 1),
+ backfill_target INTEGER NOT NULL CHECK (backfill_target >= 0),
+ backfill_sequence INTEGER NOT NULL DEFAULT 0 CHECK (backfill_sequence >= 0),
+ replay_epoch INTEGER NOT NULL CHECK (replay_epoch >= 0),
+ replay_sequence INTEGER NOT NULL CHECK (replay_sequence >= 0),
+ ready INTEGER NOT NULL DEFAULT 0 CHECK (ready IN (0, 1)),
+ CHECK (backfill_sequence <= backfill_target)
+) STRICT, WITHOUT ROWID;
+
+CREATE TABLE radroots_runtime_visibility_events (
+ source_generation BLOB NOT NULL
+ REFERENCES radroots_runtime_source_generations(generation),
+ source_sequence INTEGER NOT NULL CHECK (source_sequence > 0),
+ event_id BLOB NOT NULL UNIQUE REFERENCES radroots_runtime_events(event_id),
+ last_mutation_epoch INTEGER NOT NULL CHECK (last_mutation_epoch >= 0),
+ author BLOB NOT NULL CHECK (length(author) = 32),
+ created_at BLOB NOT NULL CHECK (length(created_at) = 8),
+ admission_stage TEXT NOT NULL CHECK (admission_stage IN ('raw', 'verified', 'visible')),
+ head_coordinate BLOB,
+ deletion_coordinate TEXT,
+ ephemeral INTEGER NOT NULL CHECK (ephemeral IN (0, 1)),
+ deletion_request INTEGER NOT NULL CHECK (deletion_request IN (0, 1)),
+ PRIMARY KEY (source_generation, source_sequence)
+) STRICT, WITHOUT ROWID;
+
+CREATE INDEX radroots_runtime_visibility_events_head_idx
+ON radroots_runtime_visibility_events(
+ source_generation, head_coordinate, created_at DESC, event_id ASC
+);
+
+CREATE INDEX radroots_runtime_visibility_events_stage_idx
+ON radroots_runtime_visibility_events(source_generation, admission_stage, source_sequence);
+
+CREATE TABLE radroots_runtime_visibility_deletion_events (
+ source_generation BLOB NOT NULL,
+ target_event BLOB NOT NULL CHECK (length(target_event) = 32),
+ author BLOB NOT NULL CHECK (length(author) = 32),
+ request_id BLOB NOT NULL
+ REFERENCES radroots_runtime_visibility_events(event_id) ON DELETE CASCADE,
+ PRIMARY KEY (source_generation, target_event, author, request_id)
+) STRICT, WITHOUT ROWID;
+
+CREATE INDEX radroots_runtime_visibility_deletion_events_request_idx
+ON radroots_runtime_visibility_deletion_events(request_id);
+
+CREATE TABLE radroots_runtime_visibility_deletion_addresses (
+ source_generation BLOB NOT NULL,
+ target_coordinate TEXT NOT NULL,
+ author BLOB NOT NULL CHECK (length(author) = 32),
+ created_at BLOB NOT NULL CHECK (length(created_at) = 8),
+ request_id BLOB NOT NULL
+ REFERENCES radroots_runtime_visibility_events(event_id) ON DELETE CASCADE,
+ PRIMARY KEY (source_generation, target_coordinate, author, created_at, request_id)
+) STRICT, WITHOUT ROWID;
+
+CREATE INDEX radroots_runtime_visibility_deletion_addresses_request_idx
+ON radroots_runtime_visibility_deletion_addresses(request_id);
+
+DROP TRIGGER radroots_runtime_events_mutation_insert;
+DROP TRIGGER radroots_runtime_events_mutation_update;
+
+CREATE TRIGGER radroots_runtime_events_mutation_insert
+AFTER INSERT ON radroots_runtime_events
+BEGIN
+ SELECT CASE WHEN (
+ SELECT mutation_epoch FROM radroots_runtime_source_generations
+ WHERE generation = NEW.source_generation
+ ) = 9223372036854775807
+ THEN RAISE(ABORT, 'runtime event mutation epoch overflow') END;
+ UPDATE radroots_runtime_source_generations
+ SET mutation_epoch = mutation_epoch + 1
+ WHERE generation = NEW.source_generation;
+ UPDATE radroots_runtime_events SET last_mutation_epoch = (
+ SELECT mutation_epoch FROM radroots_runtime_source_generations
+ WHERE generation = NEW.source_generation
+ ) WHERE event_id = NEW.event_id;
+ UPDATE radroots_runtime_visibility_progress SET ready = 0
+ WHERE source_generation = NEW.source_generation;
+END;
+
+CREATE TRIGGER radroots_runtime_events_mutation_update
+AFTER UPDATE OF admission_stage, admitted_contract_id, admitted_registry_version
+ON radroots_runtime_events
+WHEN NEW.admission_stage IS NOT OLD.admission_stage
+ OR NEW.admitted_contract_id IS NOT OLD.admitted_contract_id
+ OR NEW.admitted_registry_version IS NOT OLD.admitted_registry_version
+BEGIN
+ SELECT CASE WHEN (
+ SELECT mutation_epoch FROM radroots_runtime_source_generations
+ WHERE generation = NEW.source_generation
+ ) = 9223372036854775807
+ THEN RAISE(ABORT, 'runtime event mutation epoch overflow') END;
+ UPDATE radroots_runtime_source_generations
+ SET mutation_epoch = mutation_epoch + 1
+ WHERE generation = NEW.source_generation;
+ UPDATE radroots_runtime_events SET last_mutation_epoch = (
+ SELECT mutation_epoch FROM radroots_runtime_source_generations
+ WHERE generation = NEW.source_generation
+ ) WHERE event_id = NEW.event_id;
+ UPDATE radroots_runtime_visibility_progress SET ready = 0
+ WHERE source_generation = NEW.source_generation;
+END;
diff --git a/crates/storage_sqlite/src/migration/runtime/mod.rs b/crates/storage_sqlite/src/migration/runtime/mod.rs
@@ -6,7 +6,7 @@
/// Lowest runtime schema version this package can recognize.
pub const MINIMUM_VERSION: u32 = 1;
/// Current runtime schema version created by this package.
-pub const CURRENT_VERSION: u32 = 18;
+pub const CURRENT_VERSION: u32 = 19;
const RUNTIME_V1_SQL: &str = include_str!("0001_runtime.up.sql");
const CANONICAL_EVENT_STORAGE_V2_SQL: &str = include_str!("0002_canonical_event_storage.up.sql");
@@ -35,6 +35,9 @@ const AUTHORED_DELIVERY_RECONCILIATION_V17_SQL: &str =
const EVENT_MUTATION_EPOCH_V18_SQL: &str = include_str!("0018_event_mutation_epoch.up.sql");
+const CANONICAL_VISIBILITY_INDEX_V19_SQL: &str =
+ include_str!("0019_canonical_visibility_index.up.sql");
+
/// Stable, non-SQL description of one forward runtime migration.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct MigrationDescriptor {
@@ -989,6 +992,108 @@ const RUNTIME_V18_OBJECTS: &[&str] = &[
"radroots_runtime_source_generations_sequence_guard",
];
+const RUNTIME_V19_OBJECTS: &[&str] = &[
+ "radroots_runtime_atomic_commits",
+ "radroots_runtime_authored_artifacts",
+ "radroots_runtime_authored_artifacts_admission_ready_idx",
+ "radroots_runtime_authored_artifacts_signed_fact_guard",
+ "radroots_runtime_authored_artifacts_signing_ready_idx",
+ "radroots_runtime_authored_atomic_commits",
+ "radroots_runtime_authored_atomic_commits_delete_guard",
+ "radroots_runtime_authored_atomic_commits_update_guard",
+ "radroots_runtime_authored_atomic_target_phase_idx",
+ "radroots_runtime_authored_delivery_attempts",
+ "radroots_runtime_authored_delivery_claims",
+ "radroots_runtime_authored_delivery_claims_delete_guard",
+ "radroots_runtime_authored_delivery_claims_update_guard",
+ "radroots_runtime_authored_delivery_facts",
+ "radroots_runtime_authored_delivery_facts_delete_guard",
+ "radroots_runtime_authored_delivery_facts_update_guard",
+ "radroots_runtime_authored_delivery_plans",
+ "radroots_runtime_authored_delivery_ready_idx",
+ "radroots_runtime_authored_delivery_reconciliations",
+ "radroots_runtime_authored_delivery_reconciliations_delete_guard",
+ "radroots_runtime_authored_delivery_reconciliations_update_guard",
+ "radroots_runtime_authored_delivery_stop_guard",
+ "radroots_runtime_authored_delivery_targets",
+ "radroots_runtime_authored_draft_author_head_idx",
+ "radroots_runtime_authored_draft_revisions",
+ "radroots_runtime_authored_draft_revisions_delete_guard",
+ "radroots_runtime_authored_draft_revisions_update_guard",
+ "radroots_runtime_authored_draft_scope_head_idx",
+ "radroots_runtime_authored_migration_evidence",
+ "radroots_runtime_authored_migration_evidence_delete_guard",
+ "radroots_runtime_authored_migration_evidence_update_guard",
+ "radroots_runtime_authored_operations",
+ "radroots_runtime_delivery_evidence",
+ "radroots_runtime_delivery_evidence_item_idx",
+ "radroots_runtime_event_index_checkpoints",
+ "radroots_runtime_event_index_manifests",
+ "radroots_runtime_event_index_shards",
+ "radroots_runtime_event_provenance",
+ "radroots_runtime_event_provenance_observed_idx",
+ "radroots_runtime_events",
+ "radroots_runtime_events_admission_idx",
+ "radroots_runtime_events_contract_metadata_guard",
+ "radroots_runtime_events_contract_metadata_insert_guard",
+ "radroots_runtime_events_delete_guard",
+ "radroots_runtime_events_event_id_idx",
+ "radroots_runtime_events_mutation_idx",
+ "radroots_runtime_events_mutation_insert",
+ "radroots_runtime_events_mutation_update",
+ "radroots_runtime_events_raw_update_guard",
+ "radroots_runtime_journal_idempotency_idx",
+ "radroots_runtime_journal_operations",
+ "radroots_runtime_journal_recovery_idx",
+ "radroots_runtime_legacy_event_staging",
+ "radroots_runtime_legacy_event_staging_delete_guard",
+ "radroots_runtime_legacy_event_staging_insert_guard",
+ "radroots_runtime_legacy_event_staging_update_guard",
+ "radroots_runtime_legacy_import_commit_delete_guard",
+ "radroots_runtime_legacy_import_commit_update_guard",
+ "radroots_runtime_legacy_import_commits",
+ "radroots_runtime_legacy_import_delete_guard",
+ "radroots_runtime_legacy_import_identity_guard",
+ "radroots_runtime_legacy_import_member_delete_guard",
+ "radroots_runtime_legacy_import_member_identity_guard",
+ "radroots_runtime_legacy_import_member_state_guard",
+ "radroots_runtime_legacy_import_members",
+ "radroots_runtime_legacy_import_state_guard",
+ "radroots_runtime_legacy_import_state_idx",
+ "radroots_runtime_legacy_imports",
+ "radroots_runtime_legacy_outbox_staging",
+ "radroots_runtime_legacy_outbox_staging_delete_guard",
+ "radroots_runtime_legacy_outbox_staging_insert_guard",
+ "radroots_runtime_legacy_outbox_staging_parent_idx",
+ "radroots_runtime_legacy_outbox_staging_update_guard",
+ "radroots_runtime_outbox_items",
+ "radroots_runtime_outbox_operation_idx",
+ "radroots_runtime_outbox_ready_idx",
+ "radroots_runtime_outbox_targets",
+ "radroots_runtime_projection_checkpoints",
+ "radroots_runtime_projection_documents",
+ "radroots_runtime_projection_invalidations",
+ "radroots_runtime_projection_rebuilds",
+ "radroots_runtime_projection_rebuilds_stage_idx",
+ "radroots_runtime_projection_snapshots",
+ "radroots_runtime_projection_snapshots_created_idx",
+ "radroots_runtime_projection_snapshots_update_guard",
+ "radroots_runtime_source_generations",
+ "radroots_runtime_source_generations_active_idx",
+ "radroots_runtime_source_generations_delete_guard",
+ "radroots_runtime_source_generations_identity_guard",
+ "radroots_runtime_source_generations_mutation_guard",
+ "radroots_runtime_source_generations_sequence_guard",
+ "radroots_runtime_visibility_deletion_addresses",
+ "radroots_runtime_visibility_deletion_addresses_request_idx",
+ "radroots_runtime_visibility_deletion_events",
+ "radroots_runtime_visibility_deletion_events_request_idx",
+ "radroots_runtime_visibility_events",
+ "radroots_runtime_visibility_events_head_idx",
+ "radroots_runtime_visibility_events_stage_idx",
+ "radroots_runtime_visibility_progress",
+];
+
/// Ordered, immutable runtime migration plan.
pub const MIGRATIONS: &[MigrationDescriptor] = &[
MigrationDescriptor {
@@ -1099,6 +1204,12 @@ pub const MIGRATIONS: &[MigrationDescriptor] = &[
up_sha256: "44e8f43722bb869f824dbfa28197d17c1e8f58c6d62b8219cdd19d452eb47c3c",
owned_objects: RUNTIME_V18_OBJECTS,
},
+ MigrationDescriptor {
+ version: 19,
+ name: "canonical_visibility_index",
+ up_sha256: "fd574ba0459de5a8c36033d5332dbd647888ef35e0213bc4ee88656ac2ef49bb",
+ owned_objects: RUNTIME_V19_OBJECTS,
+ },
];
pub(crate) const fn migration_sql(version: u32) -> Option<&'static str> {
@@ -1121,6 +1232,7 @@ pub(crate) const fn migration_sql(version: u32) -> Option<&'static str> {
16 => Some(AUTHORED_DELIVERY_FACTS_V16_SQL),
17 => Some(AUTHORED_DELIVERY_RECONCILIATION_V17_SQL),
18 => Some(EVENT_MUTATION_EPOCH_V18_SQL),
+ 19 => Some(CANONICAL_VISIBILITY_INDEX_V19_SQL),
_ => None,
}
}
@@ -1164,8 +1276,8 @@ mod tests {
fn migration_plan_matches_governed_snapshot() {
let snapshot = toml::from_str::<PlanSnapshot>(PLAN_SNAPSHOT).expect("valid snapshot");
assert_eq!(MINIMUM_VERSION, 1);
- assert_eq!(CURRENT_VERSION, 18);
- assert_eq!(MIGRATIONS.len(), 18);
+ assert_eq!(CURRENT_VERSION, 19);
+ assert_eq!(MIGRATIONS.len(), 19);
let migration = MIGRATIONS[8];
assert_eq!(snapshot.schema_version, 1);
assert_eq!(snapshot.database, "runtime.sqlite");