lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

visibility_index.rs (12119B)


      1 use super::*;
      2 use radroots_storage::event::{
      3     VisibilityEventMetadata, VisibilityIndexBounds, VisibilityIndexProgress,
      4 };
      5 
      6 impl SqliteStorage {
      7     pub(super) async fn prepare_visibility_index(
      8         &self,
      9         bounds: VisibilityIndexBounds,
     10     ) -> Result<VisibilityIndexProgress, Error> {
     11         if self.mode == EventStoreMode::ReadOnly {
     12             return Err(Error::BackendUnavailable);
     13         }
     14         let mut transaction = self
     15             .pool
     16             .begin_with("BEGIN IMMEDIATE")
     17             .await
     18             .map_err(map_backend)?;
     19         let progress = self
     20             .prepare_visibility_transaction(&mut transaction, bounds)
     21             .await?;
     22         transaction.commit().await.map_err(map_backend)?;
     23         Ok(progress)
     24     }
     25 
     26     pub(super) async fn prepare_visibility_transaction(
     27         &self,
     28         transaction: &mut sqlx::Transaction<'_, Sqlite>,
     29         bounds: VisibilityIndexBounds,
     30     ) -> Result<VisibilityIndexProgress, Error> {
     31         let mut charged = 4096 + u64::from(bounds.rows()) * 128;
     32         if charged > bounds.bytes() {
     33             return Err(Error::SpaceInsufficient);
     34         }
     35         let source = sqlx::query(
     36             "SELECT mutation_epoch, sequence_head FROM radroots_runtime_source_generations
     37              WHERE generation = ? AND state = 'active'",
     38         )
     39         .bind(self.generation.as_bytes().as_slice())
     40         .fetch_optional(&mut **transaction)
     41         .await
     42         .map_err(map_backend)?
     43         .ok_or(Error::SourceGenerationChanged)?;
     44         let epoch: i64 = source.try_get("mutation_epoch").map_err(map_corrupt)?;
     45         let sequence: i64 = source.try_get("sequence_head").map_err(map_corrupt)?;
     46         sqlx::query(
     47             "INSERT OR IGNORE INTO radroots_runtime_visibility_progress
     48              (source_generation, backfill_target, replay_epoch, replay_sequence)
     49              VALUES (?, ?, ?, 9223372036854775807)",
     50         )
     51         .bind(self.generation.as_bytes().as_slice())
     52         .bind(sequence)
     53         .bind(epoch)
     54         .execute(&mut **transaction)
     55         .await
     56         .map_err(map_backend)?;
     57         let progress = sqlx::query(
     58             "SELECT schema_version, backfill_target, backfill_sequence, replay_epoch, replay_sequence
     59              FROM radroots_runtime_visibility_progress WHERE source_generation = ?",
     60         )
     61         .bind(self.generation.as_bytes().as_slice())
     62         .fetch_one(&mut **transaction).await.map_err(map_backend)?;
     63         let version: i64 = progress.try_get("schema_version").map_err(map_corrupt)?;
     64         let target: i64 = progress.try_get("backfill_target").map_err(map_corrupt)?;
     65         let mut backfill: i64 = progress.try_get("backfill_sequence").map_err(map_corrupt)?;
     66         let mut replay_epoch: i64 = progress.try_get("replay_epoch").map_err(map_corrupt)?;
     67         let mut replay_sequence: i64 = progress.try_get("replay_sequence").map_err(map_corrupt)?;
     68         if version != 1 || target > sequence || backfill > target || replay_epoch > epoch {
     69             return Err(Error::CorruptStoredEvent);
     70         }
     71         let had_pending: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM radroots_runtime_visibility_pending WHERE source_generation = ?)")
     72             .bind(self.generation.as_bytes().as_slice()).fetch_one(&mut **transaction).await.map_err(map_backend)?;
     73         let mut processed_keys = self
     74             .drain_visibility_keys(transaction, bounds.rows(), &mut charged, bounds.bytes())
     75             .await?;
     76         let remaining = if had_pending {
     77             0
     78         } else {
     79             bounds.rows() - processed_keys
     80         };
     81         let building = backfill < target;
     82         let mut query = QueryBuilder::<Sqlite>::new(
     83             "SELECT source_sequence, last_mutation_epoch, length(signed_event) AS signed_bytes
     84              FROM radroots_runtime_events WHERE source_generation = ",
     85         );
     86         query.push_bind(self.generation.as_bytes().as_slice());
     87         if building {
     88             query.push(" AND source_sequence > ").push_bind(backfill);
     89             query.push(" AND source_sequence <= ").push_bind(target);
     90             query
     91                 .push(" ORDER BY source_sequence LIMIT ")
     92                 .push_bind(i64::from(remaining));
     93         } else {
     94             query.push(" AND (last_mutation_epoch, source_sequence) > (");
     95             query
     96                 .push_bind(replay_epoch)
     97                 .push(", ")
     98                 .push_bind(replay_sequence)
     99                 .push(")");
    100             query
    101                 .push(" ORDER BY last_mutation_epoch, source_sequence LIMIT ")
    102                 .push_bind(i64::from(remaining));
    103         }
    104         let lengths = query
    105             .build()
    106             .fetch_all(&mut **transaction)
    107             .await
    108             .map_err(map_backend)?;
    109         if building && remaining > 0 && lengths.is_empty() {
    110             return Err(Error::CorruptStoredEvent);
    111         }
    112         let mut processed = 0u16;
    113         for length in lengths {
    114             let position: i64 = length.try_get("source_sequence").map_err(map_corrupt)?;
    115             let changed: i64 = length.try_get("last_mutation_epoch").map_err(map_corrupt)?;
    116             let size = u64_from_i64(length.try_get("signed_bytes").map_err(map_corrupt)?)?;
    117             let charge = VisibilityIndexBounds::charge(size)?;
    118             let next_charge = charged
    119                 .checked_add(charge)
    120                 .ok_or(Error::SpaceInsufficient)?;
    121             if next_charge > bounds.bytes() {
    122                 if processed == 0 {
    123                     return Err(Error::SpaceInsufficient);
    124                 }
    125                 break;
    126             }
    127             let row = sqlx::query(
    128                 "SELECT source_generation, source_sequence, signed_event, admission_stage,
    129                  admitted_contract_id, admitted_registry_version FROM radroots_runtime_events
    130                  WHERE source_generation = ? AND source_sequence = ?",
    131             )
    132             .bind(self.generation.as_bytes().as_slice())
    133             .bind(position)
    134             .fetch_one(&mut **transaction)
    135             .await
    136             .map_err(map_backend)?;
    137             let decoded = self.decode_event_row(&row)?;
    138             if decoded.raw_json.len() as u64 != size || changed > epoch {
    139                 return Err(Error::CorruptStoredEvent);
    140             }
    141             let metadata = VisibilityEventMetadata::from_event(&decoded.event, decoded.stage)?;
    142             self.put_visibility_metadata(transaction, position, changed, metadata)
    143                 .await?;
    144             charged = next_charge;
    145             processed += 1;
    146             if building {
    147                 backfill = position;
    148             } else {
    149                 replay_epoch = changed;
    150                 replay_sequence = position;
    151             }
    152         }
    153         let pending: bool = sqlx::query_scalar(
    154             "SELECT EXISTS (SELECT 1 FROM radroots_runtime_events
    155              WHERE source_generation = ? AND (last_mutation_epoch, source_sequence) > (?, ?))",
    156         )
    157         .bind(self.generation.as_bytes().as_slice())
    158         .bind(replay_epoch)
    159         .bind(replay_sequence)
    160         .fetch_one(&mut **transaction)
    161         .await
    162         .map_err(map_backend)?;
    163         processed_keys += self
    164             .drain_visibility_keys(
    165                 transaction,
    166                 bounds.rows() - processed - processed_keys,
    167                 &mut charged,
    168                 bounds.bytes(),
    169             )
    170             .await?;
    171         let pending_keys: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM radroots_runtime_visibility_pending WHERE source_generation = ?)")
    172             .bind(self.generation.as_bytes().as_slice()).fetch_one(&mut **transaction).await.map_err(map_backend)?;
    173         if processed == 0 && processed_keys == 0 && pending_keys {
    174             return Err(Error::SpaceInsufficient);
    175         }
    176         let complete = backfill == target && !pending && !pending_keys;
    177         if complete {
    178             let retained: i64 = sqlx::query_scalar(
    179                 "SELECT raw_events FROM radroots_runtime_visibility_progress WHERE source_generation = ?")
    180                 .bind(self.generation.as_bytes().as_slice()).fetch_one(&mut **transaction).await.map_err(map_backend)?;
    181             if retained != sequence {
    182                 return Err(Error::CorruptStoredEvent);
    183             }
    184             replay_epoch = epoch;
    185             replay_sequence = i64::MAX;
    186         }
    187         sqlx::query(
    188             "UPDATE radroots_runtime_visibility_progress SET backfill_sequence = ?,
    189              replay_epoch = ?, replay_sequence = ?, ready = ? WHERE source_generation = ?",
    190         )
    191         .bind(backfill)
    192         .bind(replay_epoch)
    193         .bind(replay_sequence)
    194         .bind(complete)
    195         .bind(self.generation.as_bytes().as_slice())
    196         .execute(&mut **transaction)
    197         .await
    198         .map_err(map_backend)?;
    199         Ok(VisibilityIndexProgress::new(
    200             EventSourceToken::new(self.generation, u64_from_i64(epoch)?),
    201             u64_from_i64(backfill)?,
    202             u64_from_i64(replay_epoch)?,
    203             processed,
    204             charged,
    205             complete,
    206         )
    207         .with_processed_keys(processed_keys))
    208     }
    209 
    210     async fn put_visibility_metadata(
    211         &self,
    212         transaction: &mut sqlx::Transaction<'_, Sqlite>,
    213         position: i64,
    214         changed: i64,
    215         metadata: VisibilityEventMetadata,
    216     ) -> Result<(), Error> {
    217         sqlx::query(
    218             "INSERT INTO radroots_runtime_visibility_events
    219              (source_generation, source_sequence, event_id, last_mutation_epoch, author, created_at,
    220               admission_stage, head_coordinate, deletion_coordinate, ephemeral, deletion_request)
    221              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    222              ON CONFLICT(source_generation, source_sequence) DO UPDATE SET
    223               last_mutation_epoch = excluded.last_mutation_epoch,
    224               admission_stage = excluded.admission_stage, head_coordinate = excluded.head_coordinate,
    225               deletion_coordinate = excluded.deletion_coordinate,
    226               ephemeral = excluded.ephemeral, deletion_request = excluded.deletion_request",
    227         )
    228         .bind(self.generation.as_bytes().as_slice()).bind(position)
    229         .bind(metadata.event_id.as_bytes().as_slice()).bind(changed)
    230         .bind(metadata.author.as_slice()).bind(metadata.created_at.to_be_bytes().as_slice())
    231         .bind(stage_name(metadata.stage)).bind(metadata.head_coordinate.as_deref())
    232         .bind(metadata.deletion_coordinate.as_deref()).bind(metadata.ephemeral)
    233         .bind(metadata.deletion_request)
    234         .execute(&mut **transaction).await.map_err(map_backend)?;
    235         // Signed event bytes and deletion tags are immutable. Replays only add identical targets.
    236         for target in metadata.event_targets {
    237             sqlx::query(
    238                 "INSERT OR IGNORE INTO radroots_runtime_visibility_deletion_events
    239                  (source_generation, target_event, author, request_id) VALUES (?, ?, ?, ?)",
    240             )
    241             .bind(self.generation.as_bytes().as_slice())
    242             .bind(target.as_bytes().as_slice())
    243             .bind(metadata.author.as_slice())
    244             .bind(metadata.event_id.as_bytes().as_slice())
    245             .execute(&mut **transaction)
    246             .await
    247             .map_err(map_backend)?;
    248         }
    249         for coordinate in metadata.address_targets {
    250             sqlx::query(
    251                 "INSERT OR IGNORE INTO radroots_runtime_visibility_deletion_addresses
    252                  (source_generation, target_coordinate, author, created_at, request_id)
    253                  VALUES (?, ?, ?, ?, ?)",
    254             )
    255             .bind(self.generation.as_bytes().as_slice())
    256             .bind(coordinate)
    257             .bind(metadata.author.as_slice())
    258             .bind(metadata.created_at.to_be_bytes().as_slice())
    259             .bind(metadata.event_id.as_bytes().as_slice())
    260             .execute(&mut **transaction)
    261             .await
    262             .map_err(map_backend)?;
    263         }
    264         self.update_visibility_membership(transaction, position)
    265             .await?;
    266         Ok(())
    267     }
    268 }