visibility_index.rs (12119B)
1 use super::*; 2 use radroots_storage::event::{ 3 VisibilityEventMetadata, VisibilityIndexBounds, VisibilityIndexProgress, 4 }; 5 6 impl SqliteStorage { 7 pub(super) async fn prepare_visibility_index( 8 &self, 9 bounds: VisibilityIndexBounds, 10 ) -> Result<VisibilityIndexProgress, Error> { 11 if self.mode == EventStoreMode::ReadOnly { 12 return Err(Error::BackendUnavailable); 13 } 14 let mut transaction = self 15 .pool 16 .begin_with("BEGIN IMMEDIATE") 17 .await 18 .map_err(map_backend)?; 19 let progress = self 20 .prepare_visibility_transaction(&mut transaction, bounds) 21 .await?; 22 transaction.commit().await.map_err(map_backend)?; 23 Ok(progress) 24 } 25 26 pub(super) async fn prepare_visibility_transaction( 27 &self, 28 transaction: &mut sqlx::Transaction<'_, Sqlite>, 29 bounds: VisibilityIndexBounds, 30 ) -> Result<VisibilityIndexProgress, Error> { 31 let mut charged = 4096 + u64::from(bounds.rows()) * 128; 32 if charged > bounds.bytes() { 33 return Err(Error::SpaceInsufficient); 34 } 35 let source = sqlx::query( 36 "SELECT mutation_epoch, sequence_head FROM radroots_runtime_source_generations 37 WHERE generation = ? AND state = 'active'", 38 ) 39 .bind(self.generation.as_bytes().as_slice()) 40 .fetch_optional(&mut **transaction) 41 .await 42 .map_err(map_backend)? 43 .ok_or(Error::SourceGenerationChanged)?; 44 let epoch: i64 = source.try_get("mutation_epoch").map_err(map_corrupt)?; 45 let sequence: i64 = source.try_get("sequence_head").map_err(map_corrupt)?; 46 sqlx::query( 47 "INSERT OR IGNORE INTO radroots_runtime_visibility_progress 48 (source_generation, backfill_target, replay_epoch, replay_sequence) 49 VALUES (?, ?, ?, 9223372036854775807)", 50 ) 51 .bind(self.generation.as_bytes().as_slice()) 52 .bind(sequence) 53 .bind(epoch) 54 .execute(&mut **transaction) 55 .await 56 .map_err(map_backend)?; 57 let progress = sqlx::query( 58 "SELECT schema_version, backfill_target, backfill_sequence, replay_epoch, replay_sequence 59 FROM radroots_runtime_visibility_progress WHERE source_generation = ?", 60 ) 61 .bind(self.generation.as_bytes().as_slice()) 62 .fetch_one(&mut **transaction).await.map_err(map_backend)?; 63 let version: i64 = progress.try_get("schema_version").map_err(map_corrupt)?; 64 let target: i64 = progress.try_get("backfill_target").map_err(map_corrupt)?; 65 let mut backfill: i64 = progress.try_get("backfill_sequence").map_err(map_corrupt)?; 66 let mut replay_epoch: i64 = progress.try_get("replay_epoch").map_err(map_corrupt)?; 67 let mut replay_sequence: i64 = progress.try_get("replay_sequence").map_err(map_corrupt)?; 68 if version != 1 || target > sequence || backfill > target || replay_epoch > epoch { 69 return Err(Error::CorruptStoredEvent); 70 } 71 let had_pending: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM radroots_runtime_visibility_pending WHERE source_generation = ?)") 72 .bind(self.generation.as_bytes().as_slice()).fetch_one(&mut **transaction).await.map_err(map_backend)?; 73 let mut processed_keys = self 74 .drain_visibility_keys(transaction, bounds.rows(), &mut charged, bounds.bytes()) 75 .await?; 76 let remaining = if had_pending { 77 0 78 } else { 79 bounds.rows() - processed_keys 80 }; 81 let building = backfill < target; 82 let mut query = QueryBuilder::<Sqlite>::new( 83 "SELECT source_sequence, last_mutation_epoch, length(signed_event) AS signed_bytes 84 FROM radroots_runtime_events WHERE source_generation = ", 85 ); 86 query.push_bind(self.generation.as_bytes().as_slice()); 87 if building { 88 query.push(" AND source_sequence > ").push_bind(backfill); 89 query.push(" AND source_sequence <= ").push_bind(target); 90 query 91 .push(" ORDER BY source_sequence LIMIT ") 92 .push_bind(i64::from(remaining)); 93 } else { 94 query.push(" AND (last_mutation_epoch, source_sequence) > ("); 95 query 96 .push_bind(replay_epoch) 97 .push(", ") 98 .push_bind(replay_sequence) 99 .push(")"); 100 query 101 .push(" ORDER BY last_mutation_epoch, source_sequence LIMIT ") 102 .push_bind(i64::from(remaining)); 103 } 104 let lengths = query 105 .build() 106 .fetch_all(&mut **transaction) 107 .await 108 .map_err(map_backend)?; 109 if building && remaining > 0 && lengths.is_empty() { 110 return Err(Error::CorruptStoredEvent); 111 } 112 let mut processed = 0u16; 113 for length in lengths { 114 let position: i64 = length.try_get("source_sequence").map_err(map_corrupt)?; 115 let changed: i64 = length.try_get("last_mutation_epoch").map_err(map_corrupt)?; 116 let size = u64_from_i64(length.try_get("signed_bytes").map_err(map_corrupt)?)?; 117 let charge = VisibilityIndexBounds::charge(size)?; 118 let next_charge = charged 119 .checked_add(charge) 120 .ok_or(Error::SpaceInsufficient)?; 121 if next_charge > bounds.bytes() { 122 if processed == 0 { 123 return Err(Error::SpaceInsufficient); 124 } 125 break; 126 } 127 let row = sqlx::query( 128 "SELECT source_generation, source_sequence, signed_event, admission_stage, 129 admitted_contract_id, admitted_registry_version FROM radroots_runtime_events 130 WHERE source_generation = ? AND source_sequence = ?", 131 ) 132 .bind(self.generation.as_bytes().as_slice()) 133 .bind(position) 134 .fetch_one(&mut **transaction) 135 .await 136 .map_err(map_backend)?; 137 let decoded = self.decode_event_row(&row)?; 138 if decoded.raw_json.len() as u64 != size || changed > epoch { 139 return Err(Error::CorruptStoredEvent); 140 } 141 let metadata = VisibilityEventMetadata::from_event(&decoded.event, decoded.stage)?; 142 self.put_visibility_metadata(transaction, position, changed, metadata) 143 .await?; 144 charged = next_charge; 145 processed += 1; 146 if building { 147 backfill = position; 148 } else { 149 replay_epoch = changed; 150 replay_sequence = position; 151 } 152 } 153 let pending: bool = sqlx::query_scalar( 154 "SELECT EXISTS (SELECT 1 FROM radroots_runtime_events 155 WHERE source_generation = ? AND (last_mutation_epoch, source_sequence) > (?, ?))", 156 ) 157 .bind(self.generation.as_bytes().as_slice()) 158 .bind(replay_epoch) 159 .bind(replay_sequence) 160 .fetch_one(&mut **transaction) 161 .await 162 .map_err(map_backend)?; 163 processed_keys += self 164 .drain_visibility_keys( 165 transaction, 166 bounds.rows() - processed - processed_keys, 167 &mut charged, 168 bounds.bytes(), 169 ) 170 .await?; 171 let pending_keys: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM radroots_runtime_visibility_pending WHERE source_generation = ?)") 172 .bind(self.generation.as_bytes().as_slice()).fetch_one(&mut **transaction).await.map_err(map_backend)?; 173 if processed == 0 && processed_keys == 0 && pending_keys { 174 return Err(Error::SpaceInsufficient); 175 } 176 let complete = backfill == target && !pending && !pending_keys; 177 if complete { 178 let retained: i64 = sqlx::query_scalar( 179 "SELECT raw_events FROM radroots_runtime_visibility_progress WHERE source_generation = ?") 180 .bind(self.generation.as_bytes().as_slice()).fetch_one(&mut **transaction).await.map_err(map_backend)?; 181 if retained != sequence { 182 return Err(Error::CorruptStoredEvent); 183 } 184 replay_epoch = epoch; 185 replay_sequence = i64::MAX; 186 } 187 sqlx::query( 188 "UPDATE radroots_runtime_visibility_progress SET backfill_sequence = ?, 189 replay_epoch = ?, replay_sequence = ?, ready = ? WHERE source_generation = ?", 190 ) 191 .bind(backfill) 192 .bind(replay_epoch) 193 .bind(replay_sequence) 194 .bind(complete) 195 .bind(self.generation.as_bytes().as_slice()) 196 .execute(&mut **transaction) 197 .await 198 .map_err(map_backend)?; 199 Ok(VisibilityIndexProgress::new( 200 EventSourceToken::new(self.generation, u64_from_i64(epoch)?), 201 u64_from_i64(backfill)?, 202 u64_from_i64(replay_epoch)?, 203 processed, 204 charged, 205 complete, 206 ) 207 .with_processed_keys(processed_keys)) 208 } 209 210 async fn put_visibility_metadata( 211 &self, 212 transaction: &mut sqlx::Transaction<'_, Sqlite>, 213 position: i64, 214 changed: i64, 215 metadata: VisibilityEventMetadata, 216 ) -> Result<(), Error> { 217 sqlx::query( 218 "INSERT INTO radroots_runtime_visibility_events 219 (source_generation, source_sequence, event_id, last_mutation_epoch, author, created_at, 220 admission_stage, head_coordinate, deletion_coordinate, ephemeral, deletion_request) 221 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) 222 ON CONFLICT(source_generation, source_sequence) DO UPDATE SET 223 last_mutation_epoch = excluded.last_mutation_epoch, 224 admission_stage = excluded.admission_stage, head_coordinate = excluded.head_coordinate, 225 deletion_coordinate = excluded.deletion_coordinate, 226 ephemeral = excluded.ephemeral, deletion_request = excluded.deletion_request", 227 ) 228 .bind(self.generation.as_bytes().as_slice()).bind(position) 229 .bind(metadata.event_id.as_bytes().as_slice()).bind(changed) 230 .bind(metadata.author.as_slice()).bind(metadata.created_at.to_be_bytes().as_slice()) 231 .bind(stage_name(metadata.stage)).bind(metadata.head_coordinate.as_deref()) 232 .bind(metadata.deletion_coordinate.as_deref()).bind(metadata.ephemeral) 233 .bind(metadata.deletion_request) 234 .execute(&mut **transaction).await.map_err(map_backend)?; 235 // Signed event bytes and deletion tags are immutable. Replays only add identical targets. 236 for target in metadata.event_targets { 237 sqlx::query( 238 "INSERT OR IGNORE INTO radroots_runtime_visibility_deletion_events 239 (source_generation, target_event, author, request_id) VALUES (?, ?, ?, ?)", 240 ) 241 .bind(self.generation.as_bytes().as_slice()) 242 .bind(target.as_bytes().as_slice()) 243 .bind(metadata.author.as_slice()) 244 .bind(metadata.event_id.as_bytes().as_slice()) 245 .execute(&mut **transaction) 246 .await 247 .map_err(map_backend)?; 248 } 249 for coordinate in metadata.address_targets { 250 sqlx::query( 251 "INSERT OR IGNORE INTO radroots_runtime_visibility_deletion_addresses 252 (source_generation, target_coordinate, author, created_at, request_id) 253 VALUES (?, ?, ?, ?, ?)", 254 ) 255 .bind(self.generation.as_bytes().as_slice()) 256 .bind(coordinate) 257 .bind(metadata.author.as_slice()) 258 .bind(metadata.created_at.to_be_bytes().as_slice()) 259 .bind(metadata.event_id.as_bytes().as_slice()) 260 .execute(&mut **transaction) 261 .await 262 .map_err(map_backend)?; 263 } 264 self.update_visibility_membership(transaction, position) 265 .await?; 266 Ok(()) 267 } 268 }