visibility_reads_tests.rs (19043B)
1 use super::*; 2 use radroots_storage::{event::VisibilityIndexBounds, memory::MemoryStorage}; 3 4 fn query(limit: u16) -> EventQuery { 5 EventQuery::all(EventQueryBounds::first(limit).unwrap()) 6 } 7 8 fn admission(event: SignedEvent) -> EventAdmission { 9 EventAdmission::visible( 10 tests::observed(event.clone(), 10, None), 11 tests::visible(&event), 12 ) 13 .unwrap() 14 } 15 16 async fn finish(storage: &SqliteStorage) { 17 for _ in 0..1000 { 18 let progress = storage 19 .maintain_visibility(VisibilityIndexBounds::new(64, 16 * 1024 * 1024).unwrap()) 20 .await 21 .unwrap(); 22 assert!(progress.processed_rows() + progress.processed_keys() <= 64); 23 assert!(progress.charged_bytes() <= 16 * 1024 * 1024); 24 if progress.is_complete() { 25 return; 26 } 27 } 28 panic!("bounded fixture maintenance failed to complete"); 29 } 30 31 #[tokio::test] 32 async fn indexed_visibility_refuses_stale_fences_without_full_source_decode() { 33 let generation = SourceGeneration::new([91; 32]).unwrap(); 34 let storage = tests::store(generation).await; 35 let token = storage.source_token().await.unwrap(); 36 let query = EventQuery::all(EventQueryBounds::first(5).unwrap()); 37 assert!( 38 storage 39 .query_visible_at(query.clone(), token) 40 .await 41 .unwrap() 42 .items() 43 .is_empty() 44 ); 45 let event = tests::signed_event_with("retained", false, 10, 1, vec![]); 46 let admission = EventAdmission::visible( 47 tests::observed(event.clone(), 10, None), 48 tests::visible(&event), 49 ) 50 .unwrap(); 51 storage.admit(admission).await.unwrap(); 52 storage.work.take(); 53 assert_eq!( 54 storage.query_visible_at(query.clone(), token).await, 55 Err(Error::SourceEpochChanged) 56 ); 57 let refused = storage.work.take(); 58 assert_eq!(refused.codec_inputs, 0); 59 assert_eq!(refused.materialization_requests, 0); 60 let current = storage.source_token().await.unwrap(); 61 assert_eq!( 62 storage 63 .query_visible_at(query, current) 64 .await 65 .unwrap() 66 .items() 67 .len(), 68 1 69 ); 70 let page = storage.work.take(); 71 assert_eq!(page.codec_inputs, 1); 72 assert_eq!(page.visibility_inputs, 0); 73 assert_eq!(page.materialization_requests, 0); 74 assert_eq!(storage.status().await.unwrap().visible_events(), 1); 75 assert_eq!( 76 storage 77 .query_visible_at( 78 EventQuery::all(EventQueryBounds::first(5).unwrap()), 79 EventSourceToken::new(SourceGeneration::new([90; 32]).unwrap(), 1) 80 ) 81 .await, 82 Err(Error::SourceEpochChanged) 83 ); 84 } 85 86 #[tokio::test] 87 async fn indexed_visibility_tiny_key_budget_preserves_pending_work_and_cannot_claim_false_ready() { 88 let generation = SourceGeneration::new([98; 32]).unwrap(); 89 let storage = tests::store(generation).await; 90 let memory = MemoryStorage::new(generation); 91 let mut tags = Vec::new(); 92 for index in 0..65 { 93 let event = tests::signed_event_with(&format!("target {index}"), false, index, 1, vec![]); 94 tags.push(vec!["e".to_owned(), event.id().to_hex()]); 95 let value = admission(event); 96 storage.admit(value.clone()).await.unwrap(); 97 memory.admit(value).await.unwrap(); 98 } 99 let deletion = admission(tests::signed_event_with("delete", false, 100, 5, tags)); 100 storage.admit(deletion.clone()).await.unwrap(); 101 memory.admit(deletion).await.unwrap(); 102 let tiny = VisibilityIndexBounds::new(1, 4096 + 128).unwrap(); 103 assert_eq!( 104 storage.maintain_visibility(tiny).await, 105 Err(Error::SpaceInsufficient) 106 ); 107 assert_eq!( 108 memory.maintain_visibility(tiny).await, 109 Err(Error::SpaceInsufficient) 110 ); 111 assert_eq!( 112 storage.query_visible(query(5)).await, 113 Err(Error::VisibilityRebuilding) 114 ); 115 assert_eq!(memory.status().await, Err(Error::VisibilityRebuilding)); 116 let enough = VisibilityIndexBounds::new(1, 4096 + 128 + 4096 + 16 * 32).unwrap(); 117 let first = storage.maintain_visibility(enough).await.unwrap(); 118 assert_eq!(first, memory.maintain_visibility(enough).await.unwrap()); 119 assert_eq!(first.processed_keys(), 1); 120 assert!(!first.is_complete()); 121 let done = storage.maintain_visibility(enough).await.unwrap(); 122 assert_eq!(done, memory.maintain_visibility(enough).await.unwrap()); 123 assert!(done.is_complete()); 124 assert_eq!(storage.status().await.unwrap().visible_events(), 1); 125 sqlx::query("UPDATE radroots_runtime_source_generations SET sequence_head = sequence_head + 1 WHERE generation = ?") 126 .bind(generation.as_bytes().as_slice()).execute(&storage.pool).await.unwrap(); 127 assert_eq!( 128 storage 129 .maintain_visibility(VisibilityIndexBounds::new(64, 16 * 1024 * 1024).unwrap()) 130 .await, 131 Err(Error::CorruptStoredEvent) 132 ); 133 assert_eq!(storage.status().await, Err(Error::CorruptStoredEvent)); 134 } 135 136 #[tokio::test] 137 async fn indexed_visibility_large_mixed_histories_have_bounded_next_pages_and_truthful_cached_counts() 138 { 139 for size in [10_000_i64, 16_384_i64] { 140 let generation = SourceGeneration::new([92; 32]).unwrap(); 141 let storage = tests::store(generation).await; 142 let mut transaction = storage.pool.begin_with("BEGIN IMMEDIATE").await.unwrap(); 143 sqlx::query( 144 "UPDATE radroots_runtime_source_generations SET sequence_head = ? WHERE generation = ?", 145 ) 146 .bind(size) 147 .bind(generation.as_bytes().as_slice()) 148 .execute(&mut *transaction) 149 .await 150 .unwrap(); 151 for index in 0..size { 152 let profile = index % 4 == 0; 153 let unsupported = index % 4 == 3; 154 let content = if profile { 155 format!(r#"{{"name":"Farm {index}"}}"#) 156 } else { 157 format!("retained {index}") 158 }; 159 let event = tests::signed_event_with( 160 &content, 161 false, 162 index as u64 + 100, 163 if profile || unsupported { 0 } else { 1 }, 164 vec![], 165 ); 166 let stage = match index % 4 { 167 2 => "raw", 168 3 => "verified", 169 _ => "visible", 170 }; 171 let contract = if stage == "visible" { 172 Some( 173 tests::visible(&event) 174 .admitted_event() 175 .validated_event() 176 .contract_id(), 177 ) 178 } else { 179 None 180 }; 181 sqlx::query("INSERT INTO radroots_runtime_events 182 (source_generation, source_sequence, event_id, admission_stage, signed_event, 183 admitted_at_unix_ms, updated_at_unix_ms, admitted_contract_id, admitted_registry_version) 184 VALUES (?, ?, ?, ?, ?, 10, 10, ?, ?)") 185 .bind(generation.as_bytes().as_slice()).bind(index + 1).bind(event.id().as_bytes().as_slice()) 186 .bind(stage).bind(event.raw_json().as_bytes()).bind(contract) 187 .bind(contract.map(|_| i64::from(radroots_event::contract::RegistryVersion::CURRENT.get()))) 188 .execute(&mut *transaction).await.unwrap(); 189 } 190 transaction.commit().await.unwrap(); 191 assert_eq!(storage.status().await, Err(Error::VisibilityRebuilding)); 192 assert_eq!( 193 storage.query_visible(query(5)).await, 194 Err(Error::VisibilityRebuilding) 195 ); 196 finish(&storage).await; 197 let oracle = storage.rebuild_visibility().await.unwrap(); 198 let status = storage.status().await.unwrap(); 199 assert_eq!(status.raw_events(), size as u64); 200 assert_eq!(status.verified_events(), (size * 3 / 4) as u64); 201 assert_eq!( 202 status.visible_events(), 203 oracle.visible_event_ids().len() as u64 204 ); 205 storage.work.take(); 206 let token = storage.source_token().await.unwrap(); 207 let mut bounds = EventQueryBounds::first(1000).unwrap(); 208 let mut returned = std::collections::BTreeSet::new(); 209 loop { 210 let page = storage 211 .query_visible_at(EventQuery::all(bounds), token) 212 .await 213 .unwrap(); 214 let counts = storage.work.take(); 215 assert_eq!(counts.materialization_requests, 0); 216 assert_eq!(counts.visibility_inputs, 0); 217 assert!(counts.key_rows_returned <= 1001); 218 assert_eq!(counts.codec_inputs, page.items().len() as u64); 219 for item in page.items() { 220 assert!(returned.insert(*item.event().id())); 221 } 222 match page.next_cursor() { 223 Some(cursor) => bounds = EventQueryBounds::first(1000).unwrap().after(cursor), 224 None => break, 225 } 226 } 227 assert_eq!( 228 returned, 229 oracle.visible_event_ids().iter().copied().collect() 230 ); 231 let first = storage.query_visible(query(5)).await.unwrap(); 232 storage.work.take(); 233 let next = storage 234 .query_visible(EventQuery::all( 235 EventQueryBounds::first(5) 236 .unwrap() 237 .after(first.next_cursor().unwrap()), 238 )) 239 .await 240 .unwrap(); 241 let counts = storage.work.take(); 242 assert_eq!(next.items().len(), 5); 243 assert_eq!(counts.key_rows_returned, 6); 244 assert_eq!(counts.codec_inputs, 5); 245 assert_eq!(counts.materialization_requests, 0); 246 let sql = storage.work.template(work::Read::Keys); 247 let mut builder = QueryBuilder::<Sqlite>::new("EXPLAIN QUERY PLAN "); 248 builder.push(sql); 249 let plans = builder 250 .build() 251 .bind(generation.as_bytes().as_slice()) 252 .bind(i64_from_u64(first.next_cursor().unwrap().sequence().get()).unwrap()) 253 .bind(6_i64) 254 .fetch_all(&storage.pool) 255 .await 256 .unwrap(); 257 assert!(plans.iter().any(|row| { 258 row.get::<String, _>("detail") 259 .contains("visibility_members_page_idx") 260 })); 261 assert!( 262 !plans 263 .iter() 264 .any(|row| row.get::<String, _>("detail").contains("USE TEMP B-TREE")) 265 ); 266 storage.work.take(); 267 assert_eq!(storage.status().await.unwrap(), status); 268 let cached = storage.work.take(); 269 assert_eq!(cached.aggregate_rows_returned, 1); 270 assert_eq!(cached.codec_inputs, 0); 271 storage.close().await.unwrap(); 272 } 273 } 274 275 #[tokio::test] 276 async fn indexed_visibility_deletion_queue_is_bounded_restartable_and_matches_memory() { 277 let generation = SourceGeneration::new([93; 32]).unwrap(); 278 let storage = tests::store(generation).await; 279 let memory = MemoryStorage::new(generation); 280 let mut tags = Vec::new(); 281 for index in 0..133 { 282 let event = tests::signed_event_with(&format!("target {index}"), false, index, 1, vec![]); 283 tags.push(vec!["e".to_owned(), event.id().to_hex()]); 284 let value = admission(event); 285 storage.admit(value.clone()).await.unwrap(); 286 memory.admit(value).await.unwrap(); 287 } 288 let deletion = admission(tests::signed_event_with("delete", false, 200, 5, tags)); 289 storage.admit(deletion.clone()).await.unwrap(); 290 memory.admit(deletion).await.unwrap(); 291 assert_eq!(storage.status().await, Err(Error::VisibilityRebuilding)); 292 assert_eq!( 293 memory.query_visible(query(5)).await, 294 Err(Error::VisibilityRebuilding) 295 ); 296 let reopened = SqliteStorage::new(storage.pool.clone(), generation, EventStoreMode::ReadWrite); 297 let bounds = VisibilityIndexBounds::new(64, 16 * 1024 * 1024).unwrap(); 298 let first = reopened.maintain_visibility(bounds).await.unwrap(); 299 assert_eq!(first, memory.maintain_visibility(bounds).await.unwrap()); 300 assert_eq!(first.processed_rows(), 0); 301 assert_eq!(first.processed_keys(), 64); 302 assert!(!first.is_complete()); 303 // Admission while target work is unfinished must retain both cursors. 304 let later = admission(tests::signed_event_with("later", false, 300, 1, vec![])); 305 reopened.admit(later.clone()).await.unwrap(); 306 memory.admit(later).await.unwrap(); 307 for _ in 0..4 { 308 let progress = reopened.maintain_visibility(bounds).await.unwrap(); 309 assert_eq!(progress, memory.maintain_visibility(bounds).await.unwrap()); 310 assert!(progress.processed_rows() + progress.processed_keys() <= 64); 311 if progress.is_complete() { 312 break; 313 } 314 } 315 assert_eq!( 316 reopened.status().await.unwrap(), 317 memory.status().await.unwrap() 318 ); 319 assert_eq!(reopened.status().await.unwrap().visible_events(), 2); 320 assert_eq!( 321 reopened.query_visible(query(5)).await.unwrap(), 322 memory.query_visible(query(5)).await.unwrap() 323 ); 324 assert_eq!( 325 reopened.rebuild_visibility().await.unwrap(), 326 memory.rebuild_visibility().await.unwrap() 327 ); 328 } 329 330 #[tokio::test] 331 async fn indexed_visibility_partial_byte_pages_preserve_continuation_and_id_navigation() { 332 let generation = SourceGeneration::new([94; 32]).unwrap(); 333 let storage = tests::store(generation).await; 334 let memory = MemoryStorage::new(generation); 335 let mut ids = Vec::new(); 336 for index in 0..12 { 337 let event = tests::signed_event_with( 338 &format!("{index}{}", "x".repeat(120_000)), 339 false, 340 index, 341 1, 342 vec![], 343 ); 344 ids.push(*event.id()); 345 let value = admission(event); 346 storage.admit(value.clone()).await.unwrap(); 347 memory.admit(value).await.unwrap(); 348 } 349 let first = storage.query_visible(query(12)).await.unwrap(); 350 assert_eq!(first, memory.query_visible(query(12)).await.unwrap()); 351 assert!(first.items().len() < 12 && !first.items().is_empty()); 352 let next_bounds = EventQueryBounds::first(12) 353 .unwrap() 354 .after(first.next_cursor().unwrap()); 355 let next = storage 356 .query_visible(EventQuery::all(next_bounds)) 357 .await 358 .unwrap(); 359 assert_eq!(first.items().len() + next.items().len(), 12); 360 assert!(next.next_cursor().is_none()); 361 storage.work.take(); 362 let selected = 363 EventQuery::for_ids(EventQueryBounds::first(5).unwrap(), vec![ids[0], ids[11]]).unwrap(); 364 let page = storage.query_visible(selected.clone()).await.unwrap(); 365 assert_eq!(page, memory.query_visible(selected).await.unwrap()); 366 let counts = storage.work.take(); 367 assert_eq!(counts.key_rows_returned, 2); 368 assert_eq!(counts.codec_inputs, 2); 369 assert!( 370 storage 371 .work 372 .template(work::Read::Keys) 373 .contains("INDEXED BY radroots_runtime_visibility_members_event_idx") 374 ); 375 } 376 377 #[tokio::test] 378 async fn indexed_visibility_fault_after_membership_rolls_back_source_counts_and_readiness() { 379 let storage = tests::store(SourceGeneration::new([95; 32]).unwrap()).await; 380 let token = storage.source_token().await.unwrap(); 381 let before = storage.status().await.unwrap(); 382 sqlx::raw_sql( 383 "CREATE TEMP TRIGGER fail_membership AFTER INSERT ON radroots_runtime_visibility_members 384 BEGIN SELECT RAISE(ABORT, 'fixture membership fault'); END", 385 ) 386 .execute(&storage.pool) 387 .await 388 .unwrap(); 389 let event = tests::signed_event_with("failure", false, 10, 1, vec![]); 390 assert_eq!( 391 storage.admit(admission(event.clone())).await, 392 Err(Error::BackendUnavailable) 393 ); 394 assert_eq!(storage.source_token().await.unwrap(), token); 395 assert_eq!(storage.status().await.unwrap(), before); 396 assert!( 397 storage 398 .query_raw(query(5)) 399 .await 400 .unwrap() 401 .items() 402 .is_empty() 403 ); 404 sqlx::query("DROP TRIGGER fail_membership") 405 .execute(&storage.pool) 406 .await 407 .unwrap(); 408 storage.admit(admission(event)).await.unwrap(); 409 assert_eq!(storage.status().await.unwrap().visible_events(), 1); 410 } 411 412 #[tokio::test] 413 async fn indexed_visibility_cancelled_pool_wait_has_no_payload_or_maintenance_effects() { 414 use std::task::{Context, Poll, Waker}; 415 let storage = tests::store(SourceGeneration::new([96; 32]).unwrap()).await; 416 storage 417 .admit(admission(tests::signed_event_with( 418 "retained", 419 false, 420 10, 421 1, 422 vec![], 423 ))) 424 .await 425 .unwrap(); 426 let token = storage.source_token().await.unwrap(); 427 storage.work.take(); 428 let held = storage.pool.acquire().await.unwrap(); 429 let mut future = storage.query_visible_at(query(5), token); 430 assert!(matches!( 431 future 432 .as_mut() 433 .poll(&mut Context::from_waker(Waker::noop())), 434 Poll::Pending 435 )); 436 drop(future); 437 drop(held); 438 assert_eq!(storage.work.take(), work::Counts::default()); 439 assert_eq!(storage.source_token().await.unwrap(), token); 440 assert_eq!( 441 storage 442 .query_visible_at(query(5), token) 443 .await 444 .unwrap() 445 .items() 446 .len(), 447 1 448 ); 449 } 450 451 #[tokio::test] 452 async fn indexed_visibility_concurrent_wal_admission_returns_one_fenced_head_or_typed_stale() { 453 let directory = tempfile::tempdir().unwrap(); 454 let paths = crate::Paths::from_directory(directory.path()).unwrap(); 455 let storage = SqliteStorage::open( 456 crate::OpenOptions::new(paths, crate::OpenMode::Create) 457 .with_source_generation(SourceGeneration::new([97; 32]).unwrap(), 100) 458 .unwrap(), 459 ) 460 .await 461 .unwrap(); 462 let mut current = tests::signed_event_with(r#"{"name":"initial"}"#, false, 1, 0, vec![]); 463 storage.admit(admission(current.clone())).await.unwrap(); 464 for index in 2..18 { 465 let next = tests::signed_event_with( 466 &format!(r#"{{"name":"Farm {index}"}}"#), 467 false, 468 index, 469 0, 470 vec![], 471 ); 472 let token = storage.source_token().await.unwrap(); 473 let (read, write) = tokio::join!( 474 storage.query_visible_at(query(5), token), 475 storage.admit(admission(next.clone())) 476 ); 477 write.unwrap(); 478 match read { 479 Ok(page) => { 480 assert_eq!(page.items().len(), 1); 481 assert_eq!(page.items()[0].event().id(), current.id()); 482 } 483 Err(Error::SourceEpochChanged) => {} 484 Err(error) => panic!("unexpected fenced read failure: {error}"), 485 } 486 let page = storage 487 .query_visible_at(query(5), storage.source_token().await.unwrap()) 488 .await 489 .unwrap(); 490 assert_eq!(page.items().len(), 1); 491 assert_eq!(page.items()[0].event().id(), next.id()); 492 current = next; 493 } 494 storage.close().await.unwrap(); 495 }