cache_retired_owner_tests.rs (20522B)
1 use super::*; 2 use crate::{OpenMode, OpenOptions, Paths}; 3 use radroots_storage::{ 4 ProjectionStore, Storage, 5 event::SourceGeneration, 6 memory::MemoryStorage, 7 projection::{ 8 InvalidationReason, ProjectionCheckpoint, ProjectionDocument, ProjectionHealth, 9 ProjectionInvalidation, ProjectionSnapshot, 10 batch::{ProjectionDocumentBatch, ProjectionWriteScope}, 11 cache::{ProjectionCacheLimits, ProjectionCacheScope, ProjectionCacheUsage}, 12 document_query::{ProjectionDocumentGenerations, ProjectionDocumentQuery}, 13 }, 14 }; 15 use sqlx::Row; 16 use tempfile::TempDir; 17 18 fn id() -> ProjectionId { 19 ProjectionId::parse("retired_owner_fixture").unwrap() 20 } 21 fn foreign() -> ProjectionId { 22 ProjectionId::parse("foreign_fixture").unwrap() 23 } 24 fn generation(value: u8) -> ProjectionGeneration { 25 ProjectionGeneration::new([value; 32]).unwrap() 26 } 27 fn limits() -> ProjectionCacheLimits { 28 ProjectionCacheLimits::new(67_108_864, 2, 32, 1_800_000).unwrap() 29 } 30 fn query() -> ProjectionDocumentQuery { 31 ProjectionDocumentQuery::new(id(), ProjectionDocumentGenerations::All, 256).unwrap() 32 } 33 async fn open(temp: &TempDir, mode: OpenMode) -> SqliteStorage { 34 let options = OpenOptions::new(Paths::from_directory(temp.path()).unwrap(), mode); 35 let options = if mode == OpenMode::Create { 36 options 37 .with_source_generation(SourceGeneration::new([9; 32]).unwrap(), 9) 38 .unwrap() 39 } else { 40 options 41 }; 42 SqliteStorage::open(options).await.unwrap() 43 } 44 fn batch( 45 source: EventSourceToken, 46 active: Option<ProjectionGeneration>, 47 target: u8, 48 start: usize, 49 count: usize, 50 ) -> ProjectionDocumentBatch { 51 ProjectionDocumentBatch::new( 52 ProjectionWriteScope::new(id(), generation(target), source, active).unwrap(), 53 (start..start + count) 54 .map(|index| ProjectionDocument::new(format!("row.{index:03}"), vec![target]).unwrap()) 55 .collect(), 56 ) 57 .unwrap() 58 } 59 async fn seed(backend: &dyn Storage) -> EventSourceToken { 60 let source = backend.source_token().await.unwrap(); 61 backend 62 .put_projection_snapshot( 63 ProjectionSnapshot::new(id(), [7; 32], generation(1), 1, vec![7; 8]).unwrap(), 64 ) 65 .await 66 .unwrap(); 67 backend 68 .configure_projection_cache(id(), limits()) 69 .await 70 .unwrap(); 71 backend 72 .promote_projection_batch( 73 batch(source, None, 1, 0, 63), 74 ProjectionCheckpoint::new(id(), generation(1), None, 63, 1).unwrap(), 75 ) 76 .await 77 .unwrap(); 78 backend 79 .put_projection_batch(batch(source, Some(generation(1)), 2, 0, 64)) 80 .await 81 .unwrap(); 82 backend 83 .promote_projection_batch( 84 batch(source, Some(generation(1)), 2, 64, 1), 85 ProjectionCheckpoint::new(id(), generation(2), None, 65, 2).unwrap(), 86 ) 87 .await 88 .unwrap(); 89 backend 90 .promote_projection_batch( 91 batch(source, Some(generation(2)), 3, 0, 1), 92 ProjectionCheckpoint::new(id(), generation(3), None, 1, 3).unwrap(), 93 ) 94 .await 95 .unwrap(); 96 backend 97 .put_projection_batch(batch(source, Some(generation(3)), 4, 0, 1)) 98 .await 99 .unwrap(); 100 backend 101 .put_projection_document( 102 foreign(), 103 generation(5), 104 ProjectionDocument::new("keep".into(), vec![5]).unwrap(), 105 ) 106 .await 107 .unwrap(); 108 source 109 } 110 async fn preserved(backend: &dyn Storage, source: EventSourceToken) { 111 assert_eq!(backend.source_token().await.unwrap(), source); 112 assert_eq!( 113 ProjectionStore::status(backend, id()) 114 .await 115 .unwrap() 116 .unwrap() 117 .generation(), 118 generation(3) 119 ); 120 for target in [3, 4] { 121 assert_eq!( 122 backend 123 .projection_document(id(), generation(target), "row.000".into()) 124 .await 125 .unwrap() 126 .unwrap() 127 .value(), 128 &[target] 129 ); 130 } 131 assert_eq!( 132 backend 133 .projection_document(foreign(), generation(5), "keep".into()) 134 .await 135 .unwrap() 136 .unwrap() 137 .value(), 138 &[5] 139 ); 140 } 141 142 #[tokio::test] 143 async fn projection_retired_owner_collection_bounds_every_pass_and_preserves_active_staging_foreign() 144 { 145 let temp = TempDir::new().unwrap(); 146 let store = open(&temp, OpenMode::Create).await; 147 let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap()); 148 for backend in [&memory as &dyn Storage, &store as &dyn Storage] { 149 let source = seed(backend).await; 150 let before = backend 151 .configure_projection_cache(id(), limits()) 152 .await 153 .unwrap(); 154 assert_eq!(before.value_bytes, 139); 155 assert_eq!(before.protected_generations, 2); 156 assert_eq!(before.legacy_snapshots, 1); 157 let mut freed = 0; 158 for (index, removed) in [64, 1, 64, 2].into_iter().enumerate() { 159 let result = backend 160 .collect_retired_projection_cache(id(), source, Some(generation(3))) 161 .await 162 .unwrap(); 163 assert_eq!(result.removed_rows, removed); 164 assert!(result.removed_rows <= 64); 165 assert_eq!(result.more, index < 3); 166 freed += result.removed_value_bytes; 167 preserved(backend, source).await; 168 } 169 assert_eq!(freed, 136); 170 let done = backend 171 .collect_retired_projection_cache(id(), source, Some(generation(3))) 172 .await 173 .unwrap(); 174 assert_eq!(done.removed_rows, 0); 175 assert_eq!(done.removed_value_bytes, 0); 176 assert!(!done.more); 177 assert_eq!( 178 done.usage, 179 ProjectionCacheUsage { 180 value_bytes: 3, 181 protected_generations: 2, 182 legacy_snapshots: 0 183 } 184 ); 185 assert_eq!( 186 backend 187 .query_projection_documents(query()) 188 .await 189 .unwrap() 190 .records() 191 .len(), 192 2 193 ); 194 } 195 store.close().await.unwrap(); 196 } 197 198 #[tokio::test] 199 async fn projection_retired_owner_collection_fences_no_work_and_refuses_before_effects() { 200 let temp = TempDir::new().unwrap(); 201 let store = open(&temp, OpenMode::Create).await; 202 let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap()); 203 for backend in [&memory as &dyn Storage, &store as &dyn Storage] { 204 let source = backend.source_token().await.unwrap(); 205 assert_eq!( 206 backend 207 .collect_retired_projection_cache(id(), source, None) 208 .await, 209 Err(Error::ProjectionRevisionConflict) 210 ); 211 let empty_id = ProjectionId::parse("empty_fixture").unwrap(); 212 backend 213 .configure_projection_cache(empty_id.clone(), limits()) 214 .await 215 .unwrap(); 216 let empty = backend 217 .collect_retired_projection_cache(empty_id.clone(), source, None) 218 .await 219 .unwrap(); 220 assert_eq!(empty.removed_rows, 0); 221 assert!(!empty.more); 222 let stale = EventSourceToken::new(source.generation(), source.epoch() + 1); 223 assert_eq!( 224 backend 225 .collect_retired_projection_cache(empty_id.clone(), stale, None) 226 .await, 227 Err(Error::SourceEpochChanged) 228 ); 229 assert_eq!( 230 backend 231 .collect_retired_projection_cache(empty_id, source, Some(generation(3))) 232 .await, 233 Err(Error::ProjectionRevisionConflict) 234 ); 235 let source = seed(backend).await; 236 let before = backend 237 .query_projection_documents(query()) 238 .await 239 .unwrap() 240 .into_records(); 241 let usage = backend 242 .configure_projection_cache(id(), limits()) 243 .await 244 .unwrap(); 245 for changed in [ 246 stale, 247 EventSourceToken::new(SourceGeneration::new([8; 32]).unwrap(), 0), 248 ] { 249 assert_eq!( 250 backend 251 .collect_retired_projection_cache(id(), changed, Some(generation(3))) 252 .await, 253 Err(Error::SourceEpochChanged) 254 ); 255 } 256 assert_eq!( 257 backend 258 .collect_retired_projection_cache(id(), source, Some(generation(2))) 259 .await, 260 Err(Error::ProjectionRevisionConflict) 261 ); 262 assert_eq!( 263 backend 264 .collect_retired_projection_cache(foreign(), source, None) 265 .await, 266 Err(Error::ProjectionRevisionConflict) 267 ); 268 assert_eq!( 269 backend 270 .query_projection_documents(query()) 271 .await 272 .unwrap() 273 .into_records(), 274 before 275 ); 276 assert_eq!( 277 backend 278 .configure_projection_cache(id(), limits()) 279 .await 280 .unwrap(), 281 usage 282 ); 283 preserved(backend, source).await; 284 } 285 store.close().await.unwrap(); 286 } 287 288 #[tokio::test] 289 async fn projection_retired_owner_collection_rolls_back_and_recovers_across_read_race_and_reopen() { 290 let temp = TempDir::new().unwrap(); 291 let store = open(&temp, OpenMode::Create).await; 292 let source = seed(&store).await; 293 let before = store 294 .query_projection_documents(query()) 295 .await 296 .unwrap() 297 .into_records(); 298 let usage = store 299 .configure_projection_cache(id(), limits()) 300 .await 301 .unwrap(); 302 sqlx::raw_sql("CREATE TRIGGER fixture_retired_delete BEFORE DELETE ON radroots_runtime_projection_documents WHEN OLD.document_key='row.010' BEGIN SELECT RAISE(ABORT,'fixture'); END;").execute(store.pool()).await.unwrap(); 303 assert!( 304 store 305 .collect_retired_projection_cache(id(), source, Some(generation(3))) 306 .await 307 .is_err() 308 ); 309 assert_eq!( 310 store 311 .query_projection_documents(query()) 312 .await 313 .unwrap() 314 .into_records(), 315 before 316 ); 317 assert_eq!( 318 store 319 .configure_projection_cache(id(), limits()) 320 .await 321 .unwrap(), 322 usage 323 ); 324 sqlx::raw_sql("DROP TRIGGER fixture_retired_delete") 325 .execute(store.pool()) 326 .await 327 .unwrap(); 328 let current = ProjectionDocumentQuery::new( 329 id(), 330 ProjectionDocumentGenerations::Exact(generation(3)), 331 20, 332 ) 333 .unwrap(); 334 let (read, collected) = tokio::join!( 335 store.query_projection_documents_at(current, source, generation(3)), 336 store.collect_retired_projection_cache(id(), source, Some(generation(3))), 337 ); 338 assert_eq!(read.unwrap().records().len(), 1); 339 assert_eq!(collected.unwrap().removed_rows, 64); 340 store.close().await.unwrap(); 341 let reopened = open(&temp, OpenMode::ReadWriteExisting).await; 342 for (removed, more) in [(1, true), (64, true), (2, false)] { 343 let result = reopened 344 .collect_retired_projection_cache(id(), source, Some(generation(3))) 345 .await 346 .unwrap(); 347 assert_eq!((result.removed_rows, result.more), (removed, more)); 348 } 349 preserved(&reopened, source).await; 350 reopened.close().await.unwrap(); 351 let readonly = open(&temp, OpenMode::ReadOnly).await; 352 assert_eq!( 353 readonly 354 .collect_retired_projection_cache(id(), source, Some(generation(3))) 355 .await, 356 Err(Error::BackendUnavailable) 357 ); 358 preserved(&readonly, source).await; 359 readonly.close().await.unwrap(); 360 } 361 362 #[tokio::test] 363 async fn projection_retired_owner_selection_uses_partial_index_without_payload_scan() { 364 let temp = TempDir::new().unwrap(); 365 let store = open(&temp, OpenMode::Create).await; 366 seed(&store).await; 367 let mut explain = sqlx::QueryBuilder::<sqlx::Sqlite>::new("EXPLAIN QUERY PLAN "); 368 explain.push(SELECTION); 369 let details = explain 370 .build() 371 .bind(id().as_str()) 372 .bind(generation(3).as_bytes().as_slice()) 373 .bind(generation(3).as_bytes().as_slice()) 374 .fetch_all(store.pool()) 375 .await 376 .unwrap(); 377 let details: Vec<String> = details 378 .iter() 379 .map(|row| row.try_get("detail").unwrap()) 380 .collect(); 381 assert!( 382 details.iter().any(|detail| detail.contains("SEARCH") 383 && detail.contains("radroots_runtime_projection_retired_generation_idx")), 384 "{details:?}" 385 ); 386 assert!( 387 !details 388 .iter() 389 .any(|detail| detail.contains("SCAN") || detail.contains("TEMP B-TREE")), 390 "{details:?}" 391 ); 392 assert_eq!( 393 store 394 .projection_document(id(), generation(1), "row.000".into()) 395 .await 396 .unwrap() 397 .unwrap() 398 .value(), 399 &[1] 400 ); 401 store.close().await.unwrap(); 402 } 403 404 #[tokio::test] 405 async fn projection_obsolete_snapshot_collection_preserves_legacy_custody_and_requires_safe_scope() 406 { 407 let temp = TempDir::new().unwrap(); 408 let store = open(&temp, OpenMode::Create).await; 409 let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap()); 410 let legacy = ProjectionId::parse("legacy_custody_fixture").unwrap(); 411 for backend in [&memory as &dyn Storage, &store as &dyn Storage] { 412 let source = backend.source_token().await.unwrap(); 413 for target in [9, 10] { 414 backend 415 .put_projection_document( 416 legacy.clone(), 417 generation(target), 418 ProjectionDocument::new("custody".into(), vec![target; 16]).unwrap(), 419 ) 420 .await 421 .unwrap(); 422 } 423 backend 424 .checkpoint( 425 ProjectionCheckpoint::new(legacy.clone(), generation(10), None, 1, 1).unwrap(), 426 ) 427 .await 428 .unwrap(); 429 for index in 1..=65 { 430 backend 431 .put_projection_snapshot( 432 ProjectionSnapshot::new( 433 legacy.clone(), 434 [index; 32], 435 generation(9), 436 1, 437 vec![7; 2], 438 ) 439 .unwrap(), 440 ) 441 .await 442 .unwrap(); 443 } 444 backend 445 .put_projection_snapshot( 446 ProjectionSnapshot::new(legacy.clone(), [200; 32], generation(10), 1, vec![8; 2]) 447 .unwrap(), 448 ) 449 .await 450 .unwrap(); 451 let old = || { 452 ProjectionCacheScope::new(legacy.clone(), generation(9), source, Some(generation(10))) 453 .unwrap() 454 }; 455 assert_eq!( 456 backend.collect_obsolete_projection_snapshots(old()).await, 457 Err(Error::ProjectionRevisionConflict) 458 ); 459 let before = backend 460 .configure_projection_cache(legacy.clone(), limits()) 461 .await 462 .unwrap(); 463 assert_eq!(before.legacy_snapshots, 66); 464 let current = || { 465 ProjectionCacheScope::new(legacy.clone(), generation(10), source, Some(generation(10))) 466 .unwrap() 467 }; 468 assert_eq!( 469 backend 470 .collect_obsolete_projection_snapshots(current()) 471 .await, 472 Err(Error::ProjectionRevisionConflict) 473 ); 474 assert_eq!( 475 backend 476 .collect_obsolete_projection_snapshots( 477 ProjectionCacheScope::new( 478 legacy.clone(), 479 generation(9), 480 EventSourceToken::new(source.generation(), 1), 481 Some(generation(10)) 482 ) 483 .unwrap() 484 ) 485 .await, 486 Err(Error::SourceEpochChanged) 487 ); 488 assert_eq!( 489 backend 490 .collect_obsolete_projection_snapshots( 491 ProjectionCacheScope::new(legacy.clone(), generation(9), source, None).unwrap() 492 ) 493 .await, 494 Err(Error::ProjectionRevisionConflict) 495 ); 496 backend 497 .put_projection_batch( 498 ProjectionDocumentBatch::new( 499 ProjectionWriteScope::new( 500 legacy.clone(), 501 generation(11), 502 source, 503 Some(generation(10)), 504 ) 505 .unwrap(), 506 vec![ProjectionDocument::new("staging".into(), vec![11]).unwrap()], 507 ) 508 .unwrap(), 509 ) 510 .await 511 .unwrap(); 512 assert_eq!( 513 backend 514 .collect_obsolete_projection_snapshots( 515 ProjectionCacheScope::new( 516 legacy.clone(), 517 generation(11), 518 source, 519 Some(generation(10)) 520 ) 521 .unwrap() 522 ) 523 .await, 524 Err(Error::ProjectionRevisionConflict) 525 ); 526 for (removed, more) in [(64, true), (1, false)] { 527 let result = backend 528 .collect_obsolete_projection_snapshots(old()) 529 .await 530 .unwrap(); 531 assert_eq!((result.removed_rows, result.more), (removed, more)); 532 assert_eq!(result.removed_value_bytes, removed * 2); 533 } 534 let invalidated = backend 535 .invalidate( 536 ProjectionInvalidation::new( 537 legacy.clone(), 538 generation(10), 539 generation(12), 540 InvalidationReason::ProjectionGenerationChanged, 541 2, 542 ) 543 .unwrap(), 544 ) 545 .await 546 .unwrap(); 547 assert_eq!(invalidated.health(), ProjectionHealth::Invalidated); 548 let result = backend 549 .collect_obsolete_projection_snapshots(current()) 550 .await 551 .unwrap(); 552 assert_eq!(result.removed_rows, 1); 553 assert_eq!(result.removed_value_bytes, 2); 554 assert!(!result.more); 555 assert_eq!(result.usage.legacy_snapshots, 0); 556 assert_eq!(result.usage.protected_generations, 2); 557 assert_eq!(backend.source_token().await.unwrap(), source); 558 assert_eq!( 559 ProjectionStore::status(backend, legacy.clone()) 560 .await 561 .unwrap(), 562 Some(invalidated) 563 ); 564 for target in [9, 10] { 565 assert_eq!( 566 backend 567 .projection_document(legacy.clone(), generation(target), "custody".into()) 568 .await 569 .unwrap() 570 .unwrap() 571 .value(), 572 &[target; 16] 573 ); 574 } 575 assert_eq!( 576 backend 577 .projection_document(legacy.clone(), generation(11), "staging".into()) 578 .await 579 .unwrap() 580 .unwrap() 581 .value(), 582 &[11] 583 ); 584 } 585 store.close().await.unwrap(); 586 let readonly = open(&temp, OpenMode::ReadOnly).await; 587 let source = radroots_storage::EventStore::source_token(&readonly) 588 .await 589 .unwrap(); 590 assert_eq!( 591 readonly 592 .collect_obsolete_projection_snapshots( 593 ProjectionCacheScope::new( 594 legacy.clone(), 595 generation(9), 596 source, 597 Some(generation(10)) 598 ) 599 .unwrap() 600 ) 601 .await, 602 Err(Error::BackendUnavailable) 603 ); 604 assert_eq!( 605 readonly 606 .projection_document(legacy, generation(9), "custody".into()) 607 .await 608 .unwrap() 609 .unwrap() 610 .value(), 611 &[9; 16] 612 ); 613 readonly.close().await.unwrap(); 614 }