lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

cache_tests.rs (54594B)


      1 use super::*;
      2 use crate::{OpenMode, OpenOptions, Paths};
      3 use radroots_storage::{
      4     EventStore, ProjectionStore,
      5     event::{EventSourceToken, SourceGeneration},
      6     memory::MemoryStorage,
      7     projection::{
      8         ProjectionCheckpoint, ProjectionDocument, ProjectionGeneration, ProjectionSnapshot,
      9         batch::{ProjectionDocumentBatch, ProjectionWriteScope},
     10     },
     11 };
     12 use tempfile::TempDir;
     13 fn id() -> ProjectionId {
     14     ProjectionId::parse("cache_fixture").unwrap()
     15 }
     16 fn generation(value: u8) -> ProjectionGeneration {
     17     ProjectionGeneration::new([value; 32]).unwrap()
     18 }
     19 fn limits() -> ProjectionCacheLimits {
     20     ProjectionCacheLimits::new(67_108_864, 2, 32, 1_800_000).unwrap()
     21 }
     22 fn batch(
     23     source: EventSourceToken,
     24     active: Option<ProjectionGeneration>,
     25     target: u8,
     26     keys: &[&str],
     27 ) -> ProjectionDocumentBatch {
     28     ProjectionDocumentBatch::new(
     29         ProjectionWriteScope::new(id(), generation(target), source, active).unwrap(),
     30         keys.iter()
     31             .map(|key| ProjectionDocument::new((*key).into(), vec![7]).unwrap())
     32             .collect(),
     33     )
     34     .unwrap()
     35 }
     36 fn scope(
     37     source: EventSourceToken,
     38     active: Option<ProjectionGeneration>,
     39     target: u8,
     40 ) -> ProjectionCacheScope {
     41     ProjectionCacheScope::new(id(), generation(target), source, active).unwrap()
     42 }
     43 fn checkpoint(target: u8, count: u64) -> ProjectionCheckpoint {
     44     ProjectionCheckpoint::new(id(), generation(target), None, count, 1).unwrap()
     45 }
     46 async fn open(temp: &TempDir) -> SqliteStorage {
     47     SqliteStorage::open(
     48         OpenOptions::new(
     49             Paths::from_directory(temp.path()).unwrap(),
     50             OpenMode::Create,
     51         )
     52         .with_source_generation(SourceGeneration::new([9; 32]).unwrap(), 9)
     53         .unwrap(),
     54     )
     55     .await
     56     .unwrap()
     57 }
     58 
     59 #[tokio::test]
     60 async fn projection_cache_protection_and_retirement_match_memory_with_bounded_collection() {
     61     let temp = TempDir::new().unwrap();
     62     let store = open(&temp).await;
     63     let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
     64     let source = store.source_token().await.unwrap();
     65     for backend in [&memory as &dyn ProjectionStore, &store] {
     66         assert_eq!(
     67             backend
     68                 .configure_projection_cache(id(), limits())
     69                 .await
     70                 .unwrap(),
     71             ProjectionCacheUsage::default()
     72         );
     73         backend
     74             .promote_projection_batch(batch(source, None, 1, &["active"]), checkpoint(1, 1))
     75             .await
     76             .unwrap();
     77         backend
     78             .put_projection_batch(batch(source, Some(generation(1)), 2, &["staging"]))
     79             .await
     80             .unwrap();
     81         assert_eq!(
     82             backend
     83                 .configure_projection_cache(id(), limits())
     84                 .await
     85                 .unwrap(),
     86             ProjectionCacheUsage {
     87                 value_bytes: 2,
     88                 protected_generations: 2,
     89                 legacy_snapshots: 0
     90             }
     91         );
     92         assert_eq!(
     93             backend
     94                 .retire_projection_generation(scope(source, Some(generation(1)), 1))
     95                 .await,
     96             Err(Error::ProjectionRevisionConflict)
     97         );
     98         assert_eq!(
     99             backend
    100                 .collect_projection_cache(scope(source, Some(generation(1)), 1))
    101                 .await
    102                 .unwrap()
    103                 .removed_rows,
    104             0
    105         );
    106         assert_eq!(
    107             backend
    108                 .collect_projection_cache(scope(source, Some(generation(1)), 2))
    109                 .await
    110                 .unwrap()
    111                 .removed_rows,
    112             0
    113         );
    114         backend
    115             .retire_projection_generation(scope(source, Some(generation(1)), 2))
    116             .await
    117             .unwrap();
    118         let collected = backend
    119             .collect_projection_cache(scope(source, Some(generation(1)), 2))
    120             .await
    121             .unwrap();
    122         assert_eq!(collected.removed_rows, 2);
    123         assert_eq!(collected.removed_value_bytes, 1);
    124         assert!(!collected.more);
    125         assert_eq!(
    126             collected.usage,
    127             ProjectionCacheUsage {
    128                 value_bytes: 1,
    129                 protected_generations: 1,
    130                 legacy_snapshots: 0
    131             }
    132         );
    133         assert_eq!(
    134             backend
    135                 .collect_projection_cache(scope(
    136                     EventSourceToken::new(source.generation(), 1),
    137                     Some(generation(1)),
    138                     2
    139                 ))
    140                 .await,
    141             Err(Error::SourceEpochChanged)
    142         );
    143         assert_eq!(
    144             backend
    145                 .collect_projection_cache(scope(source, None, 2))
    146                 .await,
    147             Err(Error::ProjectionRevisionConflict)
    148         );
    149         assert_eq!(
    150             backend
    151                 .put_projection_document(
    152                     id(),
    153                     generation(9),
    154                     ProjectionDocument::new("legacy".into(), vec![7]).unwrap()
    155                 )
    156                 .await,
    157             Err(Error::ProjectionRevisionConflict)
    158         );
    159         let keys: Vec<_> = (0..64).map(|number| format!("key.{number:02}")).collect();
    160         let keys: Vec<_> = keys.iter().map(String::as_str).collect();
    161         backend
    162             .put_projection_batch(batch(source, Some(generation(1)), 3, &keys))
    163             .await
    164             .unwrap();
    165         backend
    166             .put_projection_batch(batch(source, Some(generation(1)), 3, &["key.64"]))
    167             .await
    168             .unwrap();
    169         backend
    170             .retire_projection_generation(scope(source, Some(generation(1)), 3))
    171             .await
    172             .unwrap();
    173         let first = backend
    174             .collect_projection_cache(scope(source, Some(generation(1)), 3))
    175             .await
    176             .unwrap();
    177         assert_eq!(first.removed_rows, 64);
    178         assert!(first.more);
    179         let last = backend
    180             .collect_projection_cache(scope(source, Some(generation(1)), 3))
    181             .await
    182             .unwrap();
    183         assert_eq!(last.removed_rows, 2);
    184         assert!(!last.more);
    185         assert_eq!(last.usage.value_bytes, 1);
    186     }
    187     store.close().await.unwrap();
    188 }
    189 
    190 fn value_batch(
    191     source: EventSourceToken,
    192     active: Option<ProjectionGeneration>,
    193     target: u8,
    194     key: &str,
    195     size: usize,
    196 ) -> ProjectionDocumentBatch {
    197     ProjectionDocumentBatch::new(
    198         ProjectionWriteScope::new(id(), generation(target), source, active).unwrap(),
    199         vec![ProjectionDocument::new(key.into(), vec![7; size]).unwrap()],
    200     )
    201     .unwrap()
    202 }
    203 fn query(target: u8) -> radroots_storage::projection::document_query::ProjectionDocumentQuery {
    204     use radroots_storage::projection::document_query::{
    205         ProjectionDocumentGenerations, ProjectionDocumentQuery,
    206     };
    207     ProjectionDocumentQuery::new(
    208         id(),
    209         ProjectionDocumentGenerations::Exact(generation(target)),
    210         64,
    211     )
    212     .unwrap()
    213 }
    214 fn snapshot(number: u8, target: u8, created: u64, size: usize) -> ProjectionSnapshot {
    215     ProjectionSnapshot::new(
    216         id(),
    217         [number; 32],
    218         generation(target),
    219         created,
    220         vec![8; size],
    221     )
    222     .unwrap()
    223 }
    224 
    225 #[tokio::test]
    226 async fn projection_cache_actual_64mib_includes_legacy_active_and_staging_and_rolls_back_overflow()
    227 {
    228     let temp = TempDir::new().unwrap();
    229     let store = open(&temp).await;
    230     let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
    231     let source = store.source_token().await.unwrap();
    232     for backend in [&memory as &dyn ProjectionStore, &store] {
    233         backend
    234             .put_projection_snapshot(snapshot(1, 9, 1, 16_777_216))
    235             .await
    236             .unwrap();
    237         backend
    238             .configure_projection_cache(id(), limits())
    239             .await
    240             .unwrap();
    241         for number in 0..4 {
    242             backend
    243                 .put_projection_batch(value_batch(
    244                     source,
    245                     None,
    246                     1,
    247                     &format!("active.{number}"),
    248                     4_194_304,
    249                 ))
    250                 .await
    251                 .unwrap();
    252         }
    253         backend
    254             .promote_projection_batch(batch(source, None, 1, &[]), checkpoint(1, 4))
    255             .await
    256             .unwrap();
    257         for number in 0..8 {
    258             backend
    259                 .put_projection_batch(value_batch(
    260                     source,
    261                     Some(generation(1)),
    262                     2,
    263                     &format!("staging.{number}"),
    264                     4_194_304,
    265                 ))
    266                 .await
    267                 .unwrap();
    268         }
    269         let full = backend
    270             .configure_projection_cache(id(), limits())
    271             .await
    272             .unwrap();
    273         assert_eq!(
    274             full,
    275             ProjectionCacheUsage {
    276                 value_bytes: 67_108_864,
    277                 protected_generations: 2,
    278                 legacy_snapshots: 1
    279             }
    280         );
    281         assert_eq!(
    282             backend
    283                 .put_projection_batch(value_batch(source, Some(generation(1)), 2, "one_over", 1))
    284                 .await,
    285             Err(Error::SpaceInsufficient)
    286         );
    287         assert_eq!(
    288             backend
    289                 .configure_projection_cache(id(), limits())
    290                 .await
    291                 .unwrap(),
    292             full
    293         );
    294         assert_eq!(
    295             backend
    296                 .query_projection_documents(query(2))
    297                 .await
    298                 .unwrap()
    299                 .records()
    300                 .len(),
    301             4
    302         ); // existing page ceiling selects four4MiB values
    303         backend
    304             .collect_projection_cache(scope(source, Some(generation(1)), 9))
    305             .await
    306             .unwrap();
    307         backend
    308             .promote_projection_batch(batch(source, Some(generation(1)), 2, &[]), checkpoint(2, 8))
    309             .await
    310             .unwrap();
    311         let collected = backend
    312             .collect_projection_cache(scope(source, Some(generation(2)), 1))
    313             .await
    314             .unwrap();
    315         assert_eq!(collected.removed_value_bytes, 16_777_216);
    316         assert_eq!(
    317             collected.usage,
    318             ProjectionCacheUsage {
    319                 value_bytes: 33_554_432,
    320                 protected_generations: 1,
    321                 legacy_snapshots: 0
    322             }
    323         );
    324     }
    325     let mut transaction = store.pool().begin().await.unwrap();
    326     let logical = usage(&mut transaction).await.unwrap();
    327     transaction.rollback().await.unwrap();
    328     let page_count: i64 = sqlx::query_scalar("PRAGMA page_count")
    329         .fetch_one(store.pool())
    330         .await
    331         .unwrap();
    332     let page_size: i64 = sqlx::query_scalar("PRAGMA page_size")
    333         .fetch_one(store.pool())
    334         .await
    335         .unwrap();
    336     let wal = temp.path().join("runtime.sqlite-wal");
    337     assert!(page_count * page_size > logical.value_bytes as i64);
    338     assert!(std::fs::metadata(wal).unwrap().len() > 0);
    339     store.close().await.unwrap();
    340 }
    341 
    342 #[tokio::test]
    343 async fn projection_cache_quota_and_third_protected_wal_races_preserve_transactional_counts() {
    344     let temp = TempDir::new().unwrap();
    345     let store = open(&temp).await;
    346     let source = store.source_token().await.unwrap();
    347     let small = ProjectionCacheLimits::new(3, 2, 32, 1_800_000).unwrap();
    348     store.configure_projection_cache(id(), small).await.unwrap();
    349     store
    350         .promote_projection_batch(batch(source, None, 1, &["active"]), checkpoint(1, 1))
    351         .await
    352         .unwrap();
    353     let (left, right) = tokio::join!(
    354         store.put_projection_batch(value_batch(source, Some(generation(1)), 2, "left", 2)),
    355         store.put_projection_batch(value_batch(source, Some(generation(1)), 2, "right", 2))
    356     );
    357     assert!(matches!(
    358         (&left, &right),
    359         (Ok(()), Err(Error::SpaceInsufficient)) | (Err(Error::SpaceInsufficient), Ok(()))
    360     ));
    361     assert_eq!(
    362         store.configure_projection_cache(id(), small).await.unwrap(),
    363         ProjectionCacheUsage {
    364             value_bytes: 3,
    365             protected_generations: 2,
    366             legacy_snapshots: 0
    367         }
    368     );
    369     assert_eq!(
    370         store
    371             .query_projection_documents(query(2))
    372             .await
    373             .unwrap()
    374             .records()
    375             .len(),
    376         1
    377     );
    378     store
    379         .retire_projection_generation(scope(source, Some(generation(1)), 2))
    380         .await
    381         .unwrap();
    382     store
    383         .collect_projection_cache(scope(source, Some(generation(1)), 2))
    384         .await
    385         .unwrap();
    386     let other = ProjectionId::parse("other_cache").unwrap();
    387     store
    388         .configure_projection_cache(other.clone(), small)
    389         .await
    390         .unwrap();
    391     let second = ProjectionDocumentBatch::new(
    392         ProjectionWriteScope::new(other.clone(), generation(3), source, None).unwrap(),
    393         vec![ProjectionDocument::new("other".into(), vec![7]).unwrap()],
    394     )
    395     .unwrap();
    396     let (left, right) = tokio::join!(
    397         store.put_projection_batch(batch(source, Some(generation(1)), 2, &["new"])),
    398         store.put_projection_batch(second)
    399     );
    400     assert!(matches!(
    401         (&left, &right),
    402         (Ok(()), Err(Error::SpaceInsufficient)) | (Err(Error::SpaceInsufficient), Ok(()))
    403     ));
    404     assert_eq!(
    405         store
    406             .configure_projection_cache(id(), small)
    407             .await
    408             .unwrap()
    409             .protected_generations,
    410         2
    411     );
    412     store.close().await.unwrap();
    413 }
    414 
    415 #[tokio::test]
    416 async fn projection_cache_legacy_32_33_pressure_no_new_snapshots_and_no_sliding_replay() {
    417     let temp = TempDir::new().unwrap();
    418     let store = open(&temp).await;
    419     let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
    420     let source = store.source_token().await.unwrap();
    421     let created = now().unwrap();
    422     for backend in [&memory as &dyn ProjectionStore, &store] {
    423         backend.checkpoint(checkpoint(1, 0)).await.unwrap();
    424         for number in 1..=33 {
    425             backend
    426                 .put_projection_snapshot(snapshot(number, 1, created, 1))
    427                 .await
    428                 .unwrap();
    429         }
    430         assert_eq!(
    431             backend
    432                 .configure_projection_cache(id(), limits())
    433                 .await
    434                 .unwrap()
    435                 .legacy_snapshots,
    436             33
    437         );
    438         let replay = snapshot(2, 1, created, 1);
    439         backend
    440             .put_projection_snapshot(replay.clone())
    441             .await
    442             .unwrap();
    443         assert_eq!(
    444             backend.projection_snapshot(id(), [2; 32]).await.unwrap(),
    445             Some(replay)
    446         );
    447         assert_eq!(
    448             backend
    449                 .put_projection_snapshot(snapshot(34, 1, created, 1))
    450                 .await,
    451             Err(Error::ProjectionRevisionConflict)
    452         );
    453         assert_eq!(
    454             backend
    455                 .put_projection_snapshot(snapshot(2, 1, created + 1, 1))
    456                 .await,
    457             Err(Error::CorruptProjectionDocument)
    458         );
    459         let result = backend
    460             .collect_projection_cache(scope(source, Some(generation(1)), 1))
    461             .await
    462             .unwrap();
    463         assert_eq!(result.removed_rows, 1);
    464         assert!(!result.more);
    465         assert_eq!(result.usage.legacy_snapshots, 32);
    466         let foreign = ProjectionId::parse("legacy_other").unwrap();
    467         let extra =
    468             ProjectionSnapshot::new(foreign, [1; 32], generation(1), created, vec![8]).unwrap();
    469         assert_eq!(
    470             backend.put_projection_snapshot(extra).await,
    471             Err(Error::SpaceInsufficient)
    472         );
    473         assert_eq!(
    474             backend
    475                 .put_projection_snapshot(snapshot(1, 1, created, 1))
    476                 .await,
    477             Err(Error::ProjectionRevisionConflict)
    478         );
    479         assert_eq!(
    480             backend
    481                 .configure_projection_cache(
    482                     id(),
    483                     ProjectionCacheLimits::new(67_108_865, 2, 32, 1_800_000).unwrap()
    484                 )
    485                 .await,
    486             Err(Error::ProjectionRevisionConflict)
    487         );
    488     }
    489     store.close().await.unwrap();
    490 }
    491 
    492 #[tokio::test]
    493 async fn projection_cache_stale_and_invalidated_active_retirement_allows_successor_without_canonical_loss()
    494  {
    495     use radroots_storage::projection::{InvalidationReason, ProjectionInvalidation};
    496     let temp = TempDir::new().unwrap();
    497     let store = open(&temp).await;
    498     let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
    499     for backend in [&memory as &dyn ProjectionStore, &store] {
    500         let source = EventSourceToken::new(SourceGeneration::new([9; 32]).unwrap(), 0);
    501         backend
    502             .configure_projection_cache(id(), limits())
    503             .await
    504             .unwrap();
    505         backend
    506             .promote_projection_batch(batch(source, None, 1, &["active"]), checkpoint(1, 1))
    507             .await
    508             .unwrap();
    509         backend
    510             .invalidate(
    511                 ProjectionInvalidation::new(
    512                     id(),
    513                     generation(1),
    514                     generation(2),
    515                     InvalidationReason::OperatorRequested,
    516                     1,
    517                 )
    518                 .unwrap(),
    519             )
    520             .await
    521             .unwrap();
    522         let retired = backend
    523             .retire_projection_generation(scope(source, Some(generation(1)), 1))
    524             .await
    525             .unwrap();
    526         assert_eq!(retired.protected_generations, 0);
    527         assert_eq!(backend.status(id()).await.unwrap(), None);
    528         backend
    529             .promote_projection_batch(batch(source, None, 2, &["successor"]), checkpoint(2, 1))
    530             .await
    531             .unwrap();
    532         backend
    533             .collect_projection_cache(scope(source, Some(generation(2)), 1))
    534             .await
    535             .unwrap();
    536         assert_eq!(
    537             backend
    538                 .query_projection_documents_at(query(2), source, generation(2))
    539                 .await
    540                 .unwrap()
    541                 .records()
    542                 .len(),
    543             1
    544         );
    545     }
    546     sqlx::query("UPDATE radroots_runtime_source_generations SET mutation_epoch=mutation_epoch+1 WHERE state='active'").execute(store.pool()).await.unwrap();
    547     let current = store.source_token().await.unwrap();
    548     assert_eq!(
    549         store
    550             .retire_projection_generation(scope(
    551                 EventSourceToken::new(current.generation(), 0),
    552                 Some(generation(2)),
    553                 2
    554             ))
    555             .await,
    556         Err(Error::SourceEpochChanged)
    557     );
    558     store
    559         .retire_projection_generation(scope(current, Some(generation(2)), 2))
    560         .await
    561         .unwrap();
    562     store
    563         .promote_projection_batch(batch(current, None, 3, &["fresh"]), checkpoint(3, 1))
    564         .await
    565         .unwrap();
    566     store
    567         .collect_projection_cache(scope(current, Some(generation(3)), 2))
    568         .await
    569         .unwrap();
    570     assert_eq!(store.source_token().await.unwrap(), current);
    571     store.close().await.unwrap();
    572 }
    573 
    574 #[tokio::test]
    575 async fn projection_cache_gc_cancellation_rollback_read_race_reopen_and_readonly_preserve_active() {
    576     let temp = TempDir::new().unwrap();
    577     let store = open(&temp).await;
    578     let source = store.source_token().await.unwrap();
    579     store
    580         .configure_projection_cache(id(), limits())
    581         .await
    582         .unwrap();
    583     store
    584         .promote_projection_batch(batch(source, None, 1, &["active"]), checkpoint(1, 1))
    585         .await
    586         .unwrap();
    587     store
    588         .put_projection_batch(batch(
    589             source,
    590             Some(generation(1)),
    591             2,
    592             &["obsolete.a", "obsolete.b"],
    593         ))
    594         .await
    595         .unwrap();
    596     store
    597         .retire_projection_generation(scope(source, Some(generation(1)), 2))
    598         .await
    599         .unwrap();
    600     let mut connections = Vec::new();
    601     for _ in 0..4 {
    602         connections.push(store.pool().acquire().await.unwrap());
    603     }
    604     let mut pending = store.collect_projection_cache(scope(source, Some(generation(1)), 2));
    605     assert!(matches!(
    606         pending
    607             .as_mut()
    608             .poll(&mut std::task::Context::from_waker(std::task::Waker::noop())),
    609         std::task::Poll::Pending
    610     ));
    611     drop(pending);
    612     drop(connections);
    613     assert_eq!(
    614         store
    615             .query_projection_documents(query(2))
    616             .await
    617             .unwrap()
    618             .records()
    619             .len(),
    620         2
    621     );
    622     sqlx::raw_sql("CREATE TRIGGER fixture_cache_delete BEFORE DELETE ON radroots_runtime_projection_documents WHEN OLD.document_key='obsolete.b' BEGIN SELECT RAISE(ABORT,'fixture'); END;").execute(store.pool()).await.unwrap();
    623     let before = store
    624         .configure_projection_cache(id(), limits())
    625         .await
    626         .unwrap();
    627     assert!(
    628         store
    629             .collect_projection_cache(scope(source, Some(generation(1)), 2))
    630             .await
    631             .is_err()
    632     );
    633     assert_eq!(
    634         store
    635             .configure_projection_cache(id(), limits())
    636             .await
    637             .unwrap(),
    638         before
    639     );
    640     assert_eq!(
    641         store
    642             .query_projection_documents(query(2))
    643             .await
    644             .unwrap()
    645             .records()
    646             .len(),
    647         2
    648     );
    649     sqlx::raw_sql("DROP TRIGGER fixture_cache_delete")
    650         .execute(store.pool())
    651         .await
    652         .unwrap();
    653     let (read, collected) = tokio::join!(
    654         store.query_projection_documents_at(query(1), source, generation(1)),
    655         store.collect_projection_cache(scope(source, Some(generation(1)), 2))
    656     );
    657     assert_eq!(read.unwrap().records().len(), 1);
    658     assert_eq!(collected.unwrap().removed_rows, 3);
    659     store.close().await.unwrap();
    660     let reopened = SqliteStorage::open(OpenOptions::new(
    661         Paths::from_directory(temp.path()).unwrap(),
    662         OpenMode::ReadWriteExisting,
    663     ))
    664     .await
    665     .unwrap();
    666     assert_eq!(
    667         reopened
    668             .configure_projection_cache(id(), limits())
    669             .await
    670             .unwrap(),
    671         ProjectionCacheUsage {
    672             value_bytes: 1,
    673             protected_generations: 1,
    674             legacy_snapshots: 0
    675         }
    676     );
    677     reopened.close().await.unwrap();
    678     let readonly = SqliteStorage::open(OpenOptions::new(
    679         Paths::from_directory(temp.path()).unwrap(),
    680         OpenMode::ReadOnly,
    681     ))
    682     .await
    683     .unwrap();
    684     assert_eq!(
    685         readonly
    686             .collect_projection_cache(scope(source, Some(generation(1)), 2))
    687             .await,
    688         Err(Error::BackendUnavailable)
    689     );
    690     assert_eq!(
    691         readonly
    692             .retire_projection_generation(scope(source, Some(generation(1)), 1))
    693             .await,
    694         Err(Error::BackendUnavailable)
    695     );
    696     assert_eq!(
    697         readonly.configure_projection_cache(id(), limits()).await,
    698         Err(Error::BackendUnavailable)
    699     );
    700     assert_eq!(
    701         readonly
    702             .query_projection_documents_at(query(1), source, generation(1))
    703             .await
    704             .unwrap()
    705             .records()
    706             .len(),
    707         1
    708     );
    709     readonly.close().await.unwrap();
    710 }
    711 
    712 #[tokio::test]
    713 async fn projection_cache_owner_time_highwater_expires_future_and_rollback_without_resurrection() {
    714     let temp = TempDir::new().unwrap();
    715     let store = open(&temp).await;
    716     let created = now().unwrap();
    717     let source = store.source_token().await.unwrap();
    718     for (number, time) in [(1, 1), (2, created), (3, created + 3_600_000)] {
    719         store
    720             .put_projection_snapshot(snapshot(number, 1, time, 1))
    721             .await
    722             .unwrap();
    723     }
    724     store.checkpoint(checkpoint(1, 0)).await.unwrap();
    725     store
    726         .configure_projection_cache(id(), limits())
    727         .await
    728         .unwrap();
    729     assert!(
    730         store
    731             .projection_snapshot(id(), [2; 32])
    732             .await
    733             .unwrap()
    734             .is_some()
    735     );
    736     assert_eq!(
    737         store.projection_snapshot(id(), [3; 32]).await.unwrap(),
    738         None
    739     );
    740     let first = store
    741         .collect_projection_cache(scope(source, Some(generation(1)), 1))
    742         .await
    743         .unwrap();
    744     assert_eq!(first.removed_rows, 2);
    745     assert!(!first.more);
    746     let mut transaction = store.pool().begin_with("BEGIN IMMEDIATE").await.unwrap();
    747     assert_eq!(
    748         time(&mut transaction, created + 1_800_000).await.unwrap(),
    749         created + 1_800_000
    750     );
    751     assert_eq!(
    752         time(&mut transaction, created - 1).await.unwrap(),
    753         created + 1_800_000
    754     );
    755     transaction.commit().await.unwrap();
    756     assert_eq!(
    757         store.projection_snapshot(id(), [2; 32]).await.unwrap(),
    758         None
    759     );
    760     store
    761         .put_projection_snapshot(snapshot(2, 1, created, 1))
    762         .await
    763         .unwrap(); // exact replay cannot renew time
    764     assert_eq!(
    765         store.projection_snapshot(id(), [2; 32]).await.unwrap(),
    766         None
    767     );
    768     store.close().await.unwrap();
    769     let reopened = SqliteStorage::open(OpenOptions::new(
    770         Paths::from_directory(temp.path()).unwrap(),
    771         OpenMode::ReadWriteExisting,
    772     ))
    773     .await
    774     .unwrap();
    775     assert_eq!(
    776         reopened.projection_snapshot(id(), [2; 32]).await.unwrap(),
    777         None
    778     );
    779     assert_eq!(
    780         reopened
    781             .collect_projection_cache(scope(source, Some(generation(1)), 1))
    782             .await
    783             .unwrap()
    784             .removed_rows,
    785         1
    786     );
    787     assert_eq!(
    788         reopened
    789             .put_projection_snapshot(snapshot(2, 1, created, 1))
    790             .await,
    791         Err(Error::ProjectionRevisionConflict)
    792     );
    793     reopened.close().await.unwrap();
    794 }
    795 
    796 #[tokio::test]
    797 async fn projection_cache_snapshot_scalar_length_and_digest_admission_precedes_payload_hydration() {
    798     let temp = TempDir::new().unwrap();
    799     let store = open(&temp).await;
    800     for (number, size) in [(1, 0), (2, 16_777_217)] {
    801         let mut connection = store.pool().acquire().await.unwrap();
    802         sqlx::raw_sql("PRAGMA ignore_check_constraints=ON")
    803             .execute(&mut *connection)
    804             .await
    805             .unwrap();
    806         sqlx::query("INSERT INTO radroots_runtime_projection_snapshots(projection_id,snapshot_id,generation,created_at_unix_ms,value,value_sha256) VALUES(?,?,?,1,zeroblob(?),zeroblob(32))").bind(id().as_str()).bind([number;32].as_slice()).bind(generation(1).as_bytes().as_slice()).bind(size).execute(&mut *connection).await.unwrap();
    807         sqlx::raw_sql("PRAGMA ignore_check_constraints=OFF")
    808             .execute(&mut *connection)
    809             .await
    810             .unwrap();
    811         drop(connection);
    812         assert_eq!(
    813             store.projection_snapshot(id(), [number; 32]).await,
    814             Err(Error::CorruptProjectionDocument)
    815         );
    816     }
    817     sqlx::query("INSERT INTO radroots_runtime_projection_snapshots(projection_id,snapshot_id,generation,created_at_unix_ms,value,value_sha256) VALUES(?,?,?,1,x'01',zeroblob(32))").bind(id().as_str()).bind([3;32].as_slice()).bind(generation(1).as_bytes().as_slice()).execute(store.pool()).await.unwrap();
    818     assert_eq!(
    819         store.projection_snapshot(id(), [3; 32]).await,
    820         Err(Error::CorruptProjectionDocument)
    821     );
    822     store
    823         .configure_projection_cache(id(), limits())
    824         .await
    825         .unwrap();
    826     assert_eq!(
    827         store.projection_snapshot(id(), [2; 32]).await.unwrap(),
    828         None
    829     ); // expired oversized row is rejected before value fetch
    830     store.close().await.unwrap();
    831 }
    832 
    833 fn review_admission(verified: bool) -> radroots_storage::event::EventAdmission {
    834     use radroots_event::{SignedEvent, admission::RawEvent, wire::Nip01EventWire};
    835     use radroots_storage::event::EventAdmission;
    836     use radroots_transport::{
    837         Target, TransportId,
    838         source::{EventProvenance, ObservedEvent},
    839     };
    840     struct FixtureVerifier;
    841     impl radroots_event::admission::SignatureVerifier for FixtureVerifier {
    842         fn verify_signature(
    843             &self,
    844             _event: &radroots_event::Event,
    845         ) -> Result<(), radroots_event::Error> {
    846             Ok(())
    847         }
    848     }
    849     let mut wire = Nip01EventWire {
    850         id: "0".repeat(64),
    851         pubkey: "585591529da0bab31b3b1b1f986611cf5f435dca84f978c89ee8a40cca7103df".into(),
    852         created_at: 17,
    853         kind: 0,
    854         tags: vec![],
    855         content: r#"{"name":"review"}"#.into(),
    856         sig: "42".repeat(64),
    857         extra: Default::default(),
    858     };
    859     wire.id = wire.computed_event_id().unwrap().to_hex();
    860     let raw = serde_json::to_string(&wire).unwrap();
    861     let event = SignedEvent::from_wire_verified_id(wire, raw).unwrap();
    862     let target = Target::new(TransportId::NOSTR, "wss://fixture.example").unwrap();
    863     let observed = ObservedEvent::new(
    864         event.clone(),
    865         EventProvenance::new(TransportId::NOSTR, target.fingerprint().clone(), 1).unwrap(),
    866     );
    867     if verified {
    868         let verified = RawEvent::new(event.envelope().clone())
    869             .verify_id()
    870             .unwrap()
    871             .verify_signature(&FixtureVerifier)
    872             .unwrap();
    873         EventAdmission::verified(observed, verified).unwrap()
    874     } else {
    875         EventAdmission::raw(observed)
    876     }
    877 }
    878 
    879 #[tokio::test]
    880 async fn projection_cache_collection_preserves_canonical_authored_intent_receipt_custody_and_foreign_projection()
    881  {
    882     use radroots_storage::journal::OperationId;
    883     use radroots_storage::{
    884         Storage,
    885         atomic::{
    886             AtomicCommit, AtomicCommitDigest, AtomicCommitId, AtomicWorkflow, CommitIngested,
    887         },
    888         authored_draft::{AuthoredDraft, AuthoredDraftId, AuthoredDraftStage},
    889         journal::{IdempotencyDigest, IdempotencyKey, OperationInstanceId, PrepareOperation},
    890         private_artifact::{
    891             ArtifactCommitment, ArtifactKind, ArtifactSchemaId, DurableSecretReference,
    892             PrivateArtifactId, PrivateArtifactMetadata, RetentionPolicy,
    893         },
    894     };
    895     let temp = TempDir::new().unwrap();
    896     let store = open(&temp).await;
    897     let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
    898     for backend in [&memory as &dyn Storage, &store] {
    899         let request = AtomicCommit::new(
    900             AtomicCommitId::new([5; 16]).unwrap(),
    901             AtomicCommitDigest::new([5; 32]),
    902             1,
    903             AtomicWorkflow::Ingested(Box::new(CommitIngested::new(review_admission(false), None))),
    904         )
    905         .unwrap();
    906         let receipt = backend.commit(request.clone()).await.unwrap();
    907         let source = backend.source_token().await.unwrap();
    908         let draft = AuthoredDraft::initial(
    909             AuthoredDraftId::new([1; 16]).unwrap(),
    910             [3; 32],
    911             "fixture.preserved.v1",
    912             b"authored intent".to_vec(),
    913             AuthoredDraftStage::Draft,
    914             None,
    915             1,
    916         )
    917         .unwrap();
    918         backend
    919             .append_authored_draft(draft.clone(), None)
    920             .await
    921             .unwrap();
    922         let intent = PrepareOperation::new(
    923             OperationInstanceId::new([1; 16]).unwrap(),
    924             OperationId::SyncPush,
    925             IdempotencyKey::parse("cache-preserve").unwrap(),
    926             IdempotencyDigest::new([1; 32]),
    927             1,
    928         )
    929         .unwrap();
    930         backend.prepare(intent.clone()).await.unwrap();
    931         let custody = PrivateArtifactMetadata::new(
    932             PrivateArtifactId::new([1; 16]).unwrap(),
    933             ArtifactKind::parse("fixture.custody").unwrap(),
    934             ArtifactSchemaId::parse("fixture.custody.v1").unwrap(),
    935             ArtifactCommitment::new([1; 32]),
    936             1,
    937             DurableSecretReference::new("memory", "fixture", 1).unwrap(),
    938             RetentionPolicy::indefinite(),
    939             1,
    940         )
    941         .unwrap();
    942         backend.put_metadata(custody.clone()).await.unwrap();
    943         let foreign = ProjectionId::parse("settings_and_recovery").unwrap();
    944         backend
    945             .put_projection_document(
    946                 foreign.clone(),
    947                 generation(1),
    948                 ProjectionDocument::new("keep".into(), vec![7]).unwrap(),
    949             )
    950             .await
    951             .unwrap();
    952         backend
    953             .configure_projection_cache(id(), limits())
    954             .await
    955             .unwrap();
    956         backend
    957             .put_projection_batch(batch(source, None, 1, &["retire"]))
    958             .await
    959             .unwrap();
    960         backend
    961             .retire_projection_generation(scope(source, None, 1))
    962             .await
    963             .unwrap();
    964         assert_eq!(
    965             backend
    966                 .collect_projection_cache(
    967                     ProjectionCacheScope::new(foreign.clone(), generation(1), source, None)
    968                         .unwrap()
    969                 )
    970                 .await,
    971             Err(Error::ProjectionRevisionConflict)
    972         );
    973         assert_eq!(
    974             backend
    975                 .collect_projection_cache(scope(source, None, 1))
    976                 .await
    977                 .unwrap()
    978                 .removed_rows,
    979             2
    980         );
    981         assert_eq!(backend.source_token().await.unwrap(), source);
    982         assert_eq!(
    983             backend.receipt(request.commit_id()).await.unwrap(),
    984             Some(receipt)
    985         );
    986         assert_eq!(
    987             backend.authored_draft_head(draft.draft_id()).await.unwrap(),
    988             Some(draft)
    989         );
    990         assert!(
    991             backend
    992                 .by_idempotency_key(OperationId::SyncPush, intent.idempotency_key().clone())
    993                 .await
    994                 .unwrap()
    995                 .is_some()
    996         );
    997         assert_eq!(
    998             backend.metadata(custody.artifact_id()).await.unwrap(),
    999             Some(custody)
   1000         );
   1001         assert_eq!(
   1002             backend
   1003                 .projection_document(foreign, generation(1), "keep".into())
   1004                 .await
   1005                 .unwrap()
   1006                 .unwrap()
   1007                 .value(),
   1008             &[7]
   1009         );
   1010     }
   1011     store.close().await.unwrap();
   1012 }
   1013 
   1014 #[tokio::test]
   1015 async fn projection_cache_63_64_registration_boundary_reports_only_actual_remaining_work() {
   1016     let temp = TempDir::new().unwrap();
   1017     let store = open(&temp).await;
   1018     let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
   1019     let source = store.source_token().await.unwrap();
   1020     for backend in [&memory as &dyn ProjectionStore, &store] {
   1021         backend
   1022             .configure_projection_cache(id(), limits())
   1023             .await
   1024             .unwrap();
   1025         for (target, count) in [(1, 63), (2, 64), (3, 0)] {
   1026             let keys: Vec<_> = (0..count)
   1027                 .map(|number| format!("key.{number:02}"))
   1028                 .collect();
   1029             let keys: Vec<_> = keys.iter().map(String::as_str).collect();
   1030             backend
   1031                 .put_projection_batch(batch(source, None, target, &keys))
   1032                 .await
   1033                 .unwrap();
   1034             backend
   1035                 .retire_projection_generation(scope(source, None, target))
   1036                 .await
   1037                 .unwrap();
   1038             let first = backend
   1039                 .collect_projection_cache(scope(source, None, target))
   1040                 .await
   1041                 .unwrap();
   1042             assert_eq!(first.removed_rows, (count + 1).min(64));
   1043             assert_eq!(first.more, count == 64);
   1044             if first.more {
   1045                 let second = backend
   1046                     .collect_projection_cache(scope(source, None, target))
   1047                     .await
   1048                     .unwrap();
   1049                 assert_eq!(second.removed_rows, 1);
   1050                 assert!(!second.more);
   1051             }
   1052             assert_eq!(
   1053                 backend
   1054                     .configure_projection_cache(id(), limits())
   1055                     .await
   1056                     .unwrap(),
   1057                 ProjectionCacheUsage::default()
   1058             );
   1059         }
   1060     }
   1061     store.close().await.unwrap();
   1062 }
   1063 
   1064 async fn legacy_cache_completion(backend: &dyn ProjectionStore, registered: bool, capacity: u64) {
   1065     use radroots_storage::projection::{
   1066         InvalidationReason, ProjectionCheckpoint, ProjectionHealth, ProjectionInvalidation,
   1067         RawSourceDigest, RebuildTicket, RebuildTicketId, RebuildTransition,
   1068         batch::{ProjectionDocumentBatch, ProjectionWriteScope},
   1069     };
   1070     let id = ProjectionId::parse("legacy_cache_completion").unwrap();
   1071     let source = EventSourceToken::new(SourceGeneration::new([9; 32]).unwrap(), 0);
   1072     let limits = ProjectionCacheLimits::new(67_108_864, capacity, 32, 1_800_000).unwrap();
   1073     let checkpoint =
   1074         |target| ProjectionCheckpoint::new(id.clone(), generation(target), None, 1, 1).unwrap();
   1075     let batch = |target, active| {
   1076         ProjectionDocumentBatch::new(
   1077             ProjectionWriteScope::new(id.clone(), generation(target), source, active).unwrap(),
   1078             vec![ProjectionDocument::new("kept".into(), vec![7]).unwrap()],
   1079         )
   1080         .unwrap()
   1081     };
   1082     let scope = |target, active| {
   1083         ProjectionCacheScope::new(
   1084             id.clone(),
   1085             generation(target),
   1086             source,
   1087             Some(generation(active)),
   1088         )
   1089         .unwrap()
   1090     };
   1091     backend
   1092         .configure_projection_cache(id.clone(), limits)
   1093         .await
   1094         .unwrap();
   1095     if registered {
   1096         backend
   1097             .promote_projection_batch(batch(1, None), checkpoint(1))
   1098             .await
   1099             .unwrap();
   1100     } else {
   1101         backend.checkpoint(checkpoint(1)).await.unwrap();
   1102     }
   1103     backend
   1104         .put_projection_batch(batch(2, Some(generation(1))))
   1105         .await
   1106         .unwrap();
   1107     let invalidation = ProjectionInvalidation::new(
   1108         id.clone(),
   1109         generation(1),
   1110         generation(3),
   1111         InvalidationReason::OperatorRequested,
   1112         10,
   1113     )
   1114     .unwrap();
   1115     backend.invalidate(invalidation.clone()).await.unwrap();
   1116     let ticket = RebuildTicket::requested(
   1117         RebuildTicketId::new([3; 16]).unwrap(),
   1118         invalidation,
   1119         source.generation(),
   1120         None,
   1121         RawSourceDigest::new([1; 32]),
   1122     )
   1123     .unwrap();
   1124     backend.request_rebuild(ticket.clone()).await.unwrap();
   1125     let running = backend
   1126         .transition_rebuild(RebuildTransition::start(
   1127             ticket.ticket_id(),
   1128             ticket.revision(),
   1129             11,
   1130         ))
   1131         .await
   1132         .unwrap();
   1133     let before_status = backend.status(id.clone()).await.unwrap();
   1134     let before_usage = backend
   1135         .configure_projection_cache(id.clone(), limits)
   1136         .await
   1137         .unwrap();
   1138     assert_eq!(before_usage.protected_generations, 2);
   1139     let complete =
   1140         RebuildTransition::complete(running.ticket_id(), running.revision(), 12, checkpoint(3));
   1141     if registered && capacity == 2 {
   1142         assert_eq!(
   1143             backend.transition_rebuild(complete.clone()).await,
   1144             Err(Error::SpaceInsufficient)
   1145         );
   1146         assert_eq!(backend.status(id.clone()).await.unwrap(), before_status);
   1147         assert_eq!(
   1148             backend.rebuild(ticket.ticket_id()).await.unwrap(),
   1149             Some(running.clone())
   1150         );
   1151         assert_eq!(
   1152             backend
   1153                 .configure_projection_cache(id.clone(), limits)
   1154                 .await
   1155                 .unwrap(),
   1156             before_usage
   1157         );
   1158         assert_eq!(
   1159             backend
   1160                 .collect_projection_cache(scope(1, 1))
   1161                 .await
   1162                 .unwrap()
   1163                 .removed_rows,
   1164             0
   1165         );
   1166         assert_eq!(
   1167             backend
   1168                 .collect_projection_cache(scope(2, 1))
   1169                 .await
   1170                 .unwrap()
   1171                 .removed_rows,
   1172             0
   1173         );
   1174         assert_eq!(
   1175             backend
   1176                 .retire_projection_generation(scope(2, 1))
   1177                 .await
   1178                 .unwrap()
   1179                 .protected_generations,
   1180             1
   1181         );
   1182     }
   1183     let completed = backend.transition_rebuild(complete).await.unwrap();
   1184     assert_eq!(
   1185         backend.rebuild(ticket.ticket_id()).await.unwrap(),
   1186         Some(completed)
   1187     );
   1188     let status = backend.status(id.clone()).await.unwrap().unwrap();
   1189     assert_eq!(status.generation(), generation(3));
   1190     assert_eq!(status.health(), ProjectionHealth::Ready);
   1191     assert_eq!(status.active_rebuild(), None);
   1192     assert_eq!(status.checkpoint(), Some(&checkpoint(3)));
   1193     let expected = if registered && capacity == 3 { 3 } else { 2 };
   1194     assert_eq!(
   1195         backend
   1196             .configure_projection_cache(id.clone(), limits)
   1197             .await
   1198             .unwrap()
   1199             .protected_generations,
   1200         expected
   1201     );
   1202     assert_eq!(
   1203         backend
   1204             .collect_projection_cache(scope(3, 3))
   1205             .await
   1206             .unwrap()
   1207             .removed_rows,
   1208         0
   1209     );
   1210     if registered {
   1211         assert_eq!(
   1212             backend
   1213                 .collect_projection_cache(scope(1, 3))
   1214                 .await
   1215                 .unwrap()
   1216                 .removed_rows,
   1217             0
   1218         );
   1219         assert_eq!(
   1220             backend
   1221                 .retire_projection_generation(scope(1, 3))
   1222                 .await
   1223                 .unwrap()
   1224                 .protected_generations,
   1225             expected - 1
   1226         );
   1227         let collected = backend.collect_projection_cache(scope(1, 3)).await.unwrap();
   1228         assert_eq!(collected.removed_rows, 2);
   1229         assert_eq!(collected.removed_value_bytes, 1);
   1230         assert!(!collected.more);
   1231         assert_eq!(collected.usage.protected_generations, expected - 1);
   1232     }
   1233     if !registered || capacity == 3 {
   1234         assert_eq!(
   1235             backend
   1236                 .collect_projection_cache(scope(2, 3))
   1237                 .await
   1238                 .unwrap()
   1239                 .removed_rows,
   1240             0
   1241         );
   1242         backend
   1243             .retire_projection_generation(scope(2, 3))
   1244             .await
   1245             .unwrap();
   1246     }
   1247     let collected = backend.collect_projection_cache(scope(2, 3)).await.unwrap();
   1248     assert_eq!(collected.removed_rows, 2);
   1249     assert_eq!(collected.removed_value_bytes, 1);
   1250     assert!(!collected.more);
   1251     assert_eq!(collected.usage.protected_generations, 1);
   1252     assert_eq!(backend.status(id.clone()).await.unwrap(), Some(status));
   1253     assert_eq!(
   1254         backend
   1255             .collect_projection_cache(scope(3, 3))
   1256             .await
   1257             .unwrap()
   1258             .removed_rows,
   1259         0
   1260     );
   1261 }
   1262 
   1263 #[tokio::test]
   1264 async fn projection_cache_legacy_completion_preserves_registry_capacity_and_atomic_refusal_in_both_providers()
   1265  {
   1266     for (registered, capacity) in [(true, 2), (true, 3), (false, 2)] {
   1267         let temp = TempDir::new().unwrap();
   1268         let store = open(&temp).await;
   1269         let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
   1270         for backend in [&memory as &dyn ProjectionStore, &store] {
   1271             legacy_cache_completion(backend, registered, capacity).await;
   1272         }
   1273         store.close().await.unwrap();
   1274     }
   1275 }
   1276 
   1277 async fn cache_batch_replacement(
   1278     backend: &dyn ProjectionStore,
   1279     quota: u64,
   1280     initial: [usize; 2],
   1281     replacement: [usize; 2],
   1282     reverse: bool,
   1283 ) {
   1284     use radroots_storage::projection::batch::{ProjectionDocumentBatch, ProjectionWriteScope};
   1285     let id = ProjectionId::parse("batch_replacement").unwrap();
   1286     let source = EventSourceToken::new(SourceGeneration::new([9; 32]).unwrap(), 0);
   1287     let batch = |sizes: [usize; 2], target, reverse| {
   1288         let mut documents = vec![
   1289             ProjectionDocument::new("a".into(), vec![1; sizes[0]]).unwrap(),
   1290             ProjectionDocument::new("b".into(), vec![2; sizes[1]]).unwrap(),
   1291         ];
   1292         if reverse {
   1293             documents.reverse();
   1294         }
   1295         ProjectionDocumentBatch::new(
   1296             ProjectionWriteScope::new(id.clone(), generation(target), source, None).unwrap(),
   1297             documents,
   1298         )
   1299         .unwrap()
   1300     };
   1301     backend
   1302         .put_projection_batch(batch(initial, 1, false))
   1303         .await
   1304         .unwrap();
   1305     let limits = ProjectionCacheLimits::new(quota, 2, 32, 1_800_000).unwrap();
   1306     let before = backend
   1307         .configure_projection_cache(id.clone(), limits)
   1308         .await
   1309         .unwrap();
   1310     let status = backend.status(id.clone()).await.unwrap();
   1311     let previous_bytes = initial.iter().sum::<usize>() as u64;
   1312     let next_bytes = replacement.iter().sum::<usize>() as u64;
   1313     let result = backend
   1314         .put_projection_batch(batch(replacement, 1, reverse))
   1315         .await;
   1316     let refused = next_bytes > previous_bytes && next_bytes > quota;
   1317     let expected = if refused {
   1318         assert_eq!(result, Err(Error::SpaceInsufficient));
   1319         assert_eq!(
   1320             backend
   1321                 .configure_projection_cache(id.clone(), limits)
   1322                 .await
   1323                 .unwrap(),
   1324             before
   1325         );
   1326         initial
   1327     } else {
   1328         result.unwrap();
   1329         let current = backend
   1330             .configure_projection_cache(id.clone(), limits)
   1331             .await
   1332             .unwrap();
   1333         assert_eq!(current.value_bytes, next_bytes);
   1334         assert_eq!(current.protected_generations, 1);
   1335         replacement
   1336     };
   1337     assert_eq!(backend.status(id.clone()).await.unwrap(), status);
   1338     for (key, size, value) in [("a", expected[0], 1), ("b", expected[1], 2)] {
   1339         assert_eq!(
   1340             backend
   1341                 .projection_document(id.clone(), generation(1), key.into())
   1342                 .await
   1343                 .unwrap()
   1344                 .unwrap()
   1345                 .value(),
   1346             vec![value; size]
   1347         );
   1348     }
   1349     let current = backend
   1350         .configure_projection_cache(id.clone(), limits)
   1351         .await
   1352         .unwrap();
   1353     let over = quota.saturating_sub(current.value_bytes) + 1;
   1354     let foreign = ProjectionId::parse("direct_writer").unwrap();
   1355     assert_eq!(
   1356         backend
   1357             .put_projection_document(
   1358                 foreign.clone(),
   1359                 generation(9),
   1360                 ProjectionDocument::new("overflow".into(), vec![9; over as usize]).unwrap()
   1361             )
   1362             .await,
   1363         Err(Error::SpaceInsufficient)
   1364     );
   1365     assert_eq!(
   1366         backend
   1367             .projection_document(foreign, generation(9), "overflow".into())
   1368             .await
   1369             .unwrap(),
   1370         None
   1371     );
   1372     assert_eq!(
   1373         backend
   1374             .configure_projection_cache(id.clone(), limits)
   1375             .await
   1376             .unwrap(),
   1377         current
   1378     );
   1379     assert_eq!(
   1380         backend
   1381             .put_projection_batch(
   1382                 ProjectionDocumentBatch::new(
   1383                     ProjectionWriteScope::new(
   1384                         ProjectionId::parse("new_admission").unwrap(),
   1385                         generation(2),
   1386                         source,
   1387                         None
   1388                     )
   1389                     .unwrap(),
   1390                     vec![ProjectionDocument::new("a".into(), vec![7; over as usize]).unwrap()]
   1391                 )
   1392                 .unwrap()
   1393             )
   1394             .await,
   1395         Err(Error::SpaceInsufficient)
   1396     );
   1397     assert_eq!(
   1398         backend
   1399             .configure_projection_cache(id.clone(), limits)
   1400             .await
   1401             .unwrap(),
   1402         current
   1403     );
   1404     assert_eq!(
   1405         backend
   1406             .projection_document(
   1407                 ProjectionId::parse("new_admission").unwrap(),
   1408                 generation(2),
   1409                 "a".into()
   1410             )
   1411             .await
   1412             .unwrap(),
   1413         None
   1414     );
   1415     assert_eq!(
   1416         backend
   1417             .collect_projection_cache(
   1418                 ProjectionCacheScope::new(id.clone(), generation(1), source, None).unwrap()
   1419             )
   1420             .await
   1421             .unwrap()
   1422             .removed_rows,
   1423         0
   1424     );
   1425     backend
   1426         .retire_projection_generation(
   1427             ProjectionCacheScope::new(id.clone(), generation(1), source, None).unwrap(),
   1428         )
   1429         .await
   1430         .unwrap();
   1431     let collected = backend
   1432         .collect_projection_cache(
   1433             ProjectionCacheScope::new(id, generation(1), source, None).unwrap(),
   1434         )
   1435         .await
   1436         .unwrap();
   1437     assert_eq!(collected.removed_rows, 3);
   1438     assert!(!collected.more);
   1439     assert_eq!(collected.usage.value_bytes, 0);
   1440     assert_eq!(collected.usage.protected_generations, 0);
   1441 }
   1442 
   1443 #[tokio::test]
   1444 async fn projection_cache_batch_replacement_has_one_aggregate_policy_in_both_orders_and_providers()
   1445 {
   1446     for (quota, initial, replacement) in [
   1447         (4, [2, 2], [3, 1]),
   1448         (4, [2, 2], [2, 1]),
   1449         (6, [4, 4], [5, 2]),
   1450         (6, [4, 4], [5, 4]),
   1451         (6, [1, 1], [2, 2]),
   1452     ] {
   1453         for reverse in [false, true] {
   1454             let temp = TempDir::new().unwrap();
   1455             let store = open(&temp).await;
   1456             let memory = MemoryStorage::new(SourceGeneration::new([9; 32]).unwrap());
   1457             for backend in [&memory as &dyn ProjectionStore, &store] {
   1458                 cache_batch_replacement(backend, quota, initial, replacement, reverse).await;
   1459             }
   1460             store.close().await.unwrap();
   1461         }
   1462     }
   1463 }
   1464 
   1465 #[tokio::test]
   1466 async fn projection_cache_batch_reservation_faults_roll_back_rows_counters_registration_and_status()
   1467 {
   1468     for corrupt_reservation in [false, true] {
   1469         let temp = TempDir::new().unwrap();
   1470         let store = open(&temp).await;
   1471         let source = store.source_token().await.unwrap();
   1472         let batch = |sizes: [usize; 2]| {
   1473             ProjectionDocumentBatch::new(
   1474                 ProjectionWriteScope::new(id(), generation(1), source, None).unwrap(),
   1475                 vec![
   1476                     ProjectionDocument::new("a".into(), vec![1; sizes[0]]).unwrap(),
   1477                     ProjectionDocument::new("b".into(), vec![2; sizes[1]]).unwrap(),
   1478                 ],
   1479             )
   1480             .unwrap()
   1481         };
   1482         store.put_projection_batch(batch([2, 2])).await.unwrap();
   1483         let limits = ProjectionCacheLimits::new(4, 2, 32, 1_800_000).unwrap();
   1484         let before = store
   1485             .configure_projection_cache(id(), limits)
   1486             .await
   1487             .unwrap();
   1488         let trigger = if corrupt_reservation {
   1489             "CREATE TRIGGER fixture_batch_fault AFTER UPDATE OF value ON radroots_runtime_projection_documents WHEN NEW.document_key='a' BEGIN UPDATE radroots_runtime_projection_cache_budget SET batch_value_ceiling=batch_value_ceiling+1 WHERE singleton=1; END;"
   1490         } else {
   1491             "CREATE TRIGGER fixture_batch_fault BEFORE UPDATE OF value ON radroots_runtime_projection_documents WHEN NEW.document_key='a' BEGIN SELECT RAISE(ABORT,'fixture batch abort'); END;"
   1492         };
   1493         sqlx::raw_sql(trigger).execute(store.pool()).await.unwrap();
   1494         let result = store
   1495             .promote_projection_batch(batch([3, 1]), checkpoint(1, 2))
   1496             .await;
   1497         if corrupt_reservation {
   1498             assert_eq!(result, Err(Error::CorruptProjectionRecord));
   1499         } else {
   1500             assert!(result.is_err());
   1501         }
   1502         assert_eq!(ProjectionStore::status(&store, id()).await.unwrap(), None);
   1503         assert_eq!(
   1504             store
   1505                 .configure_projection_cache(id(), limits)
   1506                 .await
   1507                 .unwrap(),
   1508             before
   1509         );
   1510         for (key, value) in [("a", 1), ("b", 2)] {
   1511             assert_eq!(
   1512                 store
   1513                     .projection_document(id(), generation(1), key.into())
   1514                     .await
   1515                     .unwrap()
   1516                     .unwrap()
   1517                     .value(),
   1518                 vec![value; 2]
   1519             );
   1520         }
   1521         let row = sqlx::query("SELECT document_rows,value_bytes,phase FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?")
   1522             .bind(id().as_str()).bind(generation(1).as_bytes().as_slice()).fetch_one(store.pool()).await.unwrap();
   1523         assert_eq!(row.get::<i64, _>("document_rows"), 2);
   1524         assert_eq!(row.get::<i64, _>("value_bytes"), 4);
   1525         assert_eq!(row.get::<String, _>("phase"), "staging");
   1526         let ceiling: Option<i64> = sqlx::query_scalar("SELECT batch_value_ceiling FROM radroots_runtime_projection_cache_budget WHERE singleton=1").fetch_one(store.pool()).await.unwrap();
   1527         assert_eq!(ceiling, None);
   1528         sqlx::raw_sql("DROP TRIGGER fixture_batch_fault")
   1529             .execute(store.pool())
   1530             .await
   1531             .unwrap();
   1532         sqlx::query("UPDATE radroots_runtime_projection_cache_budget SET batch_value_ceiling=4 WHERE singleton=1").execute(store.pool()).await.unwrap();
   1533         assert_eq!(
   1534             store
   1535                 .promote_projection_batch(batch([3, 1]), checkpoint(1, 2))
   1536                 .await,
   1537             Err(Error::CorruptProjectionRecord)
   1538         );
   1539         assert_eq!(ProjectionStore::status(&store, id()).await.unwrap(), None);
   1540         assert_eq!(
   1541             store
   1542                 .configure_projection_cache(id(), limits)
   1543                 .await
   1544                 .unwrap(),
   1545             before
   1546         );
   1547         sqlx::query("UPDATE radroots_runtime_projection_cache_budget SET batch_value_ceiling=NULL WHERE singleton=1").execute(store.pool()).await.unwrap();
   1548         store
   1549             .promote_projection_batch(batch([3, 1]), checkpoint(1, 2))
   1550             .await
   1551             .unwrap();
   1552         let ceiling: Option<i64> = sqlx::query_scalar("SELECT batch_value_ceiling FROM radroots_runtime_projection_cache_budget WHERE singleton=1").fetch_one(store.pool()).await.unwrap();
   1553         assert_eq!(ceiling, None);
   1554         assert_eq!(
   1555             store
   1556                 .configure_projection_cache(id(), limits)
   1557                 .await
   1558                 .unwrap(),
   1559             before
   1560         );
   1561         store.close().await.unwrap();
   1562         let reopened = open(&temp).await;
   1563         assert_eq!(
   1564             reopened
   1565                 .configure_projection_cache(id(), limits)
   1566                 .await
   1567                 .unwrap(),
   1568             before
   1569         );
   1570         assert_eq!(
   1571             reopened
   1572                 .projection_document(id(), generation(1), "a".into())
   1573                 .await
   1574                 .unwrap()
   1575                 .unwrap()
   1576                 .value(),
   1577             vec![1; 3]
   1578         );
   1579         assert_eq!(
   1580             ProjectionStore::status(&reopened, id())
   1581                 .await
   1582                 .unwrap()
   1583                 .unwrap()
   1584                 .generation(),
   1585             generation(1)
   1586         );
   1587         let ceiling: Option<i64> = sqlx::query_scalar("SELECT batch_value_ceiling FROM radroots_runtime_projection_cache_budget WHERE singleton=1").fetch_one(reopened.pool()).await.unwrap();
   1588         assert_eq!(ceiling, None);
   1589         reopened.close().await.unwrap();
   1590     }
   1591 }