lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

mod.rs (69600B)


      1 use crate::SqliteStorage;
      2 use crate::backend::map_backend;
      3 use radroots_storage::{
      4     Error, ProjectionStore,
      5     event::{EventPosition, SourceGeneration},
      6     projection::{
      7         ArtifactDigest, BoxFuture, EVENT_INDEX_SHARDS_MAX, EventId, EventIdRange,
      8         EventIndexCheckpoint, EventIndexManifest, EventIndexShard, EventIndexShardCheckpoint,
      9         EventIndexShardId, InvalidationReason, ProjectionCheckpoint, ProjectionDocument,
     10         ProjectionGeneration, ProjectionHealth, ProjectionId, ProjectionInvalidation,
     11         ProjectionRevision, ProjectionSnapshot, ProjectionStatus, RawSourceDigest, RebuildFailure,
     12         RebuildStage, RebuildTicket, RebuildTicketId, RebuildTransition,
     13     },
     14 };
     15 use sqlx::{Row, Sqlite, SqliteConnection};
     16 
     17 mod batch;
     18 mod cache;
     19 mod cache_retired;
     20 mod document_query;
     21 
     22 #[cfg_attr(coverage_nightly, coverage(off))]
     23 impl ProjectionStore for SqliteStorage {
     24     fn configure_projection_cache(
     25         &self,
     26         id: ProjectionId,
     27         limits: radroots_storage::projection::cache::ProjectionCacheLimits,
     28     ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheUsage, Error>>
     29     {
     30         Box::pin(cache::configure(self, id, limits))
     31     }
     32     fn retire_projection_generation(
     33         &self,
     34         scope: radroots_storage::projection::cache::ProjectionCacheScope,
     35     ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheUsage, Error>>
     36     {
     37         Box::pin(cache::retire(self, scope))
     38     }
     39     fn collect_projection_cache(
     40         &self,
     41         scope: radroots_storage::projection::cache::ProjectionCacheScope,
     42     ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheCollection, Error>>
     43     {
     44         Box::pin(cache::collect(self, scope))
     45     }
     46     fn collect_retired_projection_cache(
     47         &self,
     48         id: ProjectionId,
     49         source: radroots_storage::event::EventSourceToken,
     50         expected_active: Option<ProjectionGeneration>,
     51     ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheCollection, Error>>
     52     {
     53         Box::pin(cache_retired::collect(self, id, source, expected_active))
     54     }
     55     fn collect_obsolete_projection_snapshots(
     56         &self,
     57         scope: radroots_storage::projection::cache::ProjectionCacheScope,
     58     ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheCollection, Error>>
     59     {
     60         Box::pin(cache::collect_kind(
     61             self,
     62             scope,
     63             cache::CollectionKind::ObsoleteSnapshots,
     64         ))
     65     }
     66     fn status(
     67         &self,
     68         projection_id: ProjectionId,
     69     ) -> BoxFuture<'_, Result<Option<ProjectionStatus>, Error>> {
     70         Box::pin(async move {
     71             sqlx::query(
     72                 "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?",
     73             )
     74             .bind(projection_id.as_str())
     75             .fetch_optional(self.pool())
     76             .await
     77             .map_err(cache::map)?
     78             .as_ref()
     79             .map(decode_status)
     80             .transpose()
     81         })
     82     }
     83 
     84     fn checkpoint(
     85         &self,
     86         checkpoint: ProjectionCheckpoint,
     87     ) -> BoxFuture<'_, Result<ProjectionStatus, Error>> {
     88         Box::pin(async move {
     89             self.require_projection_writer()?;
     90             let mut transaction = self
     91                 .pool()
     92                 .begin_with("BEGIN IMMEDIATE")
     93                 .await
     94                 .map_err(cache::map)?;
     95             let status = checkpoint_transaction(&mut transaction, checkpoint).await?;
     96             transaction.commit().await.map_err(cache::map)?;
     97             Ok(status)
     98         })
     99     }
    100 
    101     fn invalidate(
    102         &self,
    103         invalidation: ProjectionInvalidation,
    104     ) -> BoxFuture<'_, Result<ProjectionStatus, Error>> {
    105         Box::pin(async move {
    106             self.require_projection_writer()?;
    107             let mut transaction = self
    108                 .pool()
    109                 .begin_with("BEGIN IMMEDIATE")
    110                 .await
    111                 .map_err(cache::map)?;
    112             let row = sqlx::query(
    113                 "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?",
    114             )
    115             .bind(invalidation.projection_id().as_str())
    116             .fetch_optional(&mut *transaction)
    117             .await
    118             .map_err(cache::map)?
    119             .ok_or(Error::ProjectionCheckpointMismatch)?;
    120             let current = decode_status(&row)?;
    121             if current.generation() != invalidation.invalid_generation() {
    122                 return Err(Error::ProjectionCheckpointMismatch);
    123             }
    124             if let Some(existing) = load_invalidation(
    125                 &mut transaction,
    126                 invalidation.projection_id(),
    127                 invalidation.invalid_generation(),
    128             )
    129             .await?
    130             {
    131                 if existing != invalidation {
    132                     return Err(Error::ProjectionRevisionConflict);
    133                 }
    134             } else {
    135                 sqlx::query(
    136                     "INSERT INTO radroots_runtime_projection_invalidations (
    137                        projection_id, invalid_generation, replacement_generation, reason,
    138                        invalidated_at_unix_ms
    139                      ) VALUES (?, ?, ?, ?, ?)",
    140                 )
    141                 .bind(invalidation.projection_id().as_str())
    142                 .bind(invalidation.invalid_generation().as_bytes().as_slice())
    143                 .bind(invalidation.replacement_generation().as_bytes().as_slice())
    144                 .bind(reason_name(invalidation.reason()))
    145                 .bind(i64_from_u64(invalidation.invalidated_at_unix_ms())?)
    146                 .execute(&mut *transaction)
    147                 .await
    148                 .map_err(cache::map)?;
    149             }
    150             let next = ProjectionStatus::new(
    151                 invalidation.projection_id().clone(),
    152                 current.generation(),
    153                 ProjectionHealth::Invalidated,
    154                 current.checkpoint().cloned(),
    155                 None,
    156             )?;
    157             put_status_transaction(&mut transaction, &next).await?;
    158             transaction.commit().await.map_err(cache::map)?;
    159             Ok(next)
    160         })
    161     }
    162 
    163     fn request_rebuild(
    164         &self,
    165         ticket: RebuildTicket,
    166     ) -> BoxFuture<'_, Result<RebuildTicket, Error>> {
    167         Box::pin(async move {
    168             self.require_projection_writer()?;
    169             let mut transaction = self
    170                 .pool()
    171                 .begin_with("BEGIN IMMEDIATE")
    172                 .await
    173                 .map_err(cache::map)?;
    174             if let Some(row) = sqlx::query(
    175                 "SELECT * FROM radroots_runtime_projection_rebuilds WHERE ticket_id = ?",
    176             )
    177             .bind(ticket.ticket_id().as_bytes().as_slice())
    178             .fetch_optional(&mut *transaction)
    179             .await
    180             .map_err(cache::map)?
    181             {
    182                 let existing = decode_ticket(&mut transaction, &row).await?;
    183                 return if existing == ticket {
    184                     transaction.commit().await.map_err(cache::map)?;
    185                     Ok(existing)
    186                 } else {
    187                     Err(Error::ProjectionRevisionConflict)
    188                 };
    189             }
    190             let status_row = sqlx::query(
    191                 "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?",
    192             )
    193             .bind(ticket.invalidation().projection_id().as_str())
    194             .fetch_optional(&mut *transaction)
    195             .await
    196             .map_err(cache::map)?
    197             .ok_or(Error::ProjectionCheckpointMismatch)?;
    198             let status = decode_status(&status_row)?;
    199             if status.generation() != ticket.invalidation().invalid_generation()
    200                 || status.health() != ProjectionHealth::Invalidated
    201                 || load_invalidation(
    202                     &mut transaction,
    203                     ticket.invalidation().projection_id(),
    204                     ticket.invalidation().invalid_generation(),
    205                 )
    206                 .await?
    207                 .as_ref()
    208                     != Some(ticket.invalidation())
    209             {
    210                 return Err(Error::ProjectionCheckpointMismatch);
    211             }
    212             insert_ticket(&mut transaction, &ticket).await?;
    213             let next = ProjectionStatus::new(
    214                 status.projection_id().clone(),
    215                 status.generation(),
    216                 ProjectionHealth::Rebuilding,
    217                 status.checkpoint().cloned(),
    218                 Some(ticket.ticket_id()),
    219             )?;
    220             put_status_transaction(&mut transaction, &next).await?;
    221             transaction.commit().await.map_err(cache::map)?;
    222             Ok(ticket)
    223         })
    224     }
    225 
    226     fn invalidation(
    227         &self,
    228         projection_id: ProjectionId,
    229         replacement_generation: ProjectionGeneration,
    230     ) -> BoxFuture<'_, Result<Option<ProjectionInvalidation>, Error>> {
    231         Box::pin(async move {
    232             sqlx::query(
    233                 "SELECT * FROM radroots_runtime_projection_invalidations
    234                  WHERE projection_id = ? AND replacement_generation = ?
    235                  ORDER BY invalidated_at_unix_ms DESC LIMIT 1",
    236             )
    237             .bind(projection_id.as_str())
    238             .bind(replacement_generation.as_bytes().as_slice())
    239             .fetch_optional(self.pool())
    240             .await
    241             .map_err(cache::map)?
    242             .as_ref()
    243             .map(decode_invalidation)
    244             .transpose()
    245         })
    246     }
    247 
    248     fn rebuild(
    249         &self,
    250         ticket_id: RebuildTicketId,
    251     ) -> BoxFuture<'_, Result<Option<RebuildTicket>, Error>> {
    252         Box::pin(async move {
    253             let mut connection = self.pool().acquire().await.map_err(cache::map)?;
    254             let row = sqlx::query(
    255                 "SELECT * FROM radroots_runtime_projection_rebuilds WHERE ticket_id = ?",
    256             )
    257             .bind(ticket_id.as_bytes().as_slice())
    258             .fetch_optional(&mut *connection)
    259             .await
    260             .map_err(cache::map)?;
    261             match row {
    262                 Some(row) => decode_ticket(&mut connection, &row).await.map(Some),
    263                 None => Ok(None),
    264             }
    265         })
    266     }
    267 
    268     fn transition_rebuild(
    269         &self,
    270         transition: RebuildTransition,
    271     ) -> BoxFuture<'_, Result<RebuildTicket, Error>> {
    272         Box::pin(async move {
    273             self.require_projection_writer()?;
    274             let mut transaction = self
    275                 .pool()
    276                 .begin_with("BEGIN IMMEDIATE")
    277                 .await
    278                 .map_err(cache::map)?;
    279             let row = sqlx::query(
    280                 "SELECT * FROM radroots_runtime_projection_rebuilds WHERE ticket_id = ?",
    281             )
    282             .bind(transition.ticket_id().as_bytes().as_slice())
    283             .fetch_optional(&mut *transaction)
    284             .await
    285             .map_err(cache::map)?
    286             .ok_or(Error::ProjectionRevisionConflict)?;
    287             let current = decode_ticket(&mut transaction, &row).await?;
    288             let status_row = sqlx::query(
    289                 "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?",
    290             )
    291             .bind(current.invalidation().projection_id().as_str())
    292             .fetch_optional(&mut *transaction)
    293             .await
    294             .map_err(cache::map)?
    295             .ok_or(Error::CorruptProjectionRecord)?;
    296             let current_status = decode_status(&status_row)?;
    297             if current_status.generation() != current.invalidation().invalid_generation()
    298                 || current_status.health() != ProjectionHealth::Rebuilding
    299                 || current_status.active_rebuild() != Some(current.ticket_id())
    300             {
    301                 return Err(Error::CorruptProjectionRecord);
    302             }
    303             let next = current.transition(transition)?;
    304             if next.stage() == RebuildStage::Completed {
    305                 batch::refuse_registered_legacy(
    306                     &mut transaction,
    307                     next.invalidation().projection_id(),
    308                     next.invalidation().replacement_generation(),
    309                 )
    310                 .await?;
    311                 let source = sqlx::query(
    312                     "SELECT generation, sequence_head
    313                      FROM radroots_runtime_source_generations WHERE state = 'active'",
    314                 )
    315                 .fetch_one(&mut *transaction)
    316                 .await
    317                 .map_err(cache::map)?;
    318                 let generation = SourceGeneration::new(array(
    319                     source
    320                         .try_get::<Vec<u8>, _>("generation")
    321                         .map_err(map_corrupt)?,
    322                 )?)
    323                 .map_err(|_| Error::CorruptProjectionRecord)?;
    324                 let sequence = u64_from_i64(
    325                     source
    326                         .try_get::<i64, _>("sequence_head")
    327                         .map_err(map_corrupt)?,
    328                 )?;
    329                 if generation != next.source_generation()
    330                     || sequence
    331                         != next
    332                             .source_high_water()
    333                             .map_or(0, |position| position.sequence().get())
    334                 {
    335                     return Err(Error::SourceGenerationChanged);
    336                 }
    337             }
    338             update_ticket(&mut transaction, &next, current.revision()).await?;
    339             let (generation, health, checkpoint, active_rebuild) = match next.stage() {
    340                 RebuildStage::Requested | RebuildStage::Running => (
    341                     current_status.generation(),
    342                     ProjectionHealth::Rebuilding,
    343                     current_status.checkpoint().cloned(),
    344                     Some(next.ticket_id()),
    345                 ),
    346                 RebuildStage::Completed => (
    347                     next.invalidation().replacement_generation(),
    348                     ProjectionHealth::Ready,
    349                     next.checkpoint().cloned(),
    350                     None,
    351                 ),
    352                 RebuildStage::Failed => (
    353                     current_status.generation(),
    354                     ProjectionHealth::Ready,
    355                     current_status.checkpoint().cloned(),
    356                     None,
    357                 ),
    358             };
    359             let status = ProjectionStatus::new(
    360                 next.invalidation().projection_id().clone(),
    361                 generation,
    362                 health,
    363                 checkpoint,
    364                 active_rebuild,
    365             )?;
    366             put_status_transaction(&mut transaction, &status).await?;
    367             transaction.commit().await.map_err(cache::map)?;
    368             Ok(next)
    369         })
    370     }
    371 
    372     fn event_index_manifest(
    373         &self,
    374         generation: ProjectionGeneration,
    375     ) -> BoxFuture<'_, Result<Option<EventIndexManifest>, Error>> {
    376         Box::pin(async move {
    377             let mut connection = self.pool().acquire().await.map_err(cache::map)?;
    378             load_manifest(&mut connection, generation).await
    379         })
    380     }
    381 
    382     fn put_event_index_manifest(
    383         &self,
    384         manifest: EventIndexManifest,
    385     ) -> BoxFuture<'_, Result<(), Error>> {
    386         Box::pin(async move {
    387             self.require_projection_writer()?;
    388             let mut transaction = self
    389                 .pool()
    390                 .begin_with("BEGIN IMMEDIATE")
    391                 .await
    392                 .map_err(cache::map)?;
    393             if let Some(existing) = load_manifest(&mut transaction, manifest.generation()).await? {
    394                 return if existing == manifest {
    395                     transaction.commit().await.map_err(cache::map)?;
    396                     Ok(())
    397                 } else {
    398                     Err(Error::CorruptProjectionRecord)
    399                 };
    400             }
    401             sqlx::query(
    402                 "INSERT INTO radroots_runtime_event_index_manifests (
    403                    projection_generation, total_events, target_shard_size,
    404                    first_published_at_unix_s, last_published_at_unix_s
    405                  ) VALUES (?, ?, ?, ?, ?)",
    406             )
    407             .bind(manifest.generation().as_bytes().as_slice())
    408             .bind(i64_from_u64(manifest.total_events())?)
    409             .bind(i64::from(manifest.target_shard_size()))
    410             .bind(i64_from_u64(manifest.first_published_at_unix_s())?)
    411             .bind(i64_from_u64(manifest.last_published_at_unix_s())?)
    412             .execute(&mut *transaction)
    413             .await
    414             .map_err(cache::map)?;
    415             for (ordinal, shard) in manifest.shards().iter().enumerate() {
    416                 sqlx::query(
    417                     "INSERT INTO radroots_runtime_event_index_shards (
    418                        projection_generation, shard_id, ordinal, artifact_path, event_count,
    419                        first_event_id, last_event_id, first_published_at_unix_s,
    420                        last_published_at_unix_s, artifact_digest
    421                      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
    422                 )
    423                 .bind(manifest.generation().as_bytes().as_slice())
    424                 .bind(shard.shard_id().as_str())
    425                 .bind(i64::try_from(ordinal).map_err(|_| Error::CorruptProjectionRecord)?)
    426                 .bind(shard.artifact_path())
    427                 .bind(i64::from(shard.event_count()))
    428                 .bind(shard.event_ids().first().as_bytes().as_slice())
    429                 .bind(shard.event_ids().last().as_bytes().as_slice())
    430                 .bind(i64_from_u64(shard.first_published_at_unix_s())?)
    431                 .bind(i64_from_u64(shard.last_published_at_unix_s())?)
    432                 .bind(shard.sha256().as_bytes().as_slice())
    433                 .execute(&mut *transaction)
    434                 .await
    435                 .map_err(cache::map)?;
    436             }
    437             transaction.commit().await.map_err(cache::map)
    438         })
    439     }
    440 
    441     fn event_index_checkpoint(
    442         &self,
    443         generation: ProjectionGeneration,
    444     ) -> BoxFuture<'_, Result<Option<EventIndexCheckpoint>, Error>> {
    445         Box::pin(async move {
    446             sqlx::query(
    447                 "SELECT generated_at_unix_ms, checkpoint
    448                  FROM radroots_runtime_event_index_checkpoints
    449                  WHERE projection_generation = ?",
    450             )
    451             .bind(generation.as_bytes().as_slice())
    452             .fetch_optional(self.pool())
    453             .await
    454             .map_err(cache::map)?
    455             .as_ref()
    456             .map(|row| decode_index_checkpoint(row, generation))
    457             .transpose()
    458         })
    459     }
    460 
    461     fn put_event_index_checkpoint(
    462         &self,
    463         checkpoint: EventIndexCheckpoint,
    464     ) -> BoxFuture<'_, Result<(), Error>> {
    465         Box::pin(async move {
    466             self.require_projection_writer()?;
    467             let mut transaction = self
    468                 .pool()
    469                 .begin_with("BEGIN IMMEDIATE")
    470                 .await
    471                 .map_err(cache::map)?;
    472             if let Some(row) = sqlx::query(
    473                 "SELECT generated_at_unix_ms, checkpoint
    474                  FROM radroots_runtime_event_index_checkpoints
    475                  WHERE projection_generation = ?",
    476             )
    477             .bind(checkpoint.generation().as_bytes().as_slice())
    478             .fetch_optional(&mut *transaction)
    479             .await
    480             .map_err(cache::map)?
    481                 && checkpoint.generated_at_unix_ms()
    482                     < decode_index_checkpoint(&row, checkpoint.generation())?.generated_at_unix_ms()
    483             {
    484                 return Err(Error::InvalidEventIndexCheckpoint);
    485             }
    486             sqlx::query(
    487                 "INSERT INTO radroots_runtime_event_index_checkpoints (
    488                    projection_generation, generated_at_unix_ms, checkpoint
    489                  ) VALUES (?, ?, ?)
    490                  ON CONFLICT(projection_generation) DO UPDATE SET
    491                    generated_at_unix_ms = excluded.generated_at_unix_ms,
    492                    checkpoint = excluded.checkpoint",
    493             )
    494             .bind(checkpoint.generation().as_bytes().as_slice())
    495             .bind(i64_from_u64(checkpoint.generated_at_unix_ms())?)
    496             .bind(encode_index_checkpoint(&checkpoint)?)
    497             .execute(&mut *transaction)
    498             .await
    499             .map_err(cache::map)?;
    500             transaction.commit().await.map_err(cache::map)?;
    501             Ok(())
    502         })
    503     }
    504 
    505     fn put_projection_batch(
    506         &self,
    507         value: radroots_storage::projection::batch::ProjectionDocumentBatch,
    508     ) -> BoxFuture<'_, Result<(), Error>> {
    509         Box::pin(async move { batch::write(self, value, None).await.map(|_| ()) })
    510     }
    511     fn promote_projection_batch(
    512         &self,
    513         value: radroots_storage::projection::batch::ProjectionDocumentBatch,
    514         checkpoint: ProjectionCheckpoint,
    515     ) -> BoxFuture<'_, Result<ProjectionStatus, Error>> {
    516         Box::pin(async move {
    517             batch::write(self, value, Some(checkpoint))
    518                 .await?
    519                 .ok_or(Error::CorruptProjectionRecord)
    520         })
    521     }
    522     fn query_projection_documents_at(
    523         &self,
    524         query: radroots_storage::projection::document_query::ProjectionDocumentQuery,
    525         source: radroots_storage::event::EventSourceToken,
    526         active: ProjectionGeneration,
    527     ) -> BoxFuture<
    528         '_,
    529         Result<radroots_storage::projection::document_query::ProjectionDocumentPage, Error>,
    530     > {
    531         Box::pin(document_query::page_at(self, query, source, active))
    532     }
    533     fn put_projection_document(
    534         &self,
    535         projection_id: ProjectionId,
    536         generation: ProjectionGeneration,
    537         document: ProjectionDocument,
    538     ) -> BoxFuture<'_, Result<(), Error>> {
    539         Box::pin(batch::legacy_put(self, projection_id, generation, document))
    540     }
    541 
    542     fn projection_document(
    543         &self,
    544         projection_id: ProjectionId,
    545         generation: ProjectionGeneration,
    546         key: String,
    547     ) -> BoxFuture<'_, Result<Option<ProjectionDocument>, Error>> {
    548         Box::pin(batch::load_document(self, projection_id, generation, key))
    549     }
    550 
    551     fn query_projection_documents(
    552         &self,
    553         query: radroots_storage::projection::document_query::ProjectionDocumentQuery,
    554     ) -> BoxFuture<
    555         '_,
    556         Result<radroots_storage::projection::document_query::ProjectionDocumentPage, Error>,
    557     > {
    558         Box::pin(document_query::page(self, query))
    559     }
    560 
    561     fn put_projection_snapshot(
    562         &self,
    563         snapshot: ProjectionSnapshot,
    564     ) -> BoxFuture<'_, Result<(), Error>> {
    565         Box::pin(cache::put_snapshot(self, snapshot))
    566     }
    567     fn projection_snapshot(
    568         &self,
    569         id: ProjectionId,
    570         snapshot: [u8; 32],
    571     ) -> BoxFuture<'_, Result<Option<ProjectionSnapshot>, Error>> {
    572         Box::pin(cache::snapshot(self, id, snapshot))
    573     }
    574 }
    575 
    576 #[cfg_attr(coverage_nightly, coverage(off))]
    577 pub(crate) async fn checkpoint_transaction(
    578     transaction: &mut sqlx::Transaction<'_, Sqlite>,
    579     checkpoint: ProjectionCheckpoint,
    580 ) -> Result<ProjectionStatus, Error> {
    581     batch::refuse_registered_legacy(
    582         transaction,
    583         checkpoint.projection_id(),
    584         checkpoint.generation(),
    585     )
    586     .await?;
    587     let prior = sqlx::query(
    588         "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?",
    589     )
    590     .bind(checkpoint.projection_id().as_str())
    591     .fetch_optional(&mut **transaction)
    592     .await
    593     .map_err(cache::map)?
    594     .as_ref()
    595     .map(decode_status)
    596     .transpose()?;
    597     if let Some(prior) = prior.as_ref() {
    598         if prior.generation() != checkpoint.generation() {
    599             return Err(Error::ProjectionCheckpointMismatch);
    600         }
    601         if prior
    602             .checkpoint()
    603             .is_some_and(|value| !checkpoint.advances(value))
    604         {
    605             return Err(Error::ProjectionCheckpointRegression);
    606         }
    607     }
    608     let status = ProjectionStatus::new(
    609         checkpoint.projection_id().clone(),
    610         checkpoint.generation(),
    611         ProjectionHealth::Ready,
    612         Some(checkpoint),
    613         None,
    614     )?;
    615     put_status_transaction(transaction, &status).await?;
    616     Ok(status)
    617 }
    618 
    619 impl SqliteStorage {
    620     fn require_projection_writer(&self) -> Result<(), Error> {
    621         if self.event_mode() == radroots_storage::status::EventStoreMode::ReadOnly {
    622             return Err(Error::BackendUnavailable);
    623         }
    624         Ok(())
    625     }
    626 }
    627 
    628 #[cfg_attr(coverage_nightly, coverage(off))]
    629 async fn put_status_transaction(
    630     transaction: &mut sqlx::Transaction<'_, Sqlite>,
    631     status: &ProjectionStatus,
    632 ) -> Result<(), Error> {
    633     let values = checkpoint_values(status.checkpoint())?;
    634     sqlx::query(STATUS_UPSERT)
    635         .bind(status.projection_id().as_str())
    636         .bind(status.generation().as_bytes().as_slice())
    637         .bind(health_name(status.health()))
    638         .bind(values.0)
    639         .bind(values.1)
    640         .bind(values.2)
    641         .bind(values.3)
    642         .bind(
    643             status
    644                 .active_rebuild()
    645                 .map(|ticket| ticket.as_bytes().to_vec()),
    646         )
    647         .execute(&mut **transaction)
    648         .await
    649         .map_err(cache::map)?;
    650     Ok(())
    651 }
    652 
    653 const STATUS_UPSERT: &str = "INSERT INTO radroots_runtime_projection_checkpoints (
    654        projection_id, projection_generation, health, source_generation, source_sequence,
    655        projected_rows, checkpoint_updated_at_unix_ms, active_rebuild
    656      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
    657      ON CONFLICT(projection_id) DO UPDATE SET
    658        projection_generation = excluded.projection_generation,
    659        health = excluded.health,
    660        source_generation = excluded.source_generation,
    661        source_sequence = excluded.source_sequence,
    662        projected_rows = excluded.projected_rows,
    663        checkpoint_updated_at_unix_ms = excluded.checkpoint_updated_at_unix_ms,
    664        active_rebuild = excluded.active_rebuild";
    665 
    666 type CheckpointValues = (Option<Vec<u8>>, Option<i64>, Option<i64>, Option<i64>);
    667 
    668 fn checkpoint_values(checkpoint: Option<&ProjectionCheckpoint>) -> Result<CheckpointValues, Error> {
    669     let Some(checkpoint) = checkpoint else {
    670         return Ok((None, None, None, None));
    671     };
    672     let (generation, sequence) = checkpoint
    673         .source_position()
    674         .map_or((None, None), |position| {
    675             (
    676                 Some(position.generation().as_bytes().to_vec()),
    677                 Some(i64_from_u64(position.sequence().get())),
    678             )
    679         });
    680     Ok((
    681         generation,
    682         sequence.transpose()?,
    683         Some(i64_from_u64(checkpoint.projected_rows())?),
    684         Some(i64_from_u64(checkpoint.updated_at_unix_ms())?),
    685     ))
    686 }
    687 
    688 fn decode_status(row: &sqlx::sqlite::SqliteRow) -> Result<ProjectionStatus, Error> {
    689     let projection_id = ProjectionId::parse(
    690         row.try_get::<String, _>("projection_id")
    691             .map_err(map_corrupt)?,
    692     )
    693     .map_err(|_| Error::CorruptProjectionRecord)?;
    694     let generation = projection_generation(row, "projection_generation")?;
    695     let checkpoint = decode_checkpoint(row, projection_id.clone(), generation, "")?;
    696     let active = row
    697         .try_get::<Option<Vec<u8>>, _>("active_rebuild")
    698         .map_err(map_corrupt)?
    699         .map(|value| {
    700             RebuildTicketId::new(array(value)?).map_err(|_| Error::CorruptProjectionRecord)
    701         })
    702         .transpose()?;
    703     ProjectionStatus::new(
    704         projection_id,
    705         generation,
    706         health(
    707             row.try_get::<String, _>("health")
    708                 .map_err(map_corrupt)?
    709                 .as_str(),
    710         )?,
    711         checkpoint,
    712         active,
    713     )
    714 }
    715 
    716 fn decode_checkpoint(
    717     row: &sqlx::sqlite::SqliteRow,
    718     projection_id: ProjectionId,
    719     generation: ProjectionGeneration,
    720     prefix: &str,
    721 ) -> Result<Option<ProjectionCheckpoint>, Error> {
    722     let rows = row
    723         .try_get::<Option<i64>, _>(format!("{prefix}projected_rows").as_str())
    724         .map_err(map_corrupt)?;
    725     let updated = row
    726         .try_get::<Option<i64>, _>(format!("{prefix}updated_at_unix_ms").as_str())
    727         .or_else(|_| row.try_get(format!("{prefix}checkpoint_updated_at_unix_ms").as_str()))
    728         .map_err(map_corrupt)?;
    729     let source_generation = row
    730         .try_get::<Option<Vec<u8>>, _>(format!("{prefix}source_generation").as_str())
    731         .map_err(map_corrupt)?;
    732     let source_sequence = row
    733         .try_get::<Option<i64>, _>(format!("{prefix}source_sequence").as_str())
    734         .map_err(map_corrupt)?;
    735     match (rows, updated, source_generation, source_sequence) {
    736         (None, None, None, None) => Ok(None),
    737         (Some(rows), Some(updated), source_generation, source_sequence) => {
    738             let position = match (source_generation, source_sequence) {
    739                 (None, None) => None,
    740                 (Some(source_generation), Some(source_sequence)) => Some(EventPosition::new(
    741                     SourceGeneration::new(array(source_generation)?)
    742                         .map_err(|_| Error::CorruptProjectionRecord)?,
    743                     radroots_storage::event::EventSequence::new(u64_from_i64(source_sequence)?)
    744                         .map_err(|_| Error::CorruptProjectionRecord)?,
    745                 )),
    746                 _ => return Err(Error::CorruptProjectionRecord),
    747             };
    748             ProjectionCheckpoint::new(
    749                 projection_id,
    750                 generation,
    751                 position,
    752                 u64_from_i64(rows)?,
    753                 u64_from_i64(updated)?,
    754             )
    755             .map(Some)
    756             .map_err(|_| Error::CorruptProjectionRecord)
    757         }
    758         _ => Err(Error::CorruptProjectionRecord),
    759     }
    760 }
    761 
    762 #[cfg_attr(coverage_nightly, coverage(off))]
    763 async fn load_invalidation(
    764     connection: &mut SqliteConnection,
    765     projection_id: &ProjectionId,
    766     generation: ProjectionGeneration,
    767 ) -> Result<Option<ProjectionInvalidation>, Error> {
    768     sqlx::query(
    769         "SELECT * FROM radroots_runtime_projection_invalidations
    770          WHERE projection_id = ? AND invalid_generation = ?",
    771     )
    772     .bind(projection_id.as_str())
    773     .bind(generation.as_bytes().as_slice())
    774     .fetch_optional(&mut *connection)
    775     .await
    776     .map_err(cache::map)?
    777     .as_ref()
    778     .map(decode_invalidation)
    779     .transpose()
    780 }
    781 
    782 fn decode_invalidation(row: &sqlx::sqlite::SqliteRow) -> Result<ProjectionInvalidation, Error> {
    783     ProjectionInvalidation::new(
    784         ProjectionId::parse(
    785             row.try_get::<String, _>("projection_id")
    786                 .map_err(map_corrupt)?,
    787         )
    788         .map_err(|_| Error::CorruptProjectionRecord)?,
    789         projection_generation(row, "invalid_generation")?,
    790         projection_generation(row, "replacement_generation")?,
    791         reason(
    792             row.try_get::<String, _>("reason")
    793                 .map_err(map_corrupt)?
    794                 .as_str(),
    795         )?,
    796         u64_from_i64(row.try_get("invalidated_at_unix_ms").map_err(map_corrupt)?)?,
    797     )
    798     .map_err(|_| Error::CorruptProjectionRecord)
    799 }
    800 
    801 #[cfg_attr(coverage_nightly, coverage(off))]
    802 async fn insert_ticket(
    803     transaction: &mut sqlx::Transaction<'_, Sqlite>,
    804     ticket: &RebuildTicket,
    805 ) -> Result<(), Error> {
    806     let checkpoint = checkpoint_values(ticket.checkpoint())?;
    807     sqlx::query(
    808         "INSERT INTO radroots_runtime_projection_rebuilds (
    809            ticket_id, projection_id, invalid_generation, replacement_generation, revision, stage,
    810            source_generation, source_sequence, source_digest,
    811            checkpoint_source_generation, checkpoint_source_sequence, checkpoint_projected_rows,
    812            checkpoint_updated_at_unix_ms, failure, requested_at_unix_ms, updated_at_unix_ms
    813          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
    814     )
    815     .bind(ticket.ticket_id().as_bytes().as_slice())
    816     .bind(ticket.invalidation().projection_id().as_str())
    817     .bind(
    818         ticket
    819             .invalidation()
    820             .invalid_generation()
    821             .as_bytes()
    822             .as_slice(),
    823     )
    824     .bind(
    825         ticket
    826             .invalidation()
    827             .replacement_generation()
    828             .as_bytes()
    829             .as_slice(),
    830     )
    831     .bind(i64_from_u64(ticket.revision().get())?)
    832     .bind(rebuild_stage_name(ticket.stage()))
    833     .bind(ticket.source_generation().as_bytes().as_slice())
    834     .bind(
    835         ticket
    836             .source_high_water()
    837             .map(|position| i64_from_u64(position.sequence().get()))
    838             .transpose()?,
    839     )
    840     .bind(ticket.source_digest().as_bytes().as_slice())
    841     .bind(checkpoint.0)
    842     .bind(checkpoint.1)
    843     .bind(checkpoint.2)
    844     .bind(checkpoint.3)
    845     .bind(ticket.failure().map(rebuild_failure_name))
    846     .bind(i64_from_u64(ticket.requested_at_unix_ms())?)
    847     .bind(i64_from_u64(ticket.updated_at_unix_ms())?)
    848     .execute(&mut **transaction)
    849     .await
    850     .map_err(cache::map)?;
    851     Ok(())
    852 }
    853 
    854 #[cfg_attr(coverage_nightly, coverage(off))]
    855 async fn update_ticket(
    856     transaction: &mut sqlx::Transaction<'_, Sqlite>,
    857     ticket: &RebuildTicket,
    858     prior: ProjectionRevision,
    859 ) -> Result<(), Error> {
    860     let checkpoint = checkpoint_values(ticket.checkpoint())?;
    861     let result = sqlx::query(
    862         "UPDATE radroots_runtime_projection_rebuilds SET
    863            revision = ?, stage = ?, checkpoint_source_generation = ?,
    864            checkpoint_source_sequence = ?, checkpoint_projected_rows = ?,
    865            checkpoint_updated_at_unix_ms = ?, failure = ?, updated_at_unix_ms = ?
    866          WHERE ticket_id = ? AND revision = ?",
    867     )
    868     .bind(i64_from_u64(ticket.revision().get())?)
    869     .bind(rebuild_stage_name(ticket.stage()))
    870     .bind(checkpoint.0)
    871     .bind(checkpoint.1)
    872     .bind(checkpoint.2)
    873     .bind(checkpoint.3)
    874     .bind(ticket.failure().map(rebuild_failure_name))
    875     .bind(i64_from_u64(ticket.updated_at_unix_ms())?)
    876     .bind(ticket.ticket_id().as_bytes().as_slice())
    877     .bind(i64_from_u64(prior.get())?)
    878     .execute(&mut **transaction)
    879     .await
    880     .map_err(cache::map)?;
    881     if result.rows_affected() != 1 {
    882         return Err(Error::ProjectionRevisionConflict);
    883     }
    884     Ok(())
    885 }
    886 
    887 #[cfg_attr(coverage_nightly, coverage(off))]
    888 async fn decode_ticket(
    889     connection: &mut SqliteConnection,
    890     row: &sqlx::sqlite::SqliteRow,
    891 ) -> Result<RebuildTicket, Error> {
    892     let projection_id = ProjectionId::parse(
    893         row.try_get::<String, _>("projection_id")
    894             .map_err(map_corrupt)?,
    895     )
    896     .map_err(|_| Error::CorruptProjectionRecord)?;
    897     let invalid_generation = projection_generation(row, "invalid_generation")?;
    898     let invalidation = load_invalidation(connection, &projection_id, invalid_generation)
    899         .await?
    900         .ok_or(Error::CorruptProjectionRecord)?;
    901     if invalidation.replacement_generation()
    902         != projection_generation(row, "replacement_generation")?
    903     {
    904         return Err(Error::CorruptProjectionRecord);
    905     }
    906     let checkpoint = decode_checkpoint(
    907         row,
    908         projection_id,
    909         invalidation.replacement_generation(),
    910         "checkpoint_",
    911     )?;
    912     RebuildTicket::from_durable_parts(
    913         RebuildTicketId::new(array(
    914             row.try_get::<Vec<u8>, _>("ticket_id")
    915                 .map_err(map_corrupt)?,
    916         )?)
    917         .map_err(|_| Error::CorruptProjectionRecord)?,
    918         invalidation,
    919         ProjectionRevision::new(u64_from_i64(row.try_get("revision").map_err(map_corrupt)?)?)
    920             .map_err(|_| Error::CorruptProjectionRecord)?,
    921         rebuild_stage(
    922             row.try_get::<String, _>("stage")
    923                 .map_err(map_corrupt)?
    924                 .as_str(),
    925         )?,
    926         SourceGeneration::new(array(
    927             row.try_get::<Vec<u8>, _>("source_generation")
    928                 .map_err(map_corrupt)?,
    929         )?)
    930         .map_err(|_| Error::CorruptProjectionRecord)?,
    931         row.try_get::<Option<i64>, _>("source_sequence")
    932             .map_err(map_corrupt)?
    933             .map(|sequence| {
    934                 Ok(EventPosition::new(
    935                     SourceGeneration::new(array(
    936                         row.try_get::<Vec<u8>, _>("source_generation")
    937                             .map_err(map_corrupt)?,
    938                     )?)
    939                     .map_err(|_| Error::CorruptProjectionRecord)?,
    940                     radroots_storage::event::EventSequence::new(u64_from_i64(sequence)?)
    941                         .map_err(|_| Error::CorruptProjectionRecord)?,
    942                 ))
    943             })
    944             .transpose()?,
    945         RawSourceDigest::new(array(
    946             row.try_get::<Vec<u8>, _>("source_digest")
    947                 .map_err(map_corrupt)?,
    948         )?),
    949         checkpoint,
    950         row.try_get::<Option<String>, _>("failure")
    951             .map_err(map_corrupt)?
    952             .as_deref()
    953             .map(rebuild_failure)
    954             .transpose()?,
    955         u64_from_i64(row.try_get("requested_at_unix_ms").map_err(map_corrupt)?)?,
    956         u64_from_i64(row.try_get("updated_at_unix_ms").map_err(map_corrupt)?)?,
    957     )
    958 }
    959 
    960 #[cfg_attr(coverage_nightly, coverage(off))]
    961 async fn load_manifest(
    962     connection: &mut SqliteConnection,
    963     generation: ProjectionGeneration,
    964 ) -> Result<Option<EventIndexManifest>, Error> {
    965     let Some(row) = sqlx::query(
    966         "SELECT * FROM radroots_runtime_event_index_manifests
    967          WHERE projection_generation = ?",
    968     )
    969     .bind(generation.as_bytes().as_slice())
    970     .fetch_optional(&mut *connection)
    971     .await
    972     .map_err(cache::map)?
    973     else {
    974         return Ok(None);
    975     };
    976     let shards = sqlx::query(
    977         "SELECT * FROM radroots_runtime_event_index_shards
    978          WHERE projection_generation = ? ORDER BY ordinal",
    979     )
    980     .bind(generation.as_bytes().as_slice())
    981     .fetch_all(&mut *connection)
    982     .await
    983     .map_err(cache::map)?
    984     .iter()
    985     .enumerate()
    986     .map(|(ordinal, row)| {
    987         if row.try_get::<i64, _>("ordinal").map_err(map_corrupt)?
    988             != i64::try_from(ordinal).map_err(|_| Error::CorruptProjectionRecord)?
    989         {
    990             return Err(Error::CorruptProjectionRecord);
    991         }
    992         EventIndexShard::new(
    993             EventIndexShardId::parse(row.try_get::<String, _>("shard_id").map_err(map_corrupt)?)
    994                 .map_err(|_| Error::CorruptProjectionRecord)?,
    995             row.try_get::<String, _>("artifact_path")
    996                 .map_err(map_corrupt)?,
    997             u32::try_from(row.try_get::<i64, _>("event_count").map_err(map_corrupt)?)
    998                 .map_err(|_| Error::CorruptProjectionRecord)?,
    999             EventIdRange::new(
   1000                 event_id(row, "first_event_id")?,
   1001                 event_id(row, "last_event_id")?,
   1002             )
   1003             .map_err(|_| Error::CorruptProjectionRecord)?,
   1004             u64_from_i64(
   1005                 row.try_get("first_published_at_unix_s")
   1006                     .map_err(map_corrupt)?,
   1007             )?,
   1008             u64_from_i64(
   1009                 row.try_get("last_published_at_unix_s")
   1010                     .map_err(map_corrupt)?,
   1011             )?,
   1012             ArtifactDigest::new(array(
   1013                 row.try_get::<Vec<u8>, _>("artifact_digest")
   1014                     .map_err(map_corrupt)?,
   1015             )?),
   1016         )
   1017         .map_err(|_| Error::CorruptProjectionRecord)
   1018     })
   1019     .collect::<Result<Vec<_>, _>>()?;
   1020     EventIndexManifest::new(
   1021         generation,
   1022         u64_from_i64(row.try_get("total_events").map_err(map_corrupt)?)?,
   1023         u32::try_from(
   1024             row.try_get::<i64, _>("target_shard_size")
   1025                 .map_err(map_corrupt)?,
   1026         )
   1027         .map_err(|_| Error::CorruptProjectionRecord)?,
   1028         u64_from_i64(
   1029             row.try_get("first_published_at_unix_s")
   1030                 .map_err(map_corrupt)?,
   1031         )?,
   1032         u64_from_i64(
   1033             row.try_get("last_published_at_unix_s")
   1034                 .map_err(map_corrupt)?,
   1035         )?,
   1036         shards,
   1037     )
   1038     .map(Some)
   1039     .map_err(|_| Error::CorruptProjectionRecord)
   1040 }
   1041 
   1042 fn encode_index_checkpoint(checkpoint: &EventIndexCheckpoint) -> Result<Vec<u8>, Error> {
   1043     let mut bytes = vec![1];
   1044     let count =
   1045         u16::try_from(checkpoint.shards().len()).map_err(|_| Error::CorruptProjectionRecord)?;
   1046     bytes.extend_from_slice(&count.to_be_bytes());
   1047     for shard in checkpoint.shards() {
   1048         put_string(&mut bytes, shard.shard_id().as_str())?;
   1049         bytes.extend_from_slice(&shard.last_created_at_unix_s().to_be_bytes());
   1050         match shard.last_event_id() {
   1051             Some(event_id) => {
   1052                 bytes.push(1);
   1053                 bytes.extend_from_slice(event_id.as_bytes());
   1054             }
   1055             None => bytes.push(0),
   1056         }
   1057         match shard.cursor() {
   1058             Some(cursor) => {
   1059                 bytes.push(1);
   1060                 put_string(&mut bytes, cursor)?;
   1061             }
   1062             None => bytes.push(0),
   1063         }
   1064     }
   1065     Ok(bytes)
   1066 }
   1067 
   1068 fn decode_index_checkpoint(
   1069     row: &sqlx::sqlite::SqliteRow,
   1070     generation: ProjectionGeneration,
   1071 ) -> Result<EventIndexCheckpoint, Error> {
   1072     let bytes = row
   1073         .try_get::<Vec<u8>, _>("checkpoint")
   1074         .map_err(map_corrupt)?;
   1075     let mut cursor = Cursor::new(bytes.as_slice());
   1076     if cursor.byte()? != 1 {
   1077         return Err(Error::CorruptProjectionRecord);
   1078     }
   1079     let count = usize::from(cursor.u16()?);
   1080     if count > EVENT_INDEX_SHARDS_MAX {
   1081         return Err(Error::CorruptProjectionRecord);
   1082     }
   1083     let mut shards = Vec::with_capacity(count);
   1084     for _ in 0..count {
   1085         let shard_id = EventIndexShardId::parse(cursor.string()?.to_owned())
   1086             .map_err(|_| Error::CorruptProjectionRecord)?;
   1087         let last_created_at_unix_s = cursor.u64()?;
   1088         let last_event_id = match cursor.byte()? {
   1089             0 => None,
   1090             1 => Some(EventId::from_bytes(cursor.array()?)),
   1091             _ => return Err(Error::CorruptProjectionRecord),
   1092         };
   1093         let checkpoint_cursor = match cursor.byte()? {
   1094             0 => None,
   1095             1 => Some(cursor.string()?.to_owned()),
   1096             _ => return Err(Error::CorruptProjectionRecord),
   1097         };
   1098         shards.push(
   1099             EventIndexShardCheckpoint::new(
   1100                 shard_id,
   1101                 last_created_at_unix_s,
   1102                 last_event_id,
   1103                 checkpoint_cursor,
   1104             )
   1105             .map_err(|_| Error::CorruptProjectionRecord)?,
   1106         );
   1107     }
   1108     cursor.finish()?;
   1109     EventIndexCheckpoint::new(
   1110         generation,
   1111         u64_from_i64(row.try_get("generated_at_unix_ms").map_err(map_corrupt)?)?,
   1112         shards,
   1113     )
   1114     .map_err(|_| Error::CorruptProjectionRecord)
   1115 }
   1116 
   1117 pub(crate) fn encode_status_snapshot(status: &ProjectionStatus) -> Result<Vec<u8>, Error> {
   1118     let mut bytes = Vec::with_capacity(128);
   1119     bytes.push(1);
   1120     put_string(&mut bytes, status.projection_id().as_str())?;
   1121     bytes.extend_from_slice(status.generation().as_bytes());
   1122     bytes.push(match status.health() {
   1123         ProjectionHealth::Ready => 0,
   1124         ProjectionHealth::Invalidated => 1,
   1125         ProjectionHealth::Rebuilding => 2,
   1126         ProjectionHealth::Failed => 3,
   1127     });
   1128     match status.checkpoint() {
   1129         Some(checkpoint) => {
   1130             bytes.push(1);
   1131             match checkpoint.source_position() {
   1132                 Some(position) => {
   1133                     bytes.push(1);
   1134                     bytes.extend_from_slice(position.generation().as_bytes());
   1135                     bytes.extend_from_slice(&position.sequence().get().to_be_bytes());
   1136                 }
   1137                 None => bytes.push(0),
   1138             }
   1139             bytes.extend_from_slice(&checkpoint.projected_rows().to_be_bytes());
   1140             bytes.extend_from_slice(&checkpoint.updated_at_unix_ms().to_be_bytes());
   1141         }
   1142         None => bytes.push(0),
   1143     }
   1144     match status.active_rebuild() {
   1145         Some(ticket) => {
   1146             bytes.push(1);
   1147             bytes.extend_from_slice(ticket.as_bytes());
   1148         }
   1149         None => bytes.push(0),
   1150     }
   1151     Ok(bytes)
   1152 }
   1153 
   1154 pub(crate) fn decode_status_snapshot(bytes: &[u8]) -> Result<ProjectionStatus, Error> {
   1155     let mut cursor = Cursor::new(bytes);
   1156     if cursor.byte()? != 1 {
   1157         return Err(Error::CorruptProjectionRecord);
   1158     }
   1159     let projection_id =
   1160         ProjectionId::parse(cursor.string()?).map_err(|_| Error::CorruptProjectionRecord)?;
   1161     let generation =
   1162         ProjectionGeneration::new(cursor.array()?).map_err(|_| Error::CorruptProjectionRecord)?;
   1163     let health = match cursor.byte()? {
   1164         0 => ProjectionHealth::Ready,
   1165         1 => ProjectionHealth::Invalidated,
   1166         2 => ProjectionHealth::Rebuilding,
   1167         3 => ProjectionHealth::Failed,
   1168         _ => return Err(Error::CorruptProjectionRecord),
   1169     };
   1170     let checkpoint = match cursor.byte()? {
   1171         0 => None,
   1172         1 => {
   1173             let source_position = match cursor.byte()? {
   1174                 0 => None,
   1175                 1 => Some(EventPosition::new(
   1176                     SourceGeneration::new(cursor.array()?)
   1177                         .map_err(|_| Error::CorruptProjectionRecord)?,
   1178                     radroots_storage::event::EventSequence::new(cursor.u64()?)
   1179                         .map_err(|_| Error::CorruptProjectionRecord)?,
   1180                 )),
   1181                 _ => return Err(Error::CorruptProjectionRecord),
   1182             };
   1183             Some(
   1184                 ProjectionCheckpoint::new(
   1185                     projection_id.clone(),
   1186                     generation,
   1187                     source_position,
   1188                     cursor.u64()?,
   1189                     cursor.u64()?,
   1190                 )
   1191                 .map_err(|_| Error::CorruptProjectionRecord)?,
   1192             )
   1193         }
   1194         _ => return Err(Error::CorruptProjectionRecord),
   1195     };
   1196     let active_rebuild = match cursor.byte()? {
   1197         0 => None,
   1198         1 => Some(
   1199             RebuildTicketId::new(cursor.array()?).map_err(|_| Error::CorruptProjectionRecord)?,
   1200         ),
   1201         _ => return Err(Error::CorruptProjectionRecord),
   1202     };
   1203     cursor.finish()?;
   1204     ProjectionStatus::new(
   1205         projection_id,
   1206         generation,
   1207         health,
   1208         checkpoint,
   1209         active_rebuild,
   1210     )
   1211     .map_err(|_| Error::CorruptProjectionRecord)
   1212 }
   1213 
   1214 fn put_string(bytes: &mut Vec<u8>, value: &str) -> Result<(), Error> {
   1215     let length = u16::try_from(value.len()).map_err(|_| Error::CorruptProjectionRecord)?;
   1216     bytes.extend_from_slice(&length.to_be_bytes());
   1217     bytes.extend_from_slice(value.as_bytes());
   1218     Ok(())
   1219 }
   1220 
   1221 struct Cursor<'a> {
   1222     bytes: &'a [u8],
   1223     offset: usize,
   1224 }
   1225 
   1226 impl<'a> Cursor<'a> {
   1227     const fn new(bytes: &'a [u8]) -> Self {
   1228         Self { bytes, offset: 0 }
   1229     }
   1230     fn byte(&mut self) -> Result<u8, Error> {
   1231         let value = self
   1232             .bytes
   1233             .get(self.offset)
   1234             .copied()
   1235             .ok_or(Error::CorruptProjectionRecord)?;
   1236         self.offset += 1;
   1237         Ok(value)
   1238     }
   1239     fn u16(&mut self) -> Result<u16, Error> {
   1240         Ok(u16::from_be_bytes(self.array()?))
   1241     }
   1242     fn u64(&mut self) -> Result<u64, Error> {
   1243         Ok(u64::from_be_bytes(self.array()?))
   1244     }
   1245     fn string(&mut self) -> Result<&'a str, Error> {
   1246         let length = usize::from(self.u16()?);
   1247         core::str::from_utf8(self.take(length)?).map_err(|_| Error::CorruptProjectionRecord)
   1248     }
   1249     fn array<const N: usize>(&mut self) -> Result<[u8; N], Error> {
   1250         self.take(N)?
   1251             .try_into()
   1252             .map_err(|_| Error::CorruptProjectionRecord)
   1253     }
   1254     fn take(&mut self, length: usize) -> Result<&'a [u8], Error> {
   1255         let end = self
   1256             .offset
   1257             .checked_add(length)
   1258             .ok_or(Error::CorruptProjectionRecord)?;
   1259         let value = self
   1260             .bytes
   1261             .get(self.offset..end)
   1262             .ok_or(Error::CorruptProjectionRecord)?;
   1263         self.offset = end;
   1264         Ok(value)
   1265     }
   1266     fn finish(self) -> Result<(), Error> {
   1267         if self.offset == self.bytes.len() {
   1268             Ok(())
   1269         } else {
   1270             Err(Error::CorruptProjectionRecord)
   1271         }
   1272     }
   1273 }
   1274 
   1275 fn projection_generation(
   1276     row: &sqlx::sqlite::SqliteRow,
   1277     column: &str,
   1278 ) -> Result<ProjectionGeneration, Error> {
   1279     ProjectionGeneration::new(array(
   1280         row.try_get::<Vec<u8>, _>(column).map_err(map_corrupt)?,
   1281     )?)
   1282     .map_err(|_| Error::CorruptProjectionRecord)
   1283 }
   1284 
   1285 fn event_id(row: &sqlx::sqlite::SqliteRow, column: &str) -> Result<EventId, Error> {
   1286     Ok(EventId::from_bytes(array(
   1287         row.try_get::<Vec<u8>, _>(column).map_err(map_corrupt)?,
   1288     )?))
   1289 }
   1290 
   1291 const fn reason_name(value: InvalidationReason) -> &'static str {
   1292     match value {
   1293         InvalidationReason::SourceGenerationChanged => "source_generation_changed",
   1294         InvalidationReason::ProjectionGenerationChanged => "projection_generation_changed",
   1295         InvalidationReason::EventIndexManifestChanged => "event_index_manifest_changed",
   1296         InvalidationReason::IntegrityFailure => "integrity_failure",
   1297         InvalidationReason::OperatorRequested => "operator_requested",
   1298     }
   1299 }
   1300 
   1301 const fn reason(value: &str) -> Result<InvalidationReason, Error> {
   1302     match value.as_bytes() {
   1303         b"source_generation_changed" => Ok(InvalidationReason::SourceGenerationChanged),
   1304         b"projection_generation_changed" => Ok(InvalidationReason::ProjectionGenerationChanged),
   1305         b"event_index_manifest_changed" => Ok(InvalidationReason::EventIndexManifestChanged),
   1306         b"integrity_failure" => Ok(InvalidationReason::IntegrityFailure),
   1307         b"operator_requested" => Ok(InvalidationReason::OperatorRequested),
   1308         _ => Err(Error::CorruptProjectionRecord),
   1309     }
   1310 }
   1311 
   1312 const fn health_name(value: ProjectionHealth) -> &'static str {
   1313     match value {
   1314         ProjectionHealth::Ready => "ready",
   1315         ProjectionHealth::Invalidated => "invalidated",
   1316         ProjectionHealth::Rebuilding => "rebuilding",
   1317         ProjectionHealth::Failed => "failed",
   1318     }
   1319 }
   1320 
   1321 const fn health(value: &str) -> Result<ProjectionHealth, Error> {
   1322     match value.as_bytes() {
   1323         b"ready" => Ok(ProjectionHealth::Ready),
   1324         b"invalidated" => Ok(ProjectionHealth::Invalidated),
   1325         b"rebuilding" => Ok(ProjectionHealth::Rebuilding),
   1326         b"failed" => Ok(ProjectionHealth::Failed),
   1327         _ => Err(Error::CorruptProjectionRecord),
   1328     }
   1329 }
   1330 
   1331 const fn rebuild_stage_name(value: RebuildStage) -> &'static str {
   1332     match value {
   1333         RebuildStage::Requested => "requested",
   1334         RebuildStage::Running => "running",
   1335         RebuildStage::Completed => "completed",
   1336         RebuildStage::Failed => "failed",
   1337     }
   1338 }
   1339 
   1340 const fn rebuild_stage(value: &str) -> Result<RebuildStage, Error> {
   1341     match value.as_bytes() {
   1342         b"requested" => Ok(RebuildStage::Requested),
   1343         b"running" => Ok(RebuildStage::Running),
   1344         b"completed" => Ok(RebuildStage::Completed),
   1345         b"failed" => Ok(RebuildStage::Failed),
   1346         _ => Err(Error::CorruptProjectionRecord),
   1347     }
   1348 }
   1349 
   1350 const fn rebuild_failure_name(value: RebuildFailure) -> &'static str {
   1351     match value {
   1352         RebuildFailure::ReducerRejected => "reducer_rejected",
   1353         RebuildFailure::SourceChanged => "source_changed",
   1354         RebuildFailure::IntegrityFailure => "integrity_failure",
   1355         RebuildFailure::PromotionRejected => "promotion_rejected",
   1356     }
   1357 }
   1358 
   1359 const fn rebuild_failure(value: &str) -> Result<RebuildFailure, Error> {
   1360     match value.as_bytes() {
   1361         b"reducer_rejected" => Ok(RebuildFailure::ReducerRejected),
   1362         b"source_changed" => Ok(RebuildFailure::SourceChanged),
   1363         b"integrity_failure" => Ok(RebuildFailure::IntegrityFailure),
   1364         b"promotion_rejected" => Ok(RebuildFailure::PromotionRejected),
   1365         _ => Err(Error::CorruptProjectionRecord),
   1366     }
   1367 }
   1368 
   1369 fn array<const N: usize>(bytes: Vec<u8>) -> Result<[u8; N], Error> {
   1370     bytes.try_into().map_err(|_| Error::CorruptProjectionRecord)
   1371 }
   1372 
   1373 fn i64_from_u64(value: u64) -> Result<i64, Error> {
   1374     i64::try_from(value).map_err(|_| Error::CorruptProjectionRecord)
   1375 }
   1376 
   1377 fn u64_from_i64(value: i64) -> Result<u64, Error> {
   1378     u64::try_from(value).map_err(|_| Error::CorruptProjectionRecord)
   1379 }
   1380 
   1381 fn map_corrupt(_: sqlx::Error) -> Error {
   1382     Error::CorruptProjectionRecord
   1383 }
   1384 
   1385 #[cfg(test)]
   1386 #[cfg_attr(coverage_nightly, coverage(off))]
   1387 mod tests {
   1388     use super::*;
   1389     use crate::migration::runtime::{MIGRATIONS, migration_sql};
   1390     use radroots_storage::{ProjectionStore, event::EventSequence, status::EventStoreMode};
   1391     use sqlx::sqlite::SqlitePoolOptions;
   1392 
   1393     async fn store(mode: EventStoreMode) -> SqliteStorage {
   1394         let pool = SqlitePoolOptions::new()
   1395             .max_connections(1)
   1396             .connect("sqlite::memory:")
   1397             .await
   1398             .expect("memory SQLite");
   1399         sqlx::query("PRAGMA foreign_keys = ON")
   1400             .execute(&pool)
   1401             .await
   1402             .expect("foreign keys");
   1403         for migration in MIGRATIONS {
   1404             sqlx::raw_sql(migration_sql(migration.version()).expect("registered SQL"))
   1405                 .execute(&pool)
   1406                 .await
   1407                 .expect("runtime migration");
   1408         }
   1409         sqlx::query(
   1410             "INSERT INTO radroots_runtime_source_generations (
   1411                generation, sequence_head, state, created_at_unix_ms, retired_at_unix_ms
   1412              ) VALUES (?, 0, 'active', 1, NULL)",
   1413         )
   1414         .bind([41_u8; 32].as_slice())
   1415         .execute(&pool)
   1416         .await
   1417         .expect("active source generation");
   1418         SqliteStorage::new(
   1419             pool,
   1420             SourceGeneration::new([41; 32]).expect("generation"),
   1421             mode,
   1422         )
   1423     }
   1424 
   1425     fn projection_id() -> ProjectionId {
   1426         ProjectionId::parse("trade_projection").expect("projection id")
   1427     }
   1428 
   1429     fn generation(byte: u8) -> ProjectionGeneration {
   1430         ProjectionGeneration::new([byte; 32]).expect("projection generation")
   1431     }
   1432 
   1433     fn checkpoint(
   1434         generation: ProjectionGeneration,
   1435         sequence: u64,
   1436         rows: u64,
   1437         at: u64,
   1438     ) -> ProjectionCheckpoint {
   1439         ProjectionCheckpoint::new(
   1440             projection_id(),
   1441             generation,
   1442             Some(EventPosition::new(
   1443                 SourceGeneration::new([41; 32]).expect("source generation"),
   1444                 EventSequence::new(sequence).expect("sequence"),
   1445             )),
   1446             rows,
   1447             at,
   1448         )
   1449         .expect("checkpoint")
   1450     }
   1451 
   1452     fn invalidation() -> ProjectionInvalidation {
   1453         ProjectionInvalidation::new(
   1454             projection_id(),
   1455             generation(1),
   1456             generation(2),
   1457             InvalidationReason::ProjectionGenerationChanged,
   1458             200,
   1459         )
   1460         .expect("invalidation")
   1461     }
   1462 
   1463     #[tokio::test]
   1464     async fn checkpoints_invalidation_and_rebuild_lifecycle_are_durable() {
   1465         let store = store(EventStoreMode::ReadWrite).await;
   1466         assert!(
   1467             store
   1468                 .status(projection_id())
   1469                 .await
   1470                 .expect("empty status")
   1471                 .is_none()
   1472         );
   1473         let first = store
   1474             .checkpoint(checkpoint(generation(1), 1, 10, 100))
   1475             .await
   1476             .expect("first checkpoint");
   1477         assert_eq!(first.health(), ProjectionHealth::Ready);
   1478         let advanced = store
   1479             .checkpoint(checkpoint(generation(1), 2, 20, 150))
   1480             .await
   1481             .expect("advanced checkpoint");
   1482         assert_eq!(
   1483             advanced.checkpoint().expect("checkpoint").projected_rows(),
   1484             20
   1485         );
   1486         assert_eq!(
   1487             store
   1488                 .checkpoint(checkpoint(generation(1), 1, 19, 140))
   1489                 .await,
   1490             Err(Error::ProjectionCheckpointRegression)
   1491         );
   1492 
   1493         let invalidation = invalidation();
   1494         let invalidated = store
   1495             .invalidate(invalidation.clone())
   1496             .await
   1497             .expect("invalidate");
   1498         assert_eq!(invalidated.health(), ProjectionHealth::Invalidated);
   1499         let ticket = RebuildTicket::requested(
   1500             RebuildTicketId::new([3; 16]).expect("ticket"),
   1501             invalidation,
   1502             SourceGeneration::new([41; 32]).expect("source generation"),
   1503             Some(EventPosition::new(
   1504                 SourceGeneration::new([41; 32]).expect("source generation"),
   1505                 EventSequence::new(3).expect("sequence"),
   1506             )),
   1507             RawSourceDigest::new([8; 32]),
   1508         )
   1509         .expect("ticket");
   1510         let requested = store
   1511             .request_rebuild(ticket.clone())
   1512             .await
   1513             .expect("request rebuild");
   1514         assert_eq!(
   1515             store.request_rebuild(ticket).await.expect("replay"),
   1516             requested
   1517         );
   1518         let running = store
   1519             .transition_rebuild(RebuildTransition::start(
   1520                 requested.ticket_id(),
   1521                 requested.revision(),
   1522                 210,
   1523             ))
   1524             .await
   1525             .expect("start");
   1526         let progress = store
   1527             .transition_rebuild(RebuildTransition::checkpoint(
   1528                 running.ticket_id(),
   1529                 running.revision(),
   1530                 220,
   1531                 checkpoint(generation(2), 2, 20, 220),
   1532             ))
   1533             .await
   1534             .expect("progress");
   1535         assert_eq!(
   1536             store
   1537                 .transition_rebuild(RebuildTransition::fail(
   1538                     progress.ticket_id(),
   1539                     running.revision(),
   1540                     230,
   1541                     RebuildFailure::IntegrityFailure,
   1542                 ))
   1543                 .await,
   1544             Err(Error::ProjectionRevisionConflict)
   1545         );
   1546         sqlx::query(
   1547             "UPDATE radroots_runtime_source_generations SET sequence_head = 3 WHERE state = 'active'",
   1548         )
   1549         .execute(store.pool())
   1550         .await
   1551         .expect("advance source high water");
   1552         let completed = store
   1553             .transition_rebuild(RebuildTransition::complete(
   1554                 progress.ticket_id(),
   1555                 progress.revision(),
   1556                 240,
   1557                 checkpoint(generation(2), 3, 30, 240),
   1558             ))
   1559             .await
   1560             .expect("complete");
   1561         assert_eq!(completed.stage(), RebuildStage::Completed);
   1562         let status = store
   1563             .status(projection_id())
   1564             .await
   1565             .expect("status")
   1566             .expect("projection");
   1567         assert_eq!(status.health(), ProjectionHealth::Ready);
   1568         assert_eq!(status.generation(), generation(2));
   1569         assert_eq!(status.active_rebuild(), None);
   1570     }
   1571 
   1572     fn manifest(generation: ProjectionGeneration, digest: u8) -> EventIndexManifest {
   1573         EventIndexManifest::new(
   1574             generation,
   1575             4,
   1576             2,
   1577             10,
   1578             40,
   1579             vec![
   1580                 EventIndexShard::new(
   1581                     EventIndexShardId::parse("shard_a").expect("shard id"),
   1582                     "index/shard_a.bin",
   1583                     2,
   1584                     EventIdRange::new(EventId::from_bytes([1; 32]), EventId::from_bytes([2; 32]))
   1585                         .expect("range"),
   1586                     10,
   1587                     20,
   1588                     ArtifactDigest::new([digest; 32]),
   1589                 )
   1590                 .expect("first shard"),
   1591                 EventIndexShard::new(
   1592                     EventIndexShardId::parse("shard_b").expect("shard id"),
   1593                     "index/shard_b.bin",
   1594                     2,
   1595                     EventIdRange::new(EventId::from_bytes([3; 32]), EventId::from_bytes([4; 32]))
   1596                         .expect("range"),
   1597                     30,
   1598                     40,
   1599                     ArtifactDigest::new([digest.wrapping_add(1); 32]),
   1600                 )
   1601                 .expect("second shard"),
   1602             ],
   1603         )
   1604         .expect("manifest")
   1605     }
   1606 
   1607     fn index_checkpoint(generation: ProjectionGeneration, at: u64) -> EventIndexCheckpoint {
   1608         EventIndexCheckpoint::new(
   1609             generation,
   1610             at,
   1611             vec![
   1612                 EventIndexShardCheckpoint::new(
   1613                     EventIndexShardId::parse("shard_b").expect("shard id"),
   1614                     40,
   1615                     Some(EventId::from_bytes([4; 32])),
   1616                     Some("cursor-b".to_owned()),
   1617                 )
   1618                 .expect("second checkpoint"),
   1619                 EventIndexShardCheckpoint::new(
   1620                     EventIndexShardId::parse("shard_a").expect("shard id"),
   1621                     20,
   1622                     Some(EventId::from_bytes([2; 32])),
   1623                     None,
   1624                 )
   1625                 .expect("first checkpoint"),
   1626             ],
   1627         )
   1628         .expect("index checkpoint")
   1629     }
   1630 
   1631     #[tokio::test]
   1632     async fn event_index_manifest_and_checkpoint_round_trip_and_reject_regression() {
   1633         let store = store(EventStoreMode::ReadWrite).await;
   1634         let generation = generation(7);
   1635         let expected_manifest = manifest(generation, 8);
   1636         store
   1637             .put_event_index_manifest(expected_manifest.clone())
   1638             .await
   1639             .expect("put manifest");
   1640         store
   1641             .put_event_index_manifest(expected_manifest.clone())
   1642             .await
   1643             .expect("manifest replay");
   1644         assert_eq!(
   1645             store
   1646                 .event_index_manifest(generation)
   1647                 .await
   1648                 .expect("manifest lookup")
   1649                 .expect("manifest"),
   1650             expected_manifest
   1651         );
   1652         assert_eq!(
   1653             store
   1654                 .put_event_index_manifest(manifest(generation, 9))
   1655                 .await,
   1656             Err(Error::CorruptProjectionRecord)
   1657         );
   1658 
   1659         let first = index_checkpoint(generation, 100);
   1660         store
   1661             .put_event_index_checkpoint(first.clone())
   1662             .await
   1663             .expect("put checkpoint");
   1664         assert_eq!(
   1665             store
   1666                 .event_index_checkpoint(generation)
   1667                 .await
   1668                 .expect("checkpoint lookup")
   1669                 .expect("checkpoint"),
   1670             first
   1671         );
   1672         store
   1673             .put_event_index_checkpoint(index_checkpoint(generation, 110))
   1674             .await
   1675             .expect("advance checkpoint");
   1676         assert_eq!(
   1677             store
   1678                 .put_event_index_checkpoint(index_checkpoint(generation, 109))
   1679                 .await,
   1680             Err(Error::InvalidEventIndexCheckpoint)
   1681         );
   1682         for corrupt in [&[0_u8][..], &[1_u8, 0xff, 0xff][..]] {
   1683             sqlx::query(
   1684                 "UPDATE radroots_runtime_event_index_checkpoints
   1685                  SET checkpoint = ? WHERE projection_generation = ?",
   1686             )
   1687             .bind(corrupt)
   1688             .bind(generation.as_bytes().as_slice())
   1689             .execute(store.pool())
   1690             .await
   1691             .expect("forge corrupt index checkpoint");
   1692             assert_eq!(
   1693                 store.event_index_checkpoint(generation).await,
   1694                 Err(Error::CorruptProjectionRecord)
   1695             );
   1696         }
   1697     }
   1698 
   1699     #[tokio::test]
   1700     async fn materialized_documents_and_frozen_snapshots_round_trip_and_fail_closed() {
   1701         let writer = store(EventStoreMode::ReadWrite).await;
   1702         let id = projection_id();
   1703         let generation = generation(12);
   1704         let first = ProjectionDocument::new("context.alpha".into(), b"{\"cards\":[]}".to_vec())
   1705             .expect("document");
   1706         writer
   1707             .put_projection_document(id.clone(), generation, first)
   1708             .await
   1709             .expect("put document");
   1710         assert_eq!(
   1711             writer
   1712                 .projection_document(id.clone(), generation, "context.alpha".into(),)
   1713                 .await
   1714                 .expect("document lookup")
   1715                 .expect("document")
   1716                 .value(),
   1717             b"{\"cards\":[]}"
   1718         );
   1719         writer
   1720             .put_projection_document(
   1721                 id.clone(),
   1722                 generation,
   1723                 ProjectionDocument::new("context.alpha".into(), b"{\"cards\":[1]}".to_vec())
   1724                     .expect("replacement"),
   1725             )
   1726             .await
   1727             .expect("replace document");
   1728         assert_eq!(
   1729             writer
   1730                 .projection_document(id.clone(), generation, "context.alpha".into(),)
   1731                 .await
   1732                 .expect("document lookup")
   1733                 .expect("document")
   1734                 .value(),
   1735             b"{\"cards\":[1]}"
   1736         );
   1737 
   1738         let snapshot = ProjectionSnapshot::new(
   1739             id.clone(),
   1740             [13; 32],
   1741             generation,
   1742             1_000,
   1743             b"{\"frozen\":true}".to_vec(),
   1744         )
   1745         .expect("snapshot");
   1746         writer
   1747             .put_projection_snapshot(snapshot.clone())
   1748             .await
   1749             .expect("put snapshot");
   1750         writer
   1751             .put_projection_snapshot(snapshot.clone())
   1752             .await
   1753             .expect("idempotent snapshot replay");
   1754         let concurrent_snapshot = ProjectionSnapshot::new(
   1755             id.clone(),
   1756             [14; 32],
   1757             generation,
   1758             1_001,
   1759             b"{\"frozen\":\"concurrent\"}".to_vec(),
   1760         )
   1761         .expect("concurrent snapshot");
   1762         let (left, right) = tokio::join!(
   1763             writer.put_projection_snapshot(concurrent_snapshot.clone()),
   1764             writer.put_projection_snapshot(concurrent_snapshot),
   1765         );
   1766         left.expect("concurrent left snapshot insert");
   1767         right.expect("concurrent right snapshot insert");
   1768         assert_eq!(
   1769             writer
   1770                 .projection_snapshot(id.clone(), [13; 32])
   1771                 .await
   1772                 .expect("snapshot lookup"),
   1773             Some(snapshot)
   1774         );
   1775         assert_eq!(
   1776             writer
   1777                 .put_projection_snapshot(
   1778                     ProjectionSnapshot::new(
   1779                         id.clone(),
   1780                         [13; 32],
   1781                         generation,
   1782                         1_000,
   1783                         b"{\"frozen\":false}".to_vec(),
   1784                     )
   1785                     .expect("conflicting snapshot"),
   1786                 )
   1787                 .await,
   1788             Err(Error::CorruptProjectionDocument)
   1789         );
   1790 
   1791         sqlx::query(
   1792             "UPDATE radroots_runtime_projection_documents
   1793              SET value = X'00' WHERE projection_id = ?",
   1794         )
   1795         .bind(id.as_str())
   1796         .execute(writer.pool())
   1797         .await
   1798         .expect("forge corrupt document");
   1799         assert_eq!(
   1800             writer
   1801                 .projection_document(id, generation, "context.alpha".into())
   1802                 .await,
   1803             Err(Error::CorruptProjectionDocument)
   1804         );
   1805 
   1806         let read_only = store(EventStoreMode::ReadOnly).await;
   1807         assert_eq!(
   1808             read_only
   1809                 .put_projection_document(
   1810                     projection_id(),
   1811                     generation,
   1812                     ProjectionDocument::new("context.alpha".into(), vec![1]).expect("document"),
   1813                 )
   1814                 .await,
   1815             Err(Error::BackendUnavailable)
   1816         );
   1817     }
   1818 
   1819     #[tokio::test]
   1820     async fn failed_rebuild_corruption_and_read_only_mode_fail_closed() {
   1821         let store = store(EventStoreMode::ReadWrite).await;
   1822         let initial = store
   1823             .checkpoint(checkpoint(generation(1), 1, 1, 100))
   1824             .await
   1825             .expect("checkpoint");
   1826         let encoded = encode_status_snapshot(&initial).expect("encode projection status");
   1827         for end in 0..encoded.len() {
   1828             let _ = decode_status_snapshot(&encoded[..end]);
   1829         }
   1830         let mut trailing = encoded.clone();
   1831         trailing.push(0);
   1832         assert_eq!(
   1833             decode_status_snapshot(&trailing),
   1834             Err(Error::CorruptProjectionRecord)
   1835         );
   1836         for index in 0..encoded.len() {
   1837             let mut corrupt = encoded.clone();
   1838             corrupt[index] ^= 0xff;
   1839             let _ = decode_status_snapshot(&corrupt);
   1840         }
   1841         let invalidation = invalidation();
   1842         store
   1843             .invalidate(invalidation.clone())
   1844             .await
   1845             .expect("invalidate");
   1846         let ticket = store
   1847             .request_rebuild(
   1848                 RebuildTicket::requested(
   1849                     RebuildTicketId::new([9; 16]).expect("ticket"),
   1850                     invalidation,
   1851                     SourceGeneration::new([41; 32]).expect("source generation"),
   1852                     None,
   1853                     RawSourceDigest::new([8; 32]),
   1854                 )
   1855                 .expect("ticket"),
   1856             )
   1857             .await
   1858             .expect("request rebuild");
   1859         let failed = store
   1860             .transition_rebuild(RebuildTransition::fail(
   1861                 ticket.ticket_id(),
   1862                 ticket.revision(),
   1863                 210,
   1864                 RebuildFailure::IntegrityFailure,
   1865             ))
   1866             .await
   1867             .expect("fail rebuild");
   1868         assert_eq!(failed.stage(), RebuildStage::Failed);
   1869         assert_eq!(
   1870             store
   1871                 .status(projection_id())
   1872                 .await
   1873                 .expect("status")
   1874                 .expect("projection")
   1875                 .health(),
   1876             ProjectionHealth::Ready
   1877         );
   1878 
   1879         sqlx::query("PRAGMA ignore_check_constraints = ON")
   1880             .execute(store.pool())
   1881             .await
   1882             .expect("disable constraints");
   1883         sqlx::query(
   1884             "UPDATE radroots_runtime_projection_checkpoints
   1885              SET projection_generation = X'01' WHERE projection_id = ?",
   1886         )
   1887         .bind(projection_id().as_str())
   1888         .execute(store.pool())
   1889         .await
   1890         .expect("forge corruption");
   1891         assert_eq!(
   1892             store.status(projection_id()).await,
   1893             Err(Error::CorruptProjectionRecord)
   1894         );
   1895 
   1896         let read_only = SqliteStorage::new(
   1897             store.pool().clone(),
   1898             SourceGeneration::new([41; 32]).expect("generation"),
   1899             EventStoreMode::ReadOnly,
   1900         );
   1901         assert_eq!(
   1902             read_only
   1903                 .checkpoint(checkpoint(generation(3), 1, 1, 300))
   1904                 .await,
   1905             Err(Error::BackendUnavailable)
   1906         );
   1907     }
   1908 }