lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

projection.rs (23237B)


      1 //! Projection refresh and rebuild orchestration.
      2 
      3 use radroots_storage::{
      4     Error as StorageError, ProjectionStore,
      5     event::{
      6         AdmissionStage, EVENT_QUERY_LIMIT_MAX, EventPosition, EventQuery, EventQueryBounds,
      7         SourceGeneration, StoredVisibleEvent,
      8     },
      9     projection::{
     10         InvalidationReason, ProjectionCheckpoint, ProjectionGeneration, ProjectionHealth,
     11         ProjectionId, ProjectionInvalidation, ProjectionRevision, ProjectionStatus,
     12         RawSourceDigest, RebuildFailure, RebuildStage, RebuildTicket, RebuildTicketId,
     13         RebuildTransition,
     14     },
     15 };
     16 use sha2::{Digest, Sha256};
     17 
     18 use crate::{
     19     Engine,
     20     policy::{Error, OperationKind},
     21 };
     22 
     23 /// Maximum number of reducer batches in one explicit refresh call.
     24 pub const PROJECTION_REFRESH_MAX_BATCHES: u16 = 1_000;
     25 /// Maximum canonical raw events included in one rebuild source preflight.
     26 pub const PROJECTION_RAW_SOURCE_MAX_EVENTS: u64 = 1_000_000;
     27 const RAW_SOURCE_DIGEST_DOMAIN: &[u8] = b"radroots:projection:raw-source:v1\0";
     28 
     29 /// Bounded refresh request for one exact reducer generation.
     30 #[cfg_attr(feature = "serde", derive(serde::Serialize))]
     31 #[derive(Clone, Debug, Eq, PartialEq)]
     32 pub struct RefreshRequest {
     33     projection_id: ProjectionId,
     34     generation: ProjectionGeneration,
     35     batch_limit: u16,
     36     max_batches: u16,
     37 }
     38 
     39 impl RefreshRequest {
     40     pub fn new(
     41         projection_id: ProjectionId,
     42         generation: ProjectionGeneration,
     43         batch_limit: u16,
     44         max_batches: u16,
     45     ) -> Result<Self, Error> {
     46         if batch_limit == 0
     47             || batch_limit > EVENT_QUERY_LIMIT_MAX
     48             || max_batches == 0
     49             || max_batches > PROJECTION_REFRESH_MAX_BATCHES
     50         {
     51             return Err(Error::InvalidProjectionRequest);
     52         }
     53         Ok(Self {
     54             projection_id,
     55             generation,
     56             batch_limit,
     57             max_batches,
     58         })
     59     }
     60 
     61     pub const fn projection_id(&self) -> &ProjectionId {
     62         &self.projection_id
     63     }
     64 
     65     pub const fn generation(&self) -> ProjectionGeneration {
     66         self.generation
     67     }
     68 
     69     pub const fn batch_limit(&self) -> u16 {
     70         self.batch_limit
     71     }
     72 
     73     pub const fn max_batches(&self) -> u16 {
     74         self.max_batches
     75     }
     76 }
     77 
     78 #[cfg(feature = "serde")]
     79 impl<'de> serde::Deserialize<'de> for RefreshRequest {
     80     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     81     where
     82         D: serde::Deserializer<'de>,
     83     {
     84         #[derive(serde::Deserialize)]
     85         struct Wire {
     86             projection_id: ProjectionId,
     87             generation: ProjectionGeneration,
     88             batch_limit: u16,
     89             max_batches: u16,
     90         }
     91 
     92         let wire = Wire::deserialize(deserializer)?;
     93         Self::new(
     94             wire.projection_id,
     95             wire.generation,
     96             wire.batch_limit,
     97             wire.max_batches,
     98         )
     99         .map_err(serde::de::Error::custom)
    100     }
    101 }
    102 
    103 /// Owning-domain deterministic reducer capability.
    104 ///
    105 /// Reducers own domain semantics and projected row calculation. They receive
    106 /// canonical visible events in storage order and must perform no durable
    107 /// metadata mutation; sync owns the checkpoint/rebuild coordination boundary.
    108 pub trait Reducer: Send + Sync {
    109     fn projection_id(&self) -> &ProjectionId;
    110     fn generation(&self) -> ProjectionGeneration;
    111     /// Opens an isolated replacement generation. Existing readers must remain
    112     /// bound to the active generation until storage promotes the ticket.
    113     fn begin_rebuild(
    114         &self,
    115         ticket_id: RebuildTicketId,
    116         source_generation: SourceGeneration,
    117         source_digest: RawSourceDigest,
    118     ) -> Result<(), ReducerError>;
    119     fn reduce(
    120         &self,
    121         events: &[StoredVisibleEvent],
    122         prior_projected_rows: u64,
    123         rebuild_ticket: Option<RebuildTicketId>,
    124     ) -> Result<u64, ReducerError>;
    125     /// Discards an isolated replacement generation after durable failure.
    126     fn abort_rebuild(
    127         &self,
    128         ticket_id: RebuildTicketId,
    129         failure: RebuildFailure,
    130     ) -> Result<(), ReducerError>;
    131 }
    132 
    133 /// Secret-safe reducer rejection normalized at the orchestration boundary.
    134 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
    135 pub struct ReducerError;
    136 
    137 /// Refresh execution class.
    138 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
    139 #[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))]
    140 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
    141 pub enum RefreshKind {
    142     Incremental,
    143     Rebuild,
    144 }
    145 
    146 /// Deterministic state returned to the host scheduler.
    147 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
    148 #[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))]
    149 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
    150 pub enum RefreshState {
    151     Complete,
    152     Partial,
    153     Failed,
    154 }
    155 
    156 /// Normalized projection progress after one bounded refresh call.
    157 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
    158 #[derive(Clone, Debug, Eq, PartialEq)]
    159 pub struct RefreshReceipt {
    160     kind: RefreshKind,
    161     state: RefreshState,
    162     batches: u16,
    163     events_reduced: usize,
    164     checkpoint: Option<ProjectionCheckpoint>,
    165     rebuild_ticket: Option<RebuildTicketId>,
    166 }
    167 
    168 impl RefreshReceipt {
    169     pub const fn kind(&self) -> RefreshKind {
    170         self.kind
    171     }
    172     pub const fn state(&self) -> RefreshState {
    173         self.state
    174     }
    175     pub const fn batches(&self) -> u16 {
    176         self.batches
    177     }
    178     pub const fn events_reduced(&self) -> usize {
    179         self.events_reduced
    180     }
    181     pub const fn checkpoint(&self) -> Option<&ProjectionCheckpoint> {
    182         self.checkpoint.as_ref()
    183     }
    184     pub const fn rebuild_ticket(&self) -> Option<RebuildTicketId> {
    185         self.rebuild_ticket
    186     }
    187 }
    188 
    189 impl Engine {
    190     /// Runs at most the requested number of deterministic reducer batches.
    191     pub async fn refresh_projection(
    192         &self,
    193         request: RefreshRequest,
    194         reducer: &dyn Reducer,
    195     ) -> Result<RefreshReceipt, Error> {
    196         if reducer.projection_id() != request.projection_id()
    197             || reducer.generation() != request.generation()
    198         {
    199             return Err(Error::InvalidProjectionRequest);
    200         }
    201         let status = ProjectionStore::status(self.storage.as_ref(), request.projection_id.clone())
    202             .await
    203             .map_err(map_storage_error)?;
    204         let source = if status.as_ref().is_some_and(|status| {
    205             status.generation() != request.generation
    206                 || status.health() == ProjectionHealth::Rebuilding
    207         }) {
    208             Some(self.raw_source_snapshot().await?)
    209         } else {
    210             None
    211         };
    212         let mut coordination = self
    213             .projection_coordination(&request, status, source.as_ref())
    214             .await?;
    215         let kind = if coordination.ticket.is_some() {
    216             RefreshKind::Rebuild
    217         } else {
    218             RefreshKind::Incremental
    219         };
    220         let mut receipt = RefreshReceipt {
    221             kind,
    222             state: RefreshState::Partial,
    223             batches: 0,
    224             events_reduced: 0,
    225             checkpoint: coordination.checkpoint.clone(),
    226             rebuild_ticket: coordination.ticket.as_ref().map(RebuildTicket::ticket_id),
    227         };
    228 
    229         if let Some(ticket) = coordination.ticket.as_ref()
    230             && !source.is_some_and(|source| source.matches_ticket(ticket))
    231         {
    232             self.fail_rebuild(ticket, reducer, RebuildFailure::SourceChanged)
    233                 .await?;
    234             receipt.state = RefreshState::Failed;
    235             return Ok(receipt);
    236         }
    237 
    238         if coordination.started
    239             && let Some(ticket) = coordination.ticket.as_ref()
    240             && reducer
    241                 .begin_rebuild(
    242                     ticket.ticket_id(),
    243                     ticket.source_generation(),
    244                     ticket.source_digest(),
    245                 )
    246                 .is_err()
    247         {
    248             self.fail_rebuild(ticket, reducer, RebuildFailure::ReducerRejected)
    249                 .await?;
    250             receipt.state = RefreshState::Failed;
    251             return Ok(receipt);
    252         }
    253 
    254         for batch_index in 0..request.max_batches {
    255             let mut bounds =
    256                 EventQueryBounds::first(request.batch_limit).map_err(map_storage_error)?;
    257             if let Some(position) = coordination
    258                 .checkpoint
    259                 .as_ref()
    260                 .and_then(ProjectionCheckpoint::source_position)
    261             {
    262                 bounds = bounds.after(position);
    263             }
    264             let page = self
    265                 .storage
    266                 .query_visible(EventQuery::all(bounds))
    267                 .await
    268                 .map_err(map_storage_error)?;
    269             let prior_rows = coordination
    270                 .checkpoint
    271                 .as_ref()
    272                 .map_or(0, ProjectionCheckpoint::projected_rows);
    273             let projected_rows = if page.items().is_empty() {
    274                 prior_rows
    275             } else {
    276                 match reducer.reduce(
    277                     page.items(),
    278                     prior_rows,
    279                     coordination.ticket.as_ref().map(RebuildTicket::ticket_id),
    280                 ) {
    281                     Ok(rows) if rows >= prior_rows => rows,
    282                     Ok(_) => return Err(Error::InvalidReducerOutput),
    283                     Err(_) => {
    284                         if let Some(ticket) = coordination.ticket.as_ref() {
    285                             self.fail_rebuild(ticket, reducer, RebuildFailure::ReducerRejected)
    286                                 .await?;
    287                         }
    288                         receipt.state = RefreshState::Failed;
    289                         return Ok(receipt);
    290                     }
    291                 }
    292             };
    293             let source_position = page
    294                 .items()
    295                 .last()
    296                 .map(StoredVisibleEvent::position)
    297                 .or_else(|| {
    298                     coordination
    299                         .checkpoint
    300                         .as_ref()
    301                         .and_then(ProjectionCheckpoint::source_position)
    302                 });
    303             let checkpoint = ProjectionCheckpoint::new(
    304                 request.projection_id.clone(),
    305                 request.generation,
    306                 source_position,
    307                 projected_rows,
    308                 self.clock.now_unix_ms()?,
    309             )
    310             .map_err(map_storage_error)?;
    311             let complete = page.items().len() < usize::from(request.batch_limit);
    312             if let Some(ticket) = coordination.ticket.as_mut() {
    313                 if complete {
    314                     let current_source = self.raw_source_snapshot().await?;
    315                     if !current_source.matches_ticket(ticket) {
    316                         self.fail_rebuild(ticket, reducer, RebuildFailure::SourceChanged)
    317                             .await?;
    318                         receipt.state = RefreshState::Failed;
    319                         return Ok(receipt);
    320                     }
    321                 }
    322                 let transition = if complete {
    323                     RebuildTransition::complete(
    324                         ticket.ticket_id(),
    325                         ticket.revision(),
    326                         checkpoint.updated_at_unix_ms(),
    327                         checkpoint.clone(),
    328                     )
    329                 } else {
    330                     RebuildTransition::checkpoint(
    331                         ticket.ticket_id(),
    332                         ticket.revision(),
    333                         checkpoint.updated_at_unix_ms(),
    334                         checkpoint.clone(),
    335                     )
    336                 };
    337                 match self.storage.transition_rebuild(transition).await {
    338                     Ok(next) => *ticket = next,
    339                     Err(StorageError::SourceGenerationChanged) if complete => {
    340                         self.fail_rebuild(ticket, reducer, RebuildFailure::SourceChanged)
    341                             .await?;
    342                         receipt.state = RefreshState::Failed;
    343                         return Ok(receipt);
    344                     }
    345                     Err(error) => return Err(map_storage_error(error)),
    346                 }
    347             } else {
    348                 self.storage
    349                     .checkpoint(checkpoint.clone())
    350                     .await
    351                     .map_err(map_storage_error)?;
    352             }
    353             receipt.batches += 1;
    354             receipt.events_reduced += page.items().len();
    355             receipt.checkpoint = Some(checkpoint.clone());
    356             coordination.checkpoint = Some(checkpoint);
    357             if complete {
    358                 receipt.state = RefreshState::Complete;
    359                 return Ok(receipt);
    360             }
    361             if batch_index + 1 == request.max_batches {
    362                 receipt.state = RefreshState::Partial;
    363                 return Ok(receipt);
    364             }
    365         }
    366         unreachable!("validated refresh requests execute at least one batch")
    367     }
    368 
    369     async fn projection_coordination(
    370         &self,
    371         request: &RefreshRequest,
    372         status: Option<ProjectionStatus>,
    373         source: Option<&RawSourceSnapshot>,
    374     ) -> Result<ProjectionCoordination, Error> {
    375         let Some(status) = status else {
    376             return Ok(ProjectionCoordination::default());
    377         };
    378         if status.generation() == request.generation && status.health() == ProjectionHealth::Ready {
    379             return Ok(ProjectionCoordination {
    380                 checkpoint: status.checkpoint().cloned(),
    381                 ticket: None,
    382                 started: false,
    383             });
    384         }
    385         if status.health() == ProjectionHealth::Rebuilding {
    386             let ticket_id = status.active_rebuild().ok_or(Error::StorageFailed)?;
    387             let ticket = self
    388                 .storage
    389                 .rebuild(ticket_id)
    390                 .await
    391                 .map_err(map_storage_error)?
    392                 .ok_or(Error::StorageFailed)?;
    393             if ticket.invalidation().replacement_generation() != request.generation {
    394                 return Err(Error::StorageConflict);
    395             }
    396             return Ok(ProjectionCoordination {
    397                 checkpoint: ticket.checkpoint().cloned(),
    398                 ticket: Some(ticket),
    399                 started: false,
    400             });
    401         }
    402 
    403         let invalidation = if status.generation() != request.generation {
    404             if status.health() != ProjectionHealth::Ready {
    405                 return Err(Error::StorageConflict);
    406             }
    407             let invalidation = match self
    408                 .storage
    409                 .invalidation(request.projection_id.clone(), request.generation)
    410                 .await
    411                 .map_err(map_storage_error)?
    412             {
    413                 Some(existing) if existing.invalid_generation() == status.generation() => existing,
    414                 Some(_) => return Err(Error::StorageConflict),
    415                 None => ProjectionInvalidation::new(
    416                     request.projection_id.clone(),
    417                     status.generation(),
    418                     request.generation,
    419                     InvalidationReason::ProjectionGenerationChanged,
    420                     self.clock.now_unix_ms()?,
    421                 )
    422                 .map_err(map_storage_error)?,
    423             };
    424             self.storage
    425                 .invalidate(invalidation.clone())
    426                 .await
    427                 .map_err(map_storage_error)?;
    428             invalidation
    429         } else if status.health() == ProjectionHealth::Invalidated {
    430             self.storage
    431                 .invalidation(request.projection_id.clone(), request.generation)
    432                 .await
    433                 .map_err(map_storage_error)?
    434                 .ok_or(Error::StorageFailed)?
    435         } else {
    436             return Err(Error::StorageConflict);
    437         };
    438         let sync_id = self.ids.next_id(OperationKind::Projection)?;
    439         let source = source.ok_or(Error::StorageFailed)?;
    440         let ticket = RebuildTicket::requested(
    441             RebuildTicketId::new(*sync_id.as_bytes()).map_err(map_storage_error)?,
    442             invalidation,
    443             source.generation,
    444             source.high_water,
    445             source.digest,
    446         )
    447         .map_err(map_storage_error)?;
    448         let requested = self
    449             .storage
    450             .request_rebuild(ticket)
    451             .await
    452             .map_err(map_storage_error)?;
    453         let running = self
    454             .storage
    455             .transition_rebuild(RebuildTransition::start(
    456                 requested.ticket_id(),
    457                 ProjectionRevision::INITIAL,
    458                 self.clock.now_unix_ms()?,
    459             ))
    460             .await
    461             .map_err(map_storage_error)?;
    462         Ok(ProjectionCoordination {
    463             checkpoint: None,
    464             ticket: Some(running),
    465             started: true,
    466         })
    467     }
    468 
    469     async fn raw_source_snapshot(&self) -> Result<RawSourceSnapshot, Error> {
    470         let mut hasher = Sha256::new();
    471         hasher.update(RAW_SOURCE_DIGEST_DOMAIN);
    472         let mut cursor = None;
    473         let mut count = 0_u64;
    474         let mut generation = None;
    475         let mut high_water = None;
    476         loop {
    477             let mut bounds =
    478                 EventQueryBounds::first(EVENT_QUERY_LIMIT_MAX).map_err(map_storage_error)?;
    479             if let Some(position) = cursor {
    480                 bounds = bounds.after(position);
    481             }
    482             let page = self
    483                 .storage
    484                 .query_raw(EventQuery::all(bounds))
    485                 .await
    486                 .map_err(map_storage_error)?;
    487             if generation
    488                 .replace(page.generation())
    489                 .is_some_and(|prior| prior != page.generation())
    490             {
    491                 return Err(Error::StorageConflict);
    492             }
    493             hasher.update(page.generation().as_bytes());
    494             for event in page.items() {
    495                 count = count.checked_add(1).ok_or(Error::StorageFailed)?;
    496                 if count > PROJECTION_RAW_SOURCE_MAX_EVENTS {
    497                     return Err(Error::InvalidProjectionRequest);
    498                 }
    499                 let position = event.position();
    500                 hasher.update(position.sequence().get().to_be_bytes());
    501                 hasher.update([admission_stage_byte(event.stage())]);
    502                 let raw = event.event().raw_json().as_bytes();
    503                 hasher.update(
    504                     u64::try_from(raw.len())
    505                         .map_err(|_| Error::StorageFailed)?
    506                         .to_be_bytes(),
    507                 );
    508                 hasher.update(raw);
    509                 high_water = Some(position);
    510             }
    511             cursor = page.next_cursor();
    512             if cursor.is_none() {
    513                 break;
    514             }
    515         }
    516         Ok(RawSourceSnapshot {
    517             generation: generation.ok_or(Error::StorageFailed)?,
    518             high_water,
    519             digest: RawSourceDigest::new(hasher.finalize().into()),
    520         })
    521     }
    522 
    523     async fn fail_rebuild(
    524         &self,
    525         ticket: &RebuildTicket,
    526         reducer: &dyn Reducer,
    527         failure: RebuildFailure,
    528     ) -> Result<(), Error> {
    529         let failed = self
    530             .storage
    531             .transition_rebuild(RebuildTransition::fail(
    532                 ticket.ticket_id(),
    533                 ticket.revision(),
    534                 self.clock.now_unix_ms()?,
    535                 failure,
    536             ))
    537             .await
    538             .map_err(map_storage_error)?;
    539         debug_assert_eq!(failed.stage(), RebuildStage::Failed);
    540         reducer
    541             .abort_rebuild(ticket.ticket_id(), failure)
    542             .map_err(|_| Error::InvalidReducerOutput)
    543     }
    544 }
    545 
    546 #[derive(Default)]
    547 struct ProjectionCoordination {
    548     checkpoint: Option<ProjectionCheckpoint>,
    549     ticket: Option<RebuildTicket>,
    550     started: bool,
    551 }
    552 
    553 #[derive(Clone, Copy)]
    554 struct RawSourceSnapshot {
    555     generation: SourceGeneration,
    556     high_water: Option<EventPosition>,
    557     digest: RawSourceDigest,
    558 }
    559 
    560 impl RawSourceSnapshot {
    561     fn matches_ticket(self, ticket: &RebuildTicket) -> bool {
    562         self.generation == ticket.source_generation()
    563             && self.high_water == ticket.source_high_water()
    564             && self.digest == ticket.source_digest()
    565     }
    566 }
    567 
    568 const fn admission_stage_byte(stage: AdmissionStage) -> u8 {
    569     match stage {
    570         AdmissionStage::Raw => 0,
    571         AdmissionStage::Verified => 1,
    572         AdmissionStage::Visible => 2,
    573     }
    574 }
    575 
    576 fn map_storage_error(error: StorageError) -> Error {
    577     match error {
    578         StorageError::SpaceInsufficient => Error::StorageSpaceInsufficient,
    579         StorageError::ProjectionCheckpointMismatch
    580         | StorageError::ProjectionCheckpointRegression
    581         | StorageError::ProjectionRevisionConflict
    582         | StorageError::SourceGenerationChanged => Error::StorageConflict,
    583         _ => Error::StorageFailed,
    584     }
    585 }
    586 
    587 #[cfg(test)]
    588 #[cfg_attr(coverage_nightly, coverage(off))]
    589 mod tests {
    590     use super::*;
    591 
    592     #[test]
    593     fn raw_source_identity_stage_encoding_and_error_mapping_are_exact() {
    594         assert_eq!(
    595             map_storage_error(StorageError::SpaceInsufficient),
    596             Error::StorageSpaceInsufficient
    597         );
    598         let invalidation = ProjectionInvalidation::new(
    599             ProjectionId::parse("projection-helper").unwrap(),
    600             ProjectionGeneration::new([1; 32]).unwrap(),
    601             ProjectionGeneration::new([2; 32]).unwrap(),
    602             InvalidationReason::ProjectionGenerationChanged,
    603             1,
    604         )
    605         .unwrap();
    606         let source_generation = SourceGeneration::new([11; 32]).unwrap();
    607         let digest = RawSourceDigest::new([12; 32]);
    608         let ticket = RebuildTicket::requested(
    609             RebuildTicketId::new([13; 16]).unwrap(),
    610             invalidation,
    611             source_generation,
    612             None,
    613             digest,
    614         )
    615         .unwrap();
    616         let matching = RawSourceSnapshot {
    617             generation: source_generation,
    618             high_water: None,
    619             digest,
    620         };
    621         assert!(matching.matches_ticket(&ticket));
    622         assert!(
    623             !RawSourceSnapshot {
    624                 generation: SourceGeneration::new([14; 32]).unwrap(),
    625                 ..matching
    626             }
    627             .matches_ticket(&ticket)
    628         );
    629         assert!(
    630             !RawSourceSnapshot {
    631                 high_water: Some(EventPosition::new(
    632                     source_generation,
    633                     radroots_storage::event::EventSequence::new(1).unwrap(),
    634                 )),
    635                 ..matching
    636             }
    637             .matches_ticket(&ticket)
    638         );
    639         assert!(
    640             !RawSourceSnapshot {
    641                 digest: RawSourceDigest::new([15; 32]),
    642                 ..matching
    643             }
    644             .matches_ticket(&ticket)
    645         );
    646         assert_eq!(admission_stage_byte(AdmissionStage::Raw), 0);
    647         assert_eq!(admission_stage_byte(AdmissionStage::Verified), 1);
    648         assert_eq!(admission_stage_byte(AdmissionStage::Visible), 2);
    649         assert_eq!(
    650             map_storage_error(StorageError::ProjectionRevisionConflict),
    651             Error::StorageConflict
    652         );
    653         assert_eq!(
    654             map_storage_error(StorageError::BackendUnavailable),
    655             Error::StorageFailed
    656         );
    657     }
    658 }