projection.rs (23237B)
1 //! Projection refresh and rebuild orchestration. 2 3 use radroots_storage::{ 4 Error as StorageError, ProjectionStore, 5 event::{ 6 AdmissionStage, EVENT_QUERY_LIMIT_MAX, EventPosition, EventQuery, EventQueryBounds, 7 SourceGeneration, StoredVisibleEvent, 8 }, 9 projection::{ 10 InvalidationReason, ProjectionCheckpoint, ProjectionGeneration, ProjectionHealth, 11 ProjectionId, ProjectionInvalidation, ProjectionRevision, ProjectionStatus, 12 RawSourceDigest, RebuildFailure, RebuildStage, RebuildTicket, RebuildTicketId, 13 RebuildTransition, 14 }, 15 }; 16 use sha2::{Digest, Sha256}; 17 18 use crate::{ 19 Engine, 20 policy::{Error, OperationKind}, 21 }; 22 23 /// Maximum number of reducer batches in one explicit refresh call. 24 pub const PROJECTION_REFRESH_MAX_BATCHES: u16 = 1_000; 25 /// Maximum canonical raw events included in one rebuild source preflight. 26 pub const PROJECTION_RAW_SOURCE_MAX_EVENTS: u64 = 1_000_000; 27 const RAW_SOURCE_DIGEST_DOMAIN: &[u8] = b"radroots:projection:raw-source:v1\0"; 28 29 /// Bounded refresh request for one exact reducer generation. 30 #[cfg_attr(feature = "serde", derive(serde::Serialize))] 31 #[derive(Clone, Debug, Eq, PartialEq)] 32 pub struct RefreshRequest { 33 projection_id: ProjectionId, 34 generation: ProjectionGeneration, 35 batch_limit: u16, 36 max_batches: u16, 37 } 38 39 impl RefreshRequest { 40 pub fn new( 41 projection_id: ProjectionId, 42 generation: ProjectionGeneration, 43 batch_limit: u16, 44 max_batches: u16, 45 ) -> Result<Self, Error> { 46 if batch_limit == 0 47 || batch_limit > EVENT_QUERY_LIMIT_MAX 48 || max_batches == 0 49 || max_batches > PROJECTION_REFRESH_MAX_BATCHES 50 { 51 return Err(Error::InvalidProjectionRequest); 52 } 53 Ok(Self { 54 projection_id, 55 generation, 56 batch_limit, 57 max_batches, 58 }) 59 } 60 61 pub const fn projection_id(&self) -> &ProjectionId { 62 &self.projection_id 63 } 64 65 pub const fn generation(&self) -> ProjectionGeneration { 66 self.generation 67 } 68 69 pub const fn batch_limit(&self) -> u16 { 70 self.batch_limit 71 } 72 73 pub const fn max_batches(&self) -> u16 { 74 self.max_batches 75 } 76 } 77 78 #[cfg(feature = "serde")] 79 impl<'de> serde::Deserialize<'de> for RefreshRequest { 80 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> 81 where 82 D: serde::Deserializer<'de>, 83 { 84 #[derive(serde::Deserialize)] 85 struct Wire { 86 projection_id: ProjectionId, 87 generation: ProjectionGeneration, 88 batch_limit: u16, 89 max_batches: u16, 90 } 91 92 let wire = Wire::deserialize(deserializer)?; 93 Self::new( 94 wire.projection_id, 95 wire.generation, 96 wire.batch_limit, 97 wire.max_batches, 98 ) 99 .map_err(serde::de::Error::custom) 100 } 101 } 102 103 /// Owning-domain deterministic reducer capability. 104 /// 105 /// Reducers own domain semantics and projected row calculation. They receive 106 /// canonical visible events in storage order and must perform no durable 107 /// metadata mutation; sync owns the checkpoint/rebuild coordination boundary. 108 pub trait Reducer: Send + Sync { 109 fn projection_id(&self) -> &ProjectionId; 110 fn generation(&self) -> ProjectionGeneration; 111 /// Opens an isolated replacement generation. Existing readers must remain 112 /// bound to the active generation until storage promotes the ticket. 113 fn begin_rebuild( 114 &self, 115 ticket_id: RebuildTicketId, 116 source_generation: SourceGeneration, 117 source_digest: RawSourceDigest, 118 ) -> Result<(), ReducerError>; 119 fn reduce( 120 &self, 121 events: &[StoredVisibleEvent], 122 prior_projected_rows: u64, 123 rebuild_ticket: Option<RebuildTicketId>, 124 ) -> Result<u64, ReducerError>; 125 /// Discards an isolated replacement generation after durable failure. 126 fn abort_rebuild( 127 &self, 128 ticket_id: RebuildTicketId, 129 failure: RebuildFailure, 130 ) -> Result<(), ReducerError>; 131 } 132 133 /// Secret-safe reducer rejection normalized at the orchestration boundary. 134 #[derive(Clone, Copy, Debug, Eq, PartialEq)] 135 pub struct ReducerError; 136 137 /// Refresh execution class. 138 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] 139 #[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] 140 #[derive(Clone, Copy, Debug, Eq, PartialEq)] 141 pub enum RefreshKind { 142 Incremental, 143 Rebuild, 144 } 145 146 /// Deterministic state returned to the host scheduler. 147 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] 148 #[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] 149 #[derive(Clone, Copy, Debug, Eq, PartialEq)] 150 pub enum RefreshState { 151 Complete, 152 Partial, 153 Failed, 154 } 155 156 /// Normalized projection progress after one bounded refresh call. 157 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] 158 #[derive(Clone, Debug, Eq, PartialEq)] 159 pub struct RefreshReceipt { 160 kind: RefreshKind, 161 state: RefreshState, 162 batches: u16, 163 events_reduced: usize, 164 checkpoint: Option<ProjectionCheckpoint>, 165 rebuild_ticket: Option<RebuildTicketId>, 166 } 167 168 impl RefreshReceipt { 169 pub const fn kind(&self) -> RefreshKind { 170 self.kind 171 } 172 pub const fn state(&self) -> RefreshState { 173 self.state 174 } 175 pub const fn batches(&self) -> u16 { 176 self.batches 177 } 178 pub const fn events_reduced(&self) -> usize { 179 self.events_reduced 180 } 181 pub const fn checkpoint(&self) -> Option<&ProjectionCheckpoint> { 182 self.checkpoint.as_ref() 183 } 184 pub const fn rebuild_ticket(&self) -> Option<RebuildTicketId> { 185 self.rebuild_ticket 186 } 187 } 188 189 impl Engine { 190 /// Runs at most the requested number of deterministic reducer batches. 191 pub async fn refresh_projection( 192 &self, 193 request: RefreshRequest, 194 reducer: &dyn Reducer, 195 ) -> Result<RefreshReceipt, Error> { 196 if reducer.projection_id() != request.projection_id() 197 || reducer.generation() != request.generation() 198 { 199 return Err(Error::InvalidProjectionRequest); 200 } 201 let status = ProjectionStore::status(self.storage.as_ref(), request.projection_id.clone()) 202 .await 203 .map_err(map_storage_error)?; 204 let source = if status.as_ref().is_some_and(|status| { 205 status.generation() != request.generation 206 || status.health() == ProjectionHealth::Rebuilding 207 }) { 208 Some(self.raw_source_snapshot().await?) 209 } else { 210 None 211 }; 212 let mut coordination = self 213 .projection_coordination(&request, status, source.as_ref()) 214 .await?; 215 let kind = if coordination.ticket.is_some() { 216 RefreshKind::Rebuild 217 } else { 218 RefreshKind::Incremental 219 }; 220 let mut receipt = RefreshReceipt { 221 kind, 222 state: RefreshState::Partial, 223 batches: 0, 224 events_reduced: 0, 225 checkpoint: coordination.checkpoint.clone(), 226 rebuild_ticket: coordination.ticket.as_ref().map(RebuildTicket::ticket_id), 227 }; 228 229 if let Some(ticket) = coordination.ticket.as_ref() 230 && !source.is_some_and(|source| source.matches_ticket(ticket)) 231 { 232 self.fail_rebuild(ticket, reducer, RebuildFailure::SourceChanged) 233 .await?; 234 receipt.state = RefreshState::Failed; 235 return Ok(receipt); 236 } 237 238 if coordination.started 239 && let Some(ticket) = coordination.ticket.as_ref() 240 && reducer 241 .begin_rebuild( 242 ticket.ticket_id(), 243 ticket.source_generation(), 244 ticket.source_digest(), 245 ) 246 .is_err() 247 { 248 self.fail_rebuild(ticket, reducer, RebuildFailure::ReducerRejected) 249 .await?; 250 receipt.state = RefreshState::Failed; 251 return Ok(receipt); 252 } 253 254 for batch_index in 0..request.max_batches { 255 let mut bounds = 256 EventQueryBounds::first(request.batch_limit).map_err(map_storage_error)?; 257 if let Some(position) = coordination 258 .checkpoint 259 .as_ref() 260 .and_then(ProjectionCheckpoint::source_position) 261 { 262 bounds = bounds.after(position); 263 } 264 let page = self 265 .storage 266 .query_visible(EventQuery::all(bounds)) 267 .await 268 .map_err(map_storage_error)?; 269 let prior_rows = coordination 270 .checkpoint 271 .as_ref() 272 .map_or(0, ProjectionCheckpoint::projected_rows); 273 let projected_rows = if page.items().is_empty() { 274 prior_rows 275 } else { 276 match reducer.reduce( 277 page.items(), 278 prior_rows, 279 coordination.ticket.as_ref().map(RebuildTicket::ticket_id), 280 ) { 281 Ok(rows) if rows >= prior_rows => rows, 282 Ok(_) => return Err(Error::InvalidReducerOutput), 283 Err(_) => { 284 if let Some(ticket) = coordination.ticket.as_ref() { 285 self.fail_rebuild(ticket, reducer, RebuildFailure::ReducerRejected) 286 .await?; 287 } 288 receipt.state = RefreshState::Failed; 289 return Ok(receipt); 290 } 291 } 292 }; 293 let source_position = page 294 .items() 295 .last() 296 .map(StoredVisibleEvent::position) 297 .or_else(|| { 298 coordination 299 .checkpoint 300 .as_ref() 301 .and_then(ProjectionCheckpoint::source_position) 302 }); 303 let checkpoint = ProjectionCheckpoint::new( 304 request.projection_id.clone(), 305 request.generation, 306 source_position, 307 projected_rows, 308 self.clock.now_unix_ms()?, 309 ) 310 .map_err(map_storage_error)?; 311 let complete = page.items().len() < usize::from(request.batch_limit); 312 if let Some(ticket) = coordination.ticket.as_mut() { 313 if complete { 314 let current_source = self.raw_source_snapshot().await?; 315 if !current_source.matches_ticket(ticket) { 316 self.fail_rebuild(ticket, reducer, RebuildFailure::SourceChanged) 317 .await?; 318 receipt.state = RefreshState::Failed; 319 return Ok(receipt); 320 } 321 } 322 let transition = if complete { 323 RebuildTransition::complete( 324 ticket.ticket_id(), 325 ticket.revision(), 326 checkpoint.updated_at_unix_ms(), 327 checkpoint.clone(), 328 ) 329 } else { 330 RebuildTransition::checkpoint( 331 ticket.ticket_id(), 332 ticket.revision(), 333 checkpoint.updated_at_unix_ms(), 334 checkpoint.clone(), 335 ) 336 }; 337 match self.storage.transition_rebuild(transition).await { 338 Ok(next) => *ticket = next, 339 Err(StorageError::SourceGenerationChanged) if complete => { 340 self.fail_rebuild(ticket, reducer, RebuildFailure::SourceChanged) 341 .await?; 342 receipt.state = RefreshState::Failed; 343 return Ok(receipt); 344 } 345 Err(error) => return Err(map_storage_error(error)), 346 } 347 } else { 348 self.storage 349 .checkpoint(checkpoint.clone()) 350 .await 351 .map_err(map_storage_error)?; 352 } 353 receipt.batches += 1; 354 receipt.events_reduced += page.items().len(); 355 receipt.checkpoint = Some(checkpoint.clone()); 356 coordination.checkpoint = Some(checkpoint); 357 if complete { 358 receipt.state = RefreshState::Complete; 359 return Ok(receipt); 360 } 361 if batch_index + 1 == request.max_batches { 362 receipt.state = RefreshState::Partial; 363 return Ok(receipt); 364 } 365 } 366 unreachable!("validated refresh requests execute at least one batch") 367 } 368 369 async fn projection_coordination( 370 &self, 371 request: &RefreshRequest, 372 status: Option<ProjectionStatus>, 373 source: Option<&RawSourceSnapshot>, 374 ) -> Result<ProjectionCoordination, Error> { 375 let Some(status) = status else { 376 return Ok(ProjectionCoordination::default()); 377 }; 378 if status.generation() == request.generation && status.health() == ProjectionHealth::Ready { 379 return Ok(ProjectionCoordination { 380 checkpoint: status.checkpoint().cloned(), 381 ticket: None, 382 started: false, 383 }); 384 } 385 if status.health() == ProjectionHealth::Rebuilding { 386 let ticket_id = status.active_rebuild().ok_or(Error::StorageFailed)?; 387 let ticket = self 388 .storage 389 .rebuild(ticket_id) 390 .await 391 .map_err(map_storage_error)? 392 .ok_or(Error::StorageFailed)?; 393 if ticket.invalidation().replacement_generation() != request.generation { 394 return Err(Error::StorageConflict); 395 } 396 return Ok(ProjectionCoordination { 397 checkpoint: ticket.checkpoint().cloned(), 398 ticket: Some(ticket), 399 started: false, 400 }); 401 } 402 403 let invalidation = if status.generation() != request.generation { 404 if status.health() != ProjectionHealth::Ready { 405 return Err(Error::StorageConflict); 406 } 407 let invalidation = match self 408 .storage 409 .invalidation(request.projection_id.clone(), request.generation) 410 .await 411 .map_err(map_storage_error)? 412 { 413 Some(existing) if existing.invalid_generation() == status.generation() => existing, 414 Some(_) => return Err(Error::StorageConflict), 415 None => ProjectionInvalidation::new( 416 request.projection_id.clone(), 417 status.generation(), 418 request.generation, 419 InvalidationReason::ProjectionGenerationChanged, 420 self.clock.now_unix_ms()?, 421 ) 422 .map_err(map_storage_error)?, 423 }; 424 self.storage 425 .invalidate(invalidation.clone()) 426 .await 427 .map_err(map_storage_error)?; 428 invalidation 429 } else if status.health() == ProjectionHealth::Invalidated { 430 self.storage 431 .invalidation(request.projection_id.clone(), request.generation) 432 .await 433 .map_err(map_storage_error)? 434 .ok_or(Error::StorageFailed)? 435 } else { 436 return Err(Error::StorageConflict); 437 }; 438 let sync_id = self.ids.next_id(OperationKind::Projection)?; 439 let source = source.ok_or(Error::StorageFailed)?; 440 let ticket = RebuildTicket::requested( 441 RebuildTicketId::new(*sync_id.as_bytes()).map_err(map_storage_error)?, 442 invalidation, 443 source.generation, 444 source.high_water, 445 source.digest, 446 ) 447 .map_err(map_storage_error)?; 448 let requested = self 449 .storage 450 .request_rebuild(ticket) 451 .await 452 .map_err(map_storage_error)?; 453 let running = self 454 .storage 455 .transition_rebuild(RebuildTransition::start( 456 requested.ticket_id(), 457 ProjectionRevision::INITIAL, 458 self.clock.now_unix_ms()?, 459 )) 460 .await 461 .map_err(map_storage_error)?; 462 Ok(ProjectionCoordination { 463 checkpoint: None, 464 ticket: Some(running), 465 started: true, 466 }) 467 } 468 469 async fn raw_source_snapshot(&self) -> Result<RawSourceSnapshot, Error> { 470 let mut hasher = Sha256::new(); 471 hasher.update(RAW_SOURCE_DIGEST_DOMAIN); 472 let mut cursor = None; 473 let mut count = 0_u64; 474 let mut generation = None; 475 let mut high_water = None; 476 loop { 477 let mut bounds = 478 EventQueryBounds::first(EVENT_QUERY_LIMIT_MAX).map_err(map_storage_error)?; 479 if let Some(position) = cursor { 480 bounds = bounds.after(position); 481 } 482 let page = self 483 .storage 484 .query_raw(EventQuery::all(bounds)) 485 .await 486 .map_err(map_storage_error)?; 487 if generation 488 .replace(page.generation()) 489 .is_some_and(|prior| prior != page.generation()) 490 { 491 return Err(Error::StorageConflict); 492 } 493 hasher.update(page.generation().as_bytes()); 494 for event in page.items() { 495 count = count.checked_add(1).ok_or(Error::StorageFailed)?; 496 if count > PROJECTION_RAW_SOURCE_MAX_EVENTS { 497 return Err(Error::InvalidProjectionRequest); 498 } 499 let position = event.position(); 500 hasher.update(position.sequence().get().to_be_bytes()); 501 hasher.update([admission_stage_byte(event.stage())]); 502 let raw = event.event().raw_json().as_bytes(); 503 hasher.update( 504 u64::try_from(raw.len()) 505 .map_err(|_| Error::StorageFailed)? 506 .to_be_bytes(), 507 ); 508 hasher.update(raw); 509 high_water = Some(position); 510 } 511 cursor = page.next_cursor(); 512 if cursor.is_none() { 513 break; 514 } 515 } 516 Ok(RawSourceSnapshot { 517 generation: generation.ok_or(Error::StorageFailed)?, 518 high_water, 519 digest: RawSourceDigest::new(hasher.finalize().into()), 520 }) 521 } 522 523 async fn fail_rebuild( 524 &self, 525 ticket: &RebuildTicket, 526 reducer: &dyn Reducer, 527 failure: RebuildFailure, 528 ) -> Result<(), Error> { 529 let failed = self 530 .storage 531 .transition_rebuild(RebuildTransition::fail( 532 ticket.ticket_id(), 533 ticket.revision(), 534 self.clock.now_unix_ms()?, 535 failure, 536 )) 537 .await 538 .map_err(map_storage_error)?; 539 debug_assert_eq!(failed.stage(), RebuildStage::Failed); 540 reducer 541 .abort_rebuild(ticket.ticket_id(), failure) 542 .map_err(|_| Error::InvalidReducerOutput) 543 } 544 } 545 546 #[derive(Default)] 547 struct ProjectionCoordination { 548 checkpoint: Option<ProjectionCheckpoint>, 549 ticket: Option<RebuildTicket>, 550 started: bool, 551 } 552 553 #[derive(Clone, Copy)] 554 struct RawSourceSnapshot { 555 generation: SourceGeneration, 556 high_water: Option<EventPosition>, 557 digest: RawSourceDigest, 558 } 559 560 impl RawSourceSnapshot { 561 fn matches_ticket(self, ticket: &RebuildTicket) -> bool { 562 self.generation == ticket.source_generation() 563 && self.high_water == ticket.source_high_water() 564 && self.digest == ticket.source_digest() 565 } 566 } 567 568 const fn admission_stage_byte(stage: AdmissionStage) -> u8 { 569 match stage { 570 AdmissionStage::Raw => 0, 571 AdmissionStage::Verified => 1, 572 AdmissionStage::Visible => 2, 573 } 574 } 575 576 fn map_storage_error(error: StorageError) -> Error { 577 match error { 578 StorageError::SpaceInsufficient => Error::StorageSpaceInsufficient, 579 StorageError::ProjectionCheckpointMismatch 580 | StorageError::ProjectionCheckpointRegression 581 | StorageError::ProjectionRevisionConflict 582 | StorageError::SourceGenerationChanged => Error::StorageConflict, 583 _ => Error::StorageFailed, 584 } 585 } 586 587 #[cfg(test)] 588 #[cfg_attr(coverage_nightly, coverage(off))] 589 mod tests { 590 use super::*; 591 592 #[test] 593 fn raw_source_identity_stage_encoding_and_error_mapping_are_exact() { 594 assert_eq!( 595 map_storage_error(StorageError::SpaceInsufficient), 596 Error::StorageSpaceInsufficient 597 ); 598 let invalidation = ProjectionInvalidation::new( 599 ProjectionId::parse("projection-helper").unwrap(), 600 ProjectionGeneration::new([1; 32]).unwrap(), 601 ProjectionGeneration::new([2; 32]).unwrap(), 602 InvalidationReason::ProjectionGenerationChanged, 603 1, 604 ) 605 .unwrap(); 606 let source_generation = SourceGeneration::new([11; 32]).unwrap(); 607 let digest = RawSourceDigest::new([12; 32]); 608 let ticket = RebuildTicket::requested( 609 RebuildTicketId::new([13; 16]).unwrap(), 610 invalidation, 611 source_generation, 612 None, 613 digest, 614 ) 615 .unwrap(); 616 let matching = RawSourceSnapshot { 617 generation: source_generation, 618 high_water: None, 619 digest, 620 }; 621 assert!(matching.matches_ticket(&ticket)); 622 assert!( 623 !RawSourceSnapshot { 624 generation: SourceGeneration::new([14; 32]).unwrap(), 625 ..matching 626 } 627 .matches_ticket(&ticket) 628 ); 629 assert!( 630 !RawSourceSnapshot { 631 high_water: Some(EventPosition::new( 632 source_generation, 633 radroots_storage::event::EventSequence::new(1).unwrap(), 634 )), 635 ..matching 636 } 637 .matches_ticket(&ticket) 638 ); 639 assert!( 640 !RawSourceSnapshot { 641 digest: RawSourceDigest::new([15; 32]), 642 ..matching 643 } 644 .matches_ticket(&ticket) 645 ); 646 assert_eq!(admission_stage_byte(AdmissionStage::Raw), 0); 647 assert_eq!(admission_stage_byte(AdmissionStage::Verified), 1); 648 assert_eq!(admission_stage_byte(AdmissionStage::Visible), 2); 649 assert_eq!( 650 map_storage_error(StorageError::ProjectionRevisionConflict), 651 Error::StorageConflict 652 ); 653 assert_eq!( 654 map_storage_error(StorageError::BackendUnavailable), 655 Error::StorageFailed 656 ); 657 } 658 }