mod.rs (69600B)
1 use crate::SqliteStorage; 2 use crate::backend::map_backend; 3 use radroots_storage::{ 4 Error, ProjectionStore, 5 event::{EventPosition, SourceGeneration}, 6 projection::{ 7 ArtifactDigest, BoxFuture, EVENT_INDEX_SHARDS_MAX, EventId, EventIdRange, 8 EventIndexCheckpoint, EventIndexManifest, EventIndexShard, EventIndexShardCheckpoint, 9 EventIndexShardId, InvalidationReason, ProjectionCheckpoint, ProjectionDocument, 10 ProjectionGeneration, ProjectionHealth, ProjectionId, ProjectionInvalidation, 11 ProjectionRevision, ProjectionSnapshot, ProjectionStatus, RawSourceDigest, RebuildFailure, 12 RebuildStage, RebuildTicket, RebuildTicketId, RebuildTransition, 13 }, 14 }; 15 use sqlx::{Row, Sqlite, SqliteConnection}; 16 17 mod batch; 18 mod cache; 19 mod cache_retired; 20 mod document_query; 21 22 #[cfg_attr(coverage_nightly, coverage(off))] 23 impl ProjectionStore for SqliteStorage { 24 fn configure_projection_cache( 25 &self, 26 id: ProjectionId, 27 limits: radroots_storage::projection::cache::ProjectionCacheLimits, 28 ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheUsage, Error>> 29 { 30 Box::pin(cache::configure(self, id, limits)) 31 } 32 fn retire_projection_generation( 33 &self, 34 scope: radroots_storage::projection::cache::ProjectionCacheScope, 35 ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheUsage, Error>> 36 { 37 Box::pin(cache::retire(self, scope)) 38 } 39 fn collect_projection_cache( 40 &self, 41 scope: radroots_storage::projection::cache::ProjectionCacheScope, 42 ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheCollection, Error>> 43 { 44 Box::pin(cache::collect(self, scope)) 45 } 46 fn collect_retired_projection_cache( 47 &self, 48 id: ProjectionId, 49 source: radroots_storage::event::EventSourceToken, 50 expected_active: Option<ProjectionGeneration>, 51 ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheCollection, Error>> 52 { 53 Box::pin(cache_retired::collect(self, id, source, expected_active)) 54 } 55 fn collect_obsolete_projection_snapshots( 56 &self, 57 scope: radroots_storage::projection::cache::ProjectionCacheScope, 58 ) -> BoxFuture<'_, Result<radroots_storage::projection::cache::ProjectionCacheCollection, Error>> 59 { 60 Box::pin(cache::collect_kind( 61 self, 62 scope, 63 cache::CollectionKind::ObsoleteSnapshots, 64 )) 65 } 66 fn status( 67 &self, 68 projection_id: ProjectionId, 69 ) -> BoxFuture<'_, Result<Option<ProjectionStatus>, Error>> { 70 Box::pin(async move { 71 sqlx::query( 72 "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?", 73 ) 74 .bind(projection_id.as_str()) 75 .fetch_optional(self.pool()) 76 .await 77 .map_err(cache::map)? 78 .as_ref() 79 .map(decode_status) 80 .transpose() 81 }) 82 } 83 84 fn checkpoint( 85 &self, 86 checkpoint: ProjectionCheckpoint, 87 ) -> BoxFuture<'_, Result<ProjectionStatus, Error>> { 88 Box::pin(async move { 89 self.require_projection_writer()?; 90 let mut transaction = self 91 .pool() 92 .begin_with("BEGIN IMMEDIATE") 93 .await 94 .map_err(cache::map)?; 95 let status = checkpoint_transaction(&mut transaction, checkpoint).await?; 96 transaction.commit().await.map_err(cache::map)?; 97 Ok(status) 98 }) 99 } 100 101 fn invalidate( 102 &self, 103 invalidation: ProjectionInvalidation, 104 ) -> BoxFuture<'_, Result<ProjectionStatus, Error>> { 105 Box::pin(async move { 106 self.require_projection_writer()?; 107 let mut transaction = self 108 .pool() 109 .begin_with("BEGIN IMMEDIATE") 110 .await 111 .map_err(cache::map)?; 112 let row = sqlx::query( 113 "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?", 114 ) 115 .bind(invalidation.projection_id().as_str()) 116 .fetch_optional(&mut *transaction) 117 .await 118 .map_err(cache::map)? 119 .ok_or(Error::ProjectionCheckpointMismatch)?; 120 let current = decode_status(&row)?; 121 if current.generation() != invalidation.invalid_generation() { 122 return Err(Error::ProjectionCheckpointMismatch); 123 } 124 if let Some(existing) = load_invalidation( 125 &mut transaction, 126 invalidation.projection_id(), 127 invalidation.invalid_generation(), 128 ) 129 .await? 130 { 131 if existing != invalidation { 132 return Err(Error::ProjectionRevisionConflict); 133 } 134 } else { 135 sqlx::query( 136 "INSERT INTO radroots_runtime_projection_invalidations ( 137 projection_id, invalid_generation, replacement_generation, reason, 138 invalidated_at_unix_ms 139 ) VALUES (?, ?, ?, ?, ?)", 140 ) 141 .bind(invalidation.projection_id().as_str()) 142 .bind(invalidation.invalid_generation().as_bytes().as_slice()) 143 .bind(invalidation.replacement_generation().as_bytes().as_slice()) 144 .bind(reason_name(invalidation.reason())) 145 .bind(i64_from_u64(invalidation.invalidated_at_unix_ms())?) 146 .execute(&mut *transaction) 147 .await 148 .map_err(cache::map)?; 149 } 150 let next = ProjectionStatus::new( 151 invalidation.projection_id().clone(), 152 current.generation(), 153 ProjectionHealth::Invalidated, 154 current.checkpoint().cloned(), 155 None, 156 )?; 157 put_status_transaction(&mut transaction, &next).await?; 158 transaction.commit().await.map_err(cache::map)?; 159 Ok(next) 160 }) 161 } 162 163 fn request_rebuild( 164 &self, 165 ticket: RebuildTicket, 166 ) -> BoxFuture<'_, Result<RebuildTicket, Error>> { 167 Box::pin(async move { 168 self.require_projection_writer()?; 169 let mut transaction = self 170 .pool() 171 .begin_with("BEGIN IMMEDIATE") 172 .await 173 .map_err(cache::map)?; 174 if let Some(row) = sqlx::query( 175 "SELECT * FROM radroots_runtime_projection_rebuilds WHERE ticket_id = ?", 176 ) 177 .bind(ticket.ticket_id().as_bytes().as_slice()) 178 .fetch_optional(&mut *transaction) 179 .await 180 .map_err(cache::map)? 181 { 182 let existing = decode_ticket(&mut transaction, &row).await?; 183 return if existing == ticket { 184 transaction.commit().await.map_err(cache::map)?; 185 Ok(existing) 186 } else { 187 Err(Error::ProjectionRevisionConflict) 188 }; 189 } 190 let status_row = sqlx::query( 191 "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?", 192 ) 193 .bind(ticket.invalidation().projection_id().as_str()) 194 .fetch_optional(&mut *transaction) 195 .await 196 .map_err(cache::map)? 197 .ok_or(Error::ProjectionCheckpointMismatch)?; 198 let status = decode_status(&status_row)?; 199 if status.generation() != ticket.invalidation().invalid_generation() 200 || status.health() != ProjectionHealth::Invalidated 201 || load_invalidation( 202 &mut transaction, 203 ticket.invalidation().projection_id(), 204 ticket.invalidation().invalid_generation(), 205 ) 206 .await? 207 .as_ref() 208 != Some(ticket.invalidation()) 209 { 210 return Err(Error::ProjectionCheckpointMismatch); 211 } 212 insert_ticket(&mut transaction, &ticket).await?; 213 let next = ProjectionStatus::new( 214 status.projection_id().clone(), 215 status.generation(), 216 ProjectionHealth::Rebuilding, 217 status.checkpoint().cloned(), 218 Some(ticket.ticket_id()), 219 )?; 220 put_status_transaction(&mut transaction, &next).await?; 221 transaction.commit().await.map_err(cache::map)?; 222 Ok(ticket) 223 }) 224 } 225 226 fn invalidation( 227 &self, 228 projection_id: ProjectionId, 229 replacement_generation: ProjectionGeneration, 230 ) -> BoxFuture<'_, Result<Option<ProjectionInvalidation>, Error>> { 231 Box::pin(async move { 232 sqlx::query( 233 "SELECT * FROM radroots_runtime_projection_invalidations 234 WHERE projection_id = ? AND replacement_generation = ? 235 ORDER BY invalidated_at_unix_ms DESC LIMIT 1", 236 ) 237 .bind(projection_id.as_str()) 238 .bind(replacement_generation.as_bytes().as_slice()) 239 .fetch_optional(self.pool()) 240 .await 241 .map_err(cache::map)? 242 .as_ref() 243 .map(decode_invalidation) 244 .transpose() 245 }) 246 } 247 248 fn rebuild( 249 &self, 250 ticket_id: RebuildTicketId, 251 ) -> BoxFuture<'_, Result<Option<RebuildTicket>, Error>> { 252 Box::pin(async move { 253 let mut connection = self.pool().acquire().await.map_err(cache::map)?; 254 let row = sqlx::query( 255 "SELECT * FROM radroots_runtime_projection_rebuilds WHERE ticket_id = ?", 256 ) 257 .bind(ticket_id.as_bytes().as_slice()) 258 .fetch_optional(&mut *connection) 259 .await 260 .map_err(cache::map)?; 261 match row { 262 Some(row) => decode_ticket(&mut connection, &row).await.map(Some), 263 None => Ok(None), 264 } 265 }) 266 } 267 268 fn transition_rebuild( 269 &self, 270 transition: RebuildTransition, 271 ) -> BoxFuture<'_, Result<RebuildTicket, Error>> { 272 Box::pin(async move { 273 self.require_projection_writer()?; 274 let mut transaction = self 275 .pool() 276 .begin_with("BEGIN IMMEDIATE") 277 .await 278 .map_err(cache::map)?; 279 let row = sqlx::query( 280 "SELECT * FROM radroots_runtime_projection_rebuilds WHERE ticket_id = ?", 281 ) 282 .bind(transition.ticket_id().as_bytes().as_slice()) 283 .fetch_optional(&mut *transaction) 284 .await 285 .map_err(cache::map)? 286 .ok_or(Error::ProjectionRevisionConflict)?; 287 let current = decode_ticket(&mut transaction, &row).await?; 288 let status_row = sqlx::query( 289 "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?", 290 ) 291 .bind(current.invalidation().projection_id().as_str()) 292 .fetch_optional(&mut *transaction) 293 .await 294 .map_err(cache::map)? 295 .ok_or(Error::CorruptProjectionRecord)?; 296 let current_status = decode_status(&status_row)?; 297 if current_status.generation() != current.invalidation().invalid_generation() 298 || current_status.health() != ProjectionHealth::Rebuilding 299 || current_status.active_rebuild() != Some(current.ticket_id()) 300 { 301 return Err(Error::CorruptProjectionRecord); 302 } 303 let next = current.transition(transition)?; 304 if next.stage() == RebuildStage::Completed { 305 batch::refuse_registered_legacy( 306 &mut transaction, 307 next.invalidation().projection_id(), 308 next.invalidation().replacement_generation(), 309 ) 310 .await?; 311 let source = sqlx::query( 312 "SELECT generation, sequence_head 313 FROM radroots_runtime_source_generations WHERE state = 'active'", 314 ) 315 .fetch_one(&mut *transaction) 316 .await 317 .map_err(cache::map)?; 318 let generation = SourceGeneration::new(array( 319 source 320 .try_get::<Vec<u8>, _>("generation") 321 .map_err(map_corrupt)?, 322 )?) 323 .map_err(|_| Error::CorruptProjectionRecord)?; 324 let sequence = u64_from_i64( 325 source 326 .try_get::<i64, _>("sequence_head") 327 .map_err(map_corrupt)?, 328 )?; 329 if generation != next.source_generation() 330 || sequence 331 != next 332 .source_high_water() 333 .map_or(0, |position| position.sequence().get()) 334 { 335 return Err(Error::SourceGenerationChanged); 336 } 337 } 338 update_ticket(&mut transaction, &next, current.revision()).await?; 339 let (generation, health, checkpoint, active_rebuild) = match next.stage() { 340 RebuildStage::Requested | RebuildStage::Running => ( 341 current_status.generation(), 342 ProjectionHealth::Rebuilding, 343 current_status.checkpoint().cloned(), 344 Some(next.ticket_id()), 345 ), 346 RebuildStage::Completed => ( 347 next.invalidation().replacement_generation(), 348 ProjectionHealth::Ready, 349 next.checkpoint().cloned(), 350 None, 351 ), 352 RebuildStage::Failed => ( 353 current_status.generation(), 354 ProjectionHealth::Ready, 355 current_status.checkpoint().cloned(), 356 None, 357 ), 358 }; 359 let status = ProjectionStatus::new( 360 next.invalidation().projection_id().clone(), 361 generation, 362 health, 363 checkpoint, 364 active_rebuild, 365 )?; 366 put_status_transaction(&mut transaction, &status).await?; 367 transaction.commit().await.map_err(cache::map)?; 368 Ok(next) 369 }) 370 } 371 372 fn event_index_manifest( 373 &self, 374 generation: ProjectionGeneration, 375 ) -> BoxFuture<'_, Result<Option<EventIndexManifest>, Error>> { 376 Box::pin(async move { 377 let mut connection = self.pool().acquire().await.map_err(cache::map)?; 378 load_manifest(&mut connection, generation).await 379 }) 380 } 381 382 fn put_event_index_manifest( 383 &self, 384 manifest: EventIndexManifest, 385 ) -> BoxFuture<'_, Result<(), Error>> { 386 Box::pin(async move { 387 self.require_projection_writer()?; 388 let mut transaction = self 389 .pool() 390 .begin_with("BEGIN IMMEDIATE") 391 .await 392 .map_err(cache::map)?; 393 if let Some(existing) = load_manifest(&mut transaction, manifest.generation()).await? { 394 return if existing == manifest { 395 transaction.commit().await.map_err(cache::map)?; 396 Ok(()) 397 } else { 398 Err(Error::CorruptProjectionRecord) 399 }; 400 } 401 sqlx::query( 402 "INSERT INTO radroots_runtime_event_index_manifests ( 403 projection_generation, total_events, target_shard_size, 404 first_published_at_unix_s, last_published_at_unix_s 405 ) VALUES (?, ?, ?, ?, ?)", 406 ) 407 .bind(manifest.generation().as_bytes().as_slice()) 408 .bind(i64_from_u64(manifest.total_events())?) 409 .bind(i64::from(manifest.target_shard_size())) 410 .bind(i64_from_u64(manifest.first_published_at_unix_s())?) 411 .bind(i64_from_u64(manifest.last_published_at_unix_s())?) 412 .execute(&mut *transaction) 413 .await 414 .map_err(cache::map)?; 415 for (ordinal, shard) in manifest.shards().iter().enumerate() { 416 sqlx::query( 417 "INSERT INTO radroots_runtime_event_index_shards ( 418 projection_generation, shard_id, ordinal, artifact_path, event_count, 419 first_event_id, last_event_id, first_published_at_unix_s, 420 last_published_at_unix_s, artifact_digest 421 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", 422 ) 423 .bind(manifest.generation().as_bytes().as_slice()) 424 .bind(shard.shard_id().as_str()) 425 .bind(i64::try_from(ordinal).map_err(|_| Error::CorruptProjectionRecord)?) 426 .bind(shard.artifact_path()) 427 .bind(i64::from(shard.event_count())) 428 .bind(shard.event_ids().first().as_bytes().as_slice()) 429 .bind(shard.event_ids().last().as_bytes().as_slice()) 430 .bind(i64_from_u64(shard.first_published_at_unix_s())?) 431 .bind(i64_from_u64(shard.last_published_at_unix_s())?) 432 .bind(shard.sha256().as_bytes().as_slice()) 433 .execute(&mut *transaction) 434 .await 435 .map_err(cache::map)?; 436 } 437 transaction.commit().await.map_err(cache::map) 438 }) 439 } 440 441 fn event_index_checkpoint( 442 &self, 443 generation: ProjectionGeneration, 444 ) -> BoxFuture<'_, Result<Option<EventIndexCheckpoint>, Error>> { 445 Box::pin(async move { 446 sqlx::query( 447 "SELECT generated_at_unix_ms, checkpoint 448 FROM radroots_runtime_event_index_checkpoints 449 WHERE projection_generation = ?", 450 ) 451 .bind(generation.as_bytes().as_slice()) 452 .fetch_optional(self.pool()) 453 .await 454 .map_err(cache::map)? 455 .as_ref() 456 .map(|row| decode_index_checkpoint(row, generation)) 457 .transpose() 458 }) 459 } 460 461 fn put_event_index_checkpoint( 462 &self, 463 checkpoint: EventIndexCheckpoint, 464 ) -> BoxFuture<'_, Result<(), Error>> { 465 Box::pin(async move { 466 self.require_projection_writer()?; 467 let mut transaction = self 468 .pool() 469 .begin_with("BEGIN IMMEDIATE") 470 .await 471 .map_err(cache::map)?; 472 if let Some(row) = sqlx::query( 473 "SELECT generated_at_unix_ms, checkpoint 474 FROM radroots_runtime_event_index_checkpoints 475 WHERE projection_generation = ?", 476 ) 477 .bind(checkpoint.generation().as_bytes().as_slice()) 478 .fetch_optional(&mut *transaction) 479 .await 480 .map_err(cache::map)? 481 && checkpoint.generated_at_unix_ms() 482 < decode_index_checkpoint(&row, checkpoint.generation())?.generated_at_unix_ms() 483 { 484 return Err(Error::InvalidEventIndexCheckpoint); 485 } 486 sqlx::query( 487 "INSERT INTO radroots_runtime_event_index_checkpoints ( 488 projection_generation, generated_at_unix_ms, checkpoint 489 ) VALUES (?, ?, ?) 490 ON CONFLICT(projection_generation) DO UPDATE SET 491 generated_at_unix_ms = excluded.generated_at_unix_ms, 492 checkpoint = excluded.checkpoint", 493 ) 494 .bind(checkpoint.generation().as_bytes().as_slice()) 495 .bind(i64_from_u64(checkpoint.generated_at_unix_ms())?) 496 .bind(encode_index_checkpoint(&checkpoint)?) 497 .execute(&mut *transaction) 498 .await 499 .map_err(cache::map)?; 500 transaction.commit().await.map_err(cache::map)?; 501 Ok(()) 502 }) 503 } 504 505 fn put_projection_batch( 506 &self, 507 value: radroots_storage::projection::batch::ProjectionDocumentBatch, 508 ) -> BoxFuture<'_, Result<(), Error>> { 509 Box::pin(async move { batch::write(self, value, None).await.map(|_| ()) }) 510 } 511 fn promote_projection_batch( 512 &self, 513 value: radroots_storage::projection::batch::ProjectionDocumentBatch, 514 checkpoint: ProjectionCheckpoint, 515 ) -> BoxFuture<'_, Result<ProjectionStatus, Error>> { 516 Box::pin(async move { 517 batch::write(self, value, Some(checkpoint)) 518 .await? 519 .ok_or(Error::CorruptProjectionRecord) 520 }) 521 } 522 fn query_projection_documents_at( 523 &self, 524 query: radroots_storage::projection::document_query::ProjectionDocumentQuery, 525 source: radroots_storage::event::EventSourceToken, 526 active: ProjectionGeneration, 527 ) -> BoxFuture< 528 '_, 529 Result<radroots_storage::projection::document_query::ProjectionDocumentPage, Error>, 530 > { 531 Box::pin(document_query::page_at(self, query, source, active)) 532 } 533 fn put_projection_document( 534 &self, 535 projection_id: ProjectionId, 536 generation: ProjectionGeneration, 537 document: ProjectionDocument, 538 ) -> BoxFuture<'_, Result<(), Error>> { 539 Box::pin(batch::legacy_put(self, projection_id, generation, document)) 540 } 541 542 fn projection_document( 543 &self, 544 projection_id: ProjectionId, 545 generation: ProjectionGeneration, 546 key: String, 547 ) -> BoxFuture<'_, Result<Option<ProjectionDocument>, Error>> { 548 Box::pin(batch::load_document(self, projection_id, generation, key)) 549 } 550 551 fn query_projection_documents( 552 &self, 553 query: radroots_storage::projection::document_query::ProjectionDocumentQuery, 554 ) -> BoxFuture< 555 '_, 556 Result<radroots_storage::projection::document_query::ProjectionDocumentPage, Error>, 557 > { 558 Box::pin(document_query::page(self, query)) 559 } 560 561 fn put_projection_snapshot( 562 &self, 563 snapshot: ProjectionSnapshot, 564 ) -> BoxFuture<'_, Result<(), Error>> { 565 Box::pin(cache::put_snapshot(self, snapshot)) 566 } 567 fn projection_snapshot( 568 &self, 569 id: ProjectionId, 570 snapshot: [u8; 32], 571 ) -> BoxFuture<'_, Result<Option<ProjectionSnapshot>, Error>> { 572 Box::pin(cache::snapshot(self, id, snapshot)) 573 } 574 } 575 576 #[cfg_attr(coverage_nightly, coverage(off))] 577 pub(crate) async fn checkpoint_transaction( 578 transaction: &mut sqlx::Transaction<'_, Sqlite>, 579 checkpoint: ProjectionCheckpoint, 580 ) -> Result<ProjectionStatus, Error> { 581 batch::refuse_registered_legacy( 582 transaction, 583 checkpoint.projection_id(), 584 checkpoint.generation(), 585 ) 586 .await?; 587 let prior = sqlx::query( 588 "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?", 589 ) 590 .bind(checkpoint.projection_id().as_str()) 591 .fetch_optional(&mut **transaction) 592 .await 593 .map_err(cache::map)? 594 .as_ref() 595 .map(decode_status) 596 .transpose()?; 597 if let Some(prior) = prior.as_ref() { 598 if prior.generation() != checkpoint.generation() { 599 return Err(Error::ProjectionCheckpointMismatch); 600 } 601 if prior 602 .checkpoint() 603 .is_some_and(|value| !checkpoint.advances(value)) 604 { 605 return Err(Error::ProjectionCheckpointRegression); 606 } 607 } 608 let status = ProjectionStatus::new( 609 checkpoint.projection_id().clone(), 610 checkpoint.generation(), 611 ProjectionHealth::Ready, 612 Some(checkpoint), 613 None, 614 )?; 615 put_status_transaction(transaction, &status).await?; 616 Ok(status) 617 } 618 619 impl SqliteStorage { 620 fn require_projection_writer(&self) -> Result<(), Error> { 621 if self.event_mode() == radroots_storage::status::EventStoreMode::ReadOnly { 622 return Err(Error::BackendUnavailable); 623 } 624 Ok(()) 625 } 626 } 627 628 #[cfg_attr(coverage_nightly, coverage(off))] 629 async fn put_status_transaction( 630 transaction: &mut sqlx::Transaction<'_, Sqlite>, 631 status: &ProjectionStatus, 632 ) -> Result<(), Error> { 633 let values = checkpoint_values(status.checkpoint())?; 634 sqlx::query(STATUS_UPSERT) 635 .bind(status.projection_id().as_str()) 636 .bind(status.generation().as_bytes().as_slice()) 637 .bind(health_name(status.health())) 638 .bind(values.0) 639 .bind(values.1) 640 .bind(values.2) 641 .bind(values.3) 642 .bind( 643 status 644 .active_rebuild() 645 .map(|ticket| ticket.as_bytes().to_vec()), 646 ) 647 .execute(&mut **transaction) 648 .await 649 .map_err(cache::map)?; 650 Ok(()) 651 } 652 653 const STATUS_UPSERT: &str = "INSERT INTO radroots_runtime_projection_checkpoints ( 654 projection_id, projection_generation, health, source_generation, source_sequence, 655 projected_rows, checkpoint_updated_at_unix_ms, active_rebuild 656 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) 657 ON CONFLICT(projection_id) DO UPDATE SET 658 projection_generation = excluded.projection_generation, 659 health = excluded.health, 660 source_generation = excluded.source_generation, 661 source_sequence = excluded.source_sequence, 662 projected_rows = excluded.projected_rows, 663 checkpoint_updated_at_unix_ms = excluded.checkpoint_updated_at_unix_ms, 664 active_rebuild = excluded.active_rebuild"; 665 666 type CheckpointValues = (Option<Vec<u8>>, Option<i64>, Option<i64>, Option<i64>); 667 668 fn checkpoint_values(checkpoint: Option<&ProjectionCheckpoint>) -> Result<CheckpointValues, Error> { 669 let Some(checkpoint) = checkpoint else { 670 return Ok((None, None, None, None)); 671 }; 672 let (generation, sequence) = checkpoint 673 .source_position() 674 .map_or((None, None), |position| { 675 ( 676 Some(position.generation().as_bytes().to_vec()), 677 Some(i64_from_u64(position.sequence().get())), 678 ) 679 }); 680 Ok(( 681 generation, 682 sequence.transpose()?, 683 Some(i64_from_u64(checkpoint.projected_rows())?), 684 Some(i64_from_u64(checkpoint.updated_at_unix_ms())?), 685 )) 686 } 687 688 fn decode_status(row: &sqlx::sqlite::SqliteRow) -> Result<ProjectionStatus, Error> { 689 let projection_id = ProjectionId::parse( 690 row.try_get::<String, _>("projection_id") 691 .map_err(map_corrupt)?, 692 ) 693 .map_err(|_| Error::CorruptProjectionRecord)?; 694 let generation = projection_generation(row, "projection_generation")?; 695 let checkpoint = decode_checkpoint(row, projection_id.clone(), generation, "")?; 696 let active = row 697 .try_get::<Option<Vec<u8>>, _>("active_rebuild") 698 .map_err(map_corrupt)? 699 .map(|value| { 700 RebuildTicketId::new(array(value)?).map_err(|_| Error::CorruptProjectionRecord) 701 }) 702 .transpose()?; 703 ProjectionStatus::new( 704 projection_id, 705 generation, 706 health( 707 row.try_get::<String, _>("health") 708 .map_err(map_corrupt)? 709 .as_str(), 710 )?, 711 checkpoint, 712 active, 713 ) 714 } 715 716 fn decode_checkpoint( 717 row: &sqlx::sqlite::SqliteRow, 718 projection_id: ProjectionId, 719 generation: ProjectionGeneration, 720 prefix: &str, 721 ) -> Result<Option<ProjectionCheckpoint>, Error> { 722 let rows = row 723 .try_get::<Option<i64>, _>(format!("{prefix}projected_rows").as_str()) 724 .map_err(map_corrupt)?; 725 let updated = row 726 .try_get::<Option<i64>, _>(format!("{prefix}updated_at_unix_ms").as_str()) 727 .or_else(|_| row.try_get(format!("{prefix}checkpoint_updated_at_unix_ms").as_str())) 728 .map_err(map_corrupt)?; 729 let source_generation = row 730 .try_get::<Option<Vec<u8>>, _>(format!("{prefix}source_generation").as_str()) 731 .map_err(map_corrupt)?; 732 let source_sequence = row 733 .try_get::<Option<i64>, _>(format!("{prefix}source_sequence").as_str()) 734 .map_err(map_corrupt)?; 735 match (rows, updated, source_generation, source_sequence) { 736 (None, None, None, None) => Ok(None), 737 (Some(rows), Some(updated), source_generation, source_sequence) => { 738 let position = match (source_generation, source_sequence) { 739 (None, None) => None, 740 (Some(source_generation), Some(source_sequence)) => Some(EventPosition::new( 741 SourceGeneration::new(array(source_generation)?) 742 .map_err(|_| Error::CorruptProjectionRecord)?, 743 radroots_storage::event::EventSequence::new(u64_from_i64(source_sequence)?) 744 .map_err(|_| Error::CorruptProjectionRecord)?, 745 )), 746 _ => return Err(Error::CorruptProjectionRecord), 747 }; 748 ProjectionCheckpoint::new( 749 projection_id, 750 generation, 751 position, 752 u64_from_i64(rows)?, 753 u64_from_i64(updated)?, 754 ) 755 .map(Some) 756 .map_err(|_| Error::CorruptProjectionRecord) 757 } 758 _ => Err(Error::CorruptProjectionRecord), 759 } 760 } 761 762 #[cfg_attr(coverage_nightly, coverage(off))] 763 async fn load_invalidation( 764 connection: &mut SqliteConnection, 765 projection_id: &ProjectionId, 766 generation: ProjectionGeneration, 767 ) -> Result<Option<ProjectionInvalidation>, Error> { 768 sqlx::query( 769 "SELECT * FROM radroots_runtime_projection_invalidations 770 WHERE projection_id = ? AND invalid_generation = ?", 771 ) 772 .bind(projection_id.as_str()) 773 .bind(generation.as_bytes().as_slice()) 774 .fetch_optional(&mut *connection) 775 .await 776 .map_err(cache::map)? 777 .as_ref() 778 .map(decode_invalidation) 779 .transpose() 780 } 781 782 fn decode_invalidation(row: &sqlx::sqlite::SqliteRow) -> Result<ProjectionInvalidation, Error> { 783 ProjectionInvalidation::new( 784 ProjectionId::parse( 785 row.try_get::<String, _>("projection_id") 786 .map_err(map_corrupt)?, 787 ) 788 .map_err(|_| Error::CorruptProjectionRecord)?, 789 projection_generation(row, "invalid_generation")?, 790 projection_generation(row, "replacement_generation")?, 791 reason( 792 row.try_get::<String, _>("reason") 793 .map_err(map_corrupt)? 794 .as_str(), 795 )?, 796 u64_from_i64(row.try_get("invalidated_at_unix_ms").map_err(map_corrupt)?)?, 797 ) 798 .map_err(|_| Error::CorruptProjectionRecord) 799 } 800 801 #[cfg_attr(coverage_nightly, coverage(off))] 802 async fn insert_ticket( 803 transaction: &mut sqlx::Transaction<'_, Sqlite>, 804 ticket: &RebuildTicket, 805 ) -> Result<(), Error> { 806 let checkpoint = checkpoint_values(ticket.checkpoint())?; 807 sqlx::query( 808 "INSERT INTO radroots_runtime_projection_rebuilds ( 809 ticket_id, projection_id, invalid_generation, replacement_generation, revision, stage, 810 source_generation, source_sequence, source_digest, 811 checkpoint_source_generation, checkpoint_source_sequence, checkpoint_projected_rows, 812 checkpoint_updated_at_unix_ms, failure, requested_at_unix_ms, updated_at_unix_ms 813 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", 814 ) 815 .bind(ticket.ticket_id().as_bytes().as_slice()) 816 .bind(ticket.invalidation().projection_id().as_str()) 817 .bind( 818 ticket 819 .invalidation() 820 .invalid_generation() 821 .as_bytes() 822 .as_slice(), 823 ) 824 .bind( 825 ticket 826 .invalidation() 827 .replacement_generation() 828 .as_bytes() 829 .as_slice(), 830 ) 831 .bind(i64_from_u64(ticket.revision().get())?) 832 .bind(rebuild_stage_name(ticket.stage())) 833 .bind(ticket.source_generation().as_bytes().as_slice()) 834 .bind( 835 ticket 836 .source_high_water() 837 .map(|position| i64_from_u64(position.sequence().get())) 838 .transpose()?, 839 ) 840 .bind(ticket.source_digest().as_bytes().as_slice()) 841 .bind(checkpoint.0) 842 .bind(checkpoint.1) 843 .bind(checkpoint.2) 844 .bind(checkpoint.3) 845 .bind(ticket.failure().map(rebuild_failure_name)) 846 .bind(i64_from_u64(ticket.requested_at_unix_ms())?) 847 .bind(i64_from_u64(ticket.updated_at_unix_ms())?) 848 .execute(&mut **transaction) 849 .await 850 .map_err(cache::map)?; 851 Ok(()) 852 } 853 854 #[cfg_attr(coverage_nightly, coverage(off))] 855 async fn update_ticket( 856 transaction: &mut sqlx::Transaction<'_, Sqlite>, 857 ticket: &RebuildTicket, 858 prior: ProjectionRevision, 859 ) -> Result<(), Error> { 860 let checkpoint = checkpoint_values(ticket.checkpoint())?; 861 let result = sqlx::query( 862 "UPDATE radroots_runtime_projection_rebuilds SET 863 revision = ?, stage = ?, checkpoint_source_generation = ?, 864 checkpoint_source_sequence = ?, checkpoint_projected_rows = ?, 865 checkpoint_updated_at_unix_ms = ?, failure = ?, updated_at_unix_ms = ? 866 WHERE ticket_id = ? AND revision = ?", 867 ) 868 .bind(i64_from_u64(ticket.revision().get())?) 869 .bind(rebuild_stage_name(ticket.stage())) 870 .bind(checkpoint.0) 871 .bind(checkpoint.1) 872 .bind(checkpoint.2) 873 .bind(checkpoint.3) 874 .bind(ticket.failure().map(rebuild_failure_name)) 875 .bind(i64_from_u64(ticket.updated_at_unix_ms())?) 876 .bind(ticket.ticket_id().as_bytes().as_slice()) 877 .bind(i64_from_u64(prior.get())?) 878 .execute(&mut **transaction) 879 .await 880 .map_err(cache::map)?; 881 if result.rows_affected() != 1 { 882 return Err(Error::ProjectionRevisionConflict); 883 } 884 Ok(()) 885 } 886 887 #[cfg_attr(coverage_nightly, coverage(off))] 888 async fn decode_ticket( 889 connection: &mut SqliteConnection, 890 row: &sqlx::sqlite::SqliteRow, 891 ) -> Result<RebuildTicket, Error> { 892 let projection_id = ProjectionId::parse( 893 row.try_get::<String, _>("projection_id") 894 .map_err(map_corrupt)?, 895 ) 896 .map_err(|_| Error::CorruptProjectionRecord)?; 897 let invalid_generation = projection_generation(row, "invalid_generation")?; 898 let invalidation = load_invalidation(connection, &projection_id, invalid_generation) 899 .await? 900 .ok_or(Error::CorruptProjectionRecord)?; 901 if invalidation.replacement_generation() 902 != projection_generation(row, "replacement_generation")? 903 { 904 return Err(Error::CorruptProjectionRecord); 905 } 906 let checkpoint = decode_checkpoint( 907 row, 908 projection_id, 909 invalidation.replacement_generation(), 910 "checkpoint_", 911 )?; 912 RebuildTicket::from_durable_parts( 913 RebuildTicketId::new(array( 914 row.try_get::<Vec<u8>, _>("ticket_id") 915 .map_err(map_corrupt)?, 916 )?) 917 .map_err(|_| Error::CorruptProjectionRecord)?, 918 invalidation, 919 ProjectionRevision::new(u64_from_i64(row.try_get("revision").map_err(map_corrupt)?)?) 920 .map_err(|_| Error::CorruptProjectionRecord)?, 921 rebuild_stage( 922 row.try_get::<String, _>("stage") 923 .map_err(map_corrupt)? 924 .as_str(), 925 )?, 926 SourceGeneration::new(array( 927 row.try_get::<Vec<u8>, _>("source_generation") 928 .map_err(map_corrupt)?, 929 )?) 930 .map_err(|_| Error::CorruptProjectionRecord)?, 931 row.try_get::<Option<i64>, _>("source_sequence") 932 .map_err(map_corrupt)? 933 .map(|sequence| { 934 Ok(EventPosition::new( 935 SourceGeneration::new(array( 936 row.try_get::<Vec<u8>, _>("source_generation") 937 .map_err(map_corrupt)?, 938 )?) 939 .map_err(|_| Error::CorruptProjectionRecord)?, 940 radroots_storage::event::EventSequence::new(u64_from_i64(sequence)?) 941 .map_err(|_| Error::CorruptProjectionRecord)?, 942 )) 943 }) 944 .transpose()?, 945 RawSourceDigest::new(array( 946 row.try_get::<Vec<u8>, _>("source_digest") 947 .map_err(map_corrupt)?, 948 )?), 949 checkpoint, 950 row.try_get::<Option<String>, _>("failure") 951 .map_err(map_corrupt)? 952 .as_deref() 953 .map(rebuild_failure) 954 .transpose()?, 955 u64_from_i64(row.try_get("requested_at_unix_ms").map_err(map_corrupt)?)?, 956 u64_from_i64(row.try_get("updated_at_unix_ms").map_err(map_corrupt)?)?, 957 ) 958 } 959 960 #[cfg_attr(coverage_nightly, coverage(off))] 961 async fn load_manifest( 962 connection: &mut SqliteConnection, 963 generation: ProjectionGeneration, 964 ) -> Result<Option<EventIndexManifest>, Error> { 965 let Some(row) = sqlx::query( 966 "SELECT * FROM radroots_runtime_event_index_manifests 967 WHERE projection_generation = ?", 968 ) 969 .bind(generation.as_bytes().as_slice()) 970 .fetch_optional(&mut *connection) 971 .await 972 .map_err(cache::map)? 973 else { 974 return Ok(None); 975 }; 976 let shards = sqlx::query( 977 "SELECT * FROM radroots_runtime_event_index_shards 978 WHERE projection_generation = ? ORDER BY ordinal", 979 ) 980 .bind(generation.as_bytes().as_slice()) 981 .fetch_all(&mut *connection) 982 .await 983 .map_err(cache::map)? 984 .iter() 985 .enumerate() 986 .map(|(ordinal, row)| { 987 if row.try_get::<i64, _>("ordinal").map_err(map_corrupt)? 988 != i64::try_from(ordinal).map_err(|_| Error::CorruptProjectionRecord)? 989 { 990 return Err(Error::CorruptProjectionRecord); 991 } 992 EventIndexShard::new( 993 EventIndexShardId::parse(row.try_get::<String, _>("shard_id").map_err(map_corrupt)?) 994 .map_err(|_| Error::CorruptProjectionRecord)?, 995 row.try_get::<String, _>("artifact_path") 996 .map_err(map_corrupt)?, 997 u32::try_from(row.try_get::<i64, _>("event_count").map_err(map_corrupt)?) 998 .map_err(|_| Error::CorruptProjectionRecord)?, 999 EventIdRange::new( 1000 event_id(row, "first_event_id")?, 1001 event_id(row, "last_event_id")?, 1002 ) 1003 .map_err(|_| Error::CorruptProjectionRecord)?, 1004 u64_from_i64( 1005 row.try_get("first_published_at_unix_s") 1006 .map_err(map_corrupt)?, 1007 )?, 1008 u64_from_i64( 1009 row.try_get("last_published_at_unix_s") 1010 .map_err(map_corrupt)?, 1011 )?, 1012 ArtifactDigest::new(array( 1013 row.try_get::<Vec<u8>, _>("artifact_digest") 1014 .map_err(map_corrupt)?, 1015 )?), 1016 ) 1017 .map_err(|_| Error::CorruptProjectionRecord) 1018 }) 1019 .collect::<Result<Vec<_>, _>>()?; 1020 EventIndexManifest::new( 1021 generation, 1022 u64_from_i64(row.try_get("total_events").map_err(map_corrupt)?)?, 1023 u32::try_from( 1024 row.try_get::<i64, _>("target_shard_size") 1025 .map_err(map_corrupt)?, 1026 ) 1027 .map_err(|_| Error::CorruptProjectionRecord)?, 1028 u64_from_i64( 1029 row.try_get("first_published_at_unix_s") 1030 .map_err(map_corrupt)?, 1031 )?, 1032 u64_from_i64( 1033 row.try_get("last_published_at_unix_s") 1034 .map_err(map_corrupt)?, 1035 )?, 1036 shards, 1037 ) 1038 .map(Some) 1039 .map_err(|_| Error::CorruptProjectionRecord) 1040 } 1041 1042 fn encode_index_checkpoint(checkpoint: &EventIndexCheckpoint) -> Result<Vec<u8>, Error> { 1043 let mut bytes = vec![1]; 1044 let count = 1045 u16::try_from(checkpoint.shards().len()).map_err(|_| Error::CorruptProjectionRecord)?; 1046 bytes.extend_from_slice(&count.to_be_bytes()); 1047 for shard in checkpoint.shards() { 1048 put_string(&mut bytes, shard.shard_id().as_str())?; 1049 bytes.extend_from_slice(&shard.last_created_at_unix_s().to_be_bytes()); 1050 match shard.last_event_id() { 1051 Some(event_id) => { 1052 bytes.push(1); 1053 bytes.extend_from_slice(event_id.as_bytes()); 1054 } 1055 None => bytes.push(0), 1056 } 1057 match shard.cursor() { 1058 Some(cursor) => { 1059 bytes.push(1); 1060 put_string(&mut bytes, cursor)?; 1061 } 1062 None => bytes.push(0), 1063 } 1064 } 1065 Ok(bytes) 1066 } 1067 1068 fn decode_index_checkpoint( 1069 row: &sqlx::sqlite::SqliteRow, 1070 generation: ProjectionGeneration, 1071 ) -> Result<EventIndexCheckpoint, Error> { 1072 let bytes = row 1073 .try_get::<Vec<u8>, _>("checkpoint") 1074 .map_err(map_corrupt)?; 1075 let mut cursor = Cursor::new(bytes.as_slice()); 1076 if cursor.byte()? != 1 { 1077 return Err(Error::CorruptProjectionRecord); 1078 } 1079 let count = usize::from(cursor.u16()?); 1080 if count > EVENT_INDEX_SHARDS_MAX { 1081 return Err(Error::CorruptProjectionRecord); 1082 } 1083 let mut shards = Vec::with_capacity(count); 1084 for _ in 0..count { 1085 let shard_id = EventIndexShardId::parse(cursor.string()?.to_owned()) 1086 .map_err(|_| Error::CorruptProjectionRecord)?; 1087 let last_created_at_unix_s = cursor.u64()?; 1088 let last_event_id = match cursor.byte()? { 1089 0 => None, 1090 1 => Some(EventId::from_bytes(cursor.array()?)), 1091 _ => return Err(Error::CorruptProjectionRecord), 1092 }; 1093 let checkpoint_cursor = match cursor.byte()? { 1094 0 => None, 1095 1 => Some(cursor.string()?.to_owned()), 1096 _ => return Err(Error::CorruptProjectionRecord), 1097 }; 1098 shards.push( 1099 EventIndexShardCheckpoint::new( 1100 shard_id, 1101 last_created_at_unix_s, 1102 last_event_id, 1103 checkpoint_cursor, 1104 ) 1105 .map_err(|_| Error::CorruptProjectionRecord)?, 1106 ); 1107 } 1108 cursor.finish()?; 1109 EventIndexCheckpoint::new( 1110 generation, 1111 u64_from_i64(row.try_get("generated_at_unix_ms").map_err(map_corrupt)?)?, 1112 shards, 1113 ) 1114 .map_err(|_| Error::CorruptProjectionRecord) 1115 } 1116 1117 pub(crate) fn encode_status_snapshot(status: &ProjectionStatus) -> Result<Vec<u8>, Error> { 1118 let mut bytes = Vec::with_capacity(128); 1119 bytes.push(1); 1120 put_string(&mut bytes, status.projection_id().as_str())?; 1121 bytes.extend_from_slice(status.generation().as_bytes()); 1122 bytes.push(match status.health() { 1123 ProjectionHealth::Ready => 0, 1124 ProjectionHealth::Invalidated => 1, 1125 ProjectionHealth::Rebuilding => 2, 1126 ProjectionHealth::Failed => 3, 1127 }); 1128 match status.checkpoint() { 1129 Some(checkpoint) => { 1130 bytes.push(1); 1131 match checkpoint.source_position() { 1132 Some(position) => { 1133 bytes.push(1); 1134 bytes.extend_from_slice(position.generation().as_bytes()); 1135 bytes.extend_from_slice(&position.sequence().get().to_be_bytes()); 1136 } 1137 None => bytes.push(0), 1138 } 1139 bytes.extend_from_slice(&checkpoint.projected_rows().to_be_bytes()); 1140 bytes.extend_from_slice(&checkpoint.updated_at_unix_ms().to_be_bytes()); 1141 } 1142 None => bytes.push(0), 1143 } 1144 match status.active_rebuild() { 1145 Some(ticket) => { 1146 bytes.push(1); 1147 bytes.extend_from_slice(ticket.as_bytes()); 1148 } 1149 None => bytes.push(0), 1150 } 1151 Ok(bytes) 1152 } 1153 1154 pub(crate) fn decode_status_snapshot(bytes: &[u8]) -> Result<ProjectionStatus, Error> { 1155 let mut cursor = Cursor::new(bytes); 1156 if cursor.byte()? != 1 { 1157 return Err(Error::CorruptProjectionRecord); 1158 } 1159 let projection_id = 1160 ProjectionId::parse(cursor.string()?).map_err(|_| Error::CorruptProjectionRecord)?; 1161 let generation = 1162 ProjectionGeneration::new(cursor.array()?).map_err(|_| Error::CorruptProjectionRecord)?; 1163 let health = match cursor.byte()? { 1164 0 => ProjectionHealth::Ready, 1165 1 => ProjectionHealth::Invalidated, 1166 2 => ProjectionHealth::Rebuilding, 1167 3 => ProjectionHealth::Failed, 1168 _ => return Err(Error::CorruptProjectionRecord), 1169 }; 1170 let checkpoint = match cursor.byte()? { 1171 0 => None, 1172 1 => { 1173 let source_position = match cursor.byte()? { 1174 0 => None, 1175 1 => Some(EventPosition::new( 1176 SourceGeneration::new(cursor.array()?) 1177 .map_err(|_| Error::CorruptProjectionRecord)?, 1178 radroots_storage::event::EventSequence::new(cursor.u64()?) 1179 .map_err(|_| Error::CorruptProjectionRecord)?, 1180 )), 1181 _ => return Err(Error::CorruptProjectionRecord), 1182 }; 1183 Some( 1184 ProjectionCheckpoint::new( 1185 projection_id.clone(), 1186 generation, 1187 source_position, 1188 cursor.u64()?, 1189 cursor.u64()?, 1190 ) 1191 .map_err(|_| Error::CorruptProjectionRecord)?, 1192 ) 1193 } 1194 _ => return Err(Error::CorruptProjectionRecord), 1195 }; 1196 let active_rebuild = match cursor.byte()? { 1197 0 => None, 1198 1 => Some( 1199 RebuildTicketId::new(cursor.array()?).map_err(|_| Error::CorruptProjectionRecord)?, 1200 ), 1201 _ => return Err(Error::CorruptProjectionRecord), 1202 }; 1203 cursor.finish()?; 1204 ProjectionStatus::new( 1205 projection_id, 1206 generation, 1207 health, 1208 checkpoint, 1209 active_rebuild, 1210 ) 1211 .map_err(|_| Error::CorruptProjectionRecord) 1212 } 1213 1214 fn put_string(bytes: &mut Vec<u8>, value: &str) -> Result<(), Error> { 1215 let length = u16::try_from(value.len()).map_err(|_| Error::CorruptProjectionRecord)?; 1216 bytes.extend_from_slice(&length.to_be_bytes()); 1217 bytes.extend_from_slice(value.as_bytes()); 1218 Ok(()) 1219 } 1220 1221 struct Cursor<'a> { 1222 bytes: &'a [u8], 1223 offset: usize, 1224 } 1225 1226 impl<'a> Cursor<'a> { 1227 const fn new(bytes: &'a [u8]) -> Self { 1228 Self { bytes, offset: 0 } 1229 } 1230 fn byte(&mut self) -> Result<u8, Error> { 1231 let value = self 1232 .bytes 1233 .get(self.offset) 1234 .copied() 1235 .ok_or(Error::CorruptProjectionRecord)?; 1236 self.offset += 1; 1237 Ok(value) 1238 } 1239 fn u16(&mut self) -> Result<u16, Error> { 1240 Ok(u16::from_be_bytes(self.array()?)) 1241 } 1242 fn u64(&mut self) -> Result<u64, Error> { 1243 Ok(u64::from_be_bytes(self.array()?)) 1244 } 1245 fn string(&mut self) -> Result<&'a str, Error> { 1246 let length = usize::from(self.u16()?); 1247 core::str::from_utf8(self.take(length)?).map_err(|_| Error::CorruptProjectionRecord) 1248 } 1249 fn array<const N: usize>(&mut self) -> Result<[u8; N], Error> { 1250 self.take(N)? 1251 .try_into() 1252 .map_err(|_| Error::CorruptProjectionRecord) 1253 } 1254 fn take(&mut self, length: usize) -> Result<&'a [u8], Error> { 1255 let end = self 1256 .offset 1257 .checked_add(length) 1258 .ok_or(Error::CorruptProjectionRecord)?; 1259 let value = self 1260 .bytes 1261 .get(self.offset..end) 1262 .ok_or(Error::CorruptProjectionRecord)?; 1263 self.offset = end; 1264 Ok(value) 1265 } 1266 fn finish(self) -> Result<(), Error> { 1267 if self.offset == self.bytes.len() { 1268 Ok(()) 1269 } else { 1270 Err(Error::CorruptProjectionRecord) 1271 } 1272 } 1273 } 1274 1275 fn projection_generation( 1276 row: &sqlx::sqlite::SqliteRow, 1277 column: &str, 1278 ) -> Result<ProjectionGeneration, Error> { 1279 ProjectionGeneration::new(array( 1280 row.try_get::<Vec<u8>, _>(column).map_err(map_corrupt)?, 1281 )?) 1282 .map_err(|_| Error::CorruptProjectionRecord) 1283 } 1284 1285 fn event_id(row: &sqlx::sqlite::SqliteRow, column: &str) -> Result<EventId, Error> { 1286 Ok(EventId::from_bytes(array( 1287 row.try_get::<Vec<u8>, _>(column).map_err(map_corrupt)?, 1288 )?)) 1289 } 1290 1291 const fn reason_name(value: InvalidationReason) -> &'static str { 1292 match value { 1293 InvalidationReason::SourceGenerationChanged => "source_generation_changed", 1294 InvalidationReason::ProjectionGenerationChanged => "projection_generation_changed", 1295 InvalidationReason::EventIndexManifestChanged => "event_index_manifest_changed", 1296 InvalidationReason::IntegrityFailure => "integrity_failure", 1297 InvalidationReason::OperatorRequested => "operator_requested", 1298 } 1299 } 1300 1301 const fn reason(value: &str) -> Result<InvalidationReason, Error> { 1302 match value.as_bytes() { 1303 b"source_generation_changed" => Ok(InvalidationReason::SourceGenerationChanged), 1304 b"projection_generation_changed" => Ok(InvalidationReason::ProjectionGenerationChanged), 1305 b"event_index_manifest_changed" => Ok(InvalidationReason::EventIndexManifestChanged), 1306 b"integrity_failure" => Ok(InvalidationReason::IntegrityFailure), 1307 b"operator_requested" => Ok(InvalidationReason::OperatorRequested), 1308 _ => Err(Error::CorruptProjectionRecord), 1309 } 1310 } 1311 1312 const fn health_name(value: ProjectionHealth) -> &'static str { 1313 match value { 1314 ProjectionHealth::Ready => "ready", 1315 ProjectionHealth::Invalidated => "invalidated", 1316 ProjectionHealth::Rebuilding => "rebuilding", 1317 ProjectionHealth::Failed => "failed", 1318 } 1319 } 1320 1321 const fn health(value: &str) -> Result<ProjectionHealth, Error> { 1322 match value.as_bytes() { 1323 b"ready" => Ok(ProjectionHealth::Ready), 1324 b"invalidated" => Ok(ProjectionHealth::Invalidated), 1325 b"rebuilding" => Ok(ProjectionHealth::Rebuilding), 1326 b"failed" => Ok(ProjectionHealth::Failed), 1327 _ => Err(Error::CorruptProjectionRecord), 1328 } 1329 } 1330 1331 const fn rebuild_stage_name(value: RebuildStage) -> &'static str { 1332 match value { 1333 RebuildStage::Requested => "requested", 1334 RebuildStage::Running => "running", 1335 RebuildStage::Completed => "completed", 1336 RebuildStage::Failed => "failed", 1337 } 1338 } 1339 1340 const fn rebuild_stage(value: &str) -> Result<RebuildStage, Error> { 1341 match value.as_bytes() { 1342 b"requested" => Ok(RebuildStage::Requested), 1343 b"running" => Ok(RebuildStage::Running), 1344 b"completed" => Ok(RebuildStage::Completed), 1345 b"failed" => Ok(RebuildStage::Failed), 1346 _ => Err(Error::CorruptProjectionRecord), 1347 } 1348 } 1349 1350 const fn rebuild_failure_name(value: RebuildFailure) -> &'static str { 1351 match value { 1352 RebuildFailure::ReducerRejected => "reducer_rejected", 1353 RebuildFailure::SourceChanged => "source_changed", 1354 RebuildFailure::IntegrityFailure => "integrity_failure", 1355 RebuildFailure::PromotionRejected => "promotion_rejected", 1356 } 1357 } 1358 1359 const fn rebuild_failure(value: &str) -> Result<RebuildFailure, Error> { 1360 match value.as_bytes() { 1361 b"reducer_rejected" => Ok(RebuildFailure::ReducerRejected), 1362 b"source_changed" => Ok(RebuildFailure::SourceChanged), 1363 b"integrity_failure" => Ok(RebuildFailure::IntegrityFailure), 1364 b"promotion_rejected" => Ok(RebuildFailure::PromotionRejected), 1365 _ => Err(Error::CorruptProjectionRecord), 1366 } 1367 } 1368 1369 fn array<const N: usize>(bytes: Vec<u8>) -> Result<[u8; N], Error> { 1370 bytes.try_into().map_err(|_| Error::CorruptProjectionRecord) 1371 } 1372 1373 fn i64_from_u64(value: u64) -> Result<i64, Error> { 1374 i64::try_from(value).map_err(|_| Error::CorruptProjectionRecord) 1375 } 1376 1377 fn u64_from_i64(value: i64) -> Result<u64, Error> { 1378 u64::try_from(value).map_err(|_| Error::CorruptProjectionRecord) 1379 } 1380 1381 fn map_corrupt(_: sqlx::Error) -> Error { 1382 Error::CorruptProjectionRecord 1383 } 1384 1385 #[cfg(test)] 1386 #[cfg_attr(coverage_nightly, coverage(off))] 1387 mod tests { 1388 use super::*; 1389 use crate::migration::runtime::{MIGRATIONS, migration_sql}; 1390 use radroots_storage::{ProjectionStore, event::EventSequence, status::EventStoreMode}; 1391 use sqlx::sqlite::SqlitePoolOptions; 1392 1393 async fn store(mode: EventStoreMode) -> SqliteStorage { 1394 let pool = SqlitePoolOptions::new() 1395 .max_connections(1) 1396 .connect("sqlite::memory:") 1397 .await 1398 .expect("memory SQLite"); 1399 sqlx::query("PRAGMA foreign_keys = ON") 1400 .execute(&pool) 1401 .await 1402 .expect("foreign keys"); 1403 for migration in MIGRATIONS { 1404 sqlx::raw_sql(migration_sql(migration.version()).expect("registered SQL")) 1405 .execute(&pool) 1406 .await 1407 .expect("runtime migration"); 1408 } 1409 sqlx::query( 1410 "INSERT INTO radroots_runtime_source_generations ( 1411 generation, sequence_head, state, created_at_unix_ms, retired_at_unix_ms 1412 ) VALUES (?, 0, 'active', 1, NULL)", 1413 ) 1414 .bind([41_u8; 32].as_slice()) 1415 .execute(&pool) 1416 .await 1417 .expect("active source generation"); 1418 SqliteStorage::new( 1419 pool, 1420 SourceGeneration::new([41; 32]).expect("generation"), 1421 mode, 1422 ) 1423 } 1424 1425 fn projection_id() -> ProjectionId { 1426 ProjectionId::parse("trade_projection").expect("projection id") 1427 } 1428 1429 fn generation(byte: u8) -> ProjectionGeneration { 1430 ProjectionGeneration::new([byte; 32]).expect("projection generation") 1431 } 1432 1433 fn checkpoint( 1434 generation: ProjectionGeneration, 1435 sequence: u64, 1436 rows: u64, 1437 at: u64, 1438 ) -> ProjectionCheckpoint { 1439 ProjectionCheckpoint::new( 1440 projection_id(), 1441 generation, 1442 Some(EventPosition::new( 1443 SourceGeneration::new([41; 32]).expect("source generation"), 1444 EventSequence::new(sequence).expect("sequence"), 1445 )), 1446 rows, 1447 at, 1448 ) 1449 .expect("checkpoint") 1450 } 1451 1452 fn invalidation() -> ProjectionInvalidation { 1453 ProjectionInvalidation::new( 1454 projection_id(), 1455 generation(1), 1456 generation(2), 1457 InvalidationReason::ProjectionGenerationChanged, 1458 200, 1459 ) 1460 .expect("invalidation") 1461 } 1462 1463 #[tokio::test] 1464 async fn checkpoints_invalidation_and_rebuild_lifecycle_are_durable() { 1465 let store = store(EventStoreMode::ReadWrite).await; 1466 assert!( 1467 store 1468 .status(projection_id()) 1469 .await 1470 .expect("empty status") 1471 .is_none() 1472 ); 1473 let first = store 1474 .checkpoint(checkpoint(generation(1), 1, 10, 100)) 1475 .await 1476 .expect("first checkpoint"); 1477 assert_eq!(first.health(), ProjectionHealth::Ready); 1478 let advanced = store 1479 .checkpoint(checkpoint(generation(1), 2, 20, 150)) 1480 .await 1481 .expect("advanced checkpoint"); 1482 assert_eq!( 1483 advanced.checkpoint().expect("checkpoint").projected_rows(), 1484 20 1485 ); 1486 assert_eq!( 1487 store 1488 .checkpoint(checkpoint(generation(1), 1, 19, 140)) 1489 .await, 1490 Err(Error::ProjectionCheckpointRegression) 1491 ); 1492 1493 let invalidation = invalidation(); 1494 let invalidated = store 1495 .invalidate(invalidation.clone()) 1496 .await 1497 .expect("invalidate"); 1498 assert_eq!(invalidated.health(), ProjectionHealth::Invalidated); 1499 let ticket = RebuildTicket::requested( 1500 RebuildTicketId::new([3; 16]).expect("ticket"), 1501 invalidation, 1502 SourceGeneration::new([41; 32]).expect("source generation"), 1503 Some(EventPosition::new( 1504 SourceGeneration::new([41; 32]).expect("source generation"), 1505 EventSequence::new(3).expect("sequence"), 1506 )), 1507 RawSourceDigest::new([8; 32]), 1508 ) 1509 .expect("ticket"); 1510 let requested = store 1511 .request_rebuild(ticket.clone()) 1512 .await 1513 .expect("request rebuild"); 1514 assert_eq!( 1515 store.request_rebuild(ticket).await.expect("replay"), 1516 requested 1517 ); 1518 let running = store 1519 .transition_rebuild(RebuildTransition::start( 1520 requested.ticket_id(), 1521 requested.revision(), 1522 210, 1523 )) 1524 .await 1525 .expect("start"); 1526 let progress = store 1527 .transition_rebuild(RebuildTransition::checkpoint( 1528 running.ticket_id(), 1529 running.revision(), 1530 220, 1531 checkpoint(generation(2), 2, 20, 220), 1532 )) 1533 .await 1534 .expect("progress"); 1535 assert_eq!( 1536 store 1537 .transition_rebuild(RebuildTransition::fail( 1538 progress.ticket_id(), 1539 running.revision(), 1540 230, 1541 RebuildFailure::IntegrityFailure, 1542 )) 1543 .await, 1544 Err(Error::ProjectionRevisionConflict) 1545 ); 1546 sqlx::query( 1547 "UPDATE radroots_runtime_source_generations SET sequence_head = 3 WHERE state = 'active'", 1548 ) 1549 .execute(store.pool()) 1550 .await 1551 .expect("advance source high water"); 1552 let completed = store 1553 .transition_rebuild(RebuildTransition::complete( 1554 progress.ticket_id(), 1555 progress.revision(), 1556 240, 1557 checkpoint(generation(2), 3, 30, 240), 1558 )) 1559 .await 1560 .expect("complete"); 1561 assert_eq!(completed.stage(), RebuildStage::Completed); 1562 let status = store 1563 .status(projection_id()) 1564 .await 1565 .expect("status") 1566 .expect("projection"); 1567 assert_eq!(status.health(), ProjectionHealth::Ready); 1568 assert_eq!(status.generation(), generation(2)); 1569 assert_eq!(status.active_rebuild(), None); 1570 } 1571 1572 fn manifest(generation: ProjectionGeneration, digest: u8) -> EventIndexManifest { 1573 EventIndexManifest::new( 1574 generation, 1575 4, 1576 2, 1577 10, 1578 40, 1579 vec![ 1580 EventIndexShard::new( 1581 EventIndexShardId::parse("shard_a").expect("shard id"), 1582 "index/shard_a.bin", 1583 2, 1584 EventIdRange::new(EventId::from_bytes([1; 32]), EventId::from_bytes([2; 32])) 1585 .expect("range"), 1586 10, 1587 20, 1588 ArtifactDigest::new([digest; 32]), 1589 ) 1590 .expect("first shard"), 1591 EventIndexShard::new( 1592 EventIndexShardId::parse("shard_b").expect("shard id"), 1593 "index/shard_b.bin", 1594 2, 1595 EventIdRange::new(EventId::from_bytes([3; 32]), EventId::from_bytes([4; 32])) 1596 .expect("range"), 1597 30, 1598 40, 1599 ArtifactDigest::new([digest.wrapping_add(1); 32]), 1600 ) 1601 .expect("second shard"), 1602 ], 1603 ) 1604 .expect("manifest") 1605 } 1606 1607 fn index_checkpoint(generation: ProjectionGeneration, at: u64) -> EventIndexCheckpoint { 1608 EventIndexCheckpoint::new( 1609 generation, 1610 at, 1611 vec![ 1612 EventIndexShardCheckpoint::new( 1613 EventIndexShardId::parse("shard_b").expect("shard id"), 1614 40, 1615 Some(EventId::from_bytes([4; 32])), 1616 Some("cursor-b".to_owned()), 1617 ) 1618 .expect("second checkpoint"), 1619 EventIndexShardCheckpoint::new( 1620 EventIndexShardId::parse("shard_a").expect("shard id"), 1621 20, 1622 Some(EventId::from_bytes([2; 32])), 1623 None, 1624 ) 1625 .expect("first checkpoint"), 1626 ], 1627 ) 1628 .expect("index checkpoint") 1629 } 1630 1631 #[tokio::test] 1632 async fn event_index_manifest_and_checkpoint_round_trip_and_reject_regression() { 1633 let store = store(EventStoreMode::ReadWrite).await; 1634 let generation = generation(7); 1635 let expected_manifest = manifest(generation, 8); 1636 store 1637 .put_event_index_manifest(expected_manifest.clone()) 1638 .await 1639 .expect("put manifest"); 1640 store 1641 .put_event_index_manifest(expected_manifest.clone()) 1642 .await 1643 .expect("manifest replay"); 1644 assert_eq!( 1645 store 1646 .event_index_manifest(generation) 1647 .await 1648 .expect("manifest lookup") 1649 .expect("manifest"), 1650 expected_manifest 1651 ); 1652 assert_eq!( 1653 store 1654 .put_event_index_manifest(manifest(generation, 9)) 1655 .await, 1656 Err(Error::CorruptProjectionRecord) 1657 ); 1658 1659 let first = index_checkpoint(generation, 100); 1660 store 1661 .put_event_index_checkpoint(first.clone()) 1662 .await 1663 .expect("put checkpoint"); 1664 assert_eq!( 1665 store 1666 .event_index_checkpoint(generation) 1667 .await 1668 .expect("checkpoint lookup") 1669 .expect("checkpoint"), 1670 first 1671 ); 1672 store 1673 .put_event_index_checkpoint(index_checkpoint(generation, 110)) 1674 .await 1675 .expect("advance checkpoint"); 1676 assert_eq!( 1677 store 1678 .put_event_index_checkpoint(index_checkpoint(generation, 109)) 1679 .await, 1680 Err(Error::InvalidEventIndexCheckpoint) 1681 ); 1682 for corrupt in [&[0_u8][..], &[1_u8, 0xff, 0xff][..]] { 1683 sqlx::query( 1684 "UPDATE radroots_runtime_event_index_checkpoints 1685 SET checkpoint = ? WHERE projection_generation = ?", 1686 ) 1687 .bind(corrupt) 1688 .bind(generation.as_bytes().as_slice()) 1689 .execute(store.pool()) 1690 .await 1691 .expect("forge corrupt index checkpoint"); 1692 assert_eq!( 1693 store.event_index_checkpoint(generation).await, 1694 Err(Error::CorruptProjectionRecord) 1695 ); 1696 } 1697 } 1698 1699 #[tokio::test] 1700 async fn materialized_documents_and_frozen_snapshots_round_trip_and_fail_closed() { 1701 let writer = store(EventStoreMode::ReadWrite).await; 1702 let id = projection_id(); 1703 let generation = generation(12); 1704 let first = ProjectionDocument::new("context.alpha".into(), b"{\"cards\":[]}".to_vec()) 1705 .expect("document"); 1706 writer 1707 .put_projection_document(id.clone(), generation, first) 1708 .await 1709 .expect("put document"); 1710 assert_eq!( 1711 writer 1712 .projection_document(id.clone(), generation, "context.alpha".into(),) 1713 .await 1714 .expect("document lookup") 1715 .expect("document") 1716 .value(), 1717 b"{\"cards\":[]}" 1718 ); 1719 writer 1720 .put_projection_document( 1721 id.clone(), 1722 generation, 1723 ProjectionDocument::new("context.alpha".into(), b"{\"cards\":[1]}".to_vec()) 1724 .expect("replacement"), 1725 ) 1726 .await 1727 .expect("replace document"); 1728 assert_eq!( 1729 writer 1730 .projection_document(id.clone(), generation, "context.alpha".into(),) 1731 .await 1732 .expect("document lookup") 1733 .expect("document") 1734 .value(), 1735 b"{\"cards\":[1]}" 1736 ); 1737 1738 let snapshot = ProjectionSnapshot::new( 1739 id.clone(), 1740 [13; 32], 1741 generation, 1742 1_000, 1743 b"{\"frozen\":true}".to_vec(), 1744 ) 1745 .expect("snapshot"); 1746 writer 1747 .put_projection_snapshot(snapshot.clone()) 1748 .await 1749 .expect("put snapshot"); 1750 writer 1751 .put_projection_snapshot(snapshot.clone()) 1752 .await 1753 .expect("idempotent snapshot replay"); 1754 let concurrent_snapshot = ProjectionSnapshot::new( 1755 id.clone(), 1756 [14; 32], 1757 generation, 1758 1_001, 1759 b"{\"frozen\":\"concurrent\"}".to_vec(), 1760 ) 1761 .expect("concurrent snapshot"); 1762 let (left, right) = tokio::join!( 1763 writer.put_projection_snapshot(concurrent_snapshot.clone()), 1764 writer.put_projection_snapshot(concurrent_snapshot), 1765 ); 1766 left.expect("concurrent left snapshot insert"); 1767 right.expect("concurrent right snapshot insert"); 1768 assert_eq!( 1769 writer 1770 .projection_snapshot(id.clone(), [13; 32]) 1771 .await 1772 .expect("snapshot lookup"), 1773 Some(snapshot) 1774 ); 1775 assert_eq!( 1776 writer 1777 .put_projection_snapshot( 1778 ProjectionSnapshot::new( 1779 id.clone(), 1780 [13; 32], 1781 generation, 1782 1_000, 1783 b"{\"frozen\":false}".to_vec(), 1784 ) 1785 .expect("conflicting snapshot"), 1786 ) 1787 .await, 1788 Err(Error::CorruptProjectionDocument) 1789 ); 1790 1791 sqlx::query( 1792 "UPDATE radroots_runtime_projection_documents 1793 SET value = X'00' WHERE projection_id = ?", 1794 ) 1795 .bind(id.as_str()) 1796 .execute(writer.pool()) 1797 .await 1798 .expect("forge corrupt document"); 1799 assert_eq!( 1800 writer 1801 .projection_document(id, generation, "context.alpha".into()) 1802 .await, 1803 Err(Error::CorruptProjectionDocument) 1804 ); 1805 1806 let read_only = store(EventStoreMode::ReadOnly).await; 1807 assert_eq!( 1808 read_only 1809 .put_projection_document( 1810 projection_id(), 1811 generation, 1812 ProjectionDocument::new("context.alpha".into(), vec![1]).expect("document"), 1813 ) 1814 .await, 1815 Err(Error::BackendUnavailable) 1816 ); 1817 } 1818 1819 #[tokio::test] 1820 async fn failed_rebuild_corruption_and_read_only_mode_fail_closed() { 1821 let store = store(EventStoreMode::ReadWrite).await; 1822 let initial = store 1823 .checkpoint(checkpoint(generation(1), 1, 1, 100)) 1824 .await 1825 .expect("checkpoint"); 1826 let encoded = encode_status_snapshot(&initial).expect("encode projection status"); 1827 for end in 0..encoded.len() { 1828 let _ = decode_status_snapshot(&encoded[..end]); 1829 } 1830 let mut trailing = encoded.clone(); 1831 trailing.push(0); 1832 assert_eq!( 1833 decode_status_snapshot(&trailing), 1834 Err(Error::CorruptProjectionRecord) 1835 ); 1836 for index in 0..encoded.len() { 1837 let mut corrupt = encoded.clone(); 1838 corrupt[index] ^= 0xff; 1839 let _ = decode_status_snapshot(&corrupt); 1840 } 1841 let invalidation = invalidation(); 1842 store 1843 .invalidate(invalidation.clone()) 1844 .await 1845 .expect("invalidate"); 1846 let ticket = store 1847 .request_rebuild( 1848 RebuildTicket::requested( 1849 RebuildTicketId::new([9; 16]).expect("ticket"), 1850 invalidation, 1851 SourceGeneration::new([41; 32]).expect("source generation"), 1852 None, 1853 RawSourceDigest::new([8; 32]), 1854 ) 1855 .expect("ticket"), 1856 ) 1857 .await 1858 .expect("request rebuild"); 1859 let failed = store 1860 .transition_rebuild(RebuildTransition::fail( 1861 ticket.ticket_id(), 1862 ticket.revision(), 1863 210, 1864 RebuildFailure::IntegrityFailure, 1865 )) 1866 .await 1867 .expect("fail rebuild"); 1868 assert_eq!(failed.stage(), RebuildStage::Failed); 1869 assert_eq!( 1870 store 1871 .status(projection_id()) 1872 .await 1873 .expect("status") 1874 .expect("projection") 1875 .health(), 1876 ProjectionHealth::Ready 1877 ); 1878 1879 sqlx::query("PRAGMA ignore_check_constraints = ON") 1880 .execute(store.pool()) 1881 .await 1882 .expect("disable constraints"); 1883 sqlx::query( 1884 "UPDATE radroots_runtime_projection_checkpoints 1885 SET projection_generation = X'01' WHERE projection_id = ?", 1886 ) 1887 .bind(projection_id().as_str()) 1888 .execute(store.pool()) 1889 .await 1890 .expect("forge corruption"); 1891 assert_eq!( 1892 store.status(projection_id()).await, 1893 Err(Error::CorruptProjectionRecord) 1894 ); 1895 1896 let read_only = SqliteStorage::new( 1897 store.pool().clone(), 1898 SourceGeneration::new([41; 32]).expect("generation"), 1899 EventStoreMode::ReadOnly, 1900 ); 1901 assert_eq!( 1902 read_only 1903 .checkpoint(checkpoint(generation(3), 1, 1, 300)) 1904 .await, 1905 Err(Error::BackendUnavailable) 1906 ); 1907 } 1908 }