push_enqueue.rs (84387B)
1 use std::{ 2 collections::VecDeque, 3 sync::{ 4 Arc, Mutex, 5 atomic::{AtomicU64, AtomicUsize, Ordering}, 6 }, 7 }; 8 9 use futures::{FutureExt, task::noop_waker_ref}; 10 use futures_executor::block_on; 11 use radroots_event::{ 12 GenericEventDraft, SignedEvent, 13 contract::AuthorRole, 14 draft::SignedEventParts, 15 food::availability::{ 16 FoodAvailabilityDetails, FoodAvailabilityDetailsParts, FoodAvailabilityStatus, FoodContent, 17 FoodCurrency, FoodIdentifier, FoodPrice, FoodPublishedAt, FoodText, FoodUnit, 18 }, 19 }; 20 use radroots_event_codec::authoring::AuthoredEventPlan; 21 use radroots_protocol::runtime::v1::SyncRetryDecision; 22 use radroots_signing::{ 23 Actor, Error as SigningError, SignReceipt, SignRequest, Signer, SignerStatus, 24 actor::ActorSource, 25 capability::{CancellationSupport, SignerCapability, SignerKind}, 26 error::Kind as SigningErrorKind, 27 recovery::ReplayCapability, 28 request::CancellationPolicy, 29 status::SignerAvailability, 30 }; 31 use radroots_storage::{ 32 EventStore, Journal, Outbox, ProjectionStore, 33 atomic::AtomicStorage, 34 authored_atomic::AuthoredAtomicStorage, 35 authored_delivery::AuthoredDeliveryState, 36 event::{EventQuery, EventQueryBounds, SourceGeneration}, 37 journal::{IdempotencyKey, OperationInstanceId}, 38 memory::MemoryStorage, 39 status::StorageStatusProvider, 40 }; 41 use radroots_storage_sqlite::{OpenMode, OpenOptions, Paths, SqliteStorage}; 42 use radroots_sync::{ 43 Engine, PushRequest, 44 policy::{Clock, DeadlinePolicy, Error, IdSource, OperationKind, SyncId, SyncStorage}, 45 }; 46 use radroots_transport::{ 47 DeliveryReceipt, DeliveryRequest, Error as TransportError, EventSink, EventSource, FetchPage, 48 FetchRequest, SinkFailure, SinkStatus, SourceStatus, Target, TargetSet, TransportId, 49 outcome::{DeliveryOutcome, Retryability}, 50 policy::{SatisfactionClass, SatisfactionPolicy, TargetPolicy}, 51 sink::DeliveryTargetReceipt, 52 }; 53 use secp256k1::{Keypair, Message, Secp256k1, SecretKey}; 54 55 const CONTENT: &str = "frozen-content"; 56 57 #[path = "push_enqueue/signing_evidence.rs"] 58 mod signing_evidence; 59 60 #[path = "push_enqueue/delivery_evidence.rs"] 61 mod delivery_evidence; 62 63 #[path = "push_enqueue/delivery_selection.rs"] 64 mod delivery_selection; 65 66 #[path = "push_enqueue/capacity.rs"] 67 mod capacity; 68 69 #[path = "push_enqueue/wire_admission.rs"] 70 mod wire_admission; 71 72 struct MockSink; 73 74 struct FaultStorage { 75 capacity: Mutex<Option<capacity::Fault>>, 76 inner: Arc<MemoryStorage>, 77 delivery_injection: Mutex<Option<delivery_evidence::Injection>>, 78 receipt_mutation: Mutex<Option<delivery_evidence::ReceiptMutation>>, 79 history_mutation: Mutex<Option<(usize, delivery_evidence::HistoryMutation)>>, 80 fault_kind: AtomicUsize, 81 remaining: AtomicUsize, 82 admit_error: AtomicUsize, 83 prepared: Mutex<Option<radroots_storage::authored_atomic::AuthoredAtomicOutcome>>, 84 } 85 86 impl FaultStorage { 87 fn new(generation: u8) -> Self { 88 Self { 89 capacity: Mutex::new(None), 90 delivery_injection: Mutex::new(None), 91 receipt_mutation: Mutex::new(None), 92 history_mutation: Mutex::new(None), 93 inner: Arc::new(MemoryStorage::new( 94 SourceGeneration::new([generation; 32]).expect("generation"), 95 )), 96 fault_kind: AtomicUsize::new(0), 97 remaining: AtomicUsize::new(0), 98 admit_error: AtomicUsize::new(0), 99 prepared: Mutex::new(None), 100 } 101 } 102 103 fn fault_next_prepared(&self) { 104 self.fault_kind.store(1, Ordering::Relaxed); 105 self.remaining.store(1, Ordering::Relaxed); 106 } 107 108 fn fault_nth_artifact(&self, nth: usize) { 109 self.fault_kind.store(2, Ordering::Relaxed); 110 self.remaining.store(nth, Ordering::Relaxed); 111 } 112 113 fn fault_nth_plan(&self, nth: usize) { 114 self.fault_kind.store(3, Ordering::Relaxed); 115 self.remaining.store(nth, Ordering::Relaxed); 116 } 117 118 fn fault_prepared_identity(&self) { 119 self.fault_kind.store(4, Ordering::Relaxed); 120 self.remaining.store(1, Ordering::Relaxed); 121 } 122 123 fn fail_admission_with(&self, value: usize) { 124 self.admit_error.store(value, Ordering::Relaxed); 125 } 126 } 127 128 impl EventStore for FaultStorage { 129 fn status( 130 &self, 131 ) -> radroots_transport::BoxFuture< 132 '_, 133 Result<radroots_storage::status::EventStoreStatus, radroots_storage::Error>, 134 > { 135 EventStore::status(self.inner.as_ref()) 136 } 137 fn admit( 138 &self, 139 value: radroots_storage::event::EventAdmission, 140 ) -> radroots_transport::BoxFuture< 141 '_, 142 Result<radroots_storage::event::AdmissionReceipt, radroots_storage::Error>, 143 > { 144 match self.admit_error.load(Ordering::Relaxed) { 145 1 => Box::pin(async { Err(radroots_storage::Error::EventConflict) }), 146 2 => Box::pin(async { Err(radroots_storage::Error::BackendUnavailable) }), 147 3 => Box::pin(async { Err(radroots_storage::Error::SpaceInsufficient) }), 148 _ => EventStore::admit(self.inner.as_ref(), value), 149 } 150 } 151 fn query_raw( 152 &self, 153 value: radroots_storage::event::EventQuery, 154 ) -> radroots_transport::BoxFuture< 155 '_, 156 Result< 157 radroots_storage::event::EventPage<radroots_storage::event::StoredRawEvent>, 158 radroots_storage::Error, 159 >, 160 > { 161 EventStore::query_raw(self.inner.as_ref(), value) 162 } 163 fn query_verified( 164 &self, 165 value: radroots_storage::event::EventQuery, 166 ) -> radroots_transport::BoxFuture< 167 '_, 168 Result< 169 radroots_storage::event::EventPage<radroots_storage::event::StoredVerifiedEvent>, 170 radroots_storage::Error, 171 >, 172 > { 173 EventStore::query_verified(self.inner.as_ref(), value) 174 } 175 fn query_visible( 176 &self, 177 value: radroots_storage::event::EventQuery, 178 ) -> radroots_transport::BoxFuture< 179 '_, 180 Result< 181 radroots_storage::event::EventPage<radroots_storage::event::StoredVisibleEvent>, 182 radroots_storage::Error, 183 >, 184 > { 185 EventStore::query_visible(self.inner.as_ref(), value) 186 } 187 fn rebuild_visibility( 188 &self, 189 ) -> radroots_transport::BoxFuture< 190 '_, 191 Result<radroots_storage::event::VisibilitySnapshot, radroots_storage::Error>, 192 > { 193 EventStore::rebuild_visibility(self.inner.as_ref()) 194 } 195 fn query_provenance( 196 &self, 197 id: radroots_storage::event::EventId, 198 bounds: radroots_storage::event::EventQueryBounds, 199 ) -> radroots_transport::BoxFuture< 200 '_, 201 Result< 202 radroots_storage::event::EventPage<radroots_storage::event::StoredEventProvenance>, 203 radroots_storage::Error, 204 >, 205 > { 206 EventStore::query_provenance(self.inner.as_ref(), id, bounds) 207 } 208 } 209 210 impl Journal for FaultStorage { 211 fn prepare( 212 &self, 213 value: radroots_storage::journal::PrepareOperation, 214 ) -> radroots_transport::BoxFuture< 215 '_, 216 Result<radroots_storage::journal::PrepareReceipt, radroots_storage::Error>, 217 > { 218 Journal::prepare(self.inner.as_ref(), value) 219 } 220 fn operation( 221 &self, 222 id: OperationInstanceId, 223 ) -> radroots_transport::BoxFuture< 224 '_, 225 Result<Option<radroots_storage::journal::OperationRecord>, radroots_storage::Error>, 226 > { 227 Journal::operation(self.inner.as_ref(), id) 228 } 229 fn by_idempotency_key( 230 &self, 231 id: radroots_storage::journal::OperationId, 232 key: IdempotencyKey, 233 ) -> radroots_transport::BoxFuture< 234 '_, 235 Result<Option<radroots_storage::journal::OperationRecord>, radroots_storage::Error>, 236 > { 237 Journal::by_idempotency_key(self.inner.as_ref(), id, key) 238 } 239 fn transition( 240 &self, 241 value: radroots_storage::journal::JournalTransition, 242 ) -> radroots_transport::BoxFuture< 243 '_, 244 Result<radroots_storage::journal::OperationRecord, radroots_storage::Error>, 245 > { 246 Journal::transition(self.inner.as_ref(), value) 247 } 248 fn recoverable( 249 &self, 250 limit: u16, 251 ) -> radroots_transport::BoxFuture< 252 '_, 253 Result<Vec<radroots_storage::journal::OperationRecord>, radroots_storage::Error>, 254 > { 255 Journal::recoverable(self.inner.as_ref(), limit) 256 } 257 } 258 259 impl Outbox for FaultStorage { 260 fn enqueue( 261 &self, 262 value: radroots_storage::outbox::EnqueueOutboxItem, 263 ) -> radroots_transport::BoxFuture< 264 '_, 265 Result<radroots_storage::outbox::EnqueueReceipt, radroots_storage::Error>, 266 > { 267 Outbox::enqueue(self.inner.as_ref(), value) 268 } 269 fn item( 270 &self, 271 id: radroots_storage::outbox::OutboxItemId, 272 ) -> radroots_transport::BoxFuture< 273 '_, 274 Result<Option<radroots_storage::outbox::OutboxRecord>, radroots_storage::Error>, 275 > { 276 Outbox::item(self.inner.as_ref(), id) 277 } 278 fn claim( 279 &self, 280 value: radroots_storage::outbox::ClaimOutboxItems, 281 ) -> radroots_transport::BoxFuture< 282 '_, 283 Result<Vec<radroots_storage::outbox::ClaimedOutboxItem>, radroots_storage::Error>, 284 > { 285 Outbox::claim(self.inner.as_ref(), value) 286 } 287 fn record_attempt( 288 &self, 289 value: radroots_storage::outbox::DeliveryAttemptEvidence, 290 ) -> radroots_transport::BoxFuture< 291 '_, 292 Result<radroots_storage::outbox::OutboxRecord, radroots_storage::Error>, 293 > { 294 Outbox::record_attempt(self.inner.as_ref(), value) 295 } 296 fn release( 297 &self, 298 id: radroots_storage::outbox::OutboxItemId, 299 lease: radroots_storage::outbox::LeaseId, 300 revision: radroots_storage::outbox::OutboxRevision, 301 at: u64, 302 retry: Option<u64>, 303 ) -> radroots_transport::BoxFuture< 304 '_, 305 Result<radroots_storage::outbox::OutboxRecord, radroots_storage::Error>, 306 > { 307 Outbox::release(self.inner.as_ref(), id, lease, revision, at, retry) 308 } 309 fn status( 310 &self, 311 ) -> radroots_transport::BoxFuture< 312 '_, 313 Result<radroots_storage::outbox::OutboxStatus, radroots_storage::Error>, 314 > { 315 Outbox::status(self.inner.as_ref()) 316 } 317 } 318 319 impl ProjectionStore for FaultStorage { 320 fn status( 321 &self, 322 id: radroots_storage::projection::ProjectionId, 323 ) -> radroots_transport::BoxFuture< 324 '_, 325 Result<Option<radroots_storage::projection::ProjectionStatus>, radroots_storage::Error>, 326 > { 327 ProjectionStore::status(self.inner.as_ref(), id) 328 } 329 fn checkpoint( 330 &self, 331 value: radroots_storage::projection::ProjectionCheckpoint, 332 ) -> radroots_transport::BoxFuture< 333 '_, 334 Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>, 335 > { 336 ProjectionStore::checkpoint(self.inner.as_ref(), value) 337 } 338 fn invalidate( 339 &self, 340 value: radroots_storage::projection::ProjectionInvalidation, 341 ) -> radroots_transport::BoxFuture< 342 '_, 343 Result<radroots_storage::projection::ProjectionStatus, radroots_storage::Error>, 344 > { 345 ProjectionStore::invalidate(self.inner.as_ref(), value) 346 } 347 fn invalidation( 348 &self, 349 id: radroots_storage::projection::ProjectionId, 350 generation: radroots_storage::projection::ProjectionGeneration, 351 ) -> radroots_transport::BoxFuture< 352 '_, 353 Result< 354 Option<radroots_storage::projection::ProjectionInvalidation>, 355 radroots_storage::Error, 356 >, 357 > { 358 ProjectionStore::invalidation(self.inner.as_ref(), id, generation) 359 } 360 fn request_rebuild( 361 &self, 362 value: radroots_storage::projection::RebuildTicket, 363 ) -> radroots_transport::BoxFuture< 364 '_, 365 Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>, 366 > { 367 ProjectionStore::request_rebuild(self.inner.as_ref(), value) 368 } 369 fn rebuild( 370 &self, 371 id: radroots_storage::projection::RebuildTicketId, 372 ) -> radroots_transport::BoxFuture< 373 '_, 374 Result<Option<radroots_storage::projection::RebuildTicket>, radroots_storage::Error>, 375 > { 376 ProjectionStore::rebuild(self.inner.as_ref(), id) 377 } 378 fn transition_rebuild( 379 &self, 380 value: radroots_storage::projection::RebuildTransition, 381 ) -> radroots_transport::BoxFuture< 382 '_, 383 Result<radroots_storage::projection::RebuildTicket, radroots_storage::Error>, 384 > { 385 ProjectionStore::transition_rebuild(self.inner.as_ref(), value) 386 } 387 fn event_index_manifest( 388 &self, 389 generation: radroots_storage::projection::ProjectionGeneration, 390 ) -> radroots_transport::BoxFuture< 391 '_, 392 Result<Option<radroots_storage::projection::EventIndexManifest>, radroots_storage::Error>, 393 > { 394 ProjectionStore::event_index_manifest(self.inner.as_ref(), generation) 395 } 396 fn put_event_index_manifest( 397 &self, 398 value: radroots_storage::projection::EventIndexManifest, 399 ) -> radroots_transport::BoxFuture<'_, Result<(), radroots_storage::Error>> { 400 ProjectionStore::put_event_index_manifest(self.inner.as_ref(), value) 401 } 402 fn event_index_checkpoint( 403 &self, 404 generation: radroots_storage::projection::ProjectionGeneration, 405 ) -> radroots_transport::BoxFuture< 406 '_, 407 Result<Option<radroots_storage::projection::EventIndexCheckpoint>, radroots_storage::Error>, 408 > { 409 ProjectionStore::event_index_checkpoint(self.inner.as_ref(), generation) 410 } 411 fn put_event_index_checkpoint( 412 &self, 413 value: radroots_storage::projection::EventIndexCheckpoint, 414 ) -> radroots_transport::BoxFuture<'_, Result<(), radroots_storage::Error>> { 415 ProjectionStore::put_event_index_checkpoint(self.inner.as_ref(), value) 416 } 417 fn put_projection_document( 418 &self, 419 projection_id: radroots_storage::projection::ProjectionId, 420 generation: radroots_storage::projection::ProjectionGeneration, 421 document: radroots_storage::projection::ProjectionDocument, 422 ) -> radroots_transport::BoxFuture<'_, Result<(), radroots_storage::Error>> { 423 ProjectionStore::put_projection_document( 424 self.inner.as_ref(), 425 projection_id, 426 generation, 427 document, 428 ) 429 } 430 fn projection_document( 431 &self, 432 projection_id: radroots_storage::projection::ProjectionId, 433 generation: radroots_storage::projection::ProjectionGeneration, 434 key: String, 435 ) -> radroots_transport::BoxFuture< 436 '_, 437 Result<Option<radroots_storage::projection::ProjectionDocument>, radroots_storage::Error>, 438 > { 439 ProjectionStore::projection_document(self.inner.as_ref(), projection_id, generation, key) 440 } 441 fn put_projection_snapshot( 442 &self, 443 snapshot: radroots_storage::projection::ProjectionSnapshot, 444 ) -> radroots_transport::BoxFuture<'_, Result<(), radroots_storage::Error>> { 445 ProjectionStore::put_projection_snapshot(self.inner.as_ref(), snapshot) 446 } 447 fn projection_snapshot( 448 &self, 449 projection_id: radroots_storage::projection::ProjectionId, 450 snapshot_id: [u8; 32], 451 ) -> radroots_transport::BoxFuture< 452 '_, 453 Result<Option<radroots_storage::projection::ProjectionSnapshot>, radroots_storage::Error>, 454 > { 455 ProjectionStore::projection_snapshot(self.inner.as_ref(), projection_id, snapshot_id) 456 } 457 } 458 459 impl AtomicStorage for FaultStorage { 460 fn commit( 461 &self, 462 value: radroots_storage::atomic::AtomicCommit, 463 ) -> radroots_transport::BoxFuture< 464 '_, 465 Result<radroots_storage::atomic::AtomicCommitReceipt, radroots_storage::Error>, 466 > { 467 AtomicStorage::commit(self.inner.as_ref(), value) 468 } 469 fn receipt( 470 &self, 471 id: radroots_storage::atomic::AtomicCommitId, 472 ) -> radroots_transport::BoxFuture< 473 '_, 474 Result<Option<radroots_storage::atomic::AtomicCommitReceipt>, radroots_storage::Error>, 475 > { 476 AtomicStorage::receipt(self.inner.as_ref(), id) 477 } 478 } 479 480 impl StorageStatusProvider for FaultStorage { 481 fn storage_status( 482 &self, 483 ) -> radroots_transport::BoxFuture< 484 '_, 485 Result<radroots_storage::status::StorageStatus, radroots_storage::Error>, 486 > { 487 StorageStatusProvider::storage_status(self.inner.as_ref()) 488 } 489 } 490 491 impl AuthoredAtomicStorage for FaultStorage { 492 fn execute_authored( 493 &self, 494 command: radroots_storage::authored_atomic::AuthoredAtomicCommand, 495 ) -> radroots_transport::BoxFuture< 496 '_, 497 Result<radroots_storage::authored_atomic::AuthoredAtomicReceipt, radroots_storage::Error>, 498 > { 499 Box::pin(async move { 500 delivery_evidence::inject(self, &command, true).await; 501 capacity::inject(self, &command, false)?; 502 let receipt = 503 AuthoredAtomicStorage::execute_authored(self.inner.as_ref(), command.clone()) 504 .await?; 505 capacity::inject(self, &command, true)?; 506 delivery_evidence::inject(self, &command, false).await; 507 if matches!( 508 receipt.outcome(), 509 radroots_storage::authored_atomic::AuthoredAtomicOutcome::DeliveryPlan(_) 510 ) && let Some(mutation) = self.receipt_mutation.lock().unwrap().take() 511 { 512 return Ok(delivery_evidence::mutate_receipt(receipt, mutation)); 513 } 514 if let radroots_storage::authored_atomic::AuthoredAtomicOutcome::Prepared { .. } = 515 receipt.outcome() 516 { 517 let mut cache = self.prepared.lock().expect("prepared cache"); 518 if cache.is_none() { 519 *cache = Some(receipt.outcome().clone()); 520 } 521 } 522 let kind = match receipt.outcome() { 523 radroots_storage::authored_atomic::AuthoredAtomicOutcome::Prepared { .. } => 1, 524 radroots_storage::authored_atomic::AuthoredAtomicOutcome::Artifact(_) => 2, 525 radroots_storage::authored_atomic::AuthoredAtomicOutcome::DeliveryPlan(_) => 3, 526 radroots_storage::authored_atomic::AuthoredAtomicOutcome::Submitted(_) => 5, 527 }; 528 let fault = self.fault_kind.load(Ordering::Relaxed); 529 if (fault != kind && !(fault == 4 && kind == 1)) 530 || self.remaining.fetch_sub(1, Ordering::Relaxed) != 1 531 { 532 return Ok(receipt); 533 } 534 let cached = self 535 .prepared 536 .lock() 537 .expect("prepared cache") 538 .clone() 539 .expect("prepared outcome"); 540 let forged = match (fault, cached) { 541 (4, cached) => cached, 542 ( 543 1, 544 radroots_storage::authored_atomic::AuthoredAtomicOutcome::Prepared { 545 artifacts, 546 .. 547 }, 548 ) => radroots_storage::authored_atomic::AuthoredAtomicOutcome::Artifact( 549 artifacts[0].clone(), 550 ), 551 ( 552 2, 553 radroots_storage::authored_atomic::AuthoredAtomicOutcome::Prepared { 554 delivery_plans, 555 .. 556 }, 557 ) => radroots_storage::authored_atomic::AuthoredAtomicOutcome::DeliveryPlan( 558 delivery_plans[0].clone(), 559 ), 560 ( 561 3, 562 radroots_storage::authored_atomic::AuthoredAtomicOutcome::Prepared { 563 artifacts, 564 .. 565 }, 566 ) => radroots_storage::authored_atomic::AuthoredAtomicOutcome::Artifact( 567 artifacts[0].clone(), 568 ), 569 _ => unreachable!(), 570 }; 571 radroots_storage::authored_atomic::AuthoredAtomicReceipt::from_durable_parts( 572 receipt.commit_id(), 573 receipt.digest(), 574 receipt.disposition(), 575 receipt.committed_at_unix_ms(), 576 forged, 577 ) 578 }) 579 } 580 fn authored_receipt( 581 &self, 582 id: radroots_storage::atomic::AtomicCommitId, 583 ) -> radroots_transport::BoxFuture< 584 '_, 585 Result< 586 Option<radroots_storage::authored_atomic::AuthoredAtomicReceipt>, 587 radroots_storage::Error, 588 >, 589 > { 590 AuthoredAtomicStorage::authored_receipt(self.inner.as_ref(), id) 591 } 592 fn authored_operation( 593 &self, 594 id: OperationInstanceId, 595 ) -> radroots_transport::BoxFuture< 596 '_, 597 Result<Option<radroots_storage::authored::AuthoredOperation>, radroots_storage::Error>, 598 > { 599 AuthoredAtomicStorage::authored_operation(self.inner.as_ref(), id) 600 } 601 fn authored_artifact( 602 &self, 603 id: radroots_storage::authored::AuthoredArtifactId, 604 ) -> radroots_transport::BoxFuture< 605 '_, 606 Result<Option<radroots_storage::authored::AuthoredArtifact>, radroots_storage::Error>, 607 > { 608 AuthoredAtomicStorage::authored_artifact(self.inner.as_ref(), id) 609 } 610 fn authored_delivery_plan( 611 &self, 612 id: radroots_storage::authored_delivery::AuthoredDeliveryPlanId, 613 ) -> radroots_transport::BoxFuture< 614 '_, 615 Result< 616 Option<radroots_storage::authored_delivery::AuthoredDeliveryPlan>, 617 radroots_storage::Error, 618 >, 619 > { 620 AuthoredAtomicStorage::authored_delivery_plan(self.inner.as_ref(), id) 621 } 622 fn authored_delivery_history( 623 &self, 624 id: radroots_storage::authored_delivery::AuthoredDeliveryPlanId, 625 ) -> radroots_transport::BoxFuture< 626 '_, 627 Result< 628 Option<radroots_storage::authored_delivery::AuthoredDeliveryHistory>, 629 radroots_storage::Error, 630 >, 631 > { 632 Box::pin(async move { 633 let history = self.inner.authored_delivery_history(id).await?; 634 let mutation = { 635 let mut armed = self.history_mutation.lock().unwrap(); 636 if let Some((remaining, _)) = armed.as_mut() { 637 *remaining -= 1; 638 } 639 if armed.as_ref().is_some_and(|(remaining, _)| *remaining == 0) { 640 armed.take().map(|(_, mutation)| mutation) 641 } else { 642 None 643 } 644 }; 645 Ok(match (history, mutation) { 646 (Some(history), Some(mutation)) => { 647 Some(delivery_evidence::mutate_history(history, mutation)) 648 } 649 (history, _) => history, 650 }) 651 }) 652 } 653 } 654 655 struct MockSource; 656 657 impl EventSource for MockSource { 658 fn status(&self) -> radroots_transport::BoxFuture<'_, Result<SourceStatus, TransportError>> { 659 Box::pin(async { unreachable!("missing-sink check does not inspect source") }) 660 } 661 662 fn fetch( 663 &self, 664 _request: FetchRequest, 665 ) -> radroots_transport::BoxFuture<'_, Result<FetchPage, TransportError>> { 666 Box::pin(async { unreachable!("missing-sink check does not fetch") }) 667 } 668 } 669 670 impl EventSink for MockSink { 671 fn status(&self) -> radroots_transport::BoxFuture<'_, Result<SinkStatus, TransportError>> { 672 Box::pin(async { unreachable!("enqueue does not inspect sink") }) 673 } 674 fn deliver( 675 &self, 676 _request: DeliveryRequest, 677 ) -> radroots_transport::BoxFuture<'_, Result<DeliveryReceipt, SinkFailure>> { 678 Box::pin(async { unreachable!("enqueue does not deliver") }) 679 } 680 } 681 682 enum DeliveryBehavior { 683 Outcomes(Vec<DeliveryOutcome>), 684 MismatchedRequest, 685 Failure(Retryability), 686 MismatchedFailure, 687 Pending, 688 } 689 690 struct ScriptedSink { 691 behaviors: Mutex<VecDeque<DeliveryBehavior>>, 692 requests: Mutex<Vec<DeliveryRequest>>, 693 } 694 695 impl ScriptedSink { 696 fn new(behaviors: impl IntoIterator<Item = DeliveryBehavior>) -> Self { 697 Self { 698 behaviors: Mutex::new(behaviors.into_iter().collect()), 699 requests: Mutex::new(Vec::new()), 700 } 701 } 702 } 703 704 impl EventSink for ScriptedSink { 705 fn status(&self) -> radroots_transport::BoxFuture<'_, Result<SinkStatus, TransportError>> { 706 Box::pin(async { unreachable!("delivery does not inspect sink status") }) 707 } 708 709 fn deliver( 710 &self, 711 request: DeliveryRequest, 712 ) -> radroots_transport::BoxFuture<'_, Result<DeliveryReceipt, SinkFailure>> { 713 self.requests 714 .lock() 715 .expect("scripted request lock") 716 .push(request.clone()); 717 let behavior = self 718 .behaviors 719 .lock() 720 .expect("scripted behavior lock") 721 .pop_front() 722 .expect("scripted delivery behavior"); 723 Box::pin(async move { 724 match behavior { 725 DeliveryBehavior::Outcomes(outcomes) => { 726 Ok(receipt(&request, outcomes).expect("valid scripted receipt")) 727 } 728 DeliveryBehavior::MismatchedRequest => { 729 let mismatched = DeliveryRequest::new( 730 "mismatched-request", 731 request.payload().clone(), 732 request.target_set().clone(), 733 request.satisfaction().clone(), 734 request.deadline_unix_ms(), 735 ) 736 .expect("mismatched request"); 737 Ok(receipt( 738 &mismatched, 739 vec![DeliveryOutcome::accepted(); mismatched.target_set().len()], 740 ) 741 .expect("mismatched receipt")) 742 } 743 DeliveryBehavior::Failure(retryability) => Err(SinkFailure::for_request( 744 &request, 745 "scripted_failure", 746 retryability, 747 None, 748 Some("scripted failure".to_owned()), 749 Vec::new(), 750 ) 751 .expect("valid scripted failure")), 752 DeliveryBehavior::MismatchedFailure => { 753 let mismatched = DeliveryRequest::new( 754 "mismatched-failure", 755 request.payload().clone(), 756 request.target_set().clone(), 757 request.satisfaction().clone(), 758 request.deadline_unix_ms(), 759 ) 760 .expect("mismatched request"); 761 Err(SinkFailure::for_request( 762 &mismatched, 763 "mismatched_failure", 764 Retryability::Terminal, 765 None, 766 None, 767 Vec::new(), 768 ) 769 .expect("mismatched failure")) 770 } 771 DeliveryBehavior::Pending => std::future::pending().await, 772 } 773 }) 774 } 775 } 776 777 struct TestClock(AtomicU64); 778 779 impl Clock for TestClock { 780 fn now_unix_ms(&self) -> Result<u64, Error> { 781 Ok(self.0.fetch_add(1, Ordering::Relaxed)) 782 } 783 } 784 785 struct TestIds(AtomicU64); 786 787 impl IdSource for TestIds { 788 fn next_id(&self, _operation: OperationKind) -> Result<SyncId, Error> { 789 let value = self.0.fetch_add(1, Ordering::Relaxed); 790 let byte = u8::try_from(value).map_err(|_| Error::InvalidSyncId)?; 791 SyncId::new([byte; 16]) 792 } 793 } 794 795 #[derive(Clone, Copy)] 796 enum SignBehavior { 797 Success { completed_at_unix_ms: u64 }, 798 Error(SigningErrorKind), 799 Uncertain(SigningErrorKind), 800 Pending, 801 } 802 803 struct MockSigner { 804 behavior: SignBehavior, 805 replay: ReplayCapability, 806 calls: AtomicUsize, 807 } 808 809 impl MockSigner { 810 fn new(behavior: SignBehavior) -> Self { 811 Self { 812 behavior, 813 replay: ReplayCapability::LocalReplaySafe, 814 calls: AtomicUsize::new(0), 815 } 816 } 817 818 fn with_replay(behavior: SignBehavior, replay: ReplayCapability) -> Self { 819 Self { 820 behavior, 821 replay, 822 calls: AtomicUsize::new(0), 823 } 824 } 825 } 826 827 impl Signer for MockSigner { 828 fn status( 829 &self, 830 ) -> radroots_signing::signer::BoxFuture<'_, Result<SignerStatus, SigningError>> { 831 let replay = self.replay; 832 Box::pin(async move { 833 Ok(SignerStatus::new( 834 SignerAvailability::Ready, 835 vec![SignerCapability::new( 836 if replay == ReplayCapability::LocalReplaySafe { 837 SignerKind::Local 838 } else { 839 SignerKind::Remote 840 }, 841 replay, 842 CancellationSupport::BeforePublication, 843 false, 844 false, 845 )], 846 None, 847 )) 848 }) 849 } 850 851 fn sign( 852 &self, 853 request: SignRequest, 854 ) -> radroots_signing::signer::BoxFuture<'_, Result<SignReceipt, SigningError>> { 855 self.calls.fetch_add(1, Ordering::Relaxed); 856 Box::pin(async move { 857 match self.behavior { 858 SignBehavior::Success { 859 completed_at_unix_ms, 860 } => SignReceipt::from_signed_event( 861 &request, 862 signed_event(&request), 863 completed_at_unix_ms, 864 ), 865 SignBehavior::Error(kind) => Err(SigningError::new(kind)), 866 SignBehavior::Uncertain(kind) => { 867 Err(SigningError::new(kind).with_possible_remote_effect()) 868 } 869 SignBehavior::Pending => std::future::pending().await, 870 } 871 }) 872 } 873 } 874 875 #[derive(Clone, Copy)] 876 enum BoundaryViolation { 877 CompletesAfterDeadline, 878 CancelsBeforeReturn, 879 } 880 881 struct BoundaryViolatingSigner { 882 violation: BoundaryViolation, 883 clock: Arc<TestClock>, 884 } 885 886 impl Signer for BoundaryViolatingSigner { 887 fn status( 888 &self, 889 ) -> radroots_signing::signer::BoxFuture<'_, Result<SignerStatus, SigningError>> { 890 Box::pin(async { 891 Ok(SignerStatus::new( 892 SignerAvailability::Ready, 893 vec![SignerCapability::new( 894 SignerKind::Remote, 895 ReplayCapability::ExactReplayByRequestId, 896 CancellationSupport::BeforeAndAfterPublication, 897 false, 898 false, 899 )], 900 None, 901 )) 902 }) 903 } 904 905 fn sign( 906 &self, 907 request: SignRequest, 908 ) -> radroots_signing::signer::BoxFuture<'_, Result<SignReceipt, SigningError>> { 909 Box::pin(async move { 910 let deadline = request.policy().deadline_unix_ms(); 911 let receipt = 912 SignReceipt::from_signed_event(&request, signed_event(&request), deadline - 1)?; 913 match self.violation { 914 BoundaryViolation::CompletesAfterDeadline => { 915 self.clock.0.store(deadline, Ordering::Release); 916 } 917 BoundaryViolation::CancelsBeforeReturn => request.cancellation_signal().cancel(), 918 } 919 Ok(receipt) 920 }) 921 } 922 } 923 924 fn signing_keypair() -> Keypair { 925 let secret = SecretKey::from_slice(&[1; 32]).expect("secret key"); 926 Keypair::from_secret_key(&Secp256k1::new(), &secret) 927 } 928 929 fn public_key_hex() -> String { 930 signing_keypair().x_only_public_key().0.to_string() 931 } 932 933 fn signed_event(request: &SignRequest) -> SignedEvent { 934 let id = request.expected_event_id().to_hex(); 935 let pubkey = public_key_hex(); 936 let signature = Secp256k1::new() 937 .sign_schnorr_no_aux_rand( 938 &Message::from_digest(*request.expected_event_id().as_bytes()), 939 &signing_keypair(), 940 ) 941 .to_string(); 942 let raw_json = format!( 943 "{{\"id\":\"{id}\",\"pubkey\":\"{pubkey}\",\"created_at\":{},\"kind\":{},\"tags\":{:?},\"content\":{content:?},\"sig\":\"{signature}\"}}", 944 request.created_at(), 945 request.kind(), 946 request.tags(), 947 content = request.content(), 948 ); 949 SignedEvent::new(SignedEventParts { 950 id, 951 pubkey, 952 created_at: request.created_at(), 953 kind: request.kind(), 954 tags: request.tags().to_vec(), 955 content: request.content().to_owned(), 956 sig: signature, 957 raw_json, 958 }) 959 .expect("signed event") 960 } 961 962 fn request(operation_byte: u8, relay: &str) -> PushRequest { 963 request_with_policy( 964 operation_byte, 965 &[relay], 966 SatisfactionClass::Accepted, 967 TargetPolicy::any(), 968 ) 969 } 970 971 fn food_request(operation_byte: u8, relay: &str) -> PushRequest { 972 let created_at = 1_800_000_100; 973 let details = FoodAvailabilityDetails::new(FoodAvailabilityDetailsParts { 974 content: FoodContent::new("Carrots available this week.").expect("food content"), 975 identifier: FoodIdentifier::parse("nantes-carrots").expect("food identifier"), 976 title: FoodText::new("Nantes Carrots").expect("food title"), 977 summary: FoodText::new("Fresh bunches").expect("food summary"), 978 published_at: FoodPublishedAt::new(created_at).expect("published at"), 979 location: FoodText::new("Central Saanich, BC").expect("food location"), 980 price: FoodPrice::new( 981 "3", 982 FoodCurrency::parse("CAD").expect("currency"), 983 FoodUnit::Pound, 984 ) 985 .expect("food price"), 986 quantity: None, 987 status: FoodAvailabilityStatus::Active, 988 images: Vec::new(), 989 }) 990 .expect("food availability"); 991 let plan = AuthoredEventPlan::from_food_availability(&details, created_at, public_key_hex()) 992 .expect("typed food plan"); 993 let actor = Actor::new( 994 *plan.author(), 995 ActorSource::ExplicitPublicKey, 996 [AuthorRole::Seller], 997 ) 998 .expect("food actor"); 999 PushRequest::new( 1000 SyncId::new([operation_byte; 16]).expect("operation id"), 1001 IdempotencyKey::parse(format!("food-push-{operation_byte}")).expect("idempotency key"), 1002 actor, 1003 plan, 1004 TargetSet::new(vec![ 1005 Target::new(TransportId::NOSTR, relay).expect("target"), 1006 ]) 1007 .expect("targets"), 1008 SatisfactionPolicy::new(SatisfactionClass::Accepted, TargetPolicy::any()), 1009 1_800_000_300_000, 1010 CancellationPolicy::PreservePublishedRequest, 1011 ) 1012 .expect("food push request") 1013 } 1014 1015 fn request_with_policy( 1016 operation_byte: u8, 1017 relays: &[&str], 1018 class: SatisfactionClass, 1019 target_policy: TargetPolicy, 1020 ) -> PushRequest { 1021 let pubkey = public_key_hex(); 1022 let plan = AuthoredEventPlan::from_generic( 1023 GenericEventDraft::new( 1024 "radroots.social.geochat.v1", 1025 20_000, 1026 1_800_000_100, 1027 vec![], 1028 CONTENT, 1029 pubkey, 1030 ) 1031 .expect("draft"), 1032 ) 1033 .expect("authored plan"); 1034 let actor = Actor::new( 1035 *plan.author(), 1036 ActorSource::ExplicitPublicKey, 1037 [AuthorRole::Any], 1038 ) 1039 .expect("actor"); 1040 PushRequest::new( 1041 SyncId::new([operation_byte; 16]).expect("operation id"), 1042 IdempotencyKey::parse(format!("push-{operation_byte}")).expect("idempotency key"), 1043 actor, 1044 plan, 1045 TargetSet::new( 1046 relays 1047 .iter() 1048 .map(|relay| Target::new(TransportId::NOSTR, *relay).expect("target")) 1049 .collect(), 1050 ) 1051 .expect("targets"), 1052 SatisfactionPolicy::new(class, target_policy), 1053 1_800_000_300_000, 1054 CancellationPolicy::PreservePublishedRequest, 1055 ) 1056 .expect("push request") 1057 } 1058 1059 fn receipt( 1060 request: &DeliveryRequest, 1061 outcomes: Vec<DeliveryOutcome>, 1062 ) -> Result<DeliveryReceipt, TransportError> { 1063 let targets = request 1064 .target_set() 1065 .targets() 1066 .iter() 1067 .cloned() 1068 .zip(outcomes) 1069 .map(|(target, outcome)| DeliveryTargetReceipt::attempted(target, outcome)) 1070 .collect(); 1071 DeliveryReceipt::for_request(request, targets) 1072 } 1073 1074 fn setup_engine(signer: Arc<MockSigner>) -> (Engine, Arc<MemoryStorage>) { 1075 setup_engine_with_sink(signer, Arc::new(MockSink)).0 1076 } 1077 1078 fn setup_engine_with_sink( 1079 signer: Arc<MockSigner>, 1080 sink: Arc<dyn EventSink>, 1081 ) -> ((Engine, Arc<MemoryStorage>), Arc<TestClock>) { 1082 let storage = Arc::new(MemoryStorage::new( 1083 SourceGeneration::new([6; 32]).expect("generation"), 1084 )); 1085 let capability: Arc<dyn SyncStorage> = storage.clone(); 1086 let clock = Arc::new(TestClock(AtomicU64::new(1_800_000_200_000))); 1087 let engine = Engine::builder( 1088 capability, 1089 clock.clone(), 1090 Arc::new(TestIds(AtomicU64::new(10))), 1091 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 1092 ) 1093 .sink(sink) 1094 .signer(signer) 1095 .build() 1096 .expect("engine"); 1097 ((engine, storage), clock) 1098 } 1099 1100 fn fault_engine( 1101 storage: Arc<FaultStorage>, 1102 signer: Arc<MockSigner>, 1103 sink: Arc<dyn EventSink>, 1104 ) -> Engine { 1105 let capability: Arc<dyn SyncStorage> = storage; 1106 Engine::builder( 1107 capability, 1108 Arc::new(TestClock(AtomicU64::new(1_800_000_200_000))), 1109 Arc::new(TestIds(AtomicU64::new(10))), 1110 DeadlinePolicy::new(10_000, 10_000, 10_000).unwrap(), 1111 ) 1112 .sink(sink) 1113 .signer(signer) 1114 .build() 1115 .unwrap() 1116 } 1117 1118 fn execute_to_admitted(engine: &Engine, push: &PushRequest) { 1119 block_on(engine.sign_prepared(push.clone())).expect("sign prepared push"); 1120 block_on(engine.admit_signed(push.operation_id())).expect("admit signed push"); 1121 } 1122 1123 #[test] 1124 fn typed_only_authored_contract_identity_survives_signing_and_local_admission() { 1125 let signer = Arc::new(MockSigner::new(SignBehavior::Success { 1126 completed_at_unix_ms: 1_800_000_200_500, 1127 })); 1128 let (engine, storage) = setup_engine(signer); 1129 let push = food_request(9, "wss://relay.example"); 1130 1131 block_on(engine.sign_prepared(push.clone())).expect("sign typed food plan"); 1132 let admitted = block_on(engine.admit_signed(push.operation_id())) 1133 .expect("admit with the frozen typed contract identity"); 1134 assert!(admitted.artifact().admission_state().is_admitted()); 1135 let events = block_on(storage.query_visible(EventQuery::all( 1136 EventQueryBounds::first(10).expect("bounds"), 1137 ))) 1138 .expect("visible authored events"); 1139 assert_eq!(events.items().len(), 1); 1140 assert_eq!(events.items()[0].event().kind(), 30_402); 1141 } 1142 1143 #[test] 1144 fn caller_retains_late_and_cancelled_signer_facts_before_reporting_wait_outcome() { 1145 for (byte, violation, expected) in [ 1146 ( 1147 55, 1148 BoundaryViolation::CompletesAfterDeadline, 1149 Error::SignerDeadlineExceeded, 1150 ), 1151 ( 1152 56, 1153 BoundaryViolation::CancelsBeforeReturn, 1154 Error::SigningCancelled, 1155 ), 1156 ] { 1157 let storage = Arc::new(MemoryStorage::new( 1158 SourceGeneration::new([byte; 32]).expect("generation"), 1159 )); 1160 let capability: Arc<dyn SyncStorage> = storage; 1161 let clock = Arc::new(TestClock(AtomicU64::new(1_800_000_200_000))); 1162 let signer: Arc<dyn Signer> = Arc::new(BoundaryViolatingSigner { 1163 violation, 1164 clock: Arc::clone(&clock), 1165 }); 1166 let engine = Engine::builder( 1167 capability, 1168 clock, 1169 Arc::new(TestIds(AtomicU64::new(10))), 1170 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 1171 ) 1172 .sink(Arc::new(MockSink)) 1173 .signer(signer) 1174 .build() 1175 .expect("engine"); 1176 let push = request(byte, "wss://relay.example"); 1177 1178 assert_eq!(block_on(engine.sign_prepared(push.clone())), Err(expected)); 1179 let status = block_on(engine.push_status(push.operation_id())) 1180 .expect("status") 1181 .expect("prepared operation"); 1182 let signed = status 1183 .artifact() 1184 .signed() 1185 .expect("retain verified signer facts"); 1186 assert_eq!(signed.event().id(), push.plan().expected_event_id()); 1187 assert_eq!(signed.event().created_at(), push.plan().created_at()); 1188 assert_eq!( 1189 status.artifact().admission_state(), 1190 if matches!(violation, BoundaryViolation::CancelsBeforeReturn) { 1191 radroots_storage::authored::AdmissionState::Cancelled 1192 } else { 1193 radroots_storage::authored::AdmissionState::Pending 1194 } 1195 ); 1196 assert!(status.delivery_plan().attempts().is_empty()); 1197 } 1198 } 1199 1200 #[test] 1201 fn preparation_is_atomic_status_visible_and_replays_without_external_effects() { 1202 let signer = Arc::new(MockSigner::new(SignBehavior::Pending)); 1203 let (engine, storage) = setup_engine(signer.clone()); 1204 let push = request(6, "wss://relay.example"); 1205 1206 let never_polled = engine.prepare_push(push.clone()); 1207 drop(never_polled); 1208 assert!( 1209 block_on(engine.push_status(push.operation_id())) 1210 .expect("status before prepare") 1211 .is_none() 1212 ); 1213 1214 let prepared = block_on(engine.prepare_push(push.clone())).expect("prepare"); 1215 assert!(!prepared.is_replay()); 1216 assert_eq!( 1217 prepared.operation().artifact_ids(), 1218 &[prepared.artifact().artifact_id()] 1219 ); 1220 assert_eq!( 1221 prepared.delivery_plan().artifact_id(), 1222 prepared.artifact().artifact_id() 1223 ); 1224 assert!(prepared.delivery_plan().request().is_none()); 1225 assert!( 1226 block_on(Journal::operation( 1227 &*storage, 1228 OperationInstanceId::new(*push.operation_id().as_bytes()).expect("operation"), 1229 )) 1230 .expect("legacy journal lookup") 1231 .is_none() 1232 ); 1233 1234 let status = block_on(engine.push_status(push.operation_id())) 1235 .expect("status after prepare") 1236 .expect("prepared status"); 1237 assert_eq!(status.operation(), prepared.operation()); 1238 assert_eq!(status.artifact(), prepared.artifact()); 1239 assert_eq!(status.delivery_plan(), prepared.delivery_plan()); 1240 1241 let replay = block_on(engine.prepare_push(push)).expect("exact replay"); 1242 assert!(replay.is_replay()); 1243 assert_eq!(replay.operation(), prepared.operation()); 1244 assert_eq!(replay.artifact(), prepared.artifact()); 1245 assert_eq!(replay.delivery_plan(), prepared.delivery_plan()); 1246 assert_eq!(signer.calls.load(Ordering::Relaxed), 0); 1247 1248 assert_eq!( 1249 block_on(engine.prepare_push(request(6, "wss://other.example"))), 1250 Err(Error::StorageConflict) 1251 ); 1252 assert_eq!(signer.calls.load(Ordering::Relaxed), 0); 1253 } 1254 1255 #[test] 1256 fn phase_aware_cancellation_preserves_signed_artifacts_and_is_reentrant() { 1257 let unsigned_signer = Arc::new(MockSigner::new(SignBehavior::Pending)); 1258 let (unsigned_engine, _) = setup_engine(unsigned_signer.clone()); 1259 let unsigned = request(61, "wss://relay.example"); 1260 block_on(unsigned_engine.prepare_push(unsigned.clone())).expect("prepare unsigned"); 1261 let cancelled = block_on(unsigned_engine.cancel_push(unsigned.operation_id())) 1262 .expect("cancel unsigned operation"); 1263 assert!(cancelled.changed()); 1264 assert_eq!( 1265 cancelled.status().artifact().signing_state(), 1266 radroots_storage::authored::SigningState::Cancelled 1267 ); 1268 assert_eq!( 1269 cancelled.status().delivery_plan().state(), 1270 AuthoredDeliveryState::Cancelled 1271 ); 1272 assert!(cancelled.status().settlement().is_settled()); 1273 let replay = block_on(unsigned_engine.cancel_push(unsigned.operation_id())) 1274 .expect("replay cancellation"); 1275 assert!(!replay.changed()); 1276 assert_eq!(replay.status(), cancelled.status()); 1277 assert_eq!(unsigned_signer.calls.load(Ordering::Relaxed), 0); 1278 1279 let signed_signer = Arc::new(MockSigner::new(SignBehavior::Success { 1280 completed_at_unix_ms: 1_800_000_200_010, 1281 })); 1282 let (signed_engine, _) = setup_engine(signed_signer); 1283 let signed = request(62, "wss://relay.example"); 1284 block_on(signed_engine.sign_prepared(signed.clone())).expect("sign operation"); 1285 let before = block_on(signed_engine.push_status(signed.operation_id())) 1286 .unwrap() 1287 .unwrap(); 1288 let raw = before 1289 .artifact() 1290 .signed() 1291 .expect("exact signed artifact") 1292 .event() 1293 .raw_json() 1294 .to_owned(); 1295 let cancelled = block_on(signed_engine.cancel_push(signed.operation_id())) 1296 .expect("cancel signed operation"); 1297 assert_eq!( 1298 cancelled.status().artifact().admission_state(), 1299 radroots_storage::authored::AdmissionState::Cancelled 1300 ); 1301 assert_eq!( 1302 cancelled 1303 .status() 1304 .artifact() 1305 .signed() 1306 .expect("signed bytes retained") 1307 .event() 1308 .raw_json(), 1309 raw 1310 ); 1311 assert_eq!( 1312 cancelled.status().delivery_plan().state(), 1313 AuthoredDeliveryState::Cancelled 1314 ); 1315 1316 let admitted_signer = Arc::new(MockSigner::new(SignBehavior::Success { 1317 completed_at_unix_ms: 1_800_000_200_010, 1318 })); 1319 let (admitted_engine, _) = setup_engine(admitted_signer); 1320 let admitted = request(63, "wss://relay.example"); 1321 execute_to_admitted(&admitted_engine, &admitted); 1322 let cancelled = block_on(admitted_engine.cancel_push(admitted.operation_id())) 1323 .expect("cancel admitted delivery"); 1324 assert!( 1325 cancelled 1326 .status() 1327 .artifact() 1328 .admission_state() 1329 .is_admitted() 1330 ); 1331 assert_eq!( 1332 cancelled.status().delivery_plan().state(), 1333 AuthoredDeliveryState::Cancelled 1334 ); 1335 } 1336 1337 #[test] 1338 fn authored_storage_outcome_contracts_fail_closed_at_each_orchestration_phase() { 1339 let storage = Arc::new(FaultStorage::new(84)); 1340 storage.fault_next_prepared(); 1341 let engine = fault_engine( 1342 storage, 1343 Arc::new(MockSigner::new(SignBehavior::Pending)), 1344 Arc::new(MockSink), 1345 ); 1346 assert_eq!( 1347 block_on(engine.prepare_push(request(84, "wss://fault.example"))), 1348 Err(Error::StorageFailed) 1349 ); 1350 1351 let storage = Arc::new(FaultStorage::new(85)); 1352 let engine = fault_engine( 1353 storage.clone(), 1354 Arc::new(MockSigner::new(SignBehavior::Pending)), 1355 Arc::new(MockSink), 1356 ); 1357 block_on(engine.prepare_push(request(85, "wss://fault.example"))).unwrap(); 1358 storage.fault_prepared_identity(); 1359 assert_eq!( 1360 block_on(engine.prepare_push(request(86, "wss://fault.example"))), 1361 Err(Error::StorageFailed) 1362 ); 1363 1364 let storage = Arc::new(FaultStorage::new(87)); 1365 let engine = fault_engine( 1366 storage.clone(), 1367 Arc::new(MockSigner::new(SignBehavior::Success { 1368 completed_at_unix_ms: 1_800_000_200_500, 1369 })), 1370 Arc::new(MockSink), 1371 ); 1372 let push = request(87, "wss://fault.example"); 1373 block_on(engine.prepare_push(push.clone())).unwrap(); 1374 storage.fault_nth_artifact(1); 1375 assert_eq!( 1376 block_on(engine.sign_prepared(push)), 1377 Err(Error::StorageFailed) 1378 ); 1379 1380 let storage = Arc::new(FaultStorage::new(88)); 1381 let engine = fault_engine( 1382 storage.clone(), 1383 Arc::new(MockSigner::new(SignBehavior::Success { 1384 completed_at_unix_ms: 1_800_000_200_500, 1385 })), 1386 Arc::new(MockSink), 1387 ); 1388 let push = request(88, "wss://fault.example"); 1389 block_on(engine.prepare_push(push.clone())).unwrap(); 1390 storage.fault_nth_artifact(2); 1391 assert_eq!( 1392 block_on(engine.sign_prepared(push)), 1393 Err(Error::StorageFailed) 1394 ); 1395 1396 let storage = Arc::new(FaultStorage::new(89)); 1397 let engine = fault_engine( 1398 storage.clone(), 1399 Arc::new(MockSigner::new(SignBehavior::Success { 1400 completed_at_unix_ms: 1_800_000_200_500, 1401 })), 1402 Arc::new(MockSink), 1403 ); 1404 let push = request(89, "wss://fault.example"); 1405 block_on(engine.sign_prepared(push.clone())).unwrap(); 1406 storage.fault_nth_artifact(2); 1407 assert_eq!( 1408 block_on(engine.admit_signed(push.operation_id())), 1409 Err(Error::StorageFailed) 1410 ); 1411 1412 let storage = Arc::new(FaultStorage::new(90)); 1413 let engine = fault_engine( 1414 storage.clone(), 1415 Arc::new(MockSigner::new(SignBehavior::Error( 1416 SigningErrorKind::SignerRejected, 1417 ))), 1418 Arc::new(MockSink), 1419 ); 1420 let push = request(90, "wss://fault.example"); 1421 block_on(engine.prepare_push(push.clone())).unwrap(); 1422 storage.fault_nth_artifact(2); 1423 assert_eq!( 1424 block_on(engine.sign_prepared(push)), 1425 Err(Error::StorageFailed) 1426 ); 1427 1428 for (byte, nth) in [(91, 1), (92, 2)] { 1429 let storage = Arc::new(FaultStorage::new(byte)); 1430 let sink = Arc::new(ScriptedSink::new([DeliveryBehavior::Outcomes(vec![ 1431 DeliveryOutcome::accepted(), 1432 ])])); 1433 let engine = fault_engine( 1434 storage.clone(), 1435 Arc::new(MockSigner::new(SignBehavior::Success { 1436 completed_at_unix_ms: 1_800_000_200_500, 1437 })), 1438 sink, 1439 ); 1440 let push = request(byte, "wss://fault.example"); 1441 execute_to_admitted(&engine, &push); 1442 storage.fault_nth_plan(nth); 1443 assert_eq!( 1444 block_on(engine.deliver_push(push.operation_id())), 1445 Err(Error::StorageFailed) 1446 ); 1447 } 1448 } 1449 1450 #[test] 1451 fn signing_claim_recovery_respects_exact_and_non_replayable_capabilities() { 1452 let exact_storage = Arc::new(MemoryStorage::new( 1453 SourceGeneration::new([8; 32]).expect("generation"), 1454 )); 1455 let exact_pending = Arc::new(MockSigner::with_replay( 1456 SignBehavior::Pending, 1457 ReplayCapability::ExactReplayByRequestId, 1458 )); 1459 let exact_engine = Engine::builder( 1460 exact_storage.clone(), 1461 Arc::new(TestClock(AtomicU64::new(1_800_000_200_000))), 1462 Arc::new(TestIds(AtomicU64::new(100))), 1463 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 1464 ) 1465 .sink(Arc::new(MockSink)) 1466 .signer(exact_pending.clone()) 1467 .build() 1468 .expect("exact engine"); 1469 let exact_push = request(8, "wss://relay.example"); 1470 let mut exact_future = Box::pin(exact_engine.sign_prepared(exact_push.clone())).fuse(); 1471 let mut context = std::task::Context::from_waker(noop_waker_ref()); 1472 assert!(exact_future.poll_unpin(&mut context).is_pending()); 1473 drop(exact_future); 1474 let claimed = block_on(exact_engine.push_status(exact_push.operation_id())) 1475 .expect("status") 1476 .expect("claimed status"); 1477 assert!(claimed.artifact().signing_claim().is_some()); 1478 1479 let exact_success = Arc::new(MockSigner::with_replay( 1480 SignBehavior::Success { 1481 completed_at_unix_ms: 1_800_000_220_000, 1482 }, 1483 ReplayCapability::ExactReplayByRequestId, 1484 )); 1485 let exact_recovery = Engine::builder( 1486 exact_storage, 1487 Arc::new(TestClock(AtomicU64::new(1_800_000_211_000))), 1488 Arc::new(TestIds(AtomicU64::new(110))), 1489 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 1490 ) 1491 .sink(Arc::new(MockSink)) 1492 .signer(exact_success.clone()) 1493 .build() 1494 .expect("recovery engine"); 1495 let recovered = 1496 block_on(exact_recovery.sign_prepared(exact_push.clone())).expect("exact replay recovery"); 1497 assert_eq!( 1498 recovered.artifact().signing_state(), 1499 radroots_storage::authored::SigningState::Signed 1500 ); 1501 assert_eq!(exact_success.calls.load(Ordering::Relaxed), 1); 1502 let replay = block_on(exact_recovery.sign_prepared(exact_push)).expect("signed replay"); 1503 assert!(replay.is_replay()); 1504 assert_eq!(exact_success.calls.load(Ordering::Relaxed), 1); 1505 1506 let unsafe_storage = Arc::new(MemoryStorage::new( 1507 SourceGeneration::new([9; 32]).expect("generation"), 1508 )); 1509 let unsafe_pending = Arc::new(MockSigner::with_replay( 1510 SignBehavior::Pending, 1511 ReplayCapability::NonReplayable, 1512 )); 1513 let unsafe_engine = Engine::builder( 1514 unsafe_storage.clone(), 1515 Arc::new(TestClock(AtomicU64::new(1_800_000_200_000))), 1516 Arc::new(TestIds(AtomicU64::new(120))), 1517 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 1518 ) 1519 .sink(Arc::new(MockSink)) 1520 .signer(unsafe_pending) 1521 .build() 1522 .expect("unsafe engine"); 1523 let unsafe_push = request(9, "wss://relay.example"); 1524 let mut unsafe_future = Box::pin(unsafe_engine.sign_prepared(unsafe_push.clone())).fuse(); 1525 let mut context = std::task::Context::from_waker(noop_waker_ref()); 1526 assert!(unsafe_future.poll_unpin(&mut context).is_pending()); 1527 drop(unsafe_future); 1528 1529 let unsafe_success = Arc::new(MockSigner::with_replay( 1530 SignBehavior::Success { 1531 completed_at_unix_ms: 1_800_000_220_000, 1532 }, 1533 ReplayCapability::NonReplayable, 1534 )); 1535 let unsafe_recovery = Engine::builder( 1536 unsafe_storage, 1537 Arc::new(TestClock(AtomicU64::new(1_800_000_211_000))), 1538 Arc::new(TestIds(AtomicU64::new(130))), 1539 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 1540 ) 1541 .sink(Arc::new(MockSink)) 1542 .signer(unsafe_success.clone()) 1543 .build() 1544 .expect("unsafe recovery engine"); 1545 assert_eq!( 1546 block_on(unsafe_recovery.sign_prepared(unsafe_push.clone())), 1547 Err(Error::SigningIndeterminate) 1548 ); 1549 assert_eq!(unsafe_success.calls.load(Ordering::Relaxed), 0); 1550 let unsafe_status = block_on(unsafe_recovery.push_status(unsafe_push.operation_id())) 1551 .expect("unsafe status") 1552 .expect("unsafe operation"); 1553 assert_eq!( 1554 unsafe_status.artifact().signing_state(), 1555 radroots_storage::authored::SigningState::Indeterminate 1556 ); 1557 } 1558 1559 #[test] 1560 fn signing_failures_persist_retry_indeterminate_terminal_and_cancelled_states() { 1561 let exact = Arc::new(MockSigner::with_replay( 1562 SignBehavior::Uncertain(SigningErrorKind::SignerTimeout), 1563 ReplayCapability::ExactReplayByRequestId, 1564 )); 1565 let (engine, storage) = setup_engine(exact); 1566 let retryable = request(10, "wss://relay.example"); 1567 assert_eq!( 1568 block_on(engine.sign_prepared(retryable.clone())), 1569 Err(Error::SignerFailed) 1570 ); 1571 let retryable_status = block_on(engine.push_status(retryable.operation_id())) 1572 .expect("retryable status") 1573 .expect("retryable operation"); 1574 assert_eq!( 1575 retryable_status.artifact().signing_state(), 1576 radroots_storage::authored::SigningState::Retryable 1577 ); 1578 let retry_at = retryable_status 1579 .artifact() 1580 .signing_retry() 1581 .expect("retry schedule") 1582 .not_before_unix_ms(); 1583 let success = Arc::new(MockSigner::with_replay( 1584 SignBehavior::Success { 1585 completed_at_unix_ms: retry_at + 2, 1586 }, 1587 ReplayCapability::ExactReplayByRequestId, 1588 )); 1589 let recovery = Engine::builder( 1590 storage, 1591 Arc::new(TestClock(AtomicU64::new(retry_at + 1))), 1592 Arc::new(TestIds(AtomicU64::new(140))), 1593 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 1594 ) 1595 .sink(Arc::new(MockSink)) 1596 .signer(success.clone()) 1597 .build() 1598 .expect("recovery engine"); 1599 assert_eq!( 1600 block_on(recovery.sign_prepared(retryable)) 1601 .expect("retry exact request") 1602 .artifact() 1603 .signing_state(), 1604 radroots_storage::authored::SigningState::Signed 1605 ); 1606 assert_eq!(success.calls.load(Ordering::Relaxed), 1); 1607 1608 let non_replayable = Arc::new(MockSigner::with_replay( 1609 SignBehavior::Uncertain(SigningErrorKind::SignerTimeout), 1610 ReplayCapability::NonReplayable, 1611 )); 1612 let (engine, _) = setup_engine(non_replayable); 1613 let uncertain = request(11, "wss://relay.example"); 1614 assert_eq!( 1615 block_on(engine.sign_prepared(uncertain.clone())), 1616 Err(Error::SigningIndeterminate) 1617 ); 1618 assert_eq!( 1619 block_on(engine.push_status(uncertain.operation_id())) 1620 .expect("indeterminate status") 1621 .expect("indeterminate operation") 1622 .artifact() 1623 .signing_state(), 1624 radroots_storage::authored::SigningState::Indeterminate 1625 ); 1626 assert_eq!( 1627 block_on(engine.sign_prepared(uncertain)), 1628 Err(Error::SigningIndeterminate) 1629 ); 1630 1631 let cancelled = Arc::new(MockSigner::new(SignBehavior::Error( 1632 SigningErrorKind::SignerCancelled, 1633 ))); 1634 let (engine, _) = setup_engine(cancelled); 1635 let cancelled_push = request(12, "wss://relay.example"); 1636 assert_eq!( 1637 block_on(engine.sign_prepared(cancelled_push.clone())), 1638 Err(Error::SigningCancelled) 1639 ); 1640 assert_eq!( 1641 block_on(engine.push_status(cancelled_push.operation_id())) 1642 .expect("cancelled status") 1643 .expect("cancelled operation") 1644 .artifact() 1645 .signing_state(), 1646 radroots_storage::authored::SigningState::Cancelled 1647 ); 1648 assert_eq!( 1649 block_on(engine.sign_prepared(cancelled_push)), 1650 Err(Error::SignerFailed) 1651 ); 1652 1653 let terminal = Arc::new(MockSigner::new(SignBehavior::Error( 1654 SigningErrorKind::SignerRejected, 1655 ))); 1656 let (engine, _) = setup_engine(terminal); 1657 let terminal_push = request(13, "wss://relay.example"); 1658 assert_eq!( 1659 block_on(engine.sign_prepared(terminal_push.clone())), 1660 Err(Error::SignerFailed) 1661 ); 1662 assert_eq!( 1663 block_on(engine.push_status(terminal_push.operation_id())) 1664 .expect("terminal status") 1665 .expect("terminal operation") 1666 .artifact() 1667 .signing_state(), 1668 radroots_storage::authored::SigningState::FailedTerminal 1669 ); 1670 assert_eq!( 1671 block_on(engine.sign_prepared(terminal_push)), 1672 Err(Error::SignerFailed) 1673 ); 1674 1675 let deadline = Arc::new(MockSigner::new(SignBehavior::Error( 1676 SigningErrorKind::DeadlineExceeded, 1677 ))); 1678 let (engine, _) = setup_engine(deadline); 1679 let deadline_push = request(14, "wss://relay.example"); 1680 assert_eq!( 1681 block_on(engine.sign_prepared(deadline_push)), 1682 Err(Error::SignerDeadlineExceeded) 1683 ); 1684 } 1685 1686 #[tokio::test] 1687 async fn sqlite_signing_and_admission_recover_across_every_reopen_boundary() { 1688 let directory = tempfile::tempdir().expect("database directory"); 1689 let paths = Paths::from_directory(directory.path()).expect("paths"); 1690 let push = request(7, "wss://relay.example"); 1691 let original = { 1692 let store = Arc::new( 1693 SqliteStorage::open( 1694 OpenOptions::new(paths.clone(), OpenMode::Create) 1695 .with_source_generation(SourceGeneration::new([7; 32]).expect("generation"), 1) 1696 .expect("source generation"), 1697 ) 1698 .await 1699 .expect("open SQLite"), 1700 ); 1701 let capability: Arc<dyn SyncStorage> = store; 1702 let signer = Arc::new(MockSigner::new(SignBehavior::Pending)); 1703 let engine = Engine::builder( 1704 capability, 1705 Arc::new(TestClock(AtomicU64::new(1_800_000_200_000))), 1706 Arc::new(TestIds(AtomicU64::new(80))), 1707 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 1708 ) 1709 .sink(Arc::new(MockSink)) 1710 .signer(signer.clone()) 1711 .build() 1712 .expect("engine"); 1713 let prepared = engine.prepare_push(push.clone()).await.expect("prepare"); 1714 assert!(!prepared.is_replay()); 1715 assert_eq!(signer.calls.load(Ordering::Relaxed), 0); 1716 prepared 1717 }; 1718 1719 { 1720 let store = Arc::new( 1721 SqliteStorage::open(OpenOptions::new(paths.clone(), OpenMode::ReadWriteExisting)) 1722 .await 1723 .expect("reopen for signing"), 1724 ); 1725 let capability: Arc<dyn SyncStorage> = store; 1726 let signer = Arc::new(MockSigner::new(SignBehavior::Success { 1727 completed_at_unix_ms: 1_800_000_210_500, 1728 })); 1729 let engine = Engine::builder( 1730 capability, 1731 Arc::new(TestClock(AtomicU64::new(1_800_000_210_000))), 1732 Arc::new(TestIds(AtomicU64::new(90))), 1733 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 1734 ) 1735 .sink(Arc::new(MockSink)) 1736 .signer(signer.clone()) 1737 .build() 1738 .expect("engine"); 1739 let status = engine 1740 .push_status(push.operation_id()) 1741 .await 1742 .expect("status") 1743 .expect("durable preparation"); 1744 assert_eq!(status.operation(), original.operation()); 1745 assert_eq!(status.artifact(), original.artifact()); 1746 assert_eq!(status.delivery_plan(), original.delivery_plan()); 1747 let replay = engine 1748 .prepare_push(push.clone()) 1749 .await 1750 .expect("prepare replay"); 1751 assert!(replay.is_replay()); 1752 assert_eq!(signer.calls.load(Ordering::Relaxed), 0); 1753 let signed = engine 1754 .sign_prepared(push.clone()) 1755 .await 1756 .expect("sign after reopen"); 1757 assert_eq!( 1758 signed.artifact().signing_state(), 1759 radroots_storage::authored::SigningState::Signed 1760 ); 1761 assert_eq!( 1762 signed.artifact().admission_state(), 1763 radroots_storage::authored::AdmissionState::Pending 1764 ); 1765 assert!(signed.artifact().signed().is_some()); 1766 assert_eq!(signer.calls.load(Ordering::Relaxed), 1); 1767 assert_eq!( 1768 engine.deliver_push(push.operation_id()).await, 1769 Err(Error::AdmissionFailed) 1770 ); 1771 } 1772 1773 { 1774 let store = Arc::new( 1775 SqliteStorage::open(OpenOptions::new(paths.clone(), OpenMode::ReadWriteExisting)) 1776 .await 1777 .expect("reopen for admission"), 1778 ); 1779 let capability: Arc<dyn SyncStorage> = store.clone(); 1780 let signer = Arc::new(MockSigner::new(SignBehavior::Pending)); 1781 let engine = Engine::builder( 1782 capability, 1783 Arc::new(TestClock(AtomicU64::new(1_800_000_211_000))), 1784 Arc::new(TestIds(AtomicU64::new(100))), 1785 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 1786 ) 1787 .sink(Arc::new(MockSink)) 1788 .signer(signer.clone()) 1789 .build() 1790 .expect("engine"); 1791 let admitted = engine 1792 .admit_signed(push.operation_id()) 1793 .await 1794 .expect("admit after reopen"); 1795 assert!(!admitted.is_replay()); 1796 assert!(admitted.artifact().admission_state().is_admitted()); 1797 let replay = engine 1798 .admit_signed(push.operation_id()) 1799 .await 1800 .expect("admission replay"); 1801 assert!(replay.is_replay()); 1802 assert_eq!(signer.calls.load(Ordering::Relaxed), 0); 1803 let admitted_events = store 1804 .query_raw(EventQuery::all( 1805 EventQueryBounds::first(10).expect("bounds"), 1806 )) 1807 .await 1808 .expect("admitted events"); 1809 assert_eq!(admitted_events.items().len(), 1); 1810 assert_eq!( 1811 admitted_events.items()[0].stage(), 1812 radroots_storage::event::AdmissionStage::Visible 1813 ); 1814 } 1815 1816 let store = Arc::new( 1817 SqliteStorage::open(OpenOptions::new(paths, OpenMode::ReadOnly)) 1818 .await 1819 .expect("final read-only reopen"), 1820 ); 1821 let engine = Engine::builder( 1822 store, 1823 Arc::new(TestClock(AtomicU64::new(1_800_000_212_000))), 1824 Arc::new(TestIds(AtomicU64::new(110))), 1825 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 1826 ) 1827 .sink(Arc::new(MockSink)) 1828 .build() 1829 .expect("read-only engine"); 1830 let final_status = engine 1831 .push_status(push.operation_id()) 1832 .await 1833 .expect("final status") 1834 .expect("durable operation"); 1835 assert_eq!( 1836 final_status.artifact().signing_state(), 1837 radroots_storage::authored::SigningState::Signed 1838 ); 1839 assert!(final_status.artifact().admission_state().is_admitted()); 1840 assert!(final_status.delivery_plan().request().is_some()); 1841 } 1842 1843 #[test] 1844 fn authored_delivery_persists_retry_evidence_and_complete_settlement() { 1845 let sink = Arc::new(ScriptedSink::new([ 1846 DeliveryBehavior::Outcomes(vec![ 1847 DeliveryOutcome::accepted(), 1848 DeliveryOutcome::unavailable(), 1849 ]), 1850 DeliveryBehavior::Outcomes(vec![ 1851 DeliveryOutcome::accepted(), 1852 DeliveryOutcome::accepted(), 1853 ]), 1854 ])); 1855 let signer = Arc::new(MockSigner::new(SignBehavior::Success { 1856 completed_at_unix_ms: 1_800_000_200_500, 1857 })); 1858 let ((engine, _), clock) = setup_engine_with_sink(signer, sink.clone()); 1859 let push = request_with_policy( 1860 73, 1861 &["wss://one.example", "wss://two.example"], 1862 SatisfactionClass::Accepted, 1863 TargetPolicy::all(), 1864 ); 1865 execute_to_admitted(&engine, &push); 1866 let admitted = block_on(engine.push_status(push.operation_id())) 1867 .expect("admitted status") 1868 .expect("admitted operation"); 1869 assert_eq!( 1870 engine.retry_decision(admitted.delivery_plan(), 0), 1871 Err(Error::ClockUnavailable) 1872 ); 1873 assert_eq!( 1874 engine.retry_decision(admitted.delivery_plan(), 1_800_000_200_100), 1875 Ok(SyncRetryDecision::Ready) 1876 ); 1877 assert_eq!( 1878 engine.retry_decision(admitted.delivery_plan(), 1_800_000_300_000), 1879 Ok(SyncRetryDecision::Expired) 1880 ); 1881 1882 let first = block_on(engine.deliver_push(push.operation_id())).expect("first attempt"); 1883 assert!(!first.is_replay()); 1884 assert_eq!(first.plan().state(), AuthoredDeliveryState::Retryable); 1885 assert_eq!(first.plan().attempt_count(), 1); 1886 let retry_at = first 1887 .plan() 1888 .retry() 1889 .expect("durable retry") 1890 .not_before_unix_ms(); 1891 assert_eq!( 1892 engine.retry_decision(first.plan(), retry_at - 1), 1893 Ok(SyncRetryDecision::DeferredUntil { unix_ms: retry_at }) 1894 ); 1895 assert_eq!( 1896 engine.retry_decision(first.plan(), retry_at), 1897 Ok(SyncRetryDecision::Ready) 1898 ); 1899 assert_eq!( 1900 block_on(engine.deliver_push(push.operation_id())), 1901 Err(Error::DeliveryDeferred) 1902 ); 1903 let pending = block_on(engine.push_status(push.operation_id())) 1904 .expect("pending status") 1905 .expect("pending operation"); 1906 assert_eq!(pending.settlement().artifacts(), 1); 1907 assert_eq!(pending.settlement().signed(), 1); 1908 assert_eq!(pending.settlement().admitted(), 1); 1909 assert_eq!(pending.settlement().delivery_plans(), 1); 1910 assert_eq!(pending.settlement().delivery_retryable(), 1); 1911 assert!(!pending.settlement().is_settled()); 1912 1913 clock.0.store(retry_at, Ordering::Relaxed); 1914 let second = block_on(engine.deliver_push(push.operation_id())).expect("retry attempt"); 1915 assert_eq!(second.plan().state(), AuthoredDeliveryState::Satisfied); 1916 assert_eq!( 1917 engine.retry_decision(second.plan(), retry_at), 1918 Ok(SyncRetryDecision::Satisfied) 1919 ); 1920 assert_eq!(second.plan().attempt_count(), 2); 1921 assert_eq!(second.plan().attempts().len(), 2); 1922 let replay = block_on(engine.deliver_push(push.operation_id())).expect("terminal replay"); 1923 assert!(replay.is_replay()); 1924 assert_eq!(replay.plan(), second.plan()); 1925 let settled = block_on(engine.push_status(push.operation_id())) 1926 .expect("settled status") 1927 .expect("settled operation"); 1928 assert_eq!(settled.settlement().delivery_satisfied(), 1); 1929 assert!(settled.settlement().is_settled()); 1930 assert!(!settled.settlement().has_failures()); 1931 assert!(settled.settlement().is_successful()); 1932 let requests = sink.requests.lock().expect("request log"); 1933 assert_eq!(requests.len(), 2); 1934 assert_eq!(requests[0], requests[1]); 1935 } 1936 1937 #[test] 1938 fn invalid_authored_delivery_receipt_terminalizes_without_hot_loop() { 1939 let sink = Arc::new(ScriptedSink::new([DeliveryBehavior::MismatchedRequest])); 1940 let signer = Arc::new(MockSigner::new(SignBehavior::Success { 1941 completed_at_unix_ms: 1_800_000_200_500, 1942 })); 1943 let ((engine, _), _) = setup_engine_with_sink(signer, sink.clone()); 1944 let push = request(74, "wss://one.example"); 1945 execute_to_admitted(&engine, &push); 1946 1947 let failed = block_on(engine.deliver_push(push.operation_id())).expect("durable failure"); 1948 assert_eq!(failed.plan().state(), AuthoredDeliveryState::FailedTerminal); 1949 assert_eq!( 1950 engine.retry_decision(failed.plan(), 1_800_000_200_100), 1951 Ok(SyncRetryDecision::Exhausted) 1952 ); 1953 assert_eq!(failed.plan().attempt_count(), 1); 1954 assert_eq!( 1955 failed.plan().last_failure().expect("failure").code(), 1956 "invalid_transport_contract" 1957 ); 1958 let replay = block_on(engine.deliver_push(push.operation_id())).expect("failure replay"); 1959 assert!(replay.is_replay()); 1960 assert_eq!(replay.plan(), failed.plan()); 1961 assert_eq!(sink.requests.lock().expect("request log").len(), 1); 1962 let status = block_on(engine.push_status(push.operation_id())) 1963 .expect("failed status") 1964 .expect("failed operation"); 1965 assert_eq!(status.settlement().delivery_failed_terminal(), 1); 1966 assert!(status.settlement().is_settled()); 1967 assert!(status.settlement().has_failures()); 1968 assert!(!status.settlement().is_successful()); 1969 } 1970 1971 #[test] 1972 fn sink_failures_deadlines_and_missing_capabilities_are_durable_and_bounded() { 1973 for (byte, behavior, expected) in [ 1974 ( 1975 77, 1976 DeliveryBehavior::Failure(Retryability::Retryable), 1977 AuthoredDeliveryState::Retryable, 1978 ), 1979 ( 1980 78, 1981 DeliveryBehavior::Failure(Retryability::Terminal), 1982 AuthoredDeliveryState::FailedTerminal, 1983 ), 1984 ( 1985 79, 1986 DeliveryBehavior::MismatchedFailure, 1987 AuthoredDeliveryState::FailedTerminal, 1988 ), 1989 ] { 1990 let sink = Arc::new(ScriptedSink::new([behavior])); 1991 let signer = Arc::new(MockSigner::new(SignBehavior::Success { 1992 completed_at_unix_ms: 1_800_000_200_500, 1993 })); 1994 let ((engine, _), _) = setup_engine_with_sink(signer, sink); 1995 let push = request(byte, "wss://failure.example"); 1996 execute_to_admitted(&engine, &push); 1997 let delivered = 1998 block_on(engine.deliver_push(push.operation_id())).expect("durable failure"); 1999 assert_eq!(delivered.plan().state(), expected); 2000 } 2001 2002 let signer = Arc::new(MockSigner::new(SignBehavior::Success { 2003 completed_at_unix_ms: 1_800_000_200_500, 2004 })); 2005 let ((engine, _), clock) = 2006 setup_engine_with_sink(signer, Arc::new(ScriptedSink::new(std::iter::empty()))); 2007 let push = request(80, "wss://deadline.example"); 2008 execute_to_admitted(&engine, &push); 2009 clock 2010 .0 2011 .store(push.delivery_deadline_unix_ms(), Ordering::Relaxed); 2012 let expired = block_on(engine.deliver_push(push.operation_id())).expect("deadline evidence"); 2013 assert_eq!( 2014 expired.plan().state(), 2015 AuthoredDeliveryState::FailedTerminal 2016 ); 2017 assert_eq!( 2018 expired.plan().last_failure().expect("failure").code(), 2019 "delivery_deadline_exceeded" 2020 ); 2021 2022 let storage = Arc::new(MemoryStorage::new( 2023 SourceGeneration::new([81; 32]).expect("generation"), 2024 )); 2025 let capability: Arc<dyn SyncStorage> = storage; 2026 let no_signer = Engine::builder( 2027 capability, 2028 Arc::new(TestClock(AtomicU64::new(1_800_000_200_000))), 2029 Arc::new(TestIds(AtomicU64::new(10))), 2030 DeadlinePolicy::new(10_000, 10_000, 10_000).unwrap(), 2031 ) 2032 .sink(Arc::new(MockSink)) 2033 .build() 2034 .unwrap(); 2035 assert_eq!( 2036 block_on(no_signer.sign_prepared(request(81, "wss://missing.example"))), 2037 Err(Error::MissingSigner) 2038 ); 2039 2040 let storage = Arc::new(MemoryStorage::new( 2041 SourceGeneration::new([82; 32]).expect("generation"), 2042 )); 2043 let capability: Arc<dyn SyncStorage> = storage; 2044 let no_sink = Engine::builder( 2045 capability, 2046 Arc::new(TestClock(AtomicU64::new(1_800_000_200_000))), 2047 Arc::new(TestIds(AtomicU64::new(10))), 2048 DeadlinePolicy::new(10_000, 10_000, 10_000).unwrap(), 2049 ) 2050 .source(Arc::new(MockSource)) 2051 .build() 2052 .unwrap(); 2053 assert_eq!( 2054 block_on(no_sink.deliver_push(SyncId::new([82; 16]).unwrap())), 2055 Err(Error::MissingSink) 2056 ); 2057 2058 let signer = Arc::new(MockSigner::new(SignBehavior::Success { 2059 completed_at_unix_ms: 1_800_000_200_500, 2060 })); 2061 let (engine, _) = setup_engine(signer); 2062 let unsigned = request(83, "wss://unsigned.example"); 2063 block_on(engine.prepare_push(unsigned.clone())).unwrap(); 2064 assert_eq!( 2065 block_on(engine.admit_signed(unsigned.operation_id())), 2066 Err(Error::InvalidSignerOutput) 2067 ); 2068 assert_eq!( 2069 block_on(engine.deliver_push(unsigned.operation_id())), 2070 Err(Error::InvalidSignerOutput) 2071 ); 2072 } 2073 2074 #[test] 2075 fn active_authored_claims_and_admission_storage_failures_are_fenced() { 2076 let storage = Arc::new(FaultStorage::new(93)); 2077 let engine = fault_engine( 2078 storage.clone(), 2079 Arc::new(MockSigner::new(SignBehavior::Success { 2080 completed_at_unix_ms: 1_800_000_200_500, 2081 })), 2082 Arc::new(MockSink), 2083 ); 2084 let push = request(93, "wss://claim.example"); 2085 block_on(engine.prepare_push(push.clone())).unwrap(); 2086 let artifact = block_on(engine.push_status(push.operation_id())) 2087 .unwrap() 2088 .unwrap() 2089 .artifact() 2090 .clone(); 2091 let claim = radroots_storage::authored::WorkClaim::new( 2092 [93; 16], 2093 "external-signer", 2094 std::num::NonZeroU64::MIN, 2095 1_800_000_200_000, 2096 1_800_000_210_000, 2097 artifact.revision(), 2098 ) 2099 .unwrap(); 2100 block_on(AuthoredAtomicStorage::execute_authored( 2101 storage.as_ref(), 2102 radroots_storage::authored_atomic::AuthoredAtomicCommand::Claim( 2103 radroots_storage::authored_atomic::ClaimAuthoredWork::new( 2104 radroots_storage::authored_atomic::ClaimAuthoredTarget::ArtifactSigning( 2105 artifact.artifact_id(), 2106 ), 2107 claim, 2108 ), 2109 ), 2110 )) 2111 .unwrap(); 2112 assert_eq!( 2113 block_on(engine.sign_prepared(push)), 2114 Err(Error::WorkClaimConflict) 2115 ); 2116 2117 let storage = Arc::new(FaultStorage::new(94)); 2118 let engine = fault_engine( 2119 storage.clone(), 2120 Arc::new(MockSigner::new(SignBehavior::Success { 2121 completed_at_unix_ms: 1_800_000_200_500, 2122 })), 2123 Arc::new(MockSink), 2124 ); 2125 let push = request(94, "wss://claim.example"); 2126 block_on(engine.sign_prepared(push.clone())).unwrap(); 2127 let artifact = block_on(engine.push_status(push.operation_id())) 2128 .unwrap() 2129 .unwrap() 2130 .artifact() 2131 .clone(); 2132 let claim = radroots_storage::authored::WorkClaim::new( 2133 [94; 16], 2134 "external-admission", 2135 std::num::NonZeroU64::MIN, 2136 1_800_000_200_500, 2137 1_800_000_210_500, 2138 artifact.revision(), 2139 ) 2140 .unwrap(); 2141 block_on(AuthoredAtomicStorage::execute_authored( 2142 storage.as_ref(), 2143 radroots_storage::authored_atomic::AuthoredAtomicCommand::Claim( 2144 radroots_storage::authored_atomic::ClaimAuthoredWork::new( 2145 radroots_storage::authored_atomic::ClaimAuthoredTarget::ArtifactAdmission( 2146 artifact.artifact_id(), 2147 ), 2148 claim, 2149 ), 2150 ), 2151 )) 2152 .unwrap(); 2153 assert_eq!( 2154 block_on(engine.admit_signed(push.operation_id())), 2155 Err(Error::WorkClaimConflict) 2156 ); 2157 2158 for (byte, fault, expected) in [ 2159 (95, 1, radroots_storage::authored::AdmissionState::Rejected), 2160 (96, 2, radroots_storage::authored::AdmissionState::Retryable), 2161 ] { 2162 let storage = Arc::new(FaultStorage::new(byte)); 2163 let engine = fault_engine( 2164 storage.clone(), 2165 Arc::new(MockSigner::new(SignBehavior::Success { 2166 completed_at_unix_ms: 1_800_000_200_500, 2167 })), 2168 Arc::new(MockSink), 2169 ); 2170 let push = request(byte, "wss://admission-error.example"); 2171 block_on(engine.sign_prepared(push.clone())).unwrap(); 2172 storage.fail_admission_with(fault); 2173 assert_eq!( 2174 block_on(engine.admit_signed(push.operation_id())), 2175 Err(Error::AdmissionFailed), 2176 "admission fault {fault}" 2177 ); 2178 assert_eq!( 2179 block_on(engine.push_status(push.operation_id())) 2180 .unwrap() 2181 .unwrap() 2182 .artifact() 2183 .admission_state(), 2184 expected 2185 ); 2186 } 2187 } 2188 2189 #[test] 2190 fn authored_delivery_claims_fence_concurrent_and_stale_workers() { 2191 let pending_sink = Arc::new(ScriptedSink::new([DeliveryBehavior::Pending])); 2192 let signer = Arc::new(MockSigner::new(SignBehavior::Success { 2193 completed_at_unix_ms: 1_800_000_200_500, 2194 })); 2195 let ((engine, storage), clock) = setup_engine_with_sink(signer, pending_sink.clone()); 2196 let push = request(75, "wss://one.example"); 2197 execute_to_admitted(&engine, &push); 2198 2199 let mut pending = Box::pin(engine.deliver_push(push.operation_id())).fuse(); 2200 let mut context = std::task::Context::from_waker(noop_waker_ref()); 2201 assert!(pending.poll_unpin(&mut context).is_pending()); 2202 let claimed = block_on(engine.push_status(push.operation_id())) 2203 .expect("claimed status") 2204 .expect("claimed operation"); 2205 let expires_at = claimed 2206 .delivery_plan() 2207 .claim_evidence() 2208 .expect("delivery claim") 2209 .expires_at_unix_ms(); 2210 assert_eq!( 2211 engine.retry_decision(claimed.delivery_plan(), expires_at - 1), 2212 Ok(SyncRetryDecision::InFlightUntil { 2213 unix_ms: expires_at, 2214 }) 2215 ); 2216 assert_eq!( 2217 engine.retry_decision(claimed.delivery_plan(), expires_at), 2218 Ok(SyncRetryDecision::Ready) 2219 ); 2220 assert_eq!( 2221 block_on(engine.deliver_push(push.operation_id())), 2222 Err(Error::WorkClaimConflict) 2223 ); 2224 drop(pending); 2225 2226 clock.0.store(expires_at + 1, Ordering::Relaxed); 2227 let recovery_sink = Arc::new(ScriptedSink::new([DeliveryBehavior::Outcomes(vec![ 2228 DeliveryOutcome::accepted(), 2229 ])])); 2230 let capability: Arc<dyn SyncStorage> = storage; 2231 let recovery = Engine::builder( 2232 capability, 2233 clock, 2234 Arc::new(TestIds(AtomicU64::new(220))), 2235 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 2236 ) 2237 .sink(recovery_sink.clone()) 2238 .build() 2239 .expect("recovery engine"); 2240 let recovered = block_on(recovery.deliver_push(push.operation_id())).expect("stale recovery"); 2241 assert_eq!(recovered.plan().state(), AuthoredDeliveryState::Satisfied); 2242 assert_eq!( 2243 recovered.plan().claim_evidence(), 2244 None, 2245 "settlement clears the claim" 2246 ); 2247 assert_eq!(recovery_sink.requests.lock().expect("request log").len(), 1); 2248 } 2249 2250 #[tokio::test] 2251 async fn sqlite_authored_delivery_retry_survives_reopen() { 2252 let directory = tempfile::tempdir().expect("database directory"); 2253 let paths = Paths::from_directory(directory.path()).expect("paths"); 2254 let push = request_with_policy( 2255 76, 2256 &["wss://one.example", "wss://two.example"], 2257 SatisfactionClass::Accepted, 2258 TargetPolicy::all(), 2259 ); 2260 let retry_at = { 2261 let store = Arc::new( 2262 SqliteStorage::open( 2263 OpenOptions::new(paths.clone(), OpenMode::Create) 2264 .with_source_generation(SourceGeneration::new([76; 32]).expect("generation"), 1) 2265 .expect("source generation"), 2266 ) 2267 .await 2268 .expect("open SQLite"), 2269 ); 2270 let sink = Arc::new(ScriptedSink::new([DeliveryBehavior::Outcomes(vec![ 2271 DeliveryOutcome::accepted(), 2272 DeliveryOutcome::unavailable(), 2273 ])])); 2274 let engine = Engine::builder( 2275 store, 2276 Arc::new(TestClock(AtomicU64::new(1_800_000_200_000))), 2277 Arc::new(TestIds(AtomicU64::new(230))), 2278 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 2279 ) 2280 .sink(sink) 2281 .signer(Arc::new(MockSigner::new(SignBehavior::Success { 2282 completed_at_unix_ms: 1_800_000_200_500, 2283 }))) 2284 .build() 2285 .expect("engine"); 2286 engine 2287 .sign_prepared(push.clone()) 2288 .await 2289 .expect("sign prepared push"); 2290 engine 2291 .admit_signed(push.operation_id()) 2292 .await 2293 .expect("admit signed push"); 2294 let first = engine 2295 .deliver_push(push.operation_id()) 2296 .await 2297 .expect("first attempt"); 2298 assert_eq!(first.plan().attempt_count(), 1); 2299 first 2300 .plan() 2301 .retry() 2302 .expect("retry schedule") 2303 .not_before_unix_ms() 2304 }; 2305 2306 let store = Arc::new( 2307 SqliteStorage::open(OpenOptions::new(paths, OpenMode::ReadWriteExisting)) 2308 .await 2309 .expect("reopen SQLite"), 2310 ); 2311 let sink = Arc::new(ScriptedSink::new([DeliveryBehavior::Outcomes(vec![ 2312 DeliveryOutcome::accepted(), 2313 DeliveryOutcome::accepted(), 2314 ])])); 2315 let clock = Arc::new(TestClock(AtomicU64::new(retry_at - 1))); 2316 let engine = Engine::builder( 2317 store, 2318 clock.clone(), 2319 Arc::new(TestIds(AtomicU64::new(240))), 2320 DeadlinePolicy::new(10_000, 10_000, 10_000).expect("deadlines"), 2321 ) 2322 .sink(sink.clone()) 2323 .build() 2324 .expect("reopen engine"); 2325 let reopened = engine 2326 .push_status(push.operation_id()) 2327 .await 2328 .expect("reopened status") 2329 .expect("reopened operation"); 2330 assert_eq!(reopened.delivery_plan().attempt_count(), 1); 2331 assert_eq!( 2332 reopened.delivery_plan().state(), 2333 AuthoredDeliveryState::Retryable 2334 ); 2335 assert_eq!( 2336 engine.deliver_push(push.operation_id()).await, 2337 Err(Error::DeliveryDeferred) 2338 ); 2339 clock.0.store(retry_at, Ordering::Relaxed); 2340 let completed = engine 2341 .deliver_push(push.operation_id()) 2342 .await 2343 .expect("retry after reopen"); 2344 assert_eq!(completed.plan().attempt_count(), 2); 2345 assert_eq!(completed.plan().state(), AuthoredDeliveryState::Satisfied); 2346 assert_eq!(sink.requests.lock().expect("request log").len(), 1); 2347 } 2348 2349 #[test] 2350 fn captured_preparation_matches_engine_without_changing_ordinary_replay_identity() { 2351 use radroots_storage::authored_atomic::AuthoredAtomicCommand; 2352 let signer = Arc::new(MockSigner::new(SignBehavior::Pending)); 2353 let (engine, _) = setup_engine(signer.clone()); 2354 let push = request(91, "wss://captured.example"); 2355 let captured = push.authored_preparation(1_800_000_200_000).unwrap(); 2356 assert!(push.authored_preparation(0).is_err()); 2357 assert!( 2358 block_on(engine.push_status(push.operation_id())) 2359 .unwrap() 2360 .is_none() 2361 ); 2362 assert_eq!(signer.calls.load(Ordering::Relaxed), 0); 2363 let later = push.authored_preparation(1_800_000_200_001).unwrap(); 2364 assert_ne!( 2365 captured, later, 2366 "the composite command compares captured time exactly" 2367 ); 2368 let original_command = AuthoredAtomicCommand::Prepare(captured.clone()); 2369 let later_command = AuthoredAtomicCommand::Prepare(later); 2370 assert_eq!(original_command.commit_id(), later_command.commit_id()); 2371 assert_eq!(original_command.digest(), later_command.digest()); 2372 let prepared = block_on(engine.prepare_push(push)).unwrap(); 2373 assert_eq!(prepared.operation(), captured.operation()); 2374 assert_eq!( 2375 [prepared.artifact()], 2376 captured.artifacts().iter().collect::<Vec<_>>().as_slice() 2377 ); 2378 assert_eq!( 2379 [prepared.delivery_plan()], 2380 captured 2381 .delivery_plans() 2382 .iter() 2383 .collect::<Vec<_>>() 2384 .as_slice() 2385 ); 2386 assert_eq!(signer.calls.load(Ordering::Relaxed), 0); 2387 }