lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

receipt_delivery_faults.rs (20335B)


      1 use super::*;
      2 
      3 #[tokio::test]
      4 async fn a_retry_cannot_replace_the_original_publication_target_policy() {
      5     let ready = Ready::new(
      6         &[
      7             String::from("ws://127.0.0.1:9"),
      8             String::from("ws://127.0.0.1:10"),
      9         ],
     10         243,
     11     )
     12     .await;
     13     let original = ready.retain(1).await;
     14     let narrowed = TargetSet::new(vec![original.intent().targets().targets()[0].clone()]).unwrap();
     15     let replacement = DeliveryIntent::new(
     16         &"02".repeat(32),
     17         original.intent().event_id(),
     18         &narrowed,
     19         original.intent().policy(),
     20         original.intent().deadline_unix_ms(),
     21     )
     22     .unwrap();
     23     let request =
     24         DeliveryAttemptRequest::new(ready.binding.clone(), ready.work, replacement).unwrap();
     25     assert!(matches!(
     26         ready
     27             .store
     28             .prepare_delivery_attempt(request, copy(&ready.proof), None)
     29             .await,
     30         Err(OperationError::IdempotencyConflict)
     31     ));
     32     ready.store.close().await.unwrap();
     33 }
     34 
     35 fn selected_request(
     36     ready: &Ready,
     37     id: u8,
     38     policy: &SatisfactionPolicy,
     39     index: usize,
     40 ) -> DeliveryAttemptRequest {
     41     let original = ready.request(id);
     42     let selected =
     43         TargetSet::new(vec![original.intent().targets().targets()[index].clone()]).unwrap();
     44     let intent = DeliveryIntent::new(
     45         original.intent().request_id().as_str(),
     46         original.intent().event_id(),
     47         original.intent().targets(),
     48         policy,
     49         original.intent().deadline_unix_ms(),
     50     )
     51     .unwrap()
     52     .with_selection(&selected)
     53     .unwrap();
     54     DeliveryAttemptRequest::new(ready.binding.clone(), ready.work, intent).unwrap()
     55 }
     56 
     57 // Synthetic normalized adapter data exercises storage/policy composition here.
     58 // The separate actual loopback cases qualify remote frame and ACK behavior.
     59 fn synthetic_accepted(attempt: &DeliveryAttempt) -> DeliveryObservation {
     60     use radroots_transport::{
     61         DeliveryReceipt, outcome::DeliveryOutcome, sink::DeliveryTargetReceipt,
     62     };
     63     let request = attempt.transport_request().unwrap();
     64     let selected = &attempt.intent().selection()[0];
     65     let receipts = request
     66         .target_set()
     67         .targets()
     68         .iter()
     69         .map(|target| {
     70             if target.fingerprint() == selected {
     71                 DeliveryTargetReceipt::attempted(target.clone(), DeliveryOutcome::accepted())
     72             } else {
     73                 DeliveryTargetReceipt::skipped(target.clone(), DeliveryOutcome::unavailable())
     74                     .unwrap()
     75             }
     76         })
     77         .collect();
     78     let receipt = DeliveryReceipt::for_request(&request, receipts).unwrap();
     79     DeliveryObservation::from_receipt(attempt, &receipt, selected, None).unwrap()
     80 }
     81 
     82 #[tokio::test]
     83 async fn exhausted_optional_endpoint_does_not_starve_required_endpoint_or_weaken_the_original_policy()
     84  {
     85     let ready = Ready::new(
     86         &[
     87             String::from("ws://127.0.0.1:9"),
     88             String::from("ws://127.0.0.1:10"),
     89         ],
     90         244,
     91     )
     92     .await;
     93     let base = ready.request(1);
     94     let required = base.intent().targets().targets()[1].fingerprint().clone();
     95     let policy = SatisfactionPolicy::new(
     96         SatisfactionClass::Accepted,
     97         TargetPolicy::required(vec![required]).unwrap(),
     98     );
     99     for ordinal in 1..=12u8 {
    100         let selected = selected_request(&ready, ordinal, &policy, 0);
    101         let candidate = if ordinal == 1 {
    102             Some(&ready.candidate)
    103         } else {
    104             None
    105         };
    106         let DeliveryAttemptOutcome::Claimed(attempt) = ready
    107             .store
    108             .prepare_delivery_attempt(selected, copy(&ready.proof), candidate)
    109             .await
    110             .unwrap()
    111         else {
    112             panic!("new selected potential attempt expected")
    113         };
    114         assert_eq!(attempt.target_ordinals(), &[ordinal]);
    115         assert_eq!(attempt.intent().targets().len(), 2);
    116         assert_eq!(attempt.intent().selection().len(), 1);
    117     }
    118     assert!(matches!(
    119         ready
    120             .store
    121             .prepare_delivery_attempt(
    122                 selected_request(&ready, 13, &policy, 0),
    123                 copy(&ready.proof),
    124                 None
    125             )
    126             .await,
    127         Err(OperationError::DeliveryAttemptLimit)
    128     ));
    129     let DeliveryAttemptOutcome::Claimed(required) = ready
    130         .store
    131         .prepare_delivery_attempt(
    132             selected_request(&ready, 14, &policy, 1),
    133             copy(&ready.proof),
    134             None,
    135         )
    136         .await
    137         .unwrap()
    138     else {
    139         panic!("required endpoint must progress independently")
    140     };
    141     assert_eq!(required.target_ordinals(), &[1]);
    142     assert_eq!(required.raw_json(), ready.candidate.raw_json());
    143     captured(
    144         ready
    145             .store
    146             .record_delivery_observation(
    147                 ready.binding.clone(),
    148                 copy(&ready.proof),
    149                 synthetic_accepted(&required),
    150             )
    151             .await
    152             .unwrap(),
    153     );
    154     let e = ready.work.evidence();
    155     let snapshot = ready
    156         .store
    157         .operation_snapshot(
    158             ready.binding.clone(),
    159             e.operation_id(),
    160             None,
    161             copy(&ready.proof),
    162         )
    163         .await
    164         .unwrap()
    165         .unwrap();
    166     assert_eq!(snapshot.publication_accepted(ready.work), Ok(true));
    167     let receipt =
    168         radroots_execution::OperationReceipt::<radroots_domain::ChatMessage>::try_from_snapshot(
    169             &snapshot,
    170         )
    171         .unwrap();
    172     assert_eq!(receipt.delivery(), radroots_execution::Delivery::Unknown);
    173     assert!(
    174         !receipt
    175             .recovery()
    176             .contains(&radroots_execution::RecoveryRequirement::ResumeDelivery)
    177     );
    178     ready.store.close().await.unwrap();
    179 }
    180 
    181 #[tokio::test]
    182 async fn original_quorum_accumulates_acceptance_across_independent_selected_attempts() {
    183     let ready = Ready::new(
    184         &[
    185             String::from("ws://127.0.0.1:9"),
    186             String::from("ws://127.0.0.1:10"),
    187             String::from("ws://127.0.0.1:11"),
    188         ],
    189         245,
    190     )
    191     .await;
    192     let policy = SatisfactionPolicy::new(
    193         SatisfactionClass::Accepted,
    194         TargetPolicy::quorum(2).unwrap(),
    195     );
    196     for index in 0..2 {
    197         let selected = selected_request(&ready, index as u8 + 1, &policy, index);
    198         let candidate = if index == 0 {
    199             Some(&ready.candidate)
    200         } else {
    201             None
    202         };
    203         let DeliveryAttemptOutcome::Claimed(attempt) = ready
    204             .store
    205             .prepare_delivery_attempt(selected, copy(&ready.proof), candidate)
    206             .await
    207             .unwrap()
    208         else {
    209             panic!("fresh selected attempt required")
    210         };
    211         captured(
    212             ready
    213                 .store
    214                 .record_delivery_observation(
    215                     ready.binding.clone(),
    216                     copy(&ready.proof),
    217                     synthetic_accepted(&attempt),
    218                 )
    219                 .await
    220                 .unwrap(),
    221         );
    222         let e = ready.work.evidence();
    223         let snapshot = ready
    224             .store
    225             .operation_snapshot(
    226                 ready.binding.clone(),
    227                 e.operation_id(),
    228                 None,
    229                 copy(&ready.proof),
    230             )
    231             .await
    232             .unwrap()
    233             .unwrap();
    234         assert_eq!(snapshot.publication_accepted(ready.work), Ok(index == 1));
    235         let receipt = radroots_execution::OperationReceipt::<radroots_domain::ChatMessage>::try_from_snapshot(&snapshot).unwrap();
    236         assert_eq!(
    237             receipt
    238                 .recovery()
    239                 .contains(&radroots_execution::RecoveryRequirement::ResumeDelivery),
    240             index == 0
    241         );
    242     }
    243     ready.store.close().await.unwrap();
    244 }
    245 
    246 #[tokio::test]
    247 async fn a_fresh_delivery_deadline_after_first_issuance_expiry_reuses_original_bytes_and_policy() {
    248     let ready = Ready::new(&[String::from("ws://127.0.0.1:9")], 246).await;
    249     let original = ready.retain(1).await;
    250     ready
    251         .fixture
    252         .clock
    253         .now
    254         .store(ready.work.queued_at().get() + 121, Ordering::SeqCst);
    255     let next = DeliveryIntent::new(
    256         &"02".repeat(32),
    257         original.intent().event_id(),
    258         original.intent().targets(),
    259         original.intent().policy(),
    260         NonZeroU64::new((ready.work.queued_at().get() + 180) * 1000).unwrap(),
    261     )
    262     .unwrap();
    263     assert_eq!(
    264         next.publication_policy_digest(),
    265         original.intent().publication_policy_digest()
    266     );
    267     assert_ne!(next.digest(), original.intent().digest());
    268     let DeliveryAttemptOutcome::Claimed(next) = ready
    269         .store
    270         .prepare_delivery_attempt(
    271             DeliveryAttemptRequest::new(ready.binding.clone(), ready.work, next).unwrap(),
    272             copy(&ready.proof),
    273             None,
    274         )
    275         .await
    276         .unwrap()
    277     else {
    278         panic!("current-access delivery may retain a fresh attempt without reissuing S")
    279     };
    280     assert_eq!(next.raw_json(), original.raw_json());
    281     assert_eq!(next.target_ordinals(), &[2]);
    282     let retained = ready
    283         .query(&ready.store, original.intent().request_id())
    284         .await;
    285     assert_eq!(
    286         retained.intent().deadline_unix_ms(),
    287         original.intent().deadline_unix_ms()
    288     );
    289     let mut database = ready.fixture.database().await;
    290     let counts:(i64,i64,i64)=sqlx::query_as("SELECT (SELECT count(*) FROM signing_attempts),(SELECT count(*) FROM artifact_evidence),(SELECT count(*) FROM chat_messages)").fetch_one(&mut database).await.unwrap();
    291     assert_eq!(counts, (1, 1, 1));
    292     database.close().await.unwrap();
    293     ready.store.close().await.unwrap();
    294 }
    295 
    296 #[tokio::test]
    297 async fn actual_loopback_wrong_ack_timeout_disconnect_and_rejection_remain_distinct_after_reopen() {
    298     for (case, expected) in [
    299         (0, DeliveryObservationKind::TimedOut),
    300         (1, DeliveryObservationKind::Disconnected),
    301         (2, DeliveryObservationKind::Rejected),
    302     ] {
    303         let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    304         let ready = Ready::new(
    305             &[format!("ws://{}", listener.local_addr().unwrap())],
    306             237 + case,
    307         )
    308         .await;
    309         let attempt = ready.retain(1).await;
    310         let exact = String::from(attempt.raw_json());
    311         let (release, released) = tokio::sync::oneshot::channel();
    312         let server = tokio::spawn(async move {
    313             let (tcp, _) = listener.accept().await.unwrap();
    314             let mut socket = accept_async(tcp).await.unwrap();
    315             let raw = tokio::time::timeout(Duration::from_secs(5), async {
    316                 loop {
    317                     if let Message::Text(raw) = socket.next().await.unwrap().unwrap() {
    318                         break raw.to_string();
    319                     }
    320                 }
    321             })
    322             .await
    323             .unwrap();
    324             assert_eq!(raw, format!("[\"EVENT\",{exact}]"));
    325             let event: serde_json::Value = serde_json::from_str(&raw).unwrap();
    326             match case {
    327                 0 => {
    328                     socket
    329                         .send(Message::Text(
    330                             json!(["OK", "ff".repeat(32), true, ""]).to_string().into(),
    331                         ))
    332                         .await
    333                         .unwrap();
    334                     let _ = released.await;
    335                 }
    336                 1 => {
    337                     socket.close(None).await.unwrap();
    338                 }
    339                 2 => {
    340                     socket
    341                         .send(Message::Text(
    342                             json!(["OK", event[1]["id"], false, "restricted: synthetic refusal"])
    343                                 .to_string()
    344                                 .into(),
    345                         ))
    346                         .await
    347                         .unwrap();
    348                 }
    349                 _ => unreachable!(),
    350             }
    351         });
    352         let request = attempt.transport_request().unwrap();
    353         let target = request.target_set().targets()[0].fingerprint();
    354         let transport = radroots_transport_nostr_native::create_transport(ready.config.clone());
    355         let result = transport.deliver(request.clone()).await;
    356         let observation = match result {
    357             Ok(receipt) => {
    358                 assert!(!receipt.is_satisfied(&request).unwrap());
    359                 DeliveryObservation::from_receipt(&attempt, &receipt, target, None).unwrap()
    360             }
    361             Err(failure) => {
    362                 DeliveryObservation::from_failure(&attempt, &failure, target, None).unwrap()
    363             }
    364         };
    365         let _ = release.send(());
    366         server.await.unwrap();
    367         assert_eq!(observation.kind(), expected, "actual case {case}");
    368         let first = captured(
    369             ready
    370                 .store
    371                 .record_delivery_observation(ready.binding.clone(), copy(&ready.proof), observation)
    372                 .await
    373                 .unwrap(),
    374         );
    375         ready.store.close().await.unwrap();
    376         let reopened = ready.fixture.reopen(ready.input.clone()).await;
    377         assert_eq!(
    378             ready
    379                 .observations(&reopened, attempt.intent().request_id())
    380                 .await,
    381             vec![first]
    382         );
    383         let e = ready.work.evidence();
    384         let snapshot = reopened
    385             .operation_snapshot(
    386                 ready.binding.clone(),
    387                 e.operation_id(),
    388                 None,
    389                 copy(&ready.proof),
    390             )
    391             .await
    392             .unwrap()
    393             .unwrap();
    394         assert!(snapshot.installed_message().is_some());
    395         let view = radroots_execution::OperationReceipt::<radroots_domain::ChatMessage>::try_from_snapshot(&snapshot).unwrap();
    396         assert!(
    397             view.recovery()
    398                 .contains(&radroots_execution::RecoveryRequirement::ResumeDelivery)
    399         );
    400         assert_eq!(view.delivery_observations().len(), 1);
    401         reopened.close().await.unwrap();
    402     }
    403 }
    404 
    405 fn not_attempted(attempt: &DeliveryAttempt) -> DeliveryObservation {
    406     let request = attempt.transport_request().unwrap();
    407     DeliveryObservation::from_failure(
    408         attempt,
    409         &radroots_transport::SinkFailure::transport_closed(&request),
    410         request.target_set().targets()[0].fingerprint(),
    411         None,
    412     )
    413     .unwrap()
    414 }
    415 
    416 #[tokio::test]
    417 async fn actual_observation_rollback_and_commit_uncertainty_preserve_retained_outer_and_business_facts()
    418  {
    419     for commit in [false, true] {
    420         let ready = Ready::new(&[String::from("ws://127.0.0.1:9")], 240 + u8::from(commit)).await;
    421         let attempt = ready.retain(1).await;
    422         let observation = not_attempted(&attempt);
    423         let mut database = ready.fixture.database().await;
    424         if commit {
    425             sqlx::raw_sql("CREATE TABLE observation_commit_fault (parent INTEGER REFERENCES workspace_grant(singleton) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER observation_commit_fault_trigger AFTER INSERT ON artifact_delivery_observations BEGIN INSERT INTO observation_commit_fault VALUES(2); END").execute(&mut database).await.unwrap();
    426         } else {
    427             sqlx::raw_sql("CREATE TRIGGER observation_insert_fault BEFORE INSERT ON artifact_delivery_observations BEGIN SELECT RAISE(ABORT,'synthetic observation fault'); END").execute(&mut database).await.unwrap();
    428         }
    429         database.close().await.unwrap();
    430         let result = ready
    431             .store
    432             .record_delivery_observation(
    433                 ready.binding.clone(),
    434                 copy(&ready.proof),
    435                 observation.clone(),
    436             )
    437             .await;
    438         if commit {
    439             assert!(matches!(result, Ok(DeliveryObservationOutcome::Unknown(_))));
    440         } else {
    441             assert!(matches!(result, Err(OperationError::StorageFailure)));
    442         }
    443         ready.store.close().await.unwrap();
    444         let mut database = ready.fixture.database().await;
    445         if commit {
    446             sqlx::raw_sql("DROP TRIGGER observation_commit_fault_trigger; DROP TABLE observation_commit_fault").execute(&mut database).await.unwrap();
    447         } else {
    448             sqlx::query("DROP TRIGGER observation_insert_fault")
    449                 .execute(&mut database)
    450                 .await
    451                 .unwrap();
    452         }
    453         let counts:(i64,i64,i64)=sqlx::query_as("SELECT (SELECT count(*) FROM artifact_delivery_attempts),(SELECT count(*) FROM artifact_delivery_observations),(SELECT count(*) FROM chat_messages)").fetch_one(&mut database).await.unwrap();
    454         assert_eq!(counts, (1, 0, 1));
    455         database.close().await.unwrap();
    456         let reopened = ready.fixture.reopen(ready.input.clone()).await;
    457         assert_eq!(
    458             ready
    459                 .query(&reopened, attempt.intent().request_id())
    460                 .await
    461                 .raw_json(),
    462             attempt.raw_json()
    463         );
    464         assert!(
    465             ready
    466                 .observations(&reopened, attempt.intent().request_id())
    467                 .await
    468                 .is_empty()
    469         );
    470         assert_eq!(
    471             captured(
    472                 reopened
    473                     .record_delivery_observation(
    474                         ready.binding.clone(),
    475                         copy(&ready.proof),
    476                         observation.clone()
    477                     )
    478                     .await
    479                     .unwrap()
    480             ),
    481             observation
    482         );
    483         reopened.close().await.unwrap();
    484     }
    485 }
    486 
    487 #[tokio::test]
    488 async fn corrupted_delivery_ciphertext_and_oversized_observation_refuse_the_complete_snapshot() {
    489     let ready = Ready::new(&[String::from("ws://127.0.0.1:9")], 242).await;
    490     let attempt = ready.retain(1).await;
    491     captured(
    492         ready
    493             .store
    494             .record_delivery_observation(
    495                 ready.binding.clone(),
    496                 copy(&ready.proof),
    497                 not_attempted(&attempt),
    498             )
    499             .await
    500             .unwrap(),
    501     );
    502     let mut database = ready.fixture.database().await;
    503     let original: Vec<u8> =
    504         sqlx::query_scalar("SELECT protected FROM artifact_delivery_parts WHERE part=1")
    505             .fetch_one(&mut database)
    506             .await
    507             .unwrap();
    508     let mut wrong = original.clone();
    509     wrong[0] ^= 1;
    510     sqlx::query("UPDATE artifact_delivery_parts SET protected=? WHERE part=1")
    511         .bind(wrong)
    512         .execute(&mut database)
    513         .await
    514         .unwrap();
    515     let e = ready.work.evidence();
    516     assert!(
    517         ready
    518             .store
    519             .operation_snapshot(
    520                 ready.binding.clone(),
    521                 e.operation_id(),
    522                 None,
    523                 copy(&ready.proof)
    524             )
    525             .await
    526             .is_err()
    527     );
    528     sqlx::query("UPDATE artifact_delivery_parts SET protected=? WHERE part=1")
    529         .bind(original)
    530         .execute(&mut database)
    531         .await
    532         .unwrap();
    533     let original: Vec<u8> =
    534         sqlx::query_scalar("SELECT protected FROM artifact_delivery_observations")
    535             .fetch_one(&mut database)
    536             .await
    537             .unwrap();
    538     sqlx::query("PRAGMA ignore_check_constraints=ON")
    539         .execute(&mut database)
    540         .await
    541         .unwrap();
    542     sqlx::query("UPDATE artifact_delivery_observations SET protected=zeroblob(262145)")
    543         .execute(&mut database)
    544         .await
    545         .unwrap();
    546     assert!(
    547         ready
    548             .store
    549             .operation_snapshot(
    550                 ready.binding.clone(),
    551                 e.operation_id(),
    552                 None,
    553                 copy(&ready.proof)
    554             )
    555             .await
    556             .is_err()
    557     );
    558     sqlx::query("UPDATE artifact_delivery_observations SET protected=?")
    559         .bind(original)
    560         .execute(&mut database)
    561         .await
    562         .unwrap();
    563     database.close().await.unwrap();
    564     let snapshot = ready
    565         .store
    566         .operation_snapshot(
    567             ready.binding.clone(),
    568             e.operation_id(),
    569             None,
    570             copy(&ready.proof),
    571         )
    572         .await
    573         .unwrap()
    574         .unwrap();
    575     assert_eq!(snapshot.delivery_attempts().len(), 1);
    576     assert_eq!(snapshot.delivery_observations().len(), 1);
    577     ready.store.close().await.unwrap();
    578 }