lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

receipt_delivery.rs (28430B)


      1 use super::*;
      2 use futures::{SinkExt, StreamExt};
      3 use radroots_nostr::{
      4     key::parse_secret_key,
      5     signing::{LocalSigner, OsNonceSource, gift_wrap::wrap_retained_seal},
      6 };
      7 use radroots_signing::{
      8     issuance::{OutcomeKind, SigningIntent},
      9     provider::{ParticipantProvider, ProviderCall},
     10     request::CancellationSignal,
     11 };
     12 use radroots_storage::command::{
     13     ArtifactEvidence, DeliveryAttempt, DeliveryAttemptOutcome, DeliveryAttemptRequest,
     14     DeliveryIntent, DeliveryObservation, DeliveryObservationKind, DeliveryObservationOutcome,
     15     EnvelopeCandidate, EvidenceCaptureOutcome, InstallationOutcome, InstallationRequest,
     16     PublicationWork,
     17 };
     18 use radroots_transport::{
     19     EventSink, TargetSet,
     20     policy::{SatisfactionClass, SatisfactionPolicy, TargetPolicy},
     21 };
     22 use radroots_transport_nostr::{
     23     Config, RelayAccess, RelayEndpoint, RelayProfile, RelayProfileKind, RelayUrlPolicy,
     24 };
     25 use std::{
     26     num::NonZeroU64,
     27     time::{SystemTime, UNIX_EPOCH},
     28 };
     29 use tokio::net::TcpListener;
     30 use tokio_tungstenite::{accept_async, tungstenite::Message};
     31 #[path = "receipt_delivery_admission.rs"]
     32 mod admission_validation;
     33 #[path = "receipt_delivery_faults.rs"]
     34 mod faults;
     35 #[path = "receipt_delivery_validation.rs"]
     36 mod validation;
     37 
     38 fn now_ms() -> u64 {
     39     SystemTime::now()
     40         .duration_since(UNIX_EPOCH)
     41         .unwrap()
     42         .as_millis()
     43         .try_into()
     44         .unwrap()
     45 }
     46 fn signer_now() -> Result<u64, radroots_signing::Error> {
     47     Ok(now_ms())
     48 }
     49 struct Ready {
     50     fixture: Fixture,
     51     store: Arc<NativeWorkspace>,
     52     input: AttachmentInput,
     53     proof: AttachmentProof,
     54     binding: WorkspaceBinding,
     55     work: PublicationWork,
     56     candidate: EnvelopeCandidate,
     57     config: Config,
     58 }
     59 impl Ready {
     60     async fn new(urls: &[String], id: u8) -> Self {
     61         let fixture = Fixture::new();
     62         let time = now_ms() / 1000;
     63         fixture.clock.now.store(time, Ordering::SeqCst);
     64         let (store, input, proof, binding) = fixture.create_with_expiry(time + 600).await;
     65         let mut value: serde_json::Value = serde_json::from_slice(
     66             request_until(
     67                 &binding,
     68                 id,
     69                 "one exact native delivery",
     70                 "delivery",
     71                 time + 120,
     72             )
     73             .command()
     74             .protected_wire(),
     75         )
     76         .unwrap();
     77         value["input"]["peer"] = keys(2).public_key().to_hex().into();
     78         let admitted = AdmissionRequest::from_protected_wire(
     79             binding.clone(),
     80             &serde_json::to_vec(&value).unwrap(),
     81         )
     82         .unwrap();
     83         let prepared = committed(
     84             store
     85                 .admit_command_and_hold(admitted, copy(&proof))
     86                 .await
     87                 .unwrap(),
     88         );
     89         let plan = plan_with_routes(
     90             prepared.request().chat_command().unwrap(),
     91             time,
     92             "delivery",
     93             Some(urls),
     94         );
     95         let digest = *plan.digest();
     96         let recipient = prepared.request().chat_command().unwrap().peer();
     97         bound(
     98             store
     99                 .bind_signing_intents(binding.clone(), (plan).into(), copy(&proof))
    100                 .await
    101                 .unwrap(),
    102         );
    103         let claim = issuance::claim(
    104             store
    105                 .claim_attempt(
    106                     issuance::selection(&binding, &prepared, digest, recipient),
    107                     copy(&proof),
    108                 )
    109                 .await
    110                 .unwrap(),
    111         );
    112         let signer = LocalSigner::with_clock(
    113             parse_secret_key("0000000000000000000000000000000000000000000000000000000000000001")
    114                 .unwrap(),
    115             signer_now,
    116         )
    117         .unwrap()
    118         .with_nonce_source(Arc::new(OsNonceSource));
    119         let intent = SigningIntent::from_chat_seal(
    120             claim.binding().operation_id(),
    121             prepared.identity().command_digest(),
    122             claim.seal().unwrap(),
    123             ProviderCall::new(key(), claim.deadline_unix_ms(), CancellationSignal::new()),
    124         )
    125         .unwrap();
    126         let (_, result) = signer.sign_exact(intent).await.into_parts();
    127         let OutcomeKind::Obtained(artifact) = result else {
    128             panic!("synthetic participant seal must be obtained")
    129         };
    130         let seal = artifact.with_json(|raw| {
    131             ArtifactEvidence::verify(&claim, raw, artifact.observed_at_unix_ms()).unwrap()
    132         });
    133         assert!(matches!(
    134             store
    135                 .capture_evidence(input.clone(), &proof, &seal)
    136                 .await
    137                 .unwrap(),
    138             EvidenceCaptureOutcome::Committed(_)
    139         ));
    140         assert!(matches!(
    141             store
    142                 .install_transition_and_outbox(
    143                     InstallationRequest::new(
    144                         binding.clone(),
    145                         prepared.identity().operation_id(),
    146                         prepared.identity().command_digest(),
    147                         digest
    148                     ),
    149                     copy(&proof)
    150                 )
    151                 .await
    152                 .unwrap(),
    153             InstallationOutcome::Committed(_)
    154         ));
    155         let snapshot = store
    156             .operation_snapshot(
    157                 binding.clone(),
    158                 prepared.identity().operation_id(),
    159                 None,
    160                 copy(&proof),
    161             )
    162             .await
    163             .unwrap()
    164             .unwrap();
    165         let work = snapshot.publications()[0];
    166         let pending = wrap_retained_seal(
    167             seal.raw_json(),
    168             recipient,
    169             NonZeroU64::new(time).unwrap(),
    170             &OsNonceSource,
    171         )
    172         .unwrap();
    173         let candidate = pending.with_wrapping_key(|key| {
    174             EnvelopeCandidate::from_obtained(work, pending.signed_json(), key).unwrap()
    175         });
    176         let config = Config::from_profile(
    177             RelayProfile::explicit(
    178                 RelayProfileKind::Simulator,
    179                 urls.iter().map(|url| {
    180                     RelayEndpoint::new(url, RelayUrlPolicy::Local, RelayAccess::ReadWrite).unwrap()
    181                 }),
    182             )
    183             .unwrap(),
    184         )
    185         .with_timeouts(1000, 1000, 500)
    186         .unwrap();
    187         Self {
    188             fixture,
    189             store,
    190             input,
    191             proof,
    192             binding,
    193             work,
    194             candidate,
    195             config,
    196         }
    197     }
    198     fn request(&self, id: u8) -> DeliveryAttemptRequest {
    199         let targets = TargetSet::new(
    200             self.config
    201                 .relays()
    202                 .iter()
    203                 .map(|relay| relay.to_target().unwrap())
    204                 .collect(),
    205         )
    206         .unwrap();
    207         let intent = DeliveryIntent::new(
    208             &format!("{id:02x}").repeat(32),
    209             self.candidate.event_id(),
    210             &targets,
    211             &SatisfactionPolicy::new(SatisfactionClass::Accepted, TargetPolicy::all()),
    212             NonZeroU64::new(self.work.queued_at().get() * 1000 + 60_000).unwrap(),
    213         )
    214         .unwrap();
    215         DeliveryAttemptRequest::new(self.binding.clone(), self.work, intent).unwrap()
    216     }
    217     async fn retain(&self, id: u8) -> DeliveryAttempt {
    218         match self
    219             .store
    220             .prepare_delivery_attempt(self.request(id), copy(&self.proof), Some(&self.candidate))
    221             .await
    222             .unwrap()
    223         {
    224             DeliveryAttemptOutcome::Claimed(value) => *value,
    225             _ => panic!("new actual retained delivery expected"),
    226         }
    227     }
    228     async fn query(
    229         &self,
    230         store: &NativeWorkspace,
    231         id: &radroots_transport::sink::DeliveryRequestId,
    232     ) -> DeliveryAttempt {
    233         let e = self.work.evidence();
    234         store
    235             .delivery_attempt(
    236                 self.binding.clone(),
    237                 e.operation_id(),
    238                 e.command_digest(),
    239                 e.target(),
    240                 id.clone(),
    241                 copy(&self.proof),
    242             )
    243             .await
    244             .unwrap()
    245             .unwrap()
    246     }
    247     async fn observations(
    248         &self,
    249         store: &NativeWorkspace,
    250         id: &radroots_transport::sink::DeliveryRequestId,
    251     ) -> Vec<DeliveryObservation> {
    252         let e = self.work.evidence();
    253         store
    254             .delivery_observations(
    255                 self.binding.clone(),
    256                 e.operation_id(),
    257                 e.command_digest(),
    258                 e.target(),
    259                 id.clone(),
    260                 copy(&self.proof),
    261             )
    262             .await
    263             .unwrap()
    264     }
    265 }
    266 fn captured(outcome: DeliveryObservationOutcome) -> DeliveryObservation {
    267     match outcome {
    268         DeliveryObservationOutcome::Committed(value) => value,
    269         _ => panic!("confirmed original observation expected"),
    270     }
    271 }
    272 #[tokio::test]
    273 async fn native_delivery_retains_exact_outer_before_loopback_and_reopen_preserves_original_observation()
    274  {
    275     let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    276     let ready = Ready::new(&[format!("ws://{}", listener.local_addr().unwrap())], 230).await;
    277     let request = ready.request(1);
    278     let original_intent = request.intent().clone();
    279     let DeliveryAttemptOutcome::Claimed(attempt) = ready
    280         .store
    281         .prepare_delivery_attempt(request, copy(&ready.proof), Some(&ready.candidate))
    282         .await
    283         .unwrap()
    284     else {
    285         panic!("new retention required")
    286     };
    287     let replay = ready
    288         .store
    289         .prepare_delivery_attempt(
    290             DeliveryAttemptRequest::new(ready.binding.clone(), ready.work, original_intent)
    291                 .unwrap(),
    292             copy(&ready.proof),
    293             None,
    294         )
    295         .await
    296         .unwrap();
    297     assert!(matches!(replay, DeliveryAttemptOutcome::Retained(_)));
    298     let expected = format!("[\"EVENT\",{}]", attempt.raw_json());
    299     let e = ready.work.evidence();
    300     let pending = ready
    301         .store
    302         .operation_snapshot(
    303             ready.binding.clone(),
    304             e.operation_id(),
    305             None,
    306             copy(&ready.proof),
    307         )
    308         .await
    309         .unwrap()
    310         .unwrap();
    311     assert_eq!(pending.delivery_attempts().len(), 1);
    312     assert!(pending.delivery_observations().is_empty());
    313     let projected =
    314         radroots_execution::OperationReceipt::<radroots_domain::ChatMessage>::try_from_snapshot(
    315             &pending,
    316         )
    317         .unwrap();
    318     assert_eq!(projected.delivery(), radroots_execution::Delivery::Unknown);
    319     assert_eq!(
    320         projected.delivery_attempts()[0].envelope_digest(),
    321         attempt.envelope_digest()
    322     );
    323     let expected_message = pending.installed_message().unwrap().id();
    324     let server = tokio::spawn(async move {
    325         let (tcp, _) = listener.accept().await.unwrap();
    326         let mut socket = accept_async(tcp).await.unwrap();
    327         let raw = tokio::time::timeout(Duration::from_secs(5), async {
    328             loop {
    329                 if let Message::Text(raw) = socket.next().await.unwrap().unwrap() {
    330                     break raw.to_string();
    331                 }
    332             }
    333         })
    334         .await
    335         .unwrap();
    336         assert_eq!(raw, expected);
    337         let event: serde_json::Value = serde_json::from_str(&raw).unwrap();
    338         let outer = nostr::Event::from_json(event[1].to_string()).unwrap();
    339         let recovered = nostr::nips::nip59::extract_rumor(&keys(2), &outer)
    340             .await
    341             .unwrap();
    342         assert_eq!(recovered.sender, keys(1).public_key());
    343         assert_eq!(
    344             recovered.rumor.id.unwrap().as_bytes(),
    345             expected_message.as_bytes()
    346         );
    347         assert_eq!(recovered.rumor.content, "one exact native delivery");
    348         socket
    349             .send(Message::Text(
    350                 json!(["OK", "ff".repeat(32), true, ""]).to_string().into(),
    351             ))
    352             .await
    353             .unwrap();
    354         socket
    355             .send(Message::Text(
    356                 json!(["OK", event[1]["id"], true, ""]).to_string().into(),
    357             ))
    358             .await
    359             .unwrap();
    360     });
    361     let transport = radroots_transport_nostr_native::create_transport(ready.config.clone());
    362     let request = attempt.transport_request().unwrap();
    363     let receipt = transport.deliver(request.clone()).await.unwrap();
    364     receipt.validate_for_request(&request).unwrap();
    365     assert!(receipt.is_satisfied(&request).unwrap());
    366     server.await.unwrap();
    367     let target = request.target_set().targets()[0].fingerprint();
    368     let observed = DeliveryObservation::from_receipt(&attempt, &receipt, target, None).unwrap();
    369     assert_eq!(observed.kind(), DeliveryObservationKind::Accepted);
    370     let first = captured(
    371         ready
    372             .store
    373             .record_delivery_observation(ready.binding.clone(), copy(&ready.proof), observed)
    374             .await
    375             .unwrap(),
    376     );
    377     assert_eq!(first.observed_at_unix_ms(), None);
    378     let complete = ready
    379         .store
    380         .operation_snapshot(
    381             ready.binding.clone(),
    382             e.operation_id(),
    383             None,
    384             copy(&ready.proof),
    385         )
    386         .await
    387         .unwrap()
    388         .unwrap();
    389     assert_ne!(complete.checkpoint_digest(), pending.checkpoint_digest());
    390     let projected =
    391         radroots_execution::OperationReceipt::<radroots_domain::ChatMessage>::try_from_snapshot(
    392             &complete,
    393         )
    394         .unwrap();
    395     assert_eq!(
    396         projected.business_installation(),
    397         radroots_execution::BusinessInstallation::Installed
    398     );
    399     assert_eq!(projected.delivery(), radroots_execution::Delivery::Observed);
    400     assert!(
    401         !projected
    402             .recovery()
    403             .contains(&radroots_execution::RecoveryRequirement::ResumeDelivery)
    404     );
    405     assert_eq!(projected.delivery_observations().len(), 1);
    406     assert_eq!(
    407         projected.delivery_observations()[0].event_id(),
    408         attempt.intent().event_id()
    409     );
    410     assert_eq!(
    411         projected.delivery_observations()[0].request_id(),
    412         attempt.intent().request_id().as_str()
    413     );
    414     assert_eq!(projected.delivery_observations()[0].target_ordinal(), 1);
    415     assert_eq!(
    416         projected.delivery_observations()[0].outcome(),
    417         radroots_execution::TargetObservation::Accepted
    418     );
    419     let later =
    420         DeliveryObservation::from_receipt(&attempt, &receipt, target, NonZeroU64::new(now_ms()))
    421             .unwrap();
    422     assert_eq!(
    423         captured(
    424             ready
    425                 .store
    426                 .record_delivery_observation(ready.binding.clone(), copy(&ready.proof), later)
    427                 .await
    428                 .unwrap()
    429         ),
    430         first
    431     );
    432     ready.store.close().await.unwrap();
    433     let reopened = ready.fixture.reopen(ready.input.clone()).await;
    434     let retained = ready.query(&reopened, attempt.intent().request_id()).await;
    435     assert_eq!(retained.raw_json(), ready.candidate.raw_json());
    436     assert_eq!(retained.transaction_id(), attempt.transaction_id());
    437     assert_eq!(retained.target_ordinals(), &[1]);
    438     assert_eq!(
    439         ready
    440             .observations(&reopened, attempt.intent().request_id())
    441             .await,
    442         vec![first]
    443     );
    444     reopened.close().await.unwrap();
    445 }
    446 
    447 #[tokio::test]
    448 async fn predecessor_schema_six_preserves_original_installation_and_seal_without_automatic_upgrade()
    449 {
    450     let ready = Ready::new(&[String::from("ws://127.0.0.1:9")], 236).await;
    451     ready.store.close().await.unwrap();
    452     let mut database = ready.fixture.database().await;
    453     sqlx::raw_sql("DROP TABLE public_publication_work; DROP TABLE listing_installations; DROP TABLE listing_creations; DROP TABLE artifact_delivery_observations; DROP TABLE artifact_delivery_parts; DROP TABLE artifact_delivery_attempts; PRAGMA user_version=6").execute(&mut database).await.unwrap();
    454     let counts:(i64,i64,i64)=sqlx::query_as("SELECT (SELECT count(*) FROM sqlite_schema),(SELECT count(*) FROM chat_messages),(SELECT count(*) FROM artifact_evidence)").fetch_one(&mut database).await.unwrap();
    455     assert_eq!(counts, (16, 1, 1));
    456     database.close().await.unwrap();
    457     let path = ready.fixture.path.join("execution_v1.sqlite");
    458     let original = std::fs::read(&path).unwrap();
    459     assert!(matches!(
    460         NativeWorkspace::open_existing(
    461             &ready.fixture.path,
    462             ready.input,
    463             ready.fixture.crypto(),
    464             ready.fixture.clock.clone()
    465         )
    466         .await,
    467         Err(WorkspaceError::RecoveryRequired)
    468     ));
    469     assert_eq!(std::fs::read(path).unwrap(), original);
    470 }
    471 
    472 #[tokio::test]
    473 async fn native_retry_reuses_exact_bytes_and_enforces_twelve_potential_attempts_per_endpoint() {
    474     let ready = Ready::new(&[String::from("ws://127.0.0.1:9")], 231).await;
    475     let first = ready.retain(1).await;
    476     for ordinal in 2..=12u8 {
    477         let DeliveryAttemptOutcome::Claimed(attempt) = ready
    478             .store
    479             .prepare_delivery_attempt(ready.request(ordinal), copy(&ready.proof), None)
    480             .await
    481             .unwrap()
    482         else {
    483             panic!("fresh bounded retry expected")
    484         };
    485         assert_eq!(attempt.target_ordinals(), &[ordinal]);
    486         assert_eq!(attempt.raw_json(), first.raw_json());
    487         assert_eq!(attempt.envelope_digest(), first.envelope_digest());
    488         assert_ne!(attempt.transaction_id(), first.transaction_id());
    489     }
    490     assert!(matches!(
    491         ready
    492             .store
    493             .prepare_delivery_attempt(ready.request(13), copy(&ready.proof), None)
    494             .await,
    495         Err(OperationError::DeliveryAttemptLimit)
    496     ));
    497     let original = ready.query(&ready.store, first.intent().request_id()).await;
    498     assert_eq!(original.raw_json(), first.raw_json());
    499     assert_eq!(original.target_ordinals(), &[1]);
    500     ready.store.close().await.unwrap();
    501 }
    502 
    503 #[tokio::test]
    504 async fn actual_delivery_part_failure_rolls_back_attempt_and_keeps_original_installation() {
    505     let ready = Ready::new(&[String::from("ws://127.0.0.1:9")], 232).await;
    506     let mut database = ready.fixture.database().await;
    507     sqlx::raw_sql("CREATE TRIGGER refuse_delivery_part BEFORE INSERT ON artifact_delivery_parts BEGIN SELECT RAISE(ABORT,'synthetic part failure'); END").execute(&mut database).await.unwrap();
    508     assert!(matches!(
    509         ready
    510             .store
    511             .prepare_delivery_attempt(ready.request(1), copy(&ready.proof), Some(&ready.candidate))
    512             .await,
    513         Err(OperationError::StorageFailure)
    514     ));
    515     let counts:(i64,i64,i64,i64)=sqlx::query_as("SELECT (SELECT count(*) FROM artifact_delivery_attempts),(SELECT count(*) FROM artifact_delivery_parts),(SELECT count(*) FROM chat_messages),(SELECT count(*) FROM artifact_evidence)").fetch_one(&mut database).await.unwrap();
    516     assert_eq!(counts, (0, 0, 1, 1));
    517     sqlx::query("DROP TRIGGER refuse_delivery_part")
    518         .execute(&mut database)
    519         .await
    520         .unwrap();
    521     database.close().await.unwrap();
    522     assert_eq!(ready.retain(1).await.target_ordinals(), &[1]);
    523     ready.store.close().await.unwrap();
    524 }
    525 
    526 #[tokio::test]
    527 async fn actual_delivery_commit_uncertainty_latches_and_reopen_preserves_business_facts() {
    528     let ready = Ready::new(&[String::from("ws://127.0.0.1:9")], 233).await;
    529     let request = ready.request(1);
    530     let id = request.intent().request_id().clone();
    531     let mut database = ready.fixture.database().await;
    532     sqlx::raw_sql("CREATE TABLE delivery_commit_fault (parent INTEGER REFERENCES workspace_grant(singleton) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER delivery_commit_fault_trigger AFTER INSERT ON artifact_delivery_attempts BEGIN INSERT INTO delivery_commit_fault VALUES(2); END").execute(&mut database).await.unwrap();
    533     database.close().await.unwrap();
    534     let DeliveryAttemptOutcome::Unknown { operation, attempt } = ready
    535         .store
    536         .prepare_delivery_attempt(request, copy(&ready.proof), Some(&ready.candidate))
    537         .await
    538         .unwrap()
    539     else {
    540         panic!("actual commit failure must remain unknown")
    541     };
    542     assert_eq!(attempt, id);
    543     assert_eq!(
    544         operation.operation_id(),
    545         ready.work.evidence().operation_id()
    546     );
    547     assert!(operation.transaction_id().is_some());
    548     let e = ready.work.evidence();
    549     assert!(matches!(
    550         ready
    551             .store
    552             .delivery_attempt(
    553                 ready.binding.clone(),
    554                 e.operation_id(),
    555                 e.command_digest(),
    556                 e.target(),
    557                 id.clone(),
    558                 copy(&ready.proof)
    559             )
    560             .await,
    561         Err(OperationError::RecoveryRequired)
    562     ));
    563     ready.store.close().await.unwrap();
    564     let mut database = ready.fixture.database().await;
    565     sqlx::raw_sql("DROP TRIGGER delivery_commit_fault_trigger; DROP TABLE delivery_commit_fault")
    566         .execute(&mut database)
    567         .await
    568         .unwrap();
    569     database.close().await.unwrap();
    570     let reopened = ready.fixture.reopen(ready.input.clone()).await;
    571     assert!(
    572         reopened
    573             .delivery_attempt(
    574                 ready.binding.clone(),
    575                 e.operation_id(),
    576                 e.command_digest(),
    577                 e.target(),
    578                 id,
    579                 copy(&ready.proof)
    580             )
    581             .await
    582             .unwrap()
    583             .is_none()
    584     );
    585     let snapshot = reopened
    586         .operation_snapshot(
    587             ready.binding.clone(),
    588             e.operation_id(),
    589             None,
    590             copy(&ready.proof),
    591         )
    592         .await
    593         .unwrap()
    594         .unwrap();
    595     assert!(snapshot.installed_message().is_some());
    596     assert_eq!(snapshot.publications(), &[ready.work]);
    597     reopened.close().await.unwrap();
    598 }
    599 
    600 #[tokio::test]
    601 async fn admitted_delivery_survives_lost_observer_and_close_drains_original_attempt() {
    602     let ready = Ready::new(&[String::from("ws://127.0.0.1:9")], 234).await;
    603     let request = ready.request(1);
    604     let id = request.intent().request_id().clone();
    605     let (arrived, resume) = ready.fixture.clock.arm(1);
    606     let owner = ready.store.clone();
    607     let proof = copy(&ready.proof);
    608     let candidate = ready.candidate.retained_copy();
    609     let observer = tokio::spawn(async move {
    610         owner
    611             .prepare_delivery_attempt(request, proof, Some(&candidate))
    612             .await
    613     });
    614     reached(arrived).await;
    615     observer.abort();
    616     assert!(observer.await.unwrap_err().is_cancelled());
    617     resume.send(()).unwrap();
    618     ready.store.close().await.unwrap();
    619     let reopened = ready.fixture.reopen(ready.input.clone()).await;
    620     assert_eq!(
    621         ready.query(&reopened, &id).await.raw_json(),
    622         ready.candidate.raw_json()
    623     );
    624     reopened.close().await.unwrap();
    625 }
    626 
    627 #[tokio::test]
    628 async fn late_observation_append_requires_original_credential_but_no_fresh_clock_or_dispatch_grant()
    629 {
    630     let ready = Ready::new(&[String::from("ws://127.0.0.1:9")], 235).await;
    631     let attempt = ready.retain(1).await;
    632     let request = attempt.transport_request().unwrap();
    633     let failure = radroots_transport::SinkFailure::transport_closed(&request);
    634     let observation = DeliveryObservation::from_failure(
    635         &attempt,
    636         &failure,
    637         request.target_set().targets()[0].fingerprint(),
    638         None,
    639     )
    640     .unwrap();
    641     assert_eq!(observation.kind(), DeliveryObservationKind::NotAttempted);
    642     ready
    643         .store
    644         .revoke(ready.input.clone(), &ready.proof)
    645         .await
    646         .unwrap();
    647     ready.fixture.clock.now.store(0, Ordering::SeqCst);
    648     let wrong = AttachmentProof::from_protected_input(&[0x33; 32]).unwrap();
    649     assert!(
    650         ready
    651             .store
    652             .record_delivery_observation(ready.binding.clone(), wrong, observation.clone())
    653             .await
    654             .is_err()
    655     );
    656     let original = captured(
    657         ready
    658             .store
    659             .record_delivery_observation(ready.binding.clone(), copy(&ready.proof), observation)
    660             .await
    661             .unwrap(),
    662     );
    663     assert_eq!(original.observed_at_unix_ms(), None);
    664     ready.store.close().await.unwrap();
    665     // Reopening validates the protected schema, but does not renew revoked access.
    666     let reopened = ready.fixture.reopen(ready.input.clone()).await;
    667     assert!(
    668         reopened
    669             .authenticate(ready.input.clone(), &ready.proof)
    670             .await
    671             .is_err()
    672     );
    673     reopened.close().await.unwrap();
    674 }
    675 
    676 #[path = "legacy_delivery_catalog.rs"]
    677 mod legacy_delivery_catalog;
    678 
    679 #[tokio::test]
    680 async fn exact_generation_ten_reopens_private_delivery_bytes_and_continues_without_reencoding() {
    681     let ready = Ready::new(&["ws://127.0.0.1:9".into()], 236).await;
    682     let first = ready.retain(1).await;
    683     let evidence = ready.work.evidence();
    684     let checkpoint = ready
    685         .store
    686         .operation_snapshot(
    687             ready.binding.clone(),
    688             evidence.operation_id(),
    689             None,
    690             copy(&ready.proof),
    691         )
    692         .await
    693         .unwrap()
    694         .unwrap()
    695         .checkpoint_digest();
    696     ready.store.close().await.unwrap();
    697     let mut db = ready.fixture.database().await;
    698     legacy_delivery_catalog::retain_legacy_delivery_catalog(&mut db).await;
    699     sqlx::query("PRAGMA user_version=10")
    700         .execute(&mut db)
    701         .await
    702         .unwrap();
    703     let catalog: Vec<(String, String)> = sqlx::query_as(
    704         "SELECT name,sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' ORDER BY name",
    705     )
    706     .fetch_all(&mut db)
    707     .await
    708     .unwrap();
    709     let protected:Vec<Vec<u8>>=sqlx::query_scalar("SELECT protected FROM chat_delivery_attempts UNION ALL SELECT protected FROM chat_delivery_parts").fetch_all(&mut db).await.unwrap();
    710     db.close().await.unwrap();
    711     let reopened = ready.fixture.reopen(ready.input.clone()).await;
    712     let retained = reopened
    713         .delivery_attempt(
    714             ready.binding.clone(),
    715             evidence.operation_id(),
    716             evidence.command_digest(),
    717             evidence.target(),
    718             first.intent().request_id().clone(),
    719             copy(&ready.proof),
    720         )
    721         .await
    722         .unwrap()
    723         .unwrap();
    724     assert_eq!(retained.raw_json(), first.raw_json());
    725     assert_eq!(retained.transaction_id(), first.transaction_id());
    726     assert_eq!(retained.intent().digest(), first.intent().digest());
    727     let snapshot = reopened
    728         .operation_snapshot(
    729             ready.binding.clone(),
    730             evidence.operation_id(),
    731             None,
    732             copy(&ready.proof),
    733         )
    734         .await
    735         .unwrap()
    736         .unwrap();
    737     assert_eq!(snapshot.checkpoint_digest(), checkpoint);
    738     assert!(matches!(
    739         reopened
    740             .prepare_delivery_attempt(ready.request(1), copy(&ready.proof), None)
    741             .await
    742             .unwrap(),
    743         DeliveryAttemptOutcome::Retained(_)
    744     ));
    745     let second = match reopened
    746         .prepare_delivery_attempt(ready.request(2), copy(&ready.proof), None)
    747         .await
    748         .unwrap()
    749     {
    750         DeliveryAttemptOutcome::Claimed(v) => v,
    751         _ => panic!("fresh explicit private retry"),
    752     };
    753     assert_eq!(second.raw_json(), first.raw_json());
    754     assert_eq!(second.target_ordinals(), &[2]);
    755     let request = second.transport_request().unwrap();
    756     let failure = radroots_transport::SinkFailure::transport_closed(&request);
    757     let observation = DeliveryObservation::from_failure(
    758         &second,
    759         &failure,
    760         request.target_set().targets()[0].fingerprint(),
    761         None,
    762     )
    763     .unwrap();
    764     assert!(matches!(
    765         reopened
    766             .record_delivery_observation(ready.binding.clone(), copy(&ready.proof), observation)
    767             .await
    768             .unwrap(),
    769         DeliveryObservationOutcome::Committed(_)
    770     ));
    771     reopened.close().await.unwrap();
    772     let mut db = ready.fixture.database().await;
    773     let actual: Vec<(String, String)> = sqlx::query_as(
    774         "SELECT name,sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' ORDER BY name",
    775     )
    776     .fetch_all(&mut db)
    777     .await
    778     .unwrap();
    779     assert_eq!(actual, catalog);
    780     let current:Vec<Vec<u8>>=sqlx::query_scalar("SELECT protected FROM chat_delivery_attempts UNION ALL SELECT protected FROM chat_delivery_parts").fetch_all(&mut db).await.unwrap();
    781     assert!(protected.iter().all(|v| current.contains(v)));
    782     assert_eq!(
    783         sqlx::query_scalar::<_, i64>("PRAGMA user_version")
    784             .fetch_one(&mut db)
    785             .await
    786             .unwrap(),
    787         10
    788     );
    789     db.close().await.unwrap();
    790 }