native_refusal_tests.rs (11581B)
1 //! Actual ordinary-session refusal ownership, SQL faults and lost observation. 2 use super::{ 3 AcceptOrderInput, AccessPolicy, AllocationSelection, BackendKind, CapturedOrderAcceptance, 4 KeyVersion, NativeWorkspace, Ordering, ProviderError, Rig, SecretId, SecretMaterial, SecretRef, 5 StockSnapshotRequest, UnixSeconds, WorkspaceClock, WorkspaceCrypto, 6 }; 7 use sqlx::Connection; 8 9 #[tokio::test] 10 async fn actual_original_refusals_allow_explicit_cleanup_and_aborted_status_never_reissues() { 11 let r = Rig::new().await; 12 r.control.fail_peer.store(true, Ordering::SeqCst); 13 r.control.fail_self.store(true, Ordering::SeqCst); 14 let (command, original) = r.accept().await; 15 assert!(!original.aborted()); 16 assert!(original.installation().is_none()); 17 for facts in original.recipients() { 18 assert!(facts.evidence.is_none()); 19 assert_eq!( 20 facts.original_unissued.unwrap().claim, 21 facts.claim.unwrap().id 22 ); 23 } 24 assert_eq!(r.control.signs.load(Ordering::SeqCst), 2); 25 let (_, _, minimum) = r.minima().await; 26 let radroots_storage::inventory::StockAbortOutcome::Aborted(released) = r 27 .client 28 .operations(&r.scope) 29 .abort_proven_unissued(minimum, command.context().operation_id(), command.digest()) 30 .await 31 .unwrap() 32 else { 33 panic!("actual cleanup must be confirmed"); 34 }; 35 let (_, _, minimum) = r.minima().await; 36 let observed = r 37 .client 38 .orders(&r.scope) 39 .acceptance_status(minimum, command.context().operation_id(), command.digest()) 40 .await 41 .unwrap(); 42 assert!(observed.aborted()); 43 assert!(observed.installation().is_none()); 44 assert!(observed.recipients() == original.recipients()); 45 r.control.fail_peer.store(false, Ordering::SeqCst); 46 r.control.fail_self.store(false, Ordering::SeqCst); 47 let resumed = r 48 .client 49 .orders(&r.scope) 50 .resume_acceptance(minimum, command.context().operation_id(), command.digest()) 51 .await 52 .unwrap(); 53 assert!(resumed.aborted()); 54 assert!(resumed.recipients() == original.recipients()); 55 assert_eq!(r.control.signs.load(Ordering::SeqCst), 2); 56 assert_eq!(r.control.recoveries.load(Ordering::SeqCst), 0); 57 assert_eq!( 58 r.client 59 .operations(&r.scope) 60 .preparation_abort(minimum, command.context().operation_id(), command.digest()) 61 .await 62 .unwrap() 63 .unwrap() 64 .attempt(), 65 released.attempt() 66 ); 67 let mut raw = sqlx::SqliteConnection::connect(&format!( 68 "sqlite://{}", 69 r._temp 70 .path() 71 .join("stock/stock_execution_v1.sqlite") 72 .display() 73 )) 74 .await 75 .unwrap(); 76 let facts: (i64, i64, i64, i64, i64) = sqlx::query_as( 77 "SELECT (SELECT count(*) FROM stock_signing_claims),(SELECT count(*) FROM stock_original_unissued),(SELECT count(*) FROM stock_signing_evidence),(SELECT count(*) FROM stock_replay_admissions),(SELECT count(*) FROM stock_aborts)", 78 ) 79 .fetch_one(&mut raw) 80 .await 81 .unwrap(); 82 assert_eq!(facts, (2, 2, 0, 0, 1)); 83 raw.close().await.unwrap(); 84 r.client.close().await.unwrap(); 85 } 86 87 async fn fault(r: &Rig, uncertain: bool) -> sqlx::SqliteConnection { 88 let path = r._temp.path().join("stock/stock_execution_v1.sqlite"); 89 let mut connection = sqlx::SqliteConnection::connect(&format!("sqlite://{}", path.display())) 90 .await 91 .unwrap(); 92 if uncertain { 93 sqlx::raw_sql("PRAGMA foreign_keys=ON; CREATE TABLE deferred_refusal_fault(value INTEGER REFERENCES stock_authority(singleton) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER controlled_refusal_fault AFTER INSERT ON stock_original_unissued BEGIN INSERT INTO deferred_refusal_fault VALUES(2); END").execute(&mut connection).await.unwrap(); 94 } else { 95 sqlx::query("CREATE TRIGGER controlled_refusal_fault BEFORE INSERT ON stock_original_unissued BEGIN SELECT RAISE(ABORT,'original_refusal_fault'); END").execute(&mut connection).await.unwrap(); 96 } 97 connection 98 } 99 async fn remove_fault(connection: &mut sqlx::SqliteConnection) { 100 sqlx::raw_sql( 101 "DROP TRIGGER controlled_refusal_fault; DROP TABLE IF EXISTS deferred_refusal_fault", 102 ) 103 .execute(&mut *connection) 104 .await 105 .unwrap(); 106 } 107 fn assert_fact(status: &crate::orders::AcceptanceStatus, command: &CapturedOrderAcceptance) { 108 assert!(status.installation().is_some()); 109 let value = status 110 .recipients() 111 .iter() 112 .find(|v| v.original_unissued.is_some()) 113 .unwrap(); 114 let receipt = value.original_unissued.unwrap(); 115 assert_eq!(receipt.claim, value.claim.unwrap().id); 116 assert_eq!( 117 receipt.attempt.operation_id(), 118 command.context().operation_id() 119 ); 120 assert_eq!(receipt.attempt.command_digest(), command.digest()); 121 assert!(receipt.attempt.transaction_id().is_some()); 122 assert_eq!(receipt.reason, ProviderError::Cancelled); 123 assert!(value.evidence.is_none()); 124 assert!(!status.aborted()); 125 } 126 127 #[tokio::test] 128 async fn actual_refusal_append_failure_is_owned_and_explicit_drain_needs_no_clock_or_reissue() { 129 let r = Rig::new().await; 130 r.control.fail_self.store(true, Ordering::SeqCst); 131 let mut connection = fault(&r, false).await; 132 let (command, context) = r.captured().await; 133 let (private, public, minimum) = r.minima().await; 134 let input = || { 135 AcceptOrderInput::new( 136 command.input().request(), 137 command.input().terms_digest(), 138 AllocationSelection::automatic(), 139 ) 140 }; 141 let failure = r 142 .client 143 .orders(&r.scope) 144 .accept(context, minimum, private, public, input()) 145 .await 146 .unwrap_err(); 147 assert_eq!(failure.code(), crate::Code::StorageFailure); 148 assert_eq!(failure.context().admission(), crate::Admission::Confirmed); 149 assert_eq!( 150 failure.context().operation_id(), 151 Some(command.context().operation_id()) 152 ); 153 assert_eq!(r.control.signs.load(Ordering::SeqCst), 2); 154 let context = r 155 .scope 156 .command_context( 157 command.context().operation_id(), 158 command.context().expected().cloned(), 159 command.context().approval().cloned(), 160 UnixSeconds::new(230).unwrap(), 161 ) 162 .unwrap(); 163 let blocked = r 164 .client 165 .orders(&r.scope) 166 .accept(context, minimum, private, public, input()) 167 .await 168 .unwrap_err(); 169 assert_eq!(blocked.code(), crate::Code::RecoveryRequired); 170 remove_fault(&mut connection).await; 171 connection.close().await.unwrap(); 172 r.clock.0.store(u64::MAX, Ordering::SeqCst); 173 assert!(r.clock.now().is_err()); 174 r.client.stock(&r.scope).drain_evidence().await.unwrap(); 175 assert_eq!(r.control.signs.load(Ordering::SeqCst), 2); 176 assert_eq!(r.control.recoveries.load(Ordering::SeqCst), 0); 177 r.clock.0.store(150, Ordering::SeqCst); 178 let status = r 179 .client 180 .orders(&r.scope) 181 .acceptance_status(minimum, command.context().operation_id(), command.digest()) 182 .await 183 .unwrap(); 184 assert_fact(&status, &command); 185 r.client.close().await.unwrap(); 186 assert!(r.provider.upgrade().is_none()); 187 } 188 189 #[tokio::test] 190 async fn lost_refusal_waiter_and_failed_close_retain_owner_through_rollback_and_commit_uncertainty() 191 { 192 for uncertain in [false, true] { 193 let r = Rig::new().await; 194 r.control.fail_self.store(true, Ordering::SeqCst); 195 r.control.pause_refused.store(true, Ordering::SeqCst); 196 let (command, context) = r.captured().await; 197 let (private, public, minimum) = r.minima().await; 198 let input = AcceptOrderInput::new( 199 command.input().request(), 200 command.input().terms_digest(), 201 AllocationSelection::automatic(), 202 ); 203 let client = r.client.clone(); 204 let scope = r.scope.clone(); 205 let observer = tokio::spawn(async move { 206 client 207 .orders(&scope) 208 .accept(context, minimum, private, public, input) 209 .await 210 }); 211 // Ordinary acceptance now retains and delivers the installed peer 212 // before reaching this self-signing barrier. Keep the same assertions 213 // while bounding observation of that larger debug-build prefix. 214 tokio::time::timeout( 215 std::time::Duration::from_secs(120), 216 r.control.entered.notified(), 217 ) 218 .await 219 .unwrap(); 220 assert_eq!(r.control.signs.load(Ordering::SeqCst), 2); 221 observer.abort(); 222 assert!(observer.await.unwrap_err().is_cancelled()); 223 let mut connection = fault(&r, uncertain).await; 224 r.control.release.notify_one(); 225 let error = r.client.close().await.unwrap_err(); 226 assert_eq!( 227 error.code(), 228 if uncertain { 229 crate::Code::RecoveryRequired 230 } else { 231 crate::Code::StorageFailure 232 } 233 ); 234 assert_eq!(error.context().admission(), crate::Admission::Confirmed); 235 assert_eq!( 236 error.context().operation_id(), 237 Some(command.context().operation_id()) 238 ); 239 assert!(r.provider.upgrade().is_some()); 240 remove_fault(&mut connection).await; 241 connection.close().await.unwrap(); 242 r.clock.0.store(u64::MAX, Ordering::SeqCst); 243 assert!(r.clock.now().is_err()); 244 r.client.close().await.unwrap(); 245 assert!(r.provider.upgrade().is_none()); 246 assert_eq!(r.control.signs.load(Ordering::SeqCst), 2); 247 assert_eq!(r.control.recoveries.load(Ordering::SeqCst), 0); 248 r.clock.0.store(150, Ordering::SeqCst); 249 let crypto = WorkspaceCrypto::new( 250 r.vault.clone(), 251 SecretRef::new( 252 SecretId::parse("workspace-key").unwrap(), 253 BackendKind::File, 254 KeyVersion::new(1).unwrap(), 255 ), 256 SecretMaterial::from_slice(&[0x41; 32]).unwrap(), 257 AccessPolicy::standard(), 258 ) 259 .unwrap(); 260 let reopened = NativeWorkspace::open_seller_stock( 261 &r._temp.path().join("stock"), 262 r.enrollment.input().clone(), 263 crypto, 264 r.clock.clone(), 265 ) 266 .await 267 .unwrap(); 268 let binding = reopened 269 .authenticate(r.enrollment.input().clone(), r.enrollment.proof()) 270 .await 271 .unwrap(); 272 let snapshot = reopened 273 .stock_operation_snapshot( 274 StockSnapshotRequest::new( 275 binding, 276 minimum, 277 command.context().operation_id(), 278 command.digest(), 279 ), 280 r.enrollment.proof(), 281 ) 282 .await 283 .unwrap(); 284 assert!(snapshot.installation().is_some()); 285 let facts = snapshot 286 .recipients() 287 .iter() 288 .filter_map(|v| v.original_unissued()) 289 .collect::<Vec<_>>(); 290 assert_eq!(facts.len(), 1); 291 assert_eq!(facts[0].reason(), ProviderError::Cancelled); 292 assert_eq!( 293 snapshot 294 .recipients() 295 .iter() 296 .filter(|v| v.first_evidence().is_some()) 297 .count(), 298 1 299 ); 300 assert!(snapshot.aborted().is_none()); 301 reopened.close().await.unwrap(); 302 } 303 }