native_stock_delivery_tests.rs (16568B)
1 //! Ordinary stock delivery and original-material recovery against real SQLx. 2 use super::*; 3 4 #[tokio::test] 5 async fn obtained_outer_is_retained_through_statement_and_commit_failure_and_clockless_drain() { 6 use sqlx::Connection; 7 for uncertain in [false, true] { 8 let r = Rig::new().await; 9 r.control.pause_peer.store(true, Ordering::SeqCst); 10 let (command, context) = r.captured().await; 11 let (private, public, minimum) = r.minima().await; 12 let input = AcceptOrderInput::new( 13 command.input().request(), 14 command.input().terms_digest(), 15 AllocationSelection::automatic(), 16 ); 17 let client = r.client.clone(); 18 let scope = r.scope.clone(); 19 let waiter = tokio::spawn(async move { 20 client 21 .orders(&scope) 22 .accept(context, minimum, private, public, input) 23 .await 24 }); 25 tokio::time::timeout( 26 std::time::Duration::from_secs(30), 27 r.control.entered.notified(), 28 ) 29 .await 30 .unwrap(); 31 let path = r._temp.path().join("stock/stock_execution_v1.sqlite"); 32 let mut fault = sqlx::SqliteConnection::connect(&format!("sqlite://{}", path.display())) 33 .await 34 .unwrap(); 35 if uncertain { 36 sqlx::raw_sql("CREATE TABLE outer_deferred_fault(value INTEGER REFERENCES stock_authority(singleton) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER controlled_outer_failure AFTER INSERT ON stock_envelopes BEGIN INSERT INTO outer_deferred_fault VALUES(2); END").execute(&mut fault).await.unwrap(); 37 } else { 38 sqlx::query("CREATE TRIGGER controlled_outer_failure BEFORE INSERT ON stock_envelopes BEGIN SELECT RAISE(ABORT, 'controlled_outer_failure'); END").execute(&mut fault).await.unwrap(); 39 } 40 r.control.release.notify_one(); 41 let error = waiter.await.unwrap().unwrap_err(); 42 assert_eq!( 43 error.code(), 44 if uncertain { 45 crate::Code::RecoveryRequired 46 } else { 47 crate::Code::StorageFailure 48 } 49 ); 50 assert_eq!(r.io.wraps.load(Ordering::SeqCst), 1); 51 assert!(r.io.sent.lock().unwrap().is_empty()); 52 r.clock.0.store(0, Ordering::SeqCst); 53 assert!(r.client.stock(&r.scope).drain_evidence().await.is_err()); 54 assert!(r.provider.upgrade().is_some()); 55 sqlx::query("DROP TRIGGER controlled_outer_failure") 56 .execute(&mut fault) 57 .await 58 .unwrap(); 59 if uncertain { 60 sqlx::query("DROP TABLE outer_deferred_fault") 61 .execute(&mut fault) 62 .await 63 .unwrap(); 64 } 65 fault.close().await.unwrap(); 66 r.clock.0.store(0, Ordering::SeqCst); 67 r.client.stock(&r.scope).drain_evidence().await.unwrap(); 68 assert_eq!(r.io.wraps.load(Ordering::SeqCst), 1); 69 r.clock.0.store(150, Ordering::SeqCst); 70 let resumed = r 71 .client 72 .orders(&r.scope) 73 .resume_acceptance(minimum, command.context().operation_id(), command.digest()) 74 .await 75 .unwrap(); 76 assert!( 77 resumed 78 .delivery() 79 .unwrap() 80 .iter() 81 .all(|v| v.acceptance_satisfied().unwrap()) 82 ); 83 assert_eq!(r.io.wraps.load(Ordering::SeqCst), 2); 84 assert_eq!(r.control.signs.load(Ordering::SeqCst), 2); 85 r.client.close().await.unwrap(); 86 } 87 } 88 89 #[tokio::test] 90 async fn recovery_delivers_original_stock_outers_through_actual_loopback_transport() { 91 use futures::{SinkExt, StreamExt}; 92 use radroots_transport_nostr::{ 93 Config, RelayAccess, RelayEndpoint, RelayProfile, RelayProfileKind, RelayUrlPolicy, 94 }; 95 use std::time::{Duration, SystemTime, UNIX_EPOCH}; 96 use tokio_tungstenite::{accept_async, tungstenite::Message}; 97 async fn relay(listener: tokio::net::TcpListener, originals: Vec<String>, misses: usize) { 98 let (tcp, _) = listener.accept().await.unwrap(); 99 let mut socket = accept_async(tcp).await.unwrap(); 100 for index in 0..(2 + misses) { 101 let raw = tokio::time::timeout(Duration::from_secs(300), async { 102 loop { 103 if let Message::Text(raw) = socket.next().await.unwrap().unwrap() { 104 break raw; 105 } 106 } 107 }) 108 .await 109 .unwrap(); 110 let original = &originals[index % 2]; 111 assert_eq!(raw.as_str(), format!("[\"EVENT\",{original}]")); 112 let event: serde_json::Value = serde_json::from_str(original).unwrap(); 113 socket 114 .send(Message::Text( 115 serde_json::json!(["OK", "ff".repeat(32), true, ""]) 116 .to_string() 117 .into(), 118 )) 119 .await 120 .unwrap(); 121 if index >= misses { 122 socket 123 .send(Message::Text( 124 serde_json::json!(["OK", event["id"], true, ""]) 125 .to_string() 126 .into(), 127 )) 128 .await 129 .unwrap(); 130 } 131 } 132 } 133 let first_relay = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); 134 let second_relay = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); 135 let uris = vec![ 136 format!("ws://{}", first_relay.local_addr().unwrap()), 137 format!("ws://{}", second_relay.local_addr().unwrap()), 138 ]; 139 let now = SystemTime::now() 140 .duration_since(UNIX_EPOCH) 141 .unwrap() 142 .as_secs(); 143 let r = Rig::configured(Some(uris.clone()), now + 1800).await; 144 r.io.missing_ack.store(true, Ordering::SeqCst); 145 let (command, first) = r.accept().await; 146 let originals = r.io.sent.lock().unwrap().clone(); 147 assert_eq!(originals.len(), 2); 148 let accepted = tokio::spawn(relay(first_relay, originals.clone(), 0)); 149 let partial = tokio::spawn(relay(second_relay, originals.clone(), 2)); 150 let config = Config::from_profile( 151 RelayProfile::explicit( 152 RelayProfileKind::Simulator, 153 uris.iter().map(|uri| { 154 RelayEndpoint::new(uri, RelayUrlPolicy::Local, RelayAccess::ReadWrite).unwrap() 155 }), 156 ) 157 .unwrap(), 158 ) 159 .with_timeouts(1000, 1000, 500) 160 .unwrap(); 161 let transport = Arc::new(radroots_transport_nostr_native::create_transport(config)); 162 *r.io.transport.lock().unwrap() = Some(transport.clone()); 163 // Real transport uses advancing UTC; only this injected test clock follows 164 // it. Retained source events, S and original outers remain unchanged. 165 r.clock.enable_realtime(); 166 let (_, _, minimum) = r.minima().await; 167 let grant = StockGrantEnrollment::prepare( 168 &SellerAuthoritySelection::from_attachment(r.enrollment.input()), 169 "loopback recovery".into(), 170 UnixSeconds::new(now + 1500).unwrap(), 171 StockGrantScope::full_inventory(), 172 ) 173 .unwrap(); 174 r.native 175 .enroll_stock_grant( 176 r.scope.binding().clone(), 177 minimum, 178 r.enrollment.proof(), 179 &grant, 180 ) 181 .await 182 .unwrap(); 183 let (_, _, minimum) = r.minima().await; 184 let capture = StockConsentCapture::prepare( 185 SellerAuthoritySelection::from_attachment(r.enrollment.input()), 186 grant.input().grant_id(), 187 StockConsentMode::RecoverOnly, 188 StockResources::all_pools(), 189 UnixSeconds::new(now + 1200).unwrap(), 190 CapturedOrderAcceptance::parse(command.protected_wire()).unwrap(), 191 ) 192 .unwrap(); 193 r.native 194 .issue_stock_consent( 195 r.scope.binding().clone(), 196 minimum, 197 &capture, 198 r.enrollment.proof(), 199 ) 200 .await 201 .unwrap(); 202 let (_, _, minimum) = r.minima().await; 203 let recovery = r 204 .client 205 .attach(grant.input().clone(), proof(grant.proof())) 206 .await 207 .unwrap(); 208 let incomplete = r 209 .client 210 .orders(&recovery) 211 .resume_acceptance(minimum, command.context().operation_id(), command.digest()) 212 .await 213 .unwrap(); 214 for state in incomplete.delivery().unwrap() { 215 assert_eq!(state.attempt_count(), 2); 216 assert!(!state.acceptance_satisfied().unwrap()); 217 assert_eq!( 218 state 219 .targets() 220 .iter() 221 .filter(|target| target.accepted()) 222 .count(), 223 1 224 ); 225 } 226 let resumed = r 227 .client 228 .orders(&recovery) 229 .resume_acceptance( 230 incomplete.checkpoint(), 231 command.context().operation_id(), 232 command.digest(), 233 ) 234 .await 235 .unwrap(); 236 assert_eq!(resumed.installation(), first.installation()); 237 for state in resumed.delivery().unwrap() { 238 assert_eq!(state.attempt_count(), 3); 239 assert!(state.acceptance_satisfied().unwrap()); 240 let mut counts: Vec<_> = state 241 .targets() 242 .iter() 243 .map(|v| v.potential_attempts()) 244 .collect(); 245 counts.sort(); 246 assert_eq!(counts, [2, 3]); 247 } 248 let sent = r.io.sent.lock().unwrap().clone(); 249 assert_eq!(&sent[2..4], originals.as_slice()); 250 assert_eq!(&sent[4..6], originals.as_slice()); 251 assert_eq!(r.control.signs.load(Ordering::SeqCst), 2); 252 tokio::time::timeout(Duration::from_secs(10), accepted) 253 .await 254 .unwrap() 255 .unwrap(); 256 tokio::time::timeout(Duration::from_secs(10), partial) 257 .await 258 .unwrap() 259 .unwrap(); 260 transport.shutdown().await; 261 r.client.close().await.unwrap(); 262 } 263 264 #[tokio::test] 265 async fn repeated_command_does_not_resend_and_signerless_resume_uses_exact_originals() { 266 let r = Rig::new().await; 267 r.io.missing_ack.store(true, Ordering::SeqCst); 268 let (command, first) = r.accept().await; 269 assert!( 270 first 271 .delivery() 272 .unwrap() 273 .iter() 274 .all(|v| v.attempt_count() == 1 && !v.acceptance_satisfied().unwrap()) 275 ); 276 let original = r.io.sent.lock().unwrap().clone(); 277 assert_eq!(original.len(), 2); 278 let (private, public, minimum) = r.minima().await; 279 let context = r 280 .scope 281 .command_context( 282 command.context().operation_id(), 283 command.context().expected().cloned(), 284 command.context().approval().cloned(), 285 command.context().not_after(), 286 ) 287 .unwrap(); 288 let repeated = r 289 .client 290 .orders(&r.scope) 291 .accept( 292 context, 293 minimum, 294 private, 295 public, 296 AcceptOrderInput::new( 297 command.input().request(), 298 command.input().terms_digest(), 299 AllocationSelection::automatic(), 300 ), 301 ) 302 .await 303 .unwrap(); 304 assert_eq!(repeated.installation(), first.installation()); 305 assert_eq!(r.io.sent.lock().unwrap().len(), 2); 306 307 let path = r._temp.path().join("recovery_cache"); 308 std::fs::create_dir(&path).unwrap(); 309 let options = OpenOptions::new(Paths::from_directory(&path).unwrap(), OpenMode::Create) 310 .with_source_generation(SourceGeneration::new([99; 32]).unwrap(), 1) 311 .unwrap(); 312 let mut recovery = 313 Client::open(NativeConfig::public_read(options).with_workspace(r.native.clone())) 314 .await 315 .unwrap(); 316 recovery.session = recovery 317 .session 318 .clone() 319 .with_command_runtime(Arc::new(CommandRuntime::new(r.io.clone()))); 320 let scope = recovery 321 .attach(r.enrollment.input().clone(), proof(r.enrollment.proof())) 322 .await 323 .unwrap(); 324 recovery.start(StartMode::LocalOnly).await.unwrap(); 325 assert!(recovery.signer_session().is_none()); 326 r.io.missing_ack.store(false, Ordering::SeqCst); 327 let resumed = recovery 328 .orders(&scope) 329 .resume_acceptance(minimum, command.context().operation_id(), command.digest()) 330 .await 331 .unwrap(); 332 assert_eq!(resumed.installation(), first.installation()); 333 assert!( 334 resumed 335 .delivery() 336 .unwrap() 337 .iter() 338 .all(|v| v.attempt_count() == 2 && v.acceptance_satisfied().unwrap()) 339 ); 340 assert_eq!(&r.io.sent.lock().unwrap()[2..], original.as_slice()); 341 assert_eq!(r.control.signs.load(Ordering::SeqCst), 2); 342 recovery.close().await.unwrap(); 343 r.client.close().await.unwrap(); 344 } 345 346 #[tokio::test] 347 async fn obtained_delivery_survives_statement_and_commit_failure_lost_waiter_and_clock_loss() { 348 use sqlx::Connection; 349 for uncertain in [false, true] { 350 let r = Rig::new().await; 351 r.io.pause_delivery.store(true, Ordering::SeqCst); 352 let (command, context) = r.captured().await; 353 let (private, public, minimum) = r.minima().await; 354 let input = AcceptOrderInput::new( 355 command.input().request(), 356 command.input().terms_digest(), 357 AllocationSelection::automatic(), 358 ); 359 let client = r.client.clone(); 360 let scope = r.scope.clone(); 361 let waiter = tokio::spawn(async move { 362 client 363 .orders(&scope) 364 .accept(context, minimum, private, public, input) 365 .await 366 }); 367 tokio::time::timeout(std::time::Duration::from_secs(30), r.io.entered.notified()) 368 .await 369 .unwrap(); 370 assert_eq!(r.io.sent.lock().unwrap().len(), 1); 371 assert_eq!(r.control.signs.load(Ordering::SeqCst), 1); 372 waiter.abort(); 373 assert!(waiter.await.unwrap_err().is_cancelled()); 374 let path = r._temp.path().join("stock/stock_execution_v1.sqlite"); 375 let mut fault = sqlx::SqliteConnection::connect(&format!("sqlite://{}", path.display())) 376 .await 377 .unwrap(); 378 let original: (Vec<u8>, Vec<u8>) = 379 sqlx::query_as("SELECT entry_id,request_key FROM stock_delivery_attempts") 380 .fetch_one(&mut fault) 381 .await 382 .unwrap(); 383 if uncertain { 384 sqlx::raw_sql("CREATE TABLE delivery_deferred_fault(value INTEGER REFERENCES stock_authority(singleton) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER controlled_d_failure AFTER INSERT ON stock_delivery_observations BEGIN INSERT INTO delivery_deferred_fault VALUES(2); END").execute(&mut fault).await.unwrap(); 385 } else { 386 sqlx::query("CREATE TRIGGER controlled_d_failure BEFORE INSERT ON stock_delivery_observations BEGIN SELECT RAISE(ABORT, 'controlled_d_failure'); END").execute(&mut fault).await.unwrap(); 387 } 388 r.clock.0.store(0, Ordering::SeqCst); 389 r.io.release.notify_one(); 390 // Close drains the admitted task and then its retained raw result, keeping 391 // the original credential and resource lease through the actual SQL fault. 392 let error = r.client.close().await.unwrap_err(); 393 assert_eq!( 394 error.code(), 395 if uncertain { 396 crate::Code::RecoveryRequired 397 } else { 398 crate::Code::StorageFailure 399 } 400 ); 401 assert!(r.provider.upgrade().is_some()); 402 sqlx::query("DROP TRIGGER controlled_d_failure") 403 .execute(&mut fault) 404 .await 405 .unwrap(); 406 if uncertain { 407 sqlx::query("DROP TABLE delivery_deferred_fault") 408 .execute(&mut fault) 409 .await 410 .unwrap(); 411 } 412 r.client.close().await.unwrap(); 413 assert!(r.provider.upgrade().is_none()); 414 let count: i64 = sqlx::query_scalar("SELECT count(*) FROM stock_delivery_observations") 415 .fetch_one(&mut fault) 416 .await 417 .unwrap(); 418 assert_eq!(count, 1); 419 let attempts: Vec<(Vec<u8>, Vec<u8>)> = 420 sqlx::query_as("SELECT entry_id,request_key FROM stock_delivery_attempts") 421 .fetch_all(&mut fault) 422 .await 423 .unwrap(); 424 assert_eq!(attempts, vec![original]); 425 assert_eq!(r.io.sent.lock().unwrap().len(), 1); 426 assert_eq!(r.control.signs.load(Ordering::SeqCst), 1); 427 fault.close().await.unwrap(); 428 } 429 }