provider_executor.rs (25746B)
1 //! Sealed execution over the two governed signer-provider implementations. 2 3 #![allow( 4 dead_code, 5 reason = "Step 159 Unit 12 seals the executor before Unit 15 runtime graph wiring" 6 )] 7 8 use core::fmt; 9 use std::{error::Error, sync::Arc}; 10 11 use nostr::{ 12 JsonUtil as _, Keys, PublicKey, SecretKey, UnsignedEvent, 13 nips::{nip04, nip44}, 14 }; 15 use sha2::{Digest as _, Sha256}; 16 17 use crate::provider_local_signer::{ProtectedWireHex, WireCapability, WireProviderResult}; 18 use crate::provider_verification::verify_encrypted_provider_response; 19 use crate::{ 20 MYC_LOCAL_SIGNER_TRANSPORT_CONTRACT_VERSION, MYC_PROVIDER_INPUT_MAX_BYTES, MycConfigDocumentV1, 21 MycDecryptedIdentity, MycLocalSignerClient, MycProviderBinding, MycProviderCapability, 22 MycProviderCorrelationId, MycProviderDeadlineUnixMs, MycProviderKind, MycProviderOperation, 23 MycProviderOperationId, MycProviderOperationInput, MycProviderResponseObservedAtUnixMs, 24 MycProviderRole, MycRuntimeContext, MycTaskCancellation, MycVerifiedProviderResponse, 25 }; 26 #[cfg(any(target_os = "linux", target_os = "macos"))] 27 use crate::{open_myc_encrypted_identity, resolve_myc_wrapping_credential}; 28 29 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 30 pub(crate) enum MycProviderExecutionErrorKind { 31 Binding, 32 Open, 33 Operation, 34 Cancelled, 35 Transport, 36 Verification, 37 UnsupportedPlatform, 38 } 39 40 pub(crate) struct MycProviderExecutionError { 41 kind: MycProviderExecutionErrorKind, 42 } 43 44 impl MycProviderExecutionError { 45 pub(crate) const fn kind(&self) -> MycProviderExecutionErrorKind { 46 self.kind 47 } 48 } 49 50 impl fmt::Debug for MycProviderExecutionError { 51 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 52 formatter 53 .debug_struct("MycProviderExecutionError") 54 .field("kind", &self.kind) 55 .finish() 56 } 57 } 58 59 impl fmt::Display for MycProviderExecutionError { 60 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 61 formatter.write_str("Myc provider execution failed") 62 } 63 } 64 65 impl Error for MycProviderExecutionError {} 66 67 const fn execution_error(kind: MycProviderExecutionErrorKind) -> MycProviderExecutionError { 68 MycProviderExecutionError { kind } 69 } 70 71 struct OwnedBlockingTask<T> { 72 handle: Option<tokio::task::JoinHandle<T>>, 73 } 74 75 impl<T: Send + 'static> OwnedBlockingTask<T> { 76 fn spawn(task: impl FnOnce() -> T + Send + 'static) -> Self { 77 Self { 78 handle: Some(tokio::task::spawn_blocking(task)), 79 } 80 } 81 82 async fn join( 83 &mut self, 84 failure: MycProviderExecutionErrorKind, 85 ) -> Result<T, MycProviderExecutionError> { 86 let result = match self.handle.as_mut() { 87 Some(handle) => handle.await, 88 None => return Err(execution_error(failure)), 89 }; 90 self.handle.take(); 91 result.map_err(|_| execution_error(failure)) 92 } 93 } 94 95 impl<T> Drop for OwnedBlockingTask<T> { 96 fn drop(&mut self) { 97 let Some(handle) = self.handle.take() else { 98 return; 99 }; 100 handle.abort(); 101 while !handle.is_finished() { 102 std::thread::park_timeout(core::time::Duration::from_millis(1)); 103 } 104 } 105 } 106 107 enum ExecutableProvider { 108 EncryptedFile { 109 binding: MycProviderBinding, 110 identity: Arc<MycDecryptedIdentity>, 111 }, 112 LocalSigner { 113 binding: MycProviderBinding, 114 client: Box<MycLocalSignerClient>, 115 }, 116 } 117 118 impl ExecutableProvider { 119 const fn role(&self) -> MycProviderRole { 120 match self { 121 Self::EncryptedFile { binding, .. } | Self::LocalSigner { binding, .. } => { 122 binding.role() 123 } 124 } 125 } 126 } 127 128 pub(crate) struct MycProviderExecutor { 129 providers: Box<[ExecutableProvider]>, 130 } 131 132 impl MycProviderExecutor { 133 pub(crate) async fn open( 134 runtime: &MycRuntimeContext, 135 configuration: &MycConfigDocumentV1, 136 cancellation: &MycTaskCancellation, 137 ) -> Result<Self, MycProviderExecutionError> { 138 #[cfg(not(any(target_os = "linux", target_os = "macos")))] 139 { 140 let _ = (runtime, configuration, cancellation); 141 return Err(execution_error( 142 MycProviderExecutionErrorKind::UnsupportedPlatform, 143 )); 144 } 145 146 #[cfg(any(target_os = "linux", target_os = "macos"))] 147 { 148 let mut providers = 149 Vec::with_capacity(configuration.provider_contract().bindings().len()); 150 for binding in configuration.provider_contract().bindings() { 151 if cancellation.is_cancelled() { 152 return Err(execution_error(MycProviderExecutionErrorKind::Cancelled)); 153 } 154 match binding.kind() { 155 MycProviderKind::EncryptedFile => { 156 let runtime = runtime.clone(); 157 let binding = binding.clone(); 158 let worker_binding = binding.clone(); 159 let mut worker = OwnedBlockingTask::spawn(move || { 160 let credential = 161 resolve_myc_wrapping_credential(&runtime, &worker_binding) 162 .map_err(|_| ())?; 163 open_myc_encrypted_identity(&worker_binding, &credential) 164 .map_err(|_| ()) 165 }); 166 let identity = tokio::select! { 167 result = worker.join(MycProviderExecutionErrorKind::Open) => result? 168 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Open))?, 169 () = cancellation.cancelled() => { 170 let _ = worker.join(MycProviderExecutionErrorKind::Open).await; 171 return Err(execution_error(MycProviderExecutionErrorKind::Cancelled)); 172 } 173 }; 174 providers.push(ExecutableProvider::EncryptedFile { 175 binding, 176 identity: Arc::new(identity), 177 }); 178 } 179 MycProviderKind::LocalSigner => { 180 let client = MycLocalSignerClient::new(binding) 181 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Open))?; 182 providers.push(ExecutableProvider::LocalSigner { 183 binding: binding.clone(), 184 client: Box::new(client), 185 }); 186 } 187 } 188 } 189 if providers.len() != configuration.provider_contract().bindings().len() { 190 return Err(execution_error(MycProviderExecutionErrorKind::Binding)); 191 } 192 Ok(Self { 193 providers: providers.into_boxed_slice(), 194 }) 195 } 196 } 197 198 pub(crate) async fn execute( 199 &self, 200 operation: MycProviderOperation, 201 observed_at: MycProviderResponseObservedAtUnixMs, 202 cancellation: &MycTaskCancellation, 203 ) -> Result<MycVerifiedProviderResponse, MycProviderExecutionError> { 204 let provider = self 205 .providers 206 .iter() 207 .find(|provider| provider.role() == operation.role()) 208 .ok_or_else(|| execution_error(MycProviderExecutionErrorKind::Binding))?; 209 if cancellation.is_cancelled() { 210 return Err(execution_error(MycProviderExecutionErrorKind::Cancelled)); 211 } 212 match provider { 213 ExecutableProvider::EncryptedFile { binding, identity } => { 214 let binding = binding.clone(); 215 let identity = Arc::clone(identity); 216 let mut worker = OwnedBlockingTask::spawn(move || { 217 let result = execute_encrypted(&identity, &operation)?; 218 Ok::<_, MycProviderExecutionError>((operation, result)) 219 }); 220 tokio::select! { 221 joined = worker.join(MycProviderExecutionErrorKind::Operation) => { 222 let (operation, result) = joined??; 223 verify_encrypted_provider_response(&binding, &operation, observed_at, result) 224 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Verification)) 225 } 226 () = cancellation.cancelled() => { 227 // A blocking cryptographic call cannot be abandoned. Join it before 228 // returning cancellation so no protected operation is detached. 229 // The result is deliberately discarded and never becomes domain authority. 230 let _ = worker.join(MycProviderExecutionErrorKind::Operation).await; 231 Err(execution_error(MycProviderExecutionErrorKind::Cancelled)) 232 } 233 } 234 } 235 ExecutableProvider::LocalSigner { binding, client } => { 236 tokio::select! { 237 result = client.execute(&operation) => { 238 let response = result.map_err(|_| execution_error(MycProviderExecutionErrorKind::Transport))?; 239 response 240 .verify(binding, &operation, observed_at) 241 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Verification)) 242 } 243 () = cancellation.cancelled() => { 244 Err(execution_error(MycProviderExecutionErrorKind::Cancelled)) 245 } 246 } 247 } 248 } 249 } 250 251 pub(crate) fn contains_role(&self, role: MycProviderRole) -> bool { 252 self.providers 253 .iter() 254 .any(|provider| provider.role() == role) 255 } 256 257 pub(crate) async fn probe_all( 258 &self, 259 observed_at_unix_ms: u64, 260 seed: [u8; 32], 261 cancellation: &MycTaskCancellation, 262 ) -> Result<(), MycProviderExecutionError> { 263 let observed_at = MycProviderResponseObservedAtUnixMs::new(observed_at_unix_ms) 264 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Binding))?; 265 for (index, provider) in self.providers.iter().enumerate() { 266 let role = provider.role(); 267 let binding = match provider { 268 ExecutableProvider::EncryptedFile { binding, .. } 269 | ExecutableProvider::LocalSigner { binding, .. } => binding, 270 }; 271 let timeout = binding 272 .local_signer_limits() 273 .map_or(15_000, |limits| limits.request_deadline_ms()); 274 let deadline = observed_at_unix_ms 275 .checked_add(timeout) 276 .and_then(|value| MycProviderDeadlineUnixMs::new(value).ok()) 277 .ok_or_else(|| execution_error(MycProviderExecutionErrorKind::Binding))?; 278 let index = u32::try_from(index) 279 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Binding))?; 280 let operation = MycProviderOperation::new( 281 binding, 282 MycProviderOperationId::from_bytes(probe_identifier( 283 b"operation", 284 &seed, 285 index, 286 role, 287 )), 288 MycProviderCorrelationId::from_bytes(probe_identifier( 289 b"correlation", 290 &seed, 291 index, 292 role, 293 )), 294 deadline, 295 MycProviderOperationInput::describe(), 296 ) 297 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Binding))?; 298 let response = self.execute(operation, observed_at, cancellation).await?; 299 if response.role() != role || response.capability() != MycProviderCapability::Describe { 300 return Err(execution_error(MycProviderExecutionErrorKind::Verification)); 301 } 302 } 303 Ok(()) 304 } 305 } 306 307 fn probe_identifier(kind: &[u8], seed: &[u8; 32], index: u32, role: MycProviderRole) -> [u8; 32] { 308 let mut hasher = Sha256::new(); 309 hasher.update(b"radroots.myc.provider_probe.v1\0"); 310 hasher.update(u64::try_from(kind.len()).unwrap_or(u64::MAX).to_be_bytes()); 311 hasher.update(kind); 312 hasher.update(seed); 313 hasher.update(index.to_be_bytes()); 314 hasher.update(role.as_str().as_bytes()); 315 hasher.finalize().into() 316 } 317 318 impl fmt::Debug for MycProviderExecutor { 319 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 320 formatter 321 .debug_struct("MycProviderExecutor") 322 .field("provider_count", &self.providers.len()) 323 .finish() 324 } 325 } 326 327 fn execute_encrypted( 328 identity: &MycDecryptedIdentity, 329 operation: &MycProviderOperation, 330 ) -> Result<WireProviderResult, MycProviderExecutionError> { 331 if operation.provider() != MycProviderKind::EncryptedFile 332 || operation.expected_identity() != identity.public_identity() 333 { 334 return Err(execution_error(MycProviderExecutionErrorKind::Binding)); 335 } 336 let secret = SecretKey::from_slice(identity.secret_bytes()) 337 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Operation))?; 338 let keys = Keys::new(secret); 339 let input = operation.input(); 340 let peer = || { 341 input 342 .peer() 343 .ok_or_else(|| execution_error(MycProviderExecutionErrorKind::Binding)) 344 .and_then(|identity| { 345 PublicKey::from_hex(identity.as_hex()) 346 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Binding)) 347 }) 348 }; 349 let bytes = || { 350 input 351 .bytes() 352 .ok_or_else(|| execution_error(MycProviderExecutionErrorKind::Binding)) 353 }; 354 match input.capability() { 355 MycProviderCapability::Describe => Ok(WireProviderResult::Describe { 356 public_identity: identity.public_identity().as_hex().to_owned(), 357 protocol_version: MYC_LOCAL_SIGNER_TRANSPORT_CONTRACT_VERSION, 358 capabilities: MycProviderCapability::ALL 359 .into_iter() 360 .filter(|capability| capability_allowed_for_role(operation.role(), *capability)) 361 .map(WireCapability::from) 362 .collect(), 363 maximum_request_bytes: MYC_PROVIDER_INPUT_MAX_BYTES as u64, 364 }), 365 MycProviderCapability::PublicIdentity => Ok(WireProviderResult::PublicIdentity { 366 public_identity: identity.public_identity().as_hex().to_owned(), 367 }), 368 MycProviderCapability::SignEvent => { 369 let unsigned: UnsignedEvent = serde_json::from_slice(bytes()?) 370 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Operation))?; 371 let signed = unsigned 372 .sign_with_keys(&keys) 373 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Operation))?; 374 Ok(WireProviderResult::SignEvent { 375 payload_hex: ProtectedWireHex::from_bytes(signed.as_json().as_bytes()), 376 }) 377 } 378 MycProviderCapability::Nip04Encrypt => { 379 let peer = peer()?; 380 let payload = nip04::encrypt(keys.secret_key(), &peer, bytes()?) 381 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Operation))?; 382 Ok(WireProviderResult::Nip04Encrypt { 383 peer: peer.to_hex(), 384 payload_hex: ProtectedWireHex::from_bytes(payload.as_bytes()), 385 }) 386 } 387 MycProviderCapability::Nip04Decrypt => { 388 let peer = peer()?; 389 let payload = core::str::from_utf8(bytes()?) 390 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Operation))?; 391 let plaintext = nip04::decrypt(keys.secret_key(), &peer, payload) 392 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Operation))?; 393 Ok(WireProviderResult::Nip04Decrypt { 394 peer: peer.to_hex(), 395 payload_hex: ProtectedWireHex::from_bytes(plaintext.as_bytes()), 396 }) 397 } 398 MycProviderCapability::Nip44Encrypt => { 399 let peer = peer()?; 400 let payload = nip44::encrypt(keys.secret_key(), &peer, bytes()?, nip44::Version::V2) 401 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Operation))?; 402 Ok(WireProviderResult::Nip44Encrypt { 403 peer: peer.to_hex(), 404 version: 2, 405 payload_hex: ProtectedWireHex::from_bytes(payload.as_bytes()), 406 }) 407 } 408 MycProviderCapability::Nip44Decrypt => { 409 let peer = peer()?; 410 let payload = core::str::from_utf8(bytes()?) 411 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Operation))?; 412 let plaintext = nip44::decrypt(keys.secret_key(), &peer, payload) 413 .map_err(|_| execution_error(MycProviderExecutionErrorKind::Operation))?; 414 Ok(WireProviderResult::Nip44Decrypt { 415 peer: peer.to_hex(), 416 version: 2, 417 payload_hex: ProtectedWireHex::from_bytes(plaintext.as_bytes()), 418 }) 419 } 420 } 421 } 422 423 const fn capability_allowed_for_role( 424 role: MycProviderRole, 425 capability: MycProviderCapability, 426 ) -> bool { 427 match role { 428 MycProviderRole::Transport | MycProviderRole::User => true, 429 MycProviderRole::Discovery => matches!( 430 capability, 431 MycProviderCapability::Describe 432 | MycProviderCapability::PublicIdentity 433 | MycProviderCapability::SignEvent 434 ), 435 } 436 } 437 438 #[cfg(test)] 439 mod tests { 440 use nostr::{Kind, Tag, Timestamp}; 441 442 use super::*; 443 use crate::{ 444 MycConfigProfile, MycProviderCorrelationId, MycProviderDeadlineUnixMs, 445 MycProviderNip44Version, MycProviderOperationId, MycProviderOperationInput, 446 parse_myc_config_v1, 447 }; 448 449 const CONFIG: &str = include_str!("../contracts/services_hardening/config.v1.example.toml"); 450 451 fn keys(seed: u8) -> Keys { 452 Keys::parse(&format!("{seed:02x}{}", "00".repeat(31))).expect("test keys") 453 } 454 455 fn secret(seed: u8) -> [u8; 32] { 456 let mut secret = [0_u8; 32]; 457 secret[0] = seed; 458 secret 459 } 460 461 fn configuration() -> MycConfigDocumentV1 { 462 let source = CONFIG 463 .replacen( 464 "4444444444444444444444444444444444444444444444444444444444444444", 465 &keys(2).public_key().to_hex(), 466 1, 467 ) 468 .replacen( 469 "3333333333333333333333333333333333333333333333333333333333333333", 470 &keys(4).public_key().to_hex(), 471 1, 472 ); 473 parse_myc_config_v1(source.as_bytes(), MycConfigProfile::RepoLocal).expect("configuration") 474 } 475 476 fn operation( 477 binding: &MycProviderBinding, 478 seed: u8, 479 input: MycProviderOperationInput, 480 ) -> MycProviderOperation { 481 MycProviderOperation::new( 482 binding, 483 MycProviderOperationId::from_bytes([seed; 32]), 484 MycProviderCorrelationId::from_bytes([seed.wrapping_add(1); 32]), 485 MycProviderDeadlineUnixMs::new(2_000_000_000_000).expect("deadline"), 486 input, 487 ) 488 .expect("operation") 489 } 490 491 #[test] 492 fn encrypted_executor_produces_independently_verified_results() { 493 let configuration = configuration(); 494 let binding = configuration 495 .provider_contract() 496 .binding(MycProviderRole::Transport) 497 .expect("transport binding"); 498 let identity = MycDecryptedIdentity::from_test_secret(secret(2)); 499 let peer = MycDecryptedIdentity::from_test_secret(secret(5)); 500 let observed = 501 MycProviderResponseObservedAtUnixMs::new(1_999_999_999_999).expect("observed time"); 502 503 for (seed, input) in [ 504 (1, MycProviderOperationInput::describe()), 505 (2, MycProviderOperationInput::public_identity()), 506 ( 507 3, 508 MycProviderOperationInput::nip04_encrypt( 509 peer.public_identity().clone(), 510 b"nip04 protected", 511 ) 512 .expect("NIP-04 input"), 513 ), 514 ( 515 4, 516 MycProviderOperationInput::nip44_encrypt( 517 peer.public_identity().clone(), 518 MycProviderNip44Version::V2, 519 b"nip44 protected", 520 ) 521 .expect("NIP-44 input"), 522 ), 523 ] { 524 let operation = operation(binding, seed, input); 525 let result = execute_encrypted(&identity, &operation).expect("encrypted result"); 526 let verified = 527 verify_encrypted_provider_response(binding, &operation, observed, result) 528 .expect("independent verification"); 529 assert!(verified.matches_operation(&operation)); 530 } 531 532 let discovery = configuration 533 .provider_contract() 534 .binding(MycProviderRole::Discovery) 535 .expect("discovery binding"); 536 let discovery_identity = MycDecryptedIdentity::from_test_secret(secret(4)); 537 let unsigned = nostr::UnsignedEvent::new( 538 discovery_identity 539 .public_identity() 540 .as_hex() 541 .parse() 542 .expect("public key"), 543 Timestamp::from_secs(1_725_000_000), 544 Kind::Custom(31_990), 545 Vec::<Tag>::new(), 546 "{}", 547 ); 548 let operation = operation( 549 discovery, 550 5, 551 MycProviderOperationInput::sign_event(unsigned.as_json().as_bytes()) 552 .expect("sign input"), 553 ); 554 let result = execute_encrypted(&discovery_identity, &operation).expect("signature"); 555 let verified = verify_encrypted_provider_response(discovery, &operation, observed, result) 556 .expect("verified signature"); 557 assert!(verified.matches_operation(&operation)); 558 } 559 560 #[test] 561 fn capability_matrix_and_diagnostics_are_closed() { 562 for capability in MycProviderCapability::ALL { 563 assert!(capability_allowed_for_role( 564 MycProviderRole::Transport, 565 capability 566 )); 567 assert!(capability_allowed_for_role( 568 MycProviderRole::User, 569 capability 570 )); 571 assert_eq!( 572 capability_allowed_for_role(MycProviderRole::Discovery, capability), 573 matches!( 574 capability, 575 MycProviderCapability::Describe 576 | MycProviderCapability::PublicIdentity 577 | MycProviderCapability::SignEvent 578 ) 579 ); 580 } 581 for kind in [ 582 MycProviderExecutionErrorKind::Binding, 583 MycProviderExecutionErrorKind::Open, 584 MycProviderExecutionErrorKind::Operation, 585 MycProviderExecutionErrorKind::Cancelled, 586 MycProviderExecutionErrorKind::Transport, 587 MycProviderExecutionErrorKind::Verification, 588 MycProviderExecutionErrorKind::UnsupportedPlatform, 589 ] { 590 let error = execution_error(kind); 591 assert_eq!(error.kind(), kind); 592 assert!(error.source().is_none()); 593 assert!(!format!("{error} {error:?}").contains("protected")); 594 } 595 } 596 597 #[test] 598 fn provider_probe_identifiers_are_domain_seed_index_and_role_bound() { 599 let seed = [7_u8; 32]; 600 let operation = probe_identifier(b"operation", &seed, 0, MycProviderRole::Transport); 601 assert_eq!( 602 operation, 603 probe_identifier(b"operation", &seed, 0, MycProviderRole::Transport) 604 ); 605 assert_ne!( 606 operation, 607 probe_identifier(b"correlation", &seed, 0, MycProviderRole::Transport) 608 ); 609 assert_ne!( 610 operation, 611 probe_identifier(b"operation", &[8_u8; 32], 0, MycProviderRole::Transport) 612 ); 613 assert_ne!( 614 operation, 615 probe_identifier(b"operation", &seed, 1, MycProviderRole::Transport) 616 ); 617 assert_ne!( 618 operation, 619 probe_identifier(b"operation", &seed, 0, MycProviderRole::User) 620 ); 621 } 622 623 #[cfg(any(target_os = "linux", target_os = "macos"))] 624 #[tokio::test(flavor = "multi_thread", worker_threads = 2)] 625 async fn dropping_blocking_provider_work_drains_it_before_returning() { 626 use std::sync::{ 627 Arc, 628 atomic::{AtomicBool, Ordering}, 629 mpsc, 630 }; 631 632 let (entered_tx, entered_rx) = mpsc::channel(); 633 let (release_tx, release_rx) = mpsc::channel(); 634 let completed = Arc::new(AtomicBool::new(false)); 635 let worker_completed = Arc::clone(&completed); 636 let task = OwnedBlockingTask::spawn(move || { 637 entered_tx.send(()).expect("entered signal"); 638 release_rx.recv().expect("release signal"); 639 worker_completed.store(true, Ordering::SeqCst); 640 }); 641 entered_rx.recv().expect("worker entered"); 642 let releaser = std::thread::spawn(move || { 643 std::thread::sleep(core::time::Duration::from_millis(25)); 644 release_tx.send(()).expect("release worker"); 645 }); 646 647 drop(task); 648 649 releaser.join().expect("releaser joined"); 650 assert!(completed.load(Ordering::SeqCst)); 651 } 652 }