status.rs (33973B)
1 //! Stable per-relay evidence, aggregate status, and outcome normalization. 2 3 use crate::{Config, ReconnectBackoff, RelayEndpoint, RelayProfileKind, RelayUrl}; 4 use radroots_transport::{ 5 SinkStatus, SourceStatus, 6 capability::{Availability, Maturity, SinkCapabilities, SourceCapabilities}, 7 outcome::{DeliveryOutcome, DeliveryOutcomeKind, FetchTargetState, Retryability}, 8 }; 9 use std::collections::BTreeMap; 10 use std::fmt; 11 use std::sync::Mutex; 12 13 #[derive(Clone, Copy, Debug, Eq, PartialEq)] 14 enum FailureClass { 15 Rejected, 16 AuthRequired, 17 Quota, 18 RateLimited, 19 Timeout, 20 Connection, 21 Malformed, 22 Unknown, 23 } 24 25 impl FailureClass { 26 const fn code(self) -> &'static str { 27 match self { 28 Self::Rejected => "rejected", 29 Self::AuthRequired => "auth_required", 30 Self::Quota => "quota_exceeded", 31 Self::RateLimited => "rate_limited", 32 Self::Timeout => "timeout", 33 Self::Connection => "connection_failed", 34 Self::Malformed => "malformed_event", 35 Self::Unknown => "relay_failure", 36 } 37 } 38 39 const fn message(self) -> &'static str { 40 match self { 41 Self::Rejected => "relay rejected the event", 42 Self::AuthRequired => "relay authentication is required", 43 Self::Quota => "relay quota was exhausted", 44 Self::RateLimited => "relay rate limit was reached", 45 Self::Timeout => "relay operation timed out", 46 Self::Connection => "relay connection failed", 47 Self::Malformed => "relay returned a malformed event", 48 Self::Unknown => "relay operation failed", 49 } 50 } 51 } 52 53 #[derive(Clone)] 54 struct RedactedDiagnostic { 55 class: FailureClass, 56 } 57 58 impl fmt::Debug for RedactedDiagnostic { 59 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 60 formatter 61 .debug_struct("RedactedDiagnostic") 62 .field("class", &self.class) 63 .field("upstream", &"[redacted]") 64 .finish() 65 } 66 } 67 68 /// Evidence state for one relay capability direction. 69 #[derive(Clone, Copy, Debug, Eq, PartialEq)] 70 #[non_exhaustive] 71 pub enum RelayEvidenceState { 72 /// The profile does not authorize this capability direction. 73 Unsupported, 74 /// The capability is configured but no successful or failed attempt exists. 75 Unobserved, 76 /// An authorized operation is currently awaiting relay evidence. 77 Connecting, 78 /// The latest accepted observation proved the capability usable. 79 Available, 80 /// The latest accepted observation failed to prove the capability usable. 81 Unavailable, 82 } 83 84 /// Immutable evidence for one relay capability direction. 85 #[derive(Clone, Debug, Eq, PartialEq)] 86 pub struct RelayCapabilityEvidence { 87 state: RelayEvidenceState, 88 last_attempt_unix_ms: Option<u64>, 89 last_success_unix_ms: Option<u64>, 90 consecutive_failures: u32, 91 next_attempt_unix_ms: Option<u64>, 92 last_failure_retryable: Option<bool>, 93 } 94 95 impl RelayCapabilityEvidence { 96 /// Returns whether this direction is unsupported, unobserved, or observed. 97 #[must_use] 98 pub const fn state(&self) -> RelayEvidenceState { 99 self.state 100 } 101 102 /// Returns the latest monotonic accepted attempt timestamp. 103 #[must_use] 104 pub const fn last_attempt_unix_ms(&self) -> Option<u64> { 105 self.last_attempt_unix_ms 106 } 107 108 /// Returns the latest successful observation timestamp. 109 #[must_use] 110 pub const fn last_success_unix_ms(&self) -> Option<u64> { 111 self.last_success_unix_ms 112 } 113 114 /// Returns consecutive failures since the latest success. 115 #[must_use] 116 pub const fn consecutive_failures(&self) -> u32 { 117 self.consecutive_failures 118 } 119 120 /// Returns the earliest adapter-permitted reconnect time after failure. 121 #[must_use] 122 pub const fn next_attempt_unix_ms(&self) -> Option<u64> { 123 self.next_attempt_unix_ms 124 } 125 126 /// Returns the retry class of the latest failure, when one exists. 127 #[must_use] 128 pub const fn last_failure_retryable(&self) -> Option<bool> { 129 self.last_failure_retryable 130 } 131 } 132 133 #[derive(Clone, Debug)] 134 struct MutableEvidence { 135 public: RelayCapabilityEvidence, 136 } 137 138 impl MutableEvidence { 139 const fn unobserved() -> Self { 140 Self { 141 public: RelayCapabilityEvidence { 142 state: RelayEvidenceState::Unobserved, 143 last_attempt_unix_ms: None, 144 last_success_unix_ms: None, 145 consecutive_failures: 0, 146 next_attempt_unix_ms: None, 147 last_failure_retryable: None, 148 }, 149 } 150 } 151 152 const fn unsupported() -> Self { 153 Self { 154 public: RelayCapabilityEvidence { 155 state: RelayEvidenceState::Unsupported, 156 last_attempt_unix_ms: None, 157 last_success_unix_ms: None, 158 consecutive_failures: 0, 159 next_attempt_unix_ms: None, 160 last_failure_retryable: None, 161 }, 162 } 163 } 164 165 fn begin(&mut self, observed_at_unix_ms: u64) { 166 if matches!(self.public.state, RelayEvidenceState::Unsupported) 167 || self 168 .public 169 .last_attempt_unix_ms 170 .is_some_and(|current| observed_at_unix_ms < current) 171 { 172 return; 173 } 174 self.public.state = RelayEvidenceState::Connecting; 175 self.public.last_attempt_unix_ms = Some(observed_at_unix_ms); 176 } 177 178 fn record( 179 &mut self, 180 succeeded: bool, 181 retryable: bool, 182 observed_at_unix_ms: u64, 183 backoff: ReconnectBackoff, 184 ) { 185 if matches!(self.public.state, RelayEvidenceState::Unsupported) 186 || self 187 .public 188 .last_attempt_unix_ms 189 .is_some_and(|current| observed_at_unix_ms < current) 190 { 191 return; 192 } 193 self.public.last_attempt_unix_ms = Some(observed_at_unix_ms); 194 if succeeded { 195 self.public.state = RelayEvidenceState::Available; 196 self.public.last_success_unix_ms = Some(observed_at_unix_ms); 197 self.public.consecutive_failures = 0; 198 self.public.next_attempt_unix_ms = None; 199 self.public.last_failure_retryable = None; 200 } else { 201 self.public.state = RelayEvidenceState::Unavailable; 202 self.public.consecutive_failures = self.public.consecutive_failures.saturating_add(1); 203 self.public.last_failure_retryable = Some(retryable); 204 self.public.next_attempt_unix_ms = retryable.then(|| { 205 observed_at_unix_ms 206 .saturating_add(backoff.delay_ms(self.public.consecutive_failures)) 207 }); 208 } 209 } 210 211 fn may_attempt(&self, now_unix_ms: u64) -> bool { 212 !matches!(self.public.state, RelayEvidenceState::Unsupported) 213 && self.public.last_failure_retryable != Some(false) 214 && self 215 .public 216 .next_attempt_unix_ms 217 .is_none_or(|retry_at| now_unix_ms >= retry_at) 218 } 219 } 220 221 #[derive(Clone, Debug)] 222 struct MutableRelayStatus { 223 endpoint: RelayEndpoint, 224 read: MutableEvidence, 225 write: MutableEvidence, 226 } 227 228 /// Passive typed status for one configured relay. 229 #[derive(Clone, Debug, Eq, PartialEq)] 230 pub struct RelayStatus { 231 endpoint: RelayEndpoint, 232 read: RelayCapabilityEvidence, 233 write: RelayCapabilityEvidence, 234 } 235 236 impl RelayStatus { 237 /// Returns the canonical endpoint and its declared authority. 238 #[must_use] 239 pub const fn endpoint(&self) -> &RelayEndpoint { 240 &self.endpoint 241 } 242 243 /// Returns independent read evidence. 244 #[must_use] 245 pub const fn read(&self) -> &RelayCapabilityEvidence { 246 &self.read 247 } 248 249 /// Returns independent write evidence. 250 #[must_use] 251 pub const fn write(&self) -> &RelayCapabilityEvidence { 252 &self.write 253 } 254 } 255 256 /// Passive per-relay and aggregate status for one configured profile. 257 #[derive(Clone, Debug, Eq, PartialEq)] 258 pub struct RelayStatusReport { 259 profile_kind: RelayProfileKind, 260 state: RelayAggregateState, 261 relays: Vec<RelayStatus>, 262 read_availability: Availability, 263 write_availability: Availability, 264 } 265 266 /// Aggregate lifecycle derived from current per-relay evidence. 267 #[derive(Clone, Copy, Debug, Eq, PartialEq)] 268 #[non_exhaustive] 269 pub enum RelayAggregateState { 270 /// A profile is installed but no relay operation has started. 271 Configured, 272 /// At least one authorized relay operation is in flight. 273 Connecting, 274 /// Current evidence proves reads but not publication. 275 ReadOnly, 276 /// Current evidence proves both reads and publication. 277 Writable, 278 /// Some, but not all, authorized relay capabilities have current success. 279 Degraded, 280 /// Every attempted capability is temporarily unavailable and retry-bounded. 281 Offline, 282 /// Every attempted capability failed terminally for the unchanged request. 283 Failed, 284 } 285 286 impl RelayStatusReport { 287 /// Returns the host profile whose policies govern this report. 288 #[must_use] 289 pub const fn profile_kind(&self) -> RelayProfileKind { 290 self.profile_kind 291 } 292 293 /// Returns the aggregate lifecycle derived only from current evidence. 294 #[must_use] 295 pub const fn state(&self) -> RelayAggregateState { 296 self.state 297 } 298 299 /// Returns one status per configured relay in profile order. 300 #[must_use] 301 pub fn relays(&self) -> &[RelayStatus] { 302 self.relays.as_slice() 303 } 304 305 /// Returns aggregate read availability derived only from observations. 306 #[must_use] 307 pub const fn read_availability(&self) -> Availability { 308 self.read_availability 309 } 310 311 /// Returns aggregate write availability derived only from writable relays. 312 #[must_use] 313 pub const fn write_availability(&self) -> Availability { 314 self.write_availability 315 } 316 } 317 318 #[derive(Clone, Debug)] 319 struct Snapshot { 320 order: Vec<RelayUrl>, 321 relays: BTreeMap<RelayUrl, MutableRelayStatus>, 322 } 323 324 impl Snapshot { 325 fn new(config: &Config) -> Self { 326 Self { 327 order: config.relays().to_vec(), 328 relays: config 329 .endpoints() 330 .iter() 331 .map(|endpoint| { 332 ( 333 endpoint.url().clone(), 334 MutableRelayStatus { 335 endpoint: endpoint.clone(), 336 read: MutableEvidence::unobserved(), 337 write: if endpoint.access().can_write() { 338 MutableEvidence::unobserved() 339 } else { 340 MutableEvidence::unsupported() 341 }, 342 }, 343 ) 344 }) 345 .collect(), 346 } 347 } 348 } 349 350 #[derive(Debug)] 351 pub(crate) struct StatusTracker { 352 initial: Snapshot, 353 snapshot: Mutex<Snapshot>, 354 profile_kind: RelayProfileKind, 355 backoff: ReconnectBackoff, 356 } 357 358 impl StatusTracker { 359 pub(crate) fn new(config: &Config) -> Self { 360 let initial = Snapshot::new(config); 361 Self { 362 snapshot: Mutex::new(initial.clone()), 363 initial, 364 profile_kind: config.profile_kind(), 365 backoff: config.reconnect_backoff(), 366 } 367 } 368 369 pub(crate) fn begin_read(&self, relay: &RelayUrl, observed_at_unix_ms: u64) { 370 if let Ok(mut snapshot) = self.snapshot.lock() 371 && let Some(status) = snapshot.relays.get_mut(relay) 372 { 373 status.read.begin(observed_at_unix_ms); 374 } 375 } 376 377 pub(crate) fn begin_write(&self, relay: &RelayUrl, observed_at_unix_ms: u64) { 378 if let Ok(mut snapshot) = self.snapshot.lock() 379 && let Some(status) = snapshot.relays.get_mut(relay) 380 { 381 status.write.begin(observed_at_unix_ms); 382 } 383 } 384 385 pub(crate) fn record_read( 386 &self, 387 relay: &RelayUrl, 388 succeeded: bool, 389 retryable: bool, 390 observed_at_unix_ms: u64, 391 ) { 392 if let Ok(mut snapshot) = self.snapshot.lock() 393 && let Some(status) = snapshot.relays.get_mut(relay) 394 { 395 status 396 .read 397 .record(succeeded, retryable, observed_at_unix_ms, self.backoff); 398 } 399 } 400 401 pub(crate) fn record_write( 402 &self, 403 relay: &RelayUrl, 404 succeeded: bool, 405 retryable: bool, 406 observed_at_unix_ms: u64, 407 ) { 408 if let Ok(mut snapshot) = self.snapshot.lock() 409 && let Some(status) = snapshot.relays.get_mut(relay) 410 { 411 status 412 .write 413 .record(succeeded, retryable, observed_at_unix_ms, self.backoff); 414 } 415 } 416 417 pub(crate) fn may_read(&self, relay: &RelayUrl, now_unix_ms: u64) -> bool { 418 self.snapshot 419 .lock() 420 .ok() 421 .and_then(|snapshot| { 422 snapshot 423 .relays 424 .get(relay) 425 .map(|status| status.read.may_attempt(now_unix_ms)) 426 }) 427 .unwrap_or(false) 428 } 429 430 pub(crate) fn may_write(&self, relay: &RelayUrl, now_unix_ms: u64) -> bool { 431 self.snapshot 432 .lock() 433 .ok() 434 .and_then(|snapshot| { 435 snapshot 436 .relays 437 .get(relay) 438 .map(|status| status.write.may_attempt(now_unix_ms)) 439 }) 440 .unwrap_or(false) 441 } 442 443 pub(crate) fn report(&self) -> RelayStatusReport { 444 let snapshot = self 445 .snapshot 446 .lock() 447 .map(|snapshot| snapshot.clone()) 448 .unwrap_or_else(|_| self.initial.clone()); 449 let relays = self 450 .initial 451 .order 452 .iter() 453 .filter_map(|relay| snapshot.relays.get(relay)) 454 .map(|status| RelayStatus { 455 endpoint: status.endpoint.clone(), 456 read: status.read.public.clone(), 457 write: status.write.public.clone(), 458 }) 459 .collect::<Vec<_>>(); 460 RelayStatusReport { 461 profile_kind: self.profile_kind, 462 state: aggregate_state(relays.as_slice()), 463 read_availability: aggregate(relays.iter().map(RelayStatus::read)), 464 write_availability: aggregate(relays.iter().map(RelayStatus::write)), 465 relays, 466 } 467 } 468 } 469 470 fn aggregate_state(relays: &[RelayStatus]) -> RelayAggregateState { 471 let evidence = relays 472 .iter() 473 .flat_map(|relay| [relay.read(), relay.write()]) 474 .filter(|evidence| !matches!(evidence.state(), RelayEvidenceState::Unsupported)) 475 .collect::<Vec<_>>(); 476 if evidence 477 .iter() 478 .any(|evidence| matches!(evidence.state(), RelayEvidenceState::Connecting)) 479 { 480 return RelayAggregateState::Connecting; 481 } 482 if evidence 483 .iter() 484 .all(|evidence| matches!(evidence.state(), RelayEvidenceState::Unobserved)) 485 { 486 return RelayAggregateState::Configured; 487 } 488 let read = relays.iter().map(RelayStatus::read).collect::<Vec<_>>(); 489 let write = relays 490 .iter() 491 .map(RelayStatus::write) 492 .filter(|evidence| !matches!(evidence.state(), RelayEvidenceState::Unsupported)) 493 .collect::<Vec<_>>(); 494 let read_available = read 495 .iter() 496 .filter(|evidence| matches!(evidence.state(), RelayEvidenceState::Available)) 497 .count(); 498 let write_available = write 499 .iter() 500 .filter(|evidence| matches!(evidence.state(), RelayEvidenceState::Available)) 501 .count(); 502 if read_available == read.len() && !write.is_empty() && write_available == write.len() { 503 RelayAggregateState::Writable 504 } else if read_available == read.len() && write_available == 0 { 505 RelayAggregateState::ReadOnly 506 } else if read_available + write_available > 0 { 507 RelayAggregateState::Degraded 508 } else if evidence 509 .iter() 510 .any(|evidence| evidence.last_failure_retryable() == Some(true)) 511 { 512 RelayAggregateState::Offline 513 } else if evidence 514 .iter() 515 .any(|evidence| evidence.last_failure_retryable() == Some(false)) 516 { 517 RelayAggregateState::Failed 518 } else { 519 RelayAggregateState::Configured 520 } 521 } 522 523 fn aggregate<'a>(evidence: impl Iterator<Item = &'a RelayCapabilityEvidence>) -> Availability { 524 let mut supported = 0usize; 525 let mut available = 0usize; 526 for evidence in evidence { 527 if !matches!(evidence.state, RelayEvidenceState::Unsupported) { 528 supported += 1; 529 if matches!(evidence.state, RelayEvidenceState::Available) { 530 available += 1; 531 } 532 } 533 } 534 match (supported, available) { 535 (0, _) | (_, 0) => Availability::Unavailable, 536 (supported, available) if supported == available => Availability::Available, 537 _ => Availability::Degraded, 538 } 539 } 540 541 pub(crate) fn source_status(tracker: &StatusTracker) -> SourceStatus { 542 let report = tracker.report(); 543 SourceStatus::new( 544 radroots_transport::TransportId::NOSTR, 545 !report.relays().is_empty(), 546 Maturity::Preview, 547 report.read_availability(), 548 SourceCapabilities::FETCH, 549 match report.read_availability() { 550 Availability::Available => "Nostr read capability has current successful evidence", 551 Availability::Degraded => "Nostr read capability has partial successful evidence", 552 Availability::Unavailable => "Nostr read capability has no current successful evidence", 553 }, 554 ) 555 } 556 557 pub(crate) fn sink_status(tracker: &StatusTracker) -> SinkStatus { 558 let report = tracker.report(); 559 let configured = report 560 .relays() 561 .iter() 562 .any(|status| status.endpoint().access().can_write()); 563 SinkStatus::new( 564 radroots_transport::TransportId::NOSTR, 565 configured, 566 Maturity::Preview, 567 report.write_availability(), 568 SinkCapabilities::DELIVER, 569 match report.write_availability() { 570 Availability::Available => "Nostr write capability has current successful evidence", 571 Availability::Degraded => "Nostr write capability has partial successful evidence", 572 Availability::Unavailable => { 573 "Nostr write capability has no current successful evidence" 574 } 575 }, 576 ) 577 } 578 579 pub(crate) fn delivery_failure(upstream: &str) -> DeliveryOutcome { 580 // Error text can classify a failure, but cannot establish relay acceptance. 581 let class = classify(upstream).class; 582 let outcome = match class { 583 FailureClass::Rejected | FailureClass::Malformed | FailureClass::Quota => { 584 DeliveryOutcome::rejected() 585 } 586 FailureClass::AuthRequired => DeliveryOutcome::failed(Retryability::Retryable) 587 .expect("retryable authentication outcome"), 588 FailureClass::RateLimited 589 | FailureClass::Timeout 590 | FailureClass::Connection 591 | FailureClass::Unknown => DeliveryOutcome::unavailable(), 592 }; 593 outcome 594 .with_detail(class.code(), class.message()) 595 .expect("static normalized relay outcome") 596 } 597 598 pub(crate) fn fetch_failure(upstream: &str) -> (FetchTargetState, &'static str) { 599 let class = classify(upstream).class; 600 let state = match class { 601 FailureClass::Rejected | FailureClass::Malformed | FailureClass::Quota => { 602 FetchTargetState::FailedTerminal 603 } 604 _ => FetchTargetState::FailedRetryable, 605 }; 606 (state, class.message()) 607 } 608 609 fn classify(upstream: &str) -> RedactedDiagnostic { 610 let message = upstream.to_ascii_lowercase(); 611 let class = if message.contains("auth") { 612 FailureClass::AuthRequired 613 } else if message.contains("quota") { 614 FailureClass::Quota 615 } else if message.contains("blocked") 616 || message.contains("restricted") 617 || message.contains("invalid") 618 || message.contains("reject") 619 { 620 FailureClass::Rejected 621 } else if message.contains("rate") { 622 FailureClass::RateLimited 623 } else if message.contains("timeout") || message.contains("timed out") { 624 FailureClass::Timeout 625 } else if message.contains("connect") || message.contains("offline") { 626 FailureClass::Connection 627 } else if message.contains("malformed") || message.contains("decode") { 628 FailureClass::Malformed 629 } else { 630 FailureClass::Unknown 631 }; 632 RedactedDiagnostic { class } 633 } 634 635 pub(crate) fn delivery_succeeded(outcome: &DeliveryOutcome) -> bool { 636 matches!( 637 outcome.kind(), 638 DeliveryOutcomeKind::Accepted | DeliveryOutcomeKind::Delivered 639 ) 640 } 641 642 #[cfg(test)] 643 mod tests { 644 use super::*; 645 use crate::ReconnectBackoff; 646 647 fn tracker() -> (StatusTracker, RelayUrl, RelayUrl) { 648 let config = Config::from_profile( 649 crate::profile::test_profile( 650 crate::RelayProfileKind::Public, 651 crate::RelayUrlPolicy::Public, 652 ["wss://read.example", "wss://write.example"], 653 ) 654 .expect("profile"), 655 ) 656 .with_reconnect_backoff(ReconnectBackoff::new(10, 40).expect("backoff")); 657 let readable = config.relays()[0].clone(); 658 let writable = config.relays()[1].clone(); 659 (StatusTracker::new(&config), readable, writable) 660 } 661 662 #[test] 663 fn every_upstream_class_maps_to_stable_secret_safe_output() { 664 let secret = "token=very-secret-value"; 665 let cases = [ 666 ("duplicate: already have", "relay_failure"), 667 ("blocked by policy", "rejected"), 668 ("auth required", "auth_required"), 669 ("quota exceeded", "quota_exceeded"), 670 ("blocked: account QUOTA exhausted", "quota_exceeded"), 671 ("rate limited", "rate_limited"), 672 ("connection timeout", "timeout"), 673 ("connection offline", "connection_failed"), 674 ("unknown failure", "relay_failure"), 675 ]; 676 for (message, code) in cases { 677 let outcome = delivery_failure(format!("{message} {secret}").as_str()); 678 assert_eq!(outcome.code(), Some(code)); 679 assert!(!outcome.message().expect("message").contains(secret)); 680 let diagnostic = classify(format!("{message} {secret}").as_str()); 681 assert!(!format!("{diagnostic:?}").contains(secret)); 682 } 683 } 684 685 #[test] 686 fn duplicate_text_cannot_turn_an_untyped_failure_into_acceptance() { 687 for message in [ 688 "duplicate: already have this event", 689 "already have this event", 690 "blocked: not a duplicate; event was rejected", 691 "auth-required: duplicate path requires login", 692 "connection failed: duplicate acknowledgement unavailable", 693 "error: duplicate detector unavailable", 694 ] { 695 let outcome = delivery_failure(message); 696 assert!(!delivery_succeeded(&outcome), "{message}"); 697 } 698 } 699 700 #[test] 701 fn quota_refusal_requires_action_without_reclassifying_other_failures() { 702 for text in ["quota exceeded", "blocked: account QUOTA exhausted"] { 703 let outcome = delivery_failure(text); 704 assert_eq!(outcome.code(), Some("quota_exceeded")); 705 assert_eq!(outcome.kind(), DeliveryOutcomeKind::Rejected); 706 assert_eq!(outcome.retryability(), Retryability::Terminal); 707 assert_eq!( 708 fetch_failure(text), 709 ( 710 FetchTargetState::FailedTerminal, 711 "relay quota was exhausted" 712 ) 713 ); 714 } 715 assert_eq!( 716 delivery_failure("rate limited").code(), 717 Some("rate_limited") 718 ); 719 assert!(delivery_failure("rate limited").is_retryable()); 720 assert_eq!( 721 delivery_failure("auth required").code(), 722 Some("auth_required") 723 ); 724 assert_eq!( 725 delivery_failure("malformed event").code(), 726 Some("malformed_event") 727 ); 728 assert_eq!( 729 delivery_failure("unknown failure").code(), 730 Some("relay_failure") 731 ); 732 assert!(delivery_failure("unknown failure").is_retryable()); 733 } 734 735 #[test] 736 fn status_requires_directional_evidence_and_backoff_is_monotonic() { 737 let (tracker, canonical, writable) = tracker(); 738 let initial = tracker.report(); 739 assert_eq!(initial.read_availability(), Availability::Unavailable); 740 assert_eq!(initial.write_availability(), Availability::Unavailable); 741 assert_eq!(initial.state(), RelayAggregateState::Configured); 742 assert_eq!( 743 initial.relays()[0].write().state(), 744 RelayEvidenceState::Unobserved 745 ); 746 assert!(tracker.may_write(&canonical, 100)); 747 748 tracker.begin_read(&canonical, 100); 749 assert_eq!(tracker.report().state(), RelayAggregateState::Connecting); 750 tracker.record_read(&canonical, true, false, 100); 751 tracker.record_read(&writable, false, true, 100); 752 tracker.record_write(&writable, false, true, 100); 753 let partial = tracker.report(); 754 assert_eq!(partial.state(), RelayAggregateState::Degraded); 755 assert_eq!(partial.read_availability(), Availability::Degraded); 756 assert_eq!(partial.write_availability(), Availability::Unavailable); 757 assert!(!tracker.may_write(&writable, 109)); 758 assert!(tracker.may_write(&writable, 110)); 759 760 tracker.record_write(&writable, false, true, 110); 761 assert_eq!( 762 tracker.report().relays()[1].write().next_attempt_unix_ms(), 763 Some(130) 764 ); 765 tracker.record_write(&writable, true, false, 109); 766 assert_eq!( 767 tracker.report().relays()[1].write().state(), 768 RelayEvidenceState::Unavailable 769 ); 770 tracker.record_write(&writable, true, false, 130); 771 let partially_available = tracker.report(); 772 assert_eq!( 773 partially_available.write_availability(), 774 Availability::Degraded 775 ); 776 assert_eq!( 777 partially_available.relays()[1] 778 .write() 779 .consecutive_failures(), 780 0 781 ); 782 assert_eq!( 783 partially_available.relays()[1] 784 .write() 785 .last_success_unix_ms(), 786 Some(130) 787 ); 788 assert!(tracker.may_write(&writable, 130)); 789 tracker.record_write(&canonical, true, false, 130); 790 assert_eq!( 791 tracker.report().write_availability(), 792 Availability::Available 793 ); 794 assert_eq!( 795 source_status(&tracker).availability(), 796 Availability::Degraded 797 ); 798 assert_eq!( 799 sink_status(&tracker).availability(), 800 Availability::Available 801 ); 802 803 tracker.record_write(&writable, false, false, 140); 804 tracker.record_write(&canonical, false, false, 140); 805 assert!(!tracker.may_write(&writable, u64::MAX)); 806 assert!(!tracker.may_write(&canonical, u64::MAX)); 807 assert_eq!( 808 tracker.report().relays()[1] 809 .write() 810 .last_failure_retryable(), 811 Some(false) 812 ); 813 assert_eq!( 814 sink_status(&tracker).availability(), 815 Availability::Unavailable 816 ); 817 } 818 819 #[test] 820 fn normalized_failures_and_success_helpers_cover_every_state() { 821 for message in [ 822 "invalid event", 823 "restricted", 824 "rejected", 825 "malformed event", 826 "decode failed", 827 ] { 828 assert_eq!(fetch_failure(message).0, FetchTargetState::FailedTerminal); 829 } 830 assert_eq!( 831 fetch_failure("offline").0, 832 FetchTargetState::FailedRetryable 833 ); 834 assert_eq!( 835 delivery_failure("malformed event").kind(), 836 DeliveryOutcomeKind::Rejected 837 ); 838 assert!(delivery_succeeded(&DeliveryOutcome::accepted())); 839 assert!(delivery_succeeded(&DeliveryOutcome::delivered())); 840 assert!(!delivery_succeeded(&DeliveryOutcome::rejected())); 841 } 842 843 #[test] 844 fn canonical_relay_read_and_write_success_is_available() { 845 let config = Config::from_profile( 846 crate::profile::test_profile( 847 crate::RelayProfileKind::Public, 848 crate::RelayUrlPolicy::Public, 849 ["wss://relay.example"], 850 ) 851 .expect("profile"), 852 ); 853 let tracker = StatusTracker::new(&config); 854 let relay = config.relays()[0].clone(); 855 tracker.record_read(&relay, true, false, 1); 856 tracker.record_write(&relay, true, false, 1); 857 assert_eq!( 858 source_status(&tracker).availability(), 859 Availability::Available 860 ); 861 let sink = sink_status(&tracker); 862 assert!(sink.is_configured()); 863 assert_eq!(sink.availability(), Availability::Available); 864 assert_eq!(tracker.report().state(), RelayAggregateState::Writable); 865 } 866 867 #[test] 868 fn aggregate_states_and_unconfigured_targets_cover_fail_closed_edges() { 869 let (live, canonical, writable) = tracker(); 870 let unknown = 871 RelayUrl::parse("wss://unknown.example", crate::RelayUrlPolicy::Public).expect("relay"); 872 873 live.begin_read(&unknown, 1); 874 live.begin_write(&unknown, 1); 875 live.record_read(&unknown, true, false, 1); 876 live.record_write(&unknown, true, false, 1); 877 live.begin_write(&canonical, 1); 878 live.record_write(&canonical, true, false, 1); 879 assert_eq!( 880 live.report().relays()[0].write().state(), 881 RelayEvidenceState::Available 882 ); 883 884 live.begin_read(&canonical, 2); 885 live.begin_read(&canonical, 1); 886 live.record_read(&canonical, true, false, 2); 887 live.record_read(&writable, true, false, 2); 888 live.record_write(&writable, true, false, 2); 889 let writable_report = live.report(); 890 assert_eq!(writable_report.state(), RelayAggregateState::Writable); 891 assert_eq!(writable_report.read_availability(), Availability::Available); 892 assert_eq!( 893 writable_report.write_availability(), 894 Availability::Available 895 ); 896 897 let (offline, canonical, writable) = tracker(); 898 offline.record_read(&canonical, false, true, 10); 899 offline.record_read(&writable, false, true, 10); 900 offline.record_write(&canonical, false, true, 10); 901 offline.record_write(&writable, false, true, 10); 902 assert_eq!(offline.report().state(), RelayAggregateState::Offline); 903 904 let (failed, canonical, writable) = tracker(); 905 failed.record_read(&canonical, false, false, 10); 906 failed.record_read(&writable, false, false, 10); 907 failed.record_write(&canonical, false, false, 10); 908 failed.record_write(&writable, false, false, 10); 909 assert_eq!(failed.report().state(), RelayAggregateState::Failed); 910 911 assert_eq!( 912 delivery_failure("already have").code(), 913 Some("relay_failure") 914 ); 915 assert_eq!( 916 fetch_failure("timed out").0, 917 FetchTargetState::FailedRetryable 918 ); 919 } 920 921 #[test] 922 fn capability_evidence_rejects_unsupported_and_stale_observations() { 923 let backoff = ReconnectBackoff::new(10, 40).expect("backoff"); 924 925 let mut unsupported = MutableEvidence::unsupported(); 926 unsupported.begin(10); 927 unsupported.record(true, false, 10, backoff); 928 assert_eq!(unsupported.public.state(), RelayEvidenceState::Unsupported); 929 assert!(!unsupported.may_attempt(u64::MAX)); 930 931 let mut evidence = MutableEvidence::unobserved(); 932 evidence.begin(10); 933 evidence.begin(9); 934 assert_eq!(evidence.public.last_attempt_unix_ms(), Some(10)); 935 936 evidence.record(false, true, 10, backoff); 937 evidence.record(true, false, 9, backoff); 938 assert_eq!(evidence.public.state(), RelayEvidenceState::Unavailable); 939 assert!(!evidence.may_attempt(19)); 940 assert!(evidence.may_attempt(20)); 941 942 let (read_only, canonical, writable) = tracker(); 943 read_only.record_read(&canonical, true, false, 1); 944 read_only.record_read(&writable, true, false, 1); 945 let report = read_only.report(); 946 assert_eq!(report.state(), RelayAggregateState::ReadOnly); 947 assert_eq!(report.read_availability(), Availability::Available); 948 assert_eq!(report.write_availability(), Availability::Unavailable); 949 950 let read_only_profile = crate::RelayProfile::explicit( 951 crate::RelayProfileKind::Public, 952 [crate::RelayEndpoint::new( 953 "wss://read-only.example", 954 crate::RelayUrlPolicy::Public, 955 crate::RelayAccess::ReadOnly, 956 ) 957 .expect("read-only endpoint")], 958 ) 959 .expect("read-only profile"); 960 let read_only_config = Config::from_profile(read_only_profile); 961 let read_only_relay = read_only_config.relays()[0].clone(); 962 let read_only_tracker = StatusTracker::new(&read_only_config); 963 read_only_tracker.record_read(&read_only_relay, true, false, 1); 964 let report = read_only_tracker.report(); 965 assert_eq!(report.state(), RelayAggregateState::ReadOnly); 966 assert_eq!(report.read_availability(), Availability::Available); 967 assert_eq!(report.write_availability(), Availability::Unavailable); 968 assert_eq!( 969 report.relays()[0].write().state(), 970 RelayEvidenceState::Unsupported 971 ); 972 } 973 }