lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

status.rs (33973B)


      1 //! Stable per-relay evidence, aggregate status, and outcome normalization.
      2 
      3 use crate::{Config, ReconnectBackoff, RelayEndpoint, RelayProfileKind, RelayUrl};
      4 use radroots_transport::{
      5     SinkStatus, SourceStatus,
      6     capability::{Availability, Maturity, SinkCapabilities, SourceCapabilities},
      7     outcome::{DeliveryOutcome, DeliveryOutcomeKind, FetchTargetState, Retryability},
      8 };
      9 use std::collections::BTreeMap;
     10 use std::fmt;
     11 use std::sync::Mutex;
     12 
     13 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
     14 enum FailureClass {
     15     Rejected,
     16     AuthRequired,
     17     Quota,
     18     RateLimited,
     19     Timeout,
     20     Connection,
     21     Malformed,
     22     Unknown,
     23 }
     24 
     25 impl FailureClass {
     26     const fn code(self) -> &'static str {
     27         match self {
     28             Self::Rejected => "rejected",
     29             Self::AuthRequired => "auth_required",
     30             Self::Quota => "quota_exceeded",
     31             Self::RateLimited => "rate_limited",
     32             Self::Timeout => "timeout",
     33             Self::Connection => "connection_failed",
     34             Self::Malformed => "malformed_event",
     35             Self::Unknown => "relay_failure",
     36         }
     37     }
     38 
     39     const fn message(self) -> &'static str {
     40         match self {
     41             Self::Rejected => "relay rejected the event",
     42             Self::AuthRequired => "relay authentication is required",
     43             Self::Quota => "relay quota was exhausted",
     44             Self::RateLimited => "relay rate limit was reached",
     45             Self::Timeout => "relay operation timed out",
     46             Self::Connection => "relay connection failed",
     47             Self::Malformed => "relay returned a malformed event",
     48             Self::Unknown => "relay operation failed",
     49         }
     50     }
     51 }
     52 
     53 #[derive(Clone)]
     54 struct RedactedDiagnostic {
     55     class: FailureClass,
     56 }
     57 
     58 impl fmt::Debug for RedactedDiagnostic {
     59     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
     60         formatter
     61             .debug_struct("RedactedDiagnostic")
     62             .field("class", &self.class)
     63             .field("upstream", &"[redacted]")
     64             .finish()
     65     }
     66 }
     67 
     68 /// Evidence state for one relay capability direction.
     69 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
     70 #[non_exhaustive]
     71 pub enum RelayEvidenceState {
     72     /// The profile does not authorize this capability direction.
     73     Unsupported,
     74     /// The capability is configured but no successful or failed attempt exists.
     75     Unobserved,
     76     /// An authorized operation is currently awaiting relay evidence.
     77     Connecting,
     78     /// The latest accepted observation proved the capability usable.
     79     Available,
     80     /// The latest accepted observation failed to prove the capability usable.
     81     Unavailable,
     82 }
     83 
     84 /// Immutable evidence for one relay capability direction.
     85 #[derive(Clone, Debug, Eq, PartialEq)]
     86 pub struct RelayCapabilityEvidence {
     87     state: RelayEvidenceState,
     88     last_attempt_unix_ms: Option<u64>,
     89     last_success_unix_ms: Option<u64>,
     90     consecutive_failures: u32,
     91     next_attempt_unix_ms: Option<u64>,
     92     last_failure_retryable: Option<bool>,
     93 }
     94 
     95 impl RelayCapabilityEvidence {
     96     /// Returns whether this direction is unsupported, unobserved, or observed.
     97     #[must_use]
     98     pub const fn state(&self) -> RelayEvidenceState {
     99         self.state
    100     }
    101 
    102     /// Returns the latest monotonic accepted attempt timestamp.
    103     #[must_use]
    104     pub const fn last_attempt_unix_ms(&self) -> Option<u64> {
    105         self.last_attempt_unix_ms
    106     }
    107 
    108     /// Returns the latest successful observation timestamp.
    109     #[must_use]
    110     pub const fn last_success_unix_ms(&self) -> Option<u64> {
    111         self.last_success_unix_ms
    112     }
    113 
    114     /// Returns consecutive failures since the latest success.
    115     #[must_use]
    116     pub const fn consecutive_failures(&self) -> u32 {
    117         self.consecutive_failures
    118     }
    119 
    120     /// Returns the earliest adapter-permitted reconnect time after failure.
    121     #[must_use]
    122     pub const fn next_attempt_unix_ms(&self) -> Option<u64> {
    123         self.next_attempt_unix_ms
    124     }
    125 
    126     /// Returns the retry class of the latest failure, when one exists.
    127     #[must_use]
    128     pub const fn last_failure_retryable(&self) -> Option<bool> {
    129         self.last_failure_retryable
    130     }
    131 }
    132 
    133 #[derive(Clone, Debug)]
    134 struct MutableEvidence {
    135     public: RelayCapabilityEvidence,
    136 }
    137 
    138 impl MutableEvidence {
    139     const fn unobserved() -> Self {
    140         Self {
    141             public: RelayCapabilityEvidence {
    142                 state: RelayEvidenceState::Unobserved,
    143                 last_attempt_unix_ms: None,
    144                 last_success_unix_ms: None,
    145                 consecutive_failures: 0,
    146                 next_attempt_unix_ms: None,
    147                 last_failure_retryable: None,
    148             },
    149         }
    150     }
    151 
    152     const fn unsupported() -> Self {
    153         Self {
    154             public: RelayCapabilityEvidence {
    155                 state: RelayEvidenceState::Unsupported,
    156                 last_attempt_unix_ms: None,
    157                 last_success_unix_ms: None,
    158                 consecutive_failures: 0,
    159                 next_attempt_unix_ms: None,
    160                 last_failure_retryable: None,
    161             },
    162         }
    163     }
    164 
    165     fn begin(&mut self, observed_at_unix_ms: u64) {
    166         if matches!(self.public.state, RelayEvidenceState::Unsupported)
    167             || self
    168                 .public
    169                 .last_attempt_unix_ms
    170                 .is_some_and(|current| observed_at_unix_ms < current)
    171         {
    172             return;
    173         }
    174         self.public.state = RelayEvidenceState::Connecting;
    175         self.public.last_attempt_unix_ms = Some(observed_at_unix_ms);
    176     }
    177 
    178     fn record(
    179         &mut self,
    180         succeeded: bool,
    181         retryable: bool,
    182         observed_at_unix_ms: u64,
    183         backoff: ReconnectBackoff,
    184     ) {
    185         if matches!(self.public.state, RelayEvidenceState::Unsupported)
    186             || self
    187                 .public
    188                 .last_attempt_unix_ms
    189                 .is_some_and(|current| observed_at_unix_ms < current)
    190         {
    191             return;
    192         }
    193         self.public.last_attempt_unix_ms = Some(observed_at_unix_ms);
    194         if succeeded {
    195             self.public.state = RelayEvidenceState::Available;
    196             self.public.last_success_unix_ms = Some(observed_at_unix_ms);
    197             self.public.consecutive_failures = 0;
    198             self.public.next_attempt_unix_ms = None;
    199             self.public.last_failure_retryable = None;
    200         } else {
    201             self.public.state = RelayEvidenceState::Unavailable;
    202             self.public.consecutive_failures = self.public.consecutive_failures.saturating_add(1);
    203             self.public.last_failure_retryable = Some(retryable);
    204             self.public.next_attempt_unix_ms = retryable.then(|| {
    205                 observed_at_unix_ms
    206                     .saturating_add(backoff.delay_ms(self.public.consecutive_failures))
    207             });
    208         }
    209     }
    210 
    211     fn may_attempt(&self, now_unix_ms: u64) -> bool {
    212         !matches!(self.public.state, RelayEvidenceState::Unsupported)
    213             && self.public.last_failure_retryable != Some(false)
    214             && self
    215                 .public
    216                 .next_attempt_unix_ms
    217                 .is_none_or(|retry_at| now_unix_ms >= retry_at)
    218     }
    219 }
    220 
    221 #[derive(Clone, Debug)]
    222 struct MutableRelayStatus {
    223     endpoint: RelayEndpoint,
    224     read: MutableEvidence,
    225     write: MutableEvidence,
    226 }
    227 
    228 /// Passive typed status for one configured relay.
    229 #[derive(Clone, Debug, Eq, PartialEq)]
    230 pub struct RelayStatus {
    231     endpoint: RelayEndpoint,
    232     read: RelayCapabilityEvidence,
    233     write: RelayCapabilityEvidence,
    234 }
    235 
    236 impl RelayStatus {
    237     /// Returns the canonical endpoint and its declared authority.
    238     #[must_use]
    239     pub const fn endpoint(&self) -> &RelayEndpoint {
    240         &self.endpoint
    241     }
    242 
    243     /// Returns independent read evidence.
    244     #[must_use]
    245     pub const fn read(&self) -> &RelayCapabilityEvidence {
    246         &self.read
    247     }
    248 
    249     /// Returns independent write evidence.
    250     #[must_use]
    251     pub const fn write(&self) -> &RelayCapabilityEvidence {
    252         &self.write
    253     }
    254 }
    255 
    256 /// Passive per-relay and aggregate status for one configured profile.
    257 #[derive(Clone, Debug, Eq, PartialEq)]
    258 pub struct RelayStatusReport {
    259     profile_kind: RelayProfileKind,
    260     state: RelayAggregateState,
    261     relays: Vec<RelayStatus>,
    262     read_availability: Availability,
    263     write_availability: Availability,
    264 }
    265 
    266 /// Aggregate lifecycle derived from current per-relay evidence.
    267 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
    268 #[non_exhaustive]
    269 pub enum RelayAggregateState {
    270     /// A profile is installed but no relay operation has started.
    271     Configured,
    272     /// At least one authorized relay operation is in flight.
    273     Connecting,
    274     /// Current evidence proves reads but not publication.
    275     ReadOnly,
    276     /// Current evidence proves both reads and publication.
    277     Writable,
    278     /// Some, but not all, authorized relay capabilities have current success.
    279     Degraded,
    280     /// Every attempted capability is temporarily unavailable and retry-bounded.
    281     Offline,
    282     /// Every attempted capability failed terminally for the unchanged request.
    283     Failed,
    284 }
    285 
    286 impl RelayStatusReport {
    287     /// Returns the host profile whose policies govern this report.
    288     #[must_use]
    289     pub const fn profile_kind(&self) -> RelayProfileKind {
    290         self.profile_kind
    291     }
    292 
    293     /// Returns the aggregate lifecycle derived only from current evidence.
    294     #[must_use]
    295     pub const fn state(&self) -> RelayAggregateState {
    296         self.state
    297     }
    298 
    299     /// Returns one status per configured relay in profile order.
    300     #[must_use]
    301     pub fn relays(&self) -> &[RelayStatus] {
    302         self.relays.as_slice()
    303     }
    304 
    305     /// Returns aggregate read availability derived only from observations.
    306     #[must_use]
    307     pub const fn read_availability(&self) -> Availability {
    308         self.read_availability
    309     }
    310 
    311     /// Returns aggregate write availability derived only from writable relays.
    312     #[must_use]
    313     pub const fn write_availability(&self) -> Availability {
    314         self.write_availability
    315     }
    316 }
    317 
    318 #[derive(Clone, Debug)]
    319 struct Snapshot {
    320     order: Vec<RelayUrl>,
    321     relays: BTreeMap<RelayUrl, MutableRelayStatus>,
    322 }
    323 
    324 impl Snapshot {
    325     fn new(config: &Config) -> Self {
    326         Self {
    327             order: config.relays().to_vec(),
    328             relays: config
    329                 .endpoints()
    330                 .iter()
    331                 .map(|endpoint| {
    332                     (
    333                         endpoint.url().clone(),
    334                         MutableRelayStatus {
    335                             endpoint: endpoint.clone(),
    336                             read: MutableEvidence::unobserved(),
    337                             write: if endpoint.access().can_write() {
    338                                 MutableEvidence::unobserved()
    339                             } else {
    340                                 MutableEvidence::unsupported()
    341                             },
    342                         },
    343                     )
    344                 })
    345                 .collect(),
    346         }
    347     }
    348 }
    349 
    350 #[derive(Debug)]
    351 pub(crate) struct StatusTracker {
    352     initial: Snapshot,
    353     snapshot: Mutex<Snapshot>,
    354     profile_kind: RelayProfileKind,
    355     backoff: ReconnectBackoff,
    356 }
    357 
    358 impl StatusTracker {
    359     pub(crate) fn new(config: &Config) -> Self {
    360         let initial = Snapshot::new(config);
    361         Self {
    362             snapshot: Mutex::new(initial.clone()),
    363             initial,
    364             profile_kind: config.profile_kind(),
    365             backoff: config.reconnect_backoff(),
    366         }
    367     }
    368 
    369     pub(crate) fn begin_read(&self, relay: &RelayUrl, observed_at_unix_ms: u64) {
    370         if let Ok(mut snapshot) = self.snapshot.lock()
    371             && let Some(status) = snapshot.relays.get_mut(relay)
    372         {
    373             status.read.begin(observed_at_unix_ms);
    374         }
    375     }
    376 
    377     pub(crate) fn begin_write(&self, relay: &RelayUrl, observed_at_unix_ms: u64) {
    378         if let Ok(mut snapshot) = self.snapshot.lock()
    379             && let Some(status) = snapshot.relays.get_mut(relay)
    380         {
    381             status.write.begin(observed_at_unix_ms);
    382         }
    383     }
    384 
    385     pub(crate) fn record_read(
    386         &self,
    387         relay: &RelayUrl,
    388         succeeded: bool,
    389         retryable: bool,
    390         observed_at_unix_ms: u64,
    391     ) {
    392         if let Ok(mut snapshot) = self.snapshot.lock()
    393             && let Some(status) = snapshot.relays.get_mut(relay)
    394         {
    395             status
    396                 .read
    397                 .record(succeeded, retryable, observed_at_unix_ms, self.backoff);
    398         }
    399     }
    400 
    401     pub(crate) fn record_write(
    402         &self,
    403         relay: &RelayUrl,
    404         succeeded: bool,
    405         retryable: bool,
    406         observed_at_unix_ms: u64,
    407     ) {
    408         if let Ok(mut snapshot) = self.snapshot.lock()
    409             && let Some(status) = snapshot.relays.get_mut(relay)
    410         {
    411             status
    412                 .write
    413                 .record(succeeded, retryable, observed_at_unix_ms, self.backoff);
    414         }
    415     }
    416 
    417     pub(crate) fn may_read(&self, relay: &RelayUrl, now_unix_ms: u64) -> bool {
    418         self.snapshot
    419             .lock()
    420             .ok()
    421             .and_then(|snapshot| {
    422                 snapshot
    423                     .relays
    424                     .get(relay)
    425                     .map(|status| status.read.may_attempt(now_unix_ms))
    426             })
    427             .unwrap_or(false)
    428     }
    429 
    430     pub(crate) fn may_write(&self, relay: &RelayUrl, now_unix_ms: u64) -> bool {
    431         self.snapshot
    432             .lock()
    433             .ok()
    434             .and_then(|snapshot| {
    435                 snapshot
    436                     .relays
    437                     .get(relay)
    438                     .map(|status| status.write.may_attempt(now_unix_ms))
    439             })
    440             .unwrap_or(false)
    441     }
    442 
    443     pub(crate) fn report(&self) -> RelayStatusReport {
    444         let snapshot = self
    445             .snapshot
    446             .lock()
    447             .map(|snapshot| snapshot.clone())
    448             .unwrap_or_else(|_| self.initial.clone());
    449         let relays = self
    450             .initial
    451             .order
    452             .iter()
    453             .filter_map(|relay| snapshot.relays.get(relay))
    454             .map(|status| RelayStatus {
    455                 endpoint: status.endpoint.clone(),
    456                 read: status.read.public.clone(),
    457                 write: status.write.public.clone(),
    458             })
    459             .collect::<Vec<_>>();
    460         RelayStatusReport {
    461             profile_kind: self.profile_kind,
    462             state: aggregate_state(relays.as_slice()),
    463             read_availability: aggregate(relays.iter().map(RelayStatus::read)),
    464             write_availability: aggregate(relays.iter().map(RelayStatus::write)),
    465             relays,
    466         }
    467     }
    468 }
    469 
    470 fn aggregate_state(relays: &[RelayStatus]) -> RelayAggregateState {
    471     let evidence = relays
    472         .iter()
    473         .flat_map(|relay| [relay.read(), relay.write()])
    474         .filter(|evidence| !matches!(evidence.state(), RelayEvidenceState::Unsupported))
    475         .collect::<Vec<_>>();
    476     if evidence
    477         .iter()
    478         .any(|evidence| matches!(evidence.state(), RelayEvidenceState::Connecting))
    479     {
    480         return RelayAggregateState::Connecting;
    481     }
    482     if evidence
    483         .iter()
    484         .all(|evidence| matches!(evidence.state(), RelayEvidenceState::Unobserved))
    485     {
    486         return RelayAggregateState::Configured;
    487     }
    488     let read = relays.iter().map(RelayStatus::read).collect::<Vec<_>>();
    489     let write = relays
    490         .iter()
    491         .map(RelayStatus::write)
    492         .filter(|evidence| !matches!(evidence.state(), RelayEvidenceState::Unsupported))
    493         .collect::<Vec<_>>();
    494     let read_available = read
    495         .iter()
    496         .filter(|evidence| matches!(evidence.state(), RelayEvidenceState::Available))
    497         .count();
    498     let write_available = write
    499         .iter()
    500         .filter(|evidence| matches!(evidence.state(), RelayEvidenceState::Available))
    501         .count();
    502     if read_available == read.len() && !write.is_empty() && write_available == write.len() {
    503         RelayAggregateState::Writable
    504     } else if read_available == read.len() && write_available == 0 {
    505         RelayAggregateState::ReadOnly
    506     } else if read_available + write_available > 0 {
    507         RelayAggregateState::Degraded
    508     } else if evidence
    509         .iter()
    510         .any(|evidence| evidence.last_failure_retryable() == Some(true))
    511     {
    512         RelayAggregateState::Offline
    513     } else if evidence
    514         .iter()
    515         .any(|evidence| evidence.last_failure_retryable() == Some(false))
    516     {
    517         RelayAggregateState::Failed
    518     } else {
    519         RelayAggregateState::Configured
    520     }
    521 }
    522 
    523 fn aggregate<'a>(evidence: impl Iterator<Item = &'a RelayCapabilityEvidence>) -> Availability {
    524     let mut supported = 0usize;
    525     let mut available = 0usize;
    526     for evidence in evidence {
    527         if !matches!(evidence.state, RelayEvidenceState::Unsupported) {
    528             supported += 1;
    529             if matches!(evidence.state, RelayEvidenceState::Available) {
    530                 available += 1;
    531             }
    532         }
    533     }
    534     match (supported, available) {
    535         (0, _) | (_, 0) => Availability::Unavailable,
    536         (supported, available) if supported == available => Availability::Available,
    537         _ => Availability::Degraded,
    538     }
    539 }
    540 
    541 pub(crate) fn source_status(tracker: &StatusTracker) -> SourceStatus {
    542     let report = tracker.report();
    543     SourceStatus::new(
    544         radroots_transport::TransportId::NOSTR,
    545         !report.relays().is_empty(),
    546         Maturity::Preview,
    547         report.read_availability(),
    548         SourceCapabilities::FETCH,
    549         match report.read_availability() {
    550             Availability::Available => "Nostr read capability has current successful evidence",
    551             Availability::Degraded => "Nostr read capability has partial successful evidence",
    552             Availability::Unavailable => "Nostr read capability has no current successful evidence",
    553         },
    554     )
    555 }
    556 
    557 pub(crate) fn sink_status(tracker: &StatusTracker) -> SinkStatus {
    558     let report = tracker.report();
    559     let configured = report
    560         .relays()
    561         .iter()
    562         .any(|status| status.endpoint().access().can_write());
    563     SinkStatus::new(
    564         radroots_transport::TransportId::NOSTR,
    565         configured,
    566         Maturity::Preview,
    567         report.write_availability(),
    568         SinkCapabilities::DELIVER,
    569         match report.write_availability() {
    570             Availability::Available => "Nostr write capability has current successful evidence",
    571             Availability::Degraded => "Nostr write capability has partial successful evidence",
    572             Availability::Unavailable => {
    573                 "Nostr write capability has no current successful evidence"
    574             }
    575         },
    576     )
    577 }
    578 
    579 pub(crate) fn delivery_failure(upstream: &str) -> DeliveryOutcome {
    580     // Error text can classify a failure, but cannot establish relay acceptance.
    581     let class = classify(upstream).class;
    582     let outcome = match class {
    583         FailureClass::Rejected | FailureClass::Malformed | FailureClass::Quota => {
    584             DeliveryOutcome::rejected()
    585         }
    586         FailureClass::AuthRequired => DeliveryOutcome::failed(Retryability::Retryable)
    587             .expect("retryable authentication outcome"),
    588         FailureClass::RateLimited
    589         | FailureClass::Timeout
    590         | FailureClass::Connection
    591         | FailureClass::Unknown => DeliveryOutcome::unavailable(),
    592     };
    593     outcome
    594         .with_detail(class.code(), class.message())
    595         .expect("static normalized relay outcome")
    596 }
    597 
    598 pub(crate) fn fetch_failure(upstream: &str) -> (FetchTargetState, &'static str) {
    599     let class = classify(upstream).class;
    600     let state = match class {
    601         FailureClass::Rejected | FailureClass::Malformed | FailureClass::Quota => {
    602             FetchTargetState::FailedTerminal
    603         }
    604         _ => FetchTargetState::FailedRetryable,
    605     };
    606     (state, class.message())
    607 }
    608 
    609 fn classify(upstream: &str) -> RedactedDiagnostic {
    610     let message = upstream.to_ascii_lowercase();
    611     let class = if message.contains("auth") {
    612         FailureClass::AuthRequired
    613     } else if message.contains("quota") {
    614         FailureClass::Quota
    615     } else if message.contains("blocked")
    616         || message.contains("restricted")
    617         || message.contains("invalid")
    618         || message.contains("reject")
    619     {
    620         FailureClass::Rejected
    621     } else if message.contains("rate") {
    622         FailureClass::RateLimited
    623     } else if message.contains("timeout") || message.contains("timed out") {
    624         FailureClass::Timeout
    625     } else if message.contains("connect") || message.contains("offline") {
    626         FailureClass::Connection
    627     } else if message.contains("malformed") || message.contains("decode") {
    628         FailureClass::Malformed
    629     } else {
    630         FailureClass::Unknown
    631     };
    632     RedactedDiagnostic { class }
    633 }
    634 
    635 pub(crate) fn delivery_succeeded(outcome: &DeliveryOutcome) -> bool {
    636     matches!(
    637         outcome.kind(),
    638         DeliveryOutcomeKind::Accepted | DeliveryOutcomeKind::Delivered
    639     )
    640 }
    641 
    642 #[cfg(test)]
    643 mod tests {
    644     use super::*;
    645     use crate::ReconnectBackoff;
    646 
    647     fn tracker() -> (StatusTracker, RelayUrl, RelayUrl) {
    648         let config = Config::from_profile(
    649             crate::profile::test_profile(
    650                 crate::RelayProfileKind::Public,
    651                 crate::RelayUrlPolicy::Public,
    652                 ["wss://read.example", "wss://write.example"],
    653             )
    654             .expect("profile"),
    655         )
    656         .with_reconnect_backoff(ReconnectBackoff::new(10, 40).expect("backoff"));
    657         let readable = config.relays()[0].clone();
    658         let writable = config.relays()[1].clone();
    659         (StatusTracker::new(&config), readable, writable)
    660     }
    661 
    662     #[test]
    663     fn every_upstream_class_maps_to_stable_secret_safe_output() {
    664         let secret = "token=very-secret-value";
    665         let cases = [
    666             ("duplicate: already have", "relay_failure"),
    667             ("blocked by policy", "rejected"),
    668             ("auth required", "auth_required"),
    669             ("quota exceeded", "quota_exceeded"),
    670             ("blocked: account QUOTA exhausted", "quota_exceeded"),
    671             ("rate limited", "rate_limited"),
    672             ("connection timeout", "timeout"),
    673             ("connection offline", "connection_failed"),
    674             ("unknown failure", "relay_failure"),
    675         ];
    676         for (message, code) in cases {
    677             let outcome = delivery_failure(format!("{message} {secret}").as_str());
    678             assert_eq!(outcome.code(), Some(code));
    679             assert!(!outcome.message().expect("message").contains(secret));
    680             let diagnostic = classify(format!("{message} {secret}").as_str());
    681             assert!(!format!("{diagnostic:?}").contains(secret));
    682         }
    683     }
    684 
    685     #[test]
    686     fn duplicate_text_cannot_turn_an_untyped_failure_into_acceptance() {
    687         for message in [
    688             "duplicate: already have this event",
    689             "already have this event",
    690             "blocked: not a duplicate; event was rejected",
    691             "auth-required: duplicate path requires login",
    692             "connection failed: duplicate acknowledgement unavailable",
    693             "error: duplicate detector unavailable",
    694         ] {
    695             let outcome = delivery_failure(message);
    696             assert!(!delivery_succeeded(&outcome), "{message}");
    697         }
    698     }
    699 
    700     #[test]
    701     fn quota_refusal_requires_action_without_reclassifying_other_failures() {
    702         for text in ["quota exceeded", "blocked: account QUOTA exhausted"] {
    703             let outcome = delivery_failure(text);
    704             assert_eq!(outcome.code(), Some("quota_exceeded"));
    705             assert_eq!(outcome.kind(), DeliveryOutcomeKind::Rejected);
    706             assert_eq!(outcome.retryability(), Retryability::Terminal);
    707             assert_eq!(
    708                 fetch_failure(text),
    709                 (
    710                     FetchTargetState::FailedTerminal,
    711                     "relay quota was exhausted"
    712                 )
    713             );
    714         }
    715         assert_eq!(
    716             delivery_failure("rate limited").code(),
    717             Some("rate_limited")
    718         );
    719         assert!(delivery_failure("rate limited").is_retryable());
    720         assert_eq!(
    721             delivery_failure("auth required").code(),
    722             Some("auth_required")
    723         );
    724         assert_eq!(
    725             delivery_failure("malformed event").code(),
    726             Some("malformed_event")
    727         );
    728         assert_eq!(
    729             delivery_failure("unknown failure").code(),
    730             Some("relay_failure")
    731         );
    732         assert!(delivery_failure("unknown failure").is_retryable());
    733     }
    734 
    735     #[test]
    736     fn status_requires_directional_evidence_and_backoff_is_monotonic() {
    737         let (tracker, canonical, writable) = tracker();
    738         let initial = tracker.report();
    739         assert_eq!(initial.read_availability(), Availability::Unavailable);
    740         assert_eq!(initial.write_availability(), Availability::Unavailable);
    741         assert_eq!(initial.state(), RelayAggregateState::Configured);
    742         assert_eq!(
    743             initial.relays()[0].write().state(),
    744             RelayEvidenceState::Unobserved
    745         );
    746         assert!(tracker.may_write(&canonical, 100));
    747 
    748         tracker.begin_read(&canonical, 100);
    749         assert_eq!(tracker.report().state(), RelayAggregateState::Connecting);
    750         tracker.record_read(&canonical, true, false, 100);
    751         tracker.record_read(&writable, false, true, 100);
    752         tracker.record_write(&writable, false, true, 100);
    753         let partial = tracker.report();
    754         assert_eq!(partial.state(), RelayAggregateState::Degraded);
    755         assert_eq!(partial.read_availability(), Availability::Degraded);
    756         assert_eq!(partial.write_availability(), Availability::Unavailable);
    757         assert!(!tracker.may_write(&writable, 109));
    758         assert!(tracker.may_write(&writable, 110));
    759 
    760         tracker.record_write(&writable, false, true, 110);
    761         assert_eq!(
    762             tracker.report().relays()[1].write().next_attempt_unix_ms(),
    763             Some(130)
    764         );
    765         tracker.record_write(&writable, true, false, 109);
    766         assert_eq!(
    767             tracker.report().relays()[1].write().state(),
    768             RelayEvidenceState::Unavailable
    769         );
    770         tracker.record_write(&writable, true, false, 130);
    771         let partially_available = tracker.report();
    772         assert_eq!(
    773             partially_available.write_availability(),
    774             Availability::Degraded
    775         );
    776         assert_eq!(
    777             partially_available.relays()[1]
    778                 .write()
    779                 .consecutive_failures(),
    780             0
    781         );
    782         assert_eq!(
    783             partially_available.relays()[1]
    784                 .write()
    785                 .last_success_unix_ms(),
    786             Some(130)
    787         );
    788         assert!(tracker.may_write(&writable, 130));
    789         tracker.record_write(&canonical, true, false, 130);
    790         assert_eq!(
    791             tracker.report().write_availability(),
    792             Availability::Available
    793         );
    794         assert_eq!(
    795             source_status(&tracker).availability(),
    796             Availability::Degraded
    797         );
    798         assert_eq!(
    799             sink_status(&tracker).availability(),
    800             Availability::Available
    801         );
    802 
    803         tracker.record_write(&writable, false, false, 140);
    804         tracker.record_write(&canonical, false, false, 140);
    805         assert!(!tracker.may_write(&writable, u64::MAX));
    806         assert!(!tracker.may_write(&canonical, u64::MAX));
    807         assert_eq!(
    808             tracker.report().relays()[1]
    809                 .write()
    810                 .last_failure_retryable(),
    811             Some(false)
    812         );
    813         assert_eq!(
    814             sink_status(&tracker).availability(),
    815             Availability::Unavailable
    816         );
    817     }
    818 
    819     #[test]
    820     fn normalized_failures_and_success_helpers_cover_every_state() {
    821         for message in [
    822             "invalid event",
    823             "restricted",
    824             "rejected",
    825             "malformed event",
    826             "decode failed",
    827         ] {
    828             assert_eq!(fetch_failure(message).0, FetchTargetState::FailedTerminal);
    829         }
    830         assert_eq!(
    831             fetch_failure("offline").0,
    832             FetchTargetState::FailedRetryable
    833         );
    834         assert_eq!(
    835             delivery_failure("malformed event").kind(),
    836             DeliveryOutcomeKind::Rejected
    837         );
    838         assert!(delivery_succeeded(&DeliveryOutcome::accepted()));
    839         assert!(delivery_succeeded(&DeliveryOutcome::delivered()));
    840         assert!(!delivery_succeeded(&DeliveryOutcome::rejected()));
    841     }
    842 
    843     #[test]
    844     fn canonical_relay_read_and_write_success_is_available() {
    845         let config = Config::from_profile(
    846             crate::profile::test_profile(
    847                 crate::RelayProfileKind::Public,
    848                 crate::RelayUrlPolicy::Public,
    849                 ["wss://relay.example"],
    850             )
    851             .expect("profile"),
    852         );
    853         let tracker = StatusTracker::new(&config);
    854         let relay = config.relays()[0].clone();
    855         tracker.record_read(&relay, true, false, 1);
    856         tracker.record_write(&relay, true, false, 1);
    857         assert_eq!(
    858             source_status(&tracker).availability(),
    859             Availability::Available
    860         );
    861         let sink = sink_status(&tracker);
    862         assert!(sink.is_configured());
    863         assert_eq!(sink.availability(), Availability::Available);
    864         assert_eq!(tracker.report().state(), RelayAggregateState::Writable);
    865     }
    866 
    867     #[test]
    868     fn aggregate_states_and_unconfigured_targets_cover_fail_closed_edges() {
    869         let (live, canonical, writable) = tracker();
    870         let unknown =
    871             RelayUrl::parse("wss://unknown.example", crate::RelayUrlPolicy::Public).expect("relay");
    872 
    873         live.begin_read(&unknown, 1);
    874         live.begin_write(&unknown, 1);
    875         live.record_read(&unknown, true, false, 1);
    876         live.record_write(&unknown, true, false, 1);
    877         live.begin_write(&canonical, 1);
    878         live.record_write(&canonical, true, false, 1);
    879         assert_eq!(
    880             live.report().relays()[0].write().state(),
    881             RelayEvidenceState::Available
    882         );
    883 
    884         live.begin_read(&canonical, 2);
    885         live.begin_read(&canonical, 1);
    886         live.record_read(&canonical, true, false, 2);
    887         live.record_read(&writable, true, false, 2);
    888         live.record_write(&writable, true, false, 2);
    889         let writable_report = live.report();
    890         assert_eq!(writable_report.state(), RelayAggregateState::Writable);
    891         assert_eq!(writable_report.read_availability(), Availability::Available);
    892         assert_eq!(
    893             writable_report.write_availability(),
    894             Availability::Available
    895         );
    896 
    897         let (offline, canonical, writable) = tracker();
    898         offline.record_read(&canonical, false, true, 10);
    899         offline.record_read(&writable, false, true, 10);
    900         offline.record_write(&canonical, false, true, 10);
    901         offline.record_write(&writable, false, true, 10);
    902         assert_eq!(offline.report().state(), RelayAggregateState::Offline);
    903 
    904         let (failed, canonical, writable) = tracker();
    905         failed.record_read(&canonical, false, false, 10);
    906         failed.record_read(&writable, false, false, 10);
    907         failed.record_write(&canonical, false, false, 10);
    908         failed.record_write(&writable, false, false, 10);
    909         assert_eq!(failed.report().state(), RelayAggregateState::Failed);
    910 
    911         assert_eq!(
    912             delivery_failure("already have").code(),
    913             Some("relay_failure")
    914         );
    915         assert_eq!(
    916             fetch_failure("timed out").0,
    917             FetchTargetState::FailedRetryable
    918         );
    919     }
    920 
    921     #[test]
    922     fn capability_evidence_rejects_unsupported_and_stale_observations() {
    923         let backoff = ReconnectBackoff::new(10, 40).expect("backoff");
    924 
    925         let mut unsupported = MutableEvidence::unsupported();
    926         unsupported.begin(10);
    927         unsupported.record(true, false, 10, backoff);
    928         assert_eq!(unsupported.public.state(), RelayEvidenceState::Unsupported);
    929         assert!(!unsupported.may_attempt(u64::MAX));
    930 
    931         let mut evidence = MutableEvidence::unobserved();
    932         evidence.begin(10);
    933         evidence.begin(9);
    934         assert_eq!(evidence.public.last_attempt_unix_ms(), Some(10));
    935 
    936         evidence.record(false, true, 10, backoff);
    937         evidence.record(true, false, 9, backoff);
    938         assert_eq!(evidence.public.state(), RelayEvidenceState::Unavailable);
    939         assert!(!evidence.may_attempt(19));
    940         assert!(evidence.may_attempt(20));
    941 
    942         let (read_only, canonical, writable) = tracker();
    943         read_only.record_read(&canonical, true, false, 1);
    944         read_only.record_read(&writable, true, false, 1);
    945         let report = read_only.report();
    946         assert_eq!(report.state(), RelayAggregateState::ReadOnly);
    947         assert_eq!(report.read_availability(), Availability::Available);
    948         assert_eq!(report.write_availability(), Availability::Unavailable);
    949 
    950         let read_only_profile = crate::RelayProfile::explicit(
    951             crate::RelayProfileKind::Public,
    952             [crate::RelayEndpoint::new(
    953                 "wss://read-only.example",
    954                 crate::RelayUrlPolicy::Public,
    955                 crate::RelayAccess::ReadOnly,
    956             )
    957             .expect("read-only endpoint")],
    958         )
    959         .expect("read-only profile");
    960         let read_only_config = Config::from_profile(read_only_profile);
    961         let read_only_relay = read_only_config.relays()[0].clone();
    962         let read_only_tracker = StatusTracker::new(&read_only_config);
    963         read_only_tracker.record_read(&read_only_relay, true, false, 1);
    964         let report = read_only_tracker.report();
    965         assert_eq!(report.state(), RelayAggregateState::ReadOnly);
    966         assert_eq!(report.read_availability(), Availability::Available);
    967         assert_eq!(report.write_availability(), Availability::Unavailable);
    968         assert_eq!(
    969             report.relays()[0].write().state(),
    970             RelayEvidenceState::Unsupported
    971         );
    972     }
    973 }