state_delivery.rs (80129B)
1 //! Durable delivery-job identity, bounded target attempts, and lease evidence. 2 3 #[cfg(all(test, any(target_os = "linux", target_os = "macos")))] 4 #[path = "state_delivery_tests.rs"] 5 mod tests; 6 7 use core::fmt; 8 use std::error::Error; 9 10 use radroots_service_sqlite::{ 11 ServiceSqliteTransaction, ServiceSqliteTransactionError, ServiceSqliteTransactionErrorKind, 12 }; 13 use sha2::{Digest, Sha256}; 14 use sqlx::Row; 15 16 use crate::MycSignerOperationId; 17 use crate::state_repository::{ 18 MycStateRepository, MycStateRepositoryError, MycStateRepositoryErrorKind, PersistedMetadata, 19 RepositoryOperationError, require_expected_metadata, 20 }; 21 22 /// Maximum immutable relay targets on one delivery job. 23 pub const MYC_DELIVERY_TARGET_MAX_COUNT: usize = 32; 24 /// Maximum durable attempts for one target. 25 pub const MYC_DELIVERY_ATTEMPT_MAX_COUNT: u32 = 32; 26 /// Maximum UTF-8 byte length of a canonical delivery relay identifier. 27 pub const MYC_DELIVERY_RELAY_ID_MAX_BYTES: usize = 64; 28 /// Maximum caller-injected full-jitter delay for one retry. 29 pub const MYC_DELIVERY_RETRY_JITTER_MAX_MS: u64 = 300_000; 30 31 const JOB_ID_DOMAIN: &[u8] = b"radroots.myc.delivery_job.v1\0"; 32 const ATTEMPT_ID_DOMAIN: &[u8] = b"radroots.myc.delivery_attempt.v1\0"; 33 34 const READ_ACTIVE_JOB_COUNT_SQL: &str = 35 "SELECT COUNT(*) AS row_count FROM delivery_jobs WHERE status IN ('pending', 'active')"; 36 37 const READ_RUNTIME_OUTBOX_STATUS_SQL: &str = r#"SELECT 38 COUNT(CASE WHEN status IN ('pending', 'active') THEN 1 END) AS pending_count, 39 (SELECT COUNT(*) FROM delivery_targets WHERE status = 'unknown') AS unknown_count, 40 MIN(CASE WHEN status IN ('pending', 'active') THEN created_at_unix_ms ELSE NULL END) 41 AS oldest_pending_at_unix_ms, 42 typeof(MIN(CASE WHEN status IN ('pending', 'active') THEN created_at_unix_ms ELSE NULL END)) 43 AS oldest_pending_type 44 FROM delivery_jobs"#; 45 46 const READ_NEXT_READY_TARGET_SQL: &str = r#"SELECT 47 CASE WHEN typeof(t.job_id) = 'blob' AND length(t.job_id) = 32 48 THEN t.job_id ELSE NULL END AS job_id, 49 CASE WHEN typeof(t.relay_id) = 'text' 50 AND length(CAST(t.relay_id AS BLOB)) BETWEEN 1 AND 64 51 THEN t.relay_id ELSE NULL END AS relay_id 52 FROM delivery_targets t 53 JOIN delivery_jobs j ON j.job_id = t.job_id 54 WHERE j.status IN ('pending', 'active') 55 AND t.status IN ('pending', 'retryable', 'unknown') 56 AND t.attempt_count < j.max_attempts 57 AND t.active_attempt_id IS NULL 58 AND (t.next_attempt_at_unix_ms IS NULL OR t.next_attempt_at_unix_ms <= ?) 59 ORDER BY j.created_at_unix_ms, t.job_id, t.target_index 60 LIMIT 1"#; 61 62 const READ_JOB_SQL: &str = r#"SELECT 63 CASE WHEN typeof(job_id) = 'blob' AND length(job_id) = 32 64 THEN job_id ELSE NULL END AS job_id, 65 CASE WHEN typeof(source_kind) = 'text' AND length(CAST(source_kind AS BLOB)) <= 32 66 THEN source_kind ELSE NULL END AS source_kind, 67 CASE WHEN typeof(source_id) = 'blob' AND length(source_id) = 32 68 THEN source_id ELSE NULL END AS source_id, 69 CASE WHEN typeof(artifact_sha256) = 'blob' AND length(artifact_sha256) = 32 70 THEN artifact_sha256 ELSE NULL END AS artifact_sha256, 71 CASE WHEN typeof(policy_mode) = 'text' AND length(CAST(policy_mode AS BLOB)) <= 32 72 THEN policy_mode ELSE NULL END AS policy_mode, 73 required_acknowledgements, max_attempts, initial_backoff_ms, 74 maximum_backoff_ms, attempt_deadline_ms, 75 CASE WHEN typeof(status) = 'text' AND length(CAST(status AS BLOB)) <= 16 76 THEN status ELSE NULL END AS status, 77 created_at_unix_ms, updated_at_unix_ms, finalized_at_unix_ms, 78 typeof(finalized_at_unix_ms) AS finalized_at_type 79 FROM delivery_jobs 80 WHERE job_id = ? 81 LIMIT 2"#; 82 83 const READ_JOB_BY_SOURCE_SQL: &str = r#"SELECT 84 CASE WHEN typeof(job_id) = 'blob' AND length(job_id) = 32 85 THEN job_id ELSE NULL END AS job_id, 86 CASE WHEN typeof(source_kind) = 'text' AND length(CAST(source_kind AS BLOB)) <= 32 87 THEN source_kind ELSE NULL END AS source_kind, 88 CASE WHEN typeof(source_id) = 'blob' AND length(source_id) = 32 89 THEN source_id ELSE NULL END AS source_id, 90 CASE WHEN typeof(artifact_sha256) = 'blob' AND length(artifact_sha256) = 32 91 THEN artifact_sha256 ELSE NULL END AS artifact_sha256, 92 CASE WHEN typeof(policy_mode) = 'text' AND length(CAST(policy_mode AS BLOB)) <= 32 93 THEN policy_mode ELSE NULL END AS policy_mode, 94 required_acknowledgements, max_attempts, initial_backoff_ms, 95 maximum_backoff_ms, attempt_deadline_ms, 96 CASE WHEN typeof(status) = 'text' AND length(CAST(status AS BLOB)) <= 16 97 THEN status ELSE NULL END AS status, 98 created_at_unix_ms, updated_at_unix_ms, finalized_at_unix_ms, 99 typeof(finalized_at_unix_ms) AS finalized_at_type 100 FROM delivery_jobs 101 WHERE source_kind = ? AND source_id = ? 102 LIMIT 2"#; 103 104 const READ_TARGETS_SQL: &str = r#"SELECT 105 target_index, 106 CASE WHEN typeof(relay_id) = 'text' 107 AND length(CAST(relay_id AS BLOB)) BETWEEN 1 AND 64 108 THEN relay_id ELSE NULL END AS relay_id, 109 required, attempt_count, 110 CASE WHEN typeof(status) = 'text' AND length(CAST(status AS BLOB)) <= 16 111 THEN status ELSE NULL END AS status, 112 CASE WHEN typeof(active_attempt_id) = 'blob' AND length(active_attempt_id) = 32 113 THEN active_attempt_id ELSE NULL END AS active_attempt_id, 114 typeof(active_attempt_id) AS active_attempt_id_type, 115 next_attempt_at_unix_ms, 116 typeof(next_attempt_at_unix_ms) AS next_attempt_at_type, 117 updated_at_unix_ms 118 FROM delivery_targets 119 WHERE job_id = ? 120 ORDER BY target_index 121 LIMIT 33"#; 122 123 const READ_ATTEMPTS_SQL: &str = r#"SELECT 124 CASE WHEN typeof(attempt_id) = 'blob' AND length(attempt_id) = 32 125 THEN attempt_id ELSE NULL END AS attempt_id, 126 attempt_number, 127 CASE WHEN typeof(attempt_nonce) = 'blob' AND length(attempt_nonce) = 32 128 THEN attempt_nonce ELSE NULL END AS attempt_nonce, 129 CASE WHEN typeof(status) = 'text' AND length(CAST(status AS BLOB)) <= 16 130 THEN status ELSE NULL END AS status, 131 leased_at_unix_ms, lease_expires_at_unix_ms, 132 submitted_at_unix_ms, typeof(submitted_at_unix_ms) AS submitted_at_type, 133 resolved_at_unix_ms, typeof(resolved_at_unix_ms) AS resolved_at_type, 134 CASE WHEN typeof(reason_code) = 'text' AND length(CAST(reason_code AS BLOB)) <= 32 135 THEN reason_code ELSE NULL END AS reason_code, 136 typeof(reason_code) AS reason_code_type 137 FROM delivery_attempts 138 WHERE job_id = ? AND target_index = ? 139 ORDER BY attempt_number 140 LIMIT 33"#; 141 142 const READ_ATTEMPT_SQL: &str = r#"SELECT 143 CASE WHEN typeof(attempt_id) = 'blob' AND length(attempt_id) = 32 144 THEN attempt_id ELSE NULL END AS attempt_id, 145 attempt_number, 146 CASE WHEN typeof(attempt_nonce) = 'blob' AND length(attempt_nonce) = 32 147 THEN attempt_nonce ELSE NULL END AS attempt_nonce, 148 CASE WHEN typeof(status) = 'text' AND length(CAST(status AS BLOB)) <= 16 149 THEN status ELSE NULL END AS status, 150 leased_at_unix_ms, lease_expires_at_unix_ms, 151 submitted_at_unix_ms, typeof(submitted_at_unix_ms) AS submitted_at_type, 152 resolved_at_unix_ms, typeof(resolved_at_unix_ms) AS resolved_at_type, 153 CASE WHEN typeof(reason_code) = 'text' AND length(CAST(reason_code AS BLOB)) <= 32 154 THEN reason_code ELSE NULL END AS reason_code, 155 typeof(reason_code) AS reason_code_type 156 FROM delivery_attempts 157 WHERE job_id = ? AND target_index = ? AND attempt_id = ? 158 LIMIT 2"#; 159 160 const READ_ATTEMPT_BY_NONCE_SQL: &str = r#"SELECT 161 CASE WHEN typeof(attempt_id) = 'blob' AND length(attempt_id) = 32 162 THEN attempt_id ELSE NULL END AS attempt_id, 163 attempt_number, 164 CASE WHEN typeof(attempt_nonce) = 'blob' AND length(attempt_nonce) = 32 165 THEN attempt_nonce ELSE NULL END AS attempt_nonce, 166 CASE WHEN typeof(status) = 'text' AND length(CAST(status AS BLOB)) <= 16 167 THEN status ELSE NULL END AS status, 168 leased_at_unix_ms, lease_expires_at_unix_ms, 169 submitted_at_unix_ms, typeof(submitted_at_unix_ms) AS submitted_at_type, 170 resolved_at_unix_ms, typeof(resolved_at_unix_ms) AS resolved_at_type, 171 CASE WHEN typeof(reason_code) = 'text' AND length(CAST(reason_code AS BLOB)) <= 32 172 THEN reason_code ELSE NULL END AS reason_code, 173 typeof(reason_code) AS reason_code_type 174 FROM delivery_attempts 175 WHERE job_id = ? AND target_index = ? AND attempt_nonce = ? 176 LIMIT 2"#; 177 178 const INSERT_JOB_SQL: &str = r#"INSERT INTO delivery_jobs ( 179 job_id, source_kind, source_id, artifact_sha256, policy_mode, 180 required_acknowledgements, max_attempts, initial_backoff_ms, 181 maximum_backoff_ms, attempt_deadline_ms, status, 182 created_at_unix_ms, updated_at_unix_ms, finalized_at_unix_ms 183 ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, NULL)"#; 184 185 const INSERT_TARGET_SQL: &str = r#"INSERT INTO delivery_targets ( 186 job_id, target_index, relay_id, required, attempt_count, status, 187 active_attempt_id, next_attempt_at_unix_ms, updated_at_unix_ms 188 ) VALUES (?, ?, ?, ?, 0, 'pending', NULL, NULL, ?)"#; 189 190 const INSERT_ATTEMPT_SQL: &str = r#"INSERT INTO delivery_attempts ( 191 attempt_id, job_id, target_index, attempt_number, attempt_nonce, status, 192 leased_at_unix_ms, lease_expires_at_unix_ms, submitted_at_unix_ms, 193 resolved_at_unix_ms, reason_code 194 ) VALUES (?, ?, ?, ?, ?, 'leased', ?, ?, NULL, NULL, NULL)"#; 195 196 const CLAIM_TARGET_SQL: &str = r#"UPDATE delivery_targets 197 SET attempt_count = ?, status = 'leased', active_attempt_id = ?, 198 next_attempt_at_unix_ms = NULL, updated_at_unix_ms = ? 199 WHERE job_id = ? AND target_index = ? AND attempt_count = ? 200 AND status IN ('pending', 'retryable', 'unknown') 201 AND active_attempt_id IS NULL"#; 202 203 const MARK_JOB_ACTIVE_SQL: &str = r#"UPDATE delivery_jobs 204 SET status = 'active', updated_at_unix_ms = ? 205 WHERE job_id = ? AND status = 'pending'"#; 206 207 const MARK_ATTEMPT_SUBMITTED_SQL: &str = r#"UPDATE delivery_attempts 208 SET status = 'submitted', submitted_at_unix_ms = ? 209 WHERE attempt_id = ? AND job_id = ? AND target_index = ? 210 AND status = 'leased' AND lease_expires_at_unix_ms >= ?"#; 211 212 const MARK_TARGET_SUBMITTED_SQL: &str = r#"UPDATE delivery_targets 213 SET status = 'submitted', updated_at_unix_ms = ? 214 WHERE job_id = ? AND target_index = ? AND active_attempt_id = ? AND status = 'leased'"#; 215 216 const RESOLVE_ATTEMPT_SQL: &str = r#"UPDATE delivery_attempts 217 SET status = ?, resolved_at_unix_ms = ?, reason_code = ? 218 WHERE attempt_id = ? AND job_id = ? AND target_index = ? AND status = ?"#; 219 220 const RESOLVE_TARGET_SQL: &str = r#"UPDATE delivery_targets 221 SET status = ?, active_attempt_id = NULL, next_attempt_at_unix_ms = ?, 222 updated_at_unix_ms = ? 223 WHERE job_id = ? AND target_index = ? AND active_attempt_id = ? AND status = ?"#; 224 225 const FINALIZE_JOB_SQL: &str = r#"UPDATE delivery_jobs 226 SET status = ?, updated_at_unix_ms = ?, finalized_at_unix_ms = ? 227 WHERE job_id = ? AND status IN ('pending', 'active')"#; 228 229 /// Stable source-free construction failure classes. 230 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 231 pub enum MycDeliveryStateErrorKind { 232 InvalidTime, 233 InvalidRelayId, 234 InvalidPolicy, 235 InvalidRetryJitter, 236 } 237 238 impl MycDeliveryStateErrorKind { 239 /// Returns the stable machine-readable classification. 240 #[must_use] 241 pub const fn code(self) -> &'static str { 242 match self { 243 Self::InvalidTime => "delivery_time_invalid", 244 Self::InvalidRelayId => "delivery_relay_id_invalid", 245 Self::InvalidPolicy => "delivery_policy_invalid", 246 Self::InvalidRetryJitter => "delivery_retry_jitter_invalid", 247 } 248 } 249 } 250 251 /// Source-free delivery-state construction failure. 252 #[derive(Clone, Copy, PartialEq, Eq)] 253 pub struct MycDeliveryStateError { 254 kind: MycDeliveryStateErrorKind, 255 } 256 257 impl MycDeliveryStateError { 258 const fn new(kind: MycDeliveryStateErrorKind) -> Self { 259 Self { kind } 260 } 261 262 /// Returns the stable failure class. 263 #[must_use] 264 pub const fn kind(self) -> MycDeliveryStateErrorKind { 265 self.kind 266 } 267 268 /// Returns the stable machine-readable classification. 269 #[must_use] 270 pub const fn code(self) -> &'static str { 271 self.kind.code() 272 } 273 } 274 275 impl fmt::Display for MycDeliveryStateError { 276 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 277 formatter.write_str(match self.kind { 278 MycDeliveryStateErrorKind::InvalidTime => "delivery time is invalid", 279 MycDeliveryStateErrorKind::InvalidRelayId => "delivery relay identity is invalid", 280 MycDeliveryStateErrorKind::InvalidPolicy => "delivery policy is invalid", 281 MycDeliveryStateErrorKind::InvalidRetryJitter => "delivery retry jitter is invalid", 282 }) 283 } 284 } 285 286 impl fmt::Debug for MycDeliveryStateError { 287 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 288 formatter 289 .debug_struct("MycDeliveryStateError") 290 .field("kind", &self.kind) 291 .finish() 292 } 293 } 294 295 impl Error for MycDeliveryStateError {} 296 297 /// Caller-injected full-jitter delay, later relationship-checked against a job. 298 /// 299 /// The runtime entropy adapter owns generation. State code accepts only this 300 /// bounded value and persists the exact resulting retry schedule. 301 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] 302 pub struct MycDeliveryRetryJitter(u64); 303 304 impl MycDeliveryRetryJitter { 305 /// Validates one injected full-jitter delay. 306 pub fn new(value_ms: u64) -> Result<Self, MycDeliveryStateError> { 307 if value_ms > MYC_DELIVERY_RETRY_JITTER_MAX_MS { 308 return Err(MycDeliveryStateError::new( 309 MycDeliveryStateErrorKind::InvalidRetryJitter, 310 )); 311 } 312 Ok(Self(value_ms)) 313 } 314 315 /// Returns the exact injected delay in milliseconds. 316 #[must_use] 317 pub const fn get(self) -> u64 { 318 self.0 319 } 320 } 321 322 macro_rules! redacted_id { 323 ($name:ident, $debug:literal) => { 324 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 325 pub struct $name([u8; 32]); 326 327 impl $name { 328 /// Returns the exact identity bytes. 329 #[must_use] 330 pub const fn as_bytes(&self) -> &[u8; 32] { 331 &self.0 332 } 333 } 334 335 impl fmt::Debug for $name { 336 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 337 formatter.write_str($debug) 338 } 339 } 340 }; 341 } 342 343 redacted_id!(MycDeliveryJobId, "MycDeliveryJobId([redacted])"); 344 redacted_id!(MycDeliveryAttemptId, "MycDeliveryAttemptId([redacted])"); 345 redacted_id!( 346 MycDeliveryArtifactDigest, 347 "MycDeliveryArtifactDigest([redacted])" 348 ); 349 350 impl MycDeliveryJobId { 351 pub(crate) const fn from_persisted(bytes: [u8; 32]) -> Self { 352 Self(bytes) 353 } 354 } 355 356 impl MycDeliveryArtifactDigest { 357 /// Wraps an independently verified exact-artifact SHA-256 identity. 358 #[must_use] 359 pub const fn from_bytes(bytes: [u8; 32]) -> Self { 360 Self(bytes) 361 } 362 } 363 364 /// One-use injected entropy for a new delivery attempt lease. 365 pub struct MycDeliveryAttemptNonce([u8; 32]); 366 367 impl MycDeliveryAttemptNonce { 368 /// Wraps exact entropy supplied by the caller's injected boundary. 369 #[must_use] 370 pub const fn from_injected_entropy(bytes: [u8; 32]) -> Self { 371 Self(bytes) 372 } 373 } 374 375 impl fmt::Debug for MycDeliveryAttemptNonce { 376 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 377 formatter.write_str("MycDeliveryAttemptNonce([redacted])") 378 } 379 } 380 381 /// Positive UTC millisecond evidence representable by SQLite. 382 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] 383 pub struct MycDeliveryTimeUnixMs(u64); 384 385 impl MycDeliveryTimeUnixMs { 386 /// Validates one positive UTC millisecond instant. 387 pub fn new(value: u64) -> Result<Self, MycDeliveryStateError> { 388 if value == 0 || i64::try_from(value).is_err() { 389 return Err(MycDeliveryStateError::new( 390 MycDeliveryStateErrorKind::InvalidTime, 391 )); 392 } 393 Ok(Self(value)) 394 } 395 396 /// Returns the validated instant. 397 #[must_use] 398 pub const fn get(self) -> u64 { 399 self.0 400 } 401 402 pub(crate) fn sqlite_value(self) -> i64 { 403 i64::try_from(self.0).expect("validated delivery time fits SQLite") 404 } 405 } 406 407 /// Canonical configured relay identifier used as a durable target identity. 408 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] 409 pub struct MycDeliveryRelayId(Box<str>); 410 411 impl MycDeliveryRelayId { 412 /// Validates the frozen lower-snake relay grammar before allocation. 413 pub fn new(value: &str) -> Result<Self, MycDeliveryStateError> { 414 let bytes = value.as_bytes(); 415 let valid = !bytes.is_empty() 416 && bytes.len() <= MYC_DELIVERY_RELAY_ID_MAX_BYTES 417 && bytes[0].is_ascii_lowercase() 418 && bytes[bytes.len() - 1].is_ascii_alphanumeric() 419 && bytes 420 .iter() 421 .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'_') 422 && !bytes.windows(2).any(|window| window == b"__"); 423 if !valid { 424 return Err(MycDeliveryStateError::new( 425 MycDeliveryStateErrorKind::InvalidRelayId, 426 )); 427 } 428 Ok(Self(value.into())) 429 } 430 431 /// Returns the canonical identifier. 432 #[must_use] 433 pub fn as_str(&self) -> &str { 434 &self.0 435 } 436 } 437 438 impl fmt::Debug for MycDeliveryRelayId { 439 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 440 formatter.write_str("MycDeliveryRelayId([redacted])") 441 } 442 } 443 444 /// Closed delivery-policy mode copied from normalized configuration. 445 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 446 pub enum MycDeliveryPolicyMode { 447 AtLeastOneRequired, 448 AllRequired, 449 RequiredQuorum, 450 } 451 452 impl MycDeliveryPolicyMode { 453 /// Returns the exact durable spelling. 454 #[must_use] 455 pub const fn as_str(self) -> &'static str { 456 match self { 457 Self::AtLeastOneRequired => "at_least_one_required", 458 Self::AllRequired => "all_required", 459 Self::RequiredQuorum => "required_quorum", 460 } 461 } 462 463 pub(crate) fn parse(value: &str) -> Option<Self> { 464 match value { 465 "at_least_one_required" => Some(Self::AtLeastOneRequired), 466 "all_required" => Some(Self::AllRequired), 467 "required_quorum" => Some(Self::RequiredQuorum), 468 _ => None, 469 } 470 } 471 } 472 473 /// Immutable configured target identity. 474 #[derive(Clone, PartialEq, Eq)] 475 pub(crate) struct MycDeliveryTargetPolicy { 476 relay_id: MycDeliveryRelayId, 477 required: bool, 478 } 479 480 /// Immutable configured delivery and retry authority. 481 #[derive(Clone, PartialEq, Eq)] 482 pub(crate) struct MycDeliveryPolicies { 483 mode: MycDeliveryPolicyMode, 484 required_acknowledgements: u32, 485 max_attempts: u32, 486 initial_backoff_ms: u64, 487 maximum_backoff_ms: u64, 488 attempt_deadline_ms: u64, 489 outbox_maximum: usize, 490 targets: Box<[MycDeliveryTargetPolicy]>, 491 } 492 493 impl MycDeliveryPolicies { 494 #[allow(clippy::too_many_arguments)] 495 pub(crate) fn new( 496 mode: MycDeliveryPolicyMode, 497 configured_quorum: Option<u32>, 498 max_attempts: u32, 499 initial_backoff_ms: u64, 500 maximum_backoff_ms: u64, 501 attempt_deadline_ms: u64, 502 outbox_maximum: usize, 503 mut targets: Vec<(MycDeliveryRelayId, bool)>, 504 ) -> Result<Self, MycDeliveryStateError> { 505 targets.sort_by(|left, right| left.0.cmp(&right.0)); 506 let required_count = targets.iter().filter(|(_, required)| *required).count(); 507 let required_acknowledgements = match mode { 508 MycDeliveryPolicyMode::AtLeastOneRequired => 1, 509 MycDeliveryPolicyMode::AllRequired => u32::try_from(required_count).unwrap_or(u32::MAX), 510 MycDeliveryPolicyMode::RequiredQuorum => configured_quorum.unwrap_or(0), 511 }; 512 let valid = !targets.is_empty() 513 && targets.len() <= MYC_DELIVERY_TARGET_MAX_COUNT 514 && !targets.windows(2).any(|window| window[0].0 == window[1].0) 515 && required_acknowledgements != 0 516 && usize::try_from(required_acknowledgements) 517 .is_ok_and(|required| required <= targets.len()) 518 && usize::try_from(required_acknowledgements) 519 .is_ok_and(|required| required <= required_count) 520 && (1..=MYC_DELIVERY_ATTEMPT_MAX_COUNT).contains(&max_attempts) 521 && initial_backoff_ms != 0 522 && initial_backoff_ms <= maximum_backoff_ms 523 && maximum_backoff_ms <= 300_000 524 && attempt_deadline_ms != 0 525 && attempt_deadline_ms <= 30_000 526 && (1..=65_536).contains(&outbox_maximum); 527 if !valid { 528 return Err(MycDeliveryStateError::new( 529 MycDeliveryStateErrorKind::InvalidPolicy, 530 )); 531 } 532 Ok(Self { 533 mode, 534 required_acknowledgements, 535 max_attempts, 536 initial_backoff_ms, 537 maximum_backoff_ms, 538 attempt_deadline_ms, 539 outbox_maximum, 540 targets: targets 541 .into_iter() 542 .map(|(relay_id, required)| MycDeliveryTargetPolicy { relay_id, required }) 543 .collect(), 544 }) 545 } 546 547 pub(crate) const fn outbox_maximum(&self) -> usize { 548 self.outbox_maximum 549 } 550 } 551 552 impl fmt::Debug for MycDeliveryPolicies { 553 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 554 formatter 555 .debug_struct("MycDeliveryPolicies") 556 .field("mode", &self.mode) 557 .field("target_count", &self.targets.len()) 558 .finish_non_exhaustive() 559 } 560 } 561 562 /// Closed authority that produced the exact bytes retained by a delivery job. 563 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 564 pub enum MycDeliverySourceKind { 565 SignerResponse, 566 DiscoveryHandler, 567 } 568 569 impl MycDeliverySourceKind { 570 /// Returns the exact durable spelling. 571 #[must_use] 572 pub const fn as_str(self) -> &'static str { 573 match self { 574 Self::SignerResponse => "signer_response", 575 Self::DiscoveryHandler => "discovery_handler", 576 } 577 } 578 579 pub(crate) fn parse(value: &str) -> Option<Self> { 580 match value { 581 "signer_response" => Some(Self::SignerResponse), 582 "discovery_handler" => Some(Self::DiscoveryHandler), 583 _ => None, 584 } 585 } 586 } 587 588 #[derive(Clone, Copy, PartialEq, Eq)] 589 pub(crate) struct MycDeliverySource { 590 kind: MycDeliverySourceKind, 591 id: [u8; 32], 592 } 593 594 impl MycDeliverySource { 595 pub(crate) const fn signer_response(operation_id: MycSignerOperationId) -> Self { 596 Self { 597 kind: MycDeliverySourceKind::SignerResponse, 598 id: *operation_id.as_bytes(), 599 } 600 } 601 602 pub(crate) const fn discovery_handler(generation_id: [u8; 32]) -> Self { 603 Self { 604 kind: MycDeliverySourceKind::DiscoveryHandler, 605 id: generation_id, 606 } 607 } 608 609 pub(crate) const fn kind(self) -> MycDeliverySourceKind { 610 self.kind 611 } 612 613 pub(crate) const fn id(self) -> [u8; 32] { 614 self.id 615 } 616 } 617 618 impl fmt::Debug for MycDeliverySource { 619 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 620 formatter 621 .debug_struct("MycDeliverySource") 622 .field("kind", &self.kind) 623 .field("id", &"[redacted]") 624 .finish() 625 } 626 } 627 628 /// Durable job state. 629 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 630 pub enum MycDeliveryJobStatus { 631 Pending, 632 Active, 633 Delivered, 634 Failed, 635 Unknown, 636 } 637 638 impl MycDeliveryJobStatus { 639 /// Returns the exact durable spelling. 640 #[must_use] 641 pub const fn as_str(self) -> &'static str { 642 match self { 643 Self::Pending => "pending", 644 Self::Active => "active", 645 Self::Delivered => "delivered", 646 Self::Failed => "failed", 647 Self::Unknown => "unknown", 648 } 649 } 650 651 pub(crate) fn parse(value: &str) -> Option<Self> { 652 match value { 653 "pending" => Some(Self::Pending), 654 "active" => Some(Self::Active), 655 "delivered" => Some(Self::Delivered), 656 "failed" => Some(Self::Failed), 657 "unknown" => Some(Self::Unknown), 658 _ => None, 659 } 660 } 661 } 662 663 /// Durable per-target state. 664 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 665 pub enum MycDeliveryTargetStatus { 666 Pending, 667 Leased, 668 Submitted, 669 Delivered, 670 Retryable, 671 Unknown, 672 Exhausted, 673 } 674 675 impl MycDeliveryTargetStatus { 676 /// Returns the exact durable spelling. 677 #[must_use] 678 pub const fn as_str(self) -> &'static str { 679 match self { 680 Self::Pending => "pending", 681 Self::Leased => "leased", 682 Self::Submitted => "submitted", 683 Self::Delivered => "delivered", 684 Self::Retryable => "retryable", 685 Self::Unknown => "unknown", 686 Self::Exhausted => "exhausted", 687 } 688 } 689 690 fn parse(value: &str) -> Option<Self> { 691 match value { 692 "pending" => Some(Self::Pending), 693 "leased" => Some(Self::Leased), 694 "submitted" => Some(Self::Submitted), 695 "delivered" => Some(Self::Delivered), 696 "retryable" => Some(Self::Retryable), 697 "unknown" => Some(Self::Unknown), 698 "exhausted" => Some(Self::Exhausted), 699 _ => None, 700 } 701 } 702 } 703 704 /// Durable attempt state; `Unknown` is distinct from proof of failure. 705 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 706 pub enum MycDeliveryAttemptStatus { 707 Leased, 708 Submitted, 709 Delivered, 710 Failed, 711 Unknown, 712 } 713 714 impl MycDeliveryAttemptStatus { 715 /// Returns the exact durable spelling. 716 #[must_use] 717 pub const fn as_str(self) -> &'static str { 718 match self { 719 Self::Leased => "leased", 720 Self::Submitted => "submitted", 721 Self::Delivered => "delivered", 722 Self::Failed => "failed", 723 Self::Unknown => "unknown", 724 } 725 } 726 727 fn parse(value: &str) -> Option<Self> { 728 match value { 729 "leased" => Some(Self::Leased), 730 "submitted" => Some(Self::Submitted), 731 "delivered" => Some(Self::Delivered), 732 "failed" => Some(Self::Failed), 733 "unknown" => Some(Self::Unknown), 734 _ => None, 735 } 736 } 737 } 738 739 /// Closed evidence for a terminal attempt observation. 740 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 741 pub enum MycDeliveryAttemptOutcome { 742 Delivered, 743 RelayRejected, 744 TransportFailed, 745 UnknownAcknowledgement, 746 } 747 748 impl MycDeliveryAttemptOutcome { 749 const fn status(self) -> MycDeliveryAttemptStatus { 750 match self { 751 Self::Delivered => MycDeliveryAttemptStatus::Delivered, 752 Self::RelayRejected | Self::TransportFailed => MycDeliveryAttemptStatus::Failed, 753 Self::UnknownAcknowledgement => MycDeliveryAttemptStatus::Unknown, 754 } 755 } 756 757 const fn reason(self) -> &'static str { 758 match self { 759 Self::Delivered => "accepted", 760 Self::RelayRejected => "relay_rejected", 761 Self::TransportFailed => "transport_failed", 762 Self::UnknownAcknowledgement => "acknowledgement_lost", 763 } 764 } 765 } 766 767 /// Immutable summary of one retained delivery job. 768 #[derive(Clone, PartialEq, Eq)] 769 pub struct MycDeliveryJobRecord { 770 id: MycDeliveryJobId, 771 source: MycDeliverySource, 772 artifact_digest: MycDeliveryArtifactDigest, 773 policy_mode: MycDeliveryPolicyMode, 774 required_acknowledgements: u32, 775 max_attempts: u32, 776 initial_backoff_ms: u64, 777 maximum_backoff_ms: u64, 778 attempt_deadline_ms: u64, 779 status: MycDeliveryJobStatus, 780 created_at: MycDeliveryTimeUnixMs, 781 updated_at: MycDeliveryTimeUnixMs, 782 finalized_at: Option<MycDeliveryTimeUnixMs>, 783 targets: Box<[MycDeliveryTargetRecord]>, 784 } 785 786 impl MycDeliveryJobRecord { 787 pub(crate) const fn source_id(&self) -> &[u8; 32] { 788 &self.source.id 789 } 790 791 #[must_use] 792 pub const fn id(&self) -> MycDeliveryJobId { 793 self.id 794 } 795 #[must_use] 796 pub const fn operation_id(&self) -> Option<MycSignerOperationId> { 797 match self.source.kind { 798 MycDeliverySourceKind::SignerResponse => { 799 Some(MycSignerOperationId::from_persisted(self.source.id)) 800 } 801 MycDeliverySourceKind::DiscoveryHandler => None, 802 } 803 } 804 #[must_use] 805 pub const fn source_kind(&self) -> MycDeliverySourceKind { 806 self.source.kind 807 } 808 #[must_use] 809 pub const fn artifact_digest(&self) -> MycDeliveryArtifactDigest { 810 self.artifact_digest 811 } 812 #[must_use] 813 pub const fn policy_mode(&self) -> MycDeliveryPolicyMode { 814 self.policy_mode 815 } 816 #[must_use] 817 pub const fn required_acknowledgements(&self) -> u32 { 818 self.required_acknowledgements 819 } 820 #[must_use] 821 pub const fn max_attempts(&self) -> u32 { 822 self.max_attempts 823 } 824 #[must_use] 825 pub const fn initial_backoff_ms(&self) -> u64 { 826 self.initial_backoff_ms 827 } 828 #[must_use] 829 pub const fn maximum_backoff_ms(&self) -> u64 { 830 self.maximum_backoff_ms 831 } 832 #[must_use] 833 pub const fn attempt_deadline_ms(&self) -> u64 { 834 self.attempt_deadline_ms 835 } 836 #[must_use] 837 pub const fn status(&self) -> MycDeliveryJobStatus { 838 self.status 839 } 840 #[must_use] 841 pub const fn targets(&self) -> &[MycDeliveryTargetRecord] { 842 &self.targets 843 } 844 #[must_use] 845 pub const fn created_at(&self) -> MycDeliveryTimeUnixMs { 846 self.created_at 847 } 848 #[must_use] 849 pub const fn updated_at(&self) -> MycDeliveryTimeUnixMs { 850 self.updated_at 851 } 852 #[must_use] 853 pub const fn finalized_at(&self) -> Option<MycDeliveryTimeUnixMs> { 854 self.finalized_at 855 } 856 } 857 858 impl fmt::Debug for MycDeliveryJobRecord { 859 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 860 formatter 861 .debug_struct("MycDeliveryJobRecord") 862 .field("source_kind", &self.source.kind) 863 .field("status", &self.status) 864 .field("target_count", &self.targets.len()) 865 .finish() 866 } 867 } 868 869 /// Immutable summary of one configured target and its current state. 870 #[derive(Clone, PartialEq, Eq)] 871 pub struct MycDeliveryTargetRecord { 872 index: u32, 873 relay_id: MycDeliveryRelayId, 874 required: bool, 875 attempt_count: u32, 876 status: MycDeliveryTargetStatus, 877 active_attempt_id: Option<MycDeliveryAttemptId>, 878 next_attempt_at: Option<MycDeliveryTimeUnixMs>, 879 updated_at: MycDeliveryTimeUnixMs, 880 } 881 882 impl MycDeliveryTargetRecord { 883 #[must_use] 884 pub const fn index(&self) -> u32 { 885 self.index 886 } 887 #[must_use] 888 pub const fn relay_id(&self) -> &MycDeliveryRelayId { 889 &self.relay_id 890 } 891 #[must_use] 892 pub const fn required(&self) -> bool { 893 self.required 894 } 895 #[must_use] 896 pub const fn attempt_count(&self) -> u32 { 897 self.attempt_count 898 } 899 #[must_use] 900 pub const fn status(&self) -> MycDeliveryTargetStatus { 901 self.status 902 } 903 #[must_use] 904 pub const fn active_attempt_id(&self) -> Option<MycDeliveryAttemptId> { 905 self.active_attempt_id 906 } 907 #[must_use] 908 pub const fn next_attempt_at(&self) -> Option<MycDeliveryTimeUnixMs> { 909 self.next_attempt_at 910 } 911 #[must_use] 912 pub const fn updated_at(&self) -> MycDeliveryTimeUnixMs { 913 self.updated_at 914 } 915 } 916 917 impl fmt::Debug for MycDeliveryTargetRecord { 918 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 919 formatter 920 .debug_struct("MycDeliveryTargetRecord") 921 .field("required", &self.required) 922 .field("attempt_count", &self.attempt_count) 923 .field("status", &self.status) 924 .finish() 925 } 926 } 927 928 /// Immutable identity plus append-only state for one bounded delivery attempt. 929 #[derive(Clone, PartialEq, Eq)] 930 pub struct MycDeliveryAttemptRecord { 931 id: MycDeliveryAttemptId, 932 number: u32, 933 status: MycDeliveryAttemptStatus, 934 leased_at: MycDeliveryTimeUnixMs, 935 lease_expires_at: MycDeliveryTimeUnixMs, 936 submitted_at: Option<MycDeliveryTimeUnixMs>, 937 resolved_at: Option<MycDeliveryTimeUnixMs>, 938 reason: Option<&'static str>, 939 } 940 941 impl MycDeliveryAttemptRecord { 942 #[must_use] 943 pub const fn id(&self) -> MycDeliveryAttemptId { 944 self.id 945 } 946 #[must_use] 947 pub const fn number(&self) -> u32 { 948 self.number 949 } 950 #[must_use] 951 pub const fn status(&self) -> MycDeliveryAttemptStatus { 952 self.status 953 } 954 #[must_use] 955 pub const fn leased_at(&self) -> MycDeliveryTimeUnixMs { 956 self.leased_at 957 } 958 #[must_use] 959 pub const fn lease_expires_at(&self) -> MycDeliveryTimeUnixMs { 960 self.lease_expires_at 961 } 962 #[must_use] 963 pub const fn submitted_at(&self) -> Option<MycDeliveryTimeUnixMs> { 964 self.submitted_at 965 } 966 #[must_use] 967 pub const fn resolved_at(&self) -> Option<MycDeliveryTimeUnixMs> { 968 self.resolved_at 969 } 970 #[must_use] 971 pub const fn reason(&self) -> Option<&'static str> { 972 self.reason 973 } 974 } 975 976 impl fmt::Debug for MycDeliveryAttemptRecord { 977 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 978 formatter 979 .debug_struct("MycDeliveryAttemptRecord") 980 .field("number", &self.number) 981 .field("status", &self.status) 982 .finish() 983 } 984 } 985 986 /// New or idempotently replayed delivery-job creation. 987 #[derive(Clone, PartialEq, Eq)] 988 pub enum MycDeliveryJobAdmission { 989 Created(MycDeliveryJobRecord), 990 ExactReplay(MycDeliveryJobRecord), 991 } 992 993 impl MycDeliveryJobAdmission { 994 #[must_use] 995 pub const fn record(&self) -> &MycDeliveryJobRecord { 996 match self { 997 Self::Created(record) | Self::ExactReplay(record) => record, 998 } 999 } 1000 } 1001 1002 impl fmt::Debug for MycDeliveryJobAdmission { 1003 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 1004 formatter.write_str(match self { 1005 Self::Created(_) => "MycDeliveryJobAdmission::Created([redacted])", 1006 Self::ExactReplay(_) => "MycDeliveryJobAdmission::ExactReplay([redacted])", 1007 }) 1008 } 1009 } 1010 1011 /// Result of attempting to claim one target. 1012 #[derive(Clone, PartialEq, Eq)] 1013 pub enum MycDeliveryClaim { 1014 Claimed(MycDeliveryAttemptRecord), 1015 ExactReplay(MycDeliveryAttemptRecord), 1016 NotReady, 1017 Terminal, 1018 } 1019 1020 impl fmt::Debug for MycDeliveryClaim { 1021 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { 1022 formatter.write_str(match self { 1023 Self::Claimed(_) => "MycDeliveryClaim::Claimed([redacted])", 1024 Self::ExactReplay(_) => "MycDeliveryClaim::ExactReplay([redacted])", 1025 Self::NotReady => "MycDeliveryClaim::NotReady", 1026 Self::Terminal => "MycDeliveryClaim::Terminal", 1027 }) 1028 } 1029 } 1030 1031 impl MycStateRepository<'_> { 1032 pub(crate) async fn read_runtime_outbox_status( 1033 &self, 1034 ) -> Result<crate::MycOutboxStatusV1, MycStateRepositoryError> { 1035 let expected = PersistedMetadata::from(self.expected()); 1036 self.host() 1037 .transaction(move |transaction| { 1038 Box::pin(async move { 1039 verify_metadata(transaction, &expected).await?; 1040 let rows = sqlx::query(READ_RUNTIME_OUTBOX_STATUS_SQL) 1041 .fetch_all(&mut *transaction) 1042 .await 1043 .map_err(|_| DeliveryOperationError::Storage)?; 1044 let [row] = rows.as_slice() else { 1045 return Err(DeliveryOperationError::Binding); 1046 }; 1047 let count = |column| { 1048 row.try_get::<i64, _>(column) 1049 .ok() 1050 .and_then(|value| u64::try_from(value).ok()) 1051 .ok_or(DeliveryOperationError::Binding) 1052 }; 1053 let oldest = match row 1054 .try_get::<&str, _>("oldest_pending_type") 1055 .map_err(|_| DeliveryOperationError::Binding)? 1056 { 1057 "null" => None, 1058 "integer" => { 1059 let milliseconds = count("oldest_pending_at_unix_ms")?; 1060 Some( 1061 crate::MycStatusUnixSeconds::new(milliseconds / 1_000) 1062 .map_err(|_| DeliveryOperationError::Binding)?, 1063 ) 1064 } 1065 _ => return Err(DeliveryOperationError::Binding), 1066 }; 1067 Ok(crate::MycOutboxStatusV1::new( 1068 count("pending_count")?, 1069 count("unknown_count")?, 1070 oldest, 1071 )) 1072 }) 1073 }) 1074 .await 1075 .map_err(map_transaction_error) 1076 } 1077 1078 /// Returns the first exact target eligible for bounded delivery work. 1079 /// 1080 /// Selection is deterministic and performs no claim or network I/O. The 1081 /// subsequent claim transaction remains the sole lease authority. 1082 pub(crate) async fn next_ready_delivery_target( 1083 &self, 1084 observed_at: MycDeliveryTimeUnixMs, 1085 ) -> Result<Option<(MycDeliveryJobId, MycDeliveryRelayId)>, MycStateRepositoryError> { 1086 let expected = PersistedMetadata::from(self.expected()); 1087 self.host() 1088 .transaction(move |transaction| { 1089 Box::pin(async move { 1090 verify_metadata(transaction, &expected).await?; 1091 let rows = sqlx::query(READ_NEXT_READY_TARGET_SQL) 1092 .bind(observed_at.sqlite_value()) 1093 .fetch_all(&mut *transaction) 1094 .await 1095 .map_err(|_| DeliveryOperationError::Storage)?; 1096 match rows.as_slice() { 1097 [] => Ok(None), 1098 [row] => { 1099 let job_id = MycDeliveryJobId::from_persisted(blob32(row, "job_id")?); 1100 let relay_id = MycDeliveryRelayId::new(text(row, "relay_id")?) 1101 .map_err(|_| DeliveryOperationError::Binding)?; 1102 Ok(Some((job_id, relay_id))) 1103 } 1104 _ => Err(DeliveryOperationError::Binding), 1105 } 1106 }) 1107 }) 1108 .await 1109 .map_err(map_transaction_error) 1110 } 1111 1112 /// Claims one eligible target under a bounded expiring attempt lease. 1113 pub async fn claim_delivery_target( 1114 &self, 1115 job_id: MycDeliveryJobId, 1116 relay_id: &MycDeliveryRelayId, 1117 nonce: MycDeliveryAttemptNonce, 1118 claimed_at: MycDeliveryTimeUnixMs, 1119 ) -> Result<MycDeliveryClaim, MycStateRepositoryError> { 1120 let relay_id = relay_id.clone(); 1121 let expected = PersistedMetadata::from(self.expected()); 1122 self.host() 1123 .transaction(move |transaction| { 1124 Box::pin(async move { 1125 verify_metadata(transaction, &expected).await?; 1126 claim_target(transaction, job_id, &relay_id, &nonce, claimed_at).await 1127 }) 1128 }) 1129 .await 1130 .map_err(map_transaction_error) 1131 } 1132 1133 /// Records that the exact leased attempt reached the external submission boundary. 1134 pub async fn mark_delivery_attempt_submitted( 1135 &self, 1136 job_id: MycDeliveryJobId, 1137 relay_id: &MycDeliveryRelayId, 1138 attempt_id: MycDeliveryAttemptId, 1139 submitted_at: MycDeliveryTimeUnixMs, 1140 ) -> Result<MycDeliveryAttemptRecord, MycStateRepositoryError> { 1141 let relay_id = relay_id.clone(); 1142 let expected = PersistedMetadata::from(self.expected()); 1143 self.host() 1144 .transaction(move |transaction| { 1145 Box::pin(async move { 1146 verify_metadata(transaction, &expected).await?; 1147 mark_submitted(transaction, job_id, &relay_id, attempt_id, submitted_at).await 1148 }) 1149 }) 1150 .await 1151 .map_err(map_transaction_error) 1152 } 1153 1154 /// Persists delivered, proven-failed, or unknown acknowledgement evidence. 1155 pub async fn record_delivery_attempt_outcome( 1156 &self, 1157 job_id: MycDeliveryJobId, 1158 relay_id: &MycDeliveryRelayId, 1159 attempt_id: MycDeliveryAttemptId, 1160 outcome: MycDeliveryAttemptOutcome, 1161 retry_jitter: MycDeliveryRetryJitter, 1162 observed_at: MycDeliveryTimeUnixMs, 1163 ) -> Result<MycDeliveryJobRecord, MycStateRepositoryError> { 1164 let relay_id = relay_id.clone(); 1165 let expected = PersistedMetadata::from(self.expected()); 1166 self.host() 1167 .transaction(move |transaction| { 1168 Box::pin(async move { 1169 verify_metadata(transaction, &expected).await?; 1170 record_outcome( 1171 transaction, 1172 job_id, 1173 &relay_id, 1174 attempt_id, 1175 outcome, 1176 retry_jitter, 1177 observed_at, 1178 ) 1179 .await 1180 }) 1181 }) 1182 .await 1183 .map_err(map_transaction_error) 1184 } 1185 1186 /// Converts an expired pre-submit lease to failure or a submitted lease to unknown. 1187 pub async fn recover_expired_delivery_lease( 1188 &self, 1189 job_id: MycDeliveryJobId, 1190 relay_id: &MycDeliveryRelayId, 1191 attempt_id: MycDeliveryAttemptId, 1192 retry_jitter: MycDeliveryRetryJitter, 1193 observed_at: MycDeliveryTimeUnixMs, 1194 ) -> Result<MycDeliveryJobRecord, MycStateRepositoryError> { 1195 let relay_id = relay_id.clone(); 1196 let expected = PersistedMetadata::from(self.expected()); 1197 self.host() 1198 .transaction(move |transaction| { 1199 Box::pin(async move { 1200 verify_metadata(transaction, &expected).await?; 1201 recover_expired( 1202 transaction, 1203 job_id, 1204 &relay_id, 1205 attempt_id, 1206 retry_jitter, 1207 observed_at, 1208 ) 1209 .await 1210 }) 1211 }) 1212 .await 1213 .map_err(map_transaction_error) 1214 } 1215 1216 /// Reads one bounded job snapshot and its immutable target set. 1217 pub async fn read_delivery_job( 1218 &self, 1219 job_id: MycDeliveryJobId, 1220 ) -> Result<Option<MycDeliveryJobRecord>, MycStateRepositoryError> { 1221 let expected = PersistedMetadata::from(self.expected()); 1222 self.host() 1223 .transaction(move |transaction| { 1224 Box::pin(async move { 1225 verify_metadata(transaction, &expected).await?; 1226 read_job(transaction, job_id).await 1227 }) 1228 }) 1229 .await 1230 .map_err(map_transaction_error) 1231 } 1232 1233 /// Reads at most the configured 32 attempts for one retained target. 1234 pub async fn read_delivery_attempts( 1235 &self, 1236 job_id: MycDeliveryJobId, 1237 relay_id: &MycDeliveryRelayId, 1238 ) -> Result<Box<[MycDeliveryAttemptRecord]>, MycStateRepositoryError> { 1239 let relay_id = relay_id.clone(); 1240 let expected = PersistedMetadata::from(self.expected()); 1241 self.host() 1242 .transaction(move |transaction| { 1243 Box::pin(async move { 1244 verify_metadata(transaction, &expected).await?; 1245 let job = read_job(transaction, job_id) 1246 .await? 1247 .ok_or(DeliveryOperationError::Binding)?; 1248 let target = target_by_relay(&job, &relay_id)?; 1249 read_attempts(transaction, job_id, target.index).await 1250 }) 1251 }) 1252 .await 1253 .map_err(map_transaction_error) 1254 } 1255 } 1256 1257 #[derive(Clone, Copy, Debug, PartialEq, Eq)] 1258 pub(crate) enum DeliveryOperationError { 1259 Binding, 1260 Storage, 1261 } 1262 1263 async fn verify_metadata( 1264 transaction: &mut ServiceSqliteTransaction<'_>, 1265 expected: &PersistedMetadata, 1266 ) -> Result<(), DeliveryOperationError> { 1267 require_expected_metadata(transaction, expected) 1268 .await 1269 .map_err(|error| match error { 1270 RepositoryOperationError::Binding => DeliveryOperationError::Binding, 1271 RepositoryOperationError::Storage => DeliveryOperationError::Storage, 1272 }) 1273 } 1274 1275 pub(crate) async fn create_job( 1276 transaction: &mut ServiceSqliteTransaction<'_>, 1277 source: MycDeliverySource, 1278 artifact_digest: MycDeliveryArtifactDigest, 1279 created_at: MycDeliveryTimeUnixMs, 1280 policy: &MycDeliveryPolicies, 1281 ) -> Result<MycDeliveryJobAdmission, DeliveryOperationError> { 1282 if let Some(existing) = read_job_by_source(transaction, source).await? { 1283 return exact_job(&existing, source, artifact_digest, created_at, policy) 1284 .then_some(MycDeliveryJobAdmission::ExactReplay(existing)) 1285 .ok_or(DeliveryOperationError::Binding); 1286 } 1287 let active_jobs = sqlx::query(READ_ACTIVE_JOB_COUNT_SQL) 1288 .fetch_one(&mut *transaction) 1289 .await 1290 .map_err(|_| DeliveryOperationError::Storage)? 1291 .try_get::<i64, _>("row_count") 1292 .map_err(|_| DeliveryOperationError::Binding)?; 1293 if usize::try_from(active_jobs) 1294 .ok() 1295 .is_none_or(|count| count >= policy.outbox_maximum) 1296 { 1297 return Err(DeliveryOperationError::Binding); 1298 } 1299 let job_id = derive_job_id(source, artifact_digest); 1300 let result = sqlx::query(INSERT_JOB_SQL) 1301 .bind(job_id.as_bytes().as_slice()) 1302 .bind(source.kind().as_str()) 1303 .bind(source.id().as_slice()) 1304 .bind(artifact_digest.as_bytes().as_slice()) 1305 .bind(policy.mode.as_str()) 1306 .bind(i64::from(policy.required_acknowledgements)) 1307 .bind(i64::from(policy.max_attempts)) 1308 .bind( 1309 i64::try_from(policy.initial_backoff_ms) 1310 .map_err(|_| DeliveryOperationError::Binding)?, 1311 ) 1312 .bind( 1313 i64::try_from(policy.maximum_backoff_ms) 1314 .map_err(|_| DeliveryOperationError::Binding)?, 1315 ) 1316 .bind( 1317 i64::try_from(policy.attempt_deadline_ms) 1318 .map_err(|_| DeliveryOperationError::Binding)?, 1319 ) 1320 .bind(created_at.sqlite_value()) 1321 .bind(created_at.sqlite_value()) 1322 .execute(&mut *transaction) 1323 .await 1324 .map_err(|_| DeliveryOperationError::Storage)?; 1325 require_one(result.rows_affected())?; 1326 for (index, target) in policy.targets.iter().enumerate() { 1327 let index = u32::try_from(index).map_err(|_| DeliveryOperationError::Binding)?; 1328 let result = sqlx::query(INSERT_TARGET_SQL) 1329 .bind(job_id.as_bytes().as_slice()) 1330 .bind(i64::from(index)) 1331 .bind(target.relay_id.as_str()) 1332 .bind(target.required) 1333 .bind(created_at.sqlite_value()) 1334 .execute(&mut *transaction) 1335 .await 1336 .map_err(|_| DeliveryOperationError::Storage)?; 1337 require_one(result.rows_affected())?; 1338 } 1339 let record = read_job(transaction, job_id) 1340 .await? 1341 .ok_or(DeliveryOperationError::Binding)?; 1342 Ok(MycDeliveryJobAdmission::Created(record)) 1343 } 1344 1345 async fn claim_target( 1346 transaction: &mut ServiceSqliteTransaction<'_>, 1347 job_id: MycDeliveryJobId, 1348 relay_id: &MycDeliveryRelayId, 1349 nonce: &MycDeliveryAttemptNonce, 1350 claimed_at: MycDeliveryTimeUnixMs, 1351 ) -> Result<MycDeliveryClaim, DeliveryOperationError> { 1352 let job = read_job(transaction, job_id) 1353 .await? 1354 .ok_or(DeliveryOperationError::Binding)?; 1355 if matches!( 1356 job.status, 1357 MycDeliveryJobStatus::Delivered 1358 | MycDeliveryJobStatus::Failed 1359 | MycDeliveryJobStatus::Unknown 1360 ) { 1361 return Ok(MycDeliveryClaim::Terminal); 1362 } 1363 let target = target_by_relay(&job, relay_id)?.clone(); 1364 if let Some(existing) = read_attempt_by_nonce(transaction, job_id, target.index, nonce).await? { 1365 return Ok(MycDeliveryClaim::ExactReplay(existing)); 1366 } 1367 if target.active_attempt_id.is_some() 1368 || target.next_attempt_at.is_some_and(|next| next > claimed_at) 1369 { 1370 return Ok(MycDeliveryClaim::NotReady); 1371 } 1372 if matches!( 1373 target.status, 1374 MycDeliveryTargetStatus::Delivered | MycDeliveryTargetStatus::Exhausted 1375 ) || target.attempt_count >= job.max_attempts 1376 { 1377 return Ok(MycDeliveryClaim::Terminal); 1378 } 1379 let attempt_number = target.attempt_count + 1; 1380 let lease_expires_value = claimed_at 1381 .get() 1382 .checked_add(job.attempt_deadline_ms) 1383 .ok_or(DeliveryOperationError::Binding)?; 1384 let lease_expires = MycDeliveryTimeUnixMs::new(lease_expires_value) 1385 .map_err(|_| DeliveryOperationError::Binding)?; 1386 let attempt_id = derive_attempt_id(job_id, target.index, attempt_number, nonce); 1387 let result = sqlx::query(INSERT_ATTEMPT_SQL) 1388 .bind(attempt_id.as_bytes().as_slice()) 1389 .bind(job_id.as_bytes().as_slice()) 1390 .bind(i64::from(target.index)) 1391 .bind(i64::from(attempt_number)) 1392 .bind(nonce.0.as_slice()) 1393 .bind(claimed_at.sqlite_value()) 1394 .bind(lease_expires.sqlite_value()) 1395 .execute(&mut *transaction) 1396 .await 1397 .map_err(|_| DeliveryOperationError::Storage)?; 1398 require_one(result.rows_affected())?; 1399 let result = sqlx::query(CLAIM_TARGET_SQL) 1400 .bind(i64::from(attempt_number)) 1401 .bind(attempt_id.as_bytes().as_slice()) 1402 .bind(claimed_at.sqlite_value()) 1403 .bind(job_id.as_bytes().as_slice()) 1404 .bind(i64::from(target.index)) 1405 .bind(i64::from(target.attempt_count)) 1406 .execute(&mut *transaction) 1407 .await 1408 .map_err(|_| DeliveryOperationError::Storage)?; 1409 require_one(result.rows_affected())?; 1410 let result = sqlx::query(MARK_JOB_ACTIVE_SQL) 1411 .bind(claimed_at.sqlite_value()) 1412 .bind(job_id.as_bytes().as_slice()) 1413 .execute(&mut *transaction) 1414 .await 1415 .map_err(|_| DeliveryOperationError::Storage)?; 1416 if result.rows_affected() > 1 { 1417 return Err(DeliveryOperationError::Storage); 1418 } 1419 let attempt = read_attempt(transaction, job_id, target.index, attempt_id) 1420 .await? 1421 .ok_or(DeliveryOperationError::Binding)?; 1422 Ok(MycDeliveryClaim::Claimed(attempt)) 1423 } 1424 1425 async fn mark_submitted( 1426 transaction: &mut ServiceSqliteTransaction<'_>, 1427 job_id: MycDeliveryJobId, 1428 relay_id: &MycDeliveryRelayId, 1429 attempt_id: MycDeliveryAttemptId, 1430 submitted_at: MycDeliveryTimeUnixMs, 1431 ) -> Result<MycDeliveryAttemptRecord, DeliveryOperationError> { 1432 let job = read_job(transaction, job_id) 1433 .await? 1434 .ok_or(DeliveryOperationError::Binding)?; 1435 let target = target_by_relay(&job, relay_id)?; 1436 let attempt = read_attempt(transaction, job_id, target.index, attempt_id) 1437 .await? 1438 .ok_or(DeliveryOperationError::Binding)?; 1439 if attempt.status == MycDeliveryAttemptStatus::Submitted { 1440 return (attempt.submitted_at == Some(submitted_at)) 1441 .then_some(attempt) 1442 .ok_or(DeliveryOperationError::Binding); 1443 } 1444 if attempt.status != MycDeliveryAttemptStatus::Leased 1445 || submitted_at < attempt.leased_at 1446 || submitted_at > attempt.lease_expires_at 1447 || target.active_attempt_id != Some(attempt_id) 1448 || target.status != MycDeliveryTargetStatus::Leased 1449 { 1450 return Err(DeliveryOperationError::Binding); 1451 } 1452 let result = sqlx::query(MARK_ATTEMPT_SUBMITTED_SQL) 1453 .bind(submitted_at.sqlite_value()) 1454 .bind(attempt_id.as_bytes().as_slice()) 1455 .bind(job_id.as_bytes().as_slice()) 1456 .bind(i64::from(target.index)) 1457 .bind(submitted_at.sqlite_value()) 1458 .execute(&mut *transaction) 1459 .await 1460 .map_err(|_| DeliveryOperationError::Storage)?; 1461 require_one(result.rows_affected())?; 1462 let result = sqlx::query(MARK_TARGET_SUBMITTED_SQL) 1463 .bind(submitted_at.sqlite_value()) 1464 .bind(job_id.as_bytes().as_slice()) 1465 .bind(i64::from(target.index)) 1466 .bind(attempt_id.as_bytes().as_slice()) 1467 .execute(&mut *transaction) 1468 .await 1469 .map_err(|_| DeliveryOperationError::Storage)?; 1470 require_one(result.rows_affected())?; 1471 read_attempt(transaction, job_id, target.index, attempt_id) 1472 .await? 1473 .ok_or(DeliveryOperationError::Binding) 1474 } 1475 1476 async fn record_outcome( 1477 transaction: &mut ServiceSqliteTransaction<'_>, 1478 job_id: MycDeliveryJobId, 1479 relay_id: &MycDeliveryRelayId, 1480 attempt_id: MycDeliveryAttemptId, 1481 outcome: MycDeliveryAttemptOutcome, 1482 retry_jitter: MycDeliveryRetryJitter, 1483 observed_at: MycDeliveryTimeUnixMs, 1484 ) -> Result<MycDeliveryJobRecord, DeliveryOperationError> { 1485 let job = read_job(transaction, job_id) 1486 .await? 1487 .ok_or(DeliveryOperationError::Binding)?; 1488 let target = target_by_relay(&job, relay_id)?.clone(); 1489 let attempt = read_attempt(transaction, job_id, target.index, attempt_id) 1490 .await? 1491 .ok_or(DeliveryOperationError::Binding)?; 1492 if matches!( 1493 attempt.status, 1494 MycDeliveryAttemptStatus::Delivered 1495 | MycDeliveryAttemptStatus::Failed 1496 | MycDeliveryAttemptStatus::Unknown 1497 ) { 1498 return (attempt.status == outcome.status() 1499 && attempt.reason == Some(outcome.reason()) 1500 && attempt.resolved_at == Some(observed_at)) 1501 .then_some(job) 1502 .ok_or(DeliveryOperationError::Binding); 1503 } 1504 let required_prior = match outcome { 1505 MycDeliveryAttemptOutcome::Delivered 1506 | MycDeliveryAttemptOutcome::UnknownAcknowledgement => MycDeliveryAttemptStatus::Submitted, 1507 MycDeliveryAttemptOutcome::RelayRejected => MycDeliveryAttemptStatus::Submitted, 1508 MycDeliveryAttemptOutcome::TransportFailed => attempt.status, 1509 }; 1510 if attempt.status != required_prior 1511 || !matches!( 1512 required_prior, 1513 MycDeliveryAttemptStatus::Leased | MycDeliveryAttemptStatus::Submitted 1514 ) 1515 || target.active_attempt_id != Some(attempt_id) 1516 || observed_at < attempt.leased_at 1517 || attempt 1518 .submitted_at 1519 .is_some_and(|submitted_at| observed_at < submitted_at) 1520 || observed_at > attempt.lease_expires_at 1521 { 1522 return Err(DeliveryOperationError::Binding); 1523 } 1524 resolve_attempt_and_target( 1525 transaction, 1526 &job, 1527 &target, 1528 &attempt, 1529 required_prior, 1530 outcome.status(), 1531 outcome.reason(), 1532 retry_jitter, 1533 observed_at, 1534 ) 1535 .await?; 1536 finalize_job_if_terminal(transaction, job_id, observed_at).await 1537 } 1538 1539 pub(crate) async fn recover_expired( 1540 transaction: &mut ServiceSqliteTransaction<'_>, 1541 job_id: MycDeliveryJobId, 1542 relay_id: &MycDeliveryRelayId, 1543 attempt_id: MycDeliveryAttemptId, 1544 retry_jitter: MycDeliveryRetryJitter, 1545 observed_at: MycDeliveryTimeUnixMs, 1546 ) -> Result<MycDeliveryJobRecord, DeliveryOperationError> { 1547 let job = read_job(transaction, job_id) 1548 .await? 1549 .ok_or(DeliveryOperationError::Binding)?; 1550 let target = target_by_relay(&job, relay_id)?.clone(); 1551 let attempt = read_attempt(transaction, job_id, target.index, attempt_id) 1552 .await? 1553 .ok_or(DeliveryOperationError::Binding)?; 1554 if matches!( 1555 attempt.status, 1556 MycDeliveryAttemptStatus::Delivered 1557 | MycDeliveryAttemptStatus::Failed 1558 | MycDeliveryAttemptStatus::Unknown 1559 ) { 1560 return Ok(job); 1561 } 1562 if observed_at <= attempt.lease_expires_at || target.active_attempt_id != Some(attempt_id) { 1563 return Err(DeliveryOperationError::Binding); 1564 } 1565 let (terminal, reason) = match attempt.status { 1566 MycDeliveryAttemptStatus::Leased => ( 1567 MycDeliveryAttemptStatus::Failed, 1568 "lease_expired_before_submit", 1569 ), 1570 MycDeliveryAttemptStatus::Submitted => { 1571 (MycDeliveryAttemptStatus::Unknown, "acknowledgement_lost") 1572 } 1573 MycDeliveryAttemptStatus::Delivered 1574 | MycDeliveryAttemptStatus::Failed 1575 | MycDeliveryAttemptStatus::Unknown => unreachable!(), 1576 }; 1577 resolve_attempt_and_target( 1578 transaction, 1579 &job, 1580 &target, 1581 &attempt, 1582 attempt.status, 1583 terminal, 1584 reason, 1585 retry_jitter, 1586 observed_at, 1587 ) 1588 .await?; 1589 finalize_job_if_terminal(transaction, job_id, observed_at).await 1590 } 1591 1592 #[allow(clippy::too_many_arguments)] 1593 async fn resolve_attempt_and_target( 1594 transaction: &mut ServiceSqliteTransaction<'_>, 1595 job: &MycDeliveryJobRecord, 1596 target: &MycDeliveryTargetRecord, 1597 attempt: &MycDeliveryAttemptRecord, 1598 prior: MycDeliveryAttemptStatus, 1599 terminal: MycDeliveryAttemptStatus, 1600 reason: &'static str, 1601 retry_jitter: MycDeliveryRetryJitter, 1602 observed_at: MycDeliveryTimeUnixMs, 1603 ) -> Result<(), DeliveryOperationError> { 1604 let result = sqlx::query(RESOLVE_ATTEMPT_SQL) 1605 .bind(terminal.as_str()) 1606 .bind(observed_at.sqlite_value()) 1607 .bind(reason) 1608 .bind(attempt.id.as_bytes().as_slice()) 1609 .bind(job.id.as_bytes().as_slice()) 1610 .bind(i64::from(target.index)) 1611 .bind(prior.as_str()) 1612 .execute(&mut *transaction) 1613 .await 1614 .map_err(|_| DeliveryOperationError::Storage)?; 1615 require_one(result.rows_affected())?; 1616 let attempts_remaining = target.attempt_count < job.max_attempts; 1617 let schedules_retry = attempts_remaining 1618 && matches!( 1619 terminal, 1620 MycDeliveryAttemptStatus::Failed | MycDeliveryAttemptStatus::Unknown 1621 ); 1622 if !schedules_retry && retry_jitter.get() != 0 { 1623 return Err(DeliveryOperationError::Binding); 1624 } 1625 let (target_status, next_attempt) = match terminal { 1626 MycDeliveryAttemptStatus::Delivered => (MycDeliveryTargetStatus::Delivered, None), 1627 MycDeliveryAttemptStatus::Failed if attempts_remaining => ( 1628 MycDeliveryTargetStatus::Retryable, 1629 Some(next_attempt_time( 1630 job, 1631 attempt.number, 1632 retry_jitter, 1633 observed_at, 1634 )?), 1635 ), 1636 MycDeliveryAttemptStatus::Unknown => ( 1637 MycDeliveryTargetStatus::Unknown, 1638 attempts_remaining 1639 .then(|| next_attempt_time(job, attempt.number, retry_jitter, observed_at)) 1640 .transpose()?, 1641 ), 1642 MycDeliveryAttemptStatus::Failed => (MycDeliveryTargetStatus::Exhausted, None), 1643 MycDeliveryAttemptStatus::Leased | MycDeliveryAttemptStatus::Submitted => { 1644 return Err(DeliveryOperationError::Binding); 1645 } 1646 }; 1647 let result = sqlx::query(RESOLVE_TARGET_SQL) 1648 .bind(target_status.as_str()) 1649 .bind(next_attempt.map(MycDeliveryTimeUnixMs::sqlite_value)) 1650 .bind(observed_at.sqlite_value()) 1651 .bind(job.id.as_bytes().as_slice()) 1652 .bind(i64::from(target.index)) 1653 .bind(attempt.id.as_bytes().as_slice()) 1654 .bind(match prior { 1655 MycDeliveryAttemptStatus::Leased => "leased", 1656 MycDeliveryAttemptStatus::Submitted => "submitted", 1657 _ => return Err(DeliveryOperationError::Binding), 1658 }) 1659 .execute(&mut *transaction) 1660 .await 1661 .map_err(|_| DeliveryOperationError::Storage)?; 1662 require_one(result.rows_affected()) 1663 } 1664 1665 pub(crate) async fn finalize_job_if_terminal( 1666 transaction: &mut ServiceSqliteTransaction<'_>, 1667 job_id: MycDeliveryJobId, 1668 observed_at: MycDeliveryTimeUnixMs, 1669 ) -> Result<MycDeliveryJobRecord, DeliveryOperationError> { 1670 let job = read_job(transaction, job_id) 1671 .await? 1672 .ok_or(DeliveryOperationError::Binding)?; 1673 let delivered_required = job 1674 .targets 1675 .iter() 1676 .filter(|target| target.required && target.status == MycDeliveryTargetStatus::Delivered) 1677 .count(); 1678 let possible_required = job 1679 .targets 1680 .iter() 1681 .filter(|target| { 1682 target.required 1683 && target.status != MycDeliveryTargetStatus::Exhausted 1684 && !(target.status == MycDeliveryTargetStatus::Unknown 1685 && target.attempt_count >= job.max_attempts) 1686 }) 1687 .count(); 1688 let required = usize::try_from(job.required_acknowledgements) 1689 .map_err(|_| DeliveryOperationError::Binding)?; 1690 let delivered = match job.policy_mode { 1691 MycDeliveryPolicyMode::AtLeastOneRequired => delivered_required >= required, 1692 MycDeliveryPolicyMode::AllRequired | MycDeliveryPolicyMode::RequiredQuorum => { 1693 delivered_required >= required 1694 } 1695 }; 1696 let possible = match job.policy_mode { 1697 MycDeliveryPolicyMode::AtLeastOneRequired => possible_required >= required, 1698 MycDeliveryPolicyMode::AllRequired | MycDeliveryPolicyMode::RequiredQuorum => { 1699 possible_required >= required 1700 } 1701 }; 1702 if delivered || !possible { 1703 let terminal = if delivered { 1704 MycDeliveryJobStatus::Delivered 1705 } else if job.targets.iter().any(|target| { 1706 target.required 1707 && target.status == MycDeliveryTargetStatus::Unknown 1708 && target.attempt_count >= job.max_attempts 1709 }) { 1710 MycDeliveryJobStatus::Unknown 1711 } else { 1712 MycDeliveryJobStatus::Failed 1713 }; 1714 let result = sqlx::query(FINALIZE_JOB_SQL) 1715 .bind(terminal.as_str()) 1716 .bind(observed_at.sqlite_value()) 1717 .bind(observed_at.sqlite_value()) 1718 .bind(job_id.as_bytes().as_slice()) 1719 .execute(&mut *transaction) 1720 .await 1721 .map_err(|_| DeliveryOperationError::Storage)?; 1722 if result.rows_affected() > 1 { 1723 return Err(DeliveryOperationError::Storage); 1724 } 1725 } 1726 read_job(transaction, job_id) 1727 .await? 1728 .ok_or(DeliveryOperationError::Binding) 1729 } 1730 1731 fn next_attempt_time( 1732 job: &MycDeliveryJobRecord, 1733 attempt_number: u32, 1734 jitter: MycDeliveryRetryJitter, 1735 observed_at: MycDeliveryTimeUnixMs, 1736 ) -> Result<MycDeliveryTimeUnixMs, DeliveryOperationError> { 1737 let exponent = attempt_number.saturating_sub(1).min(31); 1738 let factor = 1_u64.checked_shl(exponent).unwrap_or(u64::MAX); 1739 let maximum_delay = job 1740 .initial_backoff_ms 1741 .saturating_mul(factor) 1742 .min(job.maximum_backoff_ms); 1743 if jitter.get() > maximum_delay { 1744 return Err(DeliveryOperationError::Binding); 1745 } 1746 let value = observed_at 1747 .get() 1748 .checked_add(jitter.get()) 1749 .ok_or(DeliveryOperationError::Binding)?; 1750 MycDeliveryTimeUnixMs::new(value).map_err(|_| DeliveryOperationError::Binding) 1751 } 1752 1753 async fn read_job_by_source( 1754 transaction: &mut ServiceSqliteTransaction<'_>, 1755 source: MycDeliverySource, 1756 ) -> Result<Option<MycDeliveryJobRecord>, DeliveryOperationError> { 1757 let rows = sqlx::query(READ_JOB_BY_SOURCE_SQL) 1758 .bind(source.kind().as_str()) 1759 .bind(source.id().as_slice()) 1760 .fetch_all(&mut *transaction) 1761 .await 1762 .map_err(|_| DeliveryOperationError::Storage)?; 1763 let job = read_job_rows(transaction, rows).await?; 1764 match job { 1765 Some(job) if job.source == source => Ok(Some(job)), 1766 Some(_) => Err(DeliveryOperationError::Binding), 1767 None => Ok(None), 1768 } 1769 } 1770 1771 pub(crate) async fn read_job( 1772 transaction: &mut ServiceSqliteTransaction<'_>, 1773 job_id: MycDeliveryJobId, 1774 ) -> Result<Option<MycDeliveryJobRecord>, DeliveryOperationError> { 1775 let rows = sqlx::query(READ_JOB_SQL) 1776 .bind(job_id.as_bytes().as_slice()) 1777 .fetch_all(&mut *transaction) 1778 .await 1779 .map_err(|_| DeliveryOperationError::Storage)?; 1780 read_job_rows(transaction, rows).await 1781 } 1782 1783 async fn read_job_rows( 1784 transaction: &mut ServiceSqliteTransaction<'_>, 1785 rows: Vec<sqlx::sqlite::SqliteRow>, 1786 ) -> Result<Option<MycDeliveryJobRecord>, DeliveryOperationError> { 1787 if rows.len() > 1 { 1788 return Err(DeliveryOperationError::Binding); 1789 } 1790 let Some(row) = rows.first() else { 1791 return Ok(None); 1792 }; 1793 let id = MycDeliveryJobId(blob32(row, "job_id")?); 1794 let source_kind = MycDeliverySourceKind::parse(text(row, "source_kind")?) 1795 .ok_or(DeliveryOperationError::Binding)?; 1796 let source = MycDeliverySource { 1797 kind: source_kind, 1798 id: blob32(row, "source_id")?, 1799 }; 1800 let artifact_digest = MycDeliveryArtifactDigest(blob32(row, "artifact_sha256")?); 1801 let policy_mode = MycDeliveryPolicyMode::parse(text(row, "policy_mode")?) 1802 .ok_or(DeliveryOperationError::Binding)?; 1803 let required_acknowledgements = positive_u32(row, "required_acknowledgements")?; 1804 let max_attempts = positive_u32(row, "max_attempts")?; 1805 if max_attempts > MYC_DELIVERY_ATTEMPT_MAX_COUNT { 1806 return Err(DeliveryOperationError::Binding); 1807 } 1808 let initial_backoff_ms = positive_u64(row, "initial_backoff_ms")?; 1809 let maximum_backoff_ms = positive_u64(row, "maximum_backoff_ms")?; 1810 let attempt_deadline_ms = positive_u64(row, "attempt_deadline_ms")?; 1811 if initial_backoff_ms > maximum_backoff_ms 1812 || maximum_backoff_ms > 300_000 1813 || attempt_deadline_ms > 30_000 1814 { 1815 return Err(DeliveryOperationError::Binding); 1816 } 1817 let status = 1818 MycDeliveryJobStatus::parse(text(row, "status")?).ok_or(DeliveryOperationError::Binding)?; 1819 let created_at = time(row, "created_at_unix_ms")?; 1820 let updated_at = time(row, "updated_at_unix_ms")?; 1821 let finalized_at = optional_time(row, "finalized_at_unix_ms", "finalized_at_type")?; 1822 let targets = read_targets(transaction, id).await?; 1823 let valid = !targets.is_empty() 1824 && targets.len() <= MYC_DELIVERY_TARGET_MAX_COUNT 1825 && targets 1826 .iter() 1827 .enumerate() 1828 .all(|(index, target)| usize::try_from(target.index) == Ok(index)) 1829 && matches!( 1830 status, 1831 MycDeliveryJobStatus::Delivered 1832 | MycDeliveryJobStatus::Failed 1833 | MycDeliveryJobStatus::Unknown 1834 ) == finalized_at.is_some() 1835 && targets.iter().all(|target| { 1836 target.status != MycDeliveryTargetStatus::Unknown 1837 || target.next_attempt_at.is_some() 1838 || target.attempt_count == max_attempts 1839 }); 1840 if !valid { 1841 return Err(DeliveryOperationError::Binding); 1842 } 1843 Ok(Some(MycDeliveryJobRecord { 1844 id, 1845 source, 1846 artifact_digest, 1847 policy_mode, 1848 required_acknowledgements, 1849 max_attempts, 1850 initial_backoff_ms, 1851 maximum_backoff_ms, 1852 attempt_deadline_ms, 1853 status, 1854 created_at, 1855 updated_at, 1856 finalized_at, 1857 targets, 1858 })) 1859 } 1860 1861 async fn read_targets( 1862 transaction: &mut ServiceSqliteTransaction<'_>, 1863 job_id: MycDeliveryJobId, 1864 ) -> Result<Box<[MycDeliveryTargetRecord]>, DeliveryOperationError> { 1865 let rows = sqlx::query(READ_TARGETS_SQL) 1866 .bind(job_id.as_bytes().as_slice()) 1867 .fetch_all(&mut *transaction) 1868 .await 1869 .map_err(|_| DeliveryOperationError::Storage)?; 1870 if rows.len() > MYC_DELIVERY_TARGET_MAX_COUNT { 1871 return Err(DeliveryOperationError::Binding); 1872 } 1873 rows.iter() 1874 .map(parse_target) 1875 .collect::<Result<Vec<_>, _>>() 1876 .map(Vec::into_boxed_slice) 1877 } 1878 1879 fn parse_target( 1880 row: &sqlx::sqlite::SqliteRow, 1881 ) -> Result<MycDeliveryTargetRecord, DeliveryOperationError> { 1882 let index = nonnegative_u32(row, "target_index")?; 1883 let relay_id = MycDeliveryRelayId::new(text(row, "relay_id")?) 1884 .map_err(|_| DeliveryOperationError::Binding)?; 1885 let required = bool_value(row, "required")?; 1886 let attempt_count = nonnegative_u32(row, "attempt_count")?; 1887 if attempt_count > MYC_DELIVERY_ATTEMPT_MAX_COUNT { 1888 return Err(DeliveryOperationError::Binding); 1889 } 1890 let status = MycDeliveryTargetStatus::parse(text(row, "status")?) 1891 .ok_or(DeliveryOperationError::Binding)?; 1892 let active_attempt_id = optional_blob32(row, "active_attempt_id", "active_attempt_id_type")? 1893 .map(MycDeliveryAttemptId); 1894 let next_attempt_at = optional_time(row, "next_attempt_at_unix_ms", "next_attempt_at_type")?; 1895 let updated_at = time(row, "updated_at_unix_ms")?; 1896 let active = matches!( 1897 status, 1898 MycDeliveryTargetStatus::Leased | MycDeliveryTargetStatus::Submitted 1899 ); 1900 let retryable = status == MycDeliveryTargetStatus::Retryable; 1901 if active != active_attempt_id.is_some() 1902 || (retryable && next_attempt_at.is_none()) 1903 || (!retryable && status != MycDeliveryTargetStatus::Unknown && next_attempt_at.is_some()) 1904 { 1905 return Err(DeliveryOperationError::Binding); 1906 } 1907 Ok(MycDeliveryTargetRecord { 1908 index, 1909 relay_id, 1910 required, 1911 attempt_count, 1912 status, 1913 active_attempt_id, 1914 next_attempt_at, 1915 updated_at, 1916 }) 1917 } 1918 1919 pub(crate) async fn read_attempts( 1920 transaction: &mut ServiceSqliteTransaction<'_>, 1921 job_id: MycDeliveryJobId, 1922 target_index: u32, 1923 ) -> Result<Box<[MycDeliveryAttemptRecord]>, DeliveryOperationError> { 1924 let rows = sqlx::query(READ_ATTEMPTS_SQL) 1925 .bind(job_id.as_bytes().as_slice()) 1926 .bind(i64::from(target_index)) 1927 .fetch_all(&mut *transaction) 1928 .await 1929 .map_err(|_| DeliveryOperationError::Storage)?; 1930 if rows.len() > usize::try_from(MYC_DELIVERY_ATTEMPT_MAX_COUNT).unwrap_or(usize::MAX) { 1931 return Err(DeliveryOperationError::Binding); 1932 } 1933 rows.iter() 1934 .map(parse_attempt) 1935 .collect::<Result<Vec<_>, _>>() 1936 .map(Vec::into_boxed_slice) 1937 } 1938 1939 async fn read_attempt( 1940 transaction: &mut ServiceSqliteTransaction<'_>, 1941 job_id: MycDeliveryJobId, 1942 target_index: u32, 1943 attempt_id: MycDeliveryAttemptId, 1944 ) -> Result<Option<MycDeliveryAttemptRecord>, DeliveryOperationError> { 1945 let rows = sqlx::query(READ_ATTEMPT_SQL) 1946 .bind(job_id.as_bytes().as_slice()) 1947 .bind(i64::from(target_index)) 1948 .bind(attempt_id.as_bytes().as_slice()) 1949 .fetch_all(&mut *transaction) 1950 .await 1951 .map_err(|_| DeliveryOperationError::Storage)?; 1952 one_attempt(rows) 1953 } 1954 1955 async fn read_attempt_by_nonce( 1956 transaction: &mut ServiceSqliteTransaction<'_>, 1957 job_id: MycDeliveryJobId, 1958 target_index: u32, 1959 nonce: &MycDeliveryAttemptNonce, 1960 ) -> Result<Option<MycDeliveryAttemptRecord>, DeliveryOperationError> { 1961 let rows = sqlx::query(READ_ATTEMPT_BY_NONCE_SQL) 1962 .bind(job_id.as_bytes().as_slice()) 1963 .bind(i64::from(target_index)) 1964 .bind(nonce.0.as_slice()) 1965 .fetch_all(&mut *transaction) 1966 .await 1967 .map_err(|_| DeliveryOperationError::Storage)?; 1968 one_attempt(rows) 1969 } 1970 1971 fn one_attempt( 1972 rows: Vec<sqlx::sqlite::SqliteRow>, 1973 ) -> Result<Option<MycDeliveryAttemptRecord>, DeliveryOperationError> { 1974 if rows.len() > 1 { 1975 return Err(DeliveryOperationError::Binding); 1976 } 1977 rows.first().map(parse_attempt).transpose() 1978 } 1979 1980 fn parse_attempt( 1981 row: &sqlx::sqlite::SqliteRow, 1982 ) -> Result<MycDeliveryAttemptRecord, DeliveryOperationError> { 1983 let id = MycDeliveryAttemptId(blob32(row, "attempt_id")?); 1984 let number = positive_u32(row, "attempt_number")?; 1985 if number > MYC_DELIVERY_ATTEMPT_MAX_COUNT { 1986 return Err(DeliveryOperationError::Binding); 1987 } 1988 let _nonce = blob32(row, "attempt_nonce")?; 1989 let status = MycDeliveryAttemptStatus::parse(text(row, "status")?) 1990 .ok_or(DeliveryOperationError::Binding)?; 1991 let leased_at = time(row, "leased_at_unix_ms")?; 1992 let lease_expires_at = time(row, "lease_expires_at_unix_ms")?; 1993 let submitted_at = optional_time(row, "submitted_at_unix_ms", "submitted_at_type")?; 1994 let resolved_at = optional_time(row, "resolved_at_unix_ms", "resolved_at_type")?; 1995 let reason = optional_reason(row)?; 1996 let valid = lease_expires_at > leased_at 1997 && match status { 1998 MycDeliveryAttemptStatus::Leased => { 1999 submitted_at.is_none() && resolved_at.is_none() && reason.is_none() 2000 } 2001 MycDeliveryAttemptStatus::Submitted => { 2002 submitted_at.is_some() && resolved_at.is_none() && reason.is_none() 2003 } 2004 MycDeliveryAttemptStatus::Delivered => { 2005 submitted_at.is_some() && resolved_at.is_some() && reason == Some("accepted") 2006 } 2007 MycDeliveryAttemptStatus::Failed => resolved_at.is_some() && reason.is_some(), 2008 MycDeliveryAttemptStatus::Unknown => { 2009 submitted_at.is_some() 2010 && resolved_at.is_some() 2011 && reason == Some("acknowledgement_lost") 2012 } 2013 }; 2014 valid 2015 .then_some(MycDeliveryAttemptRecord { 2016 id, 2017 number, 2018 status, 2019 leased_at, 2020 lease_expires_at, 2021 submitted_at, 2022 resolved_at, 2023 reason, 2024 }) 2025 .ok_or(DeliveryOperationError::Binding) 2026 } 2027 2028 fn optional_reason( 2029 row: &sqlx::sqlite::SqliteRow, 2030 ) -> Result<Option<&'static str>, DeliveryOperationError> { 2031 let kind = row 2032 .try_get::<&str, _>("reason_code_type") 2033 .map_err(|_| DeliveryOperationError::Binding)?; 2034 if kind == "null" { 2035 return Ok(None); 2036 } 2037 if kind != "text" { 2038 return Err(DeliveryOperationError::Binding); 2039 } 2040 match row 2041 .try_get::<Option<&str>, _>("reason_code") 2042 .map_err(|_| DeliveryOperationError::Binding)? 2043 .ok_or(DeliveryOperationError::Binding)? 2044 { 2045 "accepted" => Ok(Some("accepted")), 2046 "relay_rejected" => Ok(Some("relay_rejected")), 2047 "transport_failed" => Ok(Some("transport_failed")), 2048 "lease_expired_before_submit" => Ok(Some("lease_expired_before_submit")), 2049 "acknowledgement_lost" => Ok(Some("acknowledgement_lost")), 2050 _ => Err(DeliveryOperationError::Binding), 2051 } 2052 } 2053 2054 fn target_by_relay<'a>( 2055 job: &'a MycDeliveryJobRecord, 2056 relay_id: &MycDeliveryRelayId, 2057 ) -> Result<&'a MycDeliveryTargetRecord, DeliveryOperationError> { 2058 job.targets 2059 .iter() 2060 .find(|target| target.relay_id == *relay_id) 2061 .ok_or(DeliveryOperationError::Binding) 2062 } 2063 2064 fn exact_job( 2065 job: &MycDeliveryJobRecord, 2066 source: MycDeliverySource, 2067 artifact_digest: MycDeliveryArtifactDigest, 2068 created_at: MycDeliveryTimeUnixMs, 2069 policy: &MycDeliveryPolicies, 2070 ) -> bool { 2071 job.source == source 2072 && job.artifact_digest == artifact_digest 2073 && job.policy_mode == policy.mode 2074 && job.required_acknowledgements == policy.required_acknowledgements 2075 && job.max_attempts == policy.max_attempts 2076 && job.initial_backoff_ms == policy.initial_backoff_ms 2077 && job.maximum_backoff_ms == policy.maximum_backoff_ms 2078 && job.attempt_deadline_ms == policy.attempt_deadline_ms 2079 && job.created_at == created_at 2080 && job.targets.len() == policy.targets.len() 2081 && job 2082 .targets 2083 .iter() 2084 .zip(policy.targets.iter()) 2085 .all(|(actual, expected)| { 2086 actual.relay_id == expected.relay_id && actual.required == expected.required 2087 }) 2088 } 2089 2090 fn derive_job_id( 2091 source: MycDeliverySource, 2092 artifact: MycDeliveryArtifactDigest, 2093 ) -> MycDeliveryJobId { 2094 let mut hasher = Sha256::new(); 2095 hasher.update(JOB_ID_DOMAIN); 2096 if source.kind() == MycDeliverySourceKind::DiscoveryHandler { 2097 hasher.update(b"discovery_handler\0"); 2098 } 2099 hasher.update(source.id()); 2100 hasher.update(artifact.as_bytes()); 2101 MycDeliveryJobId(hasher.finalize().into()) 2102 } 2103 2104 fn derive_attempt_id( 2105 job_id: MycDeliveryJobId, 2106 target_index: u32, 2107 attempt_number: u32, 2108 nonce: &MycDeliveryAttemptNonce, 2109 ) -> MycDeliveryAttemptId { 2110 let mut hasher = Sha256::new(); 2111 hasher.update(ATTEMPT_ID_DOMAIN); 2112 hasher.update(job_id.as_bytes()); 2113 hasher.update(target_index.to_be_bytes()); 2114 hasher.update(attempt_number.to_be_bytes()); 2115 hasher.update(nonce.0); 2116 MycDeliveryAttemptId(hasher.finalize().into()) 2117 } 2118 2119 fn text<'a>( 2120 row: &'a sqlx::sqlite::SqliteRow, 2121 column: &str, 2122 ) -> Result<&'a str, DeliveryOperationError> { 2123 row.try_get::<Option<&str>, _>(column) 2124 .map_err(|_| DeliveryOperationError::Binding)? 2125 .ok_or(DeliveryOperationError::Binding) 2126 } 2127 2128 fn blob32(row: &sqlx::sqlite::SqliteRow, column: &str) -> Result<[u8; 32], DeliveryOperationError> { 2129 row.try_get::<Option<Vec<u8>>, _>(column) 2130 .map_err(|_| DeliveryOperationError::Binding)? 2131 .ok_or(DeliveryOperationError::Binding)? 2132 .try_into() 2133 .map_err(|_| DeliveryOperationError::Binding) 2134 } 2135 2136 fn optional_blob32( 2137 row: &sqlx::sqlite::SqliteRow, 2138 column: &str, 2139 type_column: &str, 2140 ) -> Result<Option<[u8; 32]>, DeliveryOperationError> { 2141 match row 2142 .try_get::<&str, _>(type_column) 2143 .map_err(|_| DeliveryOperationError::Binding)? 2144 { 2145 "null" => Ok(None), 2146 "blob" => blob32(row, column).map(Some), 2147 _ => Err(DeliveryOperationError::Binding), 2148 } 2149 } 2150 2151 fn positive_u32( 2152 row: &sqlx::sqlite::SqliteRow, 2153 column: &str, 2154 ) -> Result<u32, DeliveryOperationError> { 2155 nonnegative_u32(row, column).and_then(|value| { 2156 (value != 0) 2157 .then_some(value) 2158 .ok_or(DeliveryOperationError::Binding) 2159 }) 2160 } 2161 2162 fn nonnegative_u32( 2163 row: &sqlx::sqlite::SqliteRow, 2164 column: &str, 2165 ) -> Result<u32, DeliveryOperationError> { 2166 let value = row 2167 .try_get::<i64, _>(column) 2168 .map_err(|_| DeliveryOperationError::Binding)?; 2169 u32::try_from(value).map_err(|_| DeliveryOperationError::Binding) 2170 } 2171 2172 fn positive_u64( 2173 row: &sqlx::sqlite::SqliteRow, 2174 column: &str, 2175 ) -> Result<u64, DeliveryOperationError> { 2176 let value = row 2177 .try_get::<i64, _>(column) 2178 .map_err(|_| DeliveryOperationError::Binding)?; 2179 u64::try_from(value) 2180 .ok() 2181 .filter(|value| *value != 0) 2182 .ok_or(DeliveryOperationError::Binding) 2183 } 2184 2185 fn bool_value(row: &sqlx::sqlite::SqliteRow, column: &str) -> Result<bool, DeliveryOperationError> { 2186 match row 2187 .try_get::<i64, _>(column) 2188 .map_err(|_| DeliveryOperationError::Binding)? 2189 { 2190 0 => Ok(false), 2191 1 => Ok(true), 2192 _ => Err(DeliveryOperationError::Binding), 2193 } 2194 } 2195 2196 fn time( 2197 row: &sqlx::sqlite::SqliteRow, 2198 column: &str, 2199 ) -> Result<MycDeliveryTimeUnixMs, DeliveryOperationError> { 2200 let value = positive_u64(row, column)?; 2201 MycDeliveryTimeUnixMs::new(value).map_err(|_| DeliveryOperationError::Binding) 2202 } 2203 2204 fn optional_time( 2205 row: &sqlx::sqlite::SqliteRow, 2206 column: &str, 2207 type_column: &str, 2208 ) -> Result<Option<MycDeliveryTimeUnixMs>, DeliveryOperationError> { 2209 match row 2210 .try_get::<&str, _>(type_column) 2211 .map_err(|_| DeliveryOperationError::Binding)? 2212 { 2213 "null" => Ok(None), 2214 "integer" => time(row, column).map(Some), 2215 _ => Err(DeliveryOperationError::Binding), 2216 } 2217 } 2218 2219 fn require_one(rows: u64) -> Result<(), DeliveryOperationError> { 2220 (rows == 1) 2221 .then_some(()) 2222 .ok_or(DeliveryOperationError::Storage) 2223 } 2224 2225 fn map_transaction_error( 2226 error: ServiceSqliteTransactionError<DeliveryOperationError>, 2227 ) -> MycStateRepositoryError { 2228 if error.kind() == ServiceSqliteTransactionErrorKind::CommitOutcomeUnknown { 2229 return MycStateRepositoryError::new(MycStateRepositoryErrorKind::CommitOutcomeUnknown); 2230 } 2231 let kind = match error.operation_error() { 2232 Some(DeliveryOperationError::Binding) => MycStateRepositoryErrorKind::Binding, 2233 Some(DeliveryOperationError::Storage) | None => MycStateRepositoryErrorKind::Transaction, 2234 }; 2235 MycStateRepositoryError::new(kind) 2236 }