cache.rs (22413B)
1 use super::{SqliteStorage, decode_status, i64_from_u64, map_backend, map_corrupt, u64_from_i64}; 2 use radroots_storage::{ 3 Error, 4 projection::{ 5 ProjectionHealth, ProjectionId, ProjectionSnapshot, 6 cache::{ 7 PROJECTION_CACHE_COLLECTION_ROWS_MAX, ProjectionCacheCollection, ProjectionCacheLimits, 8 ProjectionCacheScope, ProjectionCacheUsage, 9 }, 10 }, 11 }; 12 use sqlx::{Row, Sqlite, Transaction}; 13 14 pub(super) fn map(error: sqlx::Error) -> Error { 15 if error 16 .as_database_error() 17 .is_some_and(|value| value.message() == "projection cache capacity exceeded") 18 { 19 Error::SpaceInsufficient 20 } else { 21 map_backend(error) 22 } 23 } 24 fn now() -> Result<u64, Error> { 25 u64::try_from( 26 std::time::SystemTime::now() 27 .duration_since(std::time::UNIX_EPOCH) 28 .map_err(|_| Error::InvalidProjectionTimestamp)? 29 .as_millis(), 30 ) 31 .map_err(|_| Error::InvalidProjectionTimestamp) 32 } 33 pub(super) async fn owner( 34 transaction: &mut Transaction<'_, Sqlite>, 35 id: &ProjectionId, 36 ) -> Result<bool, Error> { 37 let row: Option<i64> = sqlx::query_scalar( 38 "SELECT 1 FROM radroots_runtime_projection_cache_owners WHERE projection_id=?", 39 ) 40 .bind(id.as_str()) 41 .fetch_optional(&mut **transaction) 42 .await 43 .map_err(map)?; 44 Ok(row.is_some()) 45 } 46 pub(super) async fn usage( 47 transaction: &mut Transaction<'_, Sqlite>, 48 ) -> Result<ProjectionCacheUsage, Error> { 49 let row = sqlx::query("SELECT value_bytes,protected_generations,legacy_snapshots FROM radroots_runtime_projection_cache_budget WHERE singleton=1").fetch_one(&mut **transaction).await.map_err(map)?; 50 Ok(ProjectionCacheUsage { 51 value_bytes: u64_from_i64(row.try_get("value_bytes").map_err(map_corrupt)?)?, 52 protected_generations: u64_from_i64( 53 row.try_get("protected_generations").map_err(map_corrupt)?, 54 )?, 55 legacy_snapshots: u64_from_i64(row.try_get("legacy_snapshots").map_err(map_corrupt)?)?, 56 }) 57 } 58 async fn limits( 59 transaction: &mut Transaction<'_, Sqlite>, 60 ) -> Result<Option<ProjectionCacheLimits>, Error> { 61 let row = sqlx::query("SELECT max_value_bytes,max_protected_generations,max_legacy_snapshots,legacy_age_ms FROM radroots_runtime_projection_cache_budget WHERE singleton=1").fetch_one(&mut **transaction).await.map_err(map)?; 62 let bytes: Option<i64> = row.try_get("max_value_bytes").map_err(map_corrupt)?; 63 bytes 64 .map(|bytes| { 65 ProjectionCacheLimits::new( 66 u64_from_i64(bytes)?, 67 u64_from_i64( 68 row.try_get("max_protected_generations") 69 .map_err(map_corrupt)?, 70 )?, 71 u64_from_i64(row.try_get("max_legacy_snapshots").map_err(map_corrupt)?)?, 72 u64_from_i64(row.try_get("legacy_age_ms").map_err(map_corrupt)?)?, 73 ) 74 .map_err(|_| Error::CorruptProjectionRecord) 75 }) 76 .transpose() 77 } 78 async fn time(transaction: &mut Transaction<'_, Sqlite>, now: u64) -> Result<u64, Error> { 79 let clock: i64 = sqlx::query_scalar("UPDATE radroots_runtime_projection_cache_budget SET owner_time_ms=max(owner_time_ms,?) WHERE singleton=1 RETURNING owner_time_ms").bind(i64_from_u64(now)?).fetch_one(&mut **transaction).await.map_err(map)?; 80 u64_from_i64(clock) 81 } 82 pub(super) async fn configure( 83 store: &SqliteStorage, 84 id: ProjectionId, 85 requested: ProjectionCacheLimits, 86 ) -> Result<ProjectionCacheUsage, Error> { 87 store.require_projection_writer()?; 88 ProjectionId::parse(id.as_str())?; 89 let mut transaction = store 90 .pool() 91 .begin_with("BEGIN IMMEDIATE") 92 .await 93 .map_err(map)?; 94 if limits(&mut transaction) 95 .await? 96 .is_some_and(|current| current != requested) 97 { 98 return Err(Error::ProjectionRevisionConflict); 99 } 100 sqlx::query("UPDATE radroots_runtime_projection_cache_budget SET max_value_bytes=?,max_protected_generations=?,max_legacy_snapshots=?,legacy_age_ms=? WHERE singleton=1") 101 .bind(i64_from_u64(requested.value_bytes())?).bind(i64_from_u64(requested.protected_generations())?).bind(i64_from_u64(requested.legacy_snapshots())?).bind(i64_from_u64(requested.legacy_age_ms())?).execute(&mut *transaction).await.map_err(map)?; 102 sqlx::query("INSERT INTO radroots_runtime_projection_cache_owners(projection_id) VALUES(?) ON CONFLICT DO NOTHING").bind(id.as_str()).execute(&mut *transaction).await.map_err(map)?; 103 time(&mut transaction, now()?).await?; 104 let current = usage(&mut transaction).await?; 105 transaction.commit().await.map_err(map)?; 106 Ok(current) 107 } 108 pub(super) async fn fence_owner( 109 transaction: &mut Transaction<'_, Sqlite>, 110 store: &SqliteStorage, 111 id: &ProjectionId, 112 source: radroots_storage::event::EventSourceToken, 113 active: Option<radroots_storage::projection::ProjectionGeneration>, 114 ) -> Result<Option<radroots_storage::projection::ProjectionStatus>, Error> { 115 ProjectionId::parse(id.as_str())?; 116 if source.generation() != store.generation { 117 return Err(Error::SourceEpochChanged); 118 } 119 let epoch: Option<i64> = sqlx::query_scalar("SELECT mutation_epoch FROM radroots_runtime_source_generations WHERE generation=? AND state='active'").bind(source.generation().as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(map)?; 120 if epoch.map(u64_from_i64).transpose()? != Some(source.epoch()) { 121 return Err(Error::SourceEpochChanged); 122 } 123 let status = 124 sqlx::query("SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id=?") 125 .bind(id.as_str()) 126 .fetch_optional(&mut **transaction) 127 .await 128 .map_err(map)? 129 .as_ref() 130 .map(decode_status) 131 .transpose()?; 132 if status.as_ref().map(|value| value.generation()) != active { 133 return Err(Error::ProjectionRevisionConflict); 134 } 135 Ok(status) 136 } 137 pub(super) async fn retire( 138 store: &SqliteStorage, 139 scope: ProjectionCacheScope, 140 ) -> Result<ProjectionCacheUsage, Error> { 141 store.require_projection_writer()?; 142 let mut transaction = store 143 .pool() 144 .begin_with("BEGIN IMMEDIATE") 145 .await 146 .map_err(map)?; 147 let status = fence_owner( 148 &mut transaction, 149 store, 150 scope.projection_id(), 151 scope.source(), 152 scope.expected_active(), 153 ) 154 .await?; 155 let row = sqlx::query("SELECT source_generation,source_epoch,phase FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).fetch_optional(&mut *transaction).await.map_err(map)?; 156 let configured = owner(&mut transaction, scope.projection_id()).await?; 157 if !configured { 158 let phase = row 159 .as_ref() 160 .map(|row| row.try_get::<String, _>("phase").map_err(map_corrupt)) 161 .transpose()?; 162 if scope.expected_active() != Some(scope.generation()) 163 || status.as_ref().is_none_or(|value| { 164 value.health() != ProjectionHealth::Invalidated 165 || value.active_rebuild().is_some() 166 || value 167 .checkpoint() 168 .and_then(|value| value.source_position()) 169 .is_some_and(|position| position.generation() != store.generation) 170 }) 171 || phase.as_deref().is_some_and(|value| value != "retired") 172 { 173 return Err(Error::ProjectionRevisionConflict); 174 } 175 } 176 if scope.expected_active() == Some(scope.generation()) 177 && status 178 .as_ref() 179 .is_some_and(|value| value.health() == ProjectionHealth::Ready) 180 { 181 let stale = if let Some(row) = row.as_ref() { 182 let generation: Vec<u8> = row.try_get("source_generation").map_err(map_corrupt)?; 183 generation != scope.source().generation().as_bytes() 184 || u64_from_i64(row.try_get("source_epoch").map_err(map_corrupt)?)? 185 != scope.source().epoch() 186 } else { 187 false 188 }; 189 if !stale { 190 return Err(Error::ProjectionRevisionConflict); 191 } 192 } 193 sqlx::query("INSERT INTO radroots_runtime_projection_generations(projection_id,generation,source_generation,source_epoch,expected_active,phase) VALUES(?,?,?,?,NULL,'retired') ON CONFLICT(projection_id,generation) DO UPDATE SET phase='retired'") 194 .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind(scope.source().generation().as_bytes().as_slice()).bind(i64_from_u64(scope.source().epoch())?).execute(&mut *transaction).await.map_err(map)?; 195 if configured && scope.expected_active() == Some(scope.generation()) { 196 sqlx::query("DELETE FROM radroots_runtime_projection_checkpoints WHERE projection_id=? AND projection_generation=?").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).execute(&mut *transaction).await.map_err(map)?; 197 } 198 let current = usage(&mut transaction).await?; 199 transaction.commit().await.map_err(map)?; 200 Ok(current) 201 } 202 pub(super) async fn collect( 203 store: &SqliteStorage, 204 scope: ProjectionCacheScope, 205 ) -> Result<ProjectionCacheCollection, Error> { 206 collect_kind(store, scope, CollectionKind::Generation).await 207 } 208 209 #[derive(Clone, Copy, Eq, PartialEq)] 210 pub(super) enum CollectionKind { 211 Generation, 212 ObsoleteSnapshots, 213 } 214 215 pub(super) async fn collect_kind( 216 store: &SqliteStorage, 217 scope: ProjectionCacheScope, 218 kind: CollectionKind, 219 ) -> Result<ProjectionCacheCollection, Error> { 220 store.require_projection_writer()?; 221 let mut transaction = store 222 .pool() 223 .begin_with("BEGIN IMMEDIATE") 224 .await 225 .map_err(map)?; 226 let result = collect_in(&mut transaction, store, &scope, kind).await?; 227 transaction.commit().await.map_err(map)?; 228 Ok(result) 229 } 230 pub(super) async fn collect_in( 231 transaction: &mut Transaction<'_, Sqlite>, 232 store: &SqliteStorage, 233 scope: &ProjectionCacheScope, 234 kind: CollectionKind, 235 ) -> Result<ProjectionCacheCollection, Error> { 236 let status = fence_owner( 237 transaction, 238 store, 239 scope.projection_id(), 240 scope.source(), 241 scope.expected_active(), 242 ) 243 .await?; 244 if !owner(transaction, scope.projection_id()).await? { 245 return Err(Error::ProjectionRevisionConflict); 246 } 247 let phase: Option<String> = sqlx::query_scalar("SELECT phase FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(map)?; 248 let active = scope.expected_active() == Some(scope.generation()); 249 let reserved = phase.as_deref().is_some_and(|value| value != "retired"); 250 let snapshot_only = kind == CollectionKind::ObsoleteSnapshots; 251 if snapshot_only { 252 let invalidated = active 253 && status.as_ref().is_some_and(|status| { 254 status.health() == ProjectionHealth::Invalidated 255 && status.active_rebuild().is_none() 256 }); 257 let staging = phase 258 .as_deref() 259 .is_some_and(|value| value != "retired" && value != "promoted"); 260 if ((reserved || active) && !invalidated) || staging { 261 return Err(Error::ProjectionRevisionConflict); 262 } 263 } 264 let protected = !snapshot_only && (active || reserved); 265 let now = time(transaction, now()?).await?; 266 let limit = limits(transaction) 267 .await? 268 .ok_or(Error::ProjectionRevisionConflict)?; 269 let mut removed = 0u64; 270 let mut freed = 0u64; 271 if !protected && !snapshot_only { 272 let rows = sqlx::query("SELECT document_key,length(value) AS bytes FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? ORDER BY document_key LIMIT 64") 273 .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).fetch_all(&mut **transaction).await.map_err(map)?; 274 for row in rows { 275 let key: String = row.try_get("document_key").map_err(map_corrupt)?; 276 let bytes = u64_from_i64(row.try_get("bytes").map_err(map_corrupt)?)?; 277 sqlx::query("DELETE FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? AND document_key=?").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind(key).execute(&mut **transaction).await.map_err(map)?; 278 removed += 1; 279 freed = freed 280 .checked_add(bytes) 281 .ok_or(Error::CorruptProjectionRecord)?; 282 } 283 } 284 let mut current = usage(transaction).await?; 285 let rows = sqlx::query("SELECT snapshot_id,created_at_unix_ms,length(value) AS bytes FROM radroots_runtime_projection_snapshots INDEXED BY radroots_runtime_projection_snapshots_generation_created_idx WHERE projection_id=? AND generation=? ORDER BY created_at_unix_ms,snapshot_id LIMIT ?") 286 .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind((PROJECTION_CACHE_COLLECTION_ROWS_MAX as u64-removed) as i64).fetch_all(&mut **transaction).await.map_err(map)?; 287 let mut future_after = now; 288 let mut future_id = vec![255; 32]; 289 for row in rows { 290 let created = u64_from_i64(row.try_get("created_at_unix_ms").map_err(map_corrupt)?)?; 291 if protected 292 && created <= now 293 && now - created < limit.legacy_age_ms() 294 && current.legacy_snapshots <= limit.legacy_snapshots() 295 { 296 break; 297 } 298 let id: Vec<u8> = row.try_get("snapshot_id").map_err(map_corrupt)?; 299 let bytes = u64_from_i64(row.try_get("bytes").map_err(map_corrupt)?)?; 300 delete_snapshot(transaction, scope.projection_id(), &id).await?; 301 if created > now { 302 future_after = created; 303 future_id = id; 304 } 305 removed += 1; 306 freed = freed 307 .checked_add(bytes) 308 .ok_or(Error::CorruptProjectionRecord)?; 309 current.legacy_snapshots -= 1; 310 } 311 if protected && removed < PROJECTION_CACHE_COLLECTION_ROWS_MAX as u64 { 312 let rows = sqlx::query("SELECT snapshot_id,length(value) AS bytes FROM radroots_runtime_projection_snapshots INDEXED BY radroots_runtime_projection_snapshots_generation_created_idx WHERE projection_id=? AND generation=? AND (created_at_unix_ms,snapshot_id)>(?,?) ORDER BY created_at_unix_ms,snapshot_id LIMIT ?") 313 .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind(i64_from_u64(future_after)?).bind(future_id).bind((PROJECTION_CACHE_COLLECTION_ROWS_MAX as u64-removed) as i64).fetch_all(&mut **transaction).await.map_err(map)?; 314 for row in rows { 315 let id: Vec<u8> = row.try_get("snapshot_id").map_err(map_corrupt)?; 316 delete_snapshot(transaction, scope.projection_id(), &id).await?; 317 removed += 1; 318 freed = freed 319 .checked_add(u64_from_i64(row.try_get("bytes").map_err(map_corrupt)?)?) 320 .ok_or(Error::CorruptProjectionRecord)?; 321 } 322 } 323 let document_more: bool = !protected && !snapshot_only && sqlx::query_scalar::<_,i64>("SELECT EXISTS(SELECT 1 FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=?)").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).fetch_one(&mut **transaction).await.map_err(map)? != 0; 324 current = usage(transaction).await?; 325 let snapshot_sql = "SELECT EXISTS(SELECT 1 FROM radroots_runtime_projection_snapshots INDEXED BY radroots_runtime_projection_snapshots_generation_created_idx WHERE projection_id=? AND generation=? AND created_at_unix_ms<=?)"; 326 let cutoff = if !protected || current.legacy_snapshots > limit.legacy_snapshots() { 327 i64::MAX as u64 328 } else { 329 now.saturating_sub(limit.legacy_age_ms()) 330 }; 331 let expired_more: i64 = sqlx::query_scalar(snapshot_sql) 332 .bind(scope.projection_id().as_str()) 333 .bind(scope.generation().as_bytes().as_slice()) 334 .bind(i64_from_u64(cutoff)?) 335 .fetch_one(&mut **transaction) 336 .await 337 .map_err(map)?; 338 let future_more: i64 = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM radroots_runtime_projection_snapshots INDEXED BY radroots_runtime_projection_snapshots_generation_created_idx WHERE projection_id=? AND generation=? AND created_at_unix_ms>?)").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind(i64_from_u64(now)?).fetch_one(&mut **transaction).await.map_err(map)?; 339 let snapshot_more = expired_more | future_more; 340 let mut registration_more = !protected && !snapshot_only && phase.is_some(); 341 if !protected 342 && !snapshot_only 343 && !document_more 344 && snapshot_more == 0 345 && phase.is_some() 346 && removed < PROJECTION_CACHE_COLLECTION_ROWS_MAX as u64 347 { 348 sqlx::query("DELETE FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?").bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).execute(&mut **transaction).await.map_err(map)?; 349 removed += 1; 350 registration_more = false; 351 } 352 let result = ProjectionCacheCollection { 353 removed_rows: removed, 354 removed_value_bytes: freed, 355 more: document_more || snapshot_more != 0 || registration_more, 356 usage: usage(transaction).await?, 357 }; 358 Ok(result) 359 } 360 async fn delete_snapshot( 361 transaction: &mut Transaction<'_, Sqlite>, 362 id: &ProjectionId, 363 snapshot: &[u8], 364 ) -> Result<(), Error> { 365 sqlx::query( 366 "DELETE FROM radroots_runtime_projection_snapshots WHERE projection_id=? AND snapshot_id=?", 367 ) 368 .bind(id.as_str()) 369 .bind(snapshot) 370 .execute(&mut **transaction) 371 .await 372 .map_err(map)?; 373 Ok(()) 374 } 375 async fn load_snapshot( 376 transaction: &mut Transaction<'_, Sqlite>, 377 id: &ProjectionId, 378 snapshot: &[u8; 32], 379 expiry: Option<(ProjectionCacheLimits, u64)>, 380 ) -> Result<Option<ProjectionSnapshot>, Error> { 381 let row = sqlx::query("SELECT generation,created_at_unix_ms,length(value) AS bytes,CASE WHEN length(value_sha256)=32 THEN value_sha256 END AS digest FROM radroots_runtime_projection_snapshots WHERE projection_id=? AND snapshot_id=?").bind(id.as_str()).bind(snapshot.as_slice()).fetch_optional(&mut **transaction).await.map_err(map)?; 382 let Some(row) = row else { 383 return Ok(None); 384 }; 385 let created = u64_from_i64(row.try_get("created_at_unix_ms").map_err(map_corrupt)?)?; 386 if expiry.is_some_and(|(limit, now)| created > now || now - created >= limit.legacy_age_ms()) { 387 return Ok(None); 388 } 389 let bytes: i64 = row.try_get("bytes").map_err(map_corrupt)?; 390 if bytes <= 0 391 || bytes > radroots_storage::projection::PROJECTION_DOCUMENT_VALUE_MAX_BYTES as i64 392 { 393 return Err(Error::CorruptProjectionDocument); 394 } 395 let digest: [u8; 32] = row 396 .try_get::<Vec<u8>, _>("digest") 397 .map_err(|_| Error::CorruptProjectionDocument)? 398 .try_into() 399 .map_err(|_| Error::CorruptProjectionDocument)?; 400 let generation = radroots_storage::projection::ProjectionGeneration::new( 401 row.try_get::<Vec<u8>, _>("generation") 402 .map_err(map_corrupt)? 403 .try_into() 404 .map_err(|_| Error::CorruptProjectionDocument)?, 405 )?; 406 let value: Vec<u8> = sqlx::query_scalar("SELECT value FROM radroots_runtime_projection_snapshots WHERE projection_id=? AND snapshot_id=?").bind(id.as_str()).bind(snapshot.as_slice()).fetch_one(&mut **transaction).await.map_err(map)?; 407 ProjectionSnapshot::from_stored_parts(id.clone(), *snapshot, generation, created, value, digest) 408 .map(Some) 409 } 410 pub(super) async fn snapshot( 411 store: &SqliteStorage, 412 id: ProjectionId, 413 snapshot: [u8; 32], 414 ) -> Result<Option<ProjectionSnapshot>, Error> { 415 let readonly = store.event_mode() == radroots_storage::status::EventStoreMode::ReadOnly; 416 let mut transaction = if readonly { 417 store.pool().begin().await 418 } else { 419 store.pool().begin_with("BEGIN IMMEDIATE").await 420 } 421 .map_err(map)?; 422 let expiry = if owner(&mut transaction, &id).await? { 423 if readonly { 424 return Ok(None); 425 } 426 Some(( 427 limits(&mut transaction) 428 .await? 429 .ok_or(Error::CorruptProjectionRecord)?, 430 time(&mut transaction, now()?).await?, 431 )) 432 } else { 433 None 434 }; 435 let result = load_snapshot(&mut transaction, &id, &snapshot, expiry).await?; 436 if readonly { 437 transaction.rollback().await.map_err(map)?; 438 } else { 439 transaction.commit().await.map_err(map)?; 440 } 441 Ok(result) 442 } 443 pub(super) async fn put_snapshot( 444 store: &SqliteStorage, 445 snapshot: ProjectionSnapshot, 446 ) -> Result<(), Error> { 447 store.require_projection_writer()?; 448 let mut transaction = store 449 .pool() 450 .begin_with("BEGIN IMMEDIATE") 451 .await 452 .map_err(map)?; 453 if let Some(existing) = load_snapshot( 454 &mut transaction, 455 snapshot.projection_id(), 456 snapshot.snapshot_id(), 457 None, 458 ) 459 .await? 460 { 461 return if existing == snapshot { 462 Ok(()) 463 } else { 464 Err(Error::CorruptProjectionDocument) 465 }; 466 } 467 if owner(&mut transaction, snapshot.projection_id()).await? { 468 return Err(Error::ProjectionRevisionConflict); 469 } 470 sqlx::query("INSERT INTO radroots_runtime_projection_snapshots(projection_id,snapshot_id,generation,created_at_unix_ms,value,value_sha256) VALUES(?,?,?,?,?,?)") 471 .bind(snapshot.projection_id().as_str()).bind(snapshot.snapshot_id().as_slice()).bind(snapshot.generation().as_bytes().as_slice()).bind(i64_from_u64(snapshot.created_at_unix_ms())?).bind(snapshot.value()).bind(snapshot.value_sha256().as_slice()).execute(&mut *transaction).await.map_err(map)?; 472 transaction.commit().await.map_err(map) 473 } 474 475 #[cfg(test)] 476 #[path = "cache_tests.rs"] 477 mod tests; 478 479 #[cfg(test)] 480 #[path = "cache_legacy_retirement_tests.rs"] 481 mod legacy_retirement_tests;