commit 840b1f792e58382d8f1f3327cc029728ec756d3a
parent 85faca6f29319af61eefdabfae659d406fec8888
Author: triesap <tyson@radroots.org>
Date: Sat, 6 Jun 2026 13:14:33 -0700
ops: add readiness gates
- check runtime database connectivity through the store
- gate readiness on exact migration plan state
- require repository read smoke before reporting ready
- cover readiness transitions and live relay readyz output
Diffstat:
3 files changed, 100 insertions(+), 3 deletions(-)
diff --git a/crates/tangle/tests/run_integration.rs b/crates/tangle/tests/run_integration.rs
@@ -410,6 +410,12 @@ async fn tangle_run_serves_relay_clients_and_persists_surreal_state() {
assert!(seller_detail.contains(profile.id().as_str()));
assert!(seller_detail.contains("\"display_name\":\"Radroots Market\""));
assert!(seller_detail.contains("\"regions\":[\"cascadia\",\"pnw\"]"));
+ let readiness = http_get(port, "/readyz");
+ assert!(readiness.contains("200 OK"));
+ assert!(readiness.contains("\"status\":\"ready\""));
+ assert!(readiness.contains("\"database\":\"ready\""));
+ assert!(readiness.contains("\"migrations\":\"ready\""));
+ assert!(readiness.contains("\"repository\":\"ready\""));
let metrics = http_get(port, "/metrics");
assert!(metrics.contains("200 OK"));
assert!(metrics.contains("text/plain; version=0.0.4"));
diff --git a/crates/tangle_runtime/src/lib.rs b/crates/tangle_runtime/src/lib.rs
@@ -1555,8 +1555,54 @@ async fn runtime_healthz() -> Json<HealthDocument> {
healthz().await
}
-async fn runtime_readyz() -> (StatusCode, Json<ReadinessDocument>) {
- readyz(State(ReadinessState::ready())).await
+async fn runtime_readyz(
+ State(state): State<RuntimeRelayState>,
+) -> (StatusCode, Json<ReadinessDocument>) {
+ readyz(State(runtime_readiness_state(&state.store).await)).await
+}
+
+async fn runtime_readiness_state(store: &SurrealStore) -> ReadinessState {
+ let database = readiness_status(store.ping().await);
+ let migrations = if database.is_ready() {
+ readiness_status(runtime_migrations_ready(store).await)
+ } else {
+ ReadinessCheckStatus::NotReady
+ };
+ let repository = if database.is_ready() && migrations.is_ready() {
+ readiness_status(store.metrics_snapshot().await.map(|_| ()))
+ } else {
+ ReadinessCheckStatus::NotReady
+ };
+ ReadinessState::new(database, migrations, repository)
+}
+
+async fn runtime_migrations_ready(store: &SurrealStore) -> Result<(), RuntimeCommandError> {
+ let applied = store
+ .applied_migrations()
+ .await
+ .map_err(|error| RuntimeCommandError::store(error.to_string()))?;
+ let plan = base_migration_plan();
+ if applied.len() != plan.migrations().len() {
+ return Err(RuntimeCommandError::store(
+ "runtime migrations are incomplete",
+ ));
+ }
+ for (applied, expected) in applied.iter().zip(plan.migrations()) {
+ if applied.name() != expected.name() || applied.checksum() != expected.checksum() {
+ return Err(RuntimeCommandError::store(
+ "runtime migrations do not match",
+ ));
+ }
+ }
+ Ok(())
+}
+
+fn readiness_status<E>(result: Result<(), E>) -> ReadinessCheckStatus {
+ if result.is_ok() {
+ ReadinessCheckStatus::Ready
+ } else {
+ ReadinessCheckStatus::NotReady
+ }
}
async fn runtime_metrics(State(state): State<RuntimeRelayState>) -> Result<Response, ApiError> {
@@ -4897,7 +4943,7 @@ mod tests {
listing_item_document, listing_projection_query, listings_router, load_runtime_config,
metrics_router, migrate_runtime_database, parse_listing_query,
parse_marketplace_search_query, parse_runtime_config_json, relay_info_router,
- restore_runtime_store, search_document_query, websocket_router,
+ restore_runtime_store, runtime_readiness_state, search_document_query, websocket_router,
};
use axum::{body::Body, response::IntoResponse};
use http::{HeaderValue, Request, StatusCode, header};
@@ -6494,6 +6540,34 @@ mod tests {
}
#[tokio::test]
+ async fn runtime_readiness_checks_database_migrations_and_repository() {
+ let config = SurrealConnectionConfig::memory("tangle_runtime", "readiness_gates")
+ .expect("memory config");
+ let store = SurrealStore::connect_memory(&config)
+ .await
+ .expect("memory store");
+
+ let missing = runtime_readiness_state(&store).await;
+ assert_eq!(
+ missing,
+ ReadinessState::new(
+ ReadinessCheckStatus::Ready,
+ ReadinessCheckStatus::NotReady,
+ ReadinessCheckStatus::NotReady
+ )
+ );
+
+ store
+ .apply_plan(&base_migration_plan())
+ .await
+ .expect("apply plan");
+ assert_eq!(
+ runtime_readiness_state(&store).await,
+ ReadinessState::ready()
+ );
+ }
+
+ #[tokio::test]
async fn metrics_endpoint_reports_store_snapshot() {
let store = runtime_memory_store().await;
let listing = build_fixture_event(&valid_public_listing_spec()).expect("listing");
diff --git a/crates/tangle_store_surreal/src/lib.rs b/crates/tangle_store_surreal/src/lib.rs
@@ -1566,6 +1566,16 @@ impl SurrealStore {
&self.db
}
+ pub async fn ping(&self) -> Result<(), SurrealStoreError> {
+ self.db
+ .query("RETURN true;")
+ .await
+ .map_err(SurrealStoreError::from)?
+ .check()
+ .map_err(SurrealStoreError::from)?;
+ Ok(())
+ }
+
pub async fn apply_plan(
&self,
plan: &SurrealMigrationPlan,
@@ -5563,6 +5573,13 @@ mod tests {
}
#[tokio::test]
+ async fn store_ping_confirms_database_connectivity() {
+ let store = memory_store().await;
+
+ store.ping().await.expect("ping");
+ }
+
+ #[tokio::test]
async fn migration_tracking_detects_checksum_changes() {
let store = memory_store().await;
let original = migration_tracking_schema();
