batch.rs (15724B)
1 use super::{SqliteStorage, decode_status, put_status_transaction}; 2 use radroots_storage::{ 3 Error, 4 event::EventSourceToken, 5 projection::{ 6 ProjectionCheckpoint, ProjectionDocument, ProjectionGeneration, ProjectionHealth, 7 ProjectionId, ProjectionStatus, batch::ProjectionDocumentBatch, 8 }, 9 }; 10 use sqlx::{Row, Sqlite, Transaction}; 11 12 pub(super) async fn fence( 13 transaction: &mut Transaction<'_, Sqlite>, 14 store: &SqliteStorage, 15 source: EventSourceToken, 16 id: &ProjectionId, 17 expected: Option<ProjectionGeneration>, 18 ) -> Result<u64, Error> { 19 if source.generation() != store.generation { 20 return Err(Error::SourceEpochChanged); 21 } 22 let row = sqlx::query("SELECT mutation_epoch, sequence_head FROM radroots_runtime_source_generations WHERE generation = ? AND state = 'active'") 23 .bind(source.generation().as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(super::cache::map)? 24 .ok_or(Error::SourceEpochChanged)?; 25 let epoch = super::u64_from_i64(row.try_get("mutation_epoch").map_err(super::map_corrupt)?)?; 26 if epoch != source.epoch() { 27 return Err(Error::SourceEpochChanged); 28 } 29 let status = sqlx::query( 30 "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?", 31 ) 32 .bind(id.as_str()) 33 .fetch_optional(&mut **transaction) 34 .await 35 .map_err(super::cache::map)? 36 .as_ref() 37 .map(decode_status) 38 .transpose()?; 39 if status.as_ref().map(ProjectionStatus::generation) != expected 40 || status 41 .as_ref() 42 .is_some_and(|value| value.health() != ProjectionHealth::Ready) 43 { 44 return Err(Error::ProjectionRevisionConflict); 45 } 46 super::u64_from_i64(row.try_get("sequence_head").map_err(super::map_corrupt)?) 47 } 48 49 pub(super) async fn read_fence( 50 transaction: &mut Transaction<'_, Sqlite>, 51 store: &SqliteStorage, 52 source: EventSourceToken, 53 id: &ProjectionId, 54 active: ProjectionGeneration, 55 ) -> Result<(), Error> { 56 fence(transaction, store, source, id, Some(active)).await?; 57 let row = sqlx::query("SELECT source_generation, source_epoch, phase FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?") 58 .bind(id.as_str()).bind(active.as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(super::cache::map)? 59 .ok_or(Error::ProjectionRevisionConflict)?; 60 let phase: String = row.try_get("phase").map_err(super::map_corrupt)?; 61 if phase != "promoted" { 62 return Err(Error::ProjectionRevisionConflict); 63 } 64 let generation: Vec<u8> = row 65 .try_get("source_generation") 66 .map_err(super::map_corrupt)?; 67 let epoch = super::u64_from_i64(row.try_get("source_epoch").map_err(super::map_corrupt)?)?; 68 if generation != source.generation().as_bytes() || epoch != source.epoch() { 69 return Err(Error::SourceEpochChanged); 70 } 71 Ok(()) 72 } 73 74 pub(super) async fn refuse_registered_legacy( 75 transaction: &mut Transaction<'_, Sqlite>, 76 id: &ProjectionId, 77 generation: ProjectionGeneration, 78 ) -> Result<(), Error> { 79 let registered: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?") 80 .bind(id.as_str()).bind(generation.as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(super::cache::map)?; 81 if registered.is_some() { 82 return Err(Error::ProjectionRevisionConflict); 83 } 84 Ok(()) 85 } 86 87 pub(super) async fn write( 88 store: &SqliteStorage, 89 batch: ProjectionDocumentBatch, 90 checkpoint: Option<ProjectionCheckpoint>, 91 ) -> Result<Option<ProjectionStatus>, Error> { 92 store.require_projection_writer()?; 93 let mut transaction = store 94 .pool() 95 .begin_with("BEGIN IMMEDIATE") 96 .await 97 .map_err(super::cache::map)?; 98 let scope = batch.scope(); 99 let sequence = fence( 100 &mut transaction, 101 store, 102 scope.source(), 103 scope.projection_id(), 104 scope.expected_active(), 105 ) 106 .await?; 107 let prior = sqlx::query("SELECT source_generation, source_epoch, expected_active, phase, document_rows, value_bytes FROM radroots_runtime_projection_generations WHERE projection_id = ? AND generation = ?") 108 .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()) 109 .fetch_optional(&mut *transaction).await.map_err(super::cache::map)?; 110 let (mut count, mut bytes) = if let Some(row) = prior { 111 let generation: Vec<u8> = row 112 .try_get("source_generation") 113 .map_err(super::map_corrupt)?; 114 let epoch = super::u64_from_i64(row.try_get("source_epoch").map_err(super::map_corrupt)?)?; 115 let active: Option<Vec<u8>> = row.try_get("expected_active").map_err(super::map_corrupt)?; 116 let phase: String = row.try_get("phase").map_err(super::map_corrupt)?; 117 if generation != scope.source().generation().as_bytes() 118 || epoch != scope.source().epoch() 119 || active.as_deref() 120 != scope 121 .expected_active() 122 .as_ref() 123 .map(|value| value.as_bytes().as_slice()) 124 || phase != "staging" 125 { 126 return Err(Error::ProjectionRevisionConflict); 127 } 128 ( 129 super::u64_from_i64(row.try_get("document_rows").map_err(super::map_corrupt)?)?, 130 super::u64_from_i64(row.try_get("value_bytes").map_err(super::map_corrupt)?)?, 131 ) 132 } else { 133 let staging: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id = ? AND phase = 'staging' LIMIT 1") 134 .bind(scope.projection_id().as_str()).fetch_optional(&mut *transaction).await.map_err(super::cache::map)?; 135 let existing: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_documents WHERE projection_id = ? AND generation = ? LIMIT 1") 136 .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()) 137 .fetch_optional(&mut *transaction).await.map_err(super::cache::map)?; 138 if staging.is_some() || existing.is_some() { 139 return Err(Error::ProjectionRevisionConflict); 140 } 141 sqlx::query("INSERT INTO radroots_runtime_projection_generations (projection_id,generation,source_generation,source_epoch,expected_active,phase) VALUES (?,?,?,?,?,'staging')") 142 .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()) 143 .bind(scope.source().generation().as_bytes().as_slice()).bind(super::i64_from_u64(scope.source().epoch())?) 144 .bind(scope.expected_active().map(|value| value.as_bytes().to_vec())).execute(&mut *transaction).await.map_err(super::cache::map)?; 145 (0, 0) 146 }; 147 let mut writes = Vec::with_capacity(batch.documents().len()); 148 let mut replaced_bytes = 0u64; 149 let mut replacement_bytes = 0u64; 150 for document in batch.documents() { 151 let previous: Option<i64> = sqlx::query_scalar("SELECT length(value) FROM radroots_runtime_projection_documents WHERE projection_id = ? AND generation = ? AND document_key = ?") 152 .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind(document.key()) 153 .fetch_optional(&mut *transaction).await.map_err(super::cache::map)?; 154 count = count 155 .checked_add(u64::from(previous.is_none())) 156 .ok_or(Error::SpaceInsufficient)?; 157 let previous = previous.map(super::u64_from_i64).transpose()?.unwrap_or(0); 158 replaced_bytes = replaced_bytes 159 .checked_add(previous) 160 .ok_or(Error::CorruptProjectionRecord)?; 161 replacement_bytes = replacement_bytes 162 .checked_add(document.value().len() as u64) 163 .ok_or(Error::SpaceInsufficient)?; 164 writes.push((document, previous)); 165 } 166 bytes = bytes 167 .checked_sub(replaced_bytes) 168 .and_then(|size| size.checked_add(replacement_bytes)) 169 .ok_or(Error::CorruptProjectionRecord)?; 170 let budget = sqlx::query("SELECT value_bytes,max_value_bytes,batch_value_ceiling FROM radroots_runtime_projection_cache_budget WHERE singleton=1") 171 .fetch_one(&mut *transaction).await.map_err(super::cache::map)?; 172 let current_bytes = 173 super::u64_from_i64(budget.try_get("value_bytes").map_err(super::map_corrupt)?)?; 174 let limit = budget 175 .try_get::<Option<i64>, _>("max_value_bytes") 176 .map_err(super::map_corrupt)?; 177 if budget 178 .try_get::<Option<i64>, _>("batch_value_ceiling") 179 .map_err(super::map_corrupt)? 180 .is_some() 181 { 182 return Err(Error::CorruptProjectionRecord); 183 } 184 let final_bytes = current_bytes 185 .checked_sub(replaced_bytes) 186 .and_then(|value| value.checked_add(replacement_bytes)) 187 .ok_or(Error::CorruptProjectionRecord)?; 188 if final_bytes > current_bytes && limit.is_some_and(|limit| final_bytes > limit as u64) { 189 return Err(Error::SpaceInsufficient); 190 } 191 // The immutable policy is unchanged. This reservation exists only inside 192 // this transaction, admits its final aggregate and rolls back on failure. 193 sqlx::query("UPDATE radroots_runtime_projection_cache_budget SET batch_value_ceiling=? WHERE singleton=1") 194 .bind(super::i64_from_u64(final_bytes)?).execute(&mut *transaction).await.map_err(super::cache::map)?; 195 // Shrink before growth, so no intermediate increase exceeds the admitted total. 196 writes.sort_by_key(|(document, previous)| document.value().len() as u64 > *previous); 197 for (document, _) in writes { 198 put( 199 &mut transaction, 200 scope.projection_id(), 201 scope.generation(), 202 document, 203 ) 204 .await?; 205 } 206 let promoted = if let Some(checkpoint) = checkpoint { 207 if checkpoint.projection_id() != scope.projection_id() 208 || checkpoint.generation() != scope.generation() 209 || checkpoint.projected_rows() != count 210 || match checkpoint.source_position() { 211 Some(position) => { 212 position.generation() != scope.source().generation() 213 || position.sequence().get() != sequence 214 } 215 None => sequence != 0, 216 } 217 { 218 return Err(Error::ProjectionCheckpointMismatch); 219 } 220 let status = ProjectionStatus::new( 221 scope.projection_id().clone(), 222 scope.generation(), 223 ProjectionHealth::Ready, 224 Some(checkpoint), 225 None, 226 )?; 227 sqlx::query("UPDATE radroots_runtime_projection_generations SET phase = 'retired' WHERE projection_id = ? AND phase = 'promoted'") 228 .bind(scope.projection_id().as_str()).execute(&mut *transaction).await.map_err(super::cache::map)?; 229 put_status_transaction(&mut transaction, &status).await?; 230 Some(status) 231 } else { 232 None 233 }; 234 sqlx::query("UPDATE radroots_runtime_projection_generations SET document_rows = ?, value_bytes = ?, phase = ? WHERE projection_id = ? AND generation = ?") 235 .bind(super::i64_from_u64(count)?).bind(super::i64_from_u64(bytes)?) 236 .bind(if promoted.is_some() { "promoted" } else { "staging" }) 237 .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()) 238 .execute(&mut *transaction).await.map_err(super::cache::map)?; 239 let cleared = sqlx::query("UPDATE radroots_runtime_projection_cache_budget SET batch_value_ceiling=NULL WHERE singleton=1 AND batch_value_ceiling=? AND value_bytes=?") 240 .bind(super::i64_from_u64(final_bytes)?).bind(super::i64_from_u64(final_bytes)?).execute(&mut *transaction).await.map_err(super::cache::map)?; 241 if cleared.rows_affected() != 1 { 242 return Err(Error::CorruptProjectionRecord); 243 } 244 transaction.commit().await.map_err(super::cache::map)?; 245 Ok(promoted) 246 } 247 248 pub(super) async fn put( 249 transaction: &mut Transaction<'_, Sqlite>, 250 id: &ProjectionId, 251 generation: ProjectionGeneration, 252 document: &ProjectionDocument, 253 ) -> Result<(), Error> { 254 sqlx::query("INSERT INTO radroots_runtime_projection_documents (projection_id,generation,document_key,value,value_sha256) VALUES (?,?,?,?,?) ON CONFLICT(projection_id,generation,document_key) DO UPDATE SET value=excluded.value,value_sha256=excluded.value_sha256") 255 .bind(id.as_str()).bind(generation.as_bytes().as_slice()).bind(document.key()).bind(document.value()) 256 .bind(document.value_sha256().as_slice()).execute(&mut **transaction).await.map_err(super::cache::map)?; 257 Ok(()) 258 } 259 260 pub(super) async fn legacy_put( 261 store: &SqliteStorage, 262 id: ProjectionId, 263 generation: ProjectionGeneration, 264 document: ProjectionDocument, 265 ) -> Result<(), Error> { 266 store.require_projection_writer()?; 267 let mut transaction = store 268 .pool() 269 .begin_with("BEGIN IMMEDIATE") 270 .await 271 .map_err(super::cache::map)?; 272 if super::cache::owner(&mut transaction, &id).await? { 273 return Err(Error::ProjectionRevisionConflict); 274 } 275 let registered: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?") 276 .bind(id.as_str()).bind(generation.as_bytes().as_slice()).fetch_optional(&mut *transaction).await.map_err(super::cache::map)?; 277 if registered.is_some() { 278 return Err(Error::ProjectionRevisionConflict); 279 } 280 put(&mut transaction, &id, generation, &document).await?; 281 transaction.commit().await.map_err(super::cache::map) 282 } 283 284 pub(super) async fn load_document( 285 store: &SqliteStorage, 286 id: ProjectionId, 287 generation: ProjectionGeneration, 288 key: String, 289 ) -> Result<Option<ProjectionDocument>, Error> { 290 radroots_storage::projection::document_query::ProjectionDocumentRecord::corrupt( 291 generation, 292 key.clone(), 293 ) 294 .map_err(|_| Error::InvalidProjectionDocument)?; 295 let mut transaction = store.pool().begin().await.map_err(super::cache::map)?; 296 let metadata = sqlx::query("SELECT length(value) AS value_bytes, CASE WHEN length(value_sha256)=32 THEN value_sha256 END AS value_sha256 FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? AND document_key=?") 297 .bind(id.as_str()).bind(generation.as_bytes().as_slice()).bind(&key) 298 .fetch_optional(&mut *transaction).await.map_err(super::cache::map)?; 299 let document = if let Some(row) = metadata { 300 let length: i64 = row.try_get("value_bytes").map_err(super::map_corrupt)?; 301 if length <= 0 302 || length > radroots_storage::projection::PROJECTION_DOCUMENT_VALUE_MAX_BYTES as i64 303 { 304 return Err(Error::CorruptProjectionDocument); 305 } 306 let digest: Vec<u8> = row 307 .try_get("value_sha256") 308 .map_err(|_| Error::CorruptProjectionDocument)?; 309 let digest: [u8; 32] = digest 310 .try_into() 311 .map_err(|_| Error::CorruptProjectionDocument)?; 312 let value: Vec<u8> = sqlx::query_scalar("SELECT value FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? AND document_key=?") 313 .bind(id.as_str()).bind(generation.as_bytes().as_slice()).bind(&key) 314 .fetch_one(&mut *transaction).await.map_err(super::cache::map)?; 315 Some(ProjectionDocument::from_stored_parts(key, value, digest)?) 316 } else { 317 None 318 }; 319 transaction.rollback().await.map_err(super::cache::map)?; 320 Ok(document) 321 } 322 323 #[cfg(test)] 324 #[path = "batch_tests.rs"] 325 mod tests;