lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

batch.rs (15724B)


      1 use super::{SqliteStorage, decode_status, put_status_transaction};
      2 use radroots_storage::{
      3     Error,
      4     event::EventSourceToken,
      5     projection::{
      6         ProjectionCheckpoint, ProjectionDocument, ProjectionGeneration, ProjectionHealth,
      7         ProjectionId, ProjectionStatus, batch::ProjectionDocumentBatch,
      8     },
      9 };
     10 use sqlx::{Row, Sqlite, Transaction};
     11 
     12 pub(super) async fn fence(
     13     transaction: &mut Transaction<'_, Sqlite>,
     14     store: &SqliteStorage,
     15     source: EventSourceToken,
     16     id: &ProjectionId,
     17     expected: Option<ProjectionGeneration>,
     18 ) -> Result<u64, Error> {
     19     if source.generation() != store.generation {
     20         return Err(Error::SourceEpochChanged);
     21     }
     22     let row = sqlx::query("SELECT mutation_epoch, sequence_head FROM radroots_runtime_source_generations WHERE generation = ? AND state = 'active'")
     23         .bind(source.generation().as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(super::cache::map)?
     24         .ok_or(Error::SourceEpochChanged)?;
     25     let epoch = super::u64_from_i64(row.try_get("mutation_epoch").map_err(super::map_corrupt)?)?;
     26     if epoch != source.epoch() {
     27         return Err(Error::SourceEpochChanged);
     28     }
     29     let status = sqlx::query(
     30         "SELECT * FROM radroots_runtime_projection_checkpoints WHERE projection_id = ?",
     31     )
     32     .bind(id.as_str())
     33     .fetch_optional(&mut **transaction)
     34     .await
     35     .map_err(super::cache::map)?
     36     .as_ref()
     37     .map(decode_status)
     38     .transpose()?;
     39     if status.as_ref().map(ProjectionStatus::generation) != expected
     40         || status
     41             .as_ref()
     42             .is_some_and(|value| value.health() != ProjectionHealth::Ready)
     43     {
     44         return Err(Error::ProjectionRevisionConflict);
     45     }
     46     super::u64_from_i64(row.try_get("sequence_head").map_err(super::map_corrupt)?)
     47 }
     48 
     49 pub(super) async fn read_fence(
     50     transaction: &mut Transaction<'_, Sqlite>,
     51     store: &SqliteStorage,
     52     source: EventSourceToken,
     53     id: &ProjectionId,
     54     active: ProjectionGeneration,
     55 ) -> Result<(), Error> {
     56     fence(transaction, store, source, id, Some(active)).await?;
     57     let row = sqlx::query("SELECT source_generation, source_epoch, phase FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?")
     58         .bind(id.as_str()).bind(active.as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(super::cache::map)?
     59         .ok_or(Error::ProjectionRevisionConflict)?;
     60     let phase: String = row.try_get("phase").map_err(super::map_corrupt)?;
     61     if phase != "promoted" {
     62         return Err(Error::ProjectionRevisionConflict);
     63     }
     64     let generation: Vec<u8> = row
     65         .try_get("source_generation")
     66         .map_err(super::map_corrupt)?;
     67     let epoch = super::u64_from_i64(row.try_get("source_epoch").map_err(super::map_corrupt)?)?;
     68     if generation != source.generation().as_bytes() || epoch != source.epoch() {
     69         return Err(Error::SourceEpochChanged);
     70     }
     71     Ok(())
     72 }
     73 
     74 pub(super) async fn refuse_registered_legacy(
     75     transaction: &mut Transaction<'_, Sqlite>,
     76     id: &ProjectionId,
     77     generation: ProjectionGeneration,
     78 ) -> Result<(), Error> {
     79     let registered: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?")
     80         .bind(id.as_str()).bind(generation.as_bytes().as_slice()).fetch_optional(&mut **transaction).await.map_err(super::cache::map)?;
     81     if registered.is_some() {
     82         return Err(Error::ProjectionRevisionConflict);
     83     }
     84     Ok(())
     85 }
     86 
     87 pub(super) async fn write(
     88     store: &SqliteStorage,
     89     batch: ProjectionDocumentBatch,
     90     checkpoint: Option<ProjectionCheckpoint>,
     91 ) -> Result<Option<ProjectionStatus>, Error> {
     92     store.require_projection_writer()?;
     93     let mut transaction = store
     94         .pool()
     95         .begin_with("BEGIN IMMEDIATE")
     96         .await
     97         .map_err(super::cache::map)?;
     98     let scope = batch.scope();
     99     let sequence = fence(
    100         &mut transaction,
    101         store,
    102         scope.source(),
    103         scope.projection_id(),
    104         scope.expected_active(),
    105     )
    106     .await?;
    107     let prior = sqlx::query("SELECT source_generation, source_epoch, expected_active, phase, document_rows, value_bytes FROM radroots_runtime_projection_generations WHERE projection_id = ? AND generation = ?")
    108         .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice())
    109         .fetch_optional(&mut *transaction).await.map_err(super::cache::map)?;
    110     let (mut count, mut bytes) = if let Some(row) = prior {
    111         let generation: Vec<u8> = row
    112             .try_get("source_generation")
    113             .map_err(super::map_corrupt)?;
    114         let epoch = super::u64_from_i64(row.try_get("source_epoch").map_err(super::map_corrupt)?)?;
    115         let active: Option<Vec<u8>> = row.try_get("expected_active").map_err(super::map_corrupt)?;
    116         let phase: String = row.try_get("phase").map_err(super::map_corrupt)?;
    117         if generation != scope.source().generation().as_bytes()
    118             || epoch != scope.source().epoch()
    119             || active.as_deref()
    120                 != scope
    121                     .expected_active()
    122                     .as_ref()
    123                     .map(|value| value.as_bytes().as_slice())
    124             || phase != "staging"
    125         {
    126             return Err(Error::ProjectionRevisionConflict);
    127         }
    128         (
    129             super::u64_from_i64(row.try_get("document_rows").map_err(super::map_corrupt)?)?,
    130             super::u64_from_i64(row.try_get("value_bytes").map_err(super::map_corrupt)?)?,
    131         )
    132     } else {
    133         let staging: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id = ? AND phase = 'staging' LIMIT 1")
    134             .bind(scope.projection_id().as_str()).fetch_optional(&mut *transaction).await.map_err(super::cache::map)?;
    135         let existing: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_documents WHERE projection_id = ? AND generation = ? LIMIT 1")
    136             .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice())
    137             .fetch_optional(&mut *transaction).await.map_err(super::cache::map)?;
    138         if staging.is_some() || existing.is_some() {
    139             return Err(Error::ProjectionRevisionConflict);
    140         }
    141         sqlx::query("INSERT INTO radroots_runtime_projection_generations (projection_id,generation,source_generation,source_epoch,expected_active,phase) VALUES (?,?,?,?,?,'staging')")
    142             .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice())
    143             .bind(scope.source().generation().as_bytes().as_slice()).bind(super::i64_from_u64(scope.source().epoch())?)
    144             .bind(scope.expected_active().map(|value| value.as_bytes().to_vec())).execute(&mut *transaction).await.map_err(super::cache::map)?;
    145         (0, 0)
    146     };
    147     let mut writes = Vec::with_capacity(batch.documents().len());
    148     let mut replaced_bytes = 0u64;
    149     let mut replacement_bytes = 0u64;
    150     for document in batch.documents() {
    151         let previous: Option<i64> = sqlx::query_scalar("SELECT length(value) FROM radroots_runtime_projection_documents WHERE projection_id = ? AND generation = ? AND document_key = ?")
    152             .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice()).bind(document.key())
    153             .fetch_optional(&mut *transaction).await.map_err(super::cache::map)?;
    154         count = count
    155             .checked_add(u64::from(previous.is_none()))
    156             .ok_or(Error::SpaceInsufficient)?;
    157         let previous = previous.map(super::u64_from_i64).transpose()?.unwrap_or(0);
    158         replaced_bytes = replaced_bytes
    159             .checked_add(previous)
    160             .ok_or(Error::CorruptProjectionRecord)?;
    161         replacement_bytes = replacement_bytes
    162             .checked_add(document.value().len() as u64)
    163             .ok_or(Error::SpaceInsufficient)?;
    164         writes.push((document, previous));
    165     }
    166     bytes = bytes
    167         .checked_sub(replaced_bytes)
    168         .and_then(|size| size.checked_add(replacement_bytes))
    169         .ok_or(Error::CorruptProjectionRecord)?;
    170     let budget = sqlx::query("SELECT value_bytes,max_value_bytes,batch_value_ceiling FROM radroots_runtime_projection_cache_budget WHERE singleton=1")
    171         .fetch_one(&mut *transaction).await.map_err(super::cache::map)?;
    172     let current_bytes =
    173         super::u64_from_i64(budget.try_get("value_bytes").map_err(super::map_corrupt)?)?;
    174     let limit = budget
    175         .try_get::<Option<i64>, _>("max_value_bytes")
    176         .map_err(super::map_corrupt)?;
    177     if budget
    178         .try_get::<Option<i64>, _>("batch_value_ceiling")
    179         .map_err(super::map_corrupt)?
    180         .is_some()
    181     {
    182         return Err(Error::CorruptProjectionRecord);
    183     }
    184     let final_bytes = current_bytes
    185         .checked_sub(replaced_bytes)
    186         .and_then(|value| value.checked_add(replacement_bytes))
    187         .ok_or(Error::CorruptProjectionRecord)?;
    188     if final_bytes > current_bytes && limit.is_some_and(|limit| final_bytes > limit as u64) {
    189         return Err(Error::SpaceInsufficient);
    190     }
    191     // The immutable policy is unchanged. This reservation exists only inside
    192     // this transaction, admits its final aggregate and rolls back on failure.
    193     sqlx::query("UPDATE radroots_runtime_projection_cache_budget SET batch_value_ceiling=? WHERE singleton=1")
    194         .bind(super::i64_from_u64(final_bytes)?).execute(&mut *transaction).await.map_err(super::cache::map)?;
    195     // Shrink before growth, so no intermediate increase exceeds the admitted total.
    196     writes.sort_by_key(|(document, previous)| document.value().len() as u64 > *previous);
    197     for (document, _) in writes {
    198         put(
    199             &mut transaction,
    200             scope.projection_id(),
    201             scope.generation(),
    202             document,
    203         )
    204         .await?;
    205     }
    206     let promoted = if let Some(checkpoint) = checkpoint {
    207         if checkpoint.projection_id() != scope.projection_id()
    208             || checkpoint.generation() != scope.generation()
    209             || checkpoint.projected_rows() != count
    210             || match checkpoint.source_position() {
    211                 Some(position) => {
    212                     position.generation() != scope.source().generation()
    213                         || position.sequence().get() != sequence
    214                 }
    215                 None => sequence != 0,
    216             }
    217         {
    218             return Err(Error::ProjectionCheckpointMismatch);
    219         }
    220         let status = ProjectionStatus::new(
    221             scope.projection_id().clone(),
    222             scope.generation(),
    223             ProjectionHealth::Ready,
    224             Some(checkpoint),
    225             None,
    226         )?;
    227         sqlx::query("UPDATE radroots_runtime_projection_generations SET phase = 'retired' WHERE projection_id = ? AND phase = 'promoted'")
    228             .bind(scope.projection_id().as_str()).execute(&mut *transaction).await.map_err(super::cache::map)?;
    229         put_status_transaction(&mut transaction, &status).await?;
    230         Some(status)
    231     } else {
    232         None
    233     };
    234     sqlx::query("UPDATE radroots_runtime_projection_generations SET document_rows = ?, value_bytes = ?, phase = ? WHERE projection_id = ? AND generation = ?")
    235         .bind(super::i64_from_u64(count)?).bind(super::i64_from_u64(bytes)?)
    236         .bind(if promoted.is_some() { "promoted" } else { "staging" })
    237         .bind(scope.projection_id().as_str()).bind(scope.generation().as_bytes().as_slice())
    238         .execute(&mut *transaction).await.map_err(super::cache::map)?;
    239     let cleared = sqlx::query("UPDATE radroots_runtime_projection_cache_budget SET batch_value_ceiling=NULL WHERE singleton=1 AND batch_value_ceiling=? AND value_bytes=?")
    240         .bind(super::i64_from_u64(final_bytes)?).bind(super::i64_from_u64(final_bytes)?).execute(&mut *transaction).await.map_err(super::cache::map)?;
    241     if cleared.rows_affected() != 1 {
    242         return Err(Error::CorruptProjectionRecord);
    243     }
    244     transaction.commit().await.map_err(super::cache::map)?;
    245     Ok(promoted)
    246 }
    247 
    248 pub(super) async fn put(
    249     transaction: &mut Transaction<'_, Sqlite>,
    250     id: &ProjectionId,
    251     generation: ProjectionGeneration,
    252     document: &ProjectionDocument,
    253 ) -> Result<(), Error> {
    254     sqlx::query("INSERT INTO radroots_runtime_projection_documents (projection_id,generation,document_key,value,value_sha256) VALUES (?,?,?,?,?) ON CONFLICT(projection_id,generation,document_key) DO UPDATE SET value=excluded.value,value_sha256=excluded.value_sha256")
    255         .bind(id.as_str()).bind(generation.as_bytes().as_slice()).bind(document.key()).bind(document.value())
    256         .bind(document.value_sha256().as_slice()).execute(&mut **transaction).await.map_err(super::cache::map)?;
    257     Ok(())
    258 }
    259 
    260 pub(super) async fn legacy_put(
    261     store: &SqliteStorage,
    262     id: ProjectionId,
    263     generation: ProjectionGeneration,
    264     document: ProjectionDocument,
    265 ) -> Result<(), Error> {
    266     store.require_projection_writer()?;
    267     let mut transaction = store
    268         .pool()
    269         .begin_with("BEGIN IMMEDIATE")
    270         .await
    271         .map_err(super::cache::map)?;
    272     if super::cache::owner(&mut transaction, &id).await? {
    273         return Err(Error::ProjectionRevisionConflict);
    274     }
    275     let registered: Option<i64> = sqlx::query_scalar("SELECT 1 FROM radroots_runtime_projection_generations WHERE projection_id=? AND generation=?")
    276         .bind(id.as_str()).bind(generation.as_bytes().as_slice()).fetch_optional(&mut *transaction).await.map_err(super::cache::map)?;
    277     if registered.is_some() {
    278         return Err(Error::ProjectionRevisionConflict);
    279     }
    280     put(&mut transaction, &id, generation, &document).await?;
    281     transaction.commit().await.map_err(super::cache::map)
    282 }
    283 
    284 pub(super) async fn load_document(
    285     store: &SqliteStorage,
    286     id: ProjectionId,
    287     generation: ProjectionGeneration,
    288     key: String,
    289 ) -> Result<Option<ProjectionDocument>, Error> {
    290     radroots_storage::projection::document_query::ProjectionDocumentRecord::corrupt(
    291         generation,
    292         key.clone(),
    293     )
    294     .map_err(|_| Error::InvalidProjectionDocument)?;
    295     let mut transaction = store.pool().begin().await.map_err(super::cache::map)?;
    296     let metadata = sqlx::query("SELECT length(value) AS value_bytes, CASE WHEN length(value_sha256)=32 THEN value_sha256 END AS value_sha256 FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? AND document_key=?")
    297         .bind(id.as_str()).bind(generation.as_bytes().as_slice()).bind(&key)
    298         .fetch_optional(&mut *transaction).await.map_err(super::cache::map)?;
    299     let document = if let Some(row) = metadata {
    300         let length: i64 = row.try_get("value_bytes").map_err(super::map_corrupt)?;
    301         if length <= 0
    302             || length > radroots_storage::projection::PROJECTION_DOCUMENT_VALUE_MAX_BYTES as i64
    303         {
    304             return Err(Error::CorruptProjectionDocument);
    305         }
    306         let digest: Vec<u8> = row
    307             .try_get("value_sha256")
    308             .map_err(|_| Error::CorruptProjectionDocument)?;
    309         let digest: [u8; 32] = digest
    310             .try_into()
    311             .map_err(|_| Error::CorruptProjectionDocument)?;
    312         let value: Vec<u8> = sqlx::query_scalar("SELECT value FROM radroots_runtime_projection_documents WHERE projection_id=? AND generation=? AND document_key=?")
    313             .bind(id.as_str()).bind(generation.as_bytes().as_slice()).bind(&key)
    314             .fetch_one(&mut *transaction).await.map_err(super::cache::map)?;
    315         Some(ProjectionDocument::from_stored_parts(key, value, digest)?)
    316     } else {
    317         None
    318     };
    319     transaction.rollback().await.map_err(super::cache::map)?;
    320     Ok(document)
    321 }
    322 
    323 #[cfg(test)]
    324 #[path = "batch_tests.rs"]
    325 mod tests;