lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

native_stock_deficit_tests.rs (24593B)


      1 //! Real signed/installed acceptance, owner-private loss and protected reopen.
      2 use super::*;
      3 use radroots_domain::{
      4     Sequence,
      5     stock::{AvailableStockCorrection, StockTicks},
      6 };
      7 use radroots_storage::inventory::InventoryReceipt;
      8 
      9 #[tokio::test]
     10 async fn current_inspection_reports_aggregate_backing_instead_of_old_receipt_totals() {
     11     use AvailableStockCorrection::{Add, Remove, Spoil};
     12     use sqlx::Connection;
     13     let r = Rig::new().await;
     14     let (_command, _status) = r.accept().await;
     15     let (_, _, minimum) = r.minima().await;
     16     let initial = r
     17         .client
     18         .stock(&r.scope)
     19         .inspect(minimum, PoolId::from_bytes([80; 32]))
     20         .await
     21         .unwrap()
     22         .unwrap();
     23     assert_eq!(initial.checkpoint(), minimum);
     24     assert_eq!(initial.installed_obligations().remaining().get(), 3);
     25     assert_eq!(initial.installed_obligations().backed().get(), 3);
     26     assert_eq!(initial.installed_obligations().deficit(), StockTicks::ZERO);
     27     let first = committed(
     28         correct(
     29             &r,
     30             130,
     31             initial.pool().snapshot().version().get(),
     32             Spoil,
     33             "8",
     34             "first synthetic loss",
     35         )
     36         .await
     37         .unwrap(),
     38     );
     39     let current = r
     40         .client
     41         .stock(&r.scope)
     42         .inspect(first.checkpoint(), first.pool().id())
     43         .await
     44         .unwrap()
     45         .unwrap();
     46     assert_eq!(current.checkpoint(), first.checkpoint());
     47     assert_eq!(current.installed_obligations().remaining().get(), 3);
     48     assert_eq!(current.installed_obligations().backed().get(), 2);
     49     assert_eq!(current.installed_obligations().deficit().get(), 1);
     50     let second = committed(
     51         correct(
     52             &r,
     53             131,
     54             first.pool().version().get(),
     55             Remove,
     56             "1",
     57             "second synthetic loss",
     58         )
     59         .await
     60         .unwrap(),
     61     );
     62     let added = committed(
     63         correct(
     64             &r,
     65             132,
     66             second.pool().version().get(),
     67             Add,
     68             "2",
     69             "explicit new source",
     70         )
     71         .await
     72         .unwrap(),
     73     );
     74     let current = r
     75         .client
     76         .stock(&r.scope)
     77         .inspect(added.checkpoint(), added.pool().id())
     78         .await
     79         .unwrap()
     80         .unwrap();
     81     assert_eq!(first.deficits()[0].backing().deficit().get(), 1);
     82     assert_eq!(current.installed_obligations().remaining().get(), 3);
     83     assert_eq!(current.installed_obligations().backed().get(), 1);
     84     assert_eq!(current.installed_obligations().deficit().get(), 2);
     85     assert_eq!(
     86         current
     87             .pool()
     88             .snapshot()
     89             .balances()
     90             .physical()
     91             .available
     92             .get(),
     93         2
     94     );
     95     assert_eq!(
     96         format!("{current:?}"),
     97         "StockSnapshot(<private current metadata>)"
     98     );
     99     // Read fences do not append operations or reinterpret absent projections.
    100     assert_eq!(r.minima().await.2, added.checkpoint());
    101     let mut fault = sqlx::SqliteConnection::connect(&format!(
    102         "sqlite://{}",
    103         r._temp
    104             .path()
    105             .join("stock/stock_execution_v1.sqlite")
    106             .display()
    107     ))
    108     .await
    109     .unwrap();
    110     sqlx::query("DELETE FROM stock_deficits WHERE operation_id=?")
    111         .bind(first.attempt().operation_id().as_bytes().as_slice())
    112         .execute(&mut fault)
    113         .await
    114         .unwrap();
    115     assert_eq!(
    116         r.client
    117             .stock(&r.scope)
    118             .inspect(added.checkpoint(), added.pool().id())
    119             .await
    120             .unwrap_err()
    121             .code(),
    122         crate::Code::RecoveryRequired
    123     );
    124     let original = first.deficits()[0];
    125     sqlx::query("INSERT INTO stock_deficits(actor,operation_id,acceptance_id,installation_id,line_id,pool_id) VALUES(?,?,?,?,?,?)")
    126         .bind(r.enrollment.input().participant().as_bytes().as_slice()).bind(first.attempt().operation_id().as_bytes().as_slice())
    127         .bind(original.acceptance().as_bytes().as_slice()).bind(original.installation().as_bytes().as_slice())
    128         .bind(original.line().as_bytes().as_slice()).bind(original.pool().as_bytes().as_slice()).execute(&mut fault).await.unwrap();
    129     fault.close().await.unwrap();
    130     r.clock.0.store(0, Ordering::SeqCst);
    131     assert!(
    132         r.client
    133             .stock(&r.scope)
    134             .inspect(added.checkpoint(), added.pool().id())
    135             .await
    136             .is_err()
    137     );
    138     r.clock.0.store(150, Ordering::SeqCst);
    139     assert_eq!(r.control.signs.load(Ordering::SeqCst), 2);
    140     assert_eq!(r.io.wraps.load(Ordering::SeqCst), 2);
    141     assert_eq!(r.io.sent.lock().unwrap().len(), 2);
    142     r.client.close().await.unwrap();
    143     let crypto = WorkspaceCrypto::new(
    144         r.vault.clone(),
    145         SecretRef::new(
    146             SecretId::parse("workspace-key").unwrap(),
    147             BackendKind::File,
    148             KeyVersion::new(1).unwrap(),
    149         ),
    150         SecretMaterial::from_slice(&[0x41; 32]).unwrap(),
    151         AccessPolicy::standard(),
    152     )
    153     .unwrap();
    154     let reopened = NativeWorkspace::open_seller_stock(
    155         &r._temp.path().join("stock"),
    156         r.enrollment.input().clone(),
    157         crypto,
    158         r.clock.clone(),
    159     )
    160     .await
    161     .unwrap();
    162     let binding = reopened
    163         .authenticate(r.enrollment.input().clone(), r.enrollment.proof())
    164         .await
    165         .unwrap();
    166     let recovered = reopened
    167         .inspect_stock(
    168             binding,
    169             added.checkpoint(),
    170             added.pool().id(),
    171             proof(r.enrollment.proof()),
    172         )
    173         .await
    174         .unwrap()
    175         .unwrap();
    176     assert_eq!(recovered, current);
    177     reopened.close().await.unwrap();
    178 }
    179 
    180 #[tokio::test]
    181 async fn current_inspection_keeps_pending_claims_separate_from_installed_obligations() {
    182     let r = Arc::new(Rig::new().await);
    183     let (_, _, minimum) = r.minima().await;
    184     assert!(
    185         r.client
    186             .stock(&r.scope)
    187             .inspect(minimum, PoolId::from_bytes([99; 32]))
    188             .await
    189             .unwrap()
    190             .is_none()
    191     );
    192     r.control.pause_peer.store(true, Ordering::SeqCst);
    193     let task = {
    194         let r = r.clone();
    195         tokio::spawn(async move { r.accept().await })
    196     };
    197     r.control.entered.notified().await;
    198     let (_, _, minimum) = r.minima().await;
    199     let signs = r.control.signs.load(Ordering::SeqCst);
    200     let pending = r
    201         .client
    202         .stock(&r.scope)
    203         .inspect(minimum, PoolId::from_bytes([80; 32]))
    204         .await
    205         .unwrap()
    206         .unwrap();
    207     assert_eq!(
    208         pending
    209             .pool()
    210             .snapshot()
    211             .balances()
    212             .physical()
    213             .preparatory_hold
    214             .get(),
    215         3
    216     );
    217     assert_eq!(
    218         pending
    219             .pool()
    220             .snapshot()
    221             .balances()
    222             .physical()
    223             .available
    224             .get(),
    225         7
    226     );
    227     assert_eq!(
    228         pending.installed_obligations().remaining(),
    229         StockTicks::ZERO
    230     );
    231     assert_eq!(pending.installed_obligations().backed(), StockTicks::ZERO);
    232     assert_eq!(pending.installed_obligations().deficit(), StockTicks::ZERO);
    233     assert_eq!(r.control.signs.load(Ordering::SeqCst), signs);
    234     r.control.release.notify_one();
    235     task.await.unwrap();
    236     let (_, _, minimum) = r.minima().await;
    237     let installed = r
    238         .client
    239         .stock(&r.scope)
    240         .inspect(minimum, pending.pool().snapshot().id())
    241         .await
    242         .unwrap()
    243         .unwrap();
    244     assert_eq!(installed.installed_obligations().remaining().get(), 3);
    245     assert_eq!(installed.installed_obligations().backed().get(), 3);
    246     assert_eq!(
    247         installed
    248             .pool()
    249             .snapshot()
    250             .balances()
    251             .physical()
    252             .preparatory_hold,
    253         StockTicks::ZERO
    254     );
    255     r.client.close().await.unwrap();
    256 }
    257 
    258 async fn correct(
    259     r: &Rig,
    260     id: u8,
    261     version: u64,
    262     kind: AvailableStockCorrection,
    263     amount: &str,
    264     reason: &str,
    265 ) -> crate::Result<InventoryOutcome> {
    266     let (_, _, minimum) = r.minima().await;
    267     let context = r
    268         .scope
    269         .command_context(
    270             OperationId::from_bytes([id; 32]),
    271             None,
    272             None,
    273             UnixSeconds::new(230).unwrap(),
    274         )
    275         .unwrap();
    276     let input = radroots_protocol::command::PoolCorrectionInput::new(
    277         PoolId::from_bytes([80; 32]),
    278         Sequence::new(version).unwrap(),
    279         kind,
    280         amount.parse().unwrap(),
    281         reason,
    282     )
    283     .unwrap();
    284     if kind == AvailableStockCorrection::Spoil {
    285         r.client
    286             .stock(&r.scope)
    287             .spoil(context, minimum, input)
    288             .await
    289     } else {
    290         r.client
    291             .stock(&r.scope)
    292             .adjust(context, minimum, input)
    293             .await
    294     }
    295 }
    296 fn committed(outcome: InventoryOutcome) -> InventoryReceipt {
    297     let InventoryOutcome::Committed(receipt) = outcome else {
    298         panic!("actual correction must confirm its transaction");
    299     };
    300     *receipt
    301 }
    302 
    303 async fn failed_native_correction(uncertain_commit: bool) {
    304     use sqlx::Connection;
    305     let r = Rig::new().await;
    306     let (_, installed) = r.accept().await;
    307     let (_, _, minimum) = r.minima().await;
    308     let before = r
    309         .client
    310         .stock(&r.scope)
    311         .pool(minimum, PoolId::from_bytes([80; 32]))
    312         .await
    313         .unwrap()
    314         .unwrap()
    315         .snapshot();
    316     let mut fault = sqlx::SqliteConnection::connect(&format!(
    317         "sqlite://{}",
    318         r._temp
    319             .path()
    320             .join("stock/stock_execution_v1.sqlite")
    321             .display()
    322     ))
    323     .await
    324     .unwrap();
    325     let counts_sql = "SELECT (SELECT count(*) FROM stock_operations),(SELECT count(*) FROM stock_movements),(SELECT count(*) FROM stock_deficits)";
    326     let counts: (i64, i64, i64) = sqlx::query_as(counts_sql)
    327         .fetch_one(&mut fault)
    328         .await
    329         .unwrap();
    330     if uncertain_commit {
    331         sqlx::raw_sql("CREATE TABLE deficit_deferred_fault(value INTEGER REFERENCES stock_authority(singleton) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER controlled_deficit_failure AFTER INSERT ON stock_deficits BEGIN INSERT INTO deficit_deferred_fault VALUES(2); END").execute(&mut fault).await.unwrap();
    332     } else {
    333         sqlx::query("CREATE TRIGGER controlled_deficit_failure BEFORE INSERT ON stock_deficits BEGIN SELECT RAISE(ABORT, 'controlled_deficit_failure'); END").execute(&mut fault).await.unwrap();
    334     }
    335     let result = correct(
    336         &r,
    337         120,
    338         before.version().get(),
    339         AvailableStockCorrection::Spoil,
    340         "8",
    341         "synthetic loss at actual failed transaction",
    342     )
    343     .await;
    344     if uncertain_commit {
    345         let InventoryOutcome::Unknown(anchor) = result.unwrap() else {
    346             panic!("actual COMMIT failure must remain unknown");
    347         };
    348         assert_eq!(anchor.operation_id(), OperationId::from_bytes([120; 32]));
    349         assert!(anchor.transaction_id().is_some());
    350         assert_eq!(
    351             r.native
    352                 .inventory_operation(
    353                     r.scope.binding().clone(),
    354                     minimum,
    355                     anchor.operation_id(),
    356                     Some(anchor.command_digest()),
    357                     proof(r.enrollment.proof())
    358                 )
    359                 .await
    360                 .unwrap_err(),
    361             OperationError::RecoveryRequired
    362         );
    363     } else {
    364         assert_eq!(result.unwrap_err().code(), crate::Code::StorageFailure);
    365     }
    366     assert_eq!(
    367         sqlx::query_as::<_, (i64, i64, i64)>(counts_sql)
    368             .fetch_one(&mut fault)
    369             .await
    370             .unwrap(),
    371         counts
    372     );
    373     assert_eq!(r.control.signs.load(Ordering::SeqCst), 2);
    374     assert_eq!(r.io.wraps.load(Ordering::SeqCst), 2);
    375     assert_eq!(r.io.sent.lock().unwrap().len(), 2);
    376     // No obtained material is pending for an owner-private correction. Close
    377     // may release this original writer without a current clock or new effect.
    378     r.clock.0.store(0, Ordering::SeqCst);
    379     r.client.close().await.unwrap();
    380     sqlx::raw_sql(
    381         "DROP TRIGGER controlled_deficit_failure; DROP TABLE IF EXISTS deficit_deferred_fault",
    382     )
    383     .execute(&mut fault)
    384     .await
    385     .unwrap();
    386     fault.close().await.unwrap();
    387     r.clock.0.store(150, Ordering::SeqCst);
    388     let crypto = WorkspaceCrypto::new(
    389         r.vault.clone(),
    390         SecretRef::new(
    391             SecretId::parse("workspace-key").unwrap(),
    392             BackendKind::File,
    393             KeyVersion::new(1).unwrap(),
    394         ),
    395         SecretMaterial::from_slice(&[0x41; 32]).unwrap(),
    396         AccessPolicy::standard(),
    397     )
    398     .unwrap();
    399     let reopened = NativeWorkspace::open_seller_stock(
    400         &r._temp.path().join("stock"),
    401         r.enrollment.input().clone(),
    402         crypto,
    403         r.clock.clone(),
    404     )
    405     .await
    406     .unwrap();
    407     let binding = reopened
    408         .authenticate(r.enrollment.input().clone(), r.enrollment.proof())
    409         .await
    410         .unwrap();
    411     let pool = reopened
    412         .pool(
    413             binding.clone(),
    414             minimum,
    415             before.id(),
    416             proof(r.enrollment.proof()),
    417         )
    418         .await
    419         .unwrap()
    420         .unwrap();
    421     assert_eq!(pool.snapshot(), before);
    422     assert!(
    423         reopened
    424             .inventory_operation(
    425                 binding.clone(),
    426                 minimum,
    427                 OperationId::from_bytes([120; 32]),
    428                 None,
    429                 proof(r.enrollment.proof())
    430             )
    431             .await
    432             .unwrap()
    433             .is_none()
    434     );
    435     let current = reopened
    436         .stock_authority(r.enrollment.input().clone(), r.enrollment.proof())
    437         .await
    438         .unwrap();
    439     assert_eq!(current.inventory_checkpoint(), Some(installed.checkpoint()));
    440     reopened.close().await.unwrap();
    441 }
    442 
    443 #[tokio::test]
    444 async fn native_inventory_proof_work_reserves_the_live_queue_budget_until_sql_confirmation() {
    445     inventory_read_budget(false).await;
    446 }
    447 
    448 #[tokio::test]
    449 async fn native_inspection_reserves_the_live_queue_budget_until_sql_confirmation() {
    450     inventory_read_budget(true).await;
    451 }
    452 
    453 async fn inventory_read_budget(inspect: bool) {
    454     use sqlx::Connection;
    455     let r = Rig::new().await;
    456     let (_, _, minimum) = r.minima().await;
    457     let pool = PoolId::from_bytes([80; 32]);
    458     let mut blocker = sqlx::SqliteConnection::connect(&format!(
    459         "sqlite://{}",
    460         r._temp
    461             .path()
    462             .join("stock/stock_execution_v1.sqlite")
    463             .display()
    464     ))
    465     .await
    466     .unwrap();
    467     let transaction = blocker.begin_with("BEGIN IMMEDIATE").await.unwrap();
    468     let read = || async {
    469         if inspect {
    470             r.native
    471                 .inspect_stock(
    472                     r.scope.binding().clone(),
    473                     minimum,
    474                     pool,
    475                     proof(r.enrollment.proof()),
    476                 )
    477                 .await
    478                 .map(|value| value.map(|value| value.pool().clone()))
    479         } else {
    480             r.native
    481                 .pool(
    482                     r.scope.binding().clone(),
    483                     minimum,
    484                     pool,
    485                     proof(r.enrollment.proof()),
    486                 )
    487                 .await
    488         }
    489     };
    490     let mut reads: Vec<_> = (0..4).map(|_| Box::pin(read())).collect();
    491     for read in &mut reads {
    492         assert!(futures::poll!(read.as_mut()).is_pending());
    493     }
    494     let fifth = read();
    495     let error = tokio::time::timeout(std::time::Duration::from_secs(1), fifth)
    496         .await
    497         .expect("full queue must refuse without waiting for its blocked writer")
    498         .unwrap_err();
    499     assert_eq!(
    500         error,
    501         OperationError::Workspace(WorkspaceError::Unavailable)
    502     );
    503     transaction.rollback().await.unwrap();
    504     for result in futures::future::join_all(reads).await {
    505         assert_eq!(
    506             result
    507                 .unwrap()
    508                 .unwrap()
    509                 .snapshot()
    510                 .balances()
    511                 .physical()
    512                 .available
    513                 .get(),
    514             10
    515         );
    516     }
    517     blocker.close().await.unwrap();
    518     r.client.close().await.unwrap();
    519 }
    520 
    521 #[tokio::test]
    522 async fn native_deficit_statement_failure_rolls_back_pool_movement_obligation_and_root() {
    523     failed_native_correction(false).await;
    524 }
    525 
    526 #[tokio::test]
    527 async fn native_deficit_commit_failure_stays_unknown_and_reopen_retains_original_backing() {
    528     failed_native_correction(true).await;
    529 }
    530 
    531 #[tokio::test]
    532 async fn installed_backing_loss_retains_obligation_exact_replay_deficits_and_reopen() {
    533     use AvailableStockCorrection::{Add, Remove, Spoil};
    534     let r = Rig::new().await;
    535     let original_request = r.request().await;
    536     let line = &original_request.terms().lines()[0];
    537     let key =
    538         radroots_domain::stock::OptionMappingKey::new(line.source().coordinate(), line.option_id());
    539     let (acceptance, installed) = r.accept().await;
    540     let (_, _, minimum) = r.minima().await;
    541     let before = r
    542         .client
    543         .stock(&r.scope)
    544         .pool(minimum, PoolId::from_bytes([80; 32]))
    545         .await
    546         .unwrap()
    547         .unwrap()
    548         .snapshot();
    549     assert_eq!(before.balances().physical().available.get(), 7);
    550     assert_eq!(before.balances().physical().committed_hold.get(), 3);
    551     // A later mapping cannot change the original three-tick obligation into
    552     // a six-tick obligation or redirect this loss to a newly selected plan.
    553     let (_, _, mapping_minimum) = r.minima().await;
    554     let mapping = r
    555         .client
    556         .stock(&r.scope)
    557         .option_mapping(mapping_minimum, key)
    558         .await
    559         .unwrap()
    560         .unwrap();
    561     let context = r
    562         .scope
    563         .command_context(
    564             OperationId::from_bytes([109; 32]),
    565             None,
    566             None,
    567             UnixSeconds::new(230).unwrap(),
    568         )
    569         .unwrap();
    570     let changed = radroots_protocol::command::MapOptionInput::new(
    571         key,
    572         before.id(),
    573         radroots_domain::Quantity::new("2".parse().unwrap(), before.unit().unit()).unwrap(),
    574         mapping.version(),
    575     )
    576     .unwrap();
    577     r.client
    578         .stock(&r.scope)
    579         .map_option(context, mapping_minimum, changed)
    580         .await
    581         .unwrap();
    582     let first = committed(
    583         correct(
    584             &r,
    585             110,
    586             before.version().get(),
    587             Spoil,
    588             "8",
    589             "synthetic recorded spoilage",
    590         )
    591         .await
    592         .unwrap(),
    593     );
    594     let physical = first.pool().balances().physical();
    595     assert_eq!(physical.available, StockTicks::ZERO);
    596     assert_eq!(physical.committed_hold.get(), 2);
    597     assert_eq!(physical.waste.get(), 8);
    598     assert_eq!(first.pool().balances().introduced().get(), 10);
    599     assert_eq!(first.deficits().len(), 1);
    600     let deficit = first.deficits()[0];
    601     assert_eq!(deficit.acceptance(), acceptance.context().operation_id());
    602     assert_eq!(deficit.pool(), before.id());
    603     assert_eq!(deficit.backing().remaining().get(), 3);
    604     assert_eq!(deficit.backing().backed().get(), 2);
    605     assert_eq!(deficit.backing().deficit().get(), 1);
    606     assert_eq!(
    607         format!("{deficit:?}"),
    608         "StockDeficit(<private effect metadata>)"
    609     );
    610     let repeat = committed(
    611         correct(
    612             &r,
    613             110,
    614             before.version().get(),
    615             Spoil,
    616             "8",
    617             "synthetic recorded spoilage",
    618         )
    619         .await
    620         .unwrap(),
    621     );
    622     assert_eq!(repeat, first);
    623     assert_eq!(
    624         correct(
    625             &r,
    626             110,
    627             before.version().get(),
    628             Spoil,
    629             "8",
    630             "changed captured reason"
    631         )
    632         .await
    633         .unwrap_err()
    634         .code(),
    635         crate::Code::IdempotencyConflict
    636     );
    637     assert_eq!(
    638         correct(&r, 111, before.version().get(), Spoil, "1", "stale version")
    639             .await
    640             .unwrap_err()
    641             .code(),
    642         crate::Code::StaleState
    643     );
    644     let second = committed(
    645         correct(
    646             &r,
    647             112,
    648             first.pool().version().get(),
    649             Remove,
    650             "2",
    651             "synthetic recorded removal",
    652         )
    653         .await
    654         .unwrap(),
    655     );
    656     assert_eq!(
    657         second.pool().balances().physical().committed_hold,
    658         StockTicks::ZERO
    659     );
    660     assert_eq!(second.pool().balances().physical().removed.get(), 2);
    661     assert_eq!(second.deficits()[0].backing().remaining().get(), 3);
    662     assert_eq!(second.deficits()[0].backing().backed(), StockTicks::ZERO);
    663     assert_eq!(second.deficits()[0].backing().deficit().get(), 3);
    664     assert_eq!(
    665         correct(
    666             &r,
    667             113,
    668             second.pool().version().get(),
    669             Spoil,
    670             "1",
    671             "cannot deplete an unbacked obligation"
    672         )
    673         .await
    674         .unwrap_err()
    675         .code(),
    676         crate::Code::StaleState
    677     );
    678     let added = committed(
    679         correct(
    680             &r,
    681             114,
    682             second.pool().version().get(),
    683             Add,
    684             "2",
    685             "explicit new physical source",
    686         )
    687         .await
    688         .unwrap(),
    689     );
    690     assert!(added.deficits().is_empty());
    691     assert_eq!(added.pool().balances().physical().available.get(), 2);
    692     assert_eq!(
    693         added.pool().balances().physical().committed_hold,
    694         StockTicks::ZERO
    695     );
    696     assert_eq!(added.pool().balances().introduced().get(), 12);
    697     let (_, _, minimum) = r.minima().await;
    698     let old = r
    699         .client
    700         .stock(&r.scope)
    701         .operation(
    702             minimum,
    703             first.attempt().operation_id(),
    704             Some(first.attempt().command_digest()),
    705         )
    706         .await
    707         .unwrap()
    708         .unwrap();
    709     assert_eq!(old, first);
    710     // The current minimum lies after this original loss. It must not permit
    711     // trusting a forged historical deficit projection beneath that minimum.
    712     use sqlx::Connection;
    713     let mut fault = sqlx::SqliteConnection::connect(&format!(
    714         "sqlite://{}",
    715         r._temp
    716             .path()
    717             .join("stock/stock_execution_v1.sqlite")
    718             .display()
    719     ))
    720     .await
    721     .unwrap();
    722     sqlx::query("UPDATE stock_deficits SET installation_id=acceptance_id WHERE operation_id=?")
    723         .bind(first.attempt().operation_id().as_bytes().as_slice())
    724         .execute(&mut fault)
    725         .await
    726         .unwrap();
    727     assert_eq!(
    728         r.client
    729             .stock(&r.scope)
    730             .operation(minimum, first.attempt().operation_id(), None)
    731             .await
    732             .unwrap_err()
    733             .code(),
    734         crate::Code::RecoveryRequired
    735     );
    736     sqlx::query("UPDATE stock_deficits SET installation_id=? WHERE operation_id=?")
    737         .bind(deficit.installation().as_bytes().as_slice())
    738         .bind(first.attempt().operation_id().as_bytes().as_slice())
    739         .execute(&mut fault)
    740         .await
    741         .unwrap();
    742     fault.close().await.unwrap();
    743     let status = r
    744         .client
    745         .orders(&r.scope)
    746         .acceptance_status(
    747             minimum,
    748             acceptance.context().operation_id(),
    749             acceptance.digest(),
    750         )
    751         .await
    752         .unwrap();
    753     assert_eq!(status.installation(), installed.installation());
    754     assert_eq!(r.control.signs.load(Ordering::SeqCst), 2);
    755     assert_eq!(r.io.wraps.load(Ordering::SeqCst), 2);
    756     assert_eq!(r.io.sent.lock().unwrap().len(), 2);
    757     r.client.close().await.unwrap();
    758     let crypto = WorkspaceCrypto::new(
    759         r.vault.clone(),
    760         SecretRef::new(
    761             SecretId::parse("workspace-key").unwrap(),
    762             BackendKind::File,
    763             KeyVersion::new(1).unwrap(),
    764         ),
    765         SecretMaterial::from_slice(&[0x41; 32]).unwrap(),
    766         AccessPolicy::standard(),
    767     )
    768     .unwrap();
    769     let reopened = NativeWorkspace::open_seller_stock(
    770         &r._temp.path().join("stock"),
    771         r.enrollment.input().clone(),
    772         crypto,
    773         r.clock.clone(),
    774     )
    775     .await
    776     .unwrap();
    777     let binding = reopened
    778         .authenticate(r.enrollment.input().clone(), r.enrollment.proof())
    779         .await
    780         .unwrap();
    781     let recovered = reopened
    782         .inventory_operation(
    783             binding,
    784             minimum,
    785             second.attempt().operation_id(),
    786             Some(second.attempt().command_digest()),
    787             proof(r.enrollment.proof()),
    788         )
    789         .await
    790         .unwrap()
    791         .unwrap();
    792     assert_eq!(recovered, second);
    793     reopened.close().await.unwrap();
    794 }