lib

Core libraries for Radroots
git clone https://radroots.dev/git/lib.git
Log | Files | Refs | README

worker.rs (88700B)


      1 mod dispatch;
      2 use super::{
      3     AttachmentInput, AttachmentProof, WorkspaceBinding, WorkspaceClock, WorkspaceCrypto,
      4     WorkspaceError,
      5     admission::Journal,
      6     fs::{Kind, Namespace},
      7     record::GrantRecord,
      8     schema, stock,
      9 };
     10 use radroots_domain::{CommandDigest, OperationId};
     11 use radroots_storage::command::{
     12     AdmissionOutcome, AdmissionRequest, ArtifactEvidence, BoundSigningPlan, CapturedArtifact,
     13     CapturedSigningPlan, ClaimAttemptOutcome, ClaimAttemptRequest, CommitAttempt, DeliveryAttempt,
     14     DeliveryAttemptOutcome, DeliveryAttemptRequest, DeliveryObservation,
     15     DeliveryObservationOutcome, EnvelopeCandidate, EvidenceCaptureOutcome, InstallationOutcome,
     16     InstallationRequest, OperationError, OperationSnapshot, PlanBindingOutcome, PreparedOperation,
     17     SigningAttempt, SigningTarget,
     18 };
     19 use radroots_transport::sink::DeliveryRequestId;
     20 use sqlx::{
     21     Connection, Row, SqliteConnection,
     22     sqlite::{SqliteConnectOptions, SqliteJournalMode, SqliteSynchronous},
     23 };
     24 use std::{
     25     path::PathBuf,
     26     sync::{
     27         Arc, Mutex,
     28         atomic::{AtomicUsize, Ordering},
     29         mpsc::{self, SyncSender},
     30     },
     31     thread::JoinHandle,
     32     time::Duration,
     33 };
     34 use tokio::sync::{oneshot, watch};
     35 
     36 // Count conservative retained input bounds across queued and in-flight jobs.
     37 // Plan jobs retain both their canonical bytes and bounded decoded fields.
     38 const QUEUE_MAX_BYTES: usize = 4 * 1024 * 1024;
     39 const QUEUE_CAPACITY: usize = 32;
     40 type Reply<T, E = WorkspaceError> = oneshot::Sender<Result<T, E>>;
     41 enum Job {
     42     InstallStockAgreement(
     43         Box<stock::StockInstallationRequest>,
     44         AttachmentProof,
     45         Reply<stock::StockInstallationOutcome, OperationError>,
     46     ),
     47     StockAgreementInstallation(
     48         Box<stock::StockInstallationRequest>,
     49         AttachmentProof,
     50         Reply<Option<stock::StockInstallationReceipt>, OperationError>,
     51     ),
     52     AbortStockPreparation(
     53         Box<stock::StockAbortRequest>,
     54         AttachmentProof,
     55         Reply<stock::StockAbortOutcome, OperationError>,
     56     ),
     57     StockPreparationAbort(
     58         Box<stock::StockAbortRequest>,
     59         AttachmentProof,
     60         Reply<Option<stock::StockAbortReceipt>, OperationError>,
     61     ),
     62     StockSnapshot(
     63         Box<stock::StockSnapshotRequest>,
     64         AttachmentProof,
     65         Reply<stock::StockOperationSnapshot, OperationError>,
     66     ),
     67     StockEvidencePage(
     68         Box<stock::StockClaimRequest>,
     69         Option<radroots_domain::Sequence>,
     70         AttachmentProof,
     71         Reply<stock::StockEvidencePage, OperationError>,
     72     ),
     73     RecoverStockEvidence(
     74         Box<radroots_storage::inventory::StockEvidenceRequest>,
     75         AttachmentProof,
     76         Reply<stock::StockEvidenceOutcome, OperationError>,
     77     ),
     78     CaptureStockUnissued(
     79         Box<stock::issuance::unissued::Request>,
     80         AttachmentProof,
     81         Reply<stock::StockUnissuedOutcome, OperationError>,
     82     ),
     83     RecoverStockUnissued(
     84         Box<stock::issuance::unissued::Request>,
     85         AttachmentProof,
     86         Reply<stock::StockUnissuedOutcome, OperationError>,
     87     ),
     88     StockUnissued(
     89         Box<stock::StockClaimRequest>,
     90         AttachmentProof,
     91         Reply<Option<stock::StockUnissuedReceipt>, OperationError>,
     92     ),
     93     PrepareStockDelivery(
     94         Box<radroots_storage::inventory::StockDeliveryRequest>,
     95         AttachmentProof,
     96         Reply<radroots_storage::inventory::StockDeliveryAttemptOutcome, OperationError>,
     97     ),
     98     StockDeliveryAttempt(
     99         Box<radroots_storage::inventory::StockEnvelopeRequest>,
    100         radroots_transport::sink::DeliveryRequestId,
    101         AttachmentProof,
    102         Reply<Option<radroots_storage::inventory::StockDeliveryAttempt>, OperationError>,
    103     ),
    104     StockDeliverySummary(
    105         Box<radroots_storage::inventory::StockEnvelopeRequest>,
    106         AttachmentProof,
    107         Reply<radroots_storage::inventory::StockDeliverySummary, OperationError>,
    108     ),
    109     RecordStockDeliveryObservation(
    110         Box<WorkspaceBinding>,
    111         AttachmentProof,
    112         Box<radroots_storage::inventory::StockDeliveryObservation>,
    113         Reply<radroots_storage::inventory::StockDeliveryObservationOutcome, OperationError>,
    114     ),
    115     StockDeliveryObservations(
    116         Box<radroots_storage::inventory::StockEnvelopeRequest>,
    117         Option<radroots_domain::Sequence>,
    118         AttachmentProof,
    119         Reply<radroots_storage::inventory::StockDeliveryObservationPage, OperationError>,
    120     ),
    121     RecoverStockEnvelope(
    122         stock::envelope::CaptureRequest,
    123         AttachmentProof,
    124         Box<radroots_storage::command::PrivateEnvelopeCandidate>,
    125         Reply<radroots_storage::inventory::StockEnvelopeOutcome, OperationError>,
    126     ),
    127     RecoverStockObservation(
    128         Box<WorkspaceBinding>,
    129         AttachmentProof,
    130         Box<radroots_storage::inventory::StockDeliveryObservation>,
    131         Reply<radroots_storage::inventory::StockDeliveryObservationOutcome, OperationError>,
    132     ),
    133     CaptureStockEnvelope(
    134         stock::envelope::CaptureRequest,
    135         AttachmentProof,
    136         Box<radroots_storage::command::PrivateEnvelopeCandidate>,
    137         Reply<radroots_storage::inventory::StockEnvelopeOutcome, OperationError>,
    138     ),
    139     StockEnvelope(
    140         Box<radroots_storage::inventory::StockEnvelopeRequest>,
    141         AttachmentProof,
    142         Reply<Option<radroots_storage::inventory::StockEnvelopeSource>, OperationError>,
    143     ),
    144     PrepareStockRewrap(
    145         Box<radroots_storage::inventory::StockRewrapRequest>,
    146         AttachmentProof,
    147         Reply<radroots_storage::inventory::StockRewrapOutcome, OperationError>,
    148     ),
    149     StockRewrapAttempt(
    150         Box<radroots_storage::inventory::StockRewrapRequest>,
    151         AttachmentProof,
    152         Reply<Option<radroots_storage::inventory::StockRewrapState>, OperationError>,
    153     ),
    154     RecoverStockRewrapAdmission(
    155         Box<radroots_storage::inventory::StockRewrapRequest>,
    156         AttachmentProof,
    157         Reply<(), OperationError>,
    158     ),
    159     StockRewraps(
    160         Box<radroots_storage::inventory::StockEnvelopeRequest>,
    161         Option<radroots_domain::Sequence>,
    162         AttachmentProof,
    163         Reply<radroots_storage::inventory::StockRewrapPage, OperationError>,
    164     ),
    165     CaptureStockEvidence(
    166         Box<radroots_storage::inventory::StockEvidenceRequest>,
    167         AttachmentProof,
    168         Reply<stock::StockEvidenceOutcome, OperationError>,
    169     ),
    170     StockEvidence(
    171         Box<stock::StockClaimRequest>,
    172         [u8; 32],
    173         AttachmentProof,
    174         Reply<Option<stock::StockCapturedEvidence>, OperationError>,
    175     ),
    176     ClaimStockSigning(
    177         Box<stock::StockClaimRequest>,
    178         AttachmentProof,
    179         Reply<stock::StockClaimOutcome, OperationError>,
    180     ),
    181     AuthorizeStockExactReplay(
    182         Box<stock::StockClaimRequest>,
    183         AttachmentProof,
    184         Reply<stock::StockExactReplayPermit, OperationError>,
    185     ),
    186     StockSigningAttempt(
    187         Box<stock::StockClaimRequest>,
    188         AttachmentProof,
    189         Reply<Option<stock::StockSigningAttempt>, OperationError>,
    190     ),
    191     BindStockSigningPlan(
    192         Box<stock::StockSigningPlanRequest>,
    193         AttachmentProof,
    194         Reply<stock::StockSigningPlanOutcome, OperationError>,
    195     ),
    196     StockSigningPlan(
    197         Box<WorkspaceBinding>,
    198         radroots_storage::inventory::InventoryCheckpointRef,
    199         OperationId,
    200         CommandDigest,
    201         AttachmentProof,
    202         Reply<Option<stock::StockBoundSigningPlan>, OperationError>,
    203     ),
    204     PrepareStockAcceptance(
    205         Box<stock::StockAcceptanceRequest>,
    206         AttachmentProof,
    207         Reply<stock::StockAcceptanceOutcome, OperationError>,
    208     ),
    209     StockOrderRequest(
    210         Box<WorkspaceBinding>,
    211         radroots_storage::inventory::InventoryCheckpointRef,
    212         radroots_storage::workspace::seller_stock::StockSourceCheckpointRef,
    213         radroots_storage::workspace::seller_stock::StockListingCheckpointRef,
    214         radroots_domain::RecordId,
    215         AttachmentProof,
    216         Reply<stock::StockRequestSnapshot, OperationError>,
    217     ),
    218     StockOrderHistory(
    219         Box<WorkspaceBinding>,
    220         radroots_storage::inventory::InventoryCheckpointRef,
    221         radroots_storage::workspace::seller_stock::StockSourceCheckpointRef,
    222         radroots_storage::workspace::seller_stock::StockListingCheckpointRef,
    223         radroots_domain::causal::CausalScope,
    224         AttachmentProof,
    225         Reply<stock::StockHistorySnapshot, OperationError>,
    226     ),
    227     RetainStockListing(
    228         Box<WorkspaceBinding>,
    229         radroots_storage::workspace::seller_stock::StockListingCheckpointRef,
    230         Box<stock::StockListingCapture>,
    231         AttachmentProof,
    232         Reply<stock::StockListingOutcome, OperationError>,
    233     ),
    234     StockListing(
    235         Box<WorkspaceBinding>,
    236         radroots_storage::workspace::seller_stock::StockListingCheckpointRef,
    237         stock::public_source::Lookup,
    238         AttachmentProof,
    239         Reply<Option<stock::StockListingSnapshot>, OperationError>,
    240     ),
    241     IssueStockConsent(
    242         Box<WorkspaceBinding>,
    243         radroots_storage::inventory::InventoryCheckpointRef,
    244         Box<stock::StockConsentCapture>,
    245         AttachmentProof,
    246         Reply<stock::StockConsentOutcome, OperationError>,
    247     ),
    248     StockConsent(
    249         Box<WorkspaceBinding>,
    250         radroots_storage::inventory::InventoryCheckpointRef,
    251         OperationId,
    252         radroots_domain::GrantId,
    253         AttachmentProof,
    254         Reply<Option<stock::StockConsentReceipt>, OperationError>,
    255     ),
    256     RetainStockSource(
    257         Box<WorkspaceBinding>,
    258         radroots_storage::workspace::seller_stock::StockSourceCheckpointRef,
    259         Box<stock::StockSourceCapture>,
    260         AttachmentProof,
    261         Reply<stock::StockSourceOutcome, OperationError>,
    262     ),
    263     StockSource(
    264         Box<WorkspaceBinding>,
    265         radroots_storage::workspace::seller_stock::StockSourceCheckpointRef,
    266         radroots_domain::EnvelopeEventId,
    267         AttachmentProof,
    268         Reply<Option<stock::StockSourceSnapshot>, OperationError>,
    269     ),
    270     OptionMapping(
    271         Box<WorkspaceBinding>,
    272         radroots_storage::inventory::InventoryCheckpointRef,
    273         radroots_domain::stock::OptionMappingKey,
    274         AttachmentProof,
    275         Reply<Option<radroots_domain::stock::OptionPoolMapping>, OperationError>,
    276     ),
    277     ChangeStockGrant(
    278         Box<stock::grants::Change>,
    279         AttachmentProof,
    280         Reply<stock::StockGrantOutcome>,
    281     ),
    282     ExecuteInventory(
    283         Box<radroots_storage::inventory::InventoryRequest>,
    284         AttachmentProof,
    285         Reply<radroots_storage::inventory::InventoryOutcome, OperationError>,
    286     ),
    287     InventoryOperation(
    288         Box<WorkspaceBinding>,
    289         radroots_storage::inventory::InventoryCheckpointRef,
    290         OperationId,
    291         Option<CommandDigest>,
    292         AttachmentProof,
    293         Reply<Option<radroots_storage::inventory::InventoryReceipt>, OperationError>,
    294     ),
    295     InspectStock(
    296         Box<WorkspaceBinding>,
    297         radroots_storage::inventory::InventoryCheckpointRef,
    298         radroots_domain::PoolId,
    299         AttachmentProof,
    300         Reply<Option<radroots_storage::inventory::StockSnapshot>, OperationError>,
    301     ),
    302     Pool(
    303         Box<WorkspaceBinding>,
    304         radroots_storage::inventory::InventoryCheckpointRef,
    305         radroots_domain::PoolId,
    306         AttachmentProof,
    307         Reply<Option<radroots_domain::stock::PoolView>, OperationError>,
    308     ),
    309     StockAuthority(
    310         AttachmentInput,
    311         AttachmentProof,
    312         Reply<super::StockAuthorityReceipt>,
    313     ),
    314     CaptureDeliveryObservation(
    315         Box<WorkspaceBinding>,
    316         AttachmentProof,
    317         Box<DeliveryObservation>,
    318         Reply<DeliveryObservationOutcome, OperationError>,
    319     ),
    320     QueryDeliveryObservations(
    321         Box<WorkspaceBinding>,
    322         OperationId,
    323         CommandDigest,
    324         SigningTarget,
    325         DeliveryRequestId,
    326         AttachmentProof,
    327         Reply<Vec<DeliveryObservation>, OperationError>,
    328     ),
    329     PrepareDelivery(
    330         Box<DeliveryAttemptRequest>,
    331         AttachmentProof,
    332         Option<Box<EnvelopeCandidate>>,
    333         Reply<DeliveryAttemptOutcome, OperationError>,
    334     ),
    335     QueryDelivery(
    336         Box<WorkspaceBinding>,
    337         OperationId,
    338         CommandDigest,
    339         SigningTarget,
    340         DeliveryRequestId,
    341         AttachmentProof,
    342         Reply<Option<DeliveryAttempt>, OperationError>,
    343     ),
    344     Install(
    345         Box<InstallationRequest>,
    346         AttachmentProof,
    347         Reply<InstallationOutcome, OperationError>,
    348     ),
    349     Snapshot(
    350         Box<WorkspaceBinding>,
    351         OperationId,
    352         Option<CommandDigest>,
    353         AttachmentProof,
    354         Reply<Option<OperationSnapshot>, OperationError>,
    355     ),
    356     Claim(
    357         Box<ClaimAttemptRequest>,
    358         AttachmentProof,
    359         Reply<ClaimAttemptOutcome, OperationError>,
    360     ),
    361     Attempt(
    362         Box<ClaimAttemptRequest>,
    363         AttachmentProof,
    364         Reply<Option<SigningAttempt>, OperationError>,
    365     ),
    366     Capture(
    367         Box<AttachmentInput>,
    368         AttachmentProof,
    369         Box<ArtifactEvidence>,
    370         Reply<EvidenceCaptureOutcome, OperationError>,
    371     ),
    372     RecoverCapture(
    373         Box<AttachmentInput>,
    374         AttachmentProof,
    375         Box<ArtifactEvidence>,
    376         Reply<EvidenceCaptureOutcome, OperationError>,
    377     ),
    378     Captured(
    379         Box<ClaimAttemptRequest>,
    380         AttachmentProof,
    381         Reply<Option<CapturedArtifact>, OperationError>,
    382     ),
    383     BindPlan(
    384         Box<WorkspaceBinding>,
    385         CapturedSigningPlan,
    386         AttachmentProof,
    387         Reply<PlanBindingOutcome, OperationError>,
    388     ),
    389     SigningPlan(
    390         Box<WorkspaceBinding>,
    391         OperationId,
    392         CommandDigest,
    393         AttachmentProof,
    394         Reply<Option<BoundSigningPlan>, OperationError>,
    395     ),
    396     Authenticate(AttachmentInput, AttachmentProof, Reply<WorkspaceBinding>),
    397     Revoke(AttachmentInput, AttachmentProof, Reply<()>),
    398     Admit(
    399         Box<AdmissionRequest>,
    400         AttachmentProof,
    401         Reply<AdmissionOutcome, OperationError>,
    402     ),
    403     Operation(
    404         Box<WorkspaceBinding>,
    405         OperationId,
    406         Option<CommandDigest>,
    407         AttachmentProof,
    408         Reply<Option<PreparedOperation>, OperationError>,
    409     ),
    410 }
    411 impl Job {
    412     fn reject_recovery(self) {
    413         match self {
    414             Self::InstallStockAgreement(_, _, reply) => {
    415                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    416             }
    417             Self::StockAgreementInstallation(_, _, reply) => {
    418                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    419             }
    420             Self::AbortStockPreparation(_, _, reply) => {
    421                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    422             }
    423             Self::StockPreparationAbort(_, _, reply) => {
    424                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    425             }
    426             Self::StockSnapshot(_, _, reply) => {
    427                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    428             }
    429             Self::StockEvidencePage(_, _, _, reply) => {
    430                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    431             }
    432             Self::CaptureStockUnissued(_, _, reply) | Self::RecoverStockUnissued(_, _, reply) => {
    433                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    434             }
    435             Self::StockUnissued(_, _, reply) => {
    436                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    437             }
    438             Self::CaptureStockEvidence(_, _, reply) | Self::RecoverStockEvidence(_, _, reply) => {
    439                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    440             }
    441             Self::PrepareStockDelivery(_, _, reply) => {
    442                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    443             }
    444             Self::StockDeliveryAttempt(_, _, _, reply) => {
    445                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    446             }
    447             Self::StockDeliverySummary(_, _, reply) => {
    448                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    449             }
    450             Self::RecordStockDeliveryObservation(_, _, _, reply) => {
    451                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    452             }
    453             Self::StockDeliveryObservations(_, _, _, reply) => {
    454                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    455             }
    456             Self::RecoverStockEnvelope(_, _, _, reply) => {
    457                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    458             }
    459             Self::RecoverStockObservation(_, _, _, reply) => {
    460                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    461             }
    462             Self::CaptureStockEnvelope(_, _, _, reply) => {
    463                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    464             }
    465             Self::StockEnvelope(_, _, reply) => {
    466                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    467             }
    468             Self::PrepareStockRewrap(_, _, reply) => {
    469                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    470             }
    471             Self::StockRewrapAttempt(_, _, reply) => {
    472                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    473             }
    474             Self::RecoverStockRewrapAdmission(_, _, reply) => {
    475                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    476             }
    477             Self::StockRewraps(_, _, _, reply) => {
    478                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    479             }
    480             Self::StockEvidence(_, _, _, reply) => {
    481                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    482             }
    483             Self::ClaimStockSigning(_, _, reply) => {
    484                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    485             }
    486             Self::AuthorizeStockExactReplay(_, _, reply) => {
    487                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    488             }
    489             Self::StockSigningAttempt(_, _, reply) => {
    490                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    491             }
    492             Self::BindStockSigningPlan(_, _, reply) => {
    493                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    494             }
    495             Self::StockSigningPlan(_, _, _, _, _, reply) => {
    496                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    497             }
    498             Self::PrepareStockAcceptance(_, _, reply) => {
    499                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    500             }
    501             Self::StockOrderRequest(_, _, _, _, _, _, reply) => {
    502                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    503             }
    504             Self::StockOrderHistory(_, _, _, _, _, _, reply) => {
    505                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    506             }
    507             Self::RetainStockListing(_, _, _, _, reply) => {
    508                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    509             }
    510             Self::StockListing(_, _, _, _, reply) => {
    511                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    512             }
    513             Self::IssueStockConsent(_, _, _, _, reply) => {
    514                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    515             }
    516             Self::StockConsent(_, _, _, _, _, reply) => {
    517                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    518             }
    519             Self::RetainStockSource(_, _, _, _, reply) => {
    520                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    521             }
    522             Self::StockSource(_, _, _, _, reply) => {
    523                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    524             }
    525             Self::OptionMapping(_, _, _, _, reply) => {
    526                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    527             }
    528             Self::ChangeStockGrant(_, _, reply) => {
    529                 let _ = reply.send(Err(WorkspaceError::RecoveryRequired));
    530             }
    531             Self::ExecuteInventory(_, _, reply) => {
    532                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    533             }
    534             Self::InventoryOperation(_, _, _, _, _, reply) => {
    535                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    536             }
    537             Self::InspectStock(_, _, _, _, reply) => {
    538                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    539             }
    540             Self::Pool(_, _, _, _, reply) => {
    541                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    542             }
    543             Self::StockAuthority(_, _, reply) => {
    544                 let _ = reply.send(Err(WorkspaceError::RecoveryRequired));
    545             }
    546             Self::CaptureDeliveryObservation(_, _, _, reply) => {
    547                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    548             }
    549             Self::QueryDeliveryObservations(_, _, _, _, _, _, reply) => {
    550                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    551             }
    552             Self::PrepareDelivery(_, _, _, reply) => {
    553                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    554             }
    555             Self::QueryDelivery(_, _, _, _, _, _, reply) => {
    556                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    557             }
    558             Self::Install(_, _, reply) => {
    559                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    560             }
    561             Self::Snapshot(_, _, _, _, reply) => {
    562                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    563             }
    564             Self::Claim(_, _, reply) => {
    565                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    566             }
    567             Self::Attempt(_, _, reply) => {
    568                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    569             }
    570             Self::Capture(_, _, _, reply) | Self::RecoverCapture(_, _, _, reply) => {
    571                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    572             }
    573             Self::Captured(_, _, reply) => {
    574                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    575             }
    576             Self::BindPlan(_, _, _, reply) => {
    577                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    578             }
    579             Self::SigningPlan(_, _, _, _, reply) => {
    580                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    581             }
    582             Self::Authenticate(_, _, reply) => {
    583                 let _ = reply.send(Err(WorkspaceError::RecoveryRequired));
    584             }
    585             Self::Revoke(_, _, reply) => {
    586                 let _ = reply.send(Err(WorkspaceError::RecoveryRequired));
    587             }
    588             Self::Admit(_, _, reply) => {
    589                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    590             }
    591             Self::Operation(_, _, _, _, reply) => {
    592                 let _ = reply.send(Err(OperationError::RecoveryRequired));
    593             }
    594         }
    595     }
    596 }
    597 struct Budget {
    598     used: Arc<AtomicUsize>,
    599     bytes: usize,
    600 }
    601 impl Drop for Budget {
    602     fn drop(&mut self) {
    603         self.used.fetch_sub(self.bytes, Ordering::AcqRel);
    604     }
    605 }
    606 struct Queued {
    607     job: Job,
    608     _budget: Budget,
    609 }
    610 pub(super) struct Handle {
    611     sender: Mutex<Option<SyncSender<Queued>>>,
    612     queued_bytes: Arc<AtomicUsize>,
    613     completed: watch::Receiver<Option<Result<(), WorkspaceError>>>,
    614     thread: Mutex<Option<JoinHandle<()>>>,
    615 }
    616 impl Handle {
    617     pub(super) async fn launch(
    618         path: PathBuf,
    619         input: AttachmentInput,
    620         initial: Option<GrantRecord>,
    621         crypto: WorkspaceCrypto,
    622         clock: Arc<dyn WorkspaceClock>,
    623     ) -> Result<Self, WorkspaceError> {
    624         Self::launch_kind(path, input, initial, None, Kind::Outbox, crypto, clock).await
    625     }
    626     pub(super) async fn launch_stock(
    627         path: PathBuf,
    628         input: AttachmentInput,
    629         initial: Option<(
    630             GrantRecord,
    631             radroots_domain::TransactionId,
    632             radroots_domain::UnixSeconds,
    633         )>,
    634         crypto: WorkspaceCrypto,
    635         clock: Arc<dyn WorkspaceClock>,
    636     ) -> Result<Self, WorkspaceError> {
    637         let initial = initial.map(|(grant, transaction, expires_at)| stock::Initial {
    638             grant,
    639             transaction,
    640             expires_at,
    641             clock: clock.clone(),
    642         });
    643         Self::launch_kind(path, input, None, initial, Kind::SellerStock, crypto, clock).await
    644     }
    645     async fn launch_kind(
    646         path: PathBuf,
    647         input: AttachmentInput,
    648         initial: Option<GrantRecord>,
    649         stock_initial: Option<stock::Initial>,
    650         kind: Kind,
    651         crypto: WorkspaceCrypto,
    652         clock: Arc<dyn WorkspaceClock>,
    653     ) -> Result<Self, WorkspaceError> {
    654         let creating = initial.is_some() || stock_initial.is_some();
    655         let (sender, requests) = mpsc::sync_channel::<Queued>(QUEUE_CAPACITY);
    656         let queued_bytes = Arc::new(AtomicUsize::new(0));
    657         let (ready, started) = oneshot::channel();
    658         let (completion, completed) = watch::channel(None);
    659         let thread = std::thread::Builder::new()
    660             .name("radroots_workspace".into())
    661             .spawn(move || {
    662                 let runtime = match tokio::runtime::Builder::new_current_thread()
    663                     .enable_all()
    664                     .build()
    665                 {
    666                     Ok(runtime) => runtime,
    667                     Err(_) => {
    668                         let _ = ready.send(Err(WorkspaceError::Unavailable));
    669                         let _ = completion.send(Some(Err(WorkspaceError::Unavailable)));
    670                         return;
    671                     }
    672                 };
    673                 // Construction, every request and explicit close complete on this
    674                 // owner even when an async observer disappears between await points.
    675                 let namespace = match Namespace::acquire_kind(&path, creating, kind) {
    676                     Ok(namespace) => namespace,
    677                     Err(error) => {
    678                         let _ = ready.send(Err(error));
    679                         let _ = completion.send(Some(Err(error)));
    680                         return;
    681                     }
    682                 };
    683                 let result = runtime.block_on(Box::pin(async {
    684                     if !creating {
    685                         let preflight = preflight_existing(&namespace, &input, &crypto).await;
    686                         if let Err(error) = preflight {
    687                             let _ = ready.send(Err(error));
    688                             return Err(error);
    689                         }
    690                     }
    691                     let options = SqliteConnectOptions::new()
    692                         .filename(namespace.database_path())
    693                         .create_if_missing(false)
    694                         .journal_mode(SqliteJournalMode::Wal)
    695                         .synchronous(SqliteSynchronous::Full)
    696                         .foreign_keys(true)
    697                         .busy_timeout(Duration::from_secs(5));
    698                     let mut connection = match SqliteConnection::connect_with(&options).await {
    699                         Ok(connection) => connection,
    700                         Err(_) => {
    701                             let _ = ready.send(Err(WorkspaceError::StorageFailure));
    702                             return Err(WorkspaceError::StorageFailure);
    703                         }
    704                     };
    705                     let initialized = match kind {
    706                         Kind::Outbox => {
    707                             initialize(&mut connection, &namespace, &input, initial, &crypto).await
    708                         }
    709                         Kind::SellerStock => {
    710                             stock::initialize(
    711                                 &mut connection,
    712                                 &namespace,
    713                                 &input,
    714                                 stock_initial,
    715                                 &crypto,
    716                             )
    717                             .await
    718                         }
    719                     };
    720                     if let Err(error) = initialized {
    721                         let _ = ready.send(Err(error));
    722                         // Do not release namespace authority at the first failure;
    723                         // first await the actual connection's close completion.
    724                         let _ = connection.close().await;
    725                         return Err(error);
    726                     }
    727                     let mut uncertain = false;
    728                     let mut cached_grant = None;
    729                     let mut connection = Some(connection);
    730                     if ready.send(Ok(())).is_ok() {
    731                         while let Ok(Queued { job, _budget }) = requests.recv() {
    732                             if kind == Kind::SellerStock
    733                                 && !matches!(
    734                                     &job,
    735                                     Job::Authenticate(..)
    736                                         | Job::RetainStockListing(..)
    737                                         | Job::StockListing(..)
    738                                         | Job::IssueStockConsent(..)
    739                                         | Job::StockConsent(..)
    740                                         | Job::ChangeStockGrant(..)
    741                                         | Job::ExecuteInventory(..)
    742                                         | Job::InventoryOperation(..)
    743                                         | Job::InspectStock(..)
    744                                         | Job::Pool(..)
    745                                         | Job::OptionMapping(..)
    746                                         | Job::RetainStockSource(..)
    747                                         | Job::StockSource(..)
    748                                         | Job::StockOrderHistory(..)
    749                                         | Job::StockOrderRequest(..)
    750                                         | Job::PrepareStockAcceptance(..)
    751                                         | Job::BindStockSigningPlan(..)
    752                                         | Job::StockSigningPlan(..)
    753                                         | Job::ClaimStockSigning(..)
    754                                         | Job::StockSigningAttempt(..)
    755                                         | Job::AuthorizeStockExactReplay(..)
    756                                         | Job::CaptureStockUnissued(..)
    757                                         | Job::RecoverStockUnissued(..)
    758                                         | Job::StockUnissued(..)
    759                                         | Job::PrepareStockDelivery(..)
    760                                         | Job::StockDeliveryAttempt(..)
    761                                         | Job::StockDeliverySummary(..)
    762                                         | Job::RecordStockDeliveryObservation(..)
    763                                         | Job::StockDeliveryObservations(..)
    764                                         | Job::RecoverStockEnvelope(..)
    765                                         | Job::RecoverStockObservation(..)
    766                                         | Job::CaptureStockEnvelope(..)
    767                                         | Job::StockEnvelope(..)
    768                                         | Job::PrepareStockRewrap(..)
    769                                         | Job::StockRewrapAttempt(..)
    770                                         | Job::RecoverStockRewrapAdmission(..)
    771                                         | Job::StockRewraps(..)
    772                                         | Job::CaptureStockEvidence(..)
    773                                         | Job::RecoverStockEvidence(..)
    774                                         | Job::StockEvidence(..)
    775                                         | Job::StockEvidencePage(..)
    776                                         | Job::StockSnapshot(..)
    777                                         | Job::InstallStockAgreement(..)
    778                                         | Job::StockAgreementInstallation(..)
    779                                         | Job::AbortStockPreparation(..)
    780                                         | Job::StockPreparationAbort(..)
    781                                         | Job::Revoke(..)
    782                                         | Job::StockAuthority(..)
    783                                 )
    784                             {
    785                                 job.reject_recovery();
    786                                 continue;
    787                             }
    788                             let job = match job {
    789                                 Job::RecoverStockRewrapAdmission(request, proof, reply) => {
    790                                     let result = Box::pin(
    791                                         super::recovery::RecoveryTurn {
    792                                             connection: &mut connection,
    793                                             cached_grant: &mut cached_grant,
    794                                             namespace: &namespace,
    795                                             selected: &input,
    796                                             crypto: &crypto,
    797                                             clock: clock.as_ref(),
    798                                             uncertain: &mut uncertain,
    799                                             options: &options,
    800                                         }
    801                                         .reconcile_rewrap_admission(*request, proof),
    802                                     )
    803                                     .await;
    804                                     let _ = reply.send(result);
    805                                     continue;
    806                                 }
    807                                 Job::RecoverStockEnvelope(request, proof, candidate, reply) => {
    808                                     let result = Box::pin(
    809                                         super::recovery::RecoveryTurn {
    810                                             connection: &mut connection,
    811                                             cached_grant: &mut cached_grant,
    812                                             namespace: &namespace,
    813                                             selected: &input,
    814                                             crypto: &crypto,
    815                                             clock: clock.as_ref(),
    816                                             uncertain: &mut uncertain,
    817                                             options: &options,
    818                                         }
    819                                         .capture_envelope(request, proof, *candidate),
    820                                     )
    821                                     .await;
    822                                     let _ = reply.send(result);
    823                                     continue;
    824                                 }
    825                                 Job::RecoverStockObservation(
    826                                     binding,
    827                                     proof,
    828                                     observation,
    829                                     reply,
    830                                 ) => {
    831                                     let result = Box::pin(
    832                                         super::recovery::RecoveryTurn {
    833                                             connection: &mut connection,
    834                                             cached_grant: &mut cached_grant,
    835                                             namespace: &namespace,
    836                                             selected: &input,
    837                                             crypto: &crypto,
    838                                             clock: clock.as_ref(),
    839                                             uncertain: &mut uncertain,
    840                                             options: &options,
    841                                         }
    842                                         .capture_observation(*binding, proof, *observation),
    843                                     )
    844                                     .await;
    845                                     let _ = reply.send(result);
    846                                     continue;
    847                                 }
    848                                 Job::RecoverStockUnissued(request, proof, reply) => {
    849                                     let result = Box::pin(
    850                                         super::recovery::RecoveryTurn {
    851                                             connection: &mut connection,
    852                                             cached_grant: &mut cached_grant,
    853                                             namespace: &namespace,
    854                                             selected: &input,
    855                                             crypto: &crypto,
    856                                             clock: clock.as_ref(),
    857                                             uncertain: &mut uncertain,
    858                                             options: &options,
    859                                         }
    860                                         .capture_unissued(*request, proof),
    861                                     )
    862                                     .await;
    863                                     let _ = reply.send(result);
    864                                     continue;
    865                                 }
    866                                 Job::RecoverStockEvidence(request, proof, reply) => {
    867                                     let result = Box::pin(
    868                                         super::recovery::RecoveryTurn {
    869                                             connection: &mut connection,
    870                                             cached_grant: &mut cached_grant,
    871                                             namespace: &namespace,
    872                                             selected: &input,
    873                                             crypto: &crypto,
    874                                             clock: clock.as_ref(),
    875                                             uncertain: &mut uncertain,
    876                                             options: &options,
    877                                         }
    878                                         .capture_stock(*request, proof),
    879                                     )
    880                                     .await;
    881                                     let _ = reply.send(result);
    882                                     continue;
    883                                 }
    884                                 Job::RecoverCapture(selected, proof, evidence, reply) => {
    885                                     let result = super::recovery::RecoveryTurn {
    886                                         connection: &mut connection,
    887                                         cached_grant: &mut cached_grant,
    888                                         namespace: &namespace,
    889                                         selected: &input,
    890                                         crypto: &crypto,
    891                                         clock: clock.as_ref(),
    892                                         uncertain: &mut uncertain,
    893                                         options: &options,
    894                                     }
    895                                     .capture(*selected, proof, *evidence)
    896                                     .await;
    897                                     let _ = reply.send(result);
    898                                     continue;
    899                                 }
    900                                 job => job,
    901                             };
    902                             let Some(connection) = connection.as_mut() else {
    903                                 job.reject_recovery();
    904                                 continue;
    905                             };
    906                             dispatch::run(
    907                                 job,
    908                                 kind,
    909                                 stock::inventory::Turn {
    910                                     connection,
    911                                     namespace: &namespace,
    912                                     selected: &input,
    913                                     crypto: &crypto,
    914                                     clock: clock.as_ref(),
    915                                     uncertain: &mut uncertain,
    916                                 },
    917                             )
    918                             .await;
    919                         }
    920                     }
    921                     // The admission sender is gone. All admitted work has drained.
    922                     let Some(mut connection) = connection else {
    923                         return Err(WorkspaceError::RecoveryRequired);
    924                     };
    925                     let checkpoint = sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)")
    926                         .fetch_one(&mut connection)
    927                         .await;
    928                     let checked = match checkpoint {
    929                         Ok(row) if row.try_get::<i64, _>(0).ok() == Some(0) => Ok(()),
    930                         _ => Err(WorkspaceError::StorageFailure),
    931                     };
    932                     let closed = connection
    933                         .close()
    934                         .await
    935                         .map_err(|_| WorkspaceError::StorageFailure);
    936                     checked.and(closed).and_then(|()| namespace.validate())
    937                 }));
    938                 drop(namespace);
    939                 let _ = completion.send(Some(result));
    940             })
    941             .map_err(|_| WorkspaceError::Unavailable)?;
    942         let handle = Self {
    943             sender: Mutex::new(Some(sender)),
    944             queued_bytes,
    945             completed,
    946             thread: Mutex::new(Some(thread)),
    947         };
    948         match started.await {
    949             Ok(Ok(())) => Ok(handle),
    950             Ok(Err(error)) => {
    951                 let _ = handle.close().await;
    952                 Err(error)
    953             }
    954             Err(_) => {
    955                 let _ = handle.close().await;
    956                 Err(WorkspaceError::Unavailable)
    957             }
    958         }
    959     }
    960     pub(super) async fn change_stock_grant(
    961         &self,
    962         change: stock::grants::Change,
    963         proof: AttachmentProof,
    964     ) -> Result<stock::StockGrantOutcome, WorkspaceError> {
    965         let transaction = change.transaction;
    966         let (reply, receive) = oneshot::channel();
    967         self.send(Job::ChangeStockGrant(Box::new(change), proof, reply))?;
    968         receive
    969             .await
    970             .unwrap_or(Ok(stock::StockGrantOutcome::Unknown(transaction)))
    971     }
    972     pub(super) async fn stock_authority(
    973         &self,
    974         input: AttachmentInput,
    975         proof: AttachmentProof,
    976     ) -> Result<super::StockAuthorityReceipt, WorkspaceError> {
    977         let (reply, result) = oneshot::channel();
    978         self.send(Job::StockAuthority(input, proof, reply))?;
    979         result.await.map_err(|_| WorkspaceError::RecoveryRequired)?
    980     }
    981     pub(super) async fn execute_inventory(
    982         &self,
    983         request: radroots_storage::inventory::InventoryRequest,
    984         proof: AttachmentProof,
    985     ) -> Result<radroots_storage::inventory::InventoryOutcome, OperationError> {
    986         let c = request.command().context();
    987         let anchor = CommitAttempt::new(
    988             c.workspace_id(),
    989             c.operation_id(),
    990             request.command().digest(),
    991             None,
    992         );
    993         let (reply, result) = oneshot::channel();
    994         self.send(Job::ExecuteInventory(Box::new(request), proof, reply))?;
    995         result
    996             .await
    997             .unwrap_or(Ok(radroots_storage::inventory::InventoryOutcome::Unknown(
    998                 anchor,
    999             )))
   1000     }
   1001     pub(super) async fn inventory_operation(
   1002         &self,
   1003         binding: WorkspaceBinding,
   1004         minimum: radroots_storage::inventory::InventoryCheckpointRef,
   1005         operation: OperationId,
   1006         digest: Option<CommandDigest>,
   1007         proof: AttachmentProof,
   1008     ) -> Result<Option<radroots_storage::inventory::InventoryReceipt>, OperationError> {
   1009         let (reply, result) = oneshot::channel();
   1010         self.send(Job::InventoryOperation(
   1011             Box::new(binding),
   1012             minimum,
   1013             operation,
   1014             digest,
   1015             proof,
   1016             reply,
   1017         ))?;
   1018         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1019     }
   1020     pub(super) async fn issue_stock_consent(
   1021         &self,
   1022         binding: WorkspaceBinding,
   1023         minimum: radroots_storage::inventory::InventoryCheckpointRef,
   1024         capture: stock::StockConsentCapture,
   1025         proof: AttachmentProof,
   1026     ) -> Result<stock::StockConsentOutcome, OperationError> {
   1027         let anchor = stock::StockConsentOutcome::Unknown {
   1028             control_operation: capture.control_operation_id(),
   1029             transaction: capture.transaction_id(),
   1030             command_operation: capture.command().context().operation_id(),
   1031             subject: capture.subject_grant(),
   1032         };
   1033         let (reply, result) = oneshot::channel();
   1034         self.send(Job::IssueStockConsent(
   1035             Box::new(binding),
   1036             minimum,
   1037             Box::new(capture),
   1038             proof,
   1039             reply,
   1040         ))?;
   1041         result.await.unwrap_or(Ok(anchor))
   1042     }
   1043     pub(super) async fn stock_consent(
   1044         &self,
   1045         binding: WorkspaceBinding,
   1046         minimum: radroots_storage::inventory::InventoryCheckpointRef,
   1047         operation: OperationId,
   1048         subject: radroots_domain::GrantId,
   1049         proof: AttachmentProof,
   1050     ) -> Result<Option<stock::StockConsentReceipt>, OperationError> {
   1051         let (reply, result) = oneshot::channel();
   1052         self.send(Job::StockConsent(
   1053             Box::new(binding),
   1054             minimum,
   1055             operation,
   1056             subject,
   1057             proof,
   1058             reply,
   1059         ))?;
   1060         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1061     }
   1062     pub(super) async fn retain_stock_listing(
   1063         &self,
   1064         binding: WorkspaceBinding,
   1065         minimum: radroots_storage::workspace::seller_stock::StockListingCheckpointRef,
   1066         capture: stock::StockListingCapture,
   1067         proof: AttachmentProof,
   1068     ) -> Result<stock::StockListingOutcome, OperationError> {
   1069         let anchor = stock::StockListingOutcome::Unknown {
   1070             transaction: capture.transaction_id(),
   1071             listing: capture.listing_revision(),
   1072         };
   1073         let (reply, result) = oneshot::channel();
   1074         self.send(Job::RetainStockListing(
   1075             Box::new(binding),
   1076             minimum,
   1077             Box::new(capture),
   1078             proof,
   1079             reply,
   1080         ))?;
   1081         result.await.unwrap_or(Ok(anchor))
   1082     }
   1083     pub(super) async fn stock_listing(
   1084         &self,
   1085         binding: WorkspaceBinding,
   1086         minimum: radroots_storage::workspace::seller_stock::StockListingCheckpointRef,
   1087         lookup: stock::public_source::Lookup,
   1088         proof: AttachmentProof,
   1089     ) -> Result<Option<stock::StockListingSnapshot>, OperationError> {
   1090         let (reply, result) = oneshot::channel();
   1091         self.send(Job::StockListing(
   1092             Box::new(binding),
   1093             minimum,
   1094             lookup,
   1095             proof,
   1096             reply,
   1097         ))?;
   1098         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1099     }
   1100     pub(super) async fn retain_stock_source(
   1101         &self,
   1102         binding: WorkspaceBinding,
   1103         minimum: radroots_storage::workspace::seller_stock::StockSourceCheckpointRef,
   1104         capture: stock::StockSourceCapture,
   1105         proof: AttachmentProof,
   1106     ) -> Result<stock::StockSourceOutcome, OperationError> {
   1107         let anchor = stock::StockSourceOutcome::Unknown {
   1108             transaction: capture.transaction_id(),
   1109             envelope: capture.envelope_id(),
   1110         };
   1111         let (reply, result) = oneshot::channel();
   1112         self.send(Job::RetainStockSource(
   1113             Box::new(binding),
   1114             minimum,
   1115             Box::new(capture),
   1116             proof,
   1117             reply,
   1118         ))?;
   1119         result.await.unwrap_or(Ok(anchor))
   1120     }
   1121     pub(super) async fn install_stock_agreement(
   1122         &self,
   1123         request: stock::StockInstallationRequest,
   1124         proof: AttachmentProof,
   1125     ) -> Result<stock::StockInstallationOutcome, OperationError> {
   1126         let unknown = stock::StockInstallationOutcome::Unknown(request.attempt());
   1127         let (reply, result) = oneshot::channel();
   1128         self.send(Job::InstallStockAgreement(Box::new(request), proof, reply))?;
   1129         result.await.unwrap_or(Ok(unknown))
   1130     }
   1131     pub(super) async fn stock_agreement_installation(
   1132         &self,
   1133         request: stock::StockInstallationRequest,
   1134         proof: AttachmentProof,
   1135     ) -> Result<Option<stock::StockInstallationReceipt>, OperationError> {
   1136         let (reply, result) = oneshot::channel();
   1137         self.send(Job::StockAgreementInstallation(
   1138             Box::new(request),
   1139             proof,
   1140             reply,
   1141         ))?;
   1142         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1143     }
   1144     pub(super) async fn abort_proven_unissued(
   1145         &self,
   1146         request: stock::StockAbortRequest,
   1147         proof: AttachmentProof,
   1148     ) -> Result<stock::StockAbortOutcome, OperationError> {
   1149         let unknown = stock::StockAbortOutcome::Unknown(request.attempt());
   1150         let (reply, result) = oneshot::channel();
   1151         self.send(Job::AbortStockPreparation(Box::new(request), proof, reply))?;
   1152         result.await.unwrap_or(Ok(unknown))
   1153     }
   1154     pub(super) async fn stock_preparation_abort(
   1155         &self,
   1156         request: stock::StockAbortRequest,
   1157         proof: AttachmentProof,
   1158     ) -> Result<Option<stock::StockAbortReceipt>, OperationError> {
   1159         let (reply, result) = oneshot::channel();
   1160         self.send(Job::StockPreparationAbort(Box::new(request), proof, reply))?;
   1161         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1162     }
   1163     pub(super) async fn stock_operation_snapshot(
   1164         &self,
   1165         request: stock::StockSnapshotRequest,
   1166         proof: AttachmentProof,
   1167     ) -> Result<stock::StockOperationSnapshot, OperationError> {
   1168         let (reply, result) = oneshot::channel();
   1169         self.send(Job::StockSnapshot(Box::new(request), proof, reply))?;
   1170         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1171     }
   1172     pub(super) async fn stock_signing_evidence_page(
   1173         &self,
   1174         request: stock::StockClaimRequest,
   1175         after: Option<radroots_domain::Sequence>,
   1176         proof: AttachmentProof,
   1177     ) -> Result<stock::StockEvidencePage, OperationError> {
   1178         let (reply, result) = oneshot::channel();
   1179         self.send(Job::StockEvidencePage(
   1180             Box::new(request),
   1181             after,
   1182             proof,
   1183             reply,
   1184         ))?;
   1185         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1186     }
   1187     pub(super) async fn capture_stock_original_unissued(
   1188         &self,
   1189         request: stock::issuance::unissued::Request,
   1190         proof: AttachmentProof,
   1191         reconcile: bool,
   1192     ) -> Result<stock::StockUnissuedOutcome, OperationError> {
   1193         let unknown = stock::StockUnissuedOutcome::Unknown(request.attempt());
   1194         let (reply, result) = oneshot::channel();
   1195         self.send(if reconcile {
   1196             Job::RecoverStockUnissued(Box::new(request), proof, reply)
   1197         } else {
   1198             Job::CaptureStockUnissued(Box::new(request), proof, reply)
   1199         })?;
   1200         result.await.unwrap_or(Ok(unknown))
   1201     }
   1202     pub(super) async fn stock_original_unissued(
   1203         &self,
   1204         request: stock::StockClaimRequest,
   1205         proof: AttachmentProof,
   1206     ) -> Result<Option<stock::StockUnissuedReceipt>, OperationError> {
   1207         let (reply, result) = oneshot::channel();
   1208         self.send(Job::StockUnissued(Box::new(request), proof, reply))?;
   1209         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1210     }
   1211     pub(super) async fn reconcile_stock_evidence(
   1212         &self,
   1213         request: radroots_storage::inventory::StockEvidenceRequest,
   1214         proof: AttachmentProof,
   1215     ) -> Result<stock::StockEvidenceOutcome, OperationError> {
   1216         let unknown = stock::StockEvidenceOutcome::Unknown(request.attempt());
   1217         let (reply, result) = oneshot::channel();
   1218         self.send(Job::RecoverStockEvidence(Box::new(request), proof, reply))?;
   1219         result.await.unwrap_or(Ok(unknown))
   1220     }
   1221 
   1222     pub(super) async fn reconcile_stock_envelope(
   1223         &self,
   1224         request: radroots_storage::inventory::StockEnvelopeRequest,
   1225         proof: AttachmentProof,
   1226         candidate: radroots_storage::command::PrivateEnvelopeCandidate,
   1227     ) -> Result<radroots_storage::inventory::StockEnvelopeOutcome, OperationError> {
   1228         let unknown = radroots_storage::inventory::StockEnvelopeOutcome::Unknown(request.attempt());
   1229         let (reply, result) = oneshot::channel();
   1230         self.send(Job::RecoverStockEnvelope(
   1231             stock::envelope::CaptureRequest::Original(Box::new(request)),
   1232             proof,
   1233             Box::new(candidate),
   1234             reply,
   1235         ))?;
   1236         result.await.unwrap_or(Ok(unknown))
   1237     }
   1238     pub(super) async fn prepare_stock_delivery_attempt(
   1239         &self,
   1240         request: radroots_storage::inventory::StockDeliveryRequest,
   1241         proof: AttachmentProof,
   1242     ) -> Result<radroots_storage::inventory::StockDeliveryAttemptOutcome, OperationError> {
   1243         let unknown = radroots_storage::inventory::StockDeliveryAttemptOutcome::Unknown {
   1244             operation: request.attempt(),
   1245             request: request.intent().request_id().clone(),
   1246         };
   1247         let (reply, result) = oneshot::channel();
   1248         self.send(Job::PrepareStockDelivery(Box::new(request), proof, reply))?;
   1249         result.await.unwrap_or(Ok(unknown))
   1250     }
   1251     pub(super) async fn stock_delivery_attempt(
   1252         &self,
   1253         request: radroots_storage::inventory::StockEnvelopeRequest,
   1254         request_id: radroots_transport::sink::DeliveryRequestId,
   1255         proof: AttachmentProof,
   1256     ) -> Result<Option<radroots_storage::inventory::StockDeliveryAttempt>, OperationError> {
   1257         let (reply, result) = oneshot::channel();
   1258         self.send(Job::StockDeliveryAttempt(
   1259             Box::new(request),
   1260             request_id,
   1261             proof,
   1262             reply,
   1263         ))?;
   1264         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1265     }
   1266     pub(super) async fn stock_delivery_summary(
   1267         &self,
   1268         request: radroots_storage::inventory::StockEnvelopeRequest,
   1269         proof: AttachmentProof,
   1270     ) -> Result<radroots_storage::inventory::StockDeliverySummary, OperationError> {
   1271         let (reply, result) = oneshot::channel();
   1272         self.send(Job::StockDeliverySummary(Box::new(request), proof, reply))?;
   1273         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1274     }
   1275     pub(super) async fn record_stock_delivery_observation(
   1276         &self,
   1277         binding: WorkspaceBinding,
   1278         proof: AttachmentProof,
   1279         observation: radroots_storage::inventory::StockDeliveryObservation,
   1280         recovering: bool,
   1281     ) -> Result<radroots_storage::inventory::StockDeliveryObservationOutcome, OperationError> {
   1282         let source = observation.source().attempt();
   1283         let unknown = radroots_storage::inventory::StockDeliveryObservationOutcome::Unknown(
   1284             radroots_storage::command::CommitAttempt::new(
   1285                 source.workspace_id(),
   1286                 source.operation_id(),
   1287                 source.command_digest(),
   1288                 None,
   1289             ),
   1290         );
   1291         let (reply, result) = oneshot::channel();
   1292         self.send(if recovering {
   1293             Job::RecoverStockObservation(Box::new(binding), proof, Box::new(observation), reply)
   1294         } else {
   1295             Job::RecordStockDeliveryObservation(
   1296                 Box::new(binding),
   1297                 proof,
   1298                 Box::new(observation),
   1299                 reply,
   1300             )
   1301         })?;
   1302         result.await.unwrap_or(Ok(unknown))
   1303     }
   1304     pub(super) async fn stock_delivery_observations(
   1305         &self,
   1306         request: radroots_storage::inventory::StockEnvelopeRequest,
   1307         after: Option<radroots_domain::Sequence>,
   1308         proof: AttachmentProof,
   1309     ) -> Result<radroots_storage::inventory::StockDeliveryObservationPage, OperationError> {
   1310         let (reply, result) = oneshot::channel();
   1311         self.send(Job::StockDeliveryObservations(
   1312             Box::new(request),
   1313             after,
   1314             proof,
   1315             reply,
   1316         ))?;
   1317         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1318     }
   1319     pub(super) async fn capture_stock_envelope(
   1320         &self,
   1321         request: radroots_storage::inventory::StockEnvelopeRequest,
   1322         proof: AttachmentProof,
   1323         candidate: radroots_storage::command::PrivateEnvelopeCandidate,
   1324     ) -> Result<radroots_storage::inventory::StockEnvelopeOutcome, OperationError> {
   1325         let unknown = radroots_storage::inventory::StockEnvelopeOutcome::Unknown(request.attempt());
   1326         let (reply, result) = oneshot::channel();
   1327         self.send(Job::CaptureStockEnvelope(
   1328             stock::envelope::CaptureRequest::Original(Box::new(request)),
   1329             proof,
   1330             Box::new(candidate),
   1331             reply,
   1332         ))?;
   1333         result.await.unwrap_or(Ok(unknown))
   1334     }
   1335     pub(super) async fn stock_envelope(
   1336         &self,
   1337         request: radroots_storage::inventory::StockEnvelopeRequest,
   1338         proof: AttachmentProof,
   1339     ) -> Result<Option<radroots_storage::inventory::StockEnvelopeSource>, OperationError> {
   1340         let (reply, result) = oneshot::channel();
   1341         self.send(Job::StockEnvelope(Box::new(request), proof, reply))?;
   1342         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1343     }
   1344     pub(super) async fn prepare_stock_rewrap(
   1345         &self,
   1346         request: radroots_storage::inventory::StockRewrapRequest,
   1347         proof: AttachmentProof,
   1348     ) -> Result<radroots_storage::inventory::StockRewrapOutcome, OperationError> {
   1349         let unknown = radroots_storage::inventory::StockRewrapOutcome::Unknown(request.attempt());
   1350         let (reply, result) = oneshot::channel();
   1351         self.send(Job::PrepareStockRewrap(Box::new(request), proof, reply))?;
   1352         result.await.unwrap_or(Ok(unknown))
   1353     }
   1354     pub(super) async fn stock_rewrap_attempt(
   1355         &self,
   1356         request: radroots_storage::inventory::StockRewrapRequest,
   1357         proof: AttachmentProof,
   1358     ) -> Result<Option<radroots_storage::inventory::StockRewrapState>, OperationError> {
   1359         let (reply, result) = oneshot::channel();
   1360         self.send(Job::StockRewrapAttempt(Box::new(request), proof, reply))?;
   1361         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1362     }
   1363     pub(super) async fn reconcile_stock_rewrap_admission(
   1364         &self,
   1365         request: radroots_storage::inventory::StockRewrapRequest,
   1366         proof: AttachmentProof,
   1367     ) -> Result<(), OperationError> {
   1368         let (reply, result) = oneshot::channel();
   1369         self.send(Job::RecoverStockRewrapAdmission(
   1370             Box::new(request),
   1371             proof,
   1372             reply,
   1373         ))?;
   1374         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1375     }
   1376     pub(super) async fn capture_stock_rewrap(
   1377         &self,
   1378         request: radroots_storage::inventory::StockRewrapRequest,
   1379         proof: AttachmentProof,
   1380         candidate: radroots_storage::command::PrivateEnvelopeCandidate,
   1381         reconcile: bool,
   1382     ) -> Result<radroots_storage::inventory::StockEnvelopeOutcome, OperationError> {
   1383         let unknown = radroots_storage::inventory::StockEnvelopeOutcome::Unknown(request.attempt());
   1384         let (reply, result) = oneshot::channel();
   1385         let request = stock::envelope::CaptureRequest::Rewrap(Box::new(request));
   1386         self.send(if reconcile {
   1387             Job::RecoverStockEnvelope(request, proof, Box::new(candidate), reply)
   1388         } else {
   1389             Job::CaptureStockEnvelope(request, proof, Box::new(candidate), reply)
   1390         })?;
   1391         result.await.unwrap_or(Ok(unknown))
   1392     }
   1393     pub(super) async fn stock_rewraps(
   1394         &self,
   1395         request: radroots_storage::inventory::StockEnvelopeRequest,
   1396         after: Option<radroots_domain::Sequence>,
   1397         proof: AttachmentProof,
   1398     ) -> Result<radroots_storage::inventory::StockRewrapPage, OperationError> {
   1399         let (reply, result) = oneshot::channel();
   1400         self.send(Job::StockRewraps(Box::new(request), after, proof, reply))?;
   1401         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1402     }
   1403     pub(super) async fn capture_stock_signing_evidence(
   1404         &self,
   1405         evidence: radroots_storage::inventory::StockEvidenceRequest,
   1406         proof: AttachmentProof,
   1407     ) -> Result<stock::StockEvidenceOutcome, OperationError> {
   1408         let unknown = stock::StockEvidenceOutcome::Unknown(evidence.attempt());
   1409         let (reply, result) = oneshot::channel();
   1410         self.send(Job::CaptureStockEvidence(Box::new(evidence), proof, reply))?;
   1411         result.await.unwrap_or(Ok(unknown))
   1412     }
   1413     pub(super) async fn stock_signing_evidence(
   1414         &self,
   1415         request: stock::StockClaimRequest,
   1416         digest: [u8; 32],
   1417         proof: AttachmentProof,
   1418     ) -> Result<Option<stock::StockCapturedEvidence>, OperationError> {
   1419         let (reply, result) = oneshot::channel();
   1420         self.send(Job::StockEvidence(Box::new(request), digest, proof, reply))?;
   1421         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1422     }
   1423     pub(super) async fn claim_stock_signing_attempt(
   1424         &self,
   1425         request: stock::StockClaimRequest,
   1426         proof: AttachmentProof,
   1427     ) -> Result<stock::StockClaimOutcome, OperationError> {
   1428         let unknown = stock::StockClaimOutcome::Unknown(request.attempt());
   1429         let (reply, result) = oneshot::channel();
   1430         self.send(Job::ClaimStockSigning(Box::new(request), proof, reply))?;
   1431         result.await.unwrap_or(Ok(unknown))
   1432     }
   1433     pub(super) async fn authorize_stock_exact_replay(
   1434         &self,
   1435         request: stock::StockClaimRequest,
   1436         proof: AttachmentProof,
   1437     ) -> Result<stock::StockExactReplayPermit, OperationError> {
   1438         let (reply, result) = oneshot::channel();
   1439         self.send(Job::AuthorizeStockExactReplay(
   1440             Box::new(request),
   1441             proof,
   1442             reply,
   1443         ))?;
   1444         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1445     }
   1446     pub(super) async fn stock_signing_attempt(
   1447         &self,
   1448         request: stock::StockClaimRequest,
   1449         proof: AttachmentProof,
   1450     ) -> Result<Option<stock::StockSigningAttempt>, OperationError> {
   1451         let (reply, result) = oneshot::channel();
   1452         self.send(Job::StockSigningAttempt(Box::new(request), proof, reply))?;
   1453         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1454     }
   1455     pub(super) async fn bind_stock_signing_intents(
   1456         &self,
   1457         request: stock::StockSigningPlanRequest,
   1458         proof: AttachmentProof,
   1459     ) -> Result<stock::StockSigningPlanOutcome, OperationError> {
   1460         let unknown = stock::StockSigningPlanOutcome::Unknown(request.attempt());
   1461         let (reply, result) = oneshot::channel();
   1462         self.send(Job::BindStockSigningPlan(Box::new(request), proof, reply))?;
   1463         result.await.unwrap_or(Ok(unknown))
   1464     }
   1465     pub(super) async fn stock_signing_plan(
   1466         &self,
   1467         binding: WorkspaceBinding,
   1468         minimum: radroots_storage::inventory::InventoryCheckpointRef,
   1469         operation: OperationId,
   1470         digest: CommandDigest,
   1471         proof: AttachmentProof,
   1472     ) -> Result<Option<stock::StockBoundSigningPlan>, OperationError> {
   1473         let (reply, result) = oneshot::channel();
   1474         self.send(Job::StockSigningPlan(
   1475             Box::new(binding),
   1476             minimum,
   1477             operation,
   1478             digest,
   1479             proof,
   1480             reply,
   1481         ))?;
   1482         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1483     }
   1484     pub(super) async fn prepare_stock_acceptance(
   1485         &self,
   1486         request: stock::StockAcceptanceRequest,
   1487         proof: AttachmentProof,
   1488     ) -> Result<stock::StockAcceptanceOutcome, OperationError> {
   1489         let unknown = stock::StockAcceptanceOutcome::Unknown(request.attempt());
   1490         let (reply, result) = oneshot::channel();
   1491         self.send(Job::PrepareStockAcceptance(Box::new(request), proof, reply))?;
   1492         result.await.unwrap_or(Ok(unknown))
   1493     }
   1494     pub(super) async fn stock_order_request(
   1495         &self,
   1496         binding: WorkspaceBinding,
   1497         inventory_minimum: radroots_storage::inventory::InventoryCheckpointRef,
   1498         minimum: radroots_storage::workspace::seller_stock::StockSourceCheckpointRef,
   1499         listings: radroots_storage::workspace::seller_stock::StockListingCheckpointRef,
   1500         selected: radroots_domain::RecordId,
   1501         proof: AttachmentProof,
   1502     ) -> Result<stock::StockRequestSnapshot, OperationError> {
   1503         let (reply, result) = oneshot::channel();
   1504         self.send(Job::StockOrderRequest(
   1505             Box::new(binding),
   1506             inventory_minimum,
   1507             minimum,
   1508             listings,
   1509             selected,
   1510             proof,
   1511             reply,
   1512         ))?;
   1513         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1514     }
   1515     pub(super) async fn stock_order_history(
   1516         &self,
   1517         binding: WorkspaceBinding,
   1518         inventory_minimum: radroots_storage::inventory::InventoryCheckpointRef,
   1519         minimum: radroots_storage::workspace::seller_stock::StockSourceCheckpointRef,
   1520         listings: radroots_storage::workspace::seller_stock::StockListingCheckpointRef,
   1521         scope: radroots_domain::causal::CausalScope,
   1522         proof: AttachmentProof,
   1523     ) -> Result<stock::StockHistorySnapshot, OperationError> {
   1524         let (reply, result) = oneshot::channel();
   1525         self.send(Job::StockOrderHistory(
   1526             Box::new(binding),
   1527             inventory_minimum,
   1528             minimum,
   1529             listings,
   1530             scope,
   1531             proof,
   1532             reply,
   1533         ))?;
   1534         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1535     }
   1536     pub(super) async fn stock_source(
   1537         &self,
   1538         binding: WorkspaceBinding,
   1539         minimum: radroots_storage::workspace::seller_stock::StockSourceCheckpointRef,
   1540         envelope: radroots_domain::EnvelopeEventId,
   1541         proof: AttachmentProof,
   1542     ) -> Result<Option<stock::StockSourceSnapshot>, OperationError> {
   1543         let (reply, result) = oneshot::channel();
   1544         self.send(Job::StockSource(
   1545             Box::new(binding),
   1546             minimum,
   1547             envelope,
   1548             proof,
   1549             reply,
   1550         ))?;
   1551         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1552     }
   1553     pub(super) async fn option_mapping(
   1554         &self,
   1555         binding: WorkspaceBinding,
   1556         minimum: radroots_storage::inventory::InventoryCheckpointRef,
   1557         key: radroots_domain::stock::OptionMappingKey,
   1558         proof: AttachmentProof,
   1559     ) -> Result<Option<radroots_domain::stock::OptionPoolMapping>, OperationError> {
   1560         let (reply, result) = oneshot::channel();
   1561         self.send(Job::OptionMapping(
   1562             Box::new(binding),
   1563             minimum,
   1564             key,
   1565             proof,
   1566             reply,
   1567         ))?;
   1568         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1569     }
   1570     pub(super) async fn inspect_stock(
   1571         &self,
   1572         binding: WorkspaceBinding,
   1573         minimum: radroots_storage::inventory::InventoryCheckpointRef,
   1574         id: radroots_domain::PoolId,
   1575         proof: AttachmentProof,
   1576     ) -> Result<Option<radroots_storage::inventory::StockSnapshot>, OperationError> {
   1577         let (reply, result) = oneshot::channel();
   1578         self.send(Job::InspectStock(
   1579             Box::new(binding),
   1580             minimum,
   1581             id,
   1582             proof,
   1583             reply,
   1584         ))?;
   1585         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1586     }
   1587     pub(super) async fn pool(
   1588         &self,
   1589         binding: WorkspaceBinding,
   1590         minimum: radroots_storage::inventory::InventoryCheckpointRef,
   1591         id: radroots_domain::PoolId,
   1592         proof: AttachmentProof,
   1593     ) -> Result<Option<radroots_domain::stock::PoolView>, OperationError> {
   1594         let (reply, result) = oneshot::channel();
   1595         self.send(Job::Pool(Box::new(binding), minimum, id, proof, reply))?;
   1596         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1597     }
   1598     fn send(&self, job: Job) -> Result<(), WorkspaceError> {
   1599         let bytes = match &job {
   1600             Job::RetainStockListing(..)
   1601             | Job::CaptureStockUnissued(..)
   1602             | Job::RecoverStockUnissued(..)
   1603             | Job::CaptureStockEvidence(..)
   1604             | Job::RecoverStockEvidence(..) => 256 * 1024,
   1605             Job::PrepareStockDelivery(..)
   1606             | Job::StockDeliveryAttempt(..)
   1607             | Job::StockDeliverySummary(..)
   1608             | Job::RecordStockDeliveryObservation(..)
   1609             | Job::StockDeliveryObservations(..)
   1610             | Job::RecoverStockEnvelope(..)
   1611             | Job::RecoverStockObservation(..)
   1612             | Job::CaptureStockEnvelope(..)
   1613             | Job::StockEnvelope(..)
   1614             | Job::PrepareStockRewrap(..)
   1615             | Job::StockRewrapAttempt(..)
   1616             | Job::RecoverStockRewrapAdmission(..)
   1617             | Job::StockRewraps(..)
   1618             | Job::StockEvidence(..)
   1619             | Job::StockUnissued(..)
   1620             | Job::StockEvidencePage(..)
   1621             | Job::StockSnapshot(..)
   1622             | Job::InstallStockAgreement(..)
   1623             | Job::StockAgreementInstallation(..)
   1624             | Job::AbortStockPreparation(..)
   1625             | Job::StockPreparationAbort(..) => 1024 * 1024,
   1626             // Catalog19 inventory access authenticates original installed
   1627             // allocations and protected deficit history before its SQL fence.
   1628             Job::ExecuteInventory(..)
   1629             | Job::InventoryOperation(..)
   1630             | Job::InspectStock(..)
   1631             | Job::Pool(..)
   1632             | Job::OptionMapping(..) => 1024 * 1024,
   1633             Job::IssueStockConsent(..) | Job::PrepareStockAcceptance(..) => 512 * 1024,
   1634             Job::BindStockSigningPlan(..)
   1635             | Job::StockSigningPlan(..)
   1636             | Job::ClaimStockSigning(..)
   1637             | Job::StockSigningAttempt(..)
   1638             | Job::AuthorizeStockExactReplay(..) => 1024 * 1024,
   1639             Job::RetainStockSource(..) => 256 * 1024,
   1640             Job::ChangeStockGrant(..) => 64 * 1024,
   1641             Job::CaptureDeliveryObservation(..) => 96 * 1024,
   1642             Job::PrepareDelivery(..) => 1024 * 1024,
   1643             Job::BindPlan(..) => 1024 * 1024,
   1644             Job::Capture(..) | Job::RecoverCapture(..) => 256 * 1024,
   1645             // Public capture can retain a full 64 KiB wire plus decoded
   1646             // bounded fields. Include collection/descriptor ownership as well.
   1647             Job::Admit(request, ..) if request.command().listing().is_some() => 192 * 1024,
   1648             Job::Admit(..) => 96 * 1024,
   1649             _ => 16 * 1024,
   1650         };
   1651         self.queued_bytes
   1652             .try_update(Ordering::AcqRel, Ordering::Acquire, |used| {
   1653                 used.checked_add(bytes)
   1654                     .filter(|next| *next <= QUEUE_MAX_BYTES)
   1655             })
   1656             .map_err(|_| WorkspaceError::Unavailable)?;
   1657         let queued = Queued {
   1658             job,
   1659             _budget: Budget {
   1660                 used: self.queued_bytes.clone(),
   1661                 bytes,
   1662             },
   1663         };
   1664         self.sender
   1665             .lock()
   1666             .map_err(|_| WorkspaceError::Unavailable)?
   1667             .as_ref()
   1668             .ok_or(WorkspaceError::Closed)?
   1669             .try_send(queued)
   1670             .map_err(|error| match error {
   1671                 mpsc::TrySendError::Full(_) => WorkspaceError::Unavailable,
   1672                 mpsc::TrySendError::Disconnected(_) => WorkspaceError::Closed,
   1673             })
   1674     }
   1675     pub(super) async fn prepare_delivery(
   1676         &self,
   1677         request: DeliveryAttemptRequest,
   1678         proof: AttachmentProof,
   1679         candidate: Option<&EnvelopeCandidate>,
   1680     ) -> Result<DeliveryAttemptOutcome, OperationError> {
   1681         let anchor = request.identity();
   1682         let identity = request.intent().request_id().clone();
   1683         let (reply, result) = oneshot::channel();
   1684         self.send(Job::PrepareDelivery(
   1685             Box::new(request),
   1686             proof,
   1687             candidate.map(|value| Box::new(value.retained_copy())),
   1688             reply,
   1689         ))?;
   1690         match result.await {
   1691             Ok(result) => result,
   1692             Err(_) => Ok(DeliveryAttemptOutcome::Unknown {
   1693                 operation: anchor,
   1694                 attempt: identity,
   1695             }),
   1696         }
   1697     }
   1698     pub(super) async fn capture_delivery_observation(
   1699         &self,
   1700         binding: WorkspaceBinding,
   1701         proof: AttachmentProof,
   1702         observation: DeliveryObservation,
   1703     ) -> Result<DeliveryObservationOutcome, OperationError> {
   1704         let fallback = observation.clone();
   1705         let (reply, result) = oneshot::channel();
   1706         self.send(Job::CaptureDeliveryObservation(
   1707             Box::new(binding),
   1708             proof,
   1709             Box::new(observation),
   1710             reply,
   1711         ))?;
   1712         result
   1713             .await
   1714             .unwrap_or(Ok(DeliveryObservationOutcome::Unknown(fallback)))
   1715     }
   1716     pub(super) async fn query_delivery_observations(
   1717         &self,
   1718         binding: WorkspaceBinding,
   1719         operation: OperationId,
   1720         digest: CommandDigest,
   1721         target: SigningTarget,
   1722         request: DeliveryRequestId,
   1723         proof: AttachmentProof,
   1724     ) -> Result<Vec<DeliveryObservation>, OperationError> {
   1725         let (reply, result) = oneshot::channel();
   1726         self.send(Job::QueryDeliveryObservations(
   1727             Box::new(binding),
   1728             operation,
   1729             digest,
   1730             target,
   1731             request,
   1732             proof,
   1733             reply,
   1734         ))?;
   1735         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1736     }
   1737     pub(super) async fn query_delivery(
   1738         &self,
   1739         binding: WorkspaceBinding,
   1740         operation: OperationId,
   1741         digest: CommandDigest,
   1742         target: SigningTarget,
   1743         request: DeliveryRequestId,
   1744         proof: AttachmentProof,
   1745     ) -> Result<Option<DeliveryAttempt>, OperationError> {
   1746         let (reply, result) = oneshot::channel();
   1747         self.send(Job::QueryDelivery(
   1748             Box::new(binding),
   1749             operation,
   1750             digest,
   1751             target,
   1752             request,
   1753             proof,
   1754             reply,
   1755         ))?;
   1756         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1757     }
   1758     pub(super) async fn operation_snapshot(
   1759         &self,
   1760         binding: WorkspaceBinding,
   1761         operation: OperationId,
   1762         digest: Option<CommandDigest>,
   1763         proof: AttachmentProof,
   1764     ) -> Result<Option<OperationSnapshot>, OperationError> {
   1765         let (reply, result) = oneshot::channel();
   1766         self.send(Job::Snapshot(
   1767             Box::new(binding),
   1768             operation,
   1769             digest,
   1770             proof,
   1771             reply,
   1772         ))?;
   1773         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1774     }
   1775     pub(super) async fn install_chat(
   1776         &self,
   1777         request: InstallationRequest,
   1778         proof: AttachmentProof,
   1779     ) -> Result<InstallationOutcome, OperationError> {
   1780         let anchor = request.identity();
   1781         let (reply, result) = oneshot::channel();
   1782         self.send(Job::Install(Box::new(request), proof, reply))?;
   1783         match result.await {
   1784             Ok(result) => result,
   1785             Err(_) => Ok(InstallationOutcome::Unknown(anchor)),
   1786         }
   1787     }
   1788     pub(super) async fn claim_attempt(
   1789         &self,
   1790         request: ClaimAttemptRequest,
   1791         proof: AttachmentProof,
   1792     ) -> Result<ClaimAttemptOutcome, OperationError> {
   1793         let anchor = request.identity();
   1794         let (reply, result) = oneshot::channel();
   1795         self.send(Job::Claim(Box::new(request), proof, reply))?;
   1796         match result.await {
   1797             Ok(result) => result,
   1798             Err(_) => Ok(ClaimAttemptOutcome::Unknown(anchor)),
   1799         }
   1800     }
   1801     pub(super) async fn signing_attempt(
   1802         &self,
   1803         request: ClaimAttemptRequest,
   1804         proof: AttachmentProof,
   1805     ) -> Result<Option<SigningAttempt>, OperationError> {
   1806         let (reply, result) = oneshot::channel();
   1807         self.send(Job::Attempt(Box::new(request), proof, reply))?;
   1808         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1809     }
   1810     pub(super) async fn capture_evidence(
   1811         &self,
   1812         input: AttachmentInput,
   1813         proof: &AttachmentProof,
   1814         evidence: &ArtifactEvidence,
   1815     ) -> Result<EvidenceCaptureOutcome, OperationError> {
   1816         let anchor = CommitAttempt::new(
   1817             evidence.workspace_id(),
   1818             evidence.operation_id(),
   1819             evidence.command_digest(),
   1820             None,
   1821         );
   1822         let copied = proof.with_secret(|bytes| AttachmentProof::from_protected_input(bytes))?;
   1823         let (reply, result) = oneshot::channel();
   1824         self.send(Job::Capture(
   1825             Box::new(input),
   1826             copied,
   1827             Box::new(evidence.retained_copy()),
   1828             reply,
   1829         ))?;
   1830         match result.await {
   1831             Ok(result) => result,
   1832             Err(_) => Ok(EvidenceCaptureOutcome::Unknown(anchor)),
   1833         }
   1834     }
   1835     pub(super) async fn reconcile_unknown(
   1836         &self,
   1837         input: AttachmentInput,
   1838         proof: &AttachmentProof,
   1839         evidence: &ArtifactEvidence,
   1840     ) -> Result<EvidenceCaptureOutcome, OperationError> {
   1841         let anchor = CommitAttempt::new(
   1842             evidence.workspace_id(),
   1843             evidence.operation_id(),
   1844             evidence.command_digest(),
   1845             None,
   1846         );
   1847         let copied = proof.with_secret(|bytes| AttachmentProof::from_protected_input(bytes))?;
   1848         let (reply, result) = oneshot::channel();
   1849         self.send(Job::RecoverCapture(
   1850             Box::new(input),
   1851             copied,
   1852             Box::new(evidence.retained_copy()),
   1853             reply,
   1854         ))?;
   1855         result
   1856             .await
   1857             .unwrap_or(Ok(EvidenceCaptureOutcome::Unknown(anchor)))
   1858     }
   1859     pub(super) async fn obtained_evidence(
   1860         &self,
   1861         request: ClaimAttemptRequest,
   1862         proof: AttachmentProof,
   1863     ) -> Result<Option<CapturedArtifact>, OperationError> {
   1864         let (reply, result) = oneshot::channel();
   1865         self.send(Job::Captured(Box::new(request), proof, reply))?;
   1866         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1867     }
   1868     pub(super) async fn bind_plan(
   1869         &self,
   1870         binding: WorkspaceBinding,
   1871         plan: CapturedSigningPlan,
   1872         proof: AttachmentProof,
   1873     ) -> Result<PlanBindingOutcome, OperationError> {
   1874         let anchor = CommitAttempt::new(
   1875             binding.input().workspace_id(),
   1876             plan.operation_id(),
   1877             plan.command_digest(),
   1878             None,
   1879         );
   1880         let (reply, result) = oneshot::channel();
   1881         self.send(Job::BindPlan(Box::new(binding), plan, proof, reply))?;
   1882         match result.await {
   1883             Ok(result) => result,
   1884             Err(_) => Ok(PlanBindingOutcome::Unknown(anchor)),
   1885         }
   1886     }
   1887     pub(super) async fn signing_plan(
   1888         &self,
   1889         binding: WorkspaceBinding,
   1890         operation_id: OperationId,
   1891         digest: CommandDigest,
   1892         proof: AttachmentProof,
   1893     ) -> Result<Option<BoundSigningPlan>, OperationError> {
   1894         let (reply, result) = oneshot::channel();
   1895         self.send(Job::SigningPlan(
   1896             Box::new(binding),
   1897             operation_id,
   1898             digest,
   1899             proof,
   1900             reply,
   1901         ))?;
   1902         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1903     }
   1904     pub(super) async fn authenticate(
   1905         &self,
   1906         input: AttachmentInput,
   1907         proof: AttachmentProof,
   1908     ) -> Result<WorkspaceBinding, WorkspaceError> {
   1909         let (reply, result) = oneshot::channel();
   1910         self.send(Job::Authenticate(input, proof, reply))?;
   1911         result.await.map_err(|_| WorkspaceError::Unavailable)?
   1912     }
   1913     pub(super) async fn revoke(
   1914         &self,
   1915         input: AttachmentInput,
   1916         proof: &AttachmentProof,
   1917     ) -> Result<(), WorkspaceError> {
   1918         let proof = proof.with_secret(|secret| AttachmentProof::from_protected_input(secret))?;
   1919         let (reply, result) = oneshot::channel();
   1920         self.send(Job::Revoke(input, proof, reply))?;
   1921         result.await.map_err(|_| WorkspaceError::CommitUnknown)?
   1922     }
   1923     pub(super) async fn admit(
   1924         &self,
   1925         request: AdmissionRequest,
   1926         proof: AttachmentProof,
   1927     ) -> Result<AdmissionOutcome, OperationError> {
   1928         let context = request.command().context();
   1929         let anchor = CommitAttempt::new(
   1930             context.workspace_id(),
   1931             context.operation_id(),
   1932             request.command().digest(),
   1933             None,
   1934         );
   1935         let (reply, result) = oneshot::channel();
   1936         self.send(Job::Admit(Box::new(request), proof, reply))?;
   1937         match result.await {
   1938             Ok(result) => result,
   1939             Err(_) => Ok(AdmissionOutcome::Unknown(anchor)),
   1940         }
   1941     }
   1942     pub(super) async fn operation(
   1943         &self,
   1944         binding: WorkspaceBinding,
   1945         operation_id: OperationId,
   1946         digest: Option<CommandDigest>,
   1947         proof: AttachmentProof,
   1948     ) -> Result<Option<PreparedOperation>, OperationError> {
   1949         let (reply, result) = oneshot::channel();
   1950         self.send(Job::Operation(
   1951             Box::new(binding),
   1952             operation_id,
   1953             digest,
   1954             proof,
   1955             reply,
   1956         ))?;
   1957         result.await.map_err(|_| OperationError::RecoveryRequired)?
   1958     }
   1959     pub(super) async fn close(&self) -> Result<(), WorkspaceError> {
   1960         self.sender
   1961             .lock()
   1962             .map_err(|_| WorkspaceError::Unavailable)?
   1963             .take();
   1964         let mut completed = self.completed.clone();
   1965         let result = loop {
   1966             if let Some(result) = *completed.borrow_and_update() {
   1967                 break result;
   1968             }
   1969             completed
   1970                 .changed()
   1971                 .await
   1972                 .map_err(|_| WorkspaceError::Unavailable)?;
   1973         };
   1974         if let Some(thread) = self
   1975             .thread
   1976             .lock()
   1977             .map_err(|_| WorkspaceError::Unavailable)?
   1978             .take()
   1979         {
   1980             thread.join().map_err(|_| WorkspaceError::Unavailable)?;
   1981         }
   1982         result
   1983     }
   1984 }
   1985 
   1986 pub(super) async fn preflight_existing(
   1987     namespace: &Namespace,
   1988     expected: &AttachmentInput,
   1989     crypto: &WorkspaceCrypto,
   1990 ) -> Result<(), WorkspaceError> {
   1991     // Inspect through SQLite read-only before any persistent journal setting or
   1992     // writable connection. Do not turn a foreign schema into a workspace.
   1993     let options = SqliteConnectOptions::new()
   1994         .filename(namespace.database_path())
   1995         .read_only(true)
   1996         .create_if_missing(false)
   1997         .pragma("query_only", "ON")
   1998         .busy_timeout(Duration::from_secs(5));
   1999     let mut connection = SqliteConnection::connect_with(&options)
   2000         .await
   2001         .map_err(|_| WorkspaceError::RecoveryRequired)?;
   2002     let inspected = match namespace.kind() {
   2003         Kind::Outbox => initialize(&mut connection, namespace, expected, None, crypto).await,
   2004         Kind::SellerStock => {
   2005             stock::initialize(&mut connection, namespace, expected, None, crypto).await
   2006         }
   2007     };
   2008     let closed = connection
   2009         .close()
   2010         .await
   2011         .map_err(|_| WorkspaceError::StorageFailure);
   2012     inspected.and(closed)
   2013 }
   2014 
   2015 pub(super) async fn initialize(
   2016     connection: &mut SqliteConnection,
   2017     namespace: &Namespace,
   2018     expected: &AttachmentInput,
   2019     initial: Option<GrantRecord>,
   2020     crypto: &WorkspaceCrypto,
   2021 ) -> Result<(), WorkspaceError> {
   2022     namespace.validate()?;
   2023     if let Some(initial) = initial {
   2024         let plaintext = initial.encode()?;
   2025         let protected = crypto
   2026             .seal(expected.workspace_id(), expected.grant_id(), &plaintext)
   2027             .await?;
   2028         let mut transaction = connection
   2029             .begin()
   2030             .await
   2031             .map_err(|_| WorkspaceError::StorageFailure)?;
   2032         for object in schema::catalog(schema::CURRENT_VERSION)? {
   2033             sqlx::raw_sql(object.sql)
   2034                 .execute(&mut *transaction)
   2035                 .await
   2036                 .map_err(|_| WorkspaceError::StorageFailure)?;
   2037         }
   2038         sqlx::query("PRAGMA user_version=11")
   2039             .execute(&mut *transaction)
   2040             .await
   2041             .map_err(|_| WorkspaceError::StorageFailure)?;
   2042         sqlx::query(
   2043             "INSERT INTO workspace_grant(singleton,workspace,grant_id,protected) VALUES(1,?,?,?)",
   2044         )
   2045         .bind(expected.workspace_id().as_bytes().as_slice())
   2046         .bind(expected.grant_id().as_bytes().as_slice())
   2047         .bind(protected)
   2048         .execute(&mut *transaction)
   2049         .await
   2050         .map_err(|_| WorkspaceError::StorageFailure)?;
   2051         transaction
   2052             .commit()
   2053             .await
   2054             .map_err(|_| WorkspaceError::CommitUnknown)?;
   2055     }
   2056     let version: i64 = sqlx::query_scalar("PRAGMA user_version")
   2057         .fetch_one(&mut *connection)
   2058         .await
   2059         .map_err(|_| WorkspaceError::RecoveryRequired)?;
   2060     schema::admit(connection, version).await?;
   2061     let integrity: String = sqlx::query_scalar("PRAGMA quick_check(1)")
   2062         .fetch_one(&mut *connection)
   2063         .await
   2064         .map_err(|_| WorkspaceError::RecoveryRequired)?;
   2065     if integrity != "ok" {
   2066         return Err(WorkspaceError::RecoveryRequired);
   2067     }
   2068     let protected = read(connection, expected).await?;
   2069     let plaintext = crypto
   2070         .open(expected.workspace_id(), expected.grant_id(), &protected)
   2071         .await?;
   2072     plaintext.expose_secret(GrantRecord::decode_outbox)?;
   2073     namespace.validate()
   2074 }
   2075 pub(super) async fn read(
   2076     connection: &mut SqliteConnection,
   2077     expected: &AttachmentInput,
   2078 ) -> Result<Vec<u8>, WorkspaceError> {
   2079     // Refuse oversized input in SQL before materializing the protected blob.
   2080     let row = sqlx::query("SELECT CASE WHEN length(workspace)=32 THEN workspace END,CASE WHEN length(grant_id)=32 THEN grant_id END,length(protected),CASE WHEN length(protected)<=262144 THEN protected END FROM workspace_grant WHERE singleton=1")
   2081         .fetch_optional(connection).await.map_err(|_| WorkspaceError::RecoveryRequired)?.ok_or(WorkspaceError::RecoveryRequired)?;
   2082     let workspace: Vec<u8> = row
   2083         .try_get::<Option<Vec<u8>>, _>(0)
   2084         .map_err(|_| WorkspaceError::RecoveryRequired)?
   2085         .ok_or(WorkspaceError::RecoveryRequired)?;
   2086     let grant: Vec<u8> = row
   2087         .try_get::<Option<Vec<u8>>, _>(1)
   2088         .map_err(|_| WorkspaceError::RecoveryRequired)?
   2089         .ok_or(WorkspaceError::RecoveryRequired)?;
   2090     if workspace != expected.workspace_id().as_bytes() || grant != expected.grant_id().as_bytes() {
   2091         return Err(WorkspaceError::WrongNamespace);
   2092     }
   2093     let length: i64 = row
   2094         .try_get(2)
   2095         .map_err(|_| WorkspaceError::RecoveryRequired)?;
   2096     if length <= 0 || length > 262144 {
   2097         return Err(WorkspaceError::RecoveryRequired);
   2098     }
   2099     row.try_get(3).map_err(|_| WorkspaceError::RecoveryRequired)
   2100 }
   2101 async fn authenticate(
   2102     connection: &mut SqliteConnection,
   2103     namespace: &Namespace,
   2104     expected: &AttachmentInput,
   2105     crypto: &WorkspaceCrypto,
   2106     clock: &dyn WorkspaceClock,
   2107     selected: &AttachmentInput,
   2108     proof: &AttachmentProof,
   2109 ) -> Result<(GrantRecord, WorkspaceBinding), WorkspaceError> {
   2110     namespace.validate()?;
   2111     let protected = read(connection, expected).await?;
   2112     let plaintext = crypto
   2113         .open(expected.workspace_id(), expected.grant_id(), &protected)
   2114         .await?;
   2115     let record = plaintext.expose_secret(GrantRecord::decode_outbox)?;
   2116     let binding = record.authenticate(selected, proof, clock.now()?)?;
   2117     namespace.validate()?;
   2118     Ok((record, binding))
   2119 }
   2120 async fn revoke(
   2121     connection: &mut SqliteConnection,
   2122     namespace: &Namespace,
   2123     expected: &AttachmentInput,
   2124     crypto: &WorkspaceCrypto,
   2125     clock: &dyn WorkspaceClock,
   2126     selected: &AttachmentInput,
   2127     proof: &AttachmentProof,
   2128 ) -> Result<(), WorkspaceError> {
   2129     let (mut record, _) = authenticate(
   2130         connection, namespace, expected, crypto, clock, selected, proof,
   2131     )
   2132     .await?;
   2133     record.revoke()?;
   2134     let plaintext = record.encode()?;
   2135     let protected = crypto
   2136         .seal(expected.workspace_id(), expected.grant_id(), &plaintext)
   2137         .await?;
   2138     namespace.validate()?;
   2139     let mut transaction = connection
   2140         .begin()
   2141         .await
   2142         .map_err(|_| WorkspaceError::StorageFailure)?;
   2143     let changed = sqlx::query(
   2144         "UPDATE workspace_grant SET protected=? WHERE singleton=1 AND workspace=? AND grant_id=?",
   2145     )
   2146     .bind(protected)
   2147     .bind(expected.workspace_id().as_bytes().as_slice())
   2148     .bind(expected.grant_id().as_bytes().as_slice())
   2149     .execute(&mut *transaction)
   2150     .await
   2151     .map_err(|_| WorkspaceError::StorageFailure)?;
   2152     if changed.rows_affected() != 1 {
   2153         return Err(WorkspaceError::RecoveryRequired);
   2154     }
   2155     transaction
   2156         .commit()
   2157         .await
   2158         .map_err(|_| WorkspaceError::CommitUnknown)
   2159 }