app

Local-first trade for farms and co-ops
git clone https://radroots.dev/git/app.git
Log | Files | Refs | README | LICENSE

commit bddd5b988d9c7f1acc11b023b1068a5d0e6cfc08
parent 9971617b00455d761fc3ff773d9b5cc47bfacb01
Author: triesap <tyson@radroots.org>
Date:   Fri,  2 Oct 2026 18:55:12 +0000

storage: retain verified availability evidence (HCAV-020)

- Migrate to schema v3 while preserving exact v1 and v2 history
- Retain verified original wire and first provenance without local accounts
- Enforce atomic global version and payload limits with bounded corruption checks
- Align storage API v3 and FFI 4.5 with verified Kotlin and native consumers

Diffstat:
MAGENTS.md | 8+++++++-
MREADME.md | 18++++++++++++++----
Mbuild-logic/contracts/src/main/kotlin/org/harvestcircle/buildlogic/contracts/FfiCompatibilityBaseline.kt | 4++--
Abuild-logic/contracts/src/test/kotlin/org/harvestcircle/buildlogic/contracts/AvailabilityEvidenceCompatibilityTest.kt | 53+++++++++++++++++++++++++++++++++++++++++++++++++++++
Mbuild-logic/contracts/src/test/kotlin/org/harvestcircle/buildlogic/contracts/BuildContractsTest.kt | 14+++++++-------
Mbuild-logic/plugins/src/functionalTest/kotlin/org/harvestcircle/buildlogic/plugins/ConventionPluginSmokeTest.kt | 4++--
Mcore/Cargo.lock | 3+++
Mcore/compatibility/harvestcircle-ffi-v4.properties | 6+++---
Acore/compatibility/harvestcircle-storage-api-v3.txt | 90+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mcore/crates/harvestcircle_ffi/build.rs | 6+++---
Mcore/crates/harvestcircle_ffi/src/commands.rs | 2+-
Mcore/crates/harvestcircle_ffi/src/contract.rs | 2+-
Mcore/crates/harvestcircle_ffi/src/lib.rs | 2+-
Mcore/crates/harvestcircle_storage/Cargo.toml | 3+++
Acore/crates/harvestcircle_storage/src/availability_evidence.rs | 432+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acore/crates/harvestcircle_storage/src/availability_evidence_tests.rs | 2081+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mcore/crates/harvestcircle_storage/src/contract.rs | 121+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------
Mcore/crates/harvestcircle_storage/src/lib.rs | 1+
Mcore/crates/harvestcircle_storage/tests/package_boundary.rs | 39+++++++++++++++++++++++++++++++++++++--
Mcore/crates/harvestcircle_storage/tests/sqlx_storage.rs | 6+++---
Mradroots.lib.source-lock.v1.toml | 2+-
Mtools/verify-storage-api.sh | 2+-
Mtools/xtask/src/lib.rs | 12+++++++++---
23 files changed, 2868 insertions(+), 43 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md @@ -79,8 +79,14 @@ substitute. - Product state is bound only through `radroots_runtime_paths::RuntimeContext` for service `harvestcircle` and instance `desktop`; the canonical database and lock names are `state.sqlite` and `state.lock`. Fresh state initializes - at schema v1 and is migrated through the pinned current schema v2 before host + at schema v1 and is migrated through the pinned current schema v3 before host exposure; `radroots_service_sqlite` owns the governed SQLite mechanics. +- Public listing-version evidence retains exact verified original wire and first + named provenance independently of installed accounts or local signing custody. + The global public payload meter admits at most 4,096 versions and 120 MiB of + ordinary growth within its 128 MiB total; 8 MiB remains reserved for recovery. + Charges are logical payload bytes. Duplicate IDs preserve first evidence + without growth; no automatic eviction or recovery bypass is exposed. - The durable-operation journal records terminal completion time, retains terminal receipts for exactly seven days, admits at most 1,024 unfinished operations and 4,096 total rows, and deletes no more than 256 expired terminal diff --git a/README.md b/README.md @@ -101,13 +101,23 @@ envelope matches; another operation conflicts and never overwrites the existing credential. No compatibility path reads the former plaintext credential shape. The state database initializes at schema v1 and applies the pinned schema-v2 -operation-journal migration before the host is exposed. Terminal receipts carry -an explicit completion time and remain replayable for exactly seven days. +operation-journal and schema-v3 public evidence migrations before host exposure. +Terminal receipts carry an explicit completion time and remain replayable for +exactly seven days. Admission caps unfinished operations at 1,024 and all journal rows at 4,096, deletes at most 256 expired terminal receipts in one transaction, reserves each accepted operation's terminal row in place, and never evicts an in-window -receipt. The migration, resulting table and guards, and both schema snapshots -are checksum-pinned; invalid legacy rows roll the migration back atomically. +receipt. The migrations, resulting tables and guards, and all three schema +snapshots are checksum-pinned. Invalid legacy state is refused without replacement; +failed migration transactions roll back atomically. + +Public listing versions retain their exact signed wire, full unsigned event +timestamps, tolerant admission result, and first named provenance without +installing an account or local signer. Loads re-verify bounded original wire. +One durable public payload meter admits at most 4,096 versions and 120 MiB of +ordinary logical payload within a 128 MiB total, preserving an 8 MiB recovery +reserve. Exact-ID duplicates consume no additional capacity; no automatic +eviction or recovery bypass is exposed. ## Project documentation diff --git a/build-logic/contracts/src/main/kotlin/org/harvestcircle/buildlogic/contracts/FfiCompatibilityBaseline.kt b/build-logic/contracts/src/main/kotlin/org/harvestcircle/buildlogic/contracts/FfiCompatibilityBaseline.kt @@ -47,10 +47,10 @@ public class FfiCompatibilityBaseline private constructor( require(values.getValue("schema") == "harvestcircle.ffi.v4") require(values.getValue("contract.id") == "harvestcircle-desktop-ffi-v4") require(values.getValue("contract.major") == "4") - require(values.getValue("contract.minor") == "4") + require(values.getValue("contract.minor") == "5") require(values.getValue("snapshot.schema") == "1") require(values.getValue("storage.schema.minimum") == "1") - require(values.getValue("storage.schema.current") == "2") + require(values.getValue("storage.schema.current") == "3") listOf("contract.hash", "product.coordinate_digest", "source.provenance_digest").forEach { key -> require(values.getValue(key).isCanonicalHex(64)) } diff --git a/build-logic/contracts/src/test/kotlin/org/harvestcircle/buildlogic/contracts/AvailabilityEvidenceCompatibilityTest.kt b/build-logic/contracts/src/test/kotlin/org/harvestcircle/buildlogic/contracts/AvailabilityEvidenceCompatibilityTest.kt @@ -0,0 +1,53 @@ +package org.harvestcircle.buildlogic.contracts + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFails + +public class AvailabilityEvidenceCompatibilityTest { + @Test + public fun allocatedEvidenceSchemaAndMinorAreAccepted() { + val baseline = FfiCompatibilityBaseline.parse(fixture) + + assertEquals("4", baseline["contract.major"]) + assertEquals("5", baseline["contract.minor"]) + assertEquals("1", baseline["snapshot.schema"]) + assertEquals("1", baseline["storage.schema.minimum"]) + assertEquals("3", baseline["storage.schema.current"]) + assertEquals("a".repeat(64), baseline["contract.hash"]) + } + + @Test + public fun previousCurrentSchemaOrMinorAreRejected() { + listOf( + fixture.replace("contract.minor=5", "contract.minor=4"), + fixture.replace("storage.schema.current=3", "storage.schema.current=2"), + fixture.replace("contract.minor=5", "contract.minor=4").replace("storage.schema.current=3", "storage.schema.current=2"), + fixture.replace("contract.minor=5", "contract.minor=6"), + fixture.replace("storage.schema.current=3", "storage.schema.current=4"), + fixture.replace("contract.major=4", "contract.major=5"), + fixture.replace("snapshot.schema=1", "snapshot.schema=2"), + fixture.replace("storage.schema.minimum=1", "storage.schema.minimum=2"), + ).forEach { incompatible -> + assertFails { FfiCompatibilityBaseline.parse(incompatible) } + } + } + + // Synthetic parser inputs carry no native artifact or actual contract-hash claim. + private val fixture = + """ + schema=harvestcircle.ffi.v4 + contract.id=harvestcircle-desktop-ffi-v4 + contract.major=4 + contract.minor=5 + contract.hash=${"a".repeat(64)} + product.coordinate_digest=${"b".repeat(64)} + snapshot.schema=1 + storage.schema.minimum=1 + storage.schema.current=3 + product.version=0.1.0-alpha + package.version=1.0.0 + source.provenance_digest=${"c".repeat(64)} + source.foundation_baseline=${"d".repeat(40)} + """.trimIndent() + "\n" +} diff --git a/build-logic/contracts/src/test/kotlin/org/harvestcircle/buildlogic/contracts/BuildContractsTest.kt b/build-logic/contracts/src/test/kotlin/org/harvestcircle/buildlogic/contracts/BuildContractsTest.kt @@ -210,7 +210,7 @@ class BuildContractsTest { assertFails { FfiCompatibilityBaseline.parse(ffiBaseline.replace("package.version=1.0.0", "package.version=invalid")) } assertFails { FfiCompatibilityBaseline.parse(ffiBaseline.replace("schema=harvestcircle.ffi.v4", "schema=harvestcircle.ffi.v3")) } assertFails { FfiCompatibilityBaseline.parse(ffiBaseline.replace("contract.major=4", "contract.major=3")) } - assertFails { FfiCompatibilityBaseline.parse(ffiBaseline.replace("contract.minor=4", "contract.minor=3")) } + assertFails { FfiCompatibilityBaseline.parse(ffiBaseline.replace("contract.minor=5", "contract.minor=4")) } } @Test @@ -218,12 +218,12 @@ class BuildContractsTest { val baseline = FfiCompatibilityBaseline.parse(ffiBaseline) assertEquals("4", baseline["contract.major"]) - assertEquals("4", baseline["contract.minor"]) + assertEquals("5", baseline["contract.minor"]) assertEquals("1", baseline["snapshot.schema"]) assertEquals("1", baseline["storage.schema.minimum"]) - assertEquals("2", baseline["storage.schema.current"]) - assertFails { FfiCompatibilityBaseline.parse(ffiBaseline.replace("contract.minor=4", "contract.minor=3")) } - assertFails { FfiCompatibilityBaseline.parse(ffiBaseline.replace("contract.minor=4", "contract.minor=5")) } + assertEquals("3", baseline["storage.schema.current"]) + assertFails { FfiCompatibilityBaseline.parse(ffiBaseline.replace("contract.minor=5", "contract.minor=4")) } + assertFails { FfiCompatibilityBaseline.parse(ffiBaseline.replace("contract.minor=5", "contract.minor=6")) } } @Test @@ -379,12 +379,12 @@ class BuildContractsTest { schema=harvestcircle.ffi.v4 contract.id=harvestcircle-desktop-ffi-v4 contract.major=4 - contract.minor=4 + contract.minor=5 contract.hash=${"a".repeat(64)} product.coordinate_digest=${"b".repeat(64)} snapshot.schema=1 storage.schema.minimum=1 - storage.schema.current=2 + storage.schema.current=3 product.version=0.1.0-alpha package.version=1.0.0 source.provenance_digest=${"c".repeat(64)} diff --git a/build-logic/plugins/src/functionalTest/kotlin/org/harvestcircle/buildlogic/plugins/ConventionPluginSmokeTest.kt b/build-logic/plugins/src/functionalTest/kotlin/org/harvestcircle/buildlogic/plugins/ConventionPluginSmokeTest.kt @@ -676,12 +676,12 @@ class ConventionPluginSmokeTest { schema=harvestcircle.ffi.v4 contract.id=harvestcircle-desktop-ffi-v4 contract.major=4 - contract.minor=4 + contract.minor=5 contract.hash=b32b9a47d12e445e93866ae0ab668b18de503ba6c999e3a053f26dc9509ddaf9 product.coordinate_digest=bf50f9ea6c2537406de255f025463e670eb6263c295f992f7e4c4db36d957064 snapshot.schema=1 storage.schema.minimum=1 - storage.schema.current=2 + storage.schema.current=3 product.version=0.1.0-alpha package.version=1.0.0 source.provenance_digest=40b9eccd486026128f92de8d55d002a9030f235a35f9b754c98c0b0d387bd8c0 diff --git a/core/Cargo.lock b/core/Cargo.lock @@ -1077,6 +1077,9 @@ dependencies = [ "harvestcircle_domain", "harvestcircle_nostr", "harvestcircle_product", + "nostr 0.44.1", + "radroots_event", + "radroots_event_codec", "radroots_runtime_paths", "radroots_service_sqlite", "radroots_storage", diff --git a/core/compatibility/harvestcircle-ffi-v4.properties b/core/compatibility/harvestcircle-ffi-v4.properties @@ -1,12 +1,12 @@ schema=harvestcircle.ffi.v4 contract.id=harvestcircle-desktop-ffi-v4 contract.major=4 -contract.minor=4 -contract.hash=7d62bb5f894725b648503254b97f6e9a274daba06196184f1c2014f66f398efd +contract.minor=5 +contract.hash=7c76f853e46c7e345a0bef6e9f962f35a15e14efb24e099905b5fac679d8646a product.coordinate_digest=bf50f9ea6c2537406de255f025463e670eb6263c295f992f7e4c4db36d957064 snapshot.schema=1 storage.schema.minimum=1 -storage.schema.current=2 +storage.schema.current=3 product.version=0.1.0-alpha package.version=1.0.0 source.provenance_digest=6cc524ffa028e7958dabc7d721302b967b7895fb2531f2e1cb6a7efca47bbe3b diff --git a/core/compatibility/harvestcircle-storage-api-v3.txt b/core/compatibility/harvestcircle-storage-api-v3.txt @@ -0,0 +1,90 @@ +pub mod harvestcircle_storage +pub enum harvestcircle_storage::HarvestCircleStorageContractError +pub harvestcircle_storage::HarvestCircleStorageContractError::CanonicalPaths +pub harvestcircle_storage::HarvestCircleStorageContractError::ContextIdentity +pub harvestcircle_storage::HarvestCircleStorageContractError::MigrationCatalog +pub harvestcircle_storage::HarvestCircleStorageContractError::SchemaCatalog +impl core::error::Error for harvestcircle_storage::HarvestCircleStorageContractError +impl core::fmt::Display for harvestcircle_storage::HarvestCircleStorageContractError +pub fn harvestcircle_storage::HarvestCircleStorageContractError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct harvestcircle_storage::Database +impl harvestcircle_storage::Database +pub async fn harvestcircle_storage::Database::capture_online_backup(&self, &std::path::Path, radroots_service_sqlite::backup::manifest::BackupCreatedAtUnixMs) -> core::result::Result<radroots_service_sqlite::backup::manifest::ServiceBackupManifest, harvestcircle_domain::error::SafeError> +pub async fn harvestcircle_storage::Database::restore_verified_backup(&mut self, &radroots_runtime_paths::context::RuntimeContext, harvestcircle_storage::VerifiedHarvestCircleBackup, u64, &radroots_service_sqlite::migration::MigrationBuildIdentity) -> core::result::Result<(), harvestcircle_domain::error::SafeError> +impl harvestcircle_storage::Database +pub async fn harvestcircle_storage::Database::close(&self) -> core::result::Result<(), harvestcircle_domain::error::SafeError> +pub const fn harvestcircle_storage::Database::metadata(&self) -> &radroots_service_sqlite::metadata::ServiceDatabaseMetadata +pub async fn harvestcircle_storage::Database::open(&radroots_runtime_paths::context::RuntimeContext, u64, u64, &radroots_service_sqlite::migration::MigrationBuildIdentity) -> core::result::Result<Self, harvestcircle_domain::error::SafeError> +impl harvestcircle_storage::Database +pub async fn harvestcircle_storage::Database::initialize_installation_id(&self, &str) -> core::result::Result<alloc::string::String, harvestcircle_domain::error::SafeError> +pub async fn harvestcircle_storage::Database::load_installation_id(&self) -> core::result::Result<core::option::Option<alloc::string::String>, harvestcircle_domain::error::SafeError> +impl harvestcircle_storage::Database +pub async fn harvestcircle_storage::Database::load_availability_version(&self, harvestcircle_domain::availability::identity::AvailabilityEventVersion) -> core::result::Result<core::option::Option<harvestcircle_domain::availability::projection::AvailabilityVersionView>, harvestcircle_domain::error::SafeError> +pub async fn harvestcircle_storage::Database::retain_availability_version(&self, harvestcircle_domain::availability::projection::AvailabilityVersionView) -> core::result::Result<(), harvestcircle_domain::error::SafeError> +impl harvestcircle_application::ports::AppStateRepository for harvestcircle_storage::Database +pub fn harvestcircle_storage::Database::load_selected_identity(&self) -> harvestcircle_application::ports::BoxFuture<'_, core::result::Result<core::option::Option<harvestcircle_domain::key::PublicKey>, harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::save_selected_identity(&self, core::option::Option<harvestcircle_domain::key::PublicKey>) -> harvestcircle_application::ports::BoxFuture<'_, core::result::Result<(), harvestcircle_domain::error::SafeError>> +impl harvestcircle_application::ports::DurableOperationRepository for harvestcircle_storage::Database +pub fn harvestcircle_storage::Database::advance_durable_operation<'a>(&'a self, &'a harvestcircle_application::ports::DurableRequestId, harvestcircle_application::ports::DurableOperationPhase, harvestcircle_application::ports::DurableOperationPhase, harvestcircle_domain::time::UnixTimestamp, core::option::Option<harvestcircle_application::ports::OperationDiagnostic>) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<harvestcircle_application::ports::DurableIdentityOperation, harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::begin_durable_operation<'a>(&'a self, &'a harvestcircle_application::ports::DurableRequestId, harvestcircle_application::ports::DurableOperationKind, harvestcircle_domain::key::PublicKey, core::option::Option<u64>, harvestcircle_application::ports::OperationPriorState, harvestcircle_domain::time::UnixTimestamp) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<harvestcircle_application::ports::DurableOperationStart, harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::finalize_durable_operation<'a>(&'a self, &'a harvestcircle_application::ports::DurableRequestId, harvestcircle_application::ports::DurableOperationPhase, harvestcircle_application::ports::DurableTerminalOutcome, core::option::Option<u64>, harvestcircle_domain::time::UnixTimestamp) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<harvestcircle_application::ports::DurableOperationReceipt, harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::list_unfinished_durable_operations(&self) -> harvestcircle_application::ports::BoxFuture<'_, core::result::Result<alloc::vec::Vec<harvestcircle_application::ports::DurableIdentityOperation>, harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::load_durable_operation<'a>(&'a self, &'a harvestcircle_application::ports::DurableRequestId) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<core::option::Option<harvestcircle_application::ports::DurableIdentityOperation>, harvestcircle_domain::error::SafeError>> +impl harvestcircle_application::ports::IdentityNamespaceRepository for harvestcircle_storage::Database +pub fn harvestcircle_storage::Database::clear_owner(&self, harvestcircle_domain::key::PublicKey) -> harvestcircle_application::ports::BoxFuture<'_, core::result::Result<(), harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::get_value<'a>(&'a self, harvestcircle_domain::key::PublicKey, harvestcircle_application::ports::IdentityPreferenceKey) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<core::option::Option<alloc::string::String>, harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::set_value<'a>(&'a self, harvestcircle_domain::key::PublicKey, harvestcircle_application::ports::IdentityPreferenceKey, &'a str) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<(), harvestcircle_domain::error::SafeError>> +impl harvestcircle_application::ports::IdentityRepository for harvestcircle_storage::Database +pub fn harvestcircle_storage::Database::find_identity(&self, harvestcircle_domain::key::PublicKey) -> harvestcircle_application::ports::BoxFuture<'_, core::result::Result<core::option::Option<harvestcircle_domain::identity::NostrIdentity>, harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::insert_identity<'a>(&'a self, &'a harvestcircle_domain::identity::NostrIdentity) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<(), harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::list_identities(&self) -> harvestcircle_application::ports::BoxFuture<'_, core::result::Result<alloc::vec::Vec<harvestcircle_domain::identity::NostrIdentity>, harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::remove_identity(&self, harvestcircle_domain::key::PublicKey) -> harvestcircle_application::ports::BoxFuture<'_, core::result::Result<(), harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::update_identity<'a>(&'a self, &'a harvestcircle_domain::identity::NostrIdentity) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<(), harvestcircle_domain::error::SafeError>> +impl harvestcircle_application::ports::ProfileRepository for harvestcircle_storage::Database +pub fn harvestcircle_storage::Database::load_profile(&self, harvestcircle_domain::key::PublicKey) -> harvestcircle_application::ports::BoxFuture<'_, core::result::Result<core::option::Option<harvestcircle_application::ports::CachedProfile>, harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::record_refresh_status<'a>(&'a self, harvestcircle_domain::key::PublicKey, harvestcircle_domain::time::UnixTimestamp, harvestcircle_application::ports::ProfileRefreshStatus) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<(), harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::remove_profile(&self, harvestcircle_domain::key::PublicKey) -> harvestcircle_application::ports::BoxFuture<'_, core::result::Result<(), harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::Database::save_profile<'a>(&'a self, &'a harvestcircle_application::ports::CachedProfile) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<(), harvestcircle_domain::error::SafeError>> +pub struct harvestcircle_storage::HarvestCircleStorageContract +impl harvestcircle_storage::HarvestCircleStorageContract +pub fn harvestcircle_storage::HarvestCircleStorageContract::application_id(&self) -> radroots_service_sqlite::metadata::ServiceSqliteApplicationId +pub fn harvestcircle_storage::HarvestCircleStorageContract::from_runtime_context(&radroots_runtime_paths::context::RuntimeContext) -> core::result::Result<Self, harvestcircle_storage::HarvestCircleStorageContractError> +pub const fn harvestcircle_storage::HarvestCircleStorageContract::migrations(&self) -> &radroots_service_sqlite::migration::MigrationCatalog +pub const fn harvestcircle_storage::HarvestCircleStorageContract::paths(&self) -> &radroots_service_sqlite::open::ServiceSqlitePaths +pub const fn harvestcircle_storage::HarvestCircleStorageContract::schema(&self) -> &radroots_service_sqlite::integrity::catalog::SchemaCatalog +pub const fn harvestcircle_storage::HarvestCircleStorageContract::state_schema_version(&self) -> core::num::nonzero::NonZeroU32 +impl core::fmt::Debug for harvestcircle_storage::HarvestCircleStorageContract +pub fn harvestcircle_storage::HarvestCircleStorageContract::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct harvestcircle_storage::OsKeyringSecretStore +impl harvestcircle_application::secrets::SecretStore for harvestcircle_storage::OsKeyringSecretStore +pub fn harvestcircle_storage::OsKeyringSecretStore::contains(&self, harvestcircle_domain::key::PublicKey) -> harvestcircle_application::ports::BoxFuture<'_, core::result::Result<bool, harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::OsKeyringSecretStore::delete<'a>(&'a self, &'a harvestcircle_application::ports::DurableRequestId, harvestcircle_domain::key::PublicKey) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<(), harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::OsKeyringSecretStore::load(&self, harvestcircle_domain::key::PublicKey) -> harvestcircle_application::ports::BoxFuture<'_, core::result::Result<harvestcircle_domain::key::SecretKeyInput, harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::OsKeyringSecretStore::put<'a>(&'a self, &'a harvestcircle_application::ports::DurableRequestId, harvestcircle_domain::key::PublicKey, harvestcircle_domain::key::SecretKeyInput) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<(), harvestcircle_domain::error::SafeError>> +pub fn harvestcircle_storage::OsKeyringSecretStore::verify<'a>(&'a self, &'a harvestcircle_application::ports::DurableRequestId, harvestcircle_domain::key::PublicKey, harvestcircle_domain::key::SecretKeyInput) -> harvestcircle_application::ports::BoxFuture<'a, core::result::Result<(), harvestcircle_domain::error::SafeError>> +pub struct harvestcircle_storage::VerifiedHarvestCircleBackup +impl core::fmt::Debug for harvestcircle_storage::VerifiedHarvestCircleBackup +pub fn harvestcircle_storage::VerifiedHarvestCircleBackup::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub const harvestcircle_storage::CREDENTIAL_SERVICE: &str +pub const harvestcircle_storage::CURRENT_SCHEMA_VERSION: u32 +pub const harvestcircle_storage::HARVESTCIRCLE_ACTOR_MAILBOX_CAPACITY: usize +pub const harvestcircle_storage::HARVESTCIRCLE_APPLICATION_ID: u32 +pub const harvestcircle_storage::HARVESTCIRCLE_COMMAND_DEADLINE_MAX_MS: u64 +pub const harvestcircle_storage::HARVESTCIRCLE_COMMAND_DEADLINE_MIN_MS: u64 +pub const harvestcircle_storage::HARVESTCIRCLE_DURABLE_OPERATION_CAPACITY: usize +pub const harvestcircle_storage::HARVESTCIRCLE_DURABLE_OPERATION_CLEANUP_BATCH: usize +pub const harvestcircle_storage::HARVESTCIRCLE_EVENTS_PER_RELAY_CAPACITY: usize +pub const harvestcircle_storage::HARVESTCIRCLE_EVENTS_TOTAL_CAPACITY: usize +pub const harvestcircle_storage::HARVESTCIRCLE_IDENTITY_CAPACITY: usize +pub const harvestcircle_storage::HARVESTCIRCLE_INSTANCE_ID: &str +pub const harvestcircle_storage::HARVESTCIRCLE_OBSERVER_CAPACITY: usize +pub const harvestcircle_storage::HARVESTCIRCLE_PREFERENCE_VALUE_UTF8_BYTES: usize +pub const harvestcircle_storage::HARVESTCIRCLE_RELAY_ENDPOINT_CAPACITY: usize +pub const harvestcircle_storage::HARVESTCIRCLE_RELAY_URL_UTF8_BYTES: usize +pub const harvestcircle_storage::HARVESTCIRCLE_SERVICE_ID: &str +pub const harvestcircle_storage::HARVESTCIRCLE_STATE_SCHEMA_VERSION: u32 +pub const harvestcircle_storage::HARVESTCIRCLE_TERMINAL_RECEIPT_RETENTION_SECONDS: i64 +pub const harvestcircle_storage::HARVESTCIRCLE_UNFINISHED_DURABLE_OPERATION_CAPACITY: usize +pub fn harvestcircle_storage::harvestcircle_migration_catalog() -> core::result::Result<radroots_service_sqlite::migration::MigrationCatalog, harvestcircle_storage::HarvestCircleStorageContractError> +pub fn harvestcircle_storage::harvestcircle_schema_catalog() -> core::result::Result<radroots_service_sqlite::integrity::catalog::SchemaCatalog, harvestcircle_storage::HarvestCircleStorageContractError> +pub fn harvestcircle_storage::verify_harvestcircle_backup(&[u8], radroots_service_sqlite::backup::manifest::BackupManifestSha256, &std::path::Path, &radroots_service_sqlite::metadata::ServiceDatabaseIdentity, core::num::nonzero::NonZeroU64) -> core::result::Result<harvestcircle_storage::VerifiedHarvestCircleBackup, harvestcircle_domain::error::SafeError> diff --git a/core/crates/harvestcircle_ffi/build.rs b/core/crates/harvestcircle_ffi/build.rs @@ -216,10 +216,10 @@ fn validate_baseline_inputs(baseline: &BTreeMap<String, String>) { "harvestcircle-desktop-ffi-v4" ); assert_eq!(required(baseline, "contract.major"), "4"); - assert_eq!(required(baseline, "contract.minor"), "4"); + assert_eq!(required(baseline, "contract.minor"), "5"); assert_eq!(required(baseline, "snapshot.schema"), "1"); assert_eq!(required(baseline, "storage.schema.minimum"), "1"); - assert_eq!(required(baseline, "storage.schema.current"), "2"); + assert_eq!(required(baseline, "storage.schema.current"), "3"); assert_eq!( required(baseline, "product.version"), env!("CARGO_PKG_VERSION") @@ -246,7 +246,7 @@ fn validate_baseline_inputs(baseline: &BTreeMap<String, String>) { required(baseline, "source.foundation_baseline") ); - assert_eq!(required(baseline, "storage.schema.current"), "2"); + assert_eq!(required(baseline, "storage.schema.current"), "3"); } fn required<'a>(baseline: &'a BTreeMap<String, String>, key: &str) -> &'a str { diff --git a/core/crates/harvestcircle_ffi/src/commands.rs b/core/crates/harvestcircle_ffi/src/commands.rs @@ -1615,7 +1615,7 @@ mod tests { .to_path_buf(); assert!(database.ends_with("data/services/harvestcircle/desktop/state.sqlite")); assert!(!database.to_string_lossy().contains("harvestcircle.sqlite3")); - assert_eq!(CURRENT_SCHEMA_VERSION, 2); + assert_eq!(CURRENT_SCHEMA_VERSION, 3); } #[test] diff --git a/core/crates/harvestcircle_ffi/src/contract.rs b/core/crates/harvestcircle_ffi/src/contract.rs @@ -2,7 +2,7 @@ pub const FFI_CONTRACT_ID: &str = env!("HARVESTCIRCLE_FFI_CONTRACT_ID"); pub const PRODUCT_VERSION: &str = env!("HARVESTCIRCLE_PRODUCT_VERSION"); pub const DISTRIBUTION_PACKAGE_VERSION: &str = env!("HARVESTCIRCLE_PACKAGE_VERSION"); pub const FFI_CONTRACT_MAJOR: u16 = 4; -pub const FFI_CONTRACT_MINOR: u16 = 4; +pub const FFI_CONTRACT_MINOR: u16 = 5; pub const PRODUCT_COORDINATE_DIGEST: &str = env!("HARVESTCIRCLE_PRODUCT_COORDINATE_DIGEST"); pub const SNAPSHOT_SCHEMA_VERSION: u32 = 1; pub const MINIMUM_SCHEMA_VERSION: u32 = 1; diff --git a/core/crates/harvestcircle_ffi/src/lib.rs b/core/crates/harvestcircle_ffi/src/lib.rs @@ -60,7 +60,7 @@ mod tests { assert_eq!(env!("CARGO_PKG_VERSION"), "0.1.0-alpha"); assert_eq!(super::FFI_CONTRACT_ID, "harvestcircle-desktop-ffi-v4"); assert_eq!(super::FFI_CONTRACT_MAJOR, 4); - assert_eq!(super::FFI_CONTRACT_MINOR, 4); + assert_eq!(super::FFI_CONTRACT_MINOR, 5); assert_eq!(super::SNAPSHOT_SCHEMA_VERSION, 1); assert_eq!( super::PRODUCT_COORDINATE_DIGEST, diff --git a/core/crates/harvestcircle_storage/Cargo.toml b/core/crates/harvestcircle_storage/Cargo.toml @@ -15,6 +15,8 @@ include = ["src/**", "tests/**", "Cargo.toml"] harvestcircle_application.workspace = true harvestcircle_domain.workspace = true harvestcircle_product.workspace = true +radroots_event = { workspace = true, features = ["std"] } +radroots_event_codec = { workspace = true, features = ["std", "json"] } radroots_runtime_paths.workspace = true radroots_service_sqlite.workspace = true radroots_storage.workspace = true @@ -31,6 +33,7 @@ security-framework-sys = "=2.17.0" [dev-dependencies] harvestcircle_nostr.workspace = true +nostr = { git = "https://github.com/rust-nostr/nostr.git", rev = "5bba5163eb77107f82c4a8262cf29d7f33a73219", package = "nostr" } tempfile = "=3.23.0" tokio = { version = "=1.47.1", features = ["macros", "rt-multi-thread"] } diff --git a/core/crates/harvestcircle_storage/src/availability_evidence.rs b/core/crates/harvestcircle_storage/src/availability_evidence.rs @@ -0,0 +1,432 @@ +//! Exact verified public listing evidence under the sole governed SQLite host. + +use harvestcircle_domain::{ + AvailabilityEventVersion, AvailabilityObservation, AvailabilityUnsupportedReason, + AvailabilityVersionView, SafeError, SafeErrorCode, SafeMessage, UnixTimestamp, +}; +use radroots_event::envelope::event_head::EventHeadCoordinate; +use radroots_event::listing::classified::ClassifiedListingPartition; +use radroots_event::wire::{DEFAULT_RAW_JSON_MAX_BYTES, Nip01EventWire}; +use radroots_event_codec::verify::verify_nip01_event; +use radroots_service_sqlite::ServiceSqliteTransaction; +use sqlx::Row; +use sqlx::sqlite::SqliteRow; + +use crate::Database; +use crate::db::{corrupt_storage, map_transaction_error, storage_unavailable}; + +const VERSION_CAPACITY: i64 = 4096; +const TOTAL_PAYLOAD_BYTES: i64 = 134_217_728; +const ORDINARY_PAYLOAD_BYTES: i64 = 125_829_120; +const FIXED_VERSION_BYTES: usize = 92; +const RAW_IDENTIFIER_BYTES: usize = 4096; +const SOURCE_BYTES: usize = 2048; +const ADMISSION_BYTES: usize = 64; +const REJECTION_BYTES: usize = 64; + +// Every byte-bearing column is projected before any Rust hydration or decoding. +// The retained full length distinguishes a bounded prefix from complete evidence. +pub(crate) const SELECT_AVAILABILITY_VERSION_SQL: &str = "SELECT \ + substr(event_id, 1, 33) AS event_id, length(event_id) AS event_id_bytes, \ + substr(author, 1, 33) AS author, length(author) AS author_bytes, kind, \ + substr(CAST(raw_d AS BLOB), 1, 4097) AS raw_d, \ + length(CAST(raw_d AS BLOB)) AS raw_d_bytes, \ + substr(signed_at, 1, 9) AS signed_at, length(signed_at) AS signed_at_bytes, \ + substr(published_at, 1, 9) AS published_at, length(published_at) AS published_at_bytes, \ + substr(CAST(original_json AS BLOB), 1, 262145) AS original_json, \ + length(CAST(original_json AS BLOB)) AS original_json_bytes, \ + substr(CAST(admission_label AS BLOB), 1, 65) AS admission_label, \ + length(CAST(admission_label AS BLOB)) AS admission_label_bytes, \ + substr(CAST(rejection_code AS BLOB), 1, 65) AS rejection_code, \ + length(CAST(rejection_code AS BLOB)) AS rejection_code_bytes, \ + substr(CAST(source AS BLOB), 1, 2049) AS source, \ + length(CAST(source AS BLOB)) AS source_bytes, observed_at_unix_s, payload_bytes \ + FROM availability_versions WHERE event_id = ? LIMIT 2"; + +impl Database { + /// Retains the first exact signed wire and named provenance for a verified version. + /// + /// Public evidence does not install an account or authorize a local signer. + /// Duplicate IDs preserve their original evidence without consuming more quota. + pub async fn retain_availability_version( + &self, + view: AvailabilityVersionView, + ) -> Result<(), SafeError> { + self.host() + .transaction(|transaction| { + Box::pin(async move { retain_availability_version_on(transaction, view).await }) + }) + .await + .map_err(map_transaction_error) + } + + /// Loads bounded original wire and re-verifies it with the selected shared codec. + /// + /// Missing evidence returns `None`. Corrupt evidence or global accounting + /// fails closed without resetting state or constructing a verified substitute. + pub async fn load_availability_version( + &self, + version: AvailabilityEventVersion, + ) -> Result<Option<AvailabilityVersionView>, SafeError> { + self.host() + .transaction(|transaction| { + Box::pin(async move { + validate_public_usage(transaction).await?; + let row = sqlx::query(SELECT_AVAILABILITY_VERSION_SQL) + .bind(version.event_id().as_bytes().as_slice()) + .fetch_optional(&mut *transaction) + .await + .map_err(|_| corrupt_storage())?; + row.map(decode_availability_row).transpose() + }) + }) + .await + .map_err(map_transaction_error) + } +} + +/// The real insertion path, also exercised inside governed rollback fixtures. +pub(crate) async fn retain_availability_version_on( + transaction: &mut ServiceSqliteTransaction<'_>, + view: AvailabilityVersionView, +) -> Result<(), SafeError> { + let (count, bytes) = validate_public_usage(transaction).await?; + let existing = sqlx::query(SELECT_AVAILABILITY_VERSION_SQL) + .bind(view.version().event_id().as_bytes().as_slice()) + .fetch_optional(&mut *transaction) + .await + .map_err(|_| corrupt_storage())?; + if let Some(row) = existing { + let existing = decode_availability_row(row)?; + // Signature-verified ID correlation binds signed data. Original JSON + // formatting, signature variants, extras and later provenance can differ. + if existing.version() != view.version() + || existing.publisher() != view.publisher() + || existing.created_at() != view.created_at() + || existing.raw_coordinate() != view.raw_coordinate() + || existing.focused() != view.focused() + || existing.unsupported_reason() != view.unsupported_reason() + { + return Err(corrupt_storage()); + } + return Ok(()); + } + + let (label, reason) = admission(&view)?; + let charge = version_charge(&view, label, reason)?; + let next_bytes = bytes.checked_add(charge).ok_or_else(corrupt_storage)?; + if count >= VERSION_CAPACITY || next_bytes > ORDINARY_PAYLOAD_BYTES { + return Err(capacity()); + } + let reserved = sqlx::query( + "UPDATE public_payload_usage SET version_count = version_count + 1, \ + payload_bytes = payload_bytes + ? \ + WHERE singleton = 1 AND version_count = ? AND payload_bytes = ?", + ) + .bind(charge) + .bind(count) + .bind(bytes) + .execute(&mut *transaction) + .await + .map_err(|_| storage_unavailable())?; + if reserved.rows_affected() != 1 { + return Err(corrupt_storage()); + } + + let signed_at = view.created_at().as_u64().to_be_bytes(); + let published_at = view + .focused() + .map(|projection| projection.published_at().as_u64().to_be_bytes()); + // Borrow the owned verified view after quota reservation: no full-wire clone. + let inserted = sqlx::query( + "INSERT INTO availability_versions \ + (event_id, author, kind, raw_d, signed_at, published_at, original_json, \ + admission_label, rejection_code, source, observed_at_unix_s, payload_bytes) \ + VALUES (?, ?, 30402, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(view.version().event_id().as_bytes().as_slice()) + .bind(view.publisher().public_key().as_bytes().as_slice()) + .bind(identifier(&view)?) + .bind(signed_at.as_slice()) + .bind(published_at.as_ref().map(|bytes| bytes.as_slice())) + .bind(view.original_json()) + .bind(label) + .bind(reason) + .bind(view.observation().source().as_str()) + .bind(view.observation().observed_at().as_seconds()) + .bind(charge) + .execute(&mut *transaction) + .await + .map_err(|_| storage_unavailable())?; + if inserted.rows_affected() != 1 { + return Err(corrupt_storage()); + } + Ok(()) +} + +async fn validate_public_usage( + transaction: &mut ServiceSqliteTransaction<'_>, +) -> Result<(i64, i64), SafeError> { + let rows = sqlx::query( + "SELECT singleton, version_count, payload_bytes FROM public_payload_usage LIMIT 2", + ) + .fetch_all(&mut *transaction) + .await + .map_err(|_| corrupt_storage())?; + let [row] = rows.as_slice() else { + return Err(corrupt_storage()); + }; + let singleton: i64 = row.try_get("singleton").map_err(|_| corrupt_storage())?; + let count: i64 = row + .try_get("version_count") + .map_err(|_| corrupt_storage())?; + let bytes: i64 = row + .try_get("payload_bytes") + .map_err(|_| corrupt_storage())?; + if singleton != 1 + || !(0..=VERSION_CAPACITY).contains(&count) + || !(0..=TOTAL_PAYLOAD_BYTES).contains(&bytes) + { + return Err(corrupt_storage()); + } + // LIMIT bounds corruption inspection as well as normal state. The sums + // operate on byte lengths and integers without hydrating retained wire. + let actual = sqlx::query( + "SELECT count(*) AS actual_count, \ + coalesce(sum(payload_bytes), 0) AS recorded_bytes, \ + coalesce(sum(actual_charge), 0) AS actual_bytes, \ + coalesce(max(CASE WHEN payload_bytes = actual_charge THEN 0 ELSE 1 END), 0) AS mismatch \ + FROM (SELECT payload_bytes, 92 + length(CAST(original_json AS BLOB)) \ + + length(CAST(raw_d AS BLOB)) + length(CAST(source AS BLOB)) \ + + length(CAST(admission_label AS BLOB)) \ + + coalesce(length(CAST(rejection_code AS BLOB)), 0) AS actual_charge \ + FROM availability_versions LIMIT 4097)", + ) + .fetch_one(&mut *transaction) + .await + .map_err(|_| corrupt_storage())?; + let actual_count: i64 = actual + .try_get("actual_count") + .map_err(|_| corrupt_storage())?; + let recorded_bytes: i64 = actual + .try_get("recorded_bytes") + .map_err(|_| corrupt_storage())?; + let actual_bytes: i64 = actual + .try_get("actual_bytes") + .map_err(|_| corrupt_storage())?; + let mismatch: i64 = actual.try_get("mismatch").map_err(|_| corrupt_storage())?; + if actual_count != count || recorded_bytes != bytes || actual_bytes != bytes || mismatch != 0 { + return Err(corrupt_storage()); + } + Ok((count, bytes)) +} + +pub(crate) fn decode_availability_row( + row: SqliteRow, +) -> Result<AvailabilityVersionView, SafeError> { + let event_id = exact_bytes::<32>(&row, "event_id", "event_id_bytes")?; + let author = exact_bytes::<32>(&row, "author", "author_bytes")?; + let signed_at = u64::from_be_bytes(exact_bytes::<8>(&row, "signed_at", "signed_at_bytes")?); + let published_at = optional_bytes(&row, "published_at", "published_at_bytes", 8)? + .map(|bytes| { + bytes + .try_into() + .map(u64::from_be_bytes) + .map_err(|_| corrupt_storage()) + }) + .transpose()?; + let kind: i64 = row.try_get("kind").map_err(|_| corrupt_storage())?; + let observed_at: i64 = row + .try_get("observed_at_unix_s") + .map_err(|_| corrupt_storage())?; + let charge: i64 = row + .try_get("payload_bytes") + .map_err(|_| corrupt_storage())?; + if kind != 30402 || observed_at < 0 { + return Err(corrupt_storage()); + } + let raw_d = text(&row, "raw_d", "raw_d_bytes", RAW_IDENTIFIER_BYTES)?; + let source = text(&row, "source", "source_bytes", SOURCE_BYTES)?; + let label = text( + &row, + "admission_label", + "admission_label_bytes", + ADMISSION_BYTES, + )?; + let reason = optional_bytes( + &row, + "rejection_code", + "rejection_code_bytes", + REJECTION_BYTES, + )? + .map(|bytes| String::from_utf8(bytes).map_err(|_| corrupt_storage())) + .transpose()?; + if reason + .as_ref() + .is_some_and(|code| code.is_empty() || !code.is_ascii()) + { + return Err(corrupt_storage()); + } + let original_json = text( + &row, + "original_json", + "original_json_bytes", + DEFAULT_RAW_JSON_MAX_BYTES, + )?; + let verified = verify_nip01_event( + Nip01EventWire::parse_json_unverified(&original_json) + .map_err(|_| corrupt_storage())? + .into_unverified_envelope() + .map_err(|_| corrupt_storage())?, + ) + .map_err(|_| corrupt_storage())?; + let observation = AvailabilityObservation::parse( + &source, + UnixTimestamp::from_seconds(observed_at).ok_or_else(corrupt_storage)?, + ) + .map_err(|_| corrupt_storage())?; + let view = AvailabilityVersionView::from_verified(verified, &original_json, observation) + .map_err(|_| corrupt_storage())?; + let (expected_label, expected_reason) = admission(&view)?; + let expected_publication = view + .focused() + .map(|projection| projection.published_at().as_u64()); + if view.version().event_id().as_bytes() != &event_id + || view.publisher().public_key().as_bytes() != &author + || view.created_at().as_u64() != signed_at + || identifier(&view)? != raw_d + || view.observation().source().as_str() != source + || label != expected_label + || reason.as_deref() != expected_reason + || published_at != expected_publication + || charge != version_charge(&view, expected_label, expected_reason)? + { + return Err(corrupt_storage()); + } + Ok(view) +} + +fn bounded_bytes( + row: &SqliteRow, + field: &str, + length_field: &str, + maximum: usize, +) -> Result<Vec<u8>, SafeError> { + let length: i64 = row.try_get(length_field).map_err(|_| corrupt_storage())?; + let length = usize::try_from(length).map_err(|_| corrupt_storage())?; + if length > maximum { + return Err(corrupt_storage()); + } + let bytes: Vec<u8> = row.try_get(field).map_err(|_| corrupt_storage())?; + if bytes.len() != length { + return Err(corrupt_storage()); + } + Ok(bytes) +} + +fn exact_bytes<const N: usize>( + row: &SqliteRow, + field: &str, + length_field: &str, +) -> Result<[u8; N], SafeError> { + bounded_bytes(row, field, length_field, N)? + .try_into() + .map_err(|_| corrupt_storage()) +} + +fn optional_bytes( + row: &SqliteRow, + field: &str, + length_field: &str, + maximum: usize, +) -> Result<Option<Vec<u8>>, SafeError> { + let length: Option<i64> = row.try_get(length_field).map_err(|_| corrupt_storage())?; + if length.is_some() { + return bounded_bytes(row, field, length_field, maximum).map(Some); + } + let bytes: Option<Vec<u8>> = row.try_get(field).map_err(|_| corrupt_storage())?; + if bytes.is_some() { + return Err(corrupt_storage()); + } + Ok(None) +} + +fn text( + row: &SqliteRow, + field: &str, + length_field: &str, + maximum: usize, +) -> Result<String, SafeError> { + String::from_utf8(bounded_bytes(row, field, length_field, maximum)?) + .map_err(|_| corrupt_storage()) +} + +fn identifier(view: &AvailabilityVersionView) -> Result<&str, SafeError> { + match view.raw_coordinate() { + EventHeadCoordinate::Addressable { + kind: 30402, d_tag, .. + } => Ok(d_tag), + _ => Err(corrupt_storage()), + } +} + +fn admission( + view: &AvailabilityVersionView, +) -> Result<(&'static str, Option<&'static str>), SafeError> { + match (view.focused(), view.unsupported_reason().copied()) { + (Some(_), None) => Ok(("focused", None)), + (None, Some(AvailabilityUnsupportedReason::Excluded(partition))) => Ok(( + match partition { + ClassifiedListingPartition::FocusedFoodAvailability => "excluded_focused", + ClassifiedListingPartition::OperationalListing => "excluded_operational", + ClassifiedListingPartition::GenericNip99 => "excluded_generic", + ClassifiedListingPartition::Ambiguous => "excluded_ambiguous", + }, + None, + )), + (None, Some(AvailabilityUnsupportedReason::ProjectionRejected(code))) + if !code.is_empty() && code.len() <= REJECTION_BYTES && code.is_ascii() => + { + Ok(("projection_rejected", Some(code))) + } + _ => Err(corrupt_storage()), + } +} + +fn version_charge( + view: &AvailabilityVersionView, + label: &str, + reason: Option<&str>, +) -> Result<i64, SafeError> { + let identifier = identifier(view)?; + let source = view.observation().source().as_str(); + if view.original_json().len() > DEFAULT_RAW_JSON_MAX_BYTES + || identifier.len() > RAW_IDENTIFIER_BYTES + || source.is_empty() + || source.len() > SOURCE_BYTES + || label.is_empty() + || label.len() > ADMISSION_BYTES + || reason + .is_some_and(|code| code.is_empty() || code.len() > REJECTION_BYTES || !code.is_ascii()) + { + return Err(corrupt_storage()); + } + let bytes = [ + view.original_json().len(), + identifier.len(), + source.len(), + label.len(), + reason.map_or(0, str::len), + ] + .into_iter() + .try_fold(FIXED_VERSION_BYTES, usize::checked_add) + .ok_or_else(corrupt_storage)?; + i64::try_from(bytes).map_err(|_| corrupt_storage()) +} + +const fn capacity() -> SafeError { + SafeError::new( + SafeErrorCode::AvailabilityCapacity, + SafeMessage::new("The retained availability evidence is at capacity."), + ) +} diff --git a/core/crates/harvestcircle_storage/src/availability_evidence_tests.rs b/core/crates/harvestcircle_storage/src/availability_evidence_tests.rs @@ -0,0 +1,2081 @@ +//! HCAV-020: bounded public fixtures and the real governed storage boundary. + +use super::*; +use std::fs; +use std::num::NonZeroU64; +use std::sync::Arc; + +use harvestcircle_application::IdentityRepository; +use harvestcircle_domain::{ + AvailabilityEventVersion, AvailabilityObservation, AvailabilityUnsupportedReason, + AvailabilityVersionView, EventId, IdentityCreatedAt, Kind0ProfileCandidate, + LocalKeyringBinding, NostrIdentity, NostrIdentityReference, ProfileMetadata, SafeError, + SafeErrorCode, SafeMessage, SignerAvailability, UnixTimestamp, +}; +use nostr::{EventBuilder, JsonUtil, Keys, Kind, Tag, Timestamp}; +use radroots_event::envelope::event_head::EventHeadCoordinate; +use radroots_event::listing::classified::ClassifiedListingPartition; +use radroots_event::wire::{DEFAULT_RAW_JSON_MAX_BYTES, Nip01EventWire}; +use radroots_event_codec::verify::verify_nip01_event; +use radroots_runtime_paths::{ + InstanceId, RadrootsHostEnvironment, RadrootsPathProfile, RadrootsPathResolver, + RadrootsPlatform, RuntimeContextBootstrap, RuntimeContextSource, ServiceId, +}; +use radroots_service_sqlite::{ + BackupCreatedAtUnixMs, MigrationAppliedAtUnixSeconds, MigrationBuildIdentity, + ServiceDatabaseMetadata, ServiceSqliteConnectionOptions, ServiceSqliteHost, + ServiceSqliteInitializer, ServiceSqliteInitializerFuture, ServiceSqliteTransactionErrorKind, +}; +use sqlx::sqlite::{SqliteConnectOptions, SqliteRow}; +use sqlx::{Connection, Row, SqliteConnection}; +use tempfile::{TempDir, tempdir_in}; + +use crate::availability_evidence::{ + SELECT_AVAILABILITY_VERSION_SQL, decode_availability_row, retain_availability_version_on, +}; +use crate::{Database, verify_harvestcircle_backup}; + +const VERSION_CAPACITY: i64 = 4096; +const TOTAL_PAYLOAD_BYTES: i64 = 134_217_728; +const ORDINARY_PAYLOAD_BYTES: i64 = 125_829_120; +const RECOVERY_RESERVE_BYTES: i64 = 8_388_608; +const SOURCE: &str = "wss://evidence.example.invalid/public"; +const OBSERVED_AT: i64 = 1_800_000_123; +const SNAPSHOT: &str = "SELECT \ + (SELECT group_concat(hex(public_key) || ':' || npub || ':' || label || ':' || created_at_unix_s, '|') FROM account_identities) AS accounts, \ + (SELECT group_concat(hex(account_public_key) || ':' || hex(binding_public_key) || ':' || availability, '|') FROM local_signer_bindings) AS bindings, \ + (SELECT group_concat(hex(subject_public_key) || ':' || hex(event_id) || ':' || name || ':' || refreshed_at_unix_s, '|') FROM profile_cache) AS profiles, \ + (SELECT group_concat(hex(owner_public_key) || ':' || preference_key || ':' || preference_value, '|') FROM account_preferences) AS preferences, \ + (SELECT group_concat(request_id || ':' || phase || ':' || updated_at_unix_s, '|') FROM durable_operations) AS journal, \ + (SELECT hex(installation_id) FROM installation_identity WHERE singleton = 1) AS installation, \ + (SELECT hex(selected_public_key) || ':' || hex(active_account_public_key) || ':' || session_generation FROM runtime_state WHERE singleton = 1) AS runtime"; + +struct Fixture { + directory: TempDir, + context: RuntimeContext, + build: MigrationBuildIdentity, +} + +impl Fixture { + fn new() -> Self { + let directory = tempdir_in(std::env::temp_dir().canonicalize().expect("temporary root")) + .expect("isolated directory"); + let context = RuntimeContext::resolve( + &RadrootsPathResolver::new( + RadrootsPlatform::current(), + RadrootsHostEnvironment::default(), + ), + RuntimeContextBootstrap::new( + RadrootsPathProfile::RepoLocal, + Some(directory.path().canonicalize().expect("canonical root")), + RuntimeContextSource::BootstrapCli, + RuntimeContextSource::SafeDefault, + ) + .expect("bootstrap"), + ServiceId::new("harvestcircle").expect("service"), + InstanceId::new("desktop").expect("instance"), + ) + .expect("runtime context"); + fs::create_dir(directory.path().join("data")).expect("injected state root"); + let build = MigrationBuildIdentity::new( + "0.1.0-alpha", + "1111111111111111111111111111111111111111", + "189c49b74b4bafc142b00b76b296477931139e72", + "1.97.1", + "test", + "test", + 1, + 1, + 1, + 1, + 1, + ) + .expect("migration evidence"); + Self { + directory, + context, + build, + } + } + + async fn open(&self) -> Database { + Database::open(&self.context, 99, 100, &self.build) + .await + .expect("governed database") + } + + // This helper is called only after the sole governed host has closed. + async fn closed_connection(&self) -> SqliteConnection { + SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .filename(self.context.paths().state().join("state.sqlite")) + .create_if_missing(false) + .foreign_keys(true), + ) + .await + .expect("closed-host test fault connection") + } +} + +#[derive(Clone, Copy)] +enum Recipe { + Focused, + Generic, + Operational, + Ambiguous, + Rejected, +} + +fn observation(source: &str, observed_at: i64) -> AvailabilityObservation { + AvailabilityObservation::parse( + source, + UnixTimestamp::from_seconds(observed_at).expect("nonnegative observation"), + ) + .expect("bounded named public provenance") +} + +fn view_from_wire(json: &str, source: &str, observed_at: i64) -> AvailabilityVersionView { + assert!(json.len() <= 262_144); + let verified = verify_nip01_event( + Nip01EventWire::parse_json_unverified(json) + .expect("bounded public wire") + .into_unverified_envelope() + .expect("actual signed envelope"), + ) + .expect("selected shared ID and Schnorr verification"); + AvailabilityVersionView::from_verified(verified, json, observation(source, observed_at)) + .expect("actual tolerant shared/domain view") +} + +// No fixture secret escapes this builder or reaches a tested repository call. +fn public_view( + identifier: &str, + created_at: u64, + publication: u64, + recipe: Recipe, + wire_bytes: Option<usize>, +) -> AvailabilityVersionView { + let keys = Keys::generate(); + let json = public_wire( + &keys, + identifier, + created_at, + publication, + recipe, + wire_bytes, + ); + drop(keys); + view_from_wire(&json, SOURCE, OBSERVED_AT) +} + +fn public_wire( + keys: &Keys, + identifier: &str, + created_at: u64, + publication: u64, + recipe: Recipe, + wire_bytes: Option<usize>, +) -> String { + assert!(identifier.len() <= 4096); + let mut tags: Vec<Vec<String>> = vec![ + vec!["d".into(), identifier.into()], + vec!["title".into(), "Public storage listing".into()], + vec!["summary".into(), "Public evidence fixture".into()], + vec!["published_at".into(), publication.to_string()], + vec!["location".into(), "Fixture location".into()], + vec!["price".into(), "3.5".into(), "CAD".into()], + vec!["status".into(), "active".into()], + ]; + match recipe { + Recipe::Focused | Recipe::Rejected => { + tags.push(vec!["radroots:price_unit".into(), "lb".into()]) + } + Recipe::Generic => {} + Recipe::Operational => tags.push(vec!["radroots:primary_bin".into(), "public-bin".into()]), + Recipe::Ambiguous => { + tags.push(vec!["radroots:price_unit".into(), "lb".into()]); + tags.push(vec!["radroots:primary_bin".into(), "public-bin".into()]); + } + } + if matches!(recipe, Recipe::Rejected) { + tags.retain(|tag| tag[0] != "price"); + } + assert!(tags.len() <= 10 && tags.iter().map(Vec::len).sum::<usize>() <= 32); + assert!(tags.iter().flatten().all(|value| value.len() <= 4096)); + let event = EventBuilder::new( + Kind::from(30402_u16), + "Public fixture; no custody or relay effects.", + ) + .tags( + tags.into_iter() + .map(|tag| Tag::parse(tag).expect("bounded public tag")), + ) + .custom_created_at(Timestamp::from(created_at)) + .sign_with_keys(keys) + .expect("ephemeral public fixture signing"); + let compact = event.as_json(); + match wire_bytes { + None => compact, + Some(bytes) => { + assert!(bytes <= 262_144 && bytes >= compact.len() + 2); + format!("\n{}{}\n", compact, " ".repeat(bytes - compact.len() - 2)) + } + } +} + +fn focused(identifier: &str) -> AvailabilityVersionView { + public_view( + identifier, + 1_800_000_100, + 1_800_000_000, + Recipe::Focused, + None, + ) +} + +fn raw_d(view: &AvailabilityVersionView) -> &str { + match view.raw_coordinate() { + EventHeadCoordinate::Addressable { kind, d_tag, .. } => { + assert_eq!(*kind, 30402); + d_tag + } + _ => panic!("kind-30402 fixture has an addressable raw coordinate"), + } +} + +fn expected_charge(view: &AvailabilityVersionView, label: &str, reason: Option<&str>) -> i64 { + // Independent frozen accounting; this never calls a production charge helper. + i64::try_from( + 92 + view.original_json().len() + + raw_d(view).len() + + view.observation().source().as_str().len() + + label.len() + + reason.map_or(0, str::len), + ) + .expect("bounded logical payload charge") +} + +async fn rows(host: &ServiceSqliteHost, sql: &'static str) -> Vec<SqliteRow> { + host.transaction(|transaction| { + Box::pin(async move { sqlx::query(sql).fetch_all(&mut *transaction).await }) + }) + .await + .expect("governed inspection") +} + +async fn usage(database: &Database) -> (i64, i64) { + let rows = rows( + database.host(), + "SELECT version_count, payload_bytes FROM public_payload_usage WHERE singleton = 1", + ) + .await; + assert_eq!(rows.len(), 1, "one global public payload meter"); + (rows[0].get("version_count"), rows[0].get("payload_bytes")) +} + +async fn assert_usage(database: &Database, count: i64, bytes: i64) { + assert_eq!(usage(database).await, (count, bytes)); + let rows = rows(database.host(), "SELECT count(*) AS versions, coalesce(sum(payload_bytes), 0) AS charged, coalesce(sum(92 + length(CAST(original_json AS BLOB)) + length(CAST(raw_d AS BLOB)) + length(CAST(source AS BLOB)) + length(CAST(admission_label AS BLOB)) + coalesce(length(CAST(rejection_code AS BLOB)), 0)), 0) AS actual_charge FROM availability_versions").await; + assert_eq!(rows[0].get::<i64, _>("versions"), count); + assert_eq!(rows[0].get::<i64, _>("charged"), bytes); + assert_eq!(rows[0].get::<i64, _>("actual_charge"), bytes); +} + +async fn stored_row(database: &Database, version: AvailabilityEventVersion) -> SqliteRow { + database + .host() + .transaction(|transaction| { + Box::pin(async move { + sqlx::query("SELECT * FROM availability_versions WHERE event_id = ?") + .bind(version.event_id().as_bytes().as_slice()) + .fetch_one(&mut *transaction) + .await + }) + }) + .await + .expect("small valid retained row inspection") +} + +fn assert_round_trip(actual: &AvailabilityVersionView, expected: &AvailabilityVersionView) { + assert_eq!(actual.version(), expected.version()); + assert_eq!(actual.publisher(), expected.publisher()); + assert_eq!(actual.created_at(), expected.created_at()); + assert_eq!(actual.raw_coordinate(), expected.raw_coordinate()); + assert_eq!(actual.listing_coordinate(), expected.listing_coordinate()); + assert_eq!(actual.original_json(), expected.original_json()); + assert_eq!( + actual.observation().source().as_str(), + expected.observation().source().as_str() + ); + assert_eq!( + actual.observation().observed_at(), + expected.observation().observed_at() + ); + assert_eq!(actual.focused(), expected.focused()); + assert_eq!(actual.unsupported_reason(), expected.unsupported_reason()); + assert!(actual.profile_metadata().is_none()); +} + +fn legacy_catalogs() -> (MigrationCatalog, SchemaCatalog) { + let migration = MigrationDescriptor::sql( + 2, + "bound_durable_operation_receipts", + MIGRATE_DURABLE_OPERATIONS_V2_SQL, + DURABLE_OPERATIONS_V2_MIGRATION_CHECKSUM, + ) + .expect("historical migration"); + let migrations = MigrationCatalog::new([migration]).expect("historical catalog"); + let one = SchemaVersionCatalog::new( + 1, + schema_objects(CREATE_DURABLE_OPERATIONS_SQL).expect("v1 objects"), + SchemaDigest::from_bytes(VERSION_ONE_DIGEST), + ) + .expect("v1 snapshot"); + let two = SchemaVersionCatalog::new( + 2, + schema_objects(CREATE_DURABLE_OPERATIONS_V2_SQL).expect("v2 objects"), + SchemaDigest::from_bytes(VERSION_TWO_DIGEST), + ) + .expect("v2 snapshot"); + let schema = SchemaCatalog::new(&migrations, [one, two]).expect("historical schema"); + (migrations, schema) +} + +fn legacy_initializer<'a>( + initializer: &'a mut ServiceSqliteInitializer<'_>, +) -> ServiceSqliteInitializerFuture<'a, sqlx::Error> { + Box::pin(async move { + for statement in harvestcircle_initial_schema_sql() { + sqlx::query(*statement).execute(&mut *initializer).await?; + } + sqlx::query("INSERT INTO runtime_state (singleton) VALUES (1)") + .execute(&mut *initializer) + .await?; + Ok(()) + }) +} + +async fn legacy_host(fixture: &Fixture) -> (ServiceSqliteHost, ServiceDatabaseMetadata) { + let contract = + HarvestCircleStorageContract::from_runtime_context(&fixture.context).expect("paths"); + let (migrations, schema) = legacy_catalogs(); + fixture + .context + .state_directory_plan() + .expect("state plan") + .provision() + .expect("exact service suffix"); + // A public source identity fixture is not a key, counter or admission proof. + let generation = + radroots_storage::event::SourceGeneration::new([71; 32]).expect("public source identity"); + let metadata = ServiceDatabaseMetadata::new( + contract.paths(), + generation, + NonZeroU32::new(1).unwrap(), + 17, + contract.application_id(), + ) + .expect("historical metadata"); + let (opened, migration) = ServiceSqliteHost::open_or_initialize( + contract.paths(), + &metadata, + &migrations, + &schema, + ServiceSqliteConnectionOptions::reviewed(), + MigrationAppliedAtUnixSeconds::new(18).unwrap(), + &fixture.build, + &[], + legacy_initializer, + ) + .await + .expect("real governed v1-to-v2 setup"); + assert_eq!(migration.final_version(), 2); + opened.into_parts() +} + +async fn seed_legacy_state(host: &ServiceSqliteHost) -> Vec<Option<String>> { + let public = focused("legacy-public-author").publisher().public_key(); + let npub = NostrIdentityReference::derive(public) + .expect("public reference") + .npub() + .as_str() + .to_owned(); + host.transaction(|transaction| Box::pin(async move { + sqlx::query("INSERT INTO account_identities (public_key, npub, label, created_at_unix_s) VALUES (?, ?, 'retained-account', 11)").bind(public.as_bytes().as_slice()).bind(npub).execute(&mut *transaction).await?; + sqlx::query("INSERT INTO local_signer_bindings (account_public_key, binding_public_key, binding_kind, availability) VALUES (?, ?, 'local_secret', 'credential_missing')").bind(public.as_bytes().as_slice()).bind(public.as_bytes().as_slice()).execute(&mut *transaction).await?; + sqlx::query("INSERT INTO profile_cache (subject_public_key, event_id, event_created_at_unix_s, name, refreshed_at_unix_s, refresh_status) VALUES (?, ?, 12, 'retained-profile', 13, 'success')").bind(public.as_bytes().as_slice()).bind([73_u8; 32].as_slice()).execute(&mut *transaction).await?; + sqlx::query("INSERT INTO account_preferences (owner_public_key, preference_key, preference_value) VALUES (?, 'namespace_probe', 'retained-preference')").bind(public.as_bytes().as_slice()).execute(&mut *transaction).await?; + sqlx::query("INSERT INTO durable_operations (request_id, operation_kind, account_public_key, binding_public_key, phase, updated_at_unix_s) VALUES ('01890f3e-7b1c-7000-8000-000000000011', 'import', ?, ?, 'intent_recorded', 14)").bind(public.as_bytes().as_slice()).bind(public.as_bytes().as_slice()).execute(&mut *transaction).await?; + sqlx::query("INSERT INTO installation_identity (singleton, installation_id) VALUES (1, ?)").bind([72_u8; 16].as_slice()).execute(&mut *transaction).await?; + sqlx::query("UPDATE runtime_state SET selected_public_key = ?, active_account_public_key = ?, active_binding_public_key = ?, session_generation = 17 WHERE singleton = 1").bind(public.as_bytes().as_slice()).bind(public.as_bytes().as_slice()).bind(public.as_bytes().as_slice()).execute(&mut *transaction).await?; + Ok::<_, sqlx::Error>(()) + })).await.expect("legacy product state"); + snapshot(host).await +} + +async fn snapshot(host: &ServiceSqliteHost) -> Vec<Option<String>> { + let rows = rows(host, SNAPSHOT).await; + (0..7).map(|index| rows[0].get(index)).collect() +} + +async fn bounded_fault_row( + connection: &mut SqliteConnection, + version: AvailabilityEventVersion, +) -> SqliteRow { + sqlx::query(SELECT_AVAILABILITY_VERSION_SQL) + .bind(version.event_id().as_bytes().as_slice()) + .fetch_one(connection) + .await + .expect("EXACT production bounded SELECT on a closed-host fault fixture") +} + +async fn fill_count(database: &Database, count: usize) -> (AvailabilityVersionView, i64) { + assert!(count <= 4096); + let mut first = None; + let mut bytes = 0; + for index in 0..count { + let view = focused(&format!("capacity-{index}")); + bytes += expected_charge(&view, "focused", None); + if index == 0 { + first = Some(view.clone()); + } + database + .retain_availability_version(view) + .await + .expect("actual verified row admission"); + } + (first.expect("nonzero streamed capacity setup"), bytes) +} + +#[test] +fn historical_v1_v2_catalogs_preserve_exact_sql_and_digests() { + let migrations = harvestcircle_migration_catalog().expect("current migrations"); + let schema = harvestcircle_schema_catalog().expect("current schemas"); + assert_eq!(migrations.current_version(), 3); + assert_eq!(migrations.descriptors().len(), 2); + let previous = &migrations.descriptors()[0]; + assert_eq!(previous.target_version(), 2); + assert_eq!(previous.name().as_str(), "bound_durable_operation_receipts"); + assert_eq!( + *previous.checksum().as_bytes(), + [ + 107, 95, 237, 250, 255, 0, 44, 110, 142, 194, 92, 163, 84, 27, 96, 31, 210, 37, 151, + 186, 210, 83, 137, 114, 251, 20, 30, 31, 11, 136, 207, 168 + ] + ); + assert_eq!( + MigrationChecksum::for_sql(MIGRATE_DURABLE_OPERATIONS_V2_SQL), + previous.checksum() + ); + assert_eq!(schema.versions().len(), 3); + assert_eq!(schema.versions()[0].version(), 1); + assert_eq!( + *schema.versions()[0].digest().as_bytes(), + [ + 61, 122, 56, 39, 178, 126, 179, 157, 145, 167, 19, 2, 172, 134, 213, 107, 151, 196, + 212, 57, 17, 112, 163, 67, 240, 140, 61, 62, 5, 101, 14, 71 + ] + ); + assert_eq!(schema.versions()[1].version(), 2); + assert_eq!( + *schema.versions()[1].digest().as_bytes(), + [ + 78, 151, 73, 238, 2, 15, 71, 52, 11, 111, 100, 95, 135, 11, 170, 138, 84, 105, 106, + 177, 27, 7, 134, 156, 53, 68, 22, 220, 116, 199, 147, 5 + ] + ); + for (version, sql, service_count) in [ + (1, CREATE_DURABLE_OPERATIONS_SQL, 9), + (2, CREATE_DURABLE_OPERATIONS_V2_SQL, 11), + ] { + let objects = schema_objects(sql).expect("historical objects"); + assert_eq!(objects.len(), service_count); + for object in &objects { + assert_eq!( + object.digest(), + SchemaObject::computed_digest( + object.kind(), + object.name(), + object.table_name(), + match object.name() { + "account_identities" => CREATE_ACCOUNT_IDENTITIES_SQL, + "local_signer_bindings" => CREATE_LOCAL_SIGNER_BINDINGS_SQL, + "runtime_state" => CREATE_RUNTIME_STATE_SQL, + "profile_cache" => CREATE_PROFILE_CACHE_SQL, + "account_preferences" => CREATE_ACCOUNT_PREFERENCES_SQL, + "durable_operations" => sql, + "installation_identity" => CREATE_INSTALLATION_IDENTITY_SQL, + "installation_identity_no_update" => + CREATE_INSTALLATION_IDENTITY_NO_UPDATE_SQL, + "installation_identity_no_delete" => + CREATE_INSTALLATION_IDENTITY_NO_DELETE_SQL, + "durable_operations_receipt_insert_guard" => + CREATE_DURABLE_OPERATIONS_RECEIPT_INSERT_GUARD_SQL, + "durable_operations_receipt_update_guard" => + CREATE_DURABLE_OPERATIONS_RECEIPT_UPDATE_GUARD_SQL, + _ => panic!("unallocated historical object"), + } + ) + .expect("independent historical object digest") + ); + } + assert_eq!( + SchemaVersionCatalog::computed_digest(version, objects).unwrap(), + schema.versions()[version as usize - 1].digest() + ); + } + assert_eq!(harvestcircle_initial_schema_sql().len(), 9); +} + +#[tokio::test] +async fn fresh_governed_database_installs_allocated_evidence_schema() { + let fixture = Fixture::new(); + let database = fixture.open().await; + assert_eq!(database.metadata().state_schema_version().get(), 3); + assert_eq!(database.metadata().service().as_str(), "harvestcircle"); + assert_eq!(database.metadata().instance().as_str(), "desktop"); + assert_eq!(database.metadata().created_at_unix_ms(), 99); + assert_usage(&database, 0, 0).await; + let ledger = rows( + database.host(), + "SELECT version, name, checksum FROM schema_migrations ORDER BY version", + ) + .await; + assert_eq!(ledger.len(), 2); + assert_eq!(ledger[0].get::<i64, _>("version"), 2); + assert_eq!(ledger[1].get::<i64, _>("version"), 3); + assert_eq!( + ledger[1].get::<String, _>("name"), + "add_verified_listing_evidence" + ); + assert_eq!( + ledger[1].get::<Vec<u8>, _>("checksum"), + MigrationChecksum::for_sql(MIGRATE_AVAILABILITY_EVIDENCE_V3_SQL).as_bytes() + ); + let schema = harvestcircle_schema_catalog().unwrap(); + let objects = schema_objects_v3().expect("real new catalog assembly"); + let actual = rows( + database.host(), + "SELECT name, tbl_name, sql FROM sqlite_schema WHERE sql IS NOT NULL", + ) + .await; + for object in &objects { + let row = actual + .iter() + .find(|row| row.get::<String, _>("name") == object.name()) + .expect("actual schema object"); + let sql: String = row.get("sql"); + assert_eq!(row.get::<String, _>("tbl_name"), object.table_name()); + assert_eq!( + SchemaObject::computed_digest(object.kind(), object.name(), object.table_name(), &sql) + .unwrap(), + object.digest() + ); + } + assert_eq!( + SchemaVersionCatalog::computed_digest(3, objects).unwrap(), + schema.versions()[2].digest() + ); + assert_ne!(schema.versions()[2].digest(), schema.versions()[1].digest()); + let constrained = focused("sql-constraints"); + database + .retain_availability_version(constrained) + .await + .unwrap(); + database.close().await.unwrap(); + let mut connection = fixture.closed_connection().await; + assert!(sqlx::query("INSERT INTO public_payload_usage (singleton, version_count, payload_bytes) VALUES (2, 0, 0)").execute(&mut connection).await.is_err()); + for sql in [ + "UPDATE public_payload_usage SET version_count = -1", + "UPDATE public_payload_usage SET version_count = 4097", + "UPDATE public_payload_usage SET payload_bytes = -1", + "UPDATE public_payload_usage SET payload_bytes = 134217729", + "UPDATE availability_versions SET event_id = zeroblob(31)", + "UPDATE availability_versions SET author = zeroblob(31)", + "UPDATE availability_versions SET kind = 30403", + "UPDATE availability_versions SET signed_at = zeroblob(7)", + "UPDATE availability_versions SET published_at = zeroblob(9)", + "UPDATE availability_versions SET observed_at_unix_s = -1", + "UPDATE availability_versions SET admission_label = 'unknown'", + "UPDATE availability_versions SET rejection_code = 'unexpected'", + "UPDATE availability_versions SET original_json = zeroblob(10)", + ] { + assert!( + sqlx::query(sql).execute(&mut connection).await.is_err(), + "global bounds must be SQL constraints" + ); + } + for (sql, value) in [ + ( + "UPDATE availability_versions SET original_json = ?", + "x".repeat(262_145), + ), + ( + "UPDATE availability_versions SET raw_d = ?", + "é".repeat(2049), + ), + ( + "UPDATE availability_versions SET source = ?", + "x".repeat(2049), + ), + ] { + assert!( + sqlx::query(sql) + .bind(value) + .execute(&mut connection) + .await + .is_err(), + "UTF-8 limits use BLOB byte length" + ); + } + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM pragma_foreign_key_list('availability_versions')" + ) + .fetch_one(&mut connection) + .await + .unwrap(), + 0 + ); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn governed_v2_upgrade_preserves_original_state_and_source_identity() { + let fixture = Fixture::new(); + let (host, before) = legacy_host(&fixture).await; + let state = seed_legacy_state(&host).await; + assert!(state.iter().all(Option::is_some)); + host.close().await.unwrap(); + let database = fixture.open().await; + assert_eq!(database.metadata().state_schema_version().get(), 3); + assert_eq!( + database.metadata().source_generation(), + before.source_generation() + ); + assert_eq!(database.metadata().created_at_unix_ms(), 17); + assert_eq!( + database.metadata().application_id(), + before.application_id() + ); + assert_eq!(snapshot(database.host()).await, state); + assert_usage(&database, 0, 0).await; + database.close().await.unwrap(); +} + +#[tokio::test] +async fn invalid_v1_receipt_migration_rolls_back_without_partial_schema() { + let mut connection = SqliteConnection::connect("sqlite::memory:").await.unwrap(); + for sql in harvestcircle_initial_schema_sql() { + sqlx::query(*sql).execute(&mut connection).await.unwrap(); + } + sqlx::query("PRAGMA ignore_check_constraints = ON") + .execute(&mut connection) + .await + .unwrap(); + sqlx::query("INSERT INTO durable_operations (request_id, operation_kind, account_public_key, binding_public_key, phase, updated_at_unix_s) VALUES ('01890f3e-7b1c-7000-8000-000000000013', 'create', ?, ?, 'finalized', 12)") + .bind([9_u8;32].as_slice()).bind([9_u8;32].as_slice()).execute(&mut connection).await.unwrap(); + sqlx::query("PRAGMA ignore_check_constraints = OFF") + .execute(&mut connection) + .await + .unwrap(); + let mut transaction = connection.begin().await.unwrap(); + assert!( + sqlx::raw_sql(MIGRATE_DURABLE_OPERATIONS_V2_SQL) + .execute(&mut *transaction) + .await + .is_err() + ); + transaction.rollback().await.unwrap(); + assert_eq!(sqlx::query_scalar::<_, i64>("SELECT count(*) FROM pragma_table_info('durable_operations') WHERE name = 'completed_at_unix_s'").fetch_one(&mut connection).await.unwrap(), 0); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM sqlite_schema WHERE name LIKE 'durable_operations_receipt_%'" + ) + .fetch_one(&mut connection) + .await + .unwrap(), + 0 + ); + let row = sqlx::query("SELECT phase, updated_at_unix_s FROM durable_operations") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(row.get::<String, _>("phase"), "finalized"); + assert_eq!(row.get::<i64, _>("updated_at_unix_s"), 12); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn invalid_v2_database_is_refused_before_migration_without_replacement() { + let fixture = Fixture::new(); + let (host, metadata) = legacy_host(&fixture).await; + seed_legacy_state(&host).await; + host.close().await.unwrap(); + let mut connection = fixture.closed_connection().await; + sqlx::query("PRAGMA ignore_check_constraints = ON") + .execute(&mut connection) + .await + .unwrap(); + sqlx::query("UPDATE account_preferences SET preference_value = ''") + .execute(&mut connection) + .await + .unwrap(); + sqlx::query("PRAGMA ignore_check_constraints = OFF") + .execute(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + let error = match Database::open(&fixture.context, 900, 901, &fixture.build).await { + Ok(_) => panic!("invalid v2 must be refused"), + Err(error) => error, + }; + assert_eq!(error.code(), SafeErrorCode::StorageCorrupt); + // This is observed preflight refusal, not a claimed migration rollback. + let mut connection = fixture.closed_connection().await; + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT state_schema_version FROM radroots_service_metadata") + .fetch_one(&mut connection) + .await + .unwrap(), + 2 + ); + assert_eq!( + sqlx::query_scalar::<_, Vec<u8>>("SELECT source_generation FROM radroots_service_metadata") + .fetch_one(&mut connection) + .await + .unwrap(), + metadata.source_generation().as_bytes() + ); + assert_eq!( + sqlx::query_scalar::<_, String>("SELECT preference_value FROM account_preferences") + .fetch_one(&mut connection) + .await + .unwrap(), + "" + ); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT count(*) FROM account_identities") + .fetch_one(&mut connection) + .await + .unwrap(), + 1 + ); + assert_eq!(sqlx::query_scalar::<_, i64>("SELECT count(*) FROM sqlite_schema WHERE name IN ('availability_versions', 'public_payload_usage')").fetch_one(&mut connection).await.unwrap(), 0); + connection.close().await.unwrap(); +} + +async fn rollback_evidence_creation(fixture: &Fixture) -> Vec<Option<String>> { + let (host, metadata) = legacy_host(fixture).await; + let before = seed_legacy_state(&host).await; + let injected = SafeError::new( + SafeErrorCode::StorageUnavailable, + SafeMessage::new("The test operation was refused."), + ); + let error = host.transaction(|transaction| Box::pin(async move { + sqlx::raw_sql(MIGRATE_AVAILABILITY_EVIDENCE_V3_SQL).execute(&mut *transaction).await.map_err(|_| injected)?; + let objects: i64 = sqlx::query_scalar("SELECT count(*) FROM sqlite_schema WHERE name IN ('availability_versions', 'public_payload_usage')").fetch_one(&mut *transaction).await.map_err(|_| injected)?; + assert_eq!(objects, 2, "v3 DDL was actually executed before the fault"); + let meter: (i64, i64) = sqlx::query_as("SELECT version_count, payload_bytes FROM public_payload_usage").fetch_one(&mut *transaction).await.map_err(|_| injected)?; + assert_eq!(meter, (0, 0), "allocated singleton was visible within transaction"); + Err::<(), _>(injected) + })).await.expect_err("deliberate operation failure after real DDL"); + assert_eq!( + error.kind(), + ServiceSqliteTransactionErrorKind::OperationRolledBack + ); + assert_eq!(error.operation_error(), Some(&injected)); + assert_eq!(snapshot(&host).await, before); + let absent = rows(&host, "SELECT count(*) AS objects FROM sqlite_schema WHERE name IN ('availability_versions', 'public_payload_usage')").await; + assert_eq!(absent[0].get::<i64, _>("objects"), 0); + let retained = rows( + &host, + "SELECT state_schema_version, source_generation FROM radroots_service_metadata", + ) + .await; + assert_eq!(retained[0].get::<i64, _>("state_schema_version"), 2); + assert_eq!( + retained[0].get::<Vec<u8>, _>("source_generation"), + metadata.source_generation().as_bytes() + ); + host.close().await.unwrap(); + before +} + +#[tokio::test] +async fn failed_governed_evidence_creation_rolls_back_objects_and_counters() { + let fixture = Fixture::new(); + rollback_evidence_creation(&fixture).await; + let mut connection = fixture.closed_connection().await; + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT count(*) FROM schema_migrations WHERE version = 3") + .fetch_one(&mut connection) + .await + .unwrap(), + 0 + ); + connection.close().await.unwrap(); +} + +#[tokio::test] +async fn real_migration_succeeds_after_rolled_back_creation() { + let fixture = Fixture::new(); + let before = rollback_evidence_creation(&fixture).await; + let database = fixture.open().await; + assert_eq!(database.metadata().state_schema_version().get(), 3); + assert_eq!(snapshot(database.host()).await, before); + assert_usage(&database, 0, 0).await; + let public = focused("after-real-retry"); + database + .retain_availability_version(public.clone()) + .await + .unwrap(); + assert_round_trip( + &database + .load_availability_version(public.version()) + .await + .unwrap() + .unwrap(), + &public, + ); + database.close().await.unwrap(); +} + +#[tokio::test] +async fn future_schema_and_integrity_failure_preserve_original_state() { + for future in [false, true] { + let fixture = Fixture::new(); + let database = fixture.open().await; + let public = focused("preserved-on-refusal"); + database + .retain_availability_version(public.clone()) + .await + .unwrap(); + let source = database.metadata().source_generation(); + database.close().await.unwrap(); + let mut connection = fixture.closed_connection().await; + if future { + sqlx::query("UPDATE radroots_service_metadata SET state_schema_version = 4") + .execute(&mut connection) + .await + .unwrap(); + } else { + sqlx::query("CREATE TABLE unexpected_evidence (value TEXT)") + .execute(&mut connection) + .await + .unwrap(); + } + connection.close().await.unwrap(); + let error = match Database::open(&fixture.context, 700, 701, &fixture.build).await { + Ok(_) => panic!("future or foreign schema must be refused"), + Err(error) => error, + }; + assert_eq!(error.code(), SafeErrorCode::StorageCorrupt); + let mut connection = fixture.closed_connection().await; + assert_eq!( + sqlx::query_scalar::<_, Vec<u8>>( + "SELECT source_generation FROM radroots_service_metadata" + ) + .fetch_one(&mut connection) + .await + .unwrap(), + source.as_bytes() + ); + assert_eq!( + sqlx::query_scalar::<_, String>("SELECT original_json FROM availability_versions") + .fetch_one(&mut connection) + .await + .unwrap(), + public.original_json() + ); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT version_count FROM public_payload_usage") + .fetch_one(&mut connection) + .await + .unwrap(), + 1 + ); + connection.close().await.unwrap(); + } +} + +#[tokio::test] +async fn remote_version_retention_creates_no_account_or_signer() { + let public = focused("remote-public-only"); + let fixture = Fixture::new(); + let database = fixture.open().await; + assert_eq!( + database + .load_availability_version(public.version()) + .await + .unwrap(), + None + ); + database + .retain_availability_version(public.clone()) + .await + .unwrap(); + assert_round_trip( + &database + .load_availability_version(public.version()) + .await + .unwrap() + .unwrap(), + &public, + ); + let rows = rows(database.host(), "SELECT (SELECT count(*) FROM account_identities) AS accounts, (SELECT count(*) FROM local_signer_bindings) AS signers, (SELECT count(*) FROM profile_cache) AS profiles, (SELECT count(*) FROM installation_identity) AS installations, (SELECT count(*) FROM durable_operations) AS operations").await; + for column in [ + "accounts", + "signers", + "profiles", + "installations", + "operations", + ] { + assert_eq!(rows[0].get::<i64, _>(column), 0); + } + assert_usage(&database, 1, expected_charge(&public, "focused", None)).await; + database.close().await.unwrap(); + assert_eq!( + database + .load_availability_version(public.version()) + .await + .unwrap_err() + .code(), + SafeErrorCode::StorageUnavailable + ); + assert_eq!( + database + .retain_availability_version(public) + .await + .unwrap_err() + .code(), + SafeErrorCode::StorageUnavailable + ); +} + +#[tokio::test] +async fn focused_version_round_trips_exact_wire_admission_and_provenance() { + let public = focused("exact-wire"); + let original = format!( + "\n {},\"unauthenticated_extra\":\"public observed bytes\"}} \n", + public.original_json().strip_suffix('}').unwrap() + ); + let long_source = format!( + "wss://evidence.example.invalid/{}", + "x".repeat(2048 - "wss://evidence.example.invalid/".len()) + ); + assert_eq!(long_source.len(), 2048); + let expected = view_from_wire(&original, &long_source, i64::MAX); + assert!(expected.focused().is_some()); + let fixture = Fixture::new(); + let database = fixture.open().await; + database + .retain_availability_version(expected.clone()) + .await + .unwrap(); + let row = stored_row(&database, expected.version()).await; + assert_eq!(row.get::<String, _>("original_json"), original); + assert_eq!(row.get::<String, _>("raw_d"), "exact-wire"); + assert_eq!(row.get::<String, _>("source"), long_source); + assert_eq!(row.get::<i64, _>("observed_at_unix_s"), i64::MAX); + assert_eq!(row.get::<String, _>("admission_label"), "focused"); + assert_eq!(row.get::<Option<String>, _>("rejection_code"), None); + assert_eq!(row.get::<i64, _>("kind"), 30402); + assert_eq!( + row.get::<Vec<u8>, _>("event_id"), + expected.version().event_id().as_bytes() + ); + assert_eq!( + row.get::<Vec<u8>, _>("author"), + expected.publisher().public_key().as_bytes() + ); + assert_eq!( + row.get::<Vec<u8>, _>("signed_at"), + 1_800_000_100_u64.to_be_bytes() + ); + assert_eq!( + row.get::<Option<Vec<u8>>, _>("published_at"), + Some(1_800_000_000_u64.to_be_bytes().to_vec()) + ); + assert_usage(&database, 1, expected_charge(&expected, "focused", None)).await; + database.close().await.unwrap(); + let reopened = fixture.open().await; + assert_round_trip( + &reopened + .load_availability_version(expected.version()) + .await + .unwrap() + .unwrap(), + &expected, + ); + reopened.close().await.unwrap(); +} + +#[tokio::test] +async fn unsupported_empty_and_overlong_raw_coordinates_remain_evidence() { + let identifiers = [ + String::new(), + "d".repeat(4026), + "d".repeat(4096), + "é".repeat(2048), + " opaque:\0:é ".into(), + ]; + let fixture = Fixture::new(); + let database = fixture.open().await; + let mut bytes = 0; + for identifier in &identifiers { + let public = public_view(identifier, 100, 90, Recipe::Focused, None); + assert_eq!(raw_d(&public), identifier); + assert!(public.focused().is_none()); + let Some(AvailabilityUnsupportedReason::ProjectionRejected(reason)) = + public.unsupported_reason() + else { + panic!("shared strict projection rejects broader raw evidence") + }; + assert!(!reason.is_empty() && reason.len() <= 64 && reason.is_ascii()); + if identifier.is_empty() || identifier.len() > 4025 { + assert!(public.listing_coordinate().is_none()); + } + bytes += expected_charge(&public, "projection_rejected", Some(reason)); + database + .retain_availability_version(public.clone()) + .await + .unwrap(); + let row = stored_row(&database, public.version()).await; + assert_eq!(row.get::<String, _>("raw_d"), *identifier); + assert_eq!( + row.get::<String, _>("admission_label"), + "projection_rejected" + ); + assert_eq!( + row.get::<Option<String>, _>("rejection_code").as_deref(), + Some(*reason) + ); + assert_eq!(row.get::<Option<Vec<u8>>, _>("published_at"), None); + assert_round_trip( + &database + .load_availability_version(public.version()) + .await + .unwrap() + .unwrap(), + &public, + ); + } + assert_usage(&database, 5, bytes).await; + database.close().await.unwrap(); +} + +#[tokio::test] +async fn signed_and_publication_timestamps_preserve_full_u64_range() { + let values = [0, 1, i64::MAX as u64, i64::MAX as u64 + 1, u64::MAX]; + let fixture = Fixture::new(); + let database = fixture.open().await; + for (index, value) in values.into_iter().enumerate() { + let public = public_view( + &format!("unsigned-{index}"), + value, + value.max(1), + if value == 0 { + Recipe::Generic + } else { + Recipe::Focused + }, + None, + ); + assert_eq!(public.created_at().as_u64(), value); + if value != 0 { + assert_eq!(public.focused().unwrap().published_at().as_u64(), value); + } + database + .retain_availability_version(public.clone()) + .await + .unwrap(); + let row = stored_row(&database, public.version()).await; + assert_eq!(row.get::<Vec<u8>, _>("signed_at"), value.to_be_bytes()); + assert_eq!( + row.get::<Option<Vec<u8>>, _>("published_at"), + (value != 0).then(|| value.to_be_bytes().to_vec()) + ); + assert_round_trip( + &database + .load_availability_version(public.version()) + .await + .unwrap() + .unwrap(), + &public, + ); + } + let rows = rows( + database.host(), + "SELECT signed_at FROM availability_versions ORDER BY signed_at", + ) + .await; + assert_eq!( + rows.iter() + .map(|row| u64::from_be_bytes(row.get::<Vec<u8>, _>("signed_at").try_into().unwrap())) + .collect::<Vec<_>>(), + values + ); + database.close().await.unwrap(); +} + +#[tokio::test] +async fn exact_id_duplicates_preserve_first_wire_and_provenance() { + let original = focused("same-signed-id"); + let first_wire = format!(" {}\n", original.original_json()); + let first = view_from_wire(&first_wire, SOURCE, 10); + let second_wire = format!( + "{},\"extra\":\"different unauthenticated observation\"}}", + original.original_json().strip_suffix('}').unwrap() + ); + let later = view_from_wire(&second_wire, "wss://later.example.invalid/public", 20); + assert_eq!(first.version(), later.version()); + assert_ne!(first.original_json(), later.original_json()); + let fixture = Fixture::new(); + let database = fixture.open().await; + database + .retain_availability_version(first.clone()) + .await + .unwrap(); + database.retain_availability_version(later).await.unwrap(); + database + .retain_availability_version(original) + .await + .unwrap(); + assert_round_trip( + &database + .load_availability_version(first.version()) + .await + .unwrap() + .unwrap(), + &first, + ); + assert_usage(&database, 1, expected_charge(&first, "focused", None)).await; + database.close().await.unwrap(); + let reopened = fixture.open().await; + assert_round_trip( + &reopened + .load_availability_version(first.version()) + .await + .unwrap() + .unwrap(), + &first, + ); + reopened.close().await.unwrap(); +} + +#[tokio::test] +async fn distinct_versions_of_one_coordinate_preserve_history() { + let keys = Keys::generate(); + let first_wire = public_wire(&keys, "history", 100, 90, Recipe::Focused, None); + let later_wire = public_wire(&keys, "history", 101, 90, Recipe::Generic, None); + drop(keys); + let first = view_from_wire(&first_wire, SOURCE, 1); + let later = view_from_wire(&later_wire, SOURCE, 2); + assert_eq!(first.raw_coordinate(), later.raw_coordinate()); + assert_ne!(first.version(), later.version()); + assert!(first.focused().is_some()); + assert_eq!( + later.unsupported_reason(), + Some(&AvailabilityUnsupportedReason::Excluded( + ClassifiedListingPartition::GenericNip99 + )) + ); + let fixture = Fixture::new(); + let database = fixture.open().await; + database + .retain_availability_version(later.clone()) + .await + .unwrap(); + database + .retain_availability_version(first.clone()) + .await + .unwrap(); + assert_round_trip( + &database + .load_availability_version(first.version()) + .await + .unwrap() + .unwrap(), + &first, + ); + assert_round_trip( + &database + .load_availability_version(later.version()) + .await + .unwrap() + .unwrap(), + &later, + ); + assert_usage( + &database, + 2, + expected_charge(&first, "focused", None) + + expected_charge(&later, "excluded_generic", None), + ) + .await; + let heads = rows( + database.host(), + "SELECT count(*) AS heads FROM sqlite_schema WHERE name = 'availability_heads'", + ) + .await; + assert_eq!( + heads[0].get::<i64, _>("heads"), + 0, + "head selection remains a later owner" + ); + database.close().await.unwrap(); +} + +#[tokio::test] +async fn attached_display_profile_is_not_retained_as_verified_evidence() { + let public = focused("unverified-display"); + let metadata = ProfileMetadata::new( + Some("Unverified caller display".into()), + None, + None, + Some("Not signed profile evidence.".into()), + None, + ) + .unwrap(); + let profile = Kind0ProfileCandidate::new( + EventId::from_bytes([74; 32]), + public.publisher().public_key(), + UnixTimestamp::from_seconds(1).unwrap(), + metadata, + ); + let decorated = public.with_profile(Some(&profile)).unwrap(); + assert!(decorated.profile_metadata().is_some()); + let fixture = Fixture::new(); + let database = fixture.open().await; + database + .retain_availability_version(decorated) + .await + .unwrap(); + assert_round_trip( + &database + .load_availability_version(public.version()) + .await + .unwrap() + .unwrap(), + &public, + ); + let rows = rows(database.host(), "SELECT (SELECT count(*) FROM profile_cache) AS profiles, (SELECT count(*) FROM account_identities) AS accounts").await; + assert_eq!(rows[0].get::<i64, _>("profiles"), 0); + assert_eq!(rows[0].get::<i64, _>("accounts"), 0); + assert_usage(&database, 1, expected_charge(&public, "focused", None)).await; + database.close().await.unwrap(); +} + +#[tokio::test] +async fn corrupt_wire_or_signed_metadata_fails_closed_on_load() { + let public = focused("corrupt-evidence"); + let mut malformed = public.original_json().to_owned(); + let signature_start = + malformed.find("\"sig\":\"").expect("signature field") + "\"sig\":\"".len(); + let replacement = if &malformed[signature_start..signature_start + 1] == "0" { + "1" + } else { + "0" + }; + malformed.replace_range(signature_start..signature_start + 1, replacement); + let changes: [(&str, Option<String>); 9] = [ + ( + "UPDATE availability_versions SET original_json = ?", + Some(malformed), + ), + ( + "UPDATE availability_versions SET author = zeroblob(32)", + None, + ), + ( + "UPDATE availability_versions SET signed_at = x'0000000000000001'", + None, + ), + ( + "UPDATE availability_versions SET published_at = x'0000000000000001'", + None, + ), + ( + "UPDATE availability_versions SET raw_d = 'other-coordinate'", + None, + ), + ( + "UPDATE availability_versions SET source = 'invalid-source'", + None, + ), + ( + "UPDATE availability_versions SET original_json = '{}'", + None, + ), + ( + "UPDATE availability_versions SET admission_label = 'excluded_generic', published_at = NULL", + None, + ), + ( + "UPDATE availability_versions SET event_id = zeroblob(32)", + None, + ), + ]; + for (sql, text) in changes { + let fixture = Fixture::new(); + let database = fixture.open().await; + database + .retain_availability_version(public.clone()) + .await + .unwrap(); + database.close().await.unwrap(); + let mut connection = fixture.closed_connection().await; + sqlx::query("PRAGMA ignore_check_constraints = ON") + .execute(&mut connection) + .await + .unwrap(); + let mut query = sqlx::query(sql); + if let Some(text) = text { + query = query.bind(text); + } + query.execute(&mut connection).await.unwrap(); + sqlx::query("PRAGMA ignore_check_constraints = OFF") + .execute(&mut connection) + .await + .unwrap(); + let version = if sql.contains("SET event_id") { + AvailabilityEventVersion::from_hex(&"00".repeat(32)).unwrap() + } else { + public.version() + }; + let row = bounded_fault_row(&mut connection, version).await; + assert_eq!( + decode_availability_row(row).unwrap_err().code(), + SafeErrorCode::StorageCorrupt, + "actual product decoder must reject signed/metadata mismatch" + ); + connection.close().await.unwrap(); + match Database::open(&fixture.context, 200, 201, &fixture.build).await { + Ok(database) => { + assert_eq!( + database + .load_availability_version(version) + .await + .unwrap_err() + .code(), + SafeErrorCode::StorageCorrupt + ); + if version == public.version() { + assert_eq!( + database + .retain_availability_version(public.clone()) + .await + .unwrap_err() + .code(), + SafeErrorCode::StorageCorrupt, + "duplicate does not bless corrupt first evidence" + ); + } + database.close().await.unwrap(); + } + Err(error) => assert_eq!( + error.code(), + SafeErrorCode::StorageCorrupt, + "shared preflight refusal is distinct from decoder execution above" + ), + } + } +} + +#[tokio::test] +async fn oversize_or_malformed_fields_are_bounded_before_hydration() { + let public = focused("bounded-corruption"); + let fields = [ + ( + "original_json", + 262_144_usize, + "UPDATE availability_versions SET original_json = ?", + ), + ("raw_d", 4096, "UPDATE availability_versions SET raw_d = ?"), + ( + "source", + 2048, + "UPDATE availability_versions SET source = ?", + ), + ( + "admission_label", + 64, + "UPDATE availability_versions SET admission_label = ?", + ), + ( + "rejection_code", + 64, + "UPDATE availability_versions SET rejection_code = ?", + ), + ("author", 32, "UPDATE availability_versions SET author = ?"), + ( + "signed_at", + 8, + "UPDATE availability_versions SET signed_at = ?", + ), + ( + "published_at", + 8, + "UPDATE availability_versions SET published_at = ?", + ), + ]; + for (field, bound, sql) in fields { + let fixture = Fixture::new(); + let database = fixture.open().await; + database + .retain_availability_version(public.clone()) + .await + .unwrap(); + database.close().await.unwrap(); + let mut connection = fixture.closed_connection().await; + sqlx::query("PRAGMA ignore_check_constraints = ON") + .execute(&mut connection) + .await + .unwrap(); + // The identifier is exclusively this fixed test-owned inventory. + let query = sqlx::query(sql); + if matches!(field, "author" | "signed_at" | "published_at") { + query + .bind(vec![1_u8; bound + 2]) + .execute(&mut connection) + .await + .unwrap(); + } else { + query + .bind("x".repeat(bound + 2)) + .execute(&mut connection) + .await + .unwrap(); + } + sqlx::query("PRAGMA ignore_check_constraints = OFF") + .execute(&mut connection) + .await + .unwrap(); + let row = bounded_fault_row(&mut connection, public.version()).await; + let bytes: Vec<u8> = row.get(field); + assert_eq!( + bytes.len(), + bound + 1, + "real SQL projects only the bounded prefix" + ); + assert_eq!( + row.get::<i64, _>(format!("{field}_bytes").as_str()), + i64::try_from(bound + 2).unwrap(), + "full original byte length remains available without hydration" + ); + assert_eq!( + decode_availability_row(row).unwrap_err().code(), + SafeErrorCode::StorageCorrupt + ); + connection.close().await.unwrap(); + let error = match Database::open(&fixture.context, 300, 301, &fixture.build).await { + Ok(database) => { + let error = database + .load_availability_version(public.version()) + .await + .unwrap_err(); + database.close().await.unwrap(); + error + } + Err(error) => error, + }; + assert_eq!(error.code(), SafeErrorCode::StorageCorrupt); + } + for sql in [ + "UPDATE availability_versions SET original_json = CAST(x'FF' AS TEXT)", + "UPDATE availability_versions SET source = CAST(x'FF' AS TEXT)", + "UPDATE availability_versions SET kind = 30403", + "UPDATE availability_versions SET observed_at_unix_s = -1", + "UPDATE availability_versions SET admission_label = 'unknown-private-message'", + "UPDATE availability_versions SET rejection_code = 'caller-owned-message'", + ] { + let fixture = Fixture::new(); + let database = fixture.open().await; + database + .retain_availability_version(public.clone()) + .await + .unwrap(); + database.close().await.unwrap(); + let mut connection = fixture.closed_connection().await; + sqlx::query("PRAGMA ignore_check_constraints = ON") + .execute(&mut connection) + .await + .unwrap(); + sqlx::query(sql).execute(&mut connection).await.unwrap(); + sqlx::query("PRAGMA ignore_check_constraints = OFF") + .execute(&mut connection) + .await + .unwrap(); + let error = + decode_availability_row(bounded_fault_row(&mut connection, public.version()).await) + .unwrap_err(); + assert_eq!(error.code(), SafeErrorCode::StorageCorrupt); + let diagnostic = format!("{error:?}"); + assert!( + !diagnostic.contains("caller-owned-message") + && !diagnostic.contains("unknown-private-message") + ); + connection.close().await.unwrap(); + } +} + +#[tokio::test] +async fn logical_payload_charge_matches_the_frozen_formula() { + let fixture = Fixture::new(); + let database = fixture.open().await; + let cases = [ + (Recipe::Focused, "focused", None), + (Recipe::Generic, "excluded_generic", None), + (Recipe::Operational, "excluded_operational", None), + ( + Recipe::Ambiguous, + "projection_rejected", + Some("food_profile_ambiguous"), + ), + ( + Recipe::Rejected, + "projection_rejected", + Some("price_invalid"), + ), + ]; + let mut total = 0; + for (index, (recipe, label, reason)) in cases.into_iter().enumerate() { + let public = public_view(&format!("charge-{index}"), 100, 90, recipe, None); + if let Some(reason) = reason { + assert_eq!( + public.unsupported_reason(), + Some(&AvailabilityUnsupportedReason::ProjectionRejected(reason)) + ); + } + let charge = expected_charge(&public, label, reason); + total += charge; + database + .retain_availability_version(public.clone()) + .await + .unwrap(); + let row = stored_row(&database, public.version()).await; + assert_eq!(row.get::<String, _>("admission_label"), label); + assert_eq!( + row.get::<Option<String>, _>("rejection_code").as_deref(), + reason + ); + assert_eq!(row.get::<i64, _>("payload_bytes"), charge); + assert_round_trip( + &database + .load_availability_version(public.version()) + .await + .unwrap() + .unwrap(), + &public, + ); + assert_usage(&database, index as i64 + 1, total).await; + } + assert_eq!( + TOTAL_PAYLOAD_BYTES - ORDINARY_PAYLOAD_BYTES, + RECOVERY_RESERVE_BYTES + ); + database.close().await.unwrap(); + // Plausible in-range values remain corrupt when they disagree with retained facts. + let public = focused("meter-corruption"); + for sql in [ + "UPDATE public_payload_usage SET version_count = 0", + "UPDATE public_payload_usage SET payload_bytes = payload_bytes + 1", + "DELETE FROM public_payload_usage", + "UPDATE availability_versions SET payload_bytes = payload_bytes + 1", + "UPDATE availability_versions SET payload_bytes = payload_bytes + 1; UPDATE public_payload_usage SET payload_bytes = payload_bytes + 1", + ] { + let fixture = Fixture::new(); + let database = fixture.open().await; + database + .retain_availability_version(public.clone()) + .await + .unwrap(); + database.close().await.unwrap(); + let mut connection = fixture.closed_connection().await; + sqlx::query("PRAGMA ignore_check_constraints = ON") + .execute(&mut connection) + .await + .unwrap(); + sqlx::raw_sql(sql).execute(&mut connection).await.unwrap(); + sqlx::query("PRAGMA ignore_check_constraints = OFF") + .execute(&mut connection) + .await + .unwrap(); + connection.close().await.unwrap(); + match Database::open(&fixture.context, 400, 401, &fixture.build).await { + Ok(database) => { + assert_eq!( + database + .load_availability_version(public.version()) + .await + .unwrap_err() + .code(), + SafeErrorCode::StorageCorrupt + ); + assert_eq!( + database + .retain_availability_version(public.clone()) + .await + .unwrap_err() + .code(), + SafeErrorCode::StorageCorrupt + ); + database.close().await.unwrap(); + } + Err(error) => assert_eq!(error.code(), SafeErrorCode::StorageCorrupt), + } + let mut connection = fixture.closed_connection().await; + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT count(*) FROM availability_versions") + .fetch_one(&mut connection) + .await + .unwrap(), + 1, + "no reset, deletion or repair" + ); + connection.close().await.unwrap(); + } +} + +#[tokio::test] +async fn exact_version_capacity_and_plus_one_preserve_existing_rows() { + let fixture = Fixture::new(); + let database = fixture.open().await; + let (first, bytes) = fill_count(&database, 4096).await; + assert_usage(&database, VERSION_CAPACITY, bytes).await; + let excess = focused("capacity-plus-one"); + assert_eq!( + database + .retain_availability_version(excess.clone()) + .await + .unwrap_err() + .code(), + SafeErrorCode::AvailabilityCapacity + ); + assert_eq!( + database + .load_availability_version(excess.version()) + .await + .unwrap(), + None + ); + assert_round_trip( + &database + .load_availability_version(first.version()) + .await + .unwrap() + .unwrap(), + &first, + ); + assert_usage(&database, 4096, bytes).await; + database.close().await.unwrap(); + let reopened = fixture.open().await; + assert_usage(&reopened, 4096, bytes).await; + assert_round_trip( + &reopened + .load_availability_version(first.version()) + .await + .unwrap() + .unwrap(), + &first, + ); + reopened.close().await.unwrap(); +} + +#[tokio::test] +async fn ordinary_byte_capacity_preserves_the_recovery_reserve() { + assert_eq!(DEFAULT_RAW_JSON_MAX_BYTES, 262_144); + let fixture = Fixture::new(); + let database = fixture.open().await; + let mut bytes = 0; + let mut count = 0; + let mut first = None; + while bytes < ORDINARY_PAYLOAD_BYTES { + assert!(count < 600, "bounded actual-capacity fixture stream"); + let identifier = format!("byte-capacity-{count}"); + let overhead = 92 + identifier.len() + SOURCE.len() + "focused".len(); + let remaining = usize::try_from(ORDINARY_PAYLOAD_BYTES - bytes).unwrap(); + let wire_bytes = 262_144.min(remaining - overhead); + assert!( + wire_bytes >= 1024, + "last admitted row remains a genuine bounded wire recipe" + ); + let public = public_view(&identifier, 100, 90, Recipe::Focused, Some(wire_bytes)); + assert_eq!(public.original_json().len(), wire_bytes); + if first.is_none() { + first = Some(public.clone()); + } + bytes += expected_charge(&public, "focused", None); + assert!(bytes <= ORDINARY_PAYLOAD_BYTES); + database.retain_availability_version(public).await.unwrap(); + count += 1; + } + assert_eq!( + bytes, 125_829_120, + "actual original-wire payloads fill the complete ordinary budget" + ); + assert!(count < VERSION_CAPACITY); + assert_usage(&database, count, bytes).await; + let refused = focused("ordinary-byte-plus-one"); + assert_eq!( + database + .retain_availability_version(refused.clone()) + .await + .unwrap_err() + .code(), + SafeErrorCode::AvailabilityCapacity + ); + assert_eq!( + database + .load_availability_version(refused.version()) + .await + .unwrap(), + None + ); + assert_usage(&database, count, bytes).await; + assert_eq!( + TOTAL_PAYLOAD_BYTES - bytes, + 8_388_608, + "ordinary callers cannot consume recovery reserve" + ); + let first = first.unwrap(); + let alternate = view_from_wire( + first.original_json().trim(), + "wss://another.example.invalid/public", + OBSERVED_AT + 1, + ); + database + .retain_availability_version(alternate) + .await + .unwrap(); + assert_usage(&database, count, bytes).await; + assert_round_trip( + &database + .load_availability_version(first.version()) + .await + .unwrap() + .unwrap(), + &first, + ); + database.close().await.unwrap(); + let reopened = fixture.open().await; + assert_usage(&reopened, count, bytes).await; + reopened.close().await.unwrap(); +} + +#[tokio::test] +async fn duplicates_at_capacity_do_not_grow_global_accounting() { + let fixture = Fixture::new(); + let database = fixture.open().await; + let (first, bytes) = fill_count(&database, 4096).await; + let alternate_wire = format!( + "{},\"later_extra\":\"first evidence remains authoritative\"}}", + first.original_json().strip_suffix('}').unwrap() + ); + let alternate = view_from_wire( + &alternate_wire, + "wss://another.example.invalid/public", + OBSERVED_AT + 20, + ); + assert_eq!(alternate.version(), first.version()); + database + .retain_availability_version(alternate) + .await + .expect("zero-growth duplicate at actual row cap"); + database + .retain_availability_version(first.clone()) + .await + .unwrap(); + assert_usage(&database, 4096, bytes).await; + assert_round_trip( + &database + .load_availability_version(first.version()) + .await + .unwrap() + .unwrap(), + &first, + ); + database.close().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_and_failed_transactions_preserve_exact_global_accounting() { + let fixture = Fixture::new(); + let database = fixture.open().await; + let (first, before_bytes) = fill_count(&database, 4095).await; + let left = focused("last-slot-left"); + let right = focused("last-slot-right"); + let left_charge = expected_charge(&left, "focused", None); + let right_charge = expected_charge(&right, "focused", None); + let injected = SafeError::new( + SafeErrorCode::StorageUnavailable, + SafeMessage::new("The test operation was refused."), + ); + let rollback_view = left.clone(); + let error = database + .host() + .transaction(|transaction| { + Box::pin(async move { + retain_availability_version_on(transaction, rollback_view).await?; + let meter: (i64, i64) = + sqlx::query_as("SELECT version_count, payload_bytes FROM public_payload_usage") + .fetch_one(&mut *transaction) + .await + .map_err(|_| injected)?; + assert_eq!( + meter, + (4096, before_bytes + left_charge), + "real insertion and reservation precede failure" + ); + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM availability_versions") + .fetch_one(&mut *transaction) + .await + .map_err(|_| injected)?; + assert_eq!(count, 4096); + Err::<(), _>(injected) + }) + }) + .await + .unwrap_err(); + assert_eq!( + error.kind(), + ServiceSqliteTransactionErrorKind::OperationRolledBack + ); + assert_eq!(error.operation_error(), Some(&injected)); + assert_usage(&database, 4095, before_bytes).await; + assert_eq!( + database + .load_availability_version(left.version()) + .await + .unwrap(), + None + ); + let duplicate_fixture = Fixture::new(); + let duplicate_database = duplicate_fixture.open().await; + assert_usage(&duplicate_database, 0, 0).await; + let duplicate_barrier = Arc::new(tokio::sync::Barrier::new(3)); + let (first_duplicate, second_duplicate, _) = tokio::join!( + async { + duplicate_barrier.wait().await; + duplicate_database + .retain_availability_version(first.clone()) + .await + }, + async { + duplicate_barrier.wait().await; + duplicate_database + .retain_availability_version(first.clone()) + .await + }, + duplicate_barrier.wait(), + ); + assert!( + first_duplicate.is_ok(), + "first concurrent new exact-ID retention succeeds" + ); + assert!( + second_duplicate.is_ok(), + "second concurrent exact-ID retention succeeds without growth" + ); + assert_usage( + &duplicate_database, + 1, + expected_charge(&first, "focused", None), + ) + .await; + assert_round_trip( + &duplicate_database + .load_availability_version(first.version()) + .await + .unwrap() + .unwrap(), + &first, + ); + duplicate_database.close().await.unwrap(); + let barrier = Arc::new(tokio::sync::Barrier::new(3)); + let (left_result, right_result, _) = tokio::join!( + async { + barrier.wait().await; + database.retain_availability_version(left.clone()).await + }, + async { + barrier.wait().await; + database.retain_availability_version(right.clone()).await + }, + barrier.wait(), + ); + assert_ne!( + left_result.is_ok(), + right_result.is_ok(), + "one real last-slot winner" + ); + let (winner, loser, added) = match (left_result, right_result) { + (Ok(()), Err(error)) => { + assert_eq!(error.code(), SafeErrorCode::AvailabilityCapacity); + (&left, &right, left_charge) + } + (Err(error), Ok(())) => { + assert_eq!(error.code(), SafeErrorCode::AvailabilityCapacity); + (&right, &left, right_charge) + } + _ => panic!("exactly one last-slot result"), + }; + assert_usage(&database, 4096, before_bytes + added).await; + assert_round_trip( + &database + .load_availability_version(winner.version()) + .await + .unwrap() + .unwrap(), + winner, + ); + assert_eq!( + database + .load_availability_version(loser.version()) + .await + .unwrap(), + None + ); + assert_round_trip( + &database + .load_availability_version(first.version()) + .await + .unwrap() + .unwrap(), + &first, + ); + database.close().await.unwrap(); + let reopened = fixture.open().await; + assert_usage(&reopened, 4096, before_bytes + added).await; + reopened.close().await.unwrap(); +} + +#[tokio::test] +async fn reopen_and_account_removal_preserve_public_evidence() { + let public = focused("independent-of-local-account"); + let key = public.publisher().public_key(); + let identity = NostrIdentity::new( + NostrIdentityReference::derive(key).unwrap(), + LocalKeyringBinding::new(key, SignerAvailability::CredentialMissing), + None, + IdentityCreatedAt::new(UnixTimestamp::from_seconds(1).unwrap()), + None, + ) + .unwrap(); + let fixture = Fixture::new(); + let database = fixture.open().await; + database.insert_identity(&identity).await.unwrap(); + database + .retain_availability_version(public.clone()) + .await + .unwrap(); + let source = database.metadata().source_generation(); + database.close().await.unwrap(); + let reopened = fixture.open().await; + assert_eq!(reopened.metadata().source_generation(), source); + reopened.remove_identity(key).await.unwrap(); + assert!(reopened.list_identities().await.unwrap().is_empty()); + assert_round_trip( + &reopened + .load_availability_version(public.version()) + .await + .unwrap() + .unwrap(), + &public, + ); + assert_usage(&reopened, 1, expected_charge(&public, "focused", None)).await; + reopened.close().await.unwrap(); + let again = fixture.open().await; + assert_round_trip( + &again + .load_availability_version(public.version()) + .await + .unwrap() + .unwrap(), + &public, + ); + assert_usage(&again, 1, expected_charge(&public, "focused", None)).await; + again.close().await.unwrap(); +} + +#[tokio::test] +async fn governed_backup_restore_preserves_evidence_and_accounting() { + let retained = focused("captured-public-version"); + let later = public_view("post-backup-public-version", 100, 90, Recipe::Generic, None); + let fixture = Fixture::new(); + let mut database = fixture.open().await; + database + .retain_availability_version(retained.clone()) + .await + .unwrap(); + let expected_identity = database.metadata().identity(); + let parent = fixture.directory.path().join("backup-output"); + fs::create_dir(&parent).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&parent, fs::Permissions::from_mode(0o700)).unwrap(); + } + let bundle = parent.join("public-evidence"); + let manifest = database + .capture_online_backup( + &bundle, + BackupCreatedAtUnixMs::new(1_800_000_000_000).unwrap(), + ) + .await + .unwrap(); + assert!(!manifest.protected_material_included()); + let verified = verify_harvestcircle_backup( + manifest.canonical_bytes(), + manifest.digest(), + &bundle, + &expected_identity, + NonZeroU64::new(manifest.members()[0].byte_length()).unwrap(), + ) + .unwrap(); + database + .retain_availability_version(later.clone()) + .await + .unwrap(); + assert_usage( + &database, + 2, + expected_charge(&retained, "focused", None) + + expected_charge(&later, "excluded_generic", None), + ) + .await; + database + .restore_verified_backup(&fixture.context, verified, 200, &fixture.build) + .await + .unwrap(); + assert_eq!(database.metadata().identity(), expected_identity); + assert_round_trip( + &database + .load_availability_version(retained.version()) + .await + .unwrap() + .unwrap(), + &retained, + ); + assert_eq!( + database + .load_availability_version(later.version()) + .await + .unwrap(), + None + ); + assert_usage(&database, 1, expected_charge(&retained, "focused", None)).await; + database.close().await.unwrap(); + let reopened = fixture.open().await; + assert_round_trip( + &reopened + .load_availability_version(retained.version()) + .await + .unwrap() + .unwrap(), + &retained, + ); + assert_usage(&reopened, 1, expected_charge(&retained, "focused", None)).await; + reopened.close().await.unwrap(); +} diff --git a/core/crates/harvestcircle_storage/src/contract.rs b/core/crates/harvestcircle_storage/src/contract.rs @@ -14,7 +14,7 @@ pub const HARVESTCIRCLE_SERVICE_ID: &str = "harvestcircle"; pub const HARVESTCIRCLE_INSTANCE_ID: &str = "desktop"; pub const HARVESTCIRCLE_APPLICATION_ID: u32 = 0x4843_5231; pub(crate) const HARVESTCIRCLE_INITIAL_STATE_SCHEMA_VERSION: u32 = 1; -pub const HARVESTCIRCLE_STATE_SCHEMA_VERSION: u32 = 2; +pub const HARVESTCIRCLE_STATE_SCHEMA_VERSION: u32 = 3; pub const HARVESTCIRCLE_IDENTITY_CAPACITY: usize = 256; pub const HARVESTCIRCLE_UNFINISHED_DURABLE_OPERATION_CAPACITY: usize = 1_024; pub const HARVESTCIRCLE_DURABLE_OPERATION_CAPACITY: usize = 4_096; @@ -278,6 +278,52 @@ pub(crate) const CREATE_INSTALLATION_IDENTITY_SQL: &str = r#"CREATE TABLE instal installation_id BLOB NOT NULL CHECK (length(installation_id) = 16) ) STRICT"#; +pub(crate) const MIGRATE_AVAILABILITY_EVIDENCE_V3_SQL: &str = r#"UPDATE account_preferences SET preference_value = preference_value; +CREATE TABLE availability_versions ( + event_id BLOB NOT NULL PRIMARY KEY CHECK (length(event_id) = 32), + author BLOB NOT NULL CHECK (length(author) = 32), + kind INTEGER NOT NULL CHECK (kind = 30402), + raw_d TEXT NOT NULL CHECK (length(CAST(raw_d AS BLOB)) BETWEEN 0 AND 4096), + signed_at BLOB NOT NULL CHECK (length(signed_at) = 8), + published_at BLOB CHECK (published_at IS NULL OR length(published_at) = 8), + original_json TEXT NOT NULL CHECK ( + length(CAST(original_json AS BLOB)) BETWEEN 1 AND 262144 + ), + admission_label TEXT NOT NULL CHECK (admission_label IN ( + 'focused', 'excluded_focused', 'excluded_operational', 'excluded_generic', + 'excluded_ambiguous', 'projection_rejected' + )), + rejection_code TEXT CHECK ( + rejection_code IS NULL OR ( + length(CAST(rejection_code AS BLOB)) BETWEEN 1 AND 64 + AND rejection_code NOT GLOB '*[^ -~]*' + ) + ), + source TEXT NOT NULL CHECK (length(CAST(source AS BLOB)) BETWEEN 1 AND 2048), + observed_at_unix_s INTEGER NOT NULL CHECK (observed_at_unix_s >= 0), + payload_bytes INTEGER NOT NULL CHECK ( + payload_bytes = 92 + length(CAST(original_json AS BLOB)) + + length(CAST(raw_d AS BLOB)) + length(CAST(source AS BLOB)) + + length(CAST(admission_label AS BLOB)) + + coalesce(length(CAST(rejection_code AS BLOB)), 0) + ), + CHECK ( + (admission_label = 'focused' AND rejection_code IS NULL AND published_at IS NOT NULL) + OR + (admission_label IN ( + 'excluded_focused', 'excluded_operational', 'excluded_generic', 'excluded_ambiguous' + ) AND rejection_code IS NULL AND published_at IS NULL) + OR + (admission_label = 'projection_rejected' AND rejection_code IS NOT NULL AND published_at IS NULL) + ) +) STRICT; +CREATE TABLE public_payload_usage ( + singleton INTEGER NOT NULL PRIMARY KEY CHECK (singleton = 1), + version_count INTEGER NOT NULL CHECK (version_count BETWEEN 0 AND 4096), + payload_bytes INTEGER NOT NULL CHECK (payload_bytes BETWEEN 0 AND 134217728) +) STRICT; +INSERT INTO public_payload_usage (singleton, version_count, payload_bytes) VALUES (1, 0, 0)"#; + pub(crate) const CREATE_INSTALLATION_IDENTITY_NO_UPDATE_SQL: &str = r#"CREATE TRIGGER installation_identity_no_update BEFORE UPDATE ON installation_identity BEGIN @@ -366,6 +412,24 @@ const DURABLE_OPERATIONS_V2_MIGRATION_CHECKSUM: MigrationChecksum = 210, 83, 137, 114, 251, 20, 30, 31, 11, 136, 207, 168, ]); +const AVAILABILITY_VERSIONS_DIGEST: [u8; 32] = [ + 250, 239, 225, 0, 53, 126, 33, 150, 232, 105, 102, 35, 26, 221, 78, 156, 137, 219, 66, 240, 10, + 18, 96, 99, 102, 199, 219, 253, 8, 45, 171, 52, +]; +const PUBLIC_PAYLOAD_USAGE_DIGEST: [u8; 32] = [ + 36, 83, 232, 46, 105, 160, 105, 237, 4, 189, 147, 119, 164, 56, 77, 154, 100, 122, 111, 62, + 246, 160, 0, 206, 10, 136, 76, 189, 81, 68, 10, 26, +]; +const VERSION_THREE_DIGEST: [u8; 32] = [ + 53, 128, 250, 106, 143, 97, 51, 33, 195, 227, 254, 35, 19, 135, 136, 171, 229, 241, 204, 234, + 158, 180, 204, 19, 113, 95, 73, 250, 135, 241, 51, 88, +]; +const AVAILABILITY_EVIDENCE_V3_MIGRATION_CHECKSUM: MigrationChecksum = + MigrationChecksum::from_bytes([ + 254, 36, 47, 105, 27, 136, 68, 240, 103, 108, 27, 205, 97, 203, 221, 84, 234, 64, 5, 123, + 229, 227, 10, 199, 30, 127, 170, 170, 11, 2, 78, 81, + ]); + /// A sealed binding between one HarvestCircle runtime context and the governed state catalogs. /// /// External callers cannot forge alternate paths or catalogs: @@ -483,7 +547,14 @@ pub fn harvestcircle_migration_catalog() DURABLE_OPERATIONS_V2_MIGRATION_CHECKSUM, ) .map_err(|_| HarvestCircleStorageContractError::MigrationCatalog)?; - MigrationCatalog::new([migration]) + let evidence = MigrationDescriptor::sql( + 3, + "add_verified_listing_evidence", + MIGRATE_AVAILABILITY_EVIDENCE_V3_SQL, + AVAILABILITY_EVIDENCE_V3_MIGRATION_CHECKSUM, + ) + .map_err(|_| HarvestCircleStorageContractError::MigrationCatalog)?; + MigrationCatalog::new([migration, evidence]) .map_err(|_| HarvestCircleStorageContractError::MigrationCatalog) } @@ -503,12 +574,42 @@ fn schema_catalog_for( .map_err(schema_error)?; let version_two_objects = schema_objects(CREATE_DURABLE_OPERATIONS_V2_SQL)?; let version_two = SchemaVersionCatalog::new( - HARVESTCIRCLE_STATE_SCHEMA_VERSION, + 2, version_two_objects, SchemaDigest::from_bytes(VERSION_TWO_DIGEST), ) .map_err(schema_error)?; - SchemaCatalog::new(migrations, [version_one, version_two]).map_err(schema_error) + let version_three = SchemaVersionCatalog::new( + 3, + schema_objects_v3()?, + SchemaDigest::from_bytes(VERSION_THREE_DIGEST), + ) + .map_err(schema_error)?; + SchemaCatalog::new(migrations, [version_one, version_two, version_three]).map_err(schema_error) +} + +fn schema_objects_v3() -> Result<Vec<SchemaObject>, HarvestCircleStorageContractError> { + let mut objects = schema_objects(CREATE_DURABLE_OPERATIONS_V2_SQL)?; + let mut statements = MIGRATE_AVAILABILITY_EVIDENCE_V3_SQL.split(";\n").skip(1); + for (name, digest) in [ + ("availability_versions", AVAILABILITY_VERSIONS_DIGEST), + ("public_payload_usage", PUBLIC_PAYLOAD_USAGE_DIGEST), + ] { + let sql = statements + .next() + .ok_or(HarvestCircleStorageContractError::SchemaCatalog)?; + objects.push( + SchemaObject::new( + SchemaObjectKind::Table, + name, + name, + sql, + SchemaDigest::from_bytes(digest), + ) + .map_err(schema_error)?, + ); + } + Ok(objects) } fn schema_objects( @@ -610,6 +711,10 @@ const fn schema_error(_: SchemaCatalogContractError) -> HarvestCircleStorageCont } #[cfg(test)] +#[path = "availability_evidence_tests.rs"] +mod availability_evidence_tests; + +#[cfg(test)] mod tests { use super::*; use radroots_runtime_paths::{ @@ -647,10 +752,10 @@ mod tests { contract.application_id().get(), HARVESTCIRCLE_APPLICATION_ID ); - assert_eq!(contract.state_schema_version().get(), 2); - assert_eq!(contract.migrations().current_version(), 2); - assert_eq!(contract.migrations().descriptors().len(), 1); - assert_eq!(contract.schema().versions().len(), 2); + assert_eq!(contract.state_schema_version().get(), 3); + assert_eq!(contract.migrations().current_version(), 3); + assert_eq!(contract.migrations().descriptors().len(), 2); + assert_eq!(contract.schema().versions().len(), 3); assert_eq!(harvestcircle_initial_schema_sql().len(), 9); } diff --git a/core/crates/harvestcircle_storage/src/lib.rs b/core/crates/harvestcircle_storage/src/lib.rs @@ -1,6 +1,7 @@ #![doc = "HarvestCircle persistence adapters."] #![cfg_attr(coverage_nightly, feature(coverage_attribute))] +mod availability_evidence; mod backup; mod contract; mod db; diff --git a/core/crates/harvestcircle_storage/tests/package_boundary.rs b/core/crates/harvestcircle_storage/tests/package_boundary.rs @@ -14,9 +14,10 @@ fn storage_package_keeps_one_sqlite_authority_and_a_sealed_public_surface() { let root_source = read(&crate_root.join("src/lib.rs")); let contract_source = read(&crate_root.join("src/contract.rs")); let database_source = read(&crate_root.join("src/db.rs")); + let evidence_source = read(&crate_root.join("src/availability_evidence.rs")); let journal_source = read(&crate_root.join("src/journal.rs")); let keyring_source = read(&crate_root.join("src/os_keyring.rs")); - let api = read(&workspace_root.join("compatibility/harvestcircle-storage-api-v2.txt")); + let api = read(&workspace_root.join("compatibility/harvestcircle-storage-api-v3.txt")); for forbidden in ["rusqlite", "refinery", "hmac", "rustix"] { assert!( @@ -25,6 +26,12 @@ fn storage_package_keeps_one_sqlite_authority_and_a_sealed_public_surface() { ); } assert!(manifest.contains("radroots_service_sqlite.workspace = true")); + assert!(manifest.contains("radroots_event = { workspace = true, features = [\"std\"] }")); + assert!( + manifest.contains( + "radroots_event_codec = { workspace = true, features = [\"std\", \"json\"] }" + ) + ); assert!(manifest.contains("sqlx.workspace = true")); assert!(!manifest.contains("\nkeyring =")); assert!(manifest.contains( @@ -42,6 +49,7 @@ fn storage_package_keeps_one_sqlite_authority_and_a_sealed_public_surface() { } for module in [ + "availability_evidence", "backup", "contract", "db", @@ -106,11 +114,31 @@ fn storage_package_keeps_one_sqlite_authority_and_a_sealed_public_surface() { } assert!(!database_source.contains("pub fn host")); assert!(!database_source.contains("pub const fn host")); + for forbidden in [ + "SqlitePool", + "SqliteConnection", + "DELETE FROM availability_versions", + "DELETE FROM public_payload_usage", + "SecretKey", + "OsKeyringSecretStore", + "account_identities", + "local_signer_bindings", + "profile_cache", + ] { + assert!( + !evidence_source.contains(forbidden), + "public evidence crossed authority through {forbidden}" + ); + } + assert!(evidence_source.contains("verify_nip01_event(")); + assert!(evidence_source.contains("Nip01EventWire::parse_json_unverified(")); for required in [ "pub struct harvestcircle_storage::Database", "pub async fn harvestcircle_storage::Database::open", "pub async fn harvestcircle_storage::Database::close", + "pub async fn harvestcircle_storage::Database::retain_availability_version", + "pub async fn harvestcircle_storage::Database::load_availability_version", "pub async fn harvestcircle_storage::Database::capture_online_backup", "pub async fn harvestcircle_storage::Database::restore_verified_backup", "pub struct harvestcircle_storage::VerifiedHarvestCircleBackup", @@ -135,6 +163,10 @@ fn storage_package_keeps_one_sqlite_authority_and_a_sealed_public_surface() { "sqlx::", "OperationJournal", "harvestcircle_initial_schema_sql", + "SELECT_AVAILABILITY_VERSION_SQL", + "decode_availability_row", + "retain_availability_version_on", + "ServiceSqliteTransaction", "VerifiedServiceBackup", "StagedServiceRestore", "verify_backup_bundle", @@ -150,7 +182,7 @@ fn storage_package_keeps_one_sqlite_authority_and_a_sealed_public_surface() { } for required in [ - "pub const HARVESTCIRCLE_STATE_SCHEMA_VERSION: u32 = 2;", + "pub const HARVESTCIRCLE_STATE_SCHEMA_VERSION: u32 = 3;", "pub const HARVESTCIRCLE_UNFINISHED_DURABLE_OPERATION_CAPACITY: usize = 1_024;", "pub const HARVESTCIRCLE_DURABLE_OPERATION_CAPACITY: usize = 4_096;", "pub const HARVESTCIRCLE_DURABLE_OPERATION_CLEANUP_BATCH: usize = 256;", @@ -159,6 +191,9 @@ fn storage_package_keeps_one_sqlite_authority_and_a_sealed_public_surface() { "completed_at_unix_s", "durable_operations_receipt_insert_guard", "durable_operations_receipt_update_guard", + "add_verified_listing_evidence", + "CREATE TABLE availability_versions", + "CREATE TABLE public_payload_usage", ] { assert!( contract_source.contains(required), diff --git a/core/crates/harvestcircle_storage/tests/sqlx_storage.rs b/core/crates/harvestcircle_storage/tests/sqlx_storage.rs @@ -185,7 +185,7 @@ async fn utf8_boundary_identity_and_profile_metadata_round_trip_through_governed .await .expect("close before final assertions"); - assert_eq!(schema_version, 2); + assert_eq!(schema_version, 3); assert_eq!( all_identities.expect("persisted identities").len(), fixtures.len() @@ -344,7 +344,7 @@ async fn canonical_database_preserves_legacy_state_and_enforces_identity_capacit .await .expect("database"); let generation = database.metadata().source_generation(); - assert_eq!(database.metadata().state_schema_version().get(), 2); + assert_eq!(database.metadata().state_schema_version().get(), 3); let first_identity = identity(0); database @@ -388,7 +388,7 @@ async fn canonical_database_preserves_legacy_state_and_enforces_identity_capacit .await .expect("reopen"); assert_eq!(reopened.metadata().source_generation(), generation); - assert_eq!(reopened.metadata().state_schema_version().get(), 2); + assert_eq!(reopened.metadata().state_schema_version().get(), 3); assert_eq!( reopened .list_identities() diff --git a/radroots.lib.source-lock.v1.toml b/radroots.lib.source-lock.v1.toml @@ -6,4 +6,4 @@ workspace_catalog_sha256 = "ee295f2352e2577a4052d980624415aec9871197d4fc9910a4c2 version = "0.1.0-alpha" source_archive_sha256 = "c648a3ab993d10253b9073e7e86db7b8970863bdf1d394d9fc30d66825695240" lockfile = "core/Cargo.lock" -lockfile_sha256 = "3adc3bd3370fcb21821fd7e9be45554258e89088a791781a798fc39a4bdfcc31" +lockfile_sha256 = "6a32d1de0105771158647c65116fe797dfa7db7c42db515c63047d6601b846a3" diff --git a/tools/verify-storage-api.sh b/tools/verify-storage-api.sh @@ -12,4 +12,4 @@ cargo +nightly-2026-07-16 public-api \ -p harvestcircle_storage \ --all-features \ -sss >"$output" -cmp core/compatibility/harvestcircle-storage-api-v2.txt "$output" +cmp core/compatibility/harvestcircle-storage-api-v3.txt "$output" diff --git a/tools/xtask/src/lib.rs b/tools/xtask/src/lib.rs @@ -1159,7 +1159,7 @@ fn provenance_check(root: &Path, inventory: &Inventory, findings: &mut Vec<Strin "version = \"0.1.0-alpha\"\n", "source_archive_sha256 = \"c648a3ab993d10253b9073e7e86db7b8970863bdf1d394d9fc30d66825695240\"\n", "lockfile = \"core/Cargo.lock\"\n", - "lockfile_sha256 = \"3adc3bd3370fcb21821fd7e9be45554258e89088a791781a798fc39a4bdfcc31\"\n", + "lockfile_sha256 = \"6a32d1de0105771158647c65116fe797dfa7db7c42db515c63047d6601b846a3\"\n", ); let source_lock_bytes = match bounded_no_follow_bytes(root, Path::new(SOURCE_LOCK_PATH), MAX_SOURCE_LOCK_BYTES) { @@ -1274,7 +1274,7 @@ fn provenance_check(root: &Path, inventory: &Inventory, findings: &mut Vec<Strin { findings.push("app/shared/build.gradle.kts: shared KMP target boundary changed".to_owned()); } - const STORAGE_API_BASELINE: &str = "core/compatibility/harvestcircle-storage-api-v2.txt"; + const STORAGE_API_BASELINE: &str = "core/compatibility/harvestcircle-storage-api-v3.txt"; let storage_api = read_text(root, STORAGE_API_BASELINE); for required in [ "pub struct harvestcircle_storage::HarvestCircleStorageContract", @@ -1283,6 +1283,8 @@ fn provenance_check(root: &Path, inventory: &Inventory, findings: &mut Vec<Strin "pub struct harvestcircle_storage::Database", "pub async fn harvestcircle_storage::Database::open", "pub async fn harvestcircle_storage::Database::close", + "pub async fn harvestcircle_storage::Database::retain_availability_version", + "pub async fn harvestcircle_storage::Database::load_availability_version", "pub async fn harvestcircle_storage::Database::capture_online_backup", "pub async fn harvestcircle_storage::Database::restore_verified_backup", "pub struct harvestcircle_storage::VerifiedHarvestCircleBackup", @@ -1301,6 +1303,10 @@ fn provenance_check(root: &Path, inventory: &Inventory, findings: &mut Vec<Strin "sqlx::", "OperationJournal", "harvestcircle_initial_schema_sql", + "SELECT_AVAILABILITY_VERSION_SQL", + "decode_availability_row", + "retain_availability_version_on", + "ServiceSqliteTransaction", "VerifiedServiceBackup", "StagedServiceRestore", "verify_backup_bundle", @@ -2434,7 +2440,7 @@ mod tests { "version = \"0.1.0-alpha\"\n", "source_archive_sha256 = \"c648a3ab993d10253b9073e7e86db7b8970863bdf1d394d9fc30d66825695240\"\n", "lockfile = \"core/Cargo.lock\"\n", - "lockfile_sha256 = \"3adc3bd3370fcb21821fd7e9be45554258e89088a791781a798fc39a4bdfcc31\"\n", + "lockfile_sha256 = \"6a32d1de0105771158647c65116fe797dfa7db7c42db515c63047d6601b846a3\"\n", ), ); write(&root, "core/Cargo.toml", "");