commit db67342c7416e1e4b03bd20ed8f17e0cf02caa44
parent 89c88d56c832052d2cb2d8b44025ce80c7160f1c
Author: triesap <tyson@radroots.org>
Date: Sun, 14 Jun 2026 16:02:08 -0700
query: wire pocket scrape controls
Diffstat:
14 files changed, 557 insertions(+), 123 deletions(-)
diff --git a/config/tangle.example.json b/config/tangle.example.json
@@ -5,7 +5,12 @@
},
"pocket": {
"data_directory": "runtime/pocket",
- "sync_policy": "flush_on_shutdown"
+ "sync_policy": "flush_on_shutdown",
+ "query": {
+ "allow_scraping": false,
+ "allow_scrape_if_limited_to": 100,
+ "allow_scrape_if_max_seconds": 3600
+ }
},
"groups": {
"enabled": true,
diff --git a/crates/tangle/tests/version.rs b/crates/tangle/tests/version.rs
@@ -102,7 +102,12 @@ fn tangle_run_starts_server_and_stays_alive_until_shutdown() {
},
"pocket": {
"data_directory": data_dir,
- "sync_policy": "flush_on_shutdown"
+ "sync_policy": "flush_on_shutdown",
+ "query": {
+ "allow_scraping": false,
+ "allow_scrape_if_limited_to": 100,
+ "allow_scrape_if_max_seconds": 3600
+ }
},
"groups": {
"enabled": true,
diff --git a/crates/tangle_bench/src/lib.rs b/crates/tangle_bench/src/lib.rs
@@ -15,7 +15,7 @@ use tangle_runtime::relay::{
auth::BaseAuthState,
core::{BaseRelay, BaseRelayLimitSettings, BaseRelayLimits},
};
-use tangle_store_pocket::{PocketStoreConfig, PocketSyncPolicy};
+use tangle_store_pocket::{PocketQueryConfig, PocketStoreConfig, PocketSyncPolicy};
use tangle_test_support::{
FixtureKey, TANGLE_V2_RELAY_URL, tangle_v2_auth_event, tangle_v2_event, tangle_v2_group_config,
tangle_v2_group_create_event, tangle_v2_group_event, tangle_v2_put_user_event, tangle_v2_tag,
@@ -959,6 +959,7 @@ fn run_projection_rebuild_benchmark(dataset: &BenchDataset) -> Result<ScenarioRe
&materialized.store_config,
relay_limits(128),
&group_config()?,
+ PocketQueryConfig::default(),
)
.map_err(|error| error.to_string())?;
let elapsed = elapsed_micros(started);
@@ -997,6 +998,7 @@ fn run_outbox_replay_benchmark(dataset: &BenchDataset) -> Result<ScenarioReport,
&materialized.store_config,
relay_limits(128),
&group_config()?,
+ PocketQueryConfig::default(),
)
.map_err(|error| error.to_string())?;
let after_first = generated_state_counts(&reopened)?;
@@ -1005,6 +1007,7 @@ fn run_outbox_replay_benchmark(dataset: &BenchDataset) -> Result<ScenarioReport,
&materialized.store_config,
relay_limits(128),
&group_config()?,
+ PocketQueryConfig::default(),
)
.map_err(|error| error.to_string())?;
let after_second = generated_state_counts(&reopened)?;
@@ -1109,6 +1112,7 @@ fn materialize_dataset(
&store_config,
relay_limits(max_pending_events),
&group_config()?,
+ PocketQueryConfig::default(),
)
.map_err(|error| error.to_string())?;
let owner_auth = authenticated(FixtureKey::Owner)?;
diff --git a/crates/tangle_protocol/src/lib.rs b/crates/tangle_protocol/src/lib.rs
@@ -692,6 +692,12 @@ impl Filter {
filter
}
+ pub fn with_limit(&self, limit: u64) -> Self {
+ let mut filter = self.clone();
+ filter.limit = Some(limit);
+ filter
+ }
+
pub fn search(&self) -> Option<&str> {
self.search.as_deref()
}
@@ -1985,6 +1991,10 @@ mod tests {
assert_eq!(without_limit.limit(), None);
assert_eq!(without_limit.search(), filter.search());
assert!(without_limit.matches(&event));
+ let with_limit = without_limit.with_limit(2);
+ assert_eq!(with_limit.limit(), Some(2));
+ assert_eq!(with_limit.search(), filter.search());
+ assert!(with_limit.matches(&event));
assert_eq!(
filter_from_value(&filter_to_value(&filter)).expect("encoded"),
filter
diff --git a/crates/tangle_runtime/src/config.rs b/crates/tangle_runtime/src/config.rs
@@ -15,13 +15,16 @@ use serde::Deserialize;
use std::{net::SocketAddr, path::PathBuf};
use tangle_groups::GroupRuntimeConfig;
use tangle_protocol::SubscriptionId;
-use tangle_store_pocket::{PocketStoreConfig, PocketSyncPolicy};
+use tangle_store_pocket::{PocketQueryConfig, PocketStoreConfig, PocketSyncPolicy};
+
+const MAX_POCKET_QUERY_SCRAPE_WINDOW_SECONDS: u64 = 86_400;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BaseRelayRuntimeConfig {
listen_addr: SocketAddr,
relay_url: String,
pocket: PocketStoreConfig,
+ pocket_query: PocketQueryConfig,
groups: GroupRuntimeConfig,
auth_ttl_seconds: u64,
auth_created_at_skew_seconds: u64,
@@ -43,6 +46,10 @@ impl BaseRelayRuntimeConfig {
&self.pocket
}
+ pub fn pocket_query_config(&self) -> PocketQueryConfig {
+ self.pocket_query
+ }
+
pub fn groups(&self) -> &GroupRuntimeConfig {
&self.groups
}
@@ -68,7 +75,12 @@ impl BaseRelayRuntimeConfig {
}
pub fn open_relay(&self) -> Result<BaseRelay, BaseRelayError> {
- BaseRelay::open_with_groups(&self.pocket, self.limits.base_relay_limits()?, &self.groups)
+ BaseRelay::open_with_groups(
+ &self.pocket,
+ self.limits.base_relay_limits()?,
+ &self.groups,
+ self.pocket_query,
+ )
}
pub fn auth_state(&self) -> Result<BaseAuthState, BaseRelayError> {
@@ -306,6 +318,15 @@ struct BaseRelayServerConfigDocument {
struct BaseRelayPocketConfigDocument {
data_directory: String,
sync_policy: BaseRelayPocketSyncPolicyDocument,
+ query: BaseRelayPocketQueryConfigDocument,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct BaseRelayPocketQueryConfigDocument {
+ allow_scraping: bool,
+ allow_scrape_if_limited_to: u32,
+ allow_scrape_if_max_seconds: u64,
}
#[derive(Debug, Clone, Copy, Deserialize)]
@@ -433,9 +454,10 @@ pub fn parse_base_relay_runtime_config_json(
.map_err(|error| {
BaseRelayError::invalid(format!("server.listen_addr is invalid: {error}"))
})?;
+ let pocket_document = document.pocket;
let pocket = PocketStoreConfig::new(
- PathBuf::from(document.pocket.data_directory),
- match document.pocket.sync_policy {
+ PathBuf::from(pocket_document.data_directory),
+ match pocket_document.sync_policy {
BaseRelayPocketSyncPolicyDocument::FlushOnWrite => PocketSyncPolicy::FlushOnWrite,
BaseRelayPocketSyncPolicyDocument::FlushOnShutdown => PocketSyncPolicy::FlushOnShutdown,
},
@@ -447,6 +469,7 @@ pub fn parse_base_relay_runtime_config_json(
let groups = tangle_groups::parse_group_runtime_config_json(&groups_raw)
.map_err(|error| BaseRelayError::invalid(error.to_string()))?;
let limits = BaseRelayRuntimeLimitsConfig::from_document(document.limits)?;
+ let pocket_query = pocket_query_config_from_document(pocket_document.query, limits)?;
let rate_limits = base_relay_rate_limits_from_document(document.rate_limits)?;
if document.auth.created_at_skew_seconds == 0 {
return Err(BaseRelayError::invalid(
@@ -458,6 +481,7 @@ pub fn parse_base_relay_runtime_config_json(
listen_addr,
relay_url: document.server.relay_url,
pocket,
+ pocket_query,
groups,
auth_ttl_seconds: document.auth.challenge_ttl_seconds,
auth_created_at_skew_seconds: document.auth.created_at_skew_seconds,
@@ -467,6 +491,27 @@ pub fn parse_base_relay_runtime_config_json(
})
}
+fn pocket_query_config_from_document(
+ document: BaseRelayPocketQueryConfigDocument,
+ limits: BaseRelayRuntimeLimitsConfig,
+) -> Result<PocketQueryConfig, BaseRelayError> {
+ if u64::from(document.allow_scrape_if_limited_to) > limits.max_limit() {
+ return Err(BaseRelayError::invalid(
+ "pocket.query.allow_scrape_if_limited_to must be less than or equal to limits.max_limit",
+ ));
+ }
+ if document.allow_scrape_if_max_seconds > MAX_POCKET_QUERY_SCRAPE_WINDOW_SECONDS {
+ return Err(BaseRelayError::invalid(format!(
+ "pocket.query.allow_scrape_if_max_seconds must be less than or equal to {MAX_POCKET_QUERY_SCRAPE_WINDOW_SECONDS}"
+ )));
+ }
+ Ok(PocketQueryConfig::new(
+ document.allow_scraping,
+ document.allow_scrape_if_limited_to,
+ document.allow_scrape_if_max_seconds,
+ ))
+}
+
fn require_positive(field: &str, value: usize) -> Result<(), BaseRelayError> {
if value == 0 {
return Err(BaseRelayError::invalid(format!(
@@ -626,6 +671,15 @@ mod tests {
config.pocket_config().sync_policy(),
PocketSyncPolicy::FlushOnShutdown
);
+ assert!(!config.pocket_query_config().allow_scraping());
+ assert_eq!(
+ config.pocket_query_config().allow_scrape_if_limited_to(),
+ 100
+ );
+ assert_eq!(
+ config.pocket_query_config().allow_scrape_if_max_seconds(),
+ 3_600
+ );
assert!(config.groups().enabled());
assert!(!config.groups().policy().public_join());
assert!(!config.groups().policy().invites_enabled());
@@ -694,7 +748,12 @@ mod tests {
},
"pocket": {
"data_directory": "runtime/pocket",
- "sync_policy": "flush_on_shutdown"
+ "sync_policy": "flush_on_shutdown",
+ "query": {
+ "allow_scraping": false,
+ "allow_scrape_if_limited_to": 100,
+ "allow_scrape_if_max_seconds": 3600
+ }
},
"groups": {
"enabled": false
@@ -773,7 +832,12 @@ mod tests {
},
"pocket": {
"data_directory": "runtime/pocket",
- "sync_policy": "flush_on_shutdown"
+ "sync_policy": "flush_on_shutdown",
+ "query": {
+ "allow_scraping": false,
+ "allow_scrape_if_limited_to": 100,
+ "allow_scrape_if_max_seconds": 3600
+ }
},
"groups": {
"enabled": false
@@ -849,7 +913,12 @@ mod tests {
},
"pocket": {
"data_directory": "runtime/pocket",
- "sync_policy": "flush_on_shutdown"
+ "sync_policy": "flush_on_shutdown",
+ "query": {
+ "allow_scraping": false,
+ "allow_scrape_if_limited_to": 100,
+ "allow_scrape_if_max_seconds": 3600
+ }
},
"groups": {
"enabled": false
@@ -945,6 +1014,31 @@ mod tests {
}
#[test]
+ fn base_relay_runtime_config_validates_pocket_query_controls() {
+ let raw = include_str!("../../../config/tangle.example.json").replace(
+ " \"allow_scrape_if_limited_to\": 100,\n",
+ " \"allow_scrape_if_limited_to\": 501,\n",
+ );
+ assert_eq!(
+ parse_base_relay_runtime_config_json(&raw)
+ .expect_err("query scrape limit")
+ .prefixed_message(),
+ "invalid: pocket.query.allow_scrape_if_limited_to must be less than or equal to limits.max_limit"
+ );
+
+ let raw = include_str!("../../../config/tangle.example.json").replace(
+ " \"allow_scrape_if_max_seconds\": 3600\n",
+ " \"allow_scrape_if_max_seconds\": 86401\n",
+ );
+ assert_eq!(
+ parse_base_relay_runtime_config_json(&raw)
+ .expect_err("query scrape window")
+ .prefixed_message(),
+ "invalid: pocket.query.allow_scrape_if_max_seconds must be less than or equal to 86400"
+ );
+ }
+
+ #[test]
fn base_relay_runtime_config_requires_explicit_query_complexity() {
let raw = include_str!("../../../config/tangle.example.json")
.replace(" \"max_query_complexity\": 2048,\n", "");
diff --git a/crates/tangle_runtime/src/nip11.rs b/crates/tangle_runtime/src/nip11.rs
@@ -390,7 +390,12 @@ mod tests {
},
"pocket": {
"data_directory": "runtime/pocket",
- "sync_policy": "flush_on_shutdown"
+ "sync_policy": "flush_on_shutdown",
+ "query": {
+ "allow_scraping": false,
+ "allow_scrape_if_limited_to": 100,
+ "allow_scrape_if_max_seconds": 3600
+ }
},
"groups": groups,
"auth": {
diff --git a/crates/tangle_runtime/src/relay/core.rs b/crates/tangle_runtime/src/relay/core.rs
@@ -16,7 +16,9 @@ use tangle_groups::{
StoreOffset, classify_group_event, validate_client_group_event_structure,
};
use tangle_protocol::{ClientMessage, Event, Filter, RelayMessage, SubscriptionId, UnixTimestamp};
-use tangle_store_pocket::{PocketScreenResult, PocketStoreConfig, PocketStoreHandle};
+use tangle_store_pocket::{
+ PocketQueryConfig, PocketScreenResult, PocketStoreConfig, PocketStoreHandle,
+};
pub struct BaseRelay {
store: PocketStoreHandle,
@@ -24,6 +26,7 @@ pub struct BaseRelay {
groups: Option<GroupService>,
readiness: BaseRelayReadinessState,
limits: BaseRelayLimits,
+ query: PocketQueryConfig,
}
#[derive(Debug, Clone, PartialEq)]
@@ -412,28 +415,35 @@ impl BaseRelay {
pub fn open(
config: &PocketStoreConfig,
limits: BaseRelayLimits,
+ query: PocketQueryConfig,
) -> Result<Self, BaseRelayError> {
let store = PocketStoreHandle::open(config).map_err(BaseRelayError::from)?;
- Self::new(store, limits)
+ Self::new(store, limits, query)
}
pub fn open_with_groups(
config: &PocketStoreConfig,
limits: BaseRelayLimits,
groups: &GroupRuntimeConfig,
+ query: PocketQueryConfig,
) -> Result<Self, BaseRelayError> {
let store = PocketStoreHandle::open(config).map_err(BaseRelayError::from)?;
- Self::new_with_groups(store, limits, groups)
+ Self::new_with_groups(store, limits, groups, query)
}
- pub fn new(store: PocketStoreHandle, limits: BaseRelayLimits) -> Result<Self, BaseRelayError> {
- Self::new_with_groups(store, limits, &GroupRuntimeConfig::disabled())
+ pub fn new(
+ store: PocketStoreHandle,
+ limits: BaseRelayLimits,
+ query: PocketQueryConfig,
+ ) -> Result<Self, BaseRelayError> {
+ Self::new_with_groups(store, limits, &GroupRuntimeConfig::disabled(), query)
}
pub fn new_with_groups(
store: PocketStoreHandle,
limits: BaseRelayLimits,
groups: &GroupRuntimeConfig,
+ query: PocketQueryConfig,
) -> Result<Self, BaseRelayError> {
let groups = GroupService::from_config(&store, groups)?;
let subscriptions =
@@ -445,6 +455,7 @@ impl BaseRelay {
groups,
readiness,
limits,
+ query,
})
}
@@ -922,38 +933,36 @@ impl BaseRelay {
filter: &Filter,
auth: &GroupAuthContext,
) -> Result<BaseRelayEventQueryReport, BaseRelayError> {
- let pocket_filter = tangle_filter_to_pocket(filter)?;
+ let effective_filter = self.filter_with_effective_limit(filter);
+ let pocket_filter = tangle_filter_to_pocket(&effective_filter)?;
let screen_error = RefCell::new(None);
- let screened = self.store.find_events_with_screen(
- &pocket_filter,
- true,
- 0,
- u64::MAX,
- |pocket_event| {
- if screen_error.borrow().is_some() {
- return PocketScreenResult::Mismatch;
- }
- match pocket_filter.event_matches(pocket_event) {
- Ok(false) => PocketScreenResult::Mismatch,
- Ok(true) => match Self::group_read_gate_visible_to_auth(
- self.groups.as_ref(),
- pocket_event,
- auth,
- ) {
- Ok(true) => PocketScreenResult::Match,
- Ok(false) => PocketScreenResult::Redacted,
+ let screened =
+ self.store
+ .find_events_with_screen(&pocket_filter, self.query, |pocket_event| {
+ if screen_error.borrow().is_some() {
+ return PocketScreenResult::Mismatch;
+ }
+ match pocket_filter.event_matches(pocket_event) {
+ Ok(false) => PocketScreenResult::Mismatch,
+ Ok(true) => match Self::group_read_gate_visible_to_auth(
+ self.groups.as_ref(),
+ pocket_event,
+ auth,
+ ) {
+ Ok(true) => PocketScreenResult::Match,
+ Ok(false) => PocketScreenResult::Redacted,
+ Err(error) => {
+ *screen_error.borrow_mut() = Some(error);
+ PocketScreenResult::Mismatch
+ }
+ },
Err(error) => {
- *screen_error.borrow_mut() = Some(error);
+ *screen_error.borrow_mut() =
+ Some(BaseRelayError::error(error.to_string()));
PocketScreenResult::Mismatch
}
- },
- Err(error) => {
- *screen_error.borrow_mut() = Some(BaseRelayError::error(error.to_string()));
- PocketScreenResult::Mismatch
}
- }
- },
- )?;
+ })?;
if let Some(error) = screen_error.into_inner() {
return Err(error);
}
@@ -966,6 +975,13 @@ impl BaseRelay {
Ok(BaseRelayEventQueryReport::new(events, group_read_denied))
}
+ fn filter_with_effective_limit(&self, filter: &Filter) -> Filter {
+ match filter.limit() {
+ Some(_) => filter.clone(),
+ None => filter.with_limit(self.limits.default_limit()),
+ }
+ }
+
fn sort_and_dedupe_query_events(mut events: Vec<Event>) -> Vec<Event> {
events.sort_by(|left, right| {
right
@@ -1023,7 +1039,7 @@ mod tests {
ClientMessage, Event, EventId, Filter, Kind, PublicKeyHex, RelayMessage, SubscriptionId,
Tag, UnixTimestamp, UnsignedEvent, filter_from_value,
};
- use tangle_store_pocket::{PocketStoreConfig, PocketSyncPolicy};
+ use tangle_store_pocket::{PocketQueryConfig, PocketStoreConfig, PocketSyncPolicy};
#[test]
fn base_relay_stores_queries_counts_closes_and_fans_out_public_events() {
let mut relay = test_relay("base-relay-public", 4);
@@ -1075,6 +1091,60 @@ mod tests {
}
#[test]
+ fn base_relay_uses_configured_pocket_query_scrape_controls() {
+ let strict_config = test_store_config("base-relay-query-strict");
+ let mut strict = BaseRelay::open(
+ &strict_config,
+ relay_limits(4),
+ PocketQueryConfig::new(false, 0, 0),
+ )
+ .expect("strict");
+ let strict_event = signed_public_event(7, 1, Vec::new(), "strict");
+ let broad = filter_from_value(&serde_json::json!({"limit":1})).expect("filter");
+
+ assert_accepted(
+ strict
+ .handle_event(strict_event.clone())
+ .expect("strict event"),
+ &strict_event,
+ );
+ assert!(
+ strict
+ .handle_req(
+ SubscriptionId::new("strict").expect("sub"),
+ vec![broad.clone()]
+ )
+ .expect_err("strict scrape")
+ .prefixed_message()
+ .to_lowercase()
+ .contains("scraper")
+ );
+
+ let limited_config = test_store_config("base-relay-query-limited");
+ let mut limited = BaseRelay::open(
+ &limited_config,
+ relay_limits(4),
+ PocketQueryConfig::new(false, 1, 0),
+ )
+ .expect("limited");
+ let limited_event = signed_public_event(8, 1, Vec::new(), "limited");
+
+ assert_accepted(
+ limited
+ .handle_event(limited_event.clone())
+ .expect("limited event"),
+ &limited_event,
+ );
+ let messages = limited
+ .handle_req(SubscriptionId::new("limited").expect("sub"), vec![broad])
+ .expect("limited scrape");
+
+ assert!(
+ matches!(&messages[0], RelayMessage::Event { event, .. } if event.id() == limited_event.id())
+ );
+ }
+
+ #[test]
fn base_relay_fetches_events_by_store_offset() {
let relay = test_relay("base-relay-offset-lookup", 4);
let event = signed_public_event(7, 1, Vec::new(), "offset");
@@ -1169,6 +1239,7 @@ mod tests {
default_limit: 1,
})
.expect("limits"),
+ PocketQueryConfig::default(),
)
.expect("relay");
let first = signed_event_at(7, 1, Vec::new(), "one", 1_714_124_430);
@@ -1281,6 +1352,7 @@ mod tests {
default_limit: 1,
})
.expect("limits"),
+ PocketQueryConfig::default(),
)
.expect("relay");
let complex = filter_from_value(&serde_json::json!({
@@ -2230,7 +2302,8 @@ mod tests {
#[test]
fn base_relay_shutdown_closes_live_subscriptions_and_syncs_store() {
let config = test_store_config("base-relay-shutdown");
- let mut relay = BaseRelay::open(&config, relay_limits(4)).expect("relay");
+ let mut relay =
+ BaseRelay::open(&config, relay_limits(4), PocketQueryConfig::default()).expect("relay");
let event = signed_public_event(7, 1, Vec::new(), "shutdown");
let subscription_id = SubscriptionId::new("sub-shutdown").expect("sub");
@@ -2247,7 +2320,8 @@ mod tests {
assert_eq!(relay.active_subscription_count(), 0);
assert!(relay.fanout(&event).is_empty());
- let reopened = BaseRelay::open(&config, relay_limits(4)).expect("reopened");
+ let reopened = BaseRelay::open(&config, relay_limits(4), PocketQueryConfig::default())
+ .expect("reopened");
assert_eq!(count_kind(&reopened, 1), 1);
}
@@ -2296,7 +2370,9 @@ mod tests {
#[test]
fn base_relay_enforces_event_and_filter_runtime_limits() {
let config = test_store_config("base-relay-event-filter-runtime-limits");
- let mut relay = BaseRelay::open(&config, strict_relay_limits()).expect("relay");
+ let mut relay =
+ BaseRelay::open(&config, strict_relay_limits(), PocketQueryConfig::default())
+ .expect("relay");
let first = signed_public_event(7, 1, Vec::new(), "a");
let second = signed_event_at(8, 1, Vec::new(), "b", 1_714_124_434);
@@ -2374,7 +2450,9 @@ mod tests {
#[test]
fn base_relay_enforces_subscription_id_and_count_limits() {
let config = test_store_config("base-relay-subscription-limits");
- let mut relay = BaseRelay::open(&config, strict_relay_limits()).expect("relay");
+ let mut relay =
+ BaseRelay::open(&config, strict_relay_limits(), PocketQueryConfig::default())
+ .expect("relay");
assert!(
relay
@@ -2412,7 +2490,12 @@ mod tests {
fn test_relay(name: &str, max_pending_events: usize) -> BaseRelay {
let config = test_store_config(name);
- BaseRelay::open(&config, relay_limits(max_pending_events)).expect("relay")
+ BaseRelay::open(
+ &config,
+ relay_limits(max_pending_events),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay")
}
fn test_relay_with_groups(
@@ -2421,8 +2504,13 @@ mod tests {
groups: &tangle_groups::GroupRuntimeConfig,
) -> BaseRelay {
let config = test_store_config(name);
- BaseRelay::open_with_groups(&config, relay_limits(max_pending_events), groups)
- .expect("relay")
+ BaseRelay::open_with_groups(
+ &config,
+ relay_limits(max_pending_events),
+ groups,
+ PocketQueryConfig::default(),
+ )
+ .expect("relay")
}
fn relay_limits(max_pending_events: usize) -> BaseRelayLimits {
diff --git a/crates/tangle_runtime/src/runtime.rs b/crates/tangle_runtime/src/runtime.rs
@@ -2715,7 +2715,12 @@ mod tests {
},
"pocket": {
"data_directory": root.join("pocket"),
- "sync_policy": "flush_on_shutdown"
+ "sync_policy": "flush_on_shutdown",
+ "query": {
+ "allow_scraping": false,
+ "allow_scrape_if_limited_to": 100,
+ "allow_scrape_if_max_seconds": 3600
+ }
},
"groups": {
"enabled": true,
diff --git a/crates/tangle_runtime/src/server.rs b/crates/tangle_runtime/src/server.rs
@@ -524,7 +524,12 @@ mod tests {
},
"pocket": {
"data_directory": root.join("pocket"),
- "sync_policy": "flush_on_shutdown"
+ "sync_policy": "flush_on_shutdown",
+ "query": {
+ "allow_scraping": false,
+ "allow_scrape_if_limited_to": 100,
+ "allow_scrape_if_max_seconds": 3600
+ }
},
"groups": {
"enabled": true,
diff --git a/crates/tangle_runtime/src/session.rs b/crates/tangle_runtime/src/session.rs
@@ -738,7 +738,12 @@ mod tests {
},
"pocket": {
"data_directory": root.join("pocket"),
- "sync_policy": "flush_on_shutdown"
+ "sync_policy": "flush_on_shutdown",
+ "query": {
+ "allow_scraping": false,
+ "allow_scrape_if_limited_to": 100,
+ "allow_scrape_if_max_seconds": 3600
+ }
},
"groups": {
"enabled": false
diff --git a/crates/tangle_runtime/tests/base_relay_v2.rs b/crates/tangle_runtime/tests/base_relay_v2.rs
@@ -25,8 +25,9 @@ use tangle_runtime::{
},
};
use tangle_store_pocket::{
- PocketStoreConfig, PocketStoreHandle, PocketSyncPolicy, TANGLE_GROUP_CHECKPOINT_TABLE,
- TANGLE_GROUP_OUTBOX_TABLE, TANGLE_GROUP_PROJECTION_TABLE, parse_pocket_event_json,
+ PocketQueryConfig, PocketStoreConfig, PocketStoreHandle, PocketSyncPolicy,
+ TANGLE_GROUP_CHECKPOINT_TABLE, TANGLE_GROUP_OUTBOX_TABLE, TANGLE_GROUP_PROJECTION_TABLE,
+ parse_pocket_event_json,
};
use tangle_test_support::{
FixtureKey, TANGLE_V2_RELAY_SECRET_HEX, TANGLE_V2_RELAY_URL, tangle_v2_auth_event,
@@ -40,7 +41,8 @@ use tangle_test_support::{
#[test]
fn public_relay_smoke_stores_queries_counts_and_fans_out() {
let config = test_store_config("public-smoke");
- let mut relay = BaseRelay::open(&config, relay_limits(4)).expect("relay");
+ let mut relay =
+ BaseRelay::open(&config, relay_limits(4), PocketQueryConfig::default()).expect("relay");
let first =
tangle_v2_event(FixtureKey::Member, 1_714_124_433, 1, Vec::new(), "hello").expect("first");
let query_id = subscription("public-query");
@@ -177,7 +179,13 @@ fn auth_integration_covers_challenge_edges() {
fn group_auth_lifecycle_membership_and_flag_flows_pass_in_process() {
let config = test_store_config("group-flows");
let groups = group_config_with_public_join();
- let mut relay = BaseRelay::open_with_groups(&config, relay_limits(8), &groups).expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &groups,
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
let owner_auth = authenticated(FixtureKey::Owner);
let admin_auth = authenticated(FixtureKey::Admin);
let member_auth = authenticated(FixtureKey::Member);
@@ -301,8 +309,13 @@ fn group_auth_lifecycle_membership_and_flag_flows_pass_in_process() {
#[test]
fn relay_override_role_changes_generate_admin_snapshots() {
let config = test_store_config("role-admin-snapshots");
- let mut relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
let owner_auth = authenticated(FixtureKey::Owner);
let admin_auth = authenticated(FixtureKey::Admin);
let member = FixtureKey::Member.public_key().as_str().to_owned();
@@ -374,8 +387,13 @@ fn relay_override_role_changes_generate_admin_snapshots() {
#[test]
fn group_join_requests_are_denied_by_default() {
let config = test_store_config("group-public-join-default");
- let mut relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
let owner_auth = authenticated(FixtureKey::Owner);
let outsider_auth = authenticated(FixtureKey::Outsider);
let create = tangle_v2_group_create_event(FixtureKey::Owner, "Farm", 1, &[]).expect("create");
@@ -408,8 +426,13 @@ fn group_join_requests_are_denied_by_default() {
#[test]
fn metadata_flags_and_read_privacy_cover_req_count_and_fanout() {
let config = test_store_config("privacy-flags");
- let mut relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
let owner_auth = authenticated(FixtureKey::Owner);
let outsider_auth = authenticated(FixtureKey::Outsider);
@@ -573,8 +596,13 @@ fn metadata_flags_and_read_privacy_cover_req_count_and_fanout() {
#[test]
fn nip29_privacy_leak_suite_covers_relay_exposure_and_rejection_paths() {
let config = test_store_config("nip29-leak-suite");
- let mut relay =
- BaseRelay::open_with_groups(&config, relay_limits(16), &group_config()).expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(16),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
let owner_auth = authenticated(FixtureKey::Owner);
let admin_auth = authenticated(FixtureKey::Admin);
let member_auth = authenticated(FixtureKey::Member);
@@ -953,8 +981,13 @@ fn nip29_privacy_leak_suite_covers_relay_exposure_and_rejection_paths() {
#[test]
fn delete_and_secondary_privacy_surfaces_are_read_gated_or_absent() {
let config = test_store_config("delete-privacy");
- let mut relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
let owner_auth = authenticated(FixtureKey::Owner);
accept_group_create(&mut relay, "DeleteFarm", &[], 1, &owner_auth);
@@ -1037,8 +1070,13 @@ fn projection_rebuild_after_restart_matches_live_state_and_outbox_is_idempotent(
let config = test_store_config("projection-restart");
let owner_auth = authenticated(FixtureKey::Owner);
{
- let mut relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
accept_group_create(&mut relay, "RestartFarm", &[], 1, &owner_auth);
let put = tangle_v2_put_user_event(FixtureKey::Admin, "RestartFarm", FixtureKey::Member, 2)
.expect("put");
@@ -1056,8 +1094,13 @@ fn projection_rebuild_after_restart_matches_live_state_and_outbox_is_idempotent(
}
delete_group_extra_records(&config);
- let relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("reopen");
+ let relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("reopen");
assert_eq!(
relay
.readiness_state()
@@ -1100,8 +1143,13 @@ fn projection_rebuild_after_restart_matches_live_state_and_outbox_is_idempotent(
&GroupCheckpointStatus::Current { .. }
));
- let relay = BaseRelay::open_with_groups(&config, relay_limits(8), &group_config())
- .expect("second reopen");
+ let relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("second reopen");
assert_eq!(count_kind(&relay, KIND_GROUP_METADATA), 1);
assert_eq!(count_kind(&relay, KIND_GROUP_ADMINS), 1);
assert_eq!(count_kind(&relay, KIND_GROUP_MEMBERS), 1);
@@ -1117,8 +1165,13 @@ fn projection_applies_canonical_events_after_checkpoint_on_restart() {
let put = tangle_v2_put_user_event(FixtureKey::Admin, "IncrementalFarm", FixtureKey::Member, 2)
.expect("put");
{
- let mut relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
assert_accepted(
relay
.handle_event_with_auth(create.clone(), &owner_auth)
@@ -1136,8 +1189,13 @@ fn projection_applies_canonical_events_after_checkpoint_on_restart() {
let create_offset = stored_event_offset(&config, &create);
regress_member_projection_to_checkpoint(&config, create_offset, "IncrementalFarm");
- let relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("reopen");
+ let relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("reopen");
assert_eq!(
relay
.group_projection()
@@ -1168,8 +1226,13 @@ fn source_store_crash_recovery_rebuilds_projection_outbox_and_generated_events()
store_source_events(&config, &[create, put]);
- let relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("reopen");
+ let relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("reopen");
assert_eq!(
relay
.readiness_state()
@@ -1224,8 +1287,13 @@ fn rebuilt_projection_matches_live_projection_for_moderation_stream() {
let members_before;
{
- let mut relay =
- BaseRelay::open_with_groups(&config, relay_limits(16), &group_config()).expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(16),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
accept_group_create(&mut relay, "EquivFarm", &[], 1, &owner_auth);
let metadata =
tangle_v2_group_metadata_event(FixtureKey::Admin, "EquivFarm", "Market", 2, &[])
@@ -1304,8 +1372,13 @@ fn rebuilt_projection_matches_live_projection_for_moderation_stream() {
delete_group_extra_records(&config);
- let relay =
- BaseRelay::open_with_groups(&config, relay_limits(16), &group_config()).expect("reopen");
+ let relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(16),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("reopen");
assert_projection_without_checkpoint_eq(
&live_projection,
relay.group_projection().expect("projection"),
@@ -1329,8 +1402,13 @@ fn pending_and_retryable_group_outbox_records_materialize_on_restart() {
let config = test_store_config("outbox-retryable-restart");
let owner_auth = authenticated(FixtureKey::Owner);
{
- let mut relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
accept_group_create(&mut relay, "OutboxFarm", &[], 1, &owner_auth);
relay.shutdown().expect("shutdown");
}
@@ -1338,8 +1416,13 @@ fn pending_and_retryable_group_outbox_records_materialize_on_restart() {
assert_eq!(outbox_status_counts(&config).pending, 1);
assert_eq!(outbox_status_counts(&config).retryable, 1);
- let relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("reopen");
+ let relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("reopen");
assert_eq!(
relay
.readiness_state()
@@ -1360,9 +1443,13 @@ fn pending_and_retryable_group_outbox_records_materialize_on_restart() {
fn max_outbox_replay_batch_one_drains_all_pending_generated_records() {
let config = test_store_config("outbox-batch-one");
let owner_auth = authenticated(FixtureKey::Owner);
- let mut relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config_with_outbox_batch(1))
- .expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config_with_outbox_batch(1),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
accept_group_create(&mut relay, "BatchFarm", &[], 1, &owner_auth);
@@ -1379,8 +1466,13 @@ fn already_stored_generated_events_mark_outbox_stored_without_duplication_on_res
let config = test_store_config("outbox-generated-already-stored");
let owner_auth = authenticated(FixtureKey::Owner);
{
- let mut relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
accept_group_create(&mut relay, "StoredGeneratedFarm", &[], 1, &owner_auth);
relay.shutdown().expect("shutdown");
}
@@ -1392,8 +1484,13 @@ fn already_stored_generated_events_mark_outbox_stored_without_duplication_on_res
regress_outbox_records_to_pending(&config);
assert_eq!(outbox_status_counts(&config).pending, 2);
- let relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("reopen");
+ let relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("reopen");
assert_eq!(count_kind(&relay, KIND_GROUP_METADATA), 1);
assert_eq!(count_kind(&relay, KIND_GROUP_ADMINS), 1);
assert_eq!(
@@ -1416,8 +1513,13 @@ fn crash_point_recovery_states_match_live_projection_and_generated_events() {
let pending_outbox_config = test_store_config("crash-equivalence-pending-outbox");
let events = recovery_equivalence_events();
let expected = {
- let mut relay = BaseRelay::open_with_groups(&live_config, relay_limits(8), &group_config())
- .expect("live");
+ let mut relay = BaseRelay::open_with_groups(
+ &live_config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("live");
let owner_auth = authenticated(FixtureKey::Owner);
let admin_auth = authenticated(FixtureKey::Admin);
assert_accepted(
@@ -1438,17 +1540,25 @@ fn crash_point_recovery_states_match_live_projection_and_generated_events() {
};
store_source_events(&source_only_config, &events);
- let mut source_only =
- BaseRelay::open_with_groups(&source_only_config, relay_limits(8), &group_config())
- .expect("source only");
+ let mut source_only = BaseRelay::open_with_groups(
+ &source_only_config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("source only");
assert_eq!(recovery_summary(&mut source_only, "CrashFarm"), expected);
assert_eq!(outbox_status_counts(&source_only_config).stored, 5);
let offsets = store_source_events(&pending_outbox_config, &events);
seed_pending_create_outbox_records(&pending_outbox_config, &events[0], offsets[0]);
- let mut pending_outbox =
- BaseRelay::open_with_groups(&pending_outbox_config, relay_limits(8), &group_config())
- .expect("pending outbox");
+ let mut pending_outbox = BaseRelay::open_with_groups(
+ &pending_outbox_config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("pending outbox");
assert_eq!(recovery_summary(&mut pending_outbox, "CrashFarm"), expected);
let counts = outbox_status_counts(&pending_outbox_config);
assert_eq!(counts.pending, 0);
@@ -1763,8 +1873,13 @@ fn sorted_strings(values: impl IntoIterator<Item = String>) -> Vec<String> {
fn final_group_name_for_order(name: &str, edits: [&Event; 2]) -> String {
let config = test_store_config(name);
- let mut relay =
- BaseRelay::open_with_groups(&config, relay_limits(8), &group_config()).expect("relay");
+ let mut relay = BaseRelay::open_with_groups(
+ &config,
+ relay_limits(8),
+ &group_config(),
+ PocketQueryConfig::default(),
+ )
+ .expect("relay");
let auth = authenticated(FixtureKey::Owner);
accept_group_create(&mut relay, "ClockFarm", &[], 1, &auth);
for edit in edits {
@@ -1981,7 +2096,12 @@ fn runtime_config(groups_enabled: bool) -> BaseRelayRuntimeConfig {
},
"pocket": {
"data_directory": "runtime/pocket",
- "sync_policy": "flush_on_shutdown"
+ "sync_policy": "flush_on_shutdown",
+ "query": {
+ "allow_scraping": false,
+ "allow_scrape_if_limited_to": 100,
+ "allow_scrape_if_max_seconds": 3600
+ }
},
"groups": groups,
"auth": {
diff --git a/crates/tangle_runtime/tests/ops_truthfulness.rs b/crates/tangle_runtime/tests/ops_truthfulness.rs
@@ -136,7 +136,12 @@ fn runtime_config(root: &Path) -> BaseRelayRuntimeConfig {
},
"pocket": {
"data_directory": root.join("pocket"),
- "sync_policy": "flush_on_shutdown"
+ "sync_policy": "flush_on_shutdown",
+ "query": {
+ "allow_scraping": false,
+ "allow_scrape_if_limited_to": 100,
+ "allow_scrape_if_max_seconds": 3600
+ }
},
"groups": {
"enabled": true,
diff --git a/crates/tangle_runtime/tests/phase2_acceptance_targets.rs b/crates/tangle_runtime/tests/phase2_acceptance_targets.rs
@@ -1605,7 +1605,12 @@ fn runtime_config_value(root: &Path, listen_addr: SocketAddr) -> Value {
},
"pocket": {
"data_directory": root.join("pocket"),
- "sync_policy": "flush_on_shutdown"
+ "sync_policy": "flush_on_shutdown",
+ "query": {
+ "allow_scraping": false,
+ "allow_scrape_if_limited_to": 100,
+ "allow_scrape_if_max_seconds": 3600
+ }
},
"groups": {
"enabled": true,
diff --git a/crates/tangle_store_pocket/src/lib.rs b/crates/tangle_store_pocket/src/lib.rs
@@ -93,6 +93,45 @@ pub const TANGLE_POCKET_EXTRA_TABLES: [&str; 3] = [
];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct PocketQueryConfig {
+ allow_scraping: bool,
+ allow_scrape_if_limited_to: u32,
+ allow_scrape_if_max_seconds: u64,
+}
+
+impl PocketQueryConfig {
+ pub const fn new(
+ allow_scraping: bool,
+ allow_scrape_if_limited_to: u32,
+ allow_scrape_if_max_seconds: u64,
+ ) -> Self {
+ Self {
+ allow_scraping,
+ allow_scrape_if_limited_to,
+ allow_scrape_if_max_seconds,
+ }
+ }
+
+ pub fn allow_scraping(self) -> bool {
+ self.allow_scraping
+ }
+
+ pub fn allow_scrape_if_limited_to(self) -> u32 {
+ self.allow_scrape_if_limited_to
+ }
+
+ pub fn allow_scrape_if_max_seconds(self) -> u64 {
+ self.allow_scrape_if_max_seconds
+ }
+}
+
+impl Default for PocketQueryConfig {
+ fn default() -> Self {
+ Self::new(false, 100, 3_600)
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PocketDependencyBoundary {
source_repository: &'static str,
source_revision: &'static str,
@@ -173,17 +212,16 @@ impl PocketStoreHandle {
pub fn find_events(
&self,
filter: &PocketFilter,
+ query: PocketQueryConfig,
) -> Result<Vec<PocketOwnedEvent>, PocketStoreError> {
- self.find_events_with_screen(filter, true, 0, u64::MAX, |_| PocketScreenResult::Match)
+ self.find_events_with_screen(filter, query, |_| PocketScreenResult::Match)
.map(PocketScreenedEvents::into_events)
}
pub fn find_events_with_screen<F>(
&self,
filter: &PocketFilter,
- allow_scraping: bool,
- allow_scrape_if_limited_to: u32,
- allow_scrape_if_max_seconds: u64,
+ query: PocketQueryConfig,
screen: F,
) -> Result<PocketScreenedEvents, PocketStoreError>
where
@@ -193,9 +231,9 @@ impl PocketStoreHandle {
.store
.find_events(
filter,
- allow_scraping,
- allow_scrape_if_limited_to,
- allow_scrape_if_max_seconds,
+ query.allow_scraping(),
+ query.allow_scrape_if_limited_to(),
+ query.allow_scrape_if_max_seconds(),
screen,
)
.map_err(PocketStoreError::from_pocket)?;
@@ -205,8 +243,12 @@ impl PocketStoreHandle {
))
}
- pub fn count_events(&self, filter: &PocketFilter) -> Result<u64, PocketStoreError> {
- self.find_events(filter)
+ pub fn count_events(
+ &self,
+ filter: &PocketFilter,
+ query: PocketQueryConfig,
+ ) -> Result<u64, PocketStoreError> {
+ self.find_events(filter, query)
.map(|events| u64::try_from(events.len()).expect("usize count fits in u64"))
}
@@ -517,9 +559,9 @@ fn event_len_u64(event: &PocketEvent) -> Result<u64, PocketStoreError> {
mod tests {
use super::{
POCKET_SOURCE_REPOSITORY, POCKET_SOURCE_REVISION, PocketDependencyBoundary,
- PocketStoreConfig, PocketStoreHandle, PocketSyncPolicy, TANGLE_GROUP_CHECKPOINT_TABLE,
- TANGLE_GROUP_OUTBOX_TABLE, TANGLE_GROUP_PROJECTION_TABLE, TANGLE_POCKET_EXTRA_TABLES,
- parse_pocket_event_json, parse_pocket_filter_json,
+ PocketQueryConfig, PocketStoreConfig, PocketStoreHandle, PocketSyncPolicy,
+ TANGLE_GROUP_CHECKPOINT_TABLE, TANGLE_GROUP_OUTBOX_TABLE, TANGLE_GROUP_PROJECTION_TABLE,
+ TANGLE_POCKET_EXTRA_TABLES, parse_pocket_event_json, parse_pocket_filter_json,
};
use pocket_db::ScreenResult;
use std::time::{SystemTime, UNIX_EPOCH};
@@ -575,13 +617,20 @@ mod tests {
.expect("lookup")
.expect("event");
let offset_event = handle.event_by_offset(offset).expect("offset lookup");
- let found = handle.find_events(&filter).expect("find");
+ let found = handle
+ .find_events(&filter, PocketQueryConfig::default())
+ .expect("find");
assert_eq!(stored.id(), event.id());
assert_eq!(offset_event.id(), event.id());
assert_eq!(found.len(), 1);
assert_eq!(found[0].id(), event.id());
- assert_eq!(handle.count_events(&filter).expect("count"), 1);
+ assert_eq!(
+ handle
+ .count_events(&filter, PocketQueryConfig::default())
+ .expect("count"),
+ 1
+ );
let _ = std::fs::remove_dir_all(root);
}
@@ -632,7 +681,7 @@ mod tests {
handle.store_event(&redacted).expect("store redacted");
let screened = handle
- .find_events_with_screen(&filter, true, 0, u64::MAX, |event| {
+ .find_events_with_screen(&filter, PocketQueryConfig::default(), |event| {
if event.id() == visible.id() {
ScreenResult::Match
} else {
@@ -649,6 +698,35 @@ mod tests {
}
#[test]
+ fn pocket_store_query_config_controls_scraping() {
+ let root = temp_root("tangle-pocket-query-config");
+ let config = PocketStoreConfig::new(root.join("pocket"), PocketSyncPolicy::FlushOnShutdown)
+ .expect("config");
+ let handle = PocketStoreHandle::open(&config).expect("open");
+ let event =
+ parse_pocket_event_json(event_json_with("f", "6", "scrape").as_bytes()).expect("event");
+ let broad = parse_pocket_filter_json(r#"{"limit":1}"#.as_bytes()).expect("filter");
+
+ handle.store_event(&event).expect("store");
+
+ assert!(
+ handle
+ .find_events(&broad, PocketQueryConfig::new(false, 0, 0))
+ .expect_err("scrape rejected")
+ .message()
+ .contains("scraper")
+ );
+ let found = handle
+ .find_events(&broad, PocketQueryConfig::new(false, 1, 0))
+ .expect("limited scrape");
+
+ assert_eq!(found.len(), 1);
+ assert_eq!(found[0].id(), event.id());
+
+ let _ = std::fs::remove_dir_all(root);
+ }
+
+ #[test]
fn pocket_store_handle_persists_extra_table_records() {
let root = temp_root("tangle-pocket-extra");
let config = PocketStoreConfig::new(root.join("pocket"), PocketSyncPolicy::FlushOnShutdown)
