projection.rs (18535B)
1 use std::sync::{ 2 Arc, Mutex, 3 atomic::{AtomicU64, Ordering}, 4 }; 5 6 use futures_executor::block_on; 7 use radroots_event::{ 8 SignedEvent, 9 admission::{AdmissionPolicy, RawEvent, SignatureVerifier, VisibilityPolicy}, 10 draft::SignedEventParts, 11 envelope::EventEnvelope, 12 wire::compute_canonical_nip01_event_id, 13 }; 14 use radroots_storage::{ 15 EventStore, ProjectionStore, 16 event::{EventAdmission, SourceGeneration, StoredVisibleEvent}, 17 memory::MemoryStorage, 18 projection::{ 19 InvalidationReason, ProjectionGeneration, ProjectionHealth, ProjectionId, 20 ProjectionInvalidation, RawSourceDigest, RebuildFailure, RebuildTicketId, 21 }, 22 }; 23 use radroots_sync::{ 24 Engine, 25 policy::{Clock, DeadlinePolicy, Error, IdSource, OperationKind, SyncId, SyncStorage}, 26 projection::{Reducer, ReducerError, RefreshKind, RefreshRequest, RefreshState}, 27 }; 28 use radroots_transport::{ 29 Error as TransportError, EventSource, FetchPage, FetchRequest, SourceStatus, Target, 30 TransportId, 31 source::{EventProvenance, ObservedEvent}, 32 }; 33 34 const PUBKEY: &str = "585591529da0bab31b3b1b1f986611cf5f435dca84f978c89ee8a40cca7103df"; 35 const CONTENT: &str = "{\"display_name\":\"Moss Street Farm\",\"bot\":false}"; 36 37 struct MockSource; 38 39 impl EventSource for MockSource { 40 fn status(&self) -> radroots_transport::BoxFuture<'_, Result<SourceStatus, TransportError>> { 41 Box::pin(async { unreachable!("projection refresh does not inspect source") }) 42 } 43 44 fn fetch( 45 &self, 46 _request: FetchRequest, 47 ) -> radroots_transport::BoxFuture<'_, Result<FetchPage, TransportError>> { 48 Box::pin(async { unreachable!("projection refresh does not fetch") }) 49 } 50 } 51 52 struct TestClock(AtomicU64); 53 54 impl Clock for TestClock { 55 fn now_unix_ms(&self) -> Result<u64, Error> { 56 Ok(self.0.fetch_add(1, Ordering::Relaxed)) 57 } 58 } 59 60 struct TestIds(Mutex<u8>); 61 62 impl IdSource for TestIds { 63 fn next_id(&self, operation: OperationKind) -> Result<SyncId, Error> { 64 assert_eq!(operation, OperationKind::Projection); 65 let mut value = self.0.lock().expect("ids"); 66 let current = *value; 67 *value += 1; 68 SyncId::new([current; 16]) 69 } 70 } 71 72 struct Allow; 73 74 impl SignatureVerifier for Allow { 75 fn verify_signature(&self, _event: &EventEnvelope) -> Result<(), radroots_event::Error> { 76 Ok(()) 77 } 78 } 79 80 impl AdmissionPolicy for Allow { 81 type Error = core::convert::Infallible; 82 fn policy_id(&self) -> &'static str { 83 "test.projection.admission.v1" 84 } 85 fn admit( 86 &self, 87 _event: &radroots_event::admission::ContractValidatedEvent, 88 ) -> Result<(), Self::Error> { 89 Ok(()) 90 } 91 } 92 93 impl VisibilityPolicy for Allow { 94 type Error = core::convert::Infallible; 95 fn policy_id(&self) -> &'static str { 96 "test.projection.visibility.v1" 97 } 98 fn make_visible( 99 &self, 100 _event: &radroots_event::admission::AdmittedEvent, 101 ) -> Result<(), Self::Error> { 102 Ok(()) 103 } 104 } 105 106 struct CountingReducer { 107 projection_id: ProjectionId, 108 generation: ProjectionGeneration, 109 fail: bool, 110 regress: bool, 111 } 112 113 impl Reducer for CountingReducer { 114 fn projection_id(&self) -> &ProjectionId { 115 &self.projection_id 116 } 117 fn generation(&self) -> ProjectionGeneration { 118 self.generation 119 } 120 fn begin_rebuild( 121 &self, 122 _ticket_id: RebuildTicketId, 123 _source_generation: SourceGeneration, 124 _source_digest: RawSourceDigest, 125 ) -> Result<(), ReducerError> { 126 if self.fail { Err(ReducerError) } else { Ok(()) } 127 } 128 fn reduce( 129 &self, 130 events: &[StoredVisibleEvent], 131 prior_projected_rows: u64, 132 _rebuild_ticket: Option<RebuildTicketId>, 133 ) -> Result<u64, ReducerError> { 134 if self.fail { 135 return Err(ReducerError); 136 } 137 if self.regress { 138 return Ok(prior_projected_rows.saturating_sub(1)); 139 } 140 prior_projected_rows 141 .checked_add(u64::try_from(events.len()).expect("event count")) 142 .ok_or(ReducerError) 143 } 144 fn abort_rebuild( 145 &self, 146 _ticket_id: RebuildTicketId, 147 _failure: RebuildFailure, 148 ) -> Result<(), ReducerError> { 149 Ok(()) 150 } 151 } 152 153 fn setup() -> (Engine, Arc<MemoryStorage>, ProjectionId) { 154 let storage = Arc::new(MemoryStorage::new( 155 SourceGeneration::new([9; 32]).expect("generation"), 156 )); 157 let storage_capability: Arc<dyn SyncStorage> = storage.clone(); 158 let engine = Engine::builder( 159 storage_capability, 160 Arc::new(TestClock(AtomicU64::new(1_000))), 161 Arc::new(TestIds(Mutex::new(1))), 162 DeadlinePolicy::new(100, 100, 100).expect("deadlines"), 163 ) 164 .source(Arc::new(MockSource)) 165 .build() 166 .expect("engine"); 167 ( 168 engine, 169 storage, 170 ProjectionId::parse("test.projection").expect("projection id"), 171 ) 172 } 173 174 fn signed_event(created_at: u64) -> SignedEvent { 175 let tags: Vec<Vec<String>> = vec![]; 176 let id = compute_canonical_nip01_event_id(PUBKEY, created_at, 1, &tags, CONTENT) 177 .expect("event id") 178 .to_hex(); 179 let signature = "42".repeat(64); 180 let raw_json = format!( 181 "{{\"id\":\"{id}\",\"pubkey\":\"{PUBKEY}\",\"created_at\":{created_at},\"kind\":1,\"tags\":[],\"content\":{content:?},\"sig\":\"{signature}\"}}", 182 content = CONTENT, 183 ); 184 SignedEvent::new(SignedEventParts { 185 id, 186 pubkey: PUBKEY.to_owned(), 187 created_at, 188 kind: 1, 189 tags, 190 content: CONTENT.to_owned(), 191 sig: signature, 192 raw_json, 193 }) 194 .expect("signed event") 195 } 196 197 fn seed(storage: &MemoryStorage, count: u64) { 198 for offset in 0..count { 199 let event = signed_event(1_800_000_100 + offset); 200 let visible = RawEvent::new(event.envelope().clone()) 201 .verify_id() 202 .expect("id") 203 .verify_signature(&Allow) 204 .expect("signature") 205 .validate_contract() 206 .expect("contract") 207 .admit_with(&Allow) 208 .expect("admission") 209 .make_visible_with(&Allow) 210 .expect("visibility"); 211 let target = Target::new(TransportId::NOSTR, "wss://relay.example").expect("target"); 212 let provenance = EventProvenance::new( 213 TransportId::NOSTR, 214 target.fingerprint().clone(), 215 1_900_000_000_000 + offset, 216 ) 217 .expect("provenance"); 218 block_on( 219 storage.admit( 220 EventAdmission::visible(ObservedEvent::new(event, provenance), visible) 221 .expect("visible admission"), 222 ), 223 ) 224 .expect("seed event"); 225 } 226 } 227 228 fn reducer(id: &ProjectionId, generation: u8, fail: bool) -> CountingReducer { 229 CountingReducer { 230 projection_id: id.clone(), 231 generation: ProjectionGeneration::new([generation; 32]).expect("generation"), 232 fail, 233 regress: false, 234 } 235 } 236 237 #[test] 238 fn incremental_refresh_checkpoints_visible_events() { 239 let (engine, storage, id) = setup(); 240 seed(&storage, 1); 241 let reducer = reducer(&id, 1, false); 242 let request = RefreshRequest::new(id.clone(), reducer.generation(), 10, 1).expect("request"); 243 assert_eq!(request.projection_id(), &id); 244 assert_eq!(request.generation(), reducer.generation()); 245 assert_eq!(request.batch_limit(), 10); 246 assert_eq!(request.max_batches(), 1); 247 let receipt = block_on(engine.refresh_projection(request, &reducer)).expect("refresh"); 248 assert_eq!(receipt.kind(), RefreshKind::Incremental); 249 assert_eq!(receipt.state(), RefreshState::Complete); 250 assert_eq!(receipt.events_reduced(), 1); 251 assert_eq!(receipt.batches(), 1); 252 assert!(receipt.rebuild_ticket().is_none()); 253 assert_eq!( 254 receipt.checkpoint().expect("checkpoint").projected_rows(), 255 1 256 ); 257 assert_eq!( 258 block_on(ProjectionStore::status(&*storage, id)) 259 .expect("status") 260 .expect("projection") 261 .health(), 262 ProjectionHealth::Ready 263 ); 264 } 265 266 #[cfg(feature = "serde")] 267 #[test] 268 fn serialized_refresh_requests_enforce_constructor_bounds() { 269 use radroots_storage::event::EVENT_QUERY_LIMIT_MAX; 270 use radroots_sync::projection::PROJECTION_REFRESH_MAX_BATCHES; 271 272 let id = ProjectionId::parse("wire.projection").unwrap(); 273 let generation = ProjectionGeneration::new([2; 32]).unwrap(); 274 let original = RefreshRequest::new(id.clone(), generation, 1, 1).unwrap(); 275 let wire = serde_json::to_value(&original).unwrap(); 276 for (batch_limit, max_batches) in [ 277 (0, 1), 278 (EVENT_QUERY_LIMIT_MAX + 1, 1), 279 (1, 0), 280 (1, PROJECTION_REFRESH_MAX_BATCHES + 1), 281 ] { 282 assert!(RefreshRequest::new(id.clone(), generation, batch_limit, max_batches).is_err()); 283 let mut changed = wire.clone(); 284 changed["batch_limit"] = serde_json::json!(batch_limit); 285 changed["max_batches"] = serde_json::json!(max_batches); 286 assert!( 287 serde_json::from_value::<RefreshRequest>(changed).is_err(), 288 "batch_limit={batch_limit}, max_batches={max_batches}" 289 ); 290 } 291 for (batch_limit, max_batches) in [ 292 (1, 1), 293 (EVENT_QUERY_LIMIT_MAX, 1), 294 (1, PROJECTION_REFRESH_MAX_BATCHES), 295 (EVENT_QUERY_LIMIT_MAX, PROJECTION_REFRESH_MAX_BATCHES), 296 ] { 297 let request = 298 RefreshRequest::new(id.clone(), generation, batch_limit, max_batches).unwrap(); 299 let mut wire = serde_json::to_value(&request).unwrap(); 300 assert_eq!( 301 serde_json::from_value::<RefreshRequest>(wire.clone()).unwrap(), 302 request 303 ); 304 wire["application_extension"] = serde_json::json!(true); 305 assert_eq!( 306 serde_json::from_value::<RefreshRequest>(wire).unwrap(), 307 request, 308 "the existing unknown-field policy is unchanged" 309 ); 310 } 311 } 312 313 #[test] 314 fn generation_change_rebuilds_and_reducer_failure_is_durable() { 315 let (engine, storage, id) = setup(); 316 seed(&storage, 1); 317 let first = reducer(&id, 1, false); 318 block_on(engine.refresh_projection( 319 RefreshRequest::new(id.clone(), first.generation(), 10, 1).expect("request"), 320 &first, 321 )) 322 .expect("initial refresh"); 323 324 let replacement = reducer(&id, 2, false); 325 let rebuilt = block_on(engine.refresh_projection( 326 RefreshRequest::new(id.clone(), replacement.generation(), 10, 1).expect("request"), 327 &replacement, 328 )) 329 .expect("rebuild"); 330 assert_eq!(rebuilt.kind(), RefreshKind::Rebuild); 331 assert_eq!(rebuilt.state(), RefreshState::Complete); 332 333 let failing = reducer(&id, 3, true); 334 let failed = block_on(engine.refresh_projection( 335 RefreshRequest::new(id.clone(), failing.generation(), 10, 1).expect("request"), 336 &failing, 337 )) 338 .expect("normalized failure"); 339 assert_eq!(failed.state(), RefreshState::Failed); 340 assert_eq!( 341 block_on(ProjectionStore::status(&*storage, id)) 342 .expect("status") 343 .expect("projection") 344 .health(), 345 ProjectionHealth::Ready 346 ); 347 let retried_failure = block_on( 348 engine.refresh_projection( 349 RefreshRequest::new(failing.projection_id().clone(), failing.generation(), 10, 1) 350 .expect("failed generation request"), 351 &failing, 352 ), 353 ) 354 .expect("retry failed generation"); 355 assert_eq!(retried_failure.state(), RefreshState::Failed); 356 } 357 358 #[test] 359 fn obsolete_generation_request_fails_closed_after_invalidation() { 360 let (engine, storage, id) = setup(); 361 seed(&storage, 1); 362 let active = reducer(&id, 1, false); 363 block_on(engine.refresh_projection( 364 RefreshRequest::new(id.clone(), active.generation(), 10, 1).expect("request"), 365 &active, 366 )) 367 .expect("initial refresh"); 368 369 let replacement = ProjectionGeneration::new([2; 32]).expect("replacement generation"); 370 block_on(ProjectionStore::invalidate( 371 &*storage, 372 ProjectionInvalidation::new( 373 id.clone(), 374 active.generation(), 375 replacement, 376 InvalidationReason::ProjectionGenerationChanged, 377 2_000, 378 ) 379 .expect("invalidation"), 380 )) 381 .expect("invalidate active generation"); 382 383 assert_eq!( 384 block_on(engine.refresh_projection( 385 RefreshRequest::new(id, active.generation(), 10, 1).expect("obsolete request"), 386 &active, 387 )), 388 Err(Error::StorageFailed) 389 ); 390 } 391 392 #[test] 393 fn partial_rebuild_resumes_and_rejects_concurrent_generation() { 394 let (engine, storage, id) = setup(); 395 seed(&storage, 2); 396 let first = reducer(&id, 1, false); 397 block_on(engine.refresh_projection( 398 RefreshRequest::new(id.clone(), first.generation(), 10, 1).expect("request"), 399 &first, 400 )) 401 .expect("initial refresh"); 402 403 let replacement = reducer(&id, 2, false); 404 let partial = block_on(engine.refresh_projection( 405 RefreshRequest::new(id.clone(), replacement.generation(), 1, 1).expect("request"), 406 &replacement, 407 )) 408 .expect("partial rebuild"); 409 assert_eq!(partial.state(), RefreshState::Partial); 410 assert!(partial.rebuild_ticket().is_some()); 411 let visible_status = block_on(ProjectionStore::status(&*storage, id.clone())) 412 .expect("status") 413 .expect("projection"); 414 assert_eq!(visible_status.generation(), first.generation()); 415 assert_eq!(visible_status.health(), ProjectionHealth::Rebuilding); 416 417 let concurrent = reducer(&id, 3, false); 418 assert_eq!( 419 block_on(engine.refresh_projection( 420 RefreshRequest::new(id.clone(), concurrent.generation(), 1, 1).expect("request"), 421 &concurrent, 422 )), 423 Err(Error::StorageConflict) 424 ); 425 426 let second = block_on(engine.refresh_projection( 427 RefreshRequest::new(id.clone(), replacement.generation(), 1, 1).expect("request"), 428 &replacement, 429 )) 430 .expect("second batch"); 431 assert_eq!(second.state(), RefreshState::Partial); 432 let complete = block_on(engine.refresh_projection( 433 RefreshRequest::new(id.clone(), replacement.generation(), 1, 1).expect("request"), 434 &replacement, 435 )) 436 .expect("complete rebuild"); 437 assert_eq!(complete.state(), RefreshState::Complete); 438 for invalid in [ 439 RefreshRequest::new(id.clone(), replacement.generation(), 0, 1), 440 RefreshRequest::new( 441 id.clone(), 442 replacement.generation(), 443 radroots_storage::event::EVENT_QUERY_LIMIT_MAX + 1, 444 1, 445 ), 446 RefreshRequest::new(id.clone(), replacement.generation(), 1, 0), 447 RefreshRequest::new( 448 id, 449 replacement.generation(), 450 1, 451 radroots_sync::projection::PROJECTION_REFRESH_MAX_BATCHES + 1, 452 ), 453 ] { 454 assert_eq!(invalid, Err(Error::InvalidProjectionRequest)); 455 } 456 } 457 458 #[test] 459 fn source_change_fails_rebuild_and_preserves_prior_generation() { 460 let (engine, storage, id) = setup(); 461 seed(&storage, 2); 462 let active = reducer(&id, 1, false); 463 block_on(engine.refresh_projection( 464 RefreshRequest::new(id.clone(), active.generation(), 10, 1).expect("request"), 465 &active, 466 )) 467 .expect("initial refresh"); 468 469 let replacement = reducer(&id, 2, false); 470 let partial = block_on(engine.refresh_projection( 471 RefreshRequest::new(id.clone(), replacement.generation(), 1, 1).expect("request"), 472 &replacement, 473 )) 474 .expect("partial rebuild"); 475 let ticket_id = partial.rebuild_ticket().expect("ticket"); 476 seed(&storage, 3); 477 478 let failed = block_on(engine.refresh_projection( 479 RefreshRequest::new(id.clone(), replacement.generation(), 1, 1).expect("request"), 480 &replacement, 481 )) 482 .expect("source change is normalized"); 483 assert_eq!(failed.state(), RefreshState::Failed); 484 let status = block_on(ProjectionStore::status(&*storage, id)) 485 .expect("status") 486 .expect("projection"); 487 assert_eq!(status.generation(), active.generation()); 488 assert_eq!(status.health(), ProjectionHealth::Ready); 489 let ticket = block_on(storage.rebuild(ticket_id)) 490 .expect("ticket lookup") 491 .expect("durable ticket"); 492 assert_eq!(ticket.failure(), Some(RebuildFailure::SourceChanged)); 493 } 494 495 #[test] 496 fn reducer_identity_progress_and_multi_batch_boundaries_fail_closed() { 497 let (engine, storage, id) = setup(); 498 seed(&storage, 3); 499 let active_reducer = reducer(&id, 1, false); 500 let request = 501 RefreshRequest::new(id.clone(), active_reducer.generation(), 1, 2).expect("request"); 502 let wrong_id = reducer( 503 &ProjectionId::parse("different-projection").expect("projection id"), 504 1, 505 false, 506 ); 507 assert_eq!( 508 block_on(engine.refresh_projection(request.clone(), &wrong_id)), 509 Err(Error::InvalidProjectionRequest) 510 ); 511 let wrong_generation = reducer(&id, 2, false); 512 assert_eq!( 513 block_on(engine.refresh_projection(request.clone(), &wrong_generation)), 514 Err(Error::InvalidProjectionRequest) 515 ); 516 let partial = 517 block_on(engine.refresh_projection(request, &active_reducer)).expect("two batches"); 518 assert_eq!(partial.state(), RefreshState::Partial); 519 assert_eq!(partial.batches(), 2); 520 521 let (engine, storage, id) = setup(); 522 seed(&storage, 1); 523 let failing = reducer(&id, 1, true); 524 let failed = block_on(engine.refresh_projection( 525 RefreshRequest::new(id.clone(), failing.generation(), 1, 1).expect("request"), 526 &failing, 527 )) 528 .expect("normalized incremental failure"); 529 assert_eq!(failed.state(), RefreshState::Failed); 530 assert!(failed.rebuild_ticket().is_none()); 531 532 let (engine, storage, id) = setup(); 533 seed(&storage, 1); 534 let initial = reducer(&id, 1, false); 535 block_on(engine.refresh_projection( 536 RefreshRequest::new(id.clone(), initial.generation(), 1, 1).expect("request"), 537 &initial, 538 )) 539 .expect("initial projection"); 540 seed(&storage, 2); 541 let regressing = CountingReducer { 542 projection_id: id.clone(), 543 generation: initial.generation(), 544 fail: false, 545 regress: true, 546 }; 547 assert_eq!( 548 block_on(engine.refresh_projection( 549 RefreshRequest::new(id, regressing.generation(), 1, 1).expect("request"), 550 ®ressing, 551 )), 552 Err(Error::InvalidReducerOutput) 553 ); 554 }