indexer_determinism.rs (11911B)
1 use radroots_radroots_indexer::config::{Indexer, Listings, Relay, Settings}; 2 use radroots_radroots_indexer::domain::indexer::kind::IndexerEventKind; 3 use radroots_radroots_indexer::domain::indexer::models::{ 4 EventCommentIndexes, EventFollowIndexes, EventIndexes, EventJobFeedbackIndexes, 5 EventJobRequestIndexes, EventJobResultIndexes, EventListingIndexes, EventPostIndexes, 6 EventProfileIndexes, EventReactionIndexes, WriteEventIndexes, 7 }; 8 use radroots_radroots_indexer::domain::resolvers::profile::ProfileResolver; 9 use radroots_radroots_indexer::relay::event::RelayIndexerEvent; 10 use radroots_events::kinds::{KIND_JOB_REQUEST_MIN, KIND_JOB_RESULT_MIN}; 11 use std::path::Path; 12 use tempfile::tempdir; 13 14 fn settings_for(root: &Path) -> Settings { 15 Settings { 16 indexer: Indexer { 17 data_dir: root.join("data").to_string_lossy().to_string(), 18 logs_dir: root.join("logs").to_string_lossy().to_string(), 19 flush_interval: 1, 20 }, 21 relay: Relay { 22 url: String::new(), 23 database_path: String::new(), 24 }, 25 listings: Listings { 26 country_shard_size: 0, 27 profile_shard_size: 0, 28 }, 29 } 30 } 31 32 fn write_ids<T: WriteEventIndexes>( 33 indexes: &T, 34 settings: &Settings, 35 kind: IndexerEventKind, 36 ) -> Vec<String> { 37 let mut updated = Vec::new(); 38 indexes.write(settings, &mut updated).expect("write indexes"); 39 let path = kind 40 .base_path(&settings.indexer.data_dir) 41 .expect("base path") 42 .join("events.json"); 43 let raw = std::fs::read_to_string(path).expect("read events.json"); 44 serde_json::from_str(&raw).expect("parse events.json") 45 } 46 47 fn expected_ids(events: &[RelayIndexerEvent]) -> Vec<String> { 48 let mut refs: Vec<&RelayIndexerEvent> = events.iter().collect(); 49 refs.sort_unstable_by(|a, b| b.created_at.cmp(&a.created_at).then(a.id.cmp(&b.id))); 50 refs.into_iter().map(|e| e.id.clone()).collect() 51 } 52 53 fn assert_deterministic<T, F>( 54 kind: IndexerEventKind, 55 events: Vec<RelayIndexerEvent>, 56 build: F, 57 ) where 58 T: WriteEventIndexes, 59 F: Fn(&[RelayIndexerEvent]) -> T, 60 { 61 let mut reversed = events.clone(); 62 reversed.reverse(); 63 let expected = expected_ids(&events); 64 65 let dir_a = tempdir().expect("tempdir"); 66 let settings_a = settings_for(dir_a.path()); 67 let dir_b = tempdir().expect("tempdir"); 68 let settings_b = settings_for(dir_b.path()); 69 70 let indexes_a = build(&events); 71 let indexes_b = build(&reversed); 72 73 let ids_a = write_ids(&indexes_a, &settings_a, kind); 74 let ids_b = write_ids(&indexes_b, &settings_b, kind); 75 76 assert_eq!(ids_a, expected); 77 assert_eq!(ids_b, expected); 78 } 79 80 fn make_event( 81 id: &str, 82 author: &str, 83 created_at: u32, 84 kind: IndexerEventKind, 85 tags: Vec<Vec<String>>, 86 content: &str, 87 ) -> RelayIndexerEvent { 88 RelayIndexerEvent { 89 id: id.to_string(), 90 author: author.to_string(), 91 created_at, 92 pubkey: author.to_string(), 93 kind, 94 tags, 95 content: content.to_string(), 96 hash: id.to_string(), 97 sig: "sig".to_string(), 98 } 99 } 100 101 fn profile_event(id: &str, author: &str, created_at: u32, nip05: &str) -> RelayIndexerEvent { 102 let content = serde_json::json!({"name": "user", "nip05": nip05}).to_string(); 103 make_event( 104 id, 105 author, 106 created_at, 107 IndexerEventKind::Profile, 108 Vec::new(), 109 &content, 110 ) 111 } 112 113 fn listing_tags(d_tag: &str) -> Vec<Vec<String>> { 114 vec![ 115 vec!["d".to_string(), d_tag.to_string()], 116 vec!["key".to_string(), "key".to_string()], 117 vec!["title".to_string(), "title".to_string()], 118 vec!["category".to_string(), "category".to_string()], 119 ] 120 } 121 122 fn comment_tags(root_id: &str, root_author: &str) -> Vec<Vec<String>> { 123 vec![ 124 vec!["E".to_string(), root_id.to_string()], 125 vec!["K".to_string(), "1".to_string()], 126 vec!["P".to_string(), root_author.to_string()], 127 ] 128 } 129 130 fn reaction_tags(root_id: &str, root_author: &str) -> Vec<Vec<String>> { 131 vec![ 132 vec!["e".to_string(), root_id.to_string()], 133 vec!["k".to_string(), "1".to_string()], 134 vec!["p".to_string(), root_author.to_string()], 135 ] 136 } 137 138 #[test] 139 fn events_json_deterministic_profile() { 140 let author = "a".repeat(64); 141 let events = vec![ 142 profile_event("1".repeat(64).as_str(), &author, 10, "a@radroots.market"), 143 profile_event("2".repeat(64).as_str(), &author, 20, "b@radroots.market"), 144 ]; 145 assert_deterministic(IndexerEventKind::Profile, events, |raw| { 146 EventProfileIndexes::build(raw).expect("build profile") 147 }); 148 } 149 150 #[test] 151 fn events_json_deterministic_listing() { 152 let author = "b".repeat(64); 153 let events = vec![ 154 make_event( 155 "1".repeat(64).as_str(), 156 &author, 157 10, 158 IndexerEventKind::Listing, 159 listing_tags("d1"), 160 "", 161 ), 162 make_event( 163 "2".repeat(64).as_str(), 164 &author, 165 20, 166 IndexerEventKind::Listing, 167 listing_tags("d2"), 168 "", 169 ), 170 ]; 171 let profiles = ProfileResolver::default(); 172 assert_deterministic(IndexerEventKind::Listing, events, |raw| { 173 EventListingIndexes::build_with_profiles(raw, &profiles).expect("build listing") 174 }); 175 } 176 177 #[test] 178 fn events_json_deterministic_comment() { 179 let author = "c".repeat(64); 180 let root_author = "d".repeat(64); 181 let events = vec![ 182 make_event( 183 "1".repeat(64).as_str(), 184 &author, 185 10, 186 IndexerEventKind::Comment, 187 comment_tags("root1", &root_author), 188 "hello", 189 ), 190 make_event( 191 "2".repeat(64).as_str(), 192 &author, 193 20, 194 IndexerEventKind::Comment, 195 comment_tags("root2", &root_author), 196 "hi", 197 ), 198 ]; 199 let profiles = ProfileResolver::default(); 200 assert_deterministic(IndexerEventKind::Comment, events, |raw| { 201 EventCommentIndexes::build_with_profiles(raw, &profiles).expect("build comment") 202 }); 203 } 204 205 #[test] 206 fn events_json_deterministic_reaction() { 207 let author = "e".repeat(64); 208 let root_author = "f".repeat(64); 209 let events = vec![ 210 make_event( 211 "1".repeat(64).as_str(), 212 &author, 213 10, 214 IndexerEventKind::Reaction, 215 reaction_tags("root1", &root_author), 216 "+", 217 ), 218 make_event( 219 "2".repeat(64).as_str(), 220 &author, 221 20, 222 IndexerEventKind::Reaction, 223 reaction_tags("root2", &root_author), 224 "+", 225 ), 226 ]; 227 let profiles = ProfileResolver::default(); 228 assert_deterministic(IndexerEventKind::Reaction, events, |raw| { 229 EventReactionIndexes::build_with_profiles(raw, &profiles).expect("build reaction") 230 }); 231 } 232 233 #[test] 234 fn events_json_deterministic_post() { 235 let author = "1".repeat(64); 236 let events = vec![ 237 make_event( 238 "a".repeat(64).as_str(), 239 &author, 240 10, 241 IndexerEventKind::Post, 242 Vec::new(), 243 "hello", 244 ), 245 make_event( 246 "b".repeat(64).as_str(), 247 &author, 248 20, 249 IndexerEventKind::Post, 250 Vec::new(), 251 "hi", 252 ), 253 ]; 254 let profiles = ProfileResolver::default(); 255 assert_deterministic(IndexerEventKind::Post, events, |raw| { 256 EventPostIndexes::build_with_profiles(raw, &profiles).expect("build post") 257 }); 258 } 259 260 #[test] 261 fn events_json_deterministic_follow() { 262 let author = "2".repeat(64); 263 let follow = "3".repeat(64); 264 let events = vec![ 265 make_event( 266 "a".repeat(64).as_str(), 267 &author, 268 10, 269 IndexerEventKind::Follow, 270 vec![vec!["p".to_string(), follow.clone()]], 271 "", 272 ), 273 make_event( 274 "b".repeat(64).as_str(), 275 &author, 276 20, 277 IndexerEventKind::Follow, 278 vec![vec!["p".to_string(), follow]], 279 "", 280 ), 281 ]; 282 let profiles = ProfileResolver::default(); 283 assert_deterministic(IndexerEventKind::Follow, events, |raw| { 284 EventFollowIndexes::build_with_profiles(raw, &profiles).expect("build follow") 285 }); 286 } 287 288 #[test] 289 fn events_json_deterministic_job_request() { 290 let author = "4".repeat(64); 291 let events = vec![ 292 make_event( 293 "a".repeat(64).as_str(), 294 &author, 295 10, 296 IndexerEventKind::JobRequest(KIND_JOB_REQUEST_MIN), 297 Vec::new(), 298 "", 299 ), 300 make_event( 301 "b".repeat(64).as_str(), 302 &author, 303 20, 304 IndexerEventKind::JobRequest(KIND_JOB_REQUEST_MIN), 305 Vec::new(), 306 "", 307 ), 308 ]; 309 let profiles = ProfileResolver::default(); 310 assert_deterministic( 311 IndexerEventKind::JobRequest(KIND_JOB_REQUEST_MIN), 312 events, 313 |raw| EventJobRequestIndexes::build_with_profiles(raw, &profiles).expect("build job request"), 314 ); 315 } 316 317 #[test] 318 fn events_json_deterministic_job_result() { 319 let author = "5".repeat(64); 320 let events = vec![ 321 make_event( 322 "a".repeat(64).as_str(), 323 &author, 324 10, 325 IndexerEventKind::JobResult(KIND_JOB_RESULT_MIN), 326 vec![vec!["e".to_string(), "req1".to_string()]], 327 "", 328 ), 329 make_event( 330 "b".repeat(64).as_str(), 331 &author, 332 20, 333 IndexerEventKind::JobResult(KIND_JOB_RESULT_MIN), 334 vec![vec!["e".to_string(), "req2".to_string()]], 335 "", 336 ), 337 ]; 338 let profiles = ProfileResolver::default(); 339 assert_deterministic( 340 IndexerEventKind::JobResult(KIND_JOB_RESULT_MIN), 341 events, 342 |raw| EventJobResultIndexes::build_with_profiles(raw, &profiles).expect("build job result"), 343 ); 344 } 345 346 #[test] 347 fn events_json_deterministic_job_feedback() { 348 let author = "6".repeat(64); 349 let events = vec![ 350 make_event( 351 "a".repeat(64).as_str(), 352 &author, 353 10, 354 IndexerEventKind::JobFeedback, 355 vec![ 356 vec!["e".to_string(), "req1".to_string()], 357 vec!["status".to_string(), "success".to_string()], 358 ], 359 "", 360 ), 361 make_event( 362 "b".repeat(64).as_str(), 363 &author, 364 20, 365 IndexerEventKind::JobFeedback, 366 vec![ 367 vec!["e".to_string(), "req2".to_string()], 368 vec!["status".to_string(), "success".to_string()], 369 ], 370 "", 371 ), 372 ]; 373 let profiles = ProfileResolver::default(); 374 assert_deterministic(IndexerEventKind::JobFeedback, events, |raw| { 375 EventJobFeedbackIndexes::build_with_profiles(raw, &profiles).expect("build job feedback") 376 }); 377 } 378 379 #[test] 380 fn events_json_tiebreaks_by_id() { 381 let author = "7".repeat(64); 382 let events = vec![ 383 make_event( 384 "f".repeat(64).as_str(), 385 &author, 386 10, 387 IndexerEventKind::Post, 388 Vec::new(), 389 "hello", 390 ), 391 make_event( 392 "0".repeat(64).as_str(), 393 &author, 394 10, 395 IndexerEventKind::Post, 396 Vec::new(), 397 "world", 398 ), 399 ]; 400 let profiles = ProfileResolver::default(); 401 let dir = tempdir().expect("tempdir"); 402 let settings = settings_for(dir.path()); 403 let indexes = EventPostIndexes::build_with_profiles(&events, &profiles).expect("build post"); 404 let ids = write_ids(&indexes, &settings, IndexerEventKind::Post); 405 assert_eq!(ids[0], "0".repeat(64)); 406 }