commit 682596670de03a196a3f9334b40dfb96c6053dba parent be2ce93621b80ca32a936a21bd83009b153a7c66 Author: triesap <triesap@radroots.dev> Date: Mon, 3 Nov 2025 20:49:58 +0000 Add `kind 1111` comment events with metadata extraction and indexes by root, author, npub, and NIP-05, integrated into the indexing pipeline. Extend event kinds and keys, wire audit logging, persist comment indexes, enable prerender for market routes, and URL-encode route keys. Diffstat:
14 files changed, 623 insertions(+), 25 deletions(-)
diff --git a/app/src/lib/utils/profile/index.ts b/app/src/lib/utils/profile/index.ts @@ -1,39 +1,100 @@ import { _env } from "$lib/utils/_env"; import { type HttpFetch, fetch_json } from "@radroots/apps-lib"; import type { PageLoadProfileData } from "@radroots/apps-lib-market"; -import type { RadrootsListingEventMetadata, RadrootsProfileEventMetadata } from "@radroots/events-bindings"; +import type { + RadrootsCommentEventMetadata, + RadrootsListingEventMetadata, + RadrootsProfileEventMetadata +} from "@radroots/events-bindings"; import type { RadrootsEventsIndexedManifest as radroots_events_indexed_manifest } from "@radroots/events-indexed-bindings"; import { lib_nostr_npub_encode } from "@radroots/utils-nostr"; type ProfileRoutesKind = "author" | "npub" | "nip05"; +type CommentsByRoot = Record<string, RadrootsCommentEventMetadata[]>; + +export type PageLoadProfileDataWithComments = PageLoadProfileData & { + events: PageLoadProfileData["events"] & { + listing_comments: CommentsByRoot; + }; +}; + const { RADROOTS_MARKET_INDEXES_URL: idx_url } = _env; -async function fetch_listings(fetch_fn: HttpFetch, kind: ProfileRoutesKind, key: string): Promise<RadrootsListingEventMetadata[]> { +async function fetch_listings( + fetch_fn: HttpFetch, + kind: ProfileRoutesKind, + key: string +): Promise<RadrootsListingEventMetadata[]> { const manifest = await fetch_json<radroots_events_indexed_manifest>( fetch_fn, - `${idx_url}/events/30402/${kind}/${key}/manifest.json` + `${idx_url}/events/30402/${kind}/${encodeURIComponent(key)}/manifest.json` ); + if (!manifest.shards.length) return []; + const shard = manifest.shards[0]; - const shard_url = `${idx_url}/events/30402/${kind}/${key}/${shard.file}?v=${shard.sha256}`; + const shard_url = `${idx_url}/events/30402/${kind}/${encodeURIComponent( + key + )}/${shard.file}?v=${shard.sha256}`; return fetch_json<RadrootsListingEventMetadata[]>(fetch_fn, shard_url); } -export async function load_profile_indexed(fetch_fn: HttpFetch, kind: ProfileRoutesKind, key: string): Promise<PageLoadProfileData> { +async function fetch_comments_for_roots( + fetch_fn: HttpFetch, + rootIds: readonly string[] +): Promise<CommentsByRoot> { + const unique = Array.from(new Set(rootIds.map((id) => id.toLowerCase()))); + + const entries: [string, RadrootsCommentEventMetadata[]][] = await Promise.all( + unique.map(async (id): Promise<[string, RadrootsCommentEventMetadata[]]> => { + const url = `${idx_url}/events/1111/root/${encodeURIComponent( + id + )}/metadata.json`; + try { + const metas = await fetch_json<RadrootsCommentEventMetadata[]>( + fetch_fn, + url + ); + return [id, metas]; + } catch { + return [id, [] as RadrootsCommentEventMetadata[]]; + } + }) + ); + + const out: CommentsByRoot = {}; + for (const [id, metas] of entries) { + out[id] = metas; + } + + return out; +} + +export async function load_profile_indexed( + fetch_fn: HttpFetch, + kind: ProfileRoutesKind, + key: string +): Promise<PageLoadProfileDataWithComments> { const profile = await fetch_json<RadrootsProfileEventMetadata>( fetch_fn, - `${idx_url}/events/0/${kind}/${key}/metadata.json` + `${idx_url}/events/0/${kind}/${encodeURIComponent(key)}/metadata.json` ); + const listings = await fetch_listings(fetch_fn, kind, key); + const listingIds = listings.map((m) => m.id.toLowerCase()); + const listing_comments = await fetch_comments_for_roots(fetch_fn, listingIds); + const public_key = profile.author; const npub = lib_nostr_npub_encode(public_key); + return { public_key, npub, events: { profile, - listings + listings, + listing_comments } }; } diff --git a/app/src/routes/(market)/(listing)/[0=country]/+page.ts b/app/src/routes/(market)/(listing)/[0=country]/+page.ts @@ -29,4 +29,4 @@ export const load: PageLoad<PageLoadData> = async ({ fetch, params }) => { }; }; -export const prerender = idx_url ? true : false; +export const prerender = true; diff --git a/app/src/routes/(market)/(profile)/[0=nip05]/+page.ts b/app/src/routes/(market)/(profile)/[0=nip05]/+page.ts @@ -15,4 +15,4 @@ export const load: PageLoad = async ({ fetch, params }) => { return load_profile_indexed(fetch, "nip05", nip05); }; -export const prerender = idx_url ? true : false; +export const prerender = true; diff --git a/app/src/routes/(market)/(profile)/profile/[0=npub]/+page.ts b/app/src/routes/(market)/(profile)/profile/[0=npub]/+page.ts @@ -15,4 +15,4 @@ export const load: PageLoad = async ({ fetch, params }) => { return load_profile_indexed(fetch, "npub", npub); }; -export const prerender = idx_url ? true : false; +export const prerender = true; diff --git a/app/src/routes/(market)/(profile)/profile/[0=public_key]/+page.ts b/app/src/routes/(market)/(profile)/profile/[0=public_key]/+page.ts @@ -15,4 +15,4 @@ export const load: PageLoad = async ({ fetch, params }) => { return load_profile_indexed(fetch, "author", public_key); }; -export const prerender = idx_url ? true : false; +export const prerender = true; diff --git a/app/src/routes/+page.ts b/app/src/routes/+page.ts @@ -38,4 +38,4 @@ export const load: PageLoad<PageLoadData> = async ({ fetch, params }) => { return data; } -export const prerender = idx_url ? true : false; +export const prerender = true; diff --git a/indexer/src/audit.rs b/indexer/src/audit.rs @@ -10,6 +10,7 @@ use tracing::info; use crate::domain::resolvers::profile::ProfileResolver; use crate::relay::event::RelayIndexerEvent; +use radroots_events::comment::models::RadrootsCommentEventIndex; use radroots_events::listing::models::RadrootsListingEventIndex; use radroots_events::profile::models::RadrootsProfileEventIndex; @@ -386,3 +387,69 @@ pub fn log_listing_event(evt: &RadrootsListingEventIndex) { ); } } + +#[inline] +pub fn log_comment_event(evt: &RadrootsCommentEventIndex) { + let need_npub = STATE + .read() + .ok() + .map(|s| !s.filter.npubs.is_empty()) + .unwrap_or(false); + let npub_opt = if need_npub { + public_key_to_npub(&evt.event.author).ok() + } else { + None + }; + + let (need_full, need_local) = STATE + .read() + .ok() + .map(|s| { + ( + !s.filter.nip05_full.is_empty(), + !s.filter.nip05_local.is_empty(), + ) + }) + .unwrap_or((false, false)); + + let (nip05_full_opt, nip05_local_opt) = if need_full || need_local { + if let Ok(s) = STATE.read() { + if let Some(res) = s.resolver.as_ref() { + let local = res + .nip05_for_author(&evt.event.author) + .map(|s| s.to_string()); + (None, local) + } else { + (None, None) + } + } else { + (None, None) + } + } else { + (None, None) + }; + + if !should_log( + &evt.event.author, + evt.event.kind as u64, + evt.event.created_at, + &evt.event.content, + npub_opt, + nip05_full_opt, + nip05_local_opt, + ) { + return; + } + + if let Ok(json) = serde_json::to_string(evt) { + info!( + target = "audit", + kind = evt.event.kind, + id = %evt.event.id, + author = %evt.event.author, + created_at = evt.event.created_at, + processed_json = %json, + "AUDIT: processed comment" + ); + } +} diff --git a/indexer/src/domain/events/comment.rs b/indexer/src/domain/events/comment.rs @@ -0,0 +1,154 @@ +use crate::relay::event::RelayIndexerEvent; +use radroots_events::{ + comment::models::{RadrootsComment, RadrootsCommentEventIndex, RadrootsCommentEventMetadata}, + RadrootsNostrEvent, RadrootsNostrEventRef, +}; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum RadrootsCommentEventIndexError { + #[error("Failed to parse comment from tags")] + ParseError, +} + +fn parse_comment_from_tags( + tags: &[Vec<String>], + content: &str, +) -> Result<RadrootsComment, RadrootsCommentEventIndexError> { + let mut root_id: Option<String> = None; + let mut root_relays: Option<Vec<String>> = None; + let mut root_kind: Option<u32> = None; + let mut root_author: Option<String> = None; + let mut root_d: Option<String> = None; + + let mut parent_id: Option<String> = None; + let mut parent_relays: Option<Vec<String>> = None; + let mut parent_kind: Option<u32> = None; + let mut parent_author: Option<String> = None; + let mut parent_d: Option<String> = None; + + for t in tags { + if t.first().map(|k| k == "e_root").unwrap_or(false) { + if let Some(id) = t.get(1).cloned() { + root_id = Some(id); + } + if let Some(r) = t.get(2).cloned() { + root_relays = Some(vec![r]); + } + } else if t.first().map(|k| k == "e_prev").unwrap_or(false) { + if let Some(id) = t.get(1).cloned() { + parent_id = Some(id); + } + if let Some(r) = t.get(2).cloned() { + parent_relays = Some(vec![r]); + } + } else if t.first().map(|k| k == "e").unwrap_or(false) { + if root_id.is_none() { + if let Some(id) = t.get(1).cloned() { + root_id = Some(id); + } + } + if root_relays.is_none() { + if let Some(r) = t.get(2).cloned() { + root_relays = Some(vec![r]); + } + } + } else if t.first().map(|k| k == "a").unwrap_or(false) { + if let Some(arg) = t.get(1) { + let parts: Vec<&str> = arg.split(':').collect(); + if parts.len() >= 2 { + root_kind = parts[0].parse::<u32>().ok(); + root_author = Some(parts[1].to_lowercase()); + if parts.len() >= 3 && !parts[2].is_empty() { + root_d = Some(parts[2].to_string()); + } + } + } + if let Some(r) = t.get(2).cloned() { + root_relays = Some(vec![r]); + } + } + } + + if parent_id.is_none() { + parent_id = root_id.clone(); + parent_relays = root_relays.clone(); + parent_kind = root_kind; + parent_author = root_author.clone(); + parent_d = root_d.clone(); + } + + let root = RadrootsNostrEventRef { + id: root_id.ok_or(RadrootsCommentEventIndexError::ParseError)?, + author: root_author.unwrap_or_default(), + kind: root_kind.unwrap_or(1), + d_tag: root_d, + relays: root_relays, + }; + + let parent = RadrootsNostrEventRef { + id: parent_id.ok_or(RadrootsCommentEventIndexError::ParseError)?, + author: parent_author.unwrap_or_default(), + kind: parent_kind.unwrap_or(1), + d_tag: parent_d, + relays: parent_relays, + }; + + Ok(RadrootsComment { + root, + parent, + content: content.to_string(), + }) +} + +fn create_radroots_comment_event_metadata( + id: String, + author: String, + published_at: u32, + kind: u32, + content: String, + tags: Vec<Vec<String>>, +) -> Result<RadrootsCommentEventMetadata, RadrootsCommentEventIndexError> { + let comment = parse_comment_from_tags(&tags, &content)?; + Ok(RadrootsCommentEventMetadata { + id, + author, + published_at, + kind, + comment, + }) +} + +pub trait ToRadrootsCommentEventIndex { + fn to_radroots_comment_event( + self, + ) -> Result<RadrootsCommentEventIndex, RadrootsCommentEventIndexError>; +} + +impl ToRadrootsCommentEventIndex for RelayIndexerEvent { + fn to_radroots_comment_event( + self, + ) -> Result<RadrootsCommentEventIndex, RadrootsCommentEventIndexError> { + let kind_u32 = self.kind.as_u64() as u32; + let metadata = create_radroots_comment_event_metadata( + self.id.clone(), + self.author.clone(), + self.created_at, + kind_u32, + self.content.clone(), + self.tags.clone(), + )?; + Ok(RadrootsCommentEventIndex { + event: RadrootsNostrEvent { + id: self.id, + author: self.author, + created_at: self.created_at, + kind: kind_u32, + tags: self.tags, + content: self.content, + sig: self.sig, + }, + metadata, + }) + } +} diff --git a/indexer/src/domain/events/mod.rs b/indexer/src/domain/events/mod.rs @@ -1,7 +1,9 @@ +pub mod comment; pub mod listing; pub mod profile; pub mod reaction; +pub use comment::ToRadrootsCommentEventIndex; pub use listing::ToRadrootsListingEventIndex; pub use profile::ToRadrootsProfileEventIndex; pub use reaction::ToRadrootsReactionEventIndex; diff --git a/indexer/src/domain/indexer/key.rs b/indexer/src/domain/indexer/key.rs @@ -43,3 +43,11 @@ pub const REACTION_INDEX_DIRECTORY: [IndexerKey; 5] = [ IndexerKey::Npub, IndexerKey::Nip05, ]; + +pub const COMMENT_INDEX_DIRECTORY: [IndexerKey; 5] = [ + IndexerKey::Id, + IndexerKey::RootId, + IndexerKey::Author, + IndexerKey::Npub, + IndexerKey::Nip05, +]; diff --git a/indexer/src/domain/indexer/kind.rs b/indexer/src/domain/indexer/kind.rs @@ -4,7 +4,8 @@ use std::fmt; use std::path::PathBuf; use crate::domain::indexer::key::{ - IndexerKey, LISTING_INDEX_DIRECTORY, PROFILE_INDEX_DIRECTORY, REACTION_INDEX_DIRECTORY, + IndexerKey, COMMENT_INDEX_DIRECTORY, LISTING_INDEX_DIRECTORY, PROFILE_INDEX_DIRECTORY, + REACTION_INDEX_DIRECTORY, }; use crate::utils::io::{paths_join, PathsError}; @@ -13,13 +14,15 @@ pub enum IndexerEventKind { Profile, Reaction, Listing, + Comment, } impl IndexerEventKind { - pub const ALL: [IndexerEventKind; 3] = [ + pub const ALL: [IndexerEventKind; 4] = [ IndexerEventKind::Profile, IndexerEventKind::Reaction, IndexerEventKind::Listing, + IndexerEventKind::Comment, ]; pub const fn as_u64(self) -> u64 { @@ -27,6 +30,7 @@ impl IndexerEventKind { IndexerEventKind::Profile => 0, IndexerEventKind::Reaction => 7, IndexerEventKind::Listing => 30402, + IndexerEventKind::Comment => 1111, } } @@ -35,6 +39,7 @@ impl IndexerEventKind { IndexerEventKind::Profile => &PROFILE_INDEX_DIRECTORY, IndexerEventKind::Reaction => &REACTION_INDEX_DIRECTORY, IndexerEventKind::Listing => &LISTING_INDEX_DIRECTORY, + IndexerEventKind::Comment => &COMMENT_INDEX_DIRECTORY, } } @@ -86,6 +91,7 @@ impl TryFrom<u64> for IndexerEventKind { 0 => Ok(IndexerEventKind::Profile), 7 => Ok(IndexerEventKind::Reaction), 30402 => Ok(IndexerEventKind::Listing), + 1111 => Ok(IndexerEventKind::Comment), other => Err(IndexerEventKindParseError(other)), } } diff --git a/indexer/src/domain/indexer/models/comment.rs b/indexer/src/domain/indexer/models/comment.rs @@ -0,0 +1,263 @@ +use crate::domain::indexer::key::{IndexerKey, COMMENT_INDEX_DIRECTORY}; +use crate::utils::crypto::compute_hash; +use crate::utils::io::fs_mkdir; +use crate::utils::io::{write_hash, write_json}; +use crate::utils::nostr::public_key_to_npub; +use crate::utils::strings::truncate_log; +use crate::{ + audit, + domain::{ + events::comment::ToRadrootsCommentEventIndex, + indexer::{ + kind::IndexerEventKind, + models::{EventIndexes, NostrEventsStaticError, WriteEventIndexes}, + }, + resolvers::profile::ProfileResolver, + }, + relay::event::RelayIndexerEvent, + Settings, +}; +use radroots_events::comment::models::{RadrootsCommentEventIndex, RadrootsCommentEventMetadata}; +use std::{collections::BTreeMap, fs, path::PathBuf}; +use tracing::{instrument, warn}; + +macro_rules! write_if_stale { + ($path:expr, $data:expr, $updated:expr) => {{ + let hash = compute_hash(&$data)?; + let hash_path = $path.with_extension("sha256.txt"); + let needs_write = if $path.exists() && hash_path.exists() { + let stored = fs::read_to_string(&hash_path)?; + stored.trim() != hash + } else { + true + }; + if needs_write { + write_json(&$path, &$data)?; + write_hash(&$path, &hash)?; + $updated.push($path.clone()); + } + }}; +} + +#[derive(Debug)] +pub struct EventCommentIndexes { + events: Vec<RadrootsCommentEventIndex>, + events_id: BTreeMap<String, RadrootsCommentEventIndex>, + root_ids: BTreeMap<String, Vec<String>>, + author_ids: BTreeMap<String, Vec<String>>, + npub_ids: BTreeMap<String, Vec<String>>, + nip05_ids: BTreeMap<String, Vec<String>>, +} + +impl EventCommentIndexes { + pub fn build_with_profiles( + raw_events: &[RelayIndexerEvent], + profiles: &ProfileResolver, + ) -> Result<Self, NostrEventsStaticError> { + let mut events = Vec::with_capacity(raw_events.len()); + let mut events_id = BTreeMap::new(); + let mut root_ids: BTreeMap<String, Vec<String>> = BTreeMap::new(); + let mut author_ids: BTreeMap<String, Vec<String>> = BTreeMap::new(); + let mut npub_ids: BTreeMap<String, Vec<String>> = BTreeMap::new(); + let mut nip05_ids: BTreeMap<String, Vec<String>> = BTreeMap::new(); + + for raw in raw_events { + match raw.clone().to_radroots_comment_event() { + Ok(evt) => { + audit::log_comment_event(&evt); + let id = evt.metadata.id.clone(); + let author_hex = evt.metadata.author.to_lowercase(); + + let npub = public_key_to_npub(&author_hex) + .map(|s| s.to_lowercase()) + .ok(); + let author_nip05 = profiles.nip05_for_author(&author_hex).map(str::to_owned); + + let root = evt.metadata.comment.root.id.to_lowercase(); + + events_id.insert(id.clone(), evt.clone()); + events.push(evt.clone()); + + root_ids.entry(root).or_default().push(id.clone()); + author_ids.entry(author_hex).or_default().push(id.clone()); + if let Some(n) = npub { + npub_ids.entry(n).or_default().push(id.clone()); + } + if let Some(n05) = author_nip05 { + nip05_ids + .entry(n05.to_lowercase()) + .or_default() + .push(id.clone()); + } + } + Err(err) => { + warn!( + kind = raw.kind.as_u64(), + id = %raw.id, + author = %raw.author, + content = %truncate_log(&raw.content, 1000), + tags = ?raw.tags, + error = %err, + "Skipping malformed comment event" + ); + } + } + } + + let sort_ids = |ids: &mut Vec<String>, + map: &BTreeMap<String, RadrootsCommentEventIndex>| { + ids.sort_unstable_by(|a, b| { + let pa = map + .get(a) + .map(|e| e.metadata.published_at) + .unwrap_or_default(); + let pb = map + .get(b) + .map(|e| e.metadata.published_at) + .unwrap_or_default(); + pb.cmp(&pa).then(a.cmp(b)) + }); + }; + + for ids in root_ids.values_mut() { + sort_ids(ids, &events_id); + } + for ids in author_ids.values_mut() { + sort_ids(ids, &events_id); + } + for ids in npub_ids.values_mut() { + sort_ids(ids, &events_id); + } + for ids in nip05_ids.values_mut() { + sort_ids(ids, &events_id); + } + + Ok(Self { + events, + events_id, + root_ids, + author_ids, + npub_ids, + nip05_ids, + }) + } +} + +impl EventIndexes for EventCommentIndexes { + type Event = RelayIndexerEvent; + + fn subdirs() -> &'static [IndexerKey] { + &COMMENT_INDEX_DIRECTORY + } + + #[instrument(skip(raw_events), fields(event_count = raw_events.len()))] + fn build(raw_events: &[Self::Event]) -> Result<Self, NostrEventsStaticError> { + let empty = ProfileResolver::default(); + Self::build_with_profiles(raw_events, &empty) + } +} + +impl WriteEventIndexes for EventCommentIndexes { + fn write(&self, settings: &Settings, updated: &mut Vec<PathBuf>) -> anyhow::Result<()> { + let base: PathBuf = IndexerEventKind::Comment.base_path(&settings.indexer.data_dir)?; + fs_mkdir(&[&base])?; + + { + let idxs_root = base.join("events.json"); + let ids: Vec<&String> = self.events.iter().map(|e| &e.event.id).collect(); + write_if_stale!(idxs_root, ids, updated); + } + + { + let sub = base.join("id"); + fs_mkdir(&[&sub])?; + let keys: Vec<String> = self.events_id.keys().cloned().collect(); + write_if_stale!(sub.join("indexes.json"), keys, updated); + + for (id, evt) in &self.events_id { + let dir = sub.join(id.to_lowercase()); + fs_mkdir(&[&dir])?; + write_if_stale!(dir.join("event.json"), evt.event.clone(), updated); + write_if_stale!(dir.join("metadata.json"), evt.metadata.clone(), updated); + } + } + + { + let sub = base.join(IndexerKey::RootId.as_str()); + fs_mkdir(&[&sub])?; + let roots: Vec<String> = self.root_ids.keys().cloned().collect(); + write_if_stale!(sub.join("indexes.json"), roots, updated); + + for (root, ids) in &self.root_ids { + let dir = sub.join(root); + fs_mkdir(&[&dir])?; + let metas: Vec<RadrootsCommentEventMetadata> = ids + .iter() + .filter_map(|id| self.events_id.get(id)) + .map(|e| e.metadata.clone()) + .collect(); + write_if_stale!(dir.join("events.json"), ids.clone(), updated); + write_if_stale!(dir.join("metadata.json"), metas, updated); + } + } + + { + let sub = base.join(IndexerKey::Author.as_str()); + fs_mkdir(&[&sub])?; + let authors: Vec<String> = self.author_ids.keys().cloned().collect(); + write_if_stale!(sub.join("indexes.json"), authors, updated); + + for (author, ids) in &self.author_ids { + let dir = sub.join(author); + fs_mkdir(&[&dir])?; + let metas: Vec<RadrootsCommentEventMetadata> = ids + .iter() + .filter_map(|id| self.events_id.get(id)) + .map(|e| e.metadata.clone()) + .collect(); + write_if_stale!(dir.join("events.json"), ids.clone(), updated); + write_if_stale!(dir.join("metadata.json"), metas, updated); + } + } + + { + let sub = base.join(IndexerKey::Npub.as_str()); + fs_mkdir(&[&sub])?; + let npubs: Vec<String> = self.npub_ids.keys().cloned().collect(); + write_if_stale!(sub.join("indexes.json"), npubs, updated); + + for (npub, ids) in &self.npub_ids { + let dir = sub.join(npub); + fs_mkdir(&[&dir])?; + let metas: Vec<RadrootsCommentEventMetadata> = ids + .iter() + .filter_map(|id| self.events_id.get(id)) + .map(|e| e.metadata.clone()) + .collect(); + write_if_stale!(dir.join("events.json"), ids.clone(), updated); + write_if_stale!(dir.join("metadata.json"), metas, updated); + } + } + + { + let sub = base.join(IndexerKey::Nip05.as_str()); + fs_mkdir(&[&sub])?; + let names: Vec<String> = self.nip05_ids.keys().cloned().collect(); + write_if_stale!(sub.join("indexes.json"), names, updated); + + for (name, ids) in &self.nip05_ids { + let dir = sub.join(name); + fs_mkdir(&[&dir])?; + let metas: Vec<RadrootsCommentEventMetadata> = ids + .iter() + .filter_map(|id| self.events_id.get(id)) + .map(|e| e.metadata.clone()) + .collect(); + write_if_stale!(dir.join("events.json"), ids.clone(), updated); + write_if_stale!(dir.join("metadata.json"), metas, updated); + } + } + + Ok(()) + } +} diff --git a/indexer/src/domain/indexer/models/mod.rs b/indexer/src/domain/indexer/models/mod.rs @@ -1,7 +1,9 @@ +pub mod comment; pub mod listing; pub mod profile; pub mod reaction; +pub use comment::EventCommentIndexes; pub use listing::EventListingIndexes; pub use profile::EventProfileIndexes; pub use reaction::EventReactionIndexes; diff --git a/indexer/src/lib.rs b/indexer/src/lib.rs @@ -25,12 +25,14 @@ pub mod audit; #[cfg(not(feature = "audit"))] pub mod audit { use radroots_events::{ - listing::models::RadrootsListingEventIndex, profile::models::RadrootsProfileEventIndex, + comment::models::RadrootsCommentEventIndex, listing::models::RadrootsListingEventIndex, + profile::models::RadrootsProfileEventIndex, }; pub fn log_indexer_event(_: &crate::relay::event::RelayIndexerEvent) {} pub fn log_profile_event(_: &RadrootsProfileEventIndex) {} pub fn log_listing_event(_: &RadrootsListingEventIndex) {} + pub fn log_comment_event(_: &RadrootsCommentEventIndex) {} } use crate::{ @@ -38,8 +40,8 @@ use crate::{ indexer::{ kind::IndexerEventKind, models::{ - EventIndexes, EventListingIndexes, EventProfileIndexes, EventReactionIndexes, - WriteEventIndexes, + EventCommentIndexes, EventIndexes, EventListingIndexes, EventProfileIndexes, + EventReactionIndexes, WriteEventIndexes, }, }, resolvers::profile::ProfileResolver, @@ -59,6 +61,7 @@ pub async fn run(settings: Settings) -> Result<()> { let tree_events_profile = "profile_events"; let tree_events_listing = "listing_events"; let tree_events_reaction = "reaction_events"; + let tree_events_comment = "comment_events"; let tree_stats = "stats"; let last_created_at_db: u32 = db_idx @@ -77,8 +80,7 @@ pub async fn run(settings: Settings) -> Result<()> { .join(", "); let relay_query = format!( - "SELECT hex(event_hash), hex(author), created_at, kind, content \ - FROM event WHERE kind IN ({}) AND created_at > ?", + "SELECT hex(event_hash), hex(author), created_at, kind, content FROM event WHERE kind IN ({}) AND created_at > ?", event_kinds ); @@ -108,7 +110,6 @@ pub async fn run(settings: Settings) -> Result<()> { let mut need_rebuild_listing = false; - // Handle Profile Events if let Some(profile_events) = records_kind.remove(&IndexerEventKind::Profile) { if !profile_events.is_empty() { for ev in &profile_events { @@ -150,11 +151,9 @@ pub async fn run(settings: Settings) -> Result<()> { } } - // Load current profile data (used for listings and reactions) let raw_profile_events: Vec<RelayIndexerEvent> = db_idx.get_all(tree_events_profile)?; let profiles = ProfileResolver::from_metadata(&raw_profile_events); - // Handle Listing Events if let Some(listing_events) = records_kind.remove(&IndexerEventKind::Listing) { if !listing_events.is_empty() { for ev in &listing_events { @@ -195,7 +194,6 @@ pub async fn run(settings: Settings) -> Result<()> { } } - // Handle Reaction Events if let Some(reaction_events) = records_kind.remove(&IndexerEventKind::Reaction) { if !reaction_events.is_empty() { for ev in &reaction_events { @@ -235,7 +233,45 @@ pub async fn run(settings: Settings) -> Result<()> { } } - // If new profiles or listings were written, rebuild the listing indexes + if let Some(comment_events) = records_kind.remove(&IndexerEventKind::Comment) { + if !comment_events.is_empty() { + for ev in &comment_events { + last_created_at = last_created_at.max(ev.created_at); + let id = &ev.id; + let hash = &ev.hash; + let skip = if let Some(old) = db_idx.get_raw(tree_raw, id)? { + old.as_ref() == hash.as_bytes() + } else { + false + }; + if skip { + continue; + } + db_idx.insert(tree_events_comment, id, ev)?; + db_idx.insert_raw(tree_raw, id, hash.as_bytes())?; + } + + db_idx.insert_raw( + tree_stats, + "last_created_at", + &last_created_at.to_be_bytes(), + )?; + db_idx.flush()?; + + let raw_comment_events: Vec<RelayIndexerEvent> = + db_idx.get_all(tree_events_comment)?; + let comment_indexes = + EventCommentIndexes::build_with_profiles(&raw_comment_events, &profiles)?; + let mut updated_comment = Vec::new(); + comment_indexes.write(&settings, &mut updated_comment)?; + info!( + written = updated_comment.len(), + "Written {} comment index files", + updated_comment.len() + ); + } + } + if need_rebuild_listing { let raw_listing_events: Vec<RelayIndexerEvent> = db_idx.get_all(tree_events_listing)?; let listing_indexes = @@ -249,7 +285,6 @@ pub async fn run(settings: Settings) -> Result<()> { ); } - // Sleep between runs based on configuration let elapsed = iteration_start.elapsed(); let interval = Duration::from_secs(settings.indexer.flush_interval); let delay = interval.saturating_sub(elapsed);