From e4cfecbfc909c9ca4983101cf6a5855959a5d49f Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Fri, 9 Jan 2026 15:38:58 +0000 Subject: feat: Add two-tier rejected events index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements a sophisticated two-tier storage system for rejected repository announcements to enable immediate re-processing when dependencies resolve. ## Architecture **Tier 1: Hot Cache (2 minutes)** - Stores full event objects for immediate re-processing - Enables <1 second re-processing vs 24 hour wait - Auto-expires to prevent memory growth - Memory: ~200 KB typical, ~20 MB worst case **Tier 2: Cold Index (7 days)** - Stores metadata only (event_id, pubkey, identifier) - Prevents repeated downloads of rejected events - Enables invalidation when circumstances change - Memory: ~1 MB typical ## Problem Solved Without this system, maintainer announcements face a timing gap: 00:00 - Maintainer announcement rejected → Event discarded 00:02 - Owner announcement accepted (lists maintainer) → Want to re-process 00:02 - ❌ Maintainer announcement GONE → Must wait 24h for next sync With two-tier system: 00:00 - Maintainer announcement rejected → Stored in both tiers 00:02 - Owner announcement accepted → Invalidate + get from hot cache 00:02 - ✅ Re-process immediately → Accepted in <1 second ## Implementation New module: src/sync/rejected_index.rs - RejectedEventsIndex: Public API combining both tiers - HotCache: Internal struct for full event storage - ColdIndex: Internal struct for metadata storage - RejectionReason: Enum for tracking why events were rejected Key methods: - add_announcement(): Add to both tiers - contains(): Check if event is rejected - invalidate_and_get_events(): Remove from cold index, get from hot cache - cleanup_expired(): Remove expired entries from both tiers ## Testing 9 comprehensive unit tests covering: - Hot cache storage and retrieval - Hot cache expiration - Cold index metadata tracking - Cold index invalidation - Two-tier integration - Cleanup of expired entries - Hot cache misses after expiry - Multiple maintainer repositories All tests passing. ## Next Steps PR2: Switch SyncManager to use new RejectedEventsIndex PR3: Add invalidation + immediate re-processing logic PR4: Add cleanup task + Prometheus metrics Part of: Maintainer chain discovery fix See: work/SOLUTION-SUMMARY-V2.md for full design --- src/sync/mod.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'src/sync/mod.rs') diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 8b1da0e..55bea17 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -16,6 +16,7 @@ pub mod algorithms; pub mod filters; pub mod health; pub mod metrics; +pub mod rejected_index; pub mod relay_connection; pub mod self_subscriber; @@ -25,6 +26,11 @@ pub use algorithms::{AddFilters, RelaySyncNeeds}; // Re-export metrics types pub use metrics::SyncMetrics; +// Re-export rejected index types +pub use rejected_index::{RejectionReason}; +// Note: RejectedEventsIndex struct exists in rejected_index.rs but not yet used +// Current code still uses the simple HashSet type alias below + // Re-export relay connection types pub use relay_connection::{NegentropySyncResult, RelayConnection, RelayEvent}; @@ -67,7 +73,10 @@ pub type PendingSyncIndex = Arc>>>; /// Tracks EventIds of announcement events (30617/30618) that were rejected during sync. /// These events are excluded from negentropy sync and skipped during REQ+EOSE processing /// to avoid repeatedly fetching and rejecting the same events. -pub type RejectedEventsIndex = Arc>>; +/// +/// NOTE: This is a temporary simple implementation. PR2 will replace this with the +/// two-tier RejectedEventsIndex from rejected_index.rs (hot cache + cold index). +type RejectedEventsIndexSimple = Arc>>; // ============================================================================= // Supporting Data Structures @@ -436,7 +445,7 @@ pub struct SyncManager { /// In-flight subscription batches pending_sync_index: PendingSyncIndex, /// Rejected announcement event IDs (30617/30618) - excluded from sync - rejected_events_index: RejectedEventsIndex, + rejected_events_index: RejectedEventsIndexSimple, /// Active relay connections - keyed by relay URL connections: HashMap, /// Health tracker for relay connection state @@ -1992,7 +2001,7 @@ impl SyncManager { database: &SharedDatabase, write_policy: &Nip34WritePolicy, local_relay: &LocalRelay, - rejected_events_index: &RejectedEventsIndex, + rejected_events_index: &RejectedEventsIndexSimple, ) -> ProcessResult { use nostr_relay_builder::prelude::{WritePolicy, WritePolicyResult}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -2880,7 +2889,7 @@ mod tests { #[tokio::test] async fn test_rejected_events_index_tracks_announcements() { // Create a rejected events index - let rejected_index: RejectedEventsIndex = Arc::new(RwLock::new(HashSet::new())); + let rejected_index: RejectedEventsIndexSimple = Arc::new(RwLock::new(HashSet::new())); // Create test announcement event (kind 30617) let keys = Keys::generate(); -- cgit v1.2.3