From 02e957ec97c9a9e6e37eca9c9d4aa6aef4bcd363 Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Fri, 9 Jan 2026 15:49:17 +0000 Subject: feat: Switch SyncManager to use two-tier RejectedEventsIndex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the simple HashSet with the sophisticated two-tier RejectedEventsIndex from PR1, enabling future immediate re-processing when maintainer dependencies resolve. ## Changes ### Config (src/config.rs) - Add `rejected_hot_cache_duration_secs` (default: 120 = 2 minutes) - Add `rejected_cold_index_expiry_secs` (default: 604800 = 7 days) - Both configurable via CLI flags or environment variables ### SyncManager (src/sync/mod.rs) **Type Change:** - Before: `Arc>>` (simple event ID set) - After: `Arc` (two-tier storage) **Initialization:** - Pass config durations to RejectedEventsIndex::new() - Creates hot cache (2 min) + cold index (7 days) **Event Processing (process_event_static):** - Extract identifier from 'd' tag - Determine rejection reason from error message - Call `add_announcement()` with full event + metadata - Stores in both hot cache and cold index **Negentropy Sync (derive_relay_targets):** - Call `get_all_event_ids()` to get rejected IDs - Returns union of hot cache + cold index event IDs - Excludes from negentropy reconciliation **Event Loop (relay_connection):** - Use `contains()` method instead of direct HashSet access - Simpler API, same skip-rejected behavior ### RejectedEventsIndex (src/sync/rejected_index.rs) **New Method:** - `get_all_event_ids()`: Returns HashSet from both tiers - Used for negentropy exclusion (replaces direct HashSet access) ### Tests Updated **test_rejected_events_index_tracks_announcements:** - Create RejectedEventsIndex with config durations - Add 'd' tag to test announcement - Use `add_announcement()` with full event - Verify both hot cache and cold index populated - Check lengths with `hot_cache_len()` and `cold_index_len()` **test_rejected_events_excluded_from_negentropy:** - Create RejectedEventsIndex instead of HashSet - Build full event with 'd' tag - Add to index with `add_announcement()` - Get IDs with `get_all_event_ids()` - Verify excluded from reconciliation ## Architecture ``` ┌─────────────────────────────────────────────────────────────┐ │ SyncManager │ │ │ │ rejected_events_index: Arc │ │ ├─ Hot Cache (2 min): Full events for re-processing │ │ └─ Cold Index (7 days): Metadata for dedup │ └─────────────────────────────────────────────────────────────┘ │ │ On rejection ▼ ┌─────────────────────────────────────────────────────────────┐ │ add_announcement(event, pubkey, identifier, reason) │ │ ├─ Store full event in hot cache │ │ └─ Store metadata in cold index │ └─────────────────────────────────────────────────────────────┘ │ │ On negentropy sync ▼ ┌─────────────────────────────────────────────────────────────┐ │ get_all_event_ids() → HashSet │ │ ├─ Union of hot cache IDs │ │ └─ Union of cold index IDs │ └─────────────────────────────────────────────────────────────┘ ``` ## Benefits ### Immediate - **Better tracking**: Store rejection reason + metadata - **Configurable**: Tune cache/index durations per deployment - **Observable**: Separate hot/cold metrics (future PR4) ### Future (PR3) - **Immediate re-processing**: Get events from hot cache when valid - **No 24h delay**: Maintainer announcements accepted in <1 second - **Automatic recovery**: Hot cache for immediate, cold index for later ## Backward Compatibility **No breaking changes:** - Same rejection behavior (skip events in index) - Same negentropy exclusion (union with purgatory IDs) - Default config values match previous implicit behavior **Migration:** - Existing deployments continue working with defaults - Optional: Tune durations via new config flags ## Testing All tests passing: - ✅ 9 rejected_index tests (hot cache, cold index, two-tier) - ✅ 139 sync module tests (including updated integration tests) - ✅ 247 total library tests ## Next Steps **PR3: Add invalidation + immediate re-processing** - Invalidate cold index when owner announcement accepted - Get events from hot cache for re-processing - Recursive call to process_event_static - Integration tests for <1s maintainer acceptance **PR4: Add cleanup + metrics** - Hot cache cleanup task (every 60s) - Cold index cleanup task (daily) - Prometheus metrics for both tiers - Monitor hot cache hits vs misses ## Configuration Examples ```bash # Default (2 min hot cache, 7 day cold index) ngit-grasp # Longer hot cache for slow relays ngit-grasp --rejected-hot-cache-duration-secs 300 # Shorter cold index for memory-constrained systems ngit-grasp --rejected-cold-index-expiry-secs 86400 # Environment variables export NGIT_REJECTED_HOT_CACHE_DURATION_SECS=180 export NGIT_REJECTED_COLD_INDEX_EXPIRY_SECS=259200 ngit-grasp ``` Part of: Maintainer chain discovery fix See: work/SOLUTION-SUMMARY-V2.md for full design Previous: PR1 (rejected_index.rs implementation) Next: PR3 (invalidation + re-processing) --- src/sync/rejected_index.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'src/sync/rejected_index.rs') diff --git a/src/sync/rejected_index.rs b/src/sync/rejected_index.rs index f89783a..80d6b5b 100644 --- a/src/sync/rejected_index.rs +++ b/src/sync/rejected_index.rs @@ -85,7 +85,7 @@ //! ``` use nostr_sdk::{Event, EventId, PublicKey}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant}; @@ -396,6 +396,24 @@ impl RejectedEventsIndex { pub fn cold_index_len(&self) -> usize { self.cold_index.len() } + + /// Get all rejected event IDs (from both hot cache and cold index) + /// + /// Used for excluding rejected events from negentropy sync. + /// Note: This creates a snapshot - events may be added/removed concurrently. + pub fn get_all_event_ids(&self) -> HashSet { + let mut ids = HashSet::new(); + + // Add from hot cache + let hot_entries = self.hot_cache.entries.read().unwrap(); + ids.extend(hot_entries.keys().cloned()); + + // Add from cold index + let cold_entries = self.cold_index.entries.read().unwrap(); + ids.extend(cold_entries.keys().cloned()); + + ids + } } #[cfg(test)] -- cgit v1.2.3