From 02e957ec97c9a9e6e37eca9c9d4aa6aef4bcd363 Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Fri, 9 Jan 2026 15:49:17 +0000 Subject: feat: Switch SyncManager to use two-tier RejectedEventsIndex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the simple HashSet with the sophisticated two-tier RejectedEventsIndex from PR1, enabling future immediate re-processing when maintainer dependencies resolve. ## Changes ### Config (src/config.rs) - Add `rejected_hot_cache_duration_secs` (default: 120 = 2 minutes) - Add `rejected_cold_index_expiry_secs` (default: 604800 = 7 days) - Both configurable via CLI flags or environment variables ### SyncManager (src/sync/mod.rs) **Type Change:** - Before: `Arc>>` (simple event ID set) - After: `Arc` (two-tier storage) **Initialization:** - Pass config durations to RejectedEventsIndex::new() - Creates hot cache (2 min) + cold index (7 days) **Event Processing (process_event_static):** - Extract identifier from 'd' tag - Determine rejection reason from error message - Call `add_announcement()` with full event + metadata - Stores in both hot cache and cold index **Negentropy Sync (derive_relay_targets):** - Call `get_all_event_ids()` to get rejected IDs - Returns union of hot cache + cold index event IDs - Excludes from negentropy reconciliation **Event Loop (relay_connection):** - Use `contains()` method instead of direct HashSet access - Simpler API, same skip-rejected behavior ### RejectedEventsIndex (src/sync/rejected_index.rs) **New Method:** - `get_all_event_ids()`: Returns HashSet from both tiers - Used for negentropy exclusion (replaces direct HashSet access) ### Tests Updated **test_rejected_events_index_tracks_announcements:** - Create RejectedEventsIndex with config durations - Add 'd' tag to test announcement - Use `add_announcement()` with full event - Verify both hot cache and cold index populated - Check lengths with `hot_cache_len()` and `cold_index_len()` **test_rejected_events_excluded_from_negentropy:** - Create RejectedEventsIndex instead of HashSet - Build full event with 'd' tag - Add to index with `add_announcement()` - Get IDs with `get_all_event_ids()` - Verify excluded from reconciliation ## Architecture ``` ┌─────────────────────────────────────────────────────────────┐ │ SyncManager │ │ │ │ rejected_events_index: Arc │ │ ├─ Hot Cache (2 min): Full events for re-processing │ │ └─ Cold Index (7 days): Metadata for dedup │ └─────────────────────────────────────────────────────────────┘ │ │ On rejection ▼ ┌─────────────────────────────────────────────────────────────┐ │ add_announcement(event, pubkey, identifier, reason) │ │ ├─ Store full event in hot cache │ │ └─ Store metadata in cold index │ └─────────────────────────────────────────────────────────────┘ │ │ On negentropy sync ▼ ┌─────────────────────────────────────────────────────────────┐ │ get_all_event_ids() → HashSet │ │ ├─ Union of hot cache IDs │ │ └─ Union of cold index IDs │ └─────────────────────────────────────────────────────────────┘ ``` ## Benefits ### Immediate - **Better tracking**: Store rejection reason + metadata - **Configurable**: Tune cache/index durations per deployment - **Observable**: Separate hot/cold metrics (future PR4) ### Future (PR3) - **Immediate re-processing**: Get events from hot cache when valid - **No 24h delay**: Maintainer announcements accepted in <1 second - **Automatic recovery**: Hot cache for immediate, cold index for later ## Backward Compatibility **No breaking changes:** - Same rejection behavior (skip events in index) - Same negentropy exclusion (union with purgatory IDs) - Default config values match previous implicit behavior **Migration:** - Existing deployments continue working with defaults - Optional: Tune durations via new config flags ## Testing All tests passing: - ✅ 9 rejected_index tests (hot cache, cold index, two-tier) - ✅ 139 sync module tests (including updated integration tests) - ✅ 247 total library tests ## Next Steps **PR3: Add invalidation + immediate re-processing** - Invalidate cold index when owner announcement accepted - Get events from hot cache for re-processing - Recursive call to process_event_static - Integration tests for <1s maintainer acceptance **PR4: Add cleanup + metrics** - Hot cache cleanup task (every 60s) - Cold index cleanup task (daily) - Prometheus metrics for both tiers - Monitor hot cache hits vs misses ## Configuration Examples ```bash # Default (2 min hot cache, 7 day cold index) ngit-grasp # Longer hot cache for slow relays ngit-grasp --rejected-hot-cache-duration-secs 300 # Shorter cold index for memory-constrained systems ngit-grasp --rejected-cold-index-expiry-secs 86400 # Environment variables export NGIT_REJECTED_HOT_CACHE_DURATION_SECS=180 export NGIT_REJECTED_COLD_INDEX_EXPIRY_SECS=259200 ngit-grasp ``` Part of: Maintainer chain discovery fix See: work/SOLUTION-SUMMARY-V2.md for full design Previous: PR1 (rejected_index.rs implementation) Next: PR3 (invalidation + re-processing) --- src/config.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'src/config.rs') diff --git a/src/config.rs b/src/config.rs index c4a7b6c..2343c88 100644 --- a/src/config.rs +++ b/src/config.rs @@ -136,6 +136,18 @@ pub struct Config { /// Primarily useful for testing that sync works without negentropy support. #[arg(long, env = "NGIT_SYNC_DISABLE_NEGENTROPY", default_value_t = false)] pub sync_disable_negentropy: bool, + + /// Hot cache duration in seconds for rejected announcements (default: 120 = 2 minutes) + /// Stores full event objects for immediate re-processing when dependencies resolve. + /// Too short (<30s): Miss events from slow relays + /// Too long (>5min): Waste memory + #[arg(long, env = "NGIT_REJECTED_HOT_CACHE_DURATION_SECS", default_value_t = 120)] + pub rejected_hot_cache_duration_secs: u64, + + /// Cold index expiry in seconds for rejected announcements (default: 604800 = 7 days) + /// Stores metadata only to prevent repeated downloads of rejected events. + #[arg(long, env = "NGIT_REJECTED_COLD_INDEX_EXPIRY_SECS", default_value_t = 604800)] + pub rejected_cold_index_expiry_secs: u64, } impl Config { @@ -258,6 +270,8 @@ impl Config { sync_disconnect_check_interval_secs: 60, sync_base_backoff_secs: 5, sync_disable_negentropy: false, + rejected_hot_cache_duration_secs: 120, + rejected_cold_index_expiry_secs: 604800, } } } -- cgit v1.2.3