upleb.uk

Public git repos — served from a NIP-34 GRASP relay at git.upleb.uk

summaryrefslogtreecommitdiff
path: root/src/sync/mod.rs
diff options
context:
space:
mode:
authorDanConwayDev <DanConwayDev@protonmail.com>2026-01-09 15:38:58 +0000
committerDanConwayDev <DanConwayDev@protonmail.com>2026-01-09 15:38:58 +0000
commite4cfecbfc909c9ca4983101cf6a5855959a5d49f (patch)
treed56e91539c4c8a283905a651caf4c95cb130a2a1 /src/sync/mod.rs
parent29be4cb7d0fbd29325c995a76ba1b1f47beecca5 (diff)
feat: Add two-tier rejected events index
Implements a sophisticated two-tier storage system for rejected repository announcements to enable immediate re-processing when dependencies resolve. ## Architecture **Tier 1: Hot Cache (2 minutes)** - Stores full event objects for immediate re-processing - Enables <1 second re-processing vs 24 hour wait - Auto-expires to prevent memory growth - Memory: ~200 KB typical, ~20 MB worst case **Tier 2: Cold Index (7 days)** - Stores metadata only (event_id, pubkey, identifier) - Prevents repeated downloads of rejected events - Enables invalidation when circumstances change - Memory: ~1 MB typical ## Problem Solved Without this system, maintainer announcements face a timing gap: 00:00 - Maintainer announcement rejected → Event discarded 00:02 - Owner announcement accepted (lists maintainer) → Want to re-process 00:02 - ❌ Maintainer announcement GONE → Must wait 24h for next sync With two-tier system: 00:00 - Maintainer announcement rejected → Stored in both tiers 00:02 - Owner announcement accepted → Invalidate + get from hot cache 00:02 - ✅ Re-process immediately → Accepted in <1 second ## Implementation New module: src/sync/rejected_index.rs - RejectedEventsIndex: Public API combining both tiers - HotCache: Internal struct for full event storage - ColdIndex: Internal struct for metadata storage - RejectionReason: Enum for tracking why events were rejected Key methods: - add_announcement(): Add to both tiers - contains(): Check if event is rejected - invalidate_and_get_events(): Remove from cold index, get from hot cache - cleanup_expired(): Remove expired entries from both tiers ## Testing 9 comprehensive unit tests covering: - Hot cache storage and retrieval - Hot cache expiration - Cold index metadata tracking - Cold index invalidation - Two-tier integration - Cleanup of expired entries - Hot cache misses after expiry - Multiple maintainer repositories All tests passing. ## Next Steps PR2: Switch SyncManager to use new RejectedEventsIndex PR3: Add invalidation + immediate re-processing logic PR4: Add cleanup task + Prometheus metrics Part of: Maintainer chain discovery fix See: work/SOLUTION-SUMMARY-V2.md for full design
Diffstat (limited to 'src/sync/mod.rs')
-rw-r--r--src/sync/mod.rs17
1 files changed, 13 insertions, 4 deletions
diff --git a/src/sync/mod.rs b/src/sync/mod.rs
index 8b1da0e..55bea17 100644
--- a/src/sync/mod.rs
+++ b/src/sync/mod.rs
@@ -16,6 +16,7 @@ pub mod algorithms;
16pub mod filters; 16pub mod filters;
17pub mod health; 17pub mod health;
18pub mod metrics; 18pub mod metrics;
19pub mod rejected_index;
19pub mod relay_connection; 20pub mod relay_connection;
20pub mod self_subscriber; 21pub mod self_subscriber;
21 22
@@ -25,6 +26,11 @@ pub use algorithms::{AddFilters, RelaySyncNeeds};
25// Re-export metrics types 26// Re-export metrics types
26pub use metrics::SyncMetrics; 27pub use metrics::SyncMetrics;
27 28
29// Re-export rejected index types
30pub use rejected_index::{RejectionReason};
31// Note: RejectedEventsIndex struct exists in rejected_index.rs but not yet used
32// Current code still uses the simple HashSet type alias below
33
28// Re-export relay connection types 34// Re-export relay connection types
29pub use relay_connection::{NegentropySyncResult, RelayConnection, RelayEvent}; 35pub use relay_connection::{NegentropySyncResult, RelayConnection, RelayEvent};
30 36
@@ -67,7 +73,10 @@ pub type PendingSyncIndex = Arc<RwLock<HashMap<String, Vec<PendingBatch>>>>;
67/// Tracks EventIds of announcement events (30617/30618) that were rejected during sync. 73/// Tracks EventIds of announcement events (30617/30618) that were rejected during sync.
68/// These events are excluded from negentropy sync and skipped during REQ+EOSE processing 74/// These events are excluded from negentropy sync and skipped during REQ+EOSE processing
69/// to avoid repeatedly fetching and rejecting the same events. 75/// to avoid repeatedly fetching and rejecting the same events.
70pub type RejectedEventsIndex = Arc<RwLock<HashSet<EventId>>>; 76///
77/// NOTE: This is a temporary simple implementation. PR2 will replace this with the
78/// two-tier RejectedEventsIndex from rejected_index.rs (hot cache + cold index).
79type RejectedEventsIndexSimple = Arc<RwLock<HashSet<EventId>>>;
71 80
72// ============================================================================= 81// =============================================================================
73// Supporting Data Structures 82// Supporting Data Structures
@@ -436,7 +445,7 @@ pub struct SyncManager {
436 /// In-flight subscription batches 445 /// In-flight subscription batches
437 pending_sync_index: PendingSyncIndex, 446 pending_sync_index: PendingSyncIndex,
438 /// Rejected announcement event IDs (30617/30618) - excluded from sync 447 /// Rejected announcement event IDs (30617/30618) - excluded from sync
439 rejected_events_index: RejectedEventsIndex, 448 rejected_events_index: RejectedEventsIndexSimple,
440 /// Active relay connections - keyed by relay URL 449 /// Active relay connections - keyed by relay URL
441 connections: HashMap<String, RelayConnection>, 450 connections: HashMap<String, RelayConnection>,
442 /// Health tracker for relay connection state 451 /// Health tracker for relay connection state
@@ -1992,7 +2001,7 @@ impl SyncManager {
1992 database: &SharedDatabase, 2001 database: &SharedDatabase,
1993 write_policy: &Nip34WritePolicy, 2002 write_policy: &Nip34WritePolicy,
1994 local_relay: &LocalRelay, 2003 local_relay: &LocalRelay,
1995 rejected_events_index: &RejectedEventsIndex, 2004 rejected_events_index: &RejectedEventsIndexSimple,
1996 ) -> ProcessResult { 2005 ) -> ProcessResult {
1997 use nostr_relay_builder::prelude::{WritePolicy, WritePolicyResult}; 2006 use nostr_relay_builder::prelude::{WritePolicy, WritePolicyResult};
1998 use std::net::{IpAddr, Ipv4Addr, SocketAddr}; 2007 use std::net::{IpAddr, Ipv4Addr, SocketAddr};
@@ -2880,7 +2889,7 @@ mod tests {
2880 #[tokio::test] 2889 #[tokio::test]
2881 async fn test_rejected_events_index_tracks_announcements() { 2890 async fn test_rejected_events_index_tracks_announcements() {
2882 // Create a rejected events index 2891 // Create a rejected events index
2883 let rejected_index: RejectedEventsIndex = Arc::new(RwLock::new(HashSet::new())); 2892 let rejected_index: RejectedEventsIndexSimple = Arc::new(RwLock::new(HashSet::new()));
2884 2893
2885 // Create test announcement event (kind 30617) 2894 // Create test announcement event (kind 30617)
2886 let keys = Keys::generate(); 2895 let keys = Keys::generate();