diff options
| author | DanConwayDev <DanConwayDev@protonmail.com> | 2026-01-09 15:38:58 +0000 |
|---|---|---|
| committer | DanConwayDev <DanConwayDev@protonmail.com> | 2026-01-09 15:38:58 +0000 |
| commit | e4cfecbfc909c9ca4983101cf6a5855959a5d49f (patch) | |
| tree | d56e91539c4c8a283905a651caf4c95cb130a2a1 /src/sync/mod.rs | |
| parent | 29be4cb7d0fbd29325c995a76ba1b1f47beecca5 (diff) | |
feat: Add two-tier rejected events index
Implements a sophisticated two-tier storage system for rejected repository
announcements to enable immediate re-processing when dependencies resolve.
## Architecture
**Tier 1: Hot Cache (2 minutes)**
- Stores full event objects for immediate re-processing
- Enables <1 second re-processing vs 24 hour wait
- Auto-expires to prevent memory growth
- Memory: ~200 KB typical, ~20 MB worst case
**Tier 2: Cold Index (7 days)**
- Stores metadata only (event_id, pubkey, identifier)
- Prevents repeated downloads of rejected events
- Enables invalidation when circumstances change
- Memory: ~1 MB typical
## Problem Solved
Without this system, maintainer announcements face a timing gap:
00:00 - Maintainer announcement rejected → Event discarded
00:02 - Owner announcement accepted (lists maintainer) → Want to re-process
00:02 - ❌ Maintainer announcement GONE → Must wait 24h for next sync
With two-tier system:
00:00 - Maintainer announcement rejected → Stored in both tiers
00:02 - Owner announcement accepted → Invalidate + get from hot cache
00:02 - ✅ Re-process immediately → Accepted in <1 second
## Implementation
New module: src/sync/rejected_index.rs
- RejectedEventsIndex: Public API combining both tiers
- HotCache: Internal struct for full event storage
- ColdIndex: Internal struct for metadata storage
- RejectionReason: Enum for tracking why events were rejected
Key methods:
- add_announcement(): Add to both tiers
- contains(): Check if event is rejected
- invalidate_and_get_events(): Remove from cold index, get from hot cache
- cleanup_expired(): Remove expired entries from both tiers
## Testing
9 comprehensive unit tests covering:
- Hot cache storage and retrieval
- Hot cache expiration
- Cold index metadata tracking
- Cold index invalidation
- Two-tier integration
- Cleanup of expired entries
- Hot cache misses after expiry
- Multiple maintainer repositories
All tests passing.
## Next Steps
PR2: Switch SyncManager to use new RejectedEventsIndex
PR3: Add invalidation + immediate re-processing logic
PR4: Add cleanup task + Prometheus metrics
Part of: Maintainer chain discovery fix
See: work/SOLUTION-SUMMARY-V2.md for full design
Diffstat (limited to 'src/sync/mod.rs')
| -rw-r--r-- | src/sync/mod.rs | 17 |
1 files changed, 13 insertions, 4 deletions
diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 8b1da0e..55bea17 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs | |||
| @@ -16,6 +16,7 @@ pub mod algorithms; | |||
| 16 | pub mod filters; | 16 | pub mod filters; |
| 17 | pub mod health; | 17 | pub mod health; |
| 18 | pub mod metrics; | 18 | pub mod metrics; |
| 19 | pub mod rejected_index; | ||
| 19 | pub mod relay_connection; | 20 | pub mod relay_connection; |
| 20 | pub mod self_subscriber; | 21 | pub mod self_subscriber; |
| 21 | 22 | ||
| @@ -25,6 +26,11 @@ pub use algorithms::{AddFilters, RelaySyncNeeds}; | |||
| 25 | // Re-export metrics types | 26 | // Re-export metrics types |
| 26 | pub use metrics::SyncMetrics; | 27 | pub use metrics::SyncMetrics; |
| 27 | 28 | ||
| 29 | // Re-export rejected index types | ||
| 30 | pub use rejected_index::{RejectionReason}; | ||
| 31 | // Note: RejectedEventsIndex struct exists in rejected_index.rs but not yet used | ||
| 32 | // Current code still uses the simple HashSet type alias below | ||
| 33 | |||
| 28 | // Re-export relay connection types | 34 | // Re-export relay connection types |
| 29 | pub use relay_connection::{NegentropySyncResult, RelayConnection, RelayEvent}; | 35 | pub use relay_connection::{NegentropySyncResult, RelayConnection, RelayEvent}; |
| 30 | 36 | ||
| @@ -67,7 +73,10 @@ pub type PendingSyncIndex = Arc<RwLock<HashMap<String, Vec<PendingBatch>>>>; | |||
| 67 | /// Tracks EventIds of announcement events (30617/30618) that were rejected during sync. | 73 | /// Tracks EventIds of announcement events (30617/30618) that were rejected during sync. |
| 68 | /// These events are excluded from negentropy sync and skipped during REQ+EOSE processing | 74 | /// These events are excluded from negentropy sync and skipped during REQ+EOSE processing |
| 69 | /// to avoid repeatedly fetching and rejecting the same events. | 75 | /// to avoid repeatedly fetching and rejecting the same events. |
| 70 | pub type RejectedEventsIndex = Arc<RwLock<HashSet<EventId>>>; | 76 | /// |
| 77 | /// NOTE: This is a temporary simple implementation. PR2 will replace this with the | ||
| 78 | /// two-tier RejectedEventsIndex from rejected_index.rs (hot cache + cold index). | ||
| 79 | type RejectedEventsIndexSimple = Arc<RwLock<HashSet<EventId>>>; | ||
| 71 | 80 | ||
| 72 | // ============================================================================= | 81 | // ============================================================================= |
| 73 | // Supporting Data Structures | 82 | // Supporting Data Structures |
| @@ -436,7 +445,7 @@ pub struct SyncManager { | |||
| 436 | /// In-flight subscription batches | 445 | /// In-flight subscription batches |
| 437 | pending_sync_index: PendingSyncIndex, | 446 | pending_sync_index: PendingSyncIndex, |
| 438 | /// Rejected announcement event IDs (30617/30618) - excluded from sync | 447 | /// Rejected announcement event IDs (30617/30618) - excluded from sync |
| 439 | rejected_events_index: RejectedEventsIndex, | 448 | rejected_events_index: RejectedEventsIndexSimple, |
| 440 | /// Active relay connections - keyed by relay URL | 449 | /// Active relay connections - keyed by relay URL |
| 441 | connections: HashMap<String, RelayConnection>, | 450 | connections: HashMap<String, RelayConnection>, |
| 442 | /// Health tracker for relay connection state | 451 | /// Health tracker for relay connection state |
| @@ -1992,7 +2001,7 @@ impl SyncManager { | |||
| 1992 | database: &SharedDatabase, | 2001 | database: &SharedDatabase, |
| 1993 | write_policy: &Nip34WritePolicy, | 2002 | write_policy: &Nip34WritePolicy, |
| 1994 | local_relay: &LocalRelay, | 2003 | local_relay: &LocalRelay, |
| 1995 | rejected_events_index: &RejectedEventsIndex, | 2004 | rejected_events_index: &RejectedEventsIndexSimple, |
| 1996 | ) -> ProcessResult { | 2005 | ) -> ProcessResult { |
| 1997 | use nostr_relay_builder::prelude::{WritePolicy, WritePolicyResult}; | 2006 | use nostr_relay_builder::prelude::{WritePolicy, WritePolicyResult}; |
| 1998 | use std::net::{IpAddr, Ipv4Addr, SocketAddr}; | 2007 | use std::net::{IpAddr, Ipv4Addr, SocketAddr}; |
| @@ -2880,7 +2889,7 @@ mod tests { | |||
| 2880 | #[tokio::test] | 2889 | #[tokio::test] |
| 2881 | async fn test_rejected_events_index_tracks_announcements() { | 2890 | async fn test_rejected_events_index_tracks_announcements() { |
| 2882 | // Create a rejected events index | 2891 | // Create a rejected events index |
| 2883 | let rejected_index: RejectedEventsIndex = Arc::new(RwLock::new(HashSet::new())); | 2892 | let rejected_index: RejectedEventsIndexSimple = Arc::new(RwLock::new(HashSet::new())); |
| 2884 | 2893 | ||
| 2885 | // Create test announcement event (kind 30617) | 2894 | // Create test announcement event (kind 30617) |
| 2886 | let keys = Keys::generate(); | 2895 | let keys = Keys::generate(); |