From 39242bfec6f6592c478c651f2e89e88e3e66ff2a Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Fri, 19 Dec 2025 16:37:28 +0000 Subject: feat(sync): implement pagination for historic_sync REQ+EOSE flow Add automatic pagination support for non-Negentropy historic sync to handle large result sets efficiently. When a subscription receives >= 75 events, the system automatically fetches the next page using the 'until' parameter. Changes: - Add PaginationState struct to track event counts and min timestamps - Add pagination_state HashMap to PendingBatch for per-subscription tracking - Add PAGINATION_THRESHOLD constant (75 events) - Pass pending_sync_index to event processor for state updates - Track events and timestamps as they arrive - Check threshold on EOSE and launch follow-up subscriptions - Initialize pagination state when creating historic sync subscriptions - Update test fixtures in algorithms.rs The pagination continues recursively until a page returns fewer than 75 events, ensuring complete historic data retrieval without overwhelming relay limits. --- src/sync/relay_connection.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/sync/relay_connection.rs') diff --git a/src/sync/relay_connection.rs b/src/sync/relay_connection.rs index 37094be..5a61777 100644 --- a/src/sync/relay_connection.rs +++ b/src/sync/relay_connection.rs @@ -23,8 +23,8 @@ use crate::nostr::builder::SharedDatabase; /// Events from a relay connection #[derive(Debug)] pub enum RelayEvent { - /// A new event was received - Event(Event), + /// A new event was received (event, subscription_id) + Event(Event, SubscriptionId), /// End of stored events for a subscription EndOfStoredEvents(SubscriptionId), /// Connection was closed @@ -216,7 +216,7 @@ impl RelayConnection { sub_id = %subscription_id, "Received event" ); - if event_sender.send(RelayEvent::Event(*event)).await.is_err() { + if event_sender.send(RelayEvent::Event(*event, subscription_id.clone())).await.is_err() { tracing::debug!(relay = %url, "Event sender closed, stopping event loop"); break; } -- cgit v1.2.3