From 39242bfec6f6592c478c651f2e89e88e3e66ff2a Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Fri, 19 Dec 2025 16:37:28 +0000 Subject: feat(sync): implement pagination for historic_sync REQ+EOSE flow Add automatic pagination support for non-Negentropy historic sync to handle large result sets efficiently. When a subscription receives >= 75 events, the system automatically fetches the next page using the 'until' parameter. Changes: - Add PaginationState struct to track event counts and min timestamps - Add pagination_state HashMap to PendingBatch for per-subscription tracking - Add PAGINATION_THRESHOLD constant (75 events) - Pass pending_sync_index to event processor for state updates - Track events and timestamps as they arrive - Check threshold on EOSE and launch follow-up subscriptions - Initialize pagination state when creating historic sync subscriptions - Update test fixtures in algorithms.rs The pagination continues recursively until a page returns fewer than 75 events, ensuring complete historic data retrieval without overwhelming relay limits. --- src/sync/algorithms.rs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/sync/algorithms.rs') diff --git a/src/sync/algorithms.rs b/src/sync/algorithms.rs index a6e0787..f4b1f5c 100644 --- a/src/sync/algorithms.rs +++ b/src/sync/algorithms.rs @@ -401,6 +401,7 @@ mod tests { }, outstanding_subs: HashSet::new(), sync_method: SyncMethod::ReqEose, + pagination_state: HashMap::new(), }], ); @@ -512,6 +513,7 @@ mod tests { }, outstanding_subs: HashSet::new(), sync_method: SyncMethod::ReqEose, + pagination_state: HashMap::new(), }], ); -- cgit v1.2.3