From 1c5dac680b5a446e26b161208a17030f5fbd8a88 Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Wed, 7 Jan 2026 11:44:27 +0000 Subject: Add core sync functions for identifier-based purgatory synchronization Implement sync_identifier_next_url and sync_identifier_from_url functions that provide the core URL selection and fetch logic for purgatory sync. sync_identifier_next_url: - Pure URL selection logic with no side effects - Filters out our own domain and already-tried URLs - Respects domain throttling when domain parameter is None - Can target a specific domain when domain parameter is Some sync_identifier_from_url: - Fetches OIDs from a specific URL via the SyncContext - Tracks request start/completion with ThrottleManager for rate limiting - Calls process_newly_available_git_data on successful fetch Also adds get_throttled_domains_with_untried_urls helper for the main sync loop to know which DomainThrottle queues to enqueue identifiers to. These functions are designed to be called by both: - Main sync loop (tries non-throttled URLs immediately) - DomainThrottle queue processing (when capacity frees up) Includes 10 unit tests covering: - Throttled domain skipping - Tried URL skipping - Our domain filtering - Specific domain targeting - Fetch success/failure handling - Throttle request tracking --- src/purgatory/sync/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/purgatory/sync/mod.rs') diff --git a/src/purgatory/sync/mod.rs b/src/purgatory/sync/mod.rs index 1ac0cb1..d26c1f0 100644 --- a/src/purgatory/sync/mod.rs +++ b/src/purgatory/sync/mod.rs @@ -7,10 +7,15 @@ //! - Debouncing for burst event arrivals mod context; +mod functions; mod queue; mod throttle; pub use context::{ProcessResult, SyncContext}; +pub use functions::{ + get_throttled_domains_with_untried_urls, sync_identifier_from_url, sync_identifier_next_url, + ThrottledDomainInfo, +}; pub use queue::SyncQueueEntry; pub use throttle::{DomainThrottle, ThrottleManager}; -- cgit v1.2.3