diff options
| author | DanConwayDev <DanConwayDev@protonmail.com> | 2026-01-10 02:14:01 +0000 |
|---|---|---|
| committer | DanConwayDev <DanConwayDev@protonmail.com> | 2026-01-10 02:15:22 +0000 |
| commit | 1b6b669b9b82d1f81b887a32055f19c53d3bb8bf (patch) | |
| tree | 3ce1785757cb3f16dfa30d74557042973d3bf53f /src/purgatory/sync/loop.rs | |
| parent | 730f430c906c6c2d43ea8f2e5fc3b408a3de128b (diff) | |
Add naughty list for git remotes with persistent SSL/DNS errors
Implement domain-level naughty list tracking for git remotes, reusing the
existing NaughtyListTracker from relay sync. This prevents repeated attempts
to fetch from git domains with persistent infrastructure issues (SSL/TLS
certificate errors, DNS failures).
Changes:
- Updated NaughtyListTracker to track both relay URLs and git domains
- Added git_naughty_list field to RealSyncContext for error classification
- Modified fetch_oids() to classify git fetch errors and record naughty domains
- Updated sync_identifier_next_url() to filter out naughty domains during URL selection
- Added git_naughty_list parameter to ThrottleManager for domain queue processing
- Threaded naughty list through start_sync_loop and all sync functions
- Updated all tests to pass naughty list parameter
The naughty list uses 12-hour expiration (configurable) to allow domains to
recover from infrastructure issues. First occurrence logs WARN, repeats log DEBUG.
Diffstat (limited to 'src/purgatory/sync/loop.rs')
| -rw-r--r-- | src/purgatory/sync/loop.rs | 15 |
1 files changed, 13 insertions, 2 deletions
diff --git a/src/purgatory/sync/loop.rs b/src/purgatory/sync/loop.rs index 92e0594..1ab229d 100644 --- a/src/purgatory/sync/loop.rs +++ b/src/purgatory/sync/loop.rs | |||
| @@ -15,6 +15,7 @@ use tokio::task::JoinHandle; | |||
| 15 | use tracing::{debug, info}; | 15 | use tracing::{debug, info}; |
| 16 | 16 | ||
| 17 | use crate::purgatory::Purgatory; | 17 | use crate::purgatory::Purgatory; |
| 18 | use crate::sync::naughty_list::NaughtyListTracker; | ||
| 18 | 19 | ||
| 19 | use super::context::SyncContext; | 20 | use super::context::SyncContext; |
| 20 | use super::functions::sync_identifier; | 21 | use super::functions::sync_identifier; |
| @@ -37,6 +38,7 @@ impl Purgatory { | |||
| 37 | /// # Arguments | 38 | /// # Arguments |
| 38 | /// * `ctx` - The sync context providing repository data and fetch capabilities | 39 | /// * `ctx` - The sync context providing repository data and fetch capabilities |
| 39 | /// * `throttle_manager` - Used for rate limiting and domain queue management | 40 | /// * `throttle_manager` - Used for rate limiting and domain queue management |
| 41 | /// * `git_naughty_list` - Tracker for git remote domains with persistent errors | ||
| 40 | /// | 42 | /// |
| 41 | /// # Returns | 43 | /// # Returns |
| 42 | /// A `JoinHandle` for the background task (can be used to cancel the loop) | 44 | /// A `JoinHandle` for the background task (can be used to cancel the loop) |
| @@ -47,12 +49,13 @@ impl Purgatory { | |||
| 47 | /// let purgatory = Arc::new(Purgatory::new("/data/git")); | 49 | /// let purgatory = Arc::new(Purgatory::new("/data/git")); |
| 48 | /// let ctx = Arc::new(RealSyncContext::new(...)); | 50 | /// let ctx = Arc::new(RealSyncContext::new(...)); |
| 49 | /// let throttle_manager = Arc::new(ThrottleManager::new(5, 30)); | 51 | /// let throttle_manager = Arc::new(ThrottleManager::new(5, 30)); |
| 52 | /// let git_naughty_list = Arc::new(NaughtyListTracker::with_defaults()); | ||
| 50 | /// | 53 | /// |
| 51 | /// // Set context on throttle manager for queue processing | 54 | /// // Set context on throttle manager for queue processing |
| 52 | /// throttle_manager.set_context(ctx.clone()); | 55 | /// throttle_manager.set_context(ctx.clone()); |
| 53 | /// | 56 | /// |
| 54 | /// // Start the sync loop | 57 | /// // Start the sync loop |
| 55 | /// let handle = purgatory.start_sync_loop(ctx, throttle_manager); | 58 | /// let handle = purgatory.start_sync_loop(ctx, throttle_manager, git_naughty_list); |
| 56 | /// | 59 | /// |
| 57 | /// // Later, to stop the loop: | 60 | /// // Later, to stop the loop: |
| 58 | /// handle.abort(); | 61 | /// handle.abort(); |
| @@ -61,6 +64,7 @@ impl Purgatory { | |||
| 61 | self: Arc<Self>, | 64 | self: Arc<Self>, |
| 62 | ctx: Arc<dyn SyncContext>, | 65 | ctx: Arc<dyn SyncContext>, |
| 63 | throttle_manager: Arc<ThrottleManager>, | 66 | throttle_manager: Arc<ThrottleManager>, |
| 67 | git_naughty_list: Arc<NaughtyListTracker>, | ||
| 64 | ) -> JoinHandle<()> { | 68 | ) -> JoinHandle<()> { |
| 65 | info!( | 69 | info!( |
| 66 | "Starting purgatory sync loop (interval: {:?})", | 70 | "Starting purgatory sync loop (interval: {:?})", |
| @@ -121,6 +125,7 @@ impl Purgatory { | |||
| 121 | let purgatory = self.clone(); | 125 | let purgatory = self.clone(); |
| 122 | let ctx = ctx.clone(); | 126 | let ctx = ctx.clone(); |
| 123 | let throttle_manager = throttle_manager.clone(); | 127 | let throttle_manager = throttle_manager.clone(); |
| 128 | let git_naughty_list = git_naughty_list.clone(); | ||
| 124 | let id = identifier.clone(); | 129 | let id = identifier.clone(); |
| 125 | 130 | ||
| 126 | tokio::spawn(async move { | 131 | tokio::spawn(async move { |
| @@ -129,7 +134,13 @@ impl Purgatory { | |||
| 129 | "Starting sync task for identifier" | 134 | "Starting sync task for identifier" |
| 130 | ); | 135 | ); |
| 131 | 136 | ||
| 132 | let complete = sync_identifier(ctx.as_ref(), &id, &throttle_manager).await; | 137 | let complete = sync_identifier( |
| 138 | ctx.as_ref(), | ||
| 139 | &id, | ||
| 140 | &throttle_manager, | ||
| 141 | git_naughty_list.as_ref(), | ||
| 142 | ) | ||
| 143 | .await; | ||
| 133 | 144 | ||
| 134 | // Check final state and update queue | 145 | // Check final state and update queue |
| 135 | if complete || !purgatory.has_pending_events(&id) { | 146 | if complete || !purgatory.has_pending_events(&id) { |