upleb.uk

Public git repos — served from a NIP-34 GRASP relay at git.upleb.uk

summaryrefslogtreecommitdiff
path: root/src/purgatory/sync/throttle.rs
diff options
context:
space:
mode:
authorDanConwayDev <DanConwayDev@protonmail.com>2026-01-10 02:14:01 +0000
committerDanConwayDev <DanConwayDev@protonmail.com>2026-01-10 02:15:22 +0000
commit1b6b669b9b82d1f81b887a32055f19c53d3bb8bf (patch)
tree3ce1785757cb3f16dfa30d74557042973d3bf53f /src/purgatory/sync/throttle.rs
parent730f430c906c6c2d43ea8f2e5fc3b408a3de128b (diff)
Add naughty list for git remotes with persistent SSL/DNS errors
Implement domain-level naughty list tracking for git remotes, reusing the existing NaughtyListTracker from relay sync. This prevents repeated attempts to fetch from git domains with persistent infrastructure issues (SSL/TLS certificate errors, DNS failures). Changes: - Updated NaughtyListTracker to track both relay URLs and git domains - Added git_naughty_list field to RealSyncContext for error classification - Modified fetch_oids() to classify git fetch errors and record naughty domains - Updated sync_identifier_next_url() to filter out naughty domains during URL selection - Added git_naughty_list parameter to ThrottleManager for domain queue processing - Threaded naughty list through start_sync_loop and all sync functions - Updated all tests to pass naughty list parameter The naughty list uses 12-hour expiration (configurable) to allow domains to recover from infrastructure issues. First occurrence logs WARN, repeats log DEBUG.
Diffstat (limited to 'src/purgatory/sync/throttle.rs')
-rw-r--r--src/purgatory/sync/throttle.rs35
1 files changed, 32 insertions, 3 deletions
diff --git a/src/purgatory/sync/throttle.rs b/src/purgatory/sync/throttle.rs
index ad6e8ea..7f8f636 100644
--- a/src/purgatory/sync/throttle.rs
+++ b/src/purgatory/sync/throttle.rs
@@ -24,6 +24,7 @@ use tracing::debug;
24 24
25use super::context::SyncContext; 25use super::context::SyncContext;
26use super::functions::{sync_identifier_from_url, sync_identifier_next_url}; 26use super::functions::{sync_identifier_from_url, sync_identifier_next_url};
27use crate::sync::naughty_list::NaughtyListTracker;
27 28
28/// State for an identifier waiting in a domain's queue. 29/// State for an identifier waiting in a domain's queue.
29/// 30///
@@ -265,6 +266,10 @@ pub struct ThrottleManager {
265 /// Sync context for processing queued identifiers. 266 /// Sync context for processing queued identifiers.
266 /// Set once at startup via `set_context()`. 267 /// Set once at startup via `set_context()`.
267 ctx: OnceLock<Arc<dyn SyncContext>>, 268 ctx: OnceLock<Arc<dyn SyncContext>>,
269
270 /// Naughty list tracker for git remote domains with persistent errors.
271 /// Set once at startup via `set_git_naughty_list()`.
272 git_naughty_list: OnceLock<Arc<NaughtyListTracker>>,
268} 273}
269 274
270impl ThrottleManager { 275impl ThrottleManager {
@@ -279,6 +284,7 @@ impl ThrottleManager {
279 max_concurrent_per_domain: max_concurrent, 284 max_concurrent_per_domain: max_concurrent,
280 max_per_minute_per_domain: max_per_minute, 285 max_per_minute_per_domain: max_per_minute,
281 ctx: OnceLock::new(), 286 ctx: OnceLock::new(),
287 git_naughty_list: OnceLock::new(),
282 } 288 }
283 } 289 }
284 290
@@ -294,6 +300,17 @@ impl ThrottleManager {
294 let _ = self.ctx.set(ctx); 300 let _ = self.ctx.set(ctx);
295 } 301 }
296 302
303 /// Set the git naughty list tracker (called once at startup).
304 ///
305 /// The naughty list is used to filter out domains with persistent errors
306 /// during URL selection.
307 ///
308 /// # Arguments
309 /// * `git_naughty_list` - The naughty list tracker
310 pub fn set_git_naughty_list(&self, git_naughty_list: Arc<NaughtyListTracker>) {
311 let _ = self.git_naughty_list.set(git_naughty_list);
312 }
313
297 /// Check if a domain is currently throttled (at capacity). 314 /// Check if a domain is currently throttled (at capacity).
298 /// 315 ///
299 /// Returns true if the domain has no capacity for another request, 316 /// Returns true if the domain has no capacity for another request,
@@ -479,10 +496,22 @@ impl ThrottleManager {
479 .unwrap_or_default() 496 .unwrap_or_default()
480 }; 497 };
481 498
499 // Get naughty list (should be set at startup)
500 let naughty_list = self
501 .git_naughty_list
502 .get()
503 .expect("git_naughty_list not set");
504
482 // Get next URL for this identifier on this specific domain 505 // Get next URL for this identifier on this specific domain
483 let url = 506 let url = sync_identifier_next_url(
484 sync_identifier_next_url(ctx.as_ref(), identifier, Some(domain), &tried_urls, self) 507 ctx.as_ref(),
485 .await; 508 identifier,
509 Some(domain),
510 &tried_urls,
511 self,
512 naughty_list.as_ref(),
513 )
514 .await;
486 515
487 match url { 516 match url {
488 Some(url) => { 517 Some(url) => {