diff options
| author | DanConwayDev <DanConwayDev@protonmail.com> | 2026-01-10 02:14:01 +0000 |
|---|---|---|
| committer | DanConwayDev <DanConwayDev@protonmail.com> | 2026-01-10 02:15:22 +0000 |
| commit | 1b6b669b9b82d1f81b887a32055f19c53d3bb8bf (patch) | |
| tree | 3ce1785757cb3f16dfa30d74557042973d3bf53f /src/main.rs | |
| parent | 730f430c906c6c2d43ea8f2e5fc3b408a3de128b (diff) | |
Add naughty list for git remotes with persistent SSL/DNS errors
Implement domain-level naughty list tracking for git remotes, reusing the
existing NaughtyListTracker from relay sync. This prevents repeated attempts
to fetch from git domains with persistent infrastructure issues (SSL/TLS
certificate errors, DNS failures).
Changes:
- Updated NaughtyListTracker to track both relay URLs and git domains
- Added git_naughty_list field to RealSyncContext for error classification
- Modified fetch_oids() to classify git fetch errors and record naughty domains
- Updated sync_identifier_next_url() to filter out naughty domains during URL selection
- Added git_naughty_list parameter to ThrottleManager for domain queue processing
- Threaded naughty list through start_sync_loop and all sync functions
- Updated all tests to pass naughty list parameter
The naughty list uses 12-hour expiration (configurable) to allow domains to
recover from infrastructure issues. First occurrence logs WARN, repeats log DEBUG.
Diffstat (limited to 'src/main.rs')
| -rw-r--r-- | src/main.rs | 14 |
1 files changed, 10 insertions, 4 deletions
diff --git a/src/main.rs b/src/main.rs index 5e9e2d0..44545b5 100644 --- a/src/main.rs +++ b/src/main.rs | |||
| @@ -12,7 +12,7 @@ use ngit_grasp::{ | |||
| 12 | metrics::Metrics, | 12 | metrics::Metrics, |
| 13 | nostr, | 13 | nostr, |
| 14 | purgatory::{sync::RealSyncContext, sync::ThrottleManager, Purgatory}, | 14 | purgatory::{sync::RealSyncContext, sync::ThrottleManager, Purgatory}, |
| 15 | sync::SyncManager, | 15 | sync::{naughty_list::NaughtyListTracker, SyncManager}, |
| 16 | }; | 16 | }; |
| 17 | 17 | ||
| 18 | #[tokio::main] | 18 | #[tokio::main] |
| @@ -132,23 +132,29 @@ async fn main() -> Result<()> { | |||
| 132 | info!("Expired event cleanup task started (24h interval, keeps 7 days)"); | 132 | info!("Expired event cleanup task started (24h interval, keeps 7 days)"); |
| 133 | 133 | ||
| 134 | // Start purgatory sync loop for background git data fetching | 134 | // Start purgatory sync loop for background git data fetching |
| 135 | // Create naughty list tracker for git remote domains with persistent errors (12h expiration) | ||
| 136 | let git_naughty_list = Arc::new(NaughtyListTracker::with_defaults()); | ||
| 137 | |||
| 135 | let sync_ctx = Arc::new(RealSyncContext::new( | 138 | let sync_ctx = Arc::new(RealSyncContext::new( |
| 136 | purgatory.clone(), | 139 | purgatory.clone(), |
| 137 | relay_with_db.database.clone(), | 140 | relay_with_db.database.clone(), |
| 138 | PathBuf::from(config.effective_git_data_path()), | 141 | PathBuf::from(config.effective_git_data_path()), |
| 139 | Some(config.domain.clone()), | 142 | Some(config.domain.clone()), |
| 140 | Some(relay_with_db.relay.clone()), | 143 | Some(relay_with_db.relay.clone()), |
| 144 | git_naughty_list.clone(), | ||
| 141 | )); | 145 | )); |
| 142 | 146 | ||
| 143 | // Create throttle manager for rate limiting remote git servers | 147 | // Create throttle manager for rate limiting remote git servers |
| 144 | // Default: 5 concurrent requests per domain, 30 requests per minute per domain | 148 | // Default: 5 concurrent requests per domain, 30 requests per minute per domain |
| 145 | let throttle_manager = Arc::new(ThrottleManager::new(5, 30)); | 149 | let throttle_manager = Arc::new(ThrottleManager::new(5, 30)); |
| 146 | throttle_manager.set_context(sync_ctx.clone()); | 150 | throttle_manager.set_context(sync_ctx.clone()); |
| 151 | throttle_manager.set_git_naughty_list(git_naughty_list.clone()); | ||
| 147 | 152 | ||
| 148 | // Start the sync loop | 153 | // Start the sync loop |
| 149 | let _sync_loop_handle = purgatory | 154 | let _sync_loop_handle = |
| 150 | .clone() | 155 | purgatory |
| 151 | .start_sync_loop(sync_ctx, throttle_manager); | 156 | .clone() |
| 157 | .start_sync_loop(sync_ctx, throttle_manager, git_naughty_list.clone()); | ||
| 152 | info!("Purgatory sync loop started (1s interval)"); | 158 | info!("Purgatory sync loop started (1s interval)"); |
| 153 | 159 | ||
| 154 | // Setup shutdown handler for purgatory cleanup | 160 | // Setup shutdown handler for purgatory cleanup |