Merge relay.ngit.dev migration: bug fixes and migration tooling

This merge includes critical bug fixes and comprehensive migration tooling
developed during the relay.ngit.dev migration effort.

Bug Fixes:
- Fix git protocol error handling to return HTTP 200 with ERR pkt-line
- Fix naughty list false positives and DNS failure identification
- Fix database query filters in load_existing_events (remove .since())
- Fix OID fetch tracking to distinguish 0 OIDs from successful fetches
- Fix purgatory event source tracking for filtered expiry logging
- Implement OID retry logic for 'not our ref' errors

Migration Tools & Documentation:
- Complete 5-phase migration analysis pipeline with orchestration script
- Phase 1: Event fetching from source relay
- Phase 2: Git sync verification
- Phase 3: Categorization and relay comparison
- Phase 4: Log extraction (parse failures, purgatory expiry)
- Phase 5: Action classification for migration decisions
- Comprehensive migration guide with lessons learned
- Troubleshooting guide for permission and corruption issues

Configuration:
- Add NGIT_LOG_LEVEL configuration option
- Update git throttle limits to 60/minute
- Improve logging throughout for better observability

2 files changed, 110 insertions, 75 deletions

diff --git a/src/purgatory/sync/context.rs b/src/purgatory/sync/context.rs
index 33c2d12..904f8af 100644
--- a/src/purgatory/sync/context.rs
+++ b/src/purgatory/sync/context.rs
@@ -361,94 +361,121 @@ impl SyncContext for RealSyncContext {
         let naughty_list = self.git_naughty_list.clone();
 
         tokio::task::spawn_blocking(move || -> Result<Vec<String>> {
-            // git fetch <remote> <sha1> <sha2> ... - fetch all OIDs with full history
-            let mut args = vec!["fetch", &url];
-            args.extend(missing_oids.iter().map(|s| s.as_str()));
-
-            let output = Command::new("git")
-                .args(&args)
-                .current_dir(&repo_path)
-                .output();
-
-            match output {
-                Ok(result) if result.status.success() => {
-                    // Count how many OIDs we now have
-                    let fetched: Vec<String> = missing_oids
-                        .iter()
-                        .filter(|oid| crate::git::oid_exists(&repo_path, oid))
-                        .cloned()
-                        .collect();
-
-                    debug!(fetched_count = fetched.len(), "Successfully fetched OIDs");
-
-                    Ok(fetched)
+            let mut remaining_oids = missing_oids.clone();
+            let mut missing_from_remote: Vec<String> = Vec::new();
+
+            // Retry loop: keep fetching until success or no OIDs left
+            loop {
+                if remaining_oids.is_empty() {
+                    // All OIDs were missing from remote
+                    debug!(
+                        url = %url,
+                        missing_count = missing_from_remote.len(),
+                        "All requested OIDs missing from remote"
+                    );
+                    return Ok(vec![]);
                 }
-                Ok(result) => {
-                    let stderr = String::from_utf8_lossy(&result.stderr);
-
-                    // Extract domain and classify error for naughty list
-                    if let Some(domain) = extract_domain(&url) {
-                        if let Some(category) = NaughtyListTracker::classify_error(&stderr) {
-                            let is_new = naughty_list.record(&domain, category, stderr.to_string());
-
-                            if is_new {
-                                tracing::warn!(
-                                    domain = %domain,
-                                    category = %category,
-                                    error = %stderr,
-                                    "Git remote domain added to naughty list"
-                                );
-                            } else {
-                                debug!(
-                                    domain = %domain,
-                                    category = %category,
-                                    "Git remote domain still on naughty list"
-                                );
-                            }
+
+                // git fetch <remote> <sha1> <sha2> ... - fetch all OIDs with full history
+                let mut args = vec!["fetch".to_string(), url.clone()];
+                args.extend(remaining_oids.iter().cloned());
+
+                let output = Command::new("git")
+                    .args(&args)
+                    .current_dir(&repo_path)
+                    .output();
+
+                match output {
+                    Ok(result) if result.status.success() => {
+                        // Fetch succeeded - count how many OIDs we now have
+                        let fetched: Vec<String> = missing_oids
+                            .iter()
+                            .filter(|oid| crate::git::oid_exists(&repo_path, oid))
+                            .cloned()
+                            .collect();
+
+                        if !missing_from_remote.is_empty() {
+                            debug!(
+                                url = %url,
+                                fetched_count = fetched.len(),
+                                missing_count = missing_from_remote.len(),
+                                missing_oids = ?missing_from_remote,
+                                "Fetch completed after retries - some OIDs were missing from remote"
+                            );
+                        } else {
+                            debug!(url = %url, fetched_count = fetched.len(), "Successfully fetched OIDs");
                         }
+
+                        return Ok(fetched);
                     }
+                    Ok(result) => {
+                        let stderr = String::from_utf8_lossy(&result.stderr);
 
-                    // Check for "not our ref" errors and provide a clearer error message
-                    let error_msg = if stderr.contains("upload-pack: not our ref") {
-                        // Parse out the missing OID from stderr (git only reports one at a time)
-                        let missing_oid = stderr
-                            .lines()
-                            .find_map(|line| {
+                        // Check for "not our ref" error - this is retryable
+                        if stderr.contains("upload-pack: not our ref") {
+                            // Parse out the missing OID from stderr
+                            let missing_oid = stderr.lines().find_map(|line| {
                                 if line.contains("not our ref") {
                                     // Extract the OID from lines like:
                                     // "fatal: remote error: upload-pack: not our ref <oid>"
-                                    line.split("not our ref").nth(1).map(|s| s.trim().to_string())
+                                    line.split("not our ref")
+                                        .nth(1)
+                                        .map(|s| s.trim().to_string())
                                 } else {
                                     None
                                 }
                             });
 
-                        let total_requested = missing_oids.len();
+                            if let Some(ref oid) = missing_oid {
+                                // Remove the missing OID and retry with remaining
+                                remaining_oids.retain(|o| o != oid);
+                                missing_from_remote.push(oid.clone());
 
-                        if let Some(oid) = missing_oid {
-                            if total_requested > 1 {
-                                // BUG: Git stops at first missing OID, so we don't know if the others exist
-                                // We need retry logic to fetch remaining OIDs individually
-                                tracing::warn!(
+                                debug!(
                                     url = %url,
                                     missing_oid = %oid,
-                                    total_requested = total_requested,
-                                    "Git fetch failed on first missing OID - other requested OIDs may exist but were not fetched. Retry logic needed."
+                                    remaining_count = remaining_oids.len(),
+                                    "OID not found on remote, retrying with remaining OIDs"
                                 );
-                                format!("remote missing oid {} (BUG: {} other oids not attempted)", oid, total_requested - 1)
-                            } else {
-                                format!("remote missing only oid requested: {}", oid)
+
+                                continue; // Retry with remaining OIDs
+                            }
+                        }
+
+                        // Non-retryable error - record to naughty list and return error
+                        if let Some(domain) = extract_domain(&url) {
+                            if let Some(category) = NaughtyListTracker::classify_error(&stderr) {
+                                let is_new =
+                                    naughty_list.record(&domain, category, stderr.to_string());
+
+                                if is_new {
+                                    tracing::warn!(
+                                        domain = %domain,
+                                        category = %category,
+                                        error = %stderr,
+                                        "Git remote domain added to naughty list"
+                                    );
+                                } else {
+                                    debug!(
+                                        domain = %domain,
+                                        category = %category,
+                                        error = %stderr,
+                                        "Git fetch failed (domain on naughty list)"
+                                    );
+                                }
                             }
-                        } else {
-                            format!("git fetch failed: {}", stderr)
                         }
-                    } else {
-                        format!("git fetch failed: {}", stderr)
-                    };
 
-                    Err(anyhow::anyhow!("{}", error_msg))
+                        return Err(anyhow::anyhow!("git fetch failed for {}: {}", url, stderr));
+                    }
+                    Err(e) => {
+                        return Err(anyhow::anyhow!(
+                            "git fetch command error for {}: {}",
+                            url,
+                            e
+                        ))
+                    }
                 }
-                Err(e) => Err(anyhow::anyhow!("git fetch command error: {}", e)),
             }
         })
         .await
diff --git a/src/purgatory/sync/functions.rs b/src/purgatory/sync/functions.rs
index 65d29af..9207d58 100644
--- a/src/purgatory/sync/functions.rs
+++ b/src/purgatory/sync/functions.rs
@@ -368,15 +368,23 @@ pub async fn sync_identifier_from_url<C: SyncContext + ?Sized>(
     let fetch_result = ctx.fetch_oids(&target_repo, url, &needed_oids).await;
     throttle_manager.complete_request(&domain);
 
-    let oids_fetched = match fetch_result {
-        Ok(fetched) => {
+    let fetched_oids = match fetch_result {
+        Ok(fetched) if !fetched.is_empty() => {
             debug!(
                 identifier = %identifier,
                 url = %url,
                 oids_fetched = fetched.len(),
                 "Fetch succeeded"
             );
-            fetched.len()
+            fetched
+        }
+        Ok(_) => {
+            debug!(
+                identifier = %identifier,
+                url = %url,
+                "Fetch returned no OIDs (not available on remote)"
+            );
+            vec![]
         }
         Err(e) => {
             debug!(
@@ -385,13 +393,13 @@ pub async fn sync_identifier_from_url<C: SyncContext + ?Sized>(
                 error = %e,
                 "Fetch failed"
             );
-            0
+            vec![]
         }
     };
 
     // Try to process any events that can now be satisfied
-    if oids_fetched > 0 {
-        let new_oids: HashSet<String> = needed_oids.into_iter().collect();
+    if !fetched_oids.is_empty() {
+        let new_oids: HashSet<String> = fetched_oids.iter().cloned().collect();
         if let Err(e) = ctx
             .process_newly_available_git_data(&target_repo, &new_oids)
             .await
@@ -404,7 +412,7 @@ pub async fn sync_identifier_from_url<C: SyncContext + ?Sized>(
         }
     }
 
-    oids_fetched
+    fetched_oids.len()
 }
 
 /// Sync git data for an identifier.