Merge relay.ngit.dev migration: bug fixes and migration tooling

This merge includes critical bug fixes and comprehensive migration tooling
developed during the relay.ngit.dev migration effort.

Bug Fixes:
- Fix git protocol error handling to return HTTP 200 with ERR pkt-line
- Fix naughty list false positives and DNS failure identification
- Fix database query filters in load_existing_events (remove .since())
- Fix OID fetch tracking to distinguish 0 OIDs from successful fetches
- Fix purgatory event source tracking for filtered expiry logging
- Implement OID retry logic for 'not our ref' errors

Migration Tools & Documentation:
- Complete 5-phase migration analysis pipeline with orchestration script
- Phase 1: Event fetching from source relay
- Phase 2: Git sync verification
- Phase 3: Categorization and relay comparison
- Phase 4: Log extraction (parse failures, purgatory expiry)
- Phase 5: Action classification for migration decisions
- Comprehensive migration guide with lessons learned
- Troubleshooting guide for permission and corruption issues

Configuration:
- Add NGIT_LOG_LEVEL configuration option
- Update git throttle limits to 60/minute
- Improve logging throughout for better observability

1 files changed, 26 insertions, 48 deletions

diff --git a/src/config.rs b/src/config.rs
index 271a340..dd7b1e3 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -109,22 +109,25 @@ impl WhitelistEntry {
 }
 
 /// GRASP-05 Archive mode configuration
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct ArchiveConfig {
     /// Accept all repository announcements (no filtering)
     ///
     /// WARNING: Setting this to true allows anyone to mirror any repository
     /// to this relay, potentially causing storage/bandwidth exhaustion.
+    #[serde(default)]
     pub archive_all: bool,
 
     /// Whitelist entries for selective archiving
     ///
     /// If empty and archive_all is false, GRASP-05 is disabled (GRASP-01 strict mode).
+    #[serde(default)]
     pub whitelist: Vec<WhitelistEntry>,
 
     /// GRASP server domains to archive (archive all repositories from these domains)
     ///
     /// If non-empty, archives all repositories from the specified GRASP server domains.
+    #[serde(default)]
     pub grasp_services: Vec<String>,
 
     /// Read-only archive mode: relay is a read-only sync of archived repositories
@@ -132,6 +135,7 @@ pub struct ArchiveConfig {
     /// When true, the relay ONLY accepts announcements matching the archive whitelist/all.
     /// Announcements listing the relay but not in the whitelist are rejected.
     /// When false, the relay operates in GRASP-01 mode for unwhitelisted repos.
+    #[serde(default)]
     pub read_only: bool,
 }
 
@@ -146,6 +150,7 @@ impl ArchiveConfig {
     /// Returns true if:
     /// - archive_all is true, OR
     /// - announcement matches any whitelist entry
+    ///
     /// Note: grasp_services matching is handled via matches_grasp_services()
     pub fn matches(&self, npub: &str, identifier: &str) -> bool {
         if self.archive_all {
@@ -171,23 +176,13 @@ impl ArchiveConfig {
     }
 }
 
-impl Default for ArchiveConfig {
-    fn default() -> Self {
-        Self {
-            archive_all: false,
-            whitelist: Vec::new(),
-            grasp_services: Vec::new(),
-            read_only: false,
-        }
-    }
-}
-
 /// Repository whitelist configuration
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct RepositoryConfig {
     /// Whitelist entries for selective repository acceptance
     ///
     /// If empty, all repositories listing the service are accepted (GRASP-01 mode).
+    #[serde(default)]
     pub whitelist: Vec<WhitelistEntry>,
 }
 
@@ -207,21 +202,14 @@ impl RepositoryConfig {
     }
 }
 
-impl Default for RepositoryConfig {
-    fn default() -> Self {
-        Self {
-            whitelist: Vec::new(),
-        }
-    }
-}
-
 /// Repository blacklist configuration
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct BlacklistConfig {
     /// Blacklist entries for blocking specific repositories
     ///
     /// If empty, no repositories are blacklisted.
     /// Blacklist takes precedence over both archive and repository whitelists.
+    #[serde(default)]
     pub blacklist: Vec<WhitelistEntry>,
 }
 
@@ -256,21 +244,14 @@ impl BlacklistConfig {
     }
 }
 
-impl Default for BlacklistConfig {
-    fn default() -> Self {
-        Self {
-            blacklist: Vec::new(),
-        }
-    }
-}
-
 /// Event blacklist configuration for blocking events by author npub
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct EventBlacklistConfig {
     /// Blacklisted npubs - events from these authors are rejected
     ///
     /// If empty, no events are blacklisted by author.
     /// Applies to ALL event types, preventing events from reaching both the relay and purgatory.
+    #[serde(default)]
     pub blacklisted_npubs: Vec<String>,
 }
 
@@ -292,14 +273,6 @@ impl EventBlacklistConfig {
     }
 }
 
-impl Default for EventBlacklistConfig {
-    fn default() -> Self {
-        Self {
-            blacklisted_npubs: Vec::new(),
-        }
-    }
-}
-
 /// Database backend type for the relay
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default, ValueEnum)]
 #[serde(rename_all = "lowercase")]
@@ -500,6 +473,10 @@ pub struct Config {
     /// Prevents connection exhaustion DoS attacks
     #[arg(long, env = "NGIT_MAX_CONNECTIONS", default_value_t = 4096)]
     pub max_connections: usize,
+
+    /// Log level for application logging
+    #[arg(long, env = "NGIT_LOG_LEVEL", default_value = "info")]
+    pub log_level: String,
 }
 
 impl Config {
@@ -782,6 +759,7 @@ impl Config {
             repository_blacklist: String::new(),
             event_blacklist: String::new(),
             max_connections: 500,
+            log_level: "debug".to_string(),
         }
     }
 }
@@ -1103,14 +1081,14 @@ mod tests {
     fn test_archive_read_only_defaults() {
         // Default: false when no archive mode
         let config = Config::for_testing();
-        assert_eq!(config.archive_config().read_only, false);
+        assert!(!config.archive_config().read_only);
 
         // Default: true when archive_all is set
         let config = Config {
             archive_all: true,
             ..Config::for_testing()
         };
-        assert_eq!(config.archive_config().read_only, true);
+        assert!(config.archive_config().read_only);
 
         // Default: true when archive_whitelist is set
         let keys = Keys::generate();
@@ -1119,7 +1097,7 @@ mod tests {
             archive_whitelist: test_npub,
             ..Config::for_testing()
         };
-        assert_eq!(config.archive_config().read_only, true);
+        assert!(config.archive_config().read_only);
     }
 
     #[test]
@@ -1130,7 +1108,7 @@ mod tests {
             archive_read_only: Some(true),
             ..Config::for_testing()
         };
-        assert_eq!(config.archive_config().read_only, true);
+        assert!(config.archive_config().read_only);
 
         // Explicit false with archive_all (unusual but allowed)
         let config = Config {
@@ -1138,14 +1116,14 @@ mod tests {
             archive_read_only: Some(false),
             ..Config::for_testing()
         };
-        assert_eq!(config.archive_config().read_only, false);
+        assert!(!config.archive_config().read_only);
 
         // Explicit false without archive mode
         let config = Config {
             archive_read_only: Some(false),
             ..Config::for_testing()
         };
-        assert_eq!(config.archive_config().read_only, false);
+        assert!(!config.archive_config().read_only);
     }
 
     #[test]
@@ -1548,7 +1526,7 @@ mod tests {
         };
         let archive_config = config.archive_config();
         assert!(archive_config.enabled());
-        assert_eq!(archive_config.read_only, true); // Default to true
+        assert!(archive_config.read_only); // Default to true
     }
 
     #[test]
@@ -1558,7 +1536,7 @@ mod tests {
             archive_grasp_services: "git.example.com".to_string(),
             ..Config::for_testing()
         };
-        assert_eq!(config.archive_config().read_only, true);
+        assert!(config.archive_config().read_only);
     }
 
     #[test]
@@ -1569,7 +1547,7 @@ mod tests {
             archive_read_only: Some(false),
             ..Config::for_testing()
         };
-        assert_eq!(config.archive_config().read_only, false);
+        assert!(!config.archive_config().read_only);
     }
 
     #[test]