diff options
| author | DanConwayDev <DanConwayDev@protonmail.com> | 2025-12-04 15:17:04 +0000 |
|---|---|---|
| committer | DanConwayDev <DanConwayDev@protonmail.com> | 2025-12-04 15:24:19 +0000 |
| commit | fd0c87c787d0626b3546fa571541c9c809711821 (patch) | |
| tree | 934f20d973127f380b807d2bd44b25c197cf349c | |
| parent | 762cd8e815e797f173f541795de774fbbf978fc3 (diff) | |
add prometheus metrics
| -rw-r--r-- | .env.example | 20 | ||||
| -rw-r--r-- | Cargo.lock | 45 | ||||
| -rw-r--r-- | Cargo.toml | 5 | ||||
| -rw-r--r-- | README.md | 22 | ||||
| -rw-r--r-- | docs/explanation/monitoring-strategy.md | 462 | ||||
| -rw-r--r-- | docs/explanation/monitoring.md | 99 | ||||
| -rw-r--r-- | docs/grafana/ngit-grasp-dashboard.json | 675 | ||||
| -rw-r--r-- | docs/how-to/prometheus-setup.md | 178 | ||||
| -rw-r--r-- | src/config.rs | 36 | ||||
| -rw-r--r-- | src/http/mod.rs | 84 | ||||
| -rw-r--r-- | src/http/nip11.rs | 6 | ||||
| -rw-r--r-- | src/lib.rs | 1 | ||||
| -rw-r--r-- | src/main.rs | 17 | ||||
| -rw-r--r-- | src/metrics/bandwidth.rs | 301 | ||||
| -rw-r--r-- | src/metrics/connection.rs | 337 | ||||
| -rw-r--r-- | src/metrics/mod.rs | 469 |
16 files changed, 2285 insertions, 472 deletions
diff --git a/.env.example b/.env.example index 0a93b1f..9207dc4 100644 --- a/.env.example +++ b/.env.example | |||
| @@ -72,6 +72,26 @@ | |||
| 72 | # NGIT_DATABASE_BACKEND=lmdb | 72 | # NGIT_DATABASE_BACKEND=lmdb |
| 73 | 73 | ||
| 74 | # ============================================================================ | 74 | # ============================================================================ |
| 75 | # METRICS | ||
| 76 | # ============================================================================ | ||
| 77 | |||
| 78 | # Enable Prometheus metrics endpoint at /metrics | ||
| 79 | # CLI: --metrics-enabled | ||
| 80 | # Default: true | ||
| 81 | # NGIT_METRICS_ENABLED=true | ||
| 82 | |||
| 83 | # Connections per IP before flagging as potential abuse in metrics | ||
| 84 | # (display only, no rate limiting - purely for monitoring visibility) | ||
| 85 | # CLI: --metrics-connection-per-ip-abuse-threshold <count> | ||
| 86 | # Default: 10 | ||
| 87 | # NGIT_METRICS_CONNECTION_PER_IP_ABUSE_THRESHOLD=10 | ||
| 88 | |||
| 89 | # Number of top bandwidth repositories to track in metrics | ||
| 90 | # CLI: --metrics-top-n-repos <count> | ||
| 91 | # Default: 10 | ||
| 92 | # NGIT_METRICS_TOP_N_REPOS=10 | ||
| 93 | |||
| 94 | # ============================================================================ | ||
| 75 | # LOGGING | 95 | # LOGGING |
| 76 | # ============================================================================ | 96 | # ============================================================================ |
| 77 | 97 | ||
| @@ -468,6 +468,19 @@ dependencies = [ | |||
| 468 | ] | 468 | ] |
| 469 | 469 | ||
| 470 | [[package]] | 470 | [[package]] |
| 471 | name = "dashmap" | ||
| 472 | version = "5.5.3" | ||
| 473 | source = "registry+https://github.com/rust-lang/crates.io-index" | ||
| 474 | checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" | ||
| 475 | dependencies = [ | ||
| 476 | "cfg-if", | ||
| 477 | "hashbrown 0.14.5", | ||
| 478 | "lock_api", | ||
| 479 | "once_cell", | ||
| 480 | "parking_lot_core", | ||
| 481 | ] | ||
| 482 | |||
| 483 | [[package]] | ||
| 471 | name = "data-encoding" | 484 | name = "data-encoding" |
| 472 | version = "2.9.0" | 485 | version = "2.9.0" |
| 473 | source = "registry+https://github.com/rust-lang/crates.io-index" | 486 | source = "registry+https://github.com/rust-lang/crates.io-index" |
| @@ -813,6 +826,12 @@ dependencies = [ | |||
| 813 | 826 | ||
| 814 | [[package]] | 827 | [[package]] |
| 815 | name = "hashbrown" | 828 | name = "hashbrown" |
| 829 | version = "0.14.5" | ||
| 830 | source = "registry+https://github.com/rust-lang/crates.io-index" | ||
| 831 | checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" | ||
| 832 | |||
| 833 | [[package]] | ||
| 834 | name = "hashbrown" | ||
| 816 | version = "0.16.0" | 835 | version = "0.16.0" |
| 817 | source = "registry+https://github.com/rust-lang/crates.io-index" | 836 | source = "registry+https://github.com/rust-lang/crates.io-index" |
| 818 | checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" | 837 | checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" |
| @@ -1163,7 +1182,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" | |||
| 1163 | checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" | 1182 | checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" |
| 1164 | dependencies = [ | 1183 | dependencies = [ |
| 1165 | "equivalent", | 1184 | "equivalent", |
| 1166 | "hashbrown", | 1185 | "hashbrown 0.16.0", |
| 1167 | ] | 1186 | ] |
| 1168 | 1187 | ||
| 1169 | [[package]] | 1188 | [[package]] |
| @@ -1353,6 +1372,7 @@ dependencies = [ | |||
| 1353 | "anyhow", | 1372 | "anyhow", |
| 1354 | "base64 0.22.1", | 1373 | "base64 0.22.1", |
| 1355 | "clap", | 1374 | "clap", |
| 1375 | "dashmap", | ||
| 1356 | "dotenvy", | 1376 | "dotenvy", |
| 1357 | "flate2", | 1377 | "flate2", |
| 1358 | "futures-util", | 1378 | "futures-util", |
| @@ -1360,9 +1380,11 @@ dependencies = [ | |||
| 1360 | "http-body-util", | 1380 | "http-body-util", |
| 1361 | "hyper 1.8.1", | 1381 | "hyper 1.8.1", |
| 1362 | "hyper-util", | 1382 | "hyper-util", |
| 1383 | "lazy_static", | ||
| 1363 | "nostr-lmdb", | 1384 | "nostr-lmdb", |
| 1364 | "nostr-relay-builder", | 1385 | "nostr-relay-builder", |
| 1365 | "nostr-sdk 0.44.1", | 1386 | "nostr-sdk 0.44.1", |
| 1387 | "prometheus", | ||
| 1366 | "serde", | 1388 | "serde", |
| 1367 | "serde_json", | 1389 | "serde_json", |
| 1368 | "tempfile", | 1390 | "tempfile", |
| @@ -1787,6 +1809,27 @@ dependencies = [ | |||
| 1787 | ] | 1809 | ] |
| 1788 | 1810 | ||
| 1789 | [[package]] | 1811 | [[package]] |
| 1812 | name = "prometheus" | ||
| 1813 | version = "0.13.4" | ||
| 1814 | source = "registry+https://github.com/rust-lang/crates.io-index" | ||
| 1815 | checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" | ||
| 1816 | dependencies = [ | ||
| 1817 | "cfg-if", | ||
| 1818 | "fnv", | ||
| 1819 | "lazy_static", | ||
| 1820 | "memchr", | ||
| 1821 | "parking_lot", | ||
| 1822 | "protobuf", | ||
| 1823 | "thiserror 1.0.69", | ||
| 1824 | ] | ||
| 1825 | |||
| 1826 | [[package]] | ||
| 1827 | name = "protobuf" | ||
| 1828 | version = "2.28.0" | ||
| 1829 | source = "registry+https://github.com/rust-lang/crates.io-index" | ||
| 1830 | checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" | ||
| 1831 | |||
| 1832 | [[package]] | ||
| 1790 | name = "quote" | 1833 | name = "quote" |
| 1791 | version = "1.0.41" | 1834 | version = "1.0.41" |
| 1792 | source = "registry+https://github.com/rust-lang/crates.io-index" | 1835 | source = "registry+https://github.com/rust-lang/crates.io-index" |
| @@ -28,6 +28,11 @@ futures-util = "0.3" | |||
| 28 | base64 = "0.22" | 28 | base64 = "0.22" |
| 29 | flate2 = "1.0" | 29 | flate2 = "1.0" |
| 30 | 30 | ||
| 31 | # Metrics | ||
| 32 | prometheus = "0.13" | ||
| 33 | dashmap = "5" | ||
| 34 | lazy_static = "1.4" | ||
| 35 | |||
| 31 | # Serialization | 36 | # Serialization |
| 32 | serde = { version = "1.0", features = ["derive"] } | 37 | serde = { version = "1.0", features = ["derive"] } |
| 33 | serde_json = "1.0" | 38 | serde_json = "1.0" |
| @@ -94,9 +94,27 @@ Why this is useful: | |||
| 94 | 94 | ||
| 95 | dedupe git data = shared object database or (GIT_ALTERNATE_OBJECT_DIRECTORIES or .git/objects/info/alternates) | 95 | dedupe git data = shared object database or (GIT_ALTERNATE_OBJECT_DIRECTORIES or .git/objects/info/alternates) |
| 96 | 96 | ||
| 97 | ### Logging | 97 | ### Monitoring |
| 98 | 98 | ||
| 99 | See [docs/explanation/monitoring-strategy.md](docs/monitoring-strategy.md). Note: this needs refinement. | 99 | ngit-grasp exposes Prometheus metrics at `/metrics` for connection tracking, Git operations, and Nostr events. |
| 100 | |||
| 101 | **Configuration Options:** | ||
| 102 | |||
| 103 | | Option | CLI Flag | Environment Variable | Default | | ||
| 104 | |--------|----------|---------------------|---------| | ||
| 105 | | Metrics enabled | `--metrics-enabled` | `NGIT_METRICS_ENABLED` | `true` | | ||
| 106 | | Connection abuse threshold | `--metrics-connection-per-ip-abuse-threshold` | `NGIT_METRICS_CONNECTION_PER_IP_ABUSE_THRESHOLD` | `10` | | ||
| 107 | | Top N repos | `--metrics-top-n-repos` | `NGIT_METRICS_TOP_N_REPOS` | `10` | | ||
| 108 | |||
| 109 | **Key Metrics:** | ||
| 110 | - WebSocket connections (active, unique IPs, flagged abusers) | ||
| 111 | - Git operations (clone/fetch/push rates, bandwidth, authorization results) | ||
| 112 | - Nostr events (received, stored, rejected by kind) | ||
| 113 | - Top N repositories by bandwidth | ||
| 114 | |||
| 115 | **Privacy:** IP addresses are never exposed in metrics - only aggregate counts. | ||
| 116 | |||
| 117 | See [Monitoring Overview](docs/explanation/monitoring.md) and [Prometheus Setup Guide](docs/how-to/prometheus-setup.md) for deployment. | ||
| 100 | 118 | ||
| 101 | ### Delete Events | 119 | ### Delete Events |
| 102 | 120 | ||
diff --git a/docs/explanation/monitoring-strategy.md b/docs/explanation/monitoring-strategy.md deleted file mode 100644 index 4668305..0000000 --- a/docs/explanation/monitoring-strategy.md +++ /dev/null | |||
| @@ -1,462 +0,0 @@ | |||
| 1 | # Monitoring Strategy - Design Document | ||
| 2 | |||
| 3 | ## Overview | ||
| 4 | |||
| 5 | This document describes the logging and monitoring strategy for ngit-grasp, designed to help administrators: | ||
| 6 | |||
| 7 | 1. Monitor WebSocket connections per unique IP | ||
| 8 | 2. Correlate resource spikes (memory, CPU) with usage patterns | ||
| 9 | 3. Detect potential abuse (too many connections from single IP) | ||
| 10 | 4. Support future load-based scheduling of background jobs (GRASP-02 sync) | ||
| 11 | |||
| 12 | ## Architecture | ||
| 13 | |||
| 14 | ```mermaid | ||
| 15 | flowchart TB | ||
| 16 | subgraph ngit-grasp | ||
| 17 | HTTP[HTTP Service] | ||
| 18 | WS[WebSocket Handler] | ||
| 19 | GIT[Git Handlers] | ||
| 20 | RELAY[Nostr Relay] | ||
| 21 | |||
| 22 | subgraph Metrics Module | ||
| 23 | REG[Prometheus Registry] | ||
| 24 | CT[ConnectionTracker] | ||
| 25 | MC[Metric Counters] | ||
| 26 | end | ||
| 27 | |||
| 28 | ME[/metrics endpoint] | ||
| 29 | end | ||
| 30 | |||
| 31 | subgraph External | ||
| 32 | PROM[Prometheus Server] | ||
| 33 | GRAF[Grafana] | ||
| 34 | ADMIN[Admin Browser] | ||
| 35 | end | ||
| 36 | |||
| 37 | HTTP --> ME | ||
| 38 | WS --> CT | ||
| 39 | WS --> MC | ||
| 40 | GIT --> MC | ||
| 41 | RELAY --> MC | ||
| 42 | |||
| 43 | CT --> REG | ||
| 44 | MC --> REG | ||
| 45 | REG --> ME | ||
| 46 | |||
| 47 | PROM -->|scrape /metrics| ME | ||
| 48 | GRAF -->|query| PROM | ||
| 49 | ADMIN -->|view dashboards| GRAF | ||
| 50 | ``` | ||
| 51 | |||
| 52 | ## Metric Categories | ||
| 53 | |||
| 54 | ### 1. WebSocket Connection Metrics | ||
| 55 | |||
| 56 | | Metric Name | Type | Labels | Description | | ||
| 57 | |------------|------|--------|-------------| | ||
| 58 | | `ngit_websocket_connections_total` | Counter | - | Total WebSocket connections since startup | | ||
| 59 | | `ngit_websocket_connections_active` | Gauge | - | Current active WebSocket connections | | ||
| 60 | | `ngit_websocket_unique_ips` | Gauge | - | Number of unique IP addresses connected (NOT the IPs themselves) | | ||
| 61 | | `ngit_websocket_flagged_abusers` | Gauge | - | Number of IPs exceeding connection threshold | | ||
| 62 | | `ngit_websocket_connection_duration_seconds` | Histogram | - | Duration of WebSocket connections | | ||
| 63 | | `ngit_websocket_messages_received_total` | Counter | `type` | Messages received (REQ, EVENT, CLOSE) | | ||
| 64 | | `ngit_websocket_messages_sent_total` | Counter | `type` | Messages sent (EVENT, EOSE, OK, NOTICE) | | ||
| 65 | |||
| 66 | **Privacy Note:** IP addresses are NEVER exposed in metrics. The `ConnectionTracker` maintains per-IP counts internally only for abuse detection, logging warnings when thresholds are exceeded. | ||
| 67 | |||
| 68 | ### 2. Git Operation Metrics | ||
| 69 | |||
| 70 | | Metric Name | Type | Labels | Description | | ||
| 71 | |------------|------|--------|-------------| | ||
| 72 | | `ngit_git_operations_total` | Counter | `operation`, `status` | Git operations (clone, fetch, push) | | ||
| 73 | | `ngit_git_operation_duration_seconds` | Histogram | `operation` | Duration of git operations | | ||
| 74 | | `ngit_git_bytes_total` | Counter | `direction` | Total bytes in/out for git operations | | ||
| 75 | | `ngit_git_push_authorization_total` | Counter | `result` | Push auth results (allowed, denied, error) | | ||
| 76 | |||
| 77 | ### 3. Top-N Repository Bandwidth Tracking | ||
| 78 | |||
| 79 | To identify high-bandwidth repositories without creating cardinality explosion (which doesn't scale to 1000+ repos), we use a hybrid approach: | ||
| 80 | |||
| 81 | | Metric Name | Type | Labels | Description | | ||
| 82 | |------------|------|--------|-------------| | ||
| 83 | | `ngit_git_top_repos_bytes` | Gauge | `repo` | Top 10 repositories by bandwidth (refreshed every 60s) | | ||
| 84 | |||
| 85 | **How it works:** | ||
| 86 | - All per-repo bandwidth is tracked internally in a `HashMap<RepoId, u64>` | ||
| 87 | - Every 60 seconds, the top 10 are calculated and exposed to Prometheus | ||
| 88 | - Previous repo labels are cleared before setting new ones | ||
| 89 | - Prometheus only ever sees ~10 label values, keeping cardinality low | ||
| 90 | |||
| 91 | ```rust | ||
| 92 | struct BandwidthTracker { | ||
| 93 | // Internal: tracks ALL repos (memory only, not exposed) | ||
| 94 | all_repos: DashMap<String, u64>, | ||
| 95 | |||
| 96 | // Exposed to Prometheus: only top 10 | ||
| 97 | top_repos_gauge: GaugeVec, | ||
| 98 | |||
| 99 | // Refresh interval | ||
| 100 | last_refresh: Instant, | ||
| 101 | } | ||
| 102 | |||
| 103 | impl BandwidthTracker { | ||
| 104 | fn record_transfer(&self, repo_id: &str, bytes: u64) { | ||
| 105 | self.all_repos | ||
| 106 | .entry(repo_id.to_string()) | ||
| 107 | .and_modify(|v| *v += bytes) | ||
| 108 | .or_insert(bytes); | ||
| 109 | } | ||
| 110 | |||
| 111 | fn maybe_refresh_top_n(&self) { | ||
| 112 | if self.last_refresh.elapsed() > Duration::from_secs(60) { | ||
| 113 | self.refresh_top_n(); | ||
| 114 | } | ||
| 115 | } | ||
| 116 | |||
| 117 | fn refresh_top_n(&self) { | ||
| 118 | let mut sorted: Vec<_> = self.all_repos.iter() | ||
| 119 | .map(|r| (r.key().clone(), *r.value())) | ||
| 120 | .collect(); | ||
| 121 | sorted.sort_by(|a, b| b.1.cmp(&a.1)); | ||
| 122 | |||
| 123 | // Clear old labels, set new top 10 | ||
| 124 | self.top_repos_gauge.reset(); | ||
| 125 | for (repo, bytes) in sorted.into_iter().take(10) { | ||
| 126 | self.top_repos_gauge | ||
| 127 | .with_label_values(&[&repo]) | ||
| 128 | .set(bytes as i64); | ||
| 129 | } | ||
| 130 | } | ||
| 131 | } | ||
| 132 | ``` | ||
| 133 | |||
| 134 | ### 4. Nostr Event Metrics | ||
| 135 | |||
| 136 | | Metric Name | Type | Labels | Description | | ||
| 137 | |------------|------|--------|-------------| | ||
| 138 | | `ngit_events_received_total` | Counter | `kind` | Events received by kind | | ||
| 139 | | `ngit_events_stored_total` | Counter | `kind` | Events successfully stored | | ||
| 140 | | `ngit_events_rejected_total` | Counter | `kind`, `reason` | Events rejected and why | | ||
| 141 | |||
| 142 | ### 5. Repository Metrics | ||
| 143 | |||
| 144 | | Metric Name | Type | Labels | Description | | ||
| 145 | |------------|------|--------|-------------| | ||
| 146 | | `ngit_repositories_total` | Gauge | - | Total repositories hosted | | ||
| 147 | |||
| 148 | ### 6. System Health Metrics | ||
| 149 | |||
| 150 | | Metric Name | Type | Labels | Description | | ||
| 151 | |------------|------|--------|-------------| | ||
| 152 | | `ngit_uptime_seconds` | Counter | - | Seconds since startup | | ||
| 153 | | `ngit_build_info` | Gauge | `version`, `commit` | Build information | | ||
| 154 | |||
| 155 | ### 7. Future: Sync Metrics (GRASP-02) | ||
| 156 | |||
| 157 | | Metric Name | Type | Labels | Description | | ||
| 158 | |------------|------|--------|-------------| | ||
| 159 | | `ngit_sync_events_received_total` | Counter | `source` | Events from sync (live vs catchup) | | ||
| 160 | | `ngit_sync_relay_connections_active` | Gauge | - | Active outbound relay connections | | ||
| 161 | | `ngit_sync_catchup_gap_total` | Counter | - | Events found during catchup (sync failures) | | ||
| 162 | |||
| 163 | ## Connection Tracker Design | ||
| 164 | |||
| 165 | The `ConnectionTracker` maintains per-IP connection counts internally for abuse detection. **IP addresses are never exposed in metrics** - only aggregate counts. | ||
| 166 | |||
| 167 | ```mermaid | ||
| 168 | flowchart LR | ||
| 169 | subgraph ConnectionTracker | ||
| 170 | HM[Internal: HashMap IP to Count] | ||
| 171 | TH[Abuse Threshold] | ||
| 172 | CNT[Exposed: Unique IP Count] | ||
| 173 | FLAG[Exposed: Abuse Flag Count] | ||
| 174 | end | ||
| 175 | |||
| 176 | CONN[New Connection] --> CHECK{Count >= Threshold?} | ||
| 177 | CHECK -->|No| INC[Increment Count] | ||
| 178 | CHECK -->|Yes| FLAG_IT[Flag as Abuse] | ||
| 179 | FLAG_IT --> LOG[Log Warning - IP in log only] | ||
| 180 | FLAG_IT --> FLAG | ||
| 181 | |||
| 182 | DISC[Disconnection] --> DEC[Decrement Count] | ||
| 183 | DEC --> CLEAN{Count == 0?} | ||
| 184 | CLEAN -->|Yes| RM[Remove from Map] | ||
| 185 | |||
| 186 | HM --> CNT | ||
| 187 | ``` | ||
| 188 | |||
| 189 | ### Data Structure | ||
| 190 | |||
| 191 | ```rust | ||
| 192 | pub struct ConnectionTracker { | ||
| 193 | /// Active connections per IP (INTERNAL ONLY - never exposed to metrics) | ||
| 194 | connections: DashMap<IpAddr, ConnectionInfo>, | ||
| 195 | /// Threshold for abuse flagging | ||
| 196 | abuse_threshold: u32, | ||
| 197 | /// Prometheus gauges (aggregate counts only, no IPs) | ||
| 198 | active_connections: IntGauge, // Total connections | ||
| 199 | unique_ips: IntGauge, // len() of HashMap | ||
| 200 | flagged_abusers: IntGauge, // Count where flagged_as_abuse == true | ||
| 201 | } | ||
| 202 | |||
| 203 | struct ConnectionInfo { | ||
| 204 | count: u32, | ||
| 205 | first_seen: Instant, | ||
| 206 | flagged_as_abuse: bool, | ||
| 207 | } | ||
| 208 | ``` | ||
| 209 | |||
| 210 | ### What Gets Exposed vs Internal | ||
| 211 | |||
| 212 | | Data | Location | Exposed? | | ||
| 213 | |------|----------|----------| | ||
| 214 | | Total connections | Prometheus | ✅ Yes | | ||
| 215 | | Unique IP count | Prometheus | ✅ Yes | | ||
| 216 | | Flagged abuser count | Prometheus | ✅ Yes | | ||
| 217 | | Actual IP addresses | Internal HashMap | ❌ No | | ||
| 218 | | IP + abuse flag | Logs (when flagged) | ⚠️ Logs only | | ||
| 219 | |||
| 220 | ### Thread Safety | ||
| 221 | |||
| 222 | Using `DashMap` for lock-free concurrent access, as connection tracking happens across multiple tokio tasks. | ||
| 223 | |||
| 224 | ## /metrics Endpoint | ||
| 225 | |||
| 226 | The `/metrics` endpoint returns Prometheus text format: | ||
| 227 | |||
| 228 | ``` | ||
| 229 | # HELP ngit_websocket_connections_active Current active WebSocket connections | ||
| 230 | # TYPE ngit_websocket_connections_active gauge | ||
| 231 | ngit_websocket_connections_active 23 | ||
| 232 | |||
| 233 | # HELP ngit_websocket_connections_by_ip Active connections per IP | ||
| 234 | # TYPE ngit_websocket_connections_by_ip gauge | ||
| 235 | ngit_websocket_connections_by_ip{ip="192.168.1.100"} 2 | ||
| 236 | ngit_websocket_connections_by_ip{ip="10.0.0.50"} 5 | ||
| 237 | |||
| 238 | # HELP ngit_git_operations_total Git operations by type and status | ||
| 239 | # TYPE ngit_git_operations_total counter | ||
| 240 | ngit_git_operations_total{operation="clone",status="success"} 1247 | ||
| 241 | ngit_git_operations_total{operation="push",status="denied"} 12 | ||
| 242 | ``` | ||
| 243 | |||
| 244 | ## Integration Points | ||
| 245 | |||
| 246 | ### HTTP Service Integration | ||
| 247 | |||
| 248 | In [`src/http/mod.rs`](../../src/http/mod.rs): | ||
| 249 | |||
| 250 | ```rust | ||
| 251 | // Add to HttpService | ||
| 252 | struct HttpService { | ||
| 253 | // ... existing fields ... | ||
| 254 | metrics: Arc<Metrics>, | ||
| 255 | } | ||
| 256 | |||
| 257 | // Add /metrics route handling | ||
| 258 | if path == "/metrics" { | ||
| 259 | let metrics_output = self.metrics.render(); | ||
| 260 | return Ok(Response::builder() | ||
| 261 | .status(200) | ||
| 262 | .header("content-type", "text/plain; version=0.0.4") | ||
| 263 | .body(Full::new(Bytes::from(metrics_output))) | ||
| 264 | .unwrap()); | ||
| 265 | } | ||
| 266 | ``` | ||
| 267 | |||
| 268 | ### WebSocket Connection Tracking | ||
| 269 | |||
| 270 | In the WebSocket upgrade handler: | ||
| 271 | |||
| 272 | ```rust | ||
| 273 | // On connection | ||
| 274 | let ip = addr.ip(); | ||
| 275 | metrics.connection_tracker.on_connect(ip); | ||
| 276 | |||
| 277 | // Spawn connection handler | ||
| 278 | tokio::spawn(async move { | ||
| 279 | // ... handle connection ... | ||
| 280 | // On disconnect | ||
| 281 | metrics.connection_tracker.on_disconnect(ip); | ||
| 282 | }); | ||
| 283 | ``` | ||
| 284 | |||
| 285 | ### Git Handler Integration | ||
| 286 | |||
| 287 | In [`src/git/handlers.rs`](../../src/git/handlers.rs): | ||
| 288 | |||
| 289 | ```rust | ||
| 290 | // Wrap git operations with metrics | ||
| 291 | let timer = metrics.git_operation_duration.start_timer(); | ||
| 292 | let result = git::handlers::handle_upload_pack(repo_path, body_bytes).await; | ||
| 293 | timer.observe_duration(); | ||
| 294 | |||
| 295 | metrics.git_operations_total | ||
| 296 | .with_label_values(&["clone", result_status]) | ||
| 297 | .inc(); | ||
| 298 | ``` | ||
| 299 | |||
| 300 | ## Configuration | ||
| 301 | |||
| 302 | New configuration options in [`src/config.rs`](../../src/config.rs): | ||
| 303 | |||
| 304 | | Option | CLI Flag | Environment Variable | Default | Description | | ||
| 305 | |--------|----------|---------------------|---------|-------------| | ||
| 306 | | Metrics enabled | `--metrics-enabled` | `NGIT_METRICS_ENABLED` | `true` | Enable /metrics endpoint | | ||
| 307 | | Abuse threshold | `--abuse-threshold` | `NGIT_ABUSE_THRESHOLD` | `10` | Max connections per IP before flagging | | ||
| 308 | | Metrics path | `--metrics-path` | `NGIT_METRICS_PATH` | `/metrics` | Path for metrics endpoint | | ||
| 309 | |||
| 310 | ## Crate Dependencies | ||
| 311 | |||
| 312 | Add to `Cargo.toml`: | ||
| 313 | |||
| 314 | ```toml | ||
| 315 | # Metrics | ||
| 316 | prometheus = "0.13" | ||
| 317 | dashmap = "5" # Lock-free concurrent HashMap | ||
| 318 | lazy_static = "1.4" # For static metric registration | ||
| 319 | ``` | ||
| 320 | |||
| 321 | ## Module Structure | ||
| 322 | |||
| 323 | ``` | ||
| 324 | src/ | ||
| 325 | ├── metrics/ | ||
| 326 | │ ├── mod.rs # Module exports, Metrics struct | ||
| 327 | │ ├── connection.rs # ConnectionTracker implementation | ||
| 328 | │ ├── definitions.rs # Metric definitions (lazy_static!) | ||
| 329 | │ └── render.rs # Prometheus format rendering | ||
| 330 | ├── http/ | ||
| 331 | │ └── mod.rs # Add /metrics route | ||
| 332 | └── ... | ||
| 333 | ``` | ||
| 334 | |||
| 335 | ## Grafana Dashboard | ||
| 336 | |||
| 337 | A pre-built Grafana dashboard will be provided at `docs/grafana/ngit-grasp-dashboard.json` with panels for: | ||
| 338 | |||
| 339 | 1. **Overview Row** | ||
| 340 | - Active connections (gauge) | ||
| 341 | - Requests per second (graph) | ||
| 342 | - Git operations per minute (graph) | ||
| 343 | |||
| 344 | 2. **Connections Row** | ||
| 345 | - Active connections over time | ||
| 346 | - Connections by IP (top 10) | ||
| 347 | - Flagged abuse IPs (table) | ||
| 348 | |||
| 349 | 3. **Git Operations Row** | ||
| 350 | - Clone/fetch/push rates | ||
| 351 | - Push authorization results (pie chart) | ||
| 352 | - Operation duration percentiles | ||
| 353 | |||
| 354 | 4. **Events Row** | ||
| 355 | - Events received by kind | ||
| 356 | - Events rejected by reason | ||
| 357 | - Active subscriptions | ||
| 358 | |||
| 359 | ## Deployment: Prometheus on NixOS | ||
| 360 | |||
| 361 | Example NixOS configuration for Prometheus: | ||
| 362 | |||
| 363 | ```nix | ||
| 364 | services.prometheus = { | ||
| 365 | enable = true; | ||
| 366 | scrapeConfigs = [ | ||
| 367 | { | ||
| 368 | job_name = "ngit-grasp"; | ||
| 369 | static_configs = [{ | ||
| 370 | targets = [ "localhost:8080" ]; # ngit-grasp bind address | ||
| 371 | }]; | ||
| 372 | scrape_interval = "15s"; | ||
| 373 | metrics_path = "/metrics"; | ||
| 374 | } | ||
| 375 | ]; | ||
| 376 | }; | ||
| 377 | |||
| 378 | services.grafana = { | ||
| 379 | enable = true; | ||
| 380 | settings.server.http_port = 3000; | ||
| 381 | provision.datasources.settings.datasources = [{ | ||
| 382 | name = "Prometheus"; | ||
| 383 | type = "prometheus"; | ||
| 384 | url = "http://localhost:9090"; | ||
| 385 | }]; | ||
| 386 | }; | ||
| 387 | ``` | ||
| 388 | |||
| 389 | ## Future: Load-Based Sync Scheduling | ||
| 390 | |||
| 391 | The metrics infrastructure enables future load-based scheduling for GRASP-02 sync jobs: | ||
| 392 | |||
| 393 | ```mermaid | ||
| 394 | flowchart TD | ||
| 395 | SYNC[Sync Manager] --> CHECK{Check Load} | ||
| 396 | CHECK --> MET[Query Metrics] | ||
| 397 | MET --> CPU{CPU > 80%?} | ||
| 398 | CPU -->|Yes| DELAY[Delay 5 min] | ||
| 399 | CPU -->|No| CONN{Connections > N?} | ||
| 400 | CONN -->|Yes| DELAY | ||
| 401 | CONN -->|No| RUN[Run Sync Job] | ||
| 402 | DELAY --> CHECK | ||
| 403 | ``` | ||
| 404 | |||
| 405 | The `Metrics` struct will expose a method for checking load: | ||
| 406 | |||
| 407 | ```rust | ||
| 408 | impl Metrics { | ||
| 409 | /// Check if system is under high load | ||
| 410 | pub fn is_high_load(&self) -> bool { | ||
| 411 | let active = self.websocket_connections_active.get(); | ||
| 412 | active > self.config.high_load_threshold | ||
| 413 | } | ||
| 414 | } | ||
| 415 | ``` | ||
| 416 | |||
| 417 | ## Future Enhancement: Loki for Detailed Logging | ||
| 418 | |||
| 419 | For detailed per-repository investigation at scale, consider adding **Loki** (log aggregation) in a future iteration: | ||
| 420 | |||
| 421 | ```rust | ||
| 422 | // Structured logging with tracing | ||
| 423 | tracing::info!( | ||
| 424 | repo = %repo_id, | ||
| 425 | npub = %npub, | ||
| 426 | bytes = bytes_transferred, | ||
| 427 | operation = "clone", | ||
| 428 | duration_ms = elapsed.as_millis(), | ||
| 429 | "git_transfer_complete" | ||
| 430 | ); | ||
| 431 | ``` | ||
| 432 | |||
| 433 | Loki query examples: | ||
| 434 | ```logql | ||
| 435 | # Find all transfers > 10MB | ||
| 436 | {job="ngit-grasp"} |= "git_transfer_complete" | json | bytes > 10000000 | ||
| 437 | |||
| 438 | # Sum bytes by repo in last hour | ||
| 439 | sum by (repo) ( | ||
| 440 | {job="ngit-grasp"} |= "git_transfer_complete" | json | unwrap bytes | ||
| 441 | ) | ||
| 442 | ``` | ||
| 443 | |||
| 444 | This pairs with Prometheus for long-term trends while enabling ad-hoc deep dives. | ||
| 445 | |||
| 446 | ## Privacy Considerations | ||
| 447 | |||
| 448 | - IP addresses are stored only in memory (not logged to disk by default) | ||
| 449 | - Per-IP metrics can be disabled via configuration | ||
| 450 | - Consider IP anonymization for GDPR compliance if needed | ||
| 451 | |||
| 452 | ## Summary | ||
| 453 | |||
| 454 | | Component | Purpose | | ||
| 455 | |-----------|---------| | ||
| 456 | | `Metrics` struct | Central registry and access point | | ||
| 457 | | `ConnectionTracker` | Per-IP tracking with abuse detection | | ||
| 458 | | `/metrics` endpoint | Prometheus scraping interface | | ||
| 459 | | Grafana dashboard | Visualization and analysis | | ||
| 460 | | NixOS config | Easy deployment for operators | | ||
| 461 | |||
| 462 | This strategy provides comprehensive observability without requiring a separate database - Prometheus handles all time-series storage and Grafana provides the visualization layer. \ No newline at end of file | ||
diff --git a/docs/explanation/monitoring.md b/docs/explanation/monitoring.md new file mode 100644 index 0000000..3b1b1ac --- /dev/null +++ b/docs/explanation/monitoring.md | |||
| @@ -0,0 +1,99 @@ | |||
| 1 | # Monitoring | ||
| 2 | |||
| 3 | ngit-grasp exposes Prometheus metrics at `/metrics` for monitoring WebSocket connections, Git operations, Nostr events, and system health. | ||
| 4 | |||
| 5 | ## Architecture | ||
| 6 | |||
| 7 | ```mermaid | ||
| 8 | flowchart TB | ||
| 9 | subgraph ngit-grasp | ||
| 10 | HTTP[HTTP Service] | ||
| 11 | WS[WebSocket Handler] | ||
| 12 | GIT[Git Handlers] | ||
| 13 | RELAY[Nostr Relay] | ||
| 14 | |||
| 15 | subgraph Metrics Module | ||
| 16 | REG[Prometheus Registry] | ||
| 17 | CT[ConnectionTracker] | ||
| 18 | MC[Metric Counters] | ||
| 19 | end | ||
| 20 | |||
| 21 | ME[/metrics endpoint] | ||
| 22 | end | ||
| 23 | |||
| 24 | subgraph External | ||
| 25 | PROM[Prometheus Server] | ||
| 26 | GRAF[Grafana] | ||
| 27 | ADMIN[Admin Browser] | ||
| 28 | end | ||
| 29 | |||
| 30 | HTTP --> ME | ||
| 31 | WS --> CT | ||
| 32 | WS --> MC | ||
| 33 | GIT --> MC | ||
| 34 | RELAY --> MC | ||
| 35 | |||
| 36 | CT --> REG | ||
| 37 | MC --> REG | ||
| 38 | REG --> ME | ||
| 39 | |||
| 40 | PROM -->|scrape /metrics| ME | ||
| 41 | GRAF -->|query| PROM | ||
| 42 | ADMIN -->|view dashboards| GRAF | ||
| 43 | ``` | ||
| 44 | |||
| 45 | ## Configuration | ||
| 46 | |||
| 47 | | Option | CLI Flag | Environment Variable | Default | Description | | ||
| 48 | |--------|----------|---------------------|---------|-------------| | ||
| 49 | | Metrics enabled | `--metrics-enabled` | `NGIT_METRICS_ENABLED` | `true` | Enable /metrics endpoint | | ||
| 50 | | Abuse threshold | `--abuse-threshold` | `NGIT_ABUSE_THRESHOLD` | `10` | Max connections per IP before flagging | | ||
| 51 | | Top N repos | `--top-n-repos` | `NGIT_TOP_N_REPOS` | `10` | Number of top bandwidth repos to track | | ||
| 52 | |||
| 53 | ## Privacy Model | ||
| 54 | |||
| 55 | IP addresses are **never exposed in Prometheus metrics**. The connection tracker maintains per-IP counts internally only for abuse detection: | ||
| 56 | |||
| 57 | | Data | Exposed in Metrics? | | ||
| 58 | |------|---------------------| | ||
| 59 | | Total connections | ✅ Yes | | ||
| 60 | | Unique IP count | ✅ Yes | | ||
| 61 | | Flagged abuser count | ✅ Yes | | ||
| 62 | | Actual IP addresses | ❌ No (internal only) | | ||
| 63 | | IP + abuse flag | ⚠️ Logs only (when flagged) | | ||
| 64 | |||
| 65 | When an IP exceeds the abuse threshold, a warning is logged but the IP is never exposed via Prometheus. | ||
| 66 | |||
| 67 | ## Deployment | ||
| 68 | |||
| 69 | See [Prometheus Setup Guide](../how-to/prometheus-setup.md) for NixOS configuration and Grafana dashboard provisioning. | ||
| 70 | |||
| 71 | ## Future: Load-Based Sync Scheduling (GRASP-02) | ||
| 72 | |||
| 73 | The metrics infrastructure enables future load-based scheduling for GRASP-02 sync jobs: | ||
| 74 | |||
| 75 | ```mermaid | ||
| 76 | flowchart TD | ||
| 77 | SYNC[Sync Manager] --> CHECK{Check Load} | ||
| 78 | CHECK --> MET[Query Metrics] | ||
| 79 | MET --> CONN{Connections > N?} | ||
| 80 | CONN -->|Yes| DELAY[Delay 5 min] | ||
| 81 | CONN -->|No| RUN[Run Sync Job] | ||
| 82 | DELAY --> CHECK | ||
| 83 | ``` | ||
| 84 | |||
| 85 | ## Future: Loki for Detailed Logging | ||
| 86 | |||
| 87 | For detailed per-repository investigation at scale, consider adding **Loki** (log aggregation): | ||
| 88 | |||
| 89 | - Structured logging with tracing crate already in place | ||
| 90 | - Loki queries enable ad-hoc deep dives (e.g., find all transfers > 10MB) | ||
| 91 | - Pairs with Prometheus for long-term trends | ||
| 92 | |||
| 93 | ## Future: Sync Metrics (GRASP-02) | ||
| 94 | |||
| 95 | When GRASP-02 proactive sync is implemented, additional metrics will track: | ||
| 96 | |||
| 97 | - Events received from sync (live vs catchup) | ||
| 98 | - Active outbound relay connections | ||
| 99 | - Catchup gap (events found during catchup indicating sync failures) \ No newline at end of file | ||
diff --git a/docs/grafana/ngit-grasp-dashboard.json b/docs/grafana/ngit-grasp-dashboard.json new file mode 100644 index 0000000..bd1b6fe --- /dev/null +++ b/docs/grafana/ngit-grasp-dashboard.json | |||
| @@ -0,0 +1,675 @@ | |||
| 1 | { | ||
| 2 | "annotations": { | ||
| 3 | "list": [] | ||
| 4 | }, | ||
| 5 | "editable": true, | ||
| 6 | "fiscalYearStartMonth": 0, | ||
| 7 | "graphTooltip": 0, | ||
| 8 | "id": null, | ||
| 9 | "links": [], | ||
| 10 | "liveNow": false, | ||
| 11 | "panels": [ | ||
| 12 | { | ||
| 13 | "collapsed": false, | ||
| 14 | "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, | ||
| 15 | "id": 1, | ||
| 16 | "title": "Overview", | ||
| 17 | "type": "row" | ||
| 18 | }, | ||
| 19 | { | ||
| 20 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 21 | "fieldConfig": { | ||
| 22 | "defaults": { | ||
| 23 | "color": { "mode": "thresholds" }, | ||
| 24 | "mappings": [], | ||
| 25 | "thresholds": { | ||
| 26 | "mode": "absolute", | ||
| 27 | "steps": [ | ||
| 28 | { "color": "green", "value": null }, | ||
| 29 | { "color": "yellow", "value": 50 }, | ||
| 30 | { "color": "red", "value": 100 } | ||
| 31 | ] | ||
| 32 | }, | ||
| 33 | "unit": "short" | ||
| 34 | } | ||
| 35 | }, | ||
| 36 | "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, | ||
| 37 | "id": 2, | ||
| 38 | "options": { | ||
| 39 | "colorMode": "value", | ||
| 40 | "graphMode": "area", | ||
| 41 | "justifyMode": "auto", | ||
| 42 | "orientation": "auto", | ||
| 43 | "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, | ||
| 44 | "textMode": "auto" | ||
| 45 | }, | ||
| 46 | "pluginVersion": "10.0.0", | ||
| 47 | "targets": [ | ||
| 48 | { | ||
| 49 | "expr": "ngit_websocket_connections_active", | ||
| 50 | "legendFormat": "Active", | ||
| 51 | "refId": "A" | ||
| 52 | } | ||
| 53 | ], | ||
| 54 | "title": "Active Connections", | ||
| 55 | "type": "stat" | ||
| 56 | }, | ||
| 57 | { | ||
| 58 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 59 | "fieldConfig": { | ||
| 60 | "defaults": { | ||
| 61 | "color": { "mode": "thresholds" }, | ||
| 62 | "mappings": [], | ||
| 63 | "thresholds": { | ||
| 64 | "mode": "absolute", | ||
| 65 | "steps": [ | ||
| 66 | { "color": "green", "value": null } | ||
| 67 | ] | ||
| 68 | }, | ||
| 69 | "unit": "short" | ||
| 70 | } | ||
| 71 | }, | ||
| 72 | "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, | ||
| 73 | "id": 3, | ||
| 74 | "options": { | ||
| 75 | "colorMode": "value", | ||
| 76 | "graphMode": "none", | ||
| 77 | "justifyMode": "auto", | ||
| 78 | "orientation": "auto", | ||
| 79 | "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, | ||
| 80 | "textMode": "auto" | ||
| 81 | }, | ||
| 82 | "targets": [ | ||
| 83 | { | ||
| 84 | "expr": "ngit_websocket_unique_ips", | ||
| 85 | "legendFormat": "Unique IPs", | ||
| 86 | "refId": "A" | ||
| 87 | } | ||
| 88 | ], | ||
| 89 | "title": "Unique IPs", | ||
| 90 | "type": "stat" | ||
| 91 | }, | ||
| 92 | { | ||
| 93 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 94 | "fieldConfig": { | ||
| 95 | "defaults": { | ||
| 96 | "color": { "mode": "thresholds" }, | ||
| 97 | "mappings": [], | ||
| 98 | "thresholds": { | ||
| 99 | "mode": "absolute", | ||
| 100 | "steps": [ | ||
| 101 | { "color": "green", "value": null }, | ||
| 102 | { "color": "red", "value": 1 } | ||
| 103 | ] | ||
| 104 | }, | ||
| 105 | "unit": "short" | ||
| 106 | } | ||
| 107 | }, | ||
| 108 | "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, | ||
| 109 | "id": 4, | ||
| 110 | "options": { | ||
| 111 | "colorMode": "value", | ||
| 112 | "graphMode": "none", | ||
| 113 | "justifyMode": "auto", | ||
| 114 | "orientation": "auto", | ||
| 115 | "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, | ||
| 116 | "textMode": "auto" | ||
| 117 | }, | ||
| 118 | "targets": [ | ||
| 119 | { | ||
| 120 | "expr": "ngit_websocket_flagged_abusers", | ||
| 121 | "legendFormat": "Flagged", | ||
| 122 | "refId": "A" | ||
| 123 | } | ||
| 124 | ], | ||
| 125 | "title": "Flagged Abusers", | ||
| 126 | "type": "stat" | ||
| 127 | }, | ||
| 128 | { | ||
| 129 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 130 | "fieldConfig": { | ||
| 131 | "defaults": { | ||
| 132 | "color": { "mode": "thresholds" }, | ||
| 133 | "mappings": [], | ||
| 134 | "thresholds": { | ||
| 135 | "mode": "absolute", | ||
| 136 | "steps": [{ "color": "blue", "value": null }] | ||
| 137 | }, | ||
| 138 | "unit": "short" | ||
| 139 | } | ||
| 140 | }, | ||
| 141 | "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, | ||
| 142 | "id": 5, | ||
| 143 | "options": { | ||
| 144 | "colorMode": "value", | ||
| 145 | "graphMode": "none", | ||
| 146 | "justifyMode": "auto", | ||
| 147 | "orientation": "auto", | ||
| 148 | "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, | ||
| 149 | "textMode": "auto" | ||
| 150 | }, | ||
| 151 | "targets": [ | ||
| 152 | { | ||
| 153 | "expr": "ngit_repositories_total", | ||
| 154 | "legendFormat": "Repos", | ||
| 155 | "refId": "A" | ||
| 156 | } | ||
| 157 | ], | ||
| 158 | "title": "Total Repositories", | ||
| 159 | "type": "stat" | ||
| 160 | }, | ||
| 161 | { | ||
| 162 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 163 | "fieldConfig": { | ||
| 164 | "defaults": { | ||
| 165 | "color": { "mode": "thresholds" }, | ||
| 166 | "mappings": [], | ||
| 167 | "thresholds": { | ||
| 168 | "mode": "absolute", | ||
| 169 | "steps": [{ "color": "green", "value": null }] | ||
| 170 | }, | ||
| 171 | "unit": "s" | ||
| 172 | } | ||
| 173 | }, | ||
| 174 | "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, | ||
| 175 | "id": 6, | ||
| 176 | "options": { | ||
| 177 | "colorMode": "value", | ||
| 178 | "graphMode": "none", | ||
| 179 | "justifyMode": "auto", | ||
| 180 | "orientation": "auto", | ||
| 181 | "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, | ||
| 182 | "textMode": "auto" | ||
| 183 | }, | ||
| 184 | "targets": [ | ||
| 185 | { | ||
| 186 | "expr": "ngit_uptime_seconds", | ||
| 187 | "legendFormat": "Uptime", | ||
| 188 | "refId": "A" | ||
| 189 | } | ||
| 190 | ], | ||
| 191 | "title": "Uptime", | ||
| 192 | "type": "stat" | ||
| 193 | }, | ||
| 194 | { | ||
| 195 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 196 | "fieldConfig": { | ||
| 197 | "defaults": { | ||
| 198 | "color": { "mode": "thresholds" }, | ||
| 199 | "mappings": [], | ||
| 200 | "thresholds": { | ||
| 201 | "mode": "absolute", | ||
| 202 | "steps": [{ "color": "purple", "value": null }] | ||
| 203 | }, | ||
| 204 | "unit": "short" | ||
| 205 | } | ||
| 206 | }, | ||
| 207 | "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, | ||
| 208 | "id": 7, | ||
| 209 | "options": { | ||
| 210 | "colorMode": "value", | ||
| 211 | "graphMode": "none", | ||
| 212 | "justifyMode": "auto", | ||
| 213 | "orientation": "auto", | ||
| 214 | "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, | ||
| 215 | "textMode": "value_and_name" | ||
| 216 | }, | ||
| 217 | "targets": [ | ||
| 218 | { | ||
| 219 | "expr": "ngit_build_info", | ||
| 220 | "legendFormat": "{{version}}", | ||
| 221 | "refId": "A" | ||
| 222 | } | ||
| 223 | ], | ||
| 224 | "title": "Version", | ||
| 225 | "type": "stat" | ||
| 226 | }, | ||
| 227 | { | ||
| 228 | "collapsed": false, | ||
| 229 | "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, | ||
| 230 | "id": 10, | ||
| 231 | "title": "WebSocket Connections", | ||
| 232 | "type": "row" | ||
| 233 | }, | ||
| 234 | { | ||
| 235 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 236 | "fieldConfig": { | ||
| 237 | "defaults": { | ||
| 238 | "color": { "mode": "palette-classic" }, | ||
| 239 | "custom": { | ||
| 240 | "axisCenteredZero": false, | ||
| 241 | "axisColorMode": "text", | ||
| 242 | "axisLabel": "", | ||
| 243 | "axisPlacement": "auto", | ||
| 244 | "barAlignment": 0, | ||
| 245 | "drawStyle": "line", | ||
| 246 | "fillOpacity": 10, | ||
| 247 | "gradientMode": "none", | ||
| 248 | "hideFrom": { "legend": false, "tooltip": false, "viz": false }, | ||
| 249 | "lineInterpolation": "linear", | ||
| 250 | "lineWidth": 1, | ||
| 251 | "pointSize": 5, | ||
| 252 | "scaleDistribution": { "type": "linear" }, | ||
| 253 | "showPoints": "never", | ||
| 254 | "spanNulls": false, | ||
| 255 | "stacking": { "group": "A", "mode": "none" }, | ||
| 256 | "thresholdsStyle": { "mode": "off" } | ||
| 257 | }, | ||
| 258 | "mappings": [], | ||
| 259 | "thresholds": { "mode": "absolute", "steps": [] }, | ||
| 260 | "unit": "short" | ||
| 261 | } | ||
| 262 | }, | ||
| 263 | "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, | ||
| 264 | "id": 11, | ||
| 265 | "options": { | ||
| 266 | "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, | ||
| 267 | "tooltip": { "mode": "multi", "sort": "none" } | ||
| 268 | }, | ||
| 269 | "targets": [ | ||
| 270 | { | ||
| 271 | "expr": "ngit_websocket_connections_active", | ||
| 272 | "legendFormat": "Active Connections", | ||
| 273 | "refId": "A" | ||
| 274 | }, | ||
| 275 | { | ||
| 276 | "expr": "ngit_websocket_unique_ips", | ||
| 277 | "legendFormat": "Unique IPs", | ||
| 278 | "refId": "B" | ||
| 279 | } | ||
| 280 | ], | ||
| 281 | "title": "Connections Over Time", | ||
| 282 | "type": "timeseries" | ||
| 283 | }, | ||
| 284 | { | ||
| 285 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 286 | "fieldConfig": { | ||
| 287 | "defaults": { | ||
| 288 | "color": { "mode": "palette-classic" }, | ||
| 289 | "custom": { | ||
| 290 | "axisCenteredZero": false, | ||
| 291 | "axisColorMode": "text", | ||
| 292 | "axisLabel": "", | ||
| 293 | "axisPlacement": "auto", | ||
| 294 | "barAlignment": 0, | ||
| 295 | "drawStyle": "line", | ||
| 296 | "fillOpacity": 10, | ||
| 297 | "gradientMode": "none", | ||
| 298 | "hideFrom": { "legend": false, "tooltip": false, "viz": false }, | ||
| 299 | "lineInterpolation": "linear", | ||
| 300 | "lineWidth": 1, | ||
| 301 | "pointSize": 5, | ||
| 302 | "scaleDistribution": { "type": "linear" }, | ||
| 303 | "showPoints": "never", | ||
| 304 | "spanNulls": false, | ||
| 305 | "stacking": { "group": "A", "mode": "none" }, | ||
| 306 | "thresholdsStyle": { "mode": "off" } | ||
| 307 | }, | ||
| 308 | "mappings": [], | ||
| 309 | "thresholds": { "mode": "absolute", "steps": [] }, | ||
| 310 | "unit": "short" | ||
| 311 | } | ||
| 312 | }, | ||
| 313 | "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, | ||
| 314 | "id": 12, | ||
| 315 | "options": { | ||
| 316 | "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, | ||
| 317 | "tooltip": { "mode": "multi", "sort": "none" } | ||
| 318 | }, | ||
| 319 | "targets": [ | ||
| 320 | { | ||
| 321 | "expr": "rate(ngit_websocket_messages_received_total[5m])", | ||
| 322 | "legendFormat": "Received: {{type}}", | ||
| 323 | "refId": "A" | ||
| 324 | }, | ||
| 325 | { | ||
| 326 | "expr": "rate(ngit_websocket_messages_sent_total[5m])", | ||
| 327 | "legendFormat": "Sent: {{type}}", | ||
| 328 | "refId": "B" | ||
| 329 | } | ||
| 330 | ], | ||
| 331 | "title": "Message Rate (5m)", | ||
| 332 | "type": "timeseries" | ||
| 333 | }, | ||
| 334 | { | ||
| 335 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 336 | "fieldConfig": { | ||
| 337 | "defaults": { | ||
| 338 | "color": { "mode": "palette-classic" }, | ||
| 339 | "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, | ||
| 340 | "mappings": [], | ||
| 341 | "unit": "s" | ||
| 342 | } | ||
| 343 | }, | ||
| 344 | "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, | ||
| 345 | "id": 13, | ||
| 346 | "options": { | ||
| 347 | "legend": { "displayMode": "list", "placement": "right", "showLegend": true }, | ||
| 348 | "pieType": "pie", | ||
| 349 | "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, | ||
| 350 | "tooltip": { "mode": "single", "sort": "none" } | ||
| 351 | }, | ||
| 352 | "targets": [ | ||
| 353 | { | ||
| 354 | "expr": "histogram_quantile(0.5, rate(ngit_websocket_connection_duration_seconds_bucket[1h]))", | ||
| 355 | "legendFormat": "p50", | ||
| 356 | "refId": "A" | ||
| 357 | }, | ||
| 358 | { | ||
| 359 | "expr": "histogram_quantile(0.95, rate(ngit_websocket_connection_duration_seconds_bucket[1h]))", | ||
| 360 | "legendFormat": "p95", | ||
| 361 | "refId": "B" | ||
| 362 | }, | ||
| 363 | { | ||
| 364 | "expr": "histogram_quantile(0.99, rate(ngit_websocket_connection_duration_seconds_bucket[1h]))", | ||
| 365 | "legendFormat": "p99", | ||
| 366 | "refId": "C" | ||
| 367 | } | ||
| 368 | ], | ||
| 369 | "title": "Connection Duration Percentiles", | ||
| 370 | "type": "piechart" | ||
| 371 | }, | ||
| 372 | { | ||
| 373 | "collapsed": false, | ||
| 374 | "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, | ||
| 375 | "id": 20, | ||
| 376 | "title": "Git Operations", | ||
| 377 | "type": "row" | ||
| 378 | }, | ||
| 379 | { | ||
| 380 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 381 | "fieldConfig": { | ||
| 382 | "defaults": { | ||
| 383 | "color": { "mode": "palette-classic" }, | ||
| 384 | "custom": { | ||
| 385 | "axisCenteredZero": false, | ||
| 386 | "axisColorMode": "text", | ||
| 387 | "axisLabel": "", | ||
| 388 | "axisPlacement": "auto", | ||
| 389 | "barAlignment": 0, | ||
| 390 | "drawStyle": "bars", | ||
| 391 | "fillOpacity": 50, | ||
| 392 | "gradientMode": "none", | ||
| 393 | "hideFrom": { "legend": false, "tooltip": false, "viz": false }, | ||
| 394 | "lineInterpolation": "linear", | ||
| 395 | "lineWidth": 1, | ||
| 396 | "pointSize": 5, | ||
| 397 | "scaleDistribution": { "type": "linear" }, | ||
| 398 | "showPoints": "never", | ||
| 399 | "spanNulls": false, | ||
| 400 | "stacking": { "group": "A", "mode": "normal" }, | ||
| 401 | "thresholdsStyle": { "mode": "off" } | ||
| 402 | }, | ||
| 403 | "mappings": [], | ||
| 404 | "thresholds": { "mode": "absolute", "steps": [] }, | ||
| 405 | "unit": "ops" | ||
| 406 | } | ||
| 407 | }, | ||
| 408 | "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, | ||
| 409 | "id": 21, | ||
| 410 | "options": { | ||
| 411 | "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, | ||
| 412 | "tooltip": { "mode": "multi", "sort": "none" } | ||
| 413 | }, | ||
| 414 | "targets": [ | ||
| 415 | { | ||
| 416 | "expr": "rate(ngit_git_operations_total{status=\"success\"}[5m])", | ||
| 417 | "legendFormat": "{{operation}} (success)", | ||
| 418 | "refId": "A" | ||
| 419 | }, | ||
| 420 | { | ||
| 421 | "expr": "rate(ngit_git_operations_total{status=\"error\"}[5m])", | ||
| 422 | "legendFormat": "{{operation}} (error)", | ||
| 423 | "refId": "B" | ||
| 424 | } | ||
| 425 | ], | ||
| 426 | "title": "Git Operations Rate (5m)", | ||
| 427 | "type": "timeseries" | ||
| 428 | }, | ||
| 429 | { | ||
| 430 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 431 | "fieldConfig": { | ||
| 432 | "defaults": { | ||
| 433 | "color": { "mode": "palette-classic" }, | ||
| 434 | "custom": { | ||
| 435 | "axisCenteredZero": false, | ||
| 436 | "axisColorMode": "text", | ||
| 437 | "axisLabel": "", | ||
| 438 | "axisPlacement": "auto", | ||
| 439 | "barAlignment": 0, | ||
| 440 | "drawStyle": "line", | ||
| 441 | "fillOpacity": 10, | ||
| 442 | "gradientMode": "none", | ||
| 443 | "hideFrom": { "legend": false, "tooltip": false, "viz": false }, | ||
| 444 | "lineInterpolation": "linear", | ||
| 445 | "lineWidth": 1, | ||
| 446 | "pointSize": 5, | ||
| 447 | "scaleDistribution": { "type": "linear" }, | ||
| 448 | "showPoints": "never", | ||
| 449 | "spanNulls": false, | ||
| 450 | "stacking": { "group": "A", "mode": "none" }, | ||
| 451 | "thresholdsStyle": { "mode": "off" } | ||
| 452 | }, | ||
| 453 | "mappings": [], | ||
| 454 | "thresholds": { "mode": "absolute", "steps": [] }, | ||
| 455 | "unit": "bytes" | ||
| 456 | } | ||
| 457 | }, | ||
| 458 | "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 }, | ||
| 459 | "id": 22, | ||
| 460 | "options": { | ||
| 461 | "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, | ||
| 462 | "tooltip": { "mode": "multi", "sort": "none" } | ||
| 463 | }, | ||
| 464 | "targets": [ | ||
| 465 | { | ||
| 466 | "expr": "rate(ngit_git_bytes_total[5m])", | ||
| 467 | "legendFormat": "{{direction}}", | ||
| 468 | "refId": "A" | ||
| 469 | } | ||
| 470 | ], | ||
| 471 | "title": "Git Bandwidth (5m)", | ||
| 472 | "type": "timeseries" | ||
| 473 | }, | ||
| 474 | { | ||
| 475 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 476 | "fieldConfig": { | ||
| 477 | "defaults": { | ||
| 478 | "color": { "mode": "palette-classic" }, | ||
| 479 | "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, | ||
| 480 | "mappings": [], | ||
| 481 | "unit": "short" | ||
| 482 | }, | ||
| 483 | "overrides": [ | ||
| 484 | { | ||
| 485 | "matcher": { "id": "byName", "options": "denied" }, | ||
| 486 | "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] | ||
| 487 | }, | ||
| 488 | { | ||
| 489 | "matcher": { "id": "byName", "options": "allowed" }, | ||
| 490 | "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] | ||
| 491 | } | ||
| 492 | ] | ||
| 493 | }, | ||
| 494 | "gridPos": { "h": 8, "w": 6, "x": 0, "y": 31 }, | ||
| 495 | "id": 23, | ||
| 496 | "options": { | ||
| 497 | "legend": { "displayMode": "list", "placement": "right", "showLegend": true }, | ||
| 498 | "pieType": "pie", | ||
| 499 | "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, | ||
| 500 | "tooltip": { "mode": "single", "sort": "none" } | ||
| 501 | }, | ||
| 502 | "targets": [ | ||
| 503 | { | ||
| 504 | "expr": "increase(ngit_git_push_authorization_total[24h])", | ||
| 505 | "legendFormat": "{{result}}", | ||
| 506 | "refId": "A" | ||
| 507 | } | ||
| 508 | ], | ||
| 509 | "title": "Push Authorization (24h)", | ||
| 510 | "type": "piechart" | ||
| 511 | }, | ||
| 512 | { | ||
| 513 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 514 | "fieldConfig": { | ||
| 515 | "defaults": { | ||
| 516 | "color": { "mode": "palette-classic" }, | ||
| 517 | "mappings": [], | ||
| 518 | "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, | ||
| 519 | "unit": "bytes" | ||
| 520 | } | ||
| 521 | }, | ||
| 522 | "gridPos": { "h": 8, "w": 18, "x": 6, "y": 31 }, | ||
| 523 | "id": 24, | ||
| 524 | "options": { | ||
| 525 | "displayMode": "gradient", | ||
| 526 | "minVizHeight": 10, | ||
| 527 | "minVizWidth": 0, | ||
| 528 | "orientation": "horizontal", | ||
| 529 | "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, | ||
| 530 | "showUnfilled": true, | ||
| 531 | "valueMode": "color" | ||
| 532 | }, | ||
| 533 | "targets": [ | ||
| 534 | { | ||
| 535 | "expr": "topk(10, ngit_git_top_repos_bytes)", | ||
| 536 | "legendFormat": "{{repo}}", | ||
| 537 | "refId": "A" | ||
| 538 | } | ||
| 539 | ], | ||
| 540 | "title": "Top Repositories by Bandwidth", | ||
| 541 | "type": "bargauge" | ||
| 542 | }, | ||
| 543 | { | ||
| 544 | "collapsed": false, | ||
| 545 | "gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 }, | ||
| 546 | "id": 30, | ||
| 547 | "title": "Nostr Events", | ||
| 548 | "type": "row" | ||
| 549 | }, | ||
| 550 | { | ||
| 551 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 552 | "fieldConfig": { | ||
| 553 | "defaults": { | ||
| 554 | "color": { "mode": "palette-classic" }, | ||
| 555 | "custom": { | ||
| 556 | "axisCenteredZero": false, | ||
| 557 | "axisColorMode": "text", | ||
| 558 | "axisLabel": "", | ||
| 559 | "axisPlacement": "auto", | ||
| 560 | "barAlignment": 0, | ||
| 561 | "drawStyle": "line", | ||
| 562 | "fillOpacity": 10, | ||
| 563 | "gradientMode": "none", | ||
| 564 | "hideFrom": { "legend": false, "tooltip": false, "viz": false }, | ||
| 565 | "lineInterpolation": "linear", | ||
| 566 | "lineWidth": 1, | ||
| 567 | "pointSize": 5, | ||
| 568 | "scaleDistribution": { "type": "linear" }, | ||
| 569 | "showPoints": "never", | ||
| 570 | "spanNulls": false, | ||
| 571 | "stacking": { "group": "A", "mode": "none" }, | ||
| 572 | "thresholdsStyle": { "mode": "off" } | ||
| 573 | }, | ||
| 574 | "mappings": [], | ||
| 575 | "thresholds": { "mode": "absolute", "steps": [] }, | ||
| 576 | "unit": "short" | ||
| 577 | } | ||
| 578 | }, | ||
| 579 | "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, | ||
| 580 | "id": 31, | ||
| 581 | "options": { | ||
| 582 | "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, | ||
| 583 | "tooltip": { "mode": "multi", "sort": "none" } | ||
| 584 | }, | ||
| 585 | "targets": [ | ||
| 586 | { | ||
| 587 | "expr": "rate(ngit_events_received_total[5m])", | ||
| 588 | "legendFormat": "Kind {{kind}}", | ||
| 589 | "refId": "A" | ||
| 590 | } | ||
| 591 | ], | ||
| 592 | "title": "Events Received by Kind (5m)", | ||
| 593 | "type": "timeseries" | ||
| 594 | }, | ||
| 595 | { | ||
| 596 | "datasource": { "type": "prometheus", "uid": "${datasource}" }, | ||
| 597 | "fieldConfig": { | ||
| 598 | "defaults": { | ||
| 599 | "color": { "mode": "palette-classic" }, | ||
| 600 | "custom": { | ||
| 601 | "axisCenteredZero": false, | ||
| 602 | "axisColorMode": "text", | ||
| 603 | "axisLabel": "", | ||
| 604 | "axisPlacement": "auto", | ||
| 605 | "barAlignment": 0, | ||
| 606 | "drawStyle": "line", | ||
| 607 | "fillOpacity": 10, | ||
| 608 | "gradientMode": "none", | ||
| 609 | "hideFrom": { "legend": false, "tooltip": false, "viz": false }, | ||
| 610 | "lineInterpolation": "linear", | ||
| 611 | "lineWidth": 1, | ||
| 612 | "pointSize": 5, | ||
| 613 | "scaleDistribution": { "type": "linear" }, | ||
| 614 | "showPoints": "never", | ||
| 615 | "spanNulls": false, | ||
| 616 | "stacking": { "group": "A", "mode": "none" }, | ||
| 617 | "thresholdsStyle": { "mode": "off" } | ||
| 618 | }, | ||
| 619 | "mappings": [], | ||
| 620 | "thresholds": { "mode": "absolute", "steps": [] }, | ||
| 621 | "unit": "short" | ||
| 622 | } | ||
| 623 | }, | ||
| 624 | "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, | ||
| 625 | "id": 32, | ||
| 626 | "options": { | ||
| 627 | "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, | ||
| 628 | "tooltip": { "mode": "multi", "sort": "none" } | ||
| 629 | }, | ||
| 630 | "targets": [ | ||
| 631 | { | ||
| 632 | "expr": "rate(ngit_events_stored_total[5m])", | ||
| 633 | "legendFormat": "Stored: Kind {{kind}}", | ||
| 634 | "refId": "A" | ||
| 635 | }, | ||
| 636 | { | ||
| 637 | "expr": "rate(ngit_events_rejected_total[5m])", | ||
| 638 | "legendFormat": "Rejected: {{reason}}", | ||
| 639 | "refId": "B" | ||
| 640 | } | ||
| 641 | ], | ||
| 642 | "title": "Events Stored vs Rejected (5m)", | ||
| 643 | "type": "timeseries" | ||
| 644 | } | ||
| 645 | ], | ||
| 646 | "refresh": "30s", | ||
| 647 | "schemaVersion": 38, | ||
| 648 | "style": "dark", | ||
| 649 | "tags": ["ngit-grasp", "nostr", "git"], | ||
| 650 | "templating": { | ||
| 651 | "list": [ | ||
| 652 | { | ||
| 653 | "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, | ||
| 654 | "hide": 0, | ||
| 655 | "includeAll": false, | ||
| 656 | "label": "Datasource", | ||
| 657 | "multi": false, | ||
| 658 | "name": "datasource", | ||
| 659 | "options": [], | ||
| 660 | "query": "prometheus", | ||
| 661 | "refresh": 1, | ||
| 662 | "regex": "", | ||
| 663 | "skipUrlSync": false, | ||
| 664 | "type": "datasource" | ||
| 665 | } | ||
| 666 | ] | ||
| 667 | }, | ||
| 668 | "time": { "from": "now-6h", "to": "now" }, | ||
| 669 | "timepicker": {}, | ||
| 670 | "timezone": "browser", | ||
| 671 | "title": "ngit-grasp", | ||
| 672 | "uid": "ngit-grasp", | ||
| 673 | "version": 1, | ||
| 674 | "weekStart": "" | ||
| 675 | } \ No newline at end of file | ||
diff --git a/docs/how-to/prometheus-setup.md b/docs/how-to/prometheus-setup.md new file mode 100644 index 0000000..741255b --- /dev/null +++ b/docs/how-to/prometheus-setup.md | |||
| @@ -0,0 +1,178 @@ | |||
| 1 | # Prometheus and Grafana Setup | ||
| 2 | |||
| 3 | This guide shows how to configure Prometheus and Grafana to monitor ngit-grasp. | ||
| 4 | |||
| 5 | ## Prerequisites | ||
| 6 | |||
| 7 | - ngit-grasp running with metrics enabled (default: `--metrics-enabled true`) | ||
| 8 | - Prometheus server | ||
| 9 | - Grafana (optional, for dashboards) | ||
| 10 | |||
| 11 | ## Verify Metrics Endpoint | ||
| 12 | |||
| 13 | First, verify that ngit-grasp is exposing metrics: | ||
| 14 | |||
| 15 | ```bash | ||
| 16 | curl http://localhost:8080/metrics | ||
| 17 | ``` | ||
| 18 | |||
| 19 | You should see Prometheus-formatted metrics like: | ||
| 20 | |||
| 21 | ``` | ||
| 22 | # HELP ngit_websocket_connections_active Current active WebSocket connections | ||
| 23 | # TYPE ngit_websocket_connections_active gauge | ||
| 24 | ngit_websocket_connections_active 5 | ||
| 25 | |||
| 26 | # HELP ngit_git_operations_total Git operations by type and status | ||
| 27 | # TYPE ngit_git_operations_total counter | ||
| 28 | ngit_git_operations_total{operation="clone",status="success"} 42 | ||
| 29 | ``` | ||
| 30 | |||
| 31 | ## NixOS Configuration | ||
| 32 | |||
| 33 | ### Prometheus | ||
| 34 | |||
| 35 | Add ngit-grasp as a scrape target: | ||
| 36 | |||
| 37 | ```nix | ||
| 38 | services.prometheus = { | ||
| 39 | enable = true; | ||
| 40 | scrapeConfigs = [ | ||
| 41 | { | ||
| 42 | job_name = "ngit-grasp"; | ||
| 43 | static_configs = [{ | ||
| 44 | targets = [ "localhost:8080" ]; # ngit-grasp bind address | ||
| 45 | }]; | ||
| 46 | scrape_interval = "15s"; | ||
| 47 | metrics_path = "/metrics"; | ||
| 48 | } | ||
| 49 | ]; | ||
| 50 | }; | ||
| 51 | ``` | ||
| 52 | |||
| 53 | ### Grafana with Prometheus Datasource | ||
| 54 | |||
| 55 | ```nix | ||
| 56 | services.grafana = { | ||
| 57 | enable = true; | ||
| 58 | settings.server.http_port = 3000; | ||
| 59 | |||
| 60 | provision.datasources.settings.datasources = [{ | ||
| 61 | name = "Prometheus"; | ||
| 62 | type = "prometheus"; | ||
| 63 | url = "http://localhost:9090"; | ||
| 64 | isDefault = true; | ||
| 65 | }]; | ||
| 66 | |||
| 67 | # Optional: provision the ngit-grasp dashboard | ||
| 68 | provision.dashboards.settings.providers = [{ | ||
| 69 | name = "ngit-grasp"; | ||
| 70 | options.path = "/path/to/ngit-grasp/docs/grafana"; | ||
| 71 | }]; | ||
| 72 | }; | ||
| 73 | ``` | ||
| 74 | |||
| 75 | ## Docker Compose Configuration | ||
| 76 | |||
| 77 | For non-NixOS deployments: | ||
| 78 | |||
| 79 | ```yaml | ||
| 80 | version: '3.8' | ||
| 81 | services: | ||
| 82 | prometheus: | ||
| 83 | image: prom/prometheus:latest | ||
| 84 | volumes: | ||
| 85 | - ./prometheus.yml:/etc/prometheus/prometheus.yml | ||
| 86 | ports: | ||
| 87 | - "9090:9090" | ||
| 88 | |||
| 89 | grafana: | ||
| 90 | image: grafana/grafana:latest | ||
| 91 | ports: | ||
| 92 | - "3000:3000" | ||
| 93 | volumes: | ||
| 94 | - ./docs/grafana:/var/lib/grafana/dashboards | ||
| 95 | environment: | ||
| 96 | - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/ngit-grasp-dashboard.json | ||
| 97 | ``` | ||
| 98 | |||
| 99 | With `prometheus.yml`: | ||
| 100 | |||
| 101 | ```yaml | ||
| 102 | global: | ||
| 103 | scrape_interval: 15s | ||
| 104 | |||
| 105 | scrape_configs: | ||
| 106 | - job_name: 'ngit-grasp' | ||
| 107 | static_configs: | ||
| 108 | - targets: ['host.docker.internal:8080'] # or your ngit-grasp host | ||
| 109 | metrics_path: /metrics | ||
| 110 | ``` | ||
| 111 | |||
| 112 | ## Import Dashboard | ||
| 113 | |||
| 114 | 1. Open Grafana at `http://localhost:3000` | ||
| 115 | 2. Go to **Dashboards** → **Import** | ||
| 116 | 3. Upload `docs/grafana/ngit-grasp-dashboard.json` | ||
| 117 | 4. Select your Prometheus datasource | ||
| 118 | 5. Click **Import** | ||
| 119 | |||
| 120 | ## Key Metrics to Monitor | ||
| 121 | |||
| 122 | ### Connection Health | ||
| 123 | - `ngit_websocket_connections_active` - Current active connections | ||
| 124 | - `ngit_websocket_unique_ips` - Number of unique client IPs | ||
| 125 | - `ngit_websocket_flagged_abusers` - IPs exceeding connection threshold | ||
| 126 | |||
| 127 | ### Git Operations | ||
| 128 | - `ngit_git_operations_total` - Operations by type (clone/fetch/push) and status | ||
| 129 | - `ngit_git_bytes_total` - Bandwidth by direction (in/out) | ||
| 130 | - `ngit_git_top_repos_bytes` - Top N repositories by bandwidth | ||
| 131 | |||
| 132 | ### Nostr Events | ||
| 133 | - `ngit_events_received_total` - Events received by kind | ||
| 134 | - `ngit_events_stored_total` - Events successfully stored | ||
| 135 | - `ngit_events_rejected_total` - Events rejected by reason | ||
| 136 | |||
| 137 | ### System | ||
| 138 | - `ngit_uptime_seconds` - Server uptime | ||
| 139 | - `ngit_build_info` - Version and commit info | ||
| 140 | - `ngit_repositories_total` - Total hosted repositories | ||
| 141 | |||
| 142 | ## Example Alerts | ||
| 143 | |||
| 144 | Add to your Prometheus alerting rules: | ||
| 145 | |||
| 146 | ```yaml | ||
| 147 | groups: | ||
| 148 | - name: ngit-grasp | ||
| 149 | rules: | ||
| 150 | - alert: HighConnectionCount | ||
| 151 | expr: ngit_websocket_connections_active > 100 | ||
| 152 | for: 5m | ||
| 153 | labels: | ||
| 154 | severity: warning | ||
| 155 | annotations: | ||
| 156 | summary: "High number of WebSocket connections" | ||
| 157 | |||
| 158 | - alert: AbusiveIPs | ||
| 159 | expr: ngit_websocket_flagged_abusers > 0 | ||
| 160 | for: 1m | ||
| 161 | labels: | ||
| 162 | severity: warning | ||
| 163 | annotations: | ||
| 164 | summary: "{{ $value }} IPs flagged for excessive connections" | ||
| 165 | |||
| 166 | - alert: PushAuthorizationFailures | ||
| 167 | expr: rate(ngit_git_operations_total{operation="push",status="denied"}[5m]) > 0.1 | ||
| 168 | for: 5m | ||
| 169 | labels: | ||
| 170 | severity: info | ||
| 171 | annotations: | ||
| 172 | summary: "Elevated push authorization failures" | ||
| 173 | ``` | ||
| 174 | |||
| 175 | ## See Also | ||
| 176 | |||
| 177 | - [Monitoring Overview](../explanation/monitoring.md) - Architecture and design | ||
| 178 | - [Configuration Reference](../reference/configuration.md) - All config options \ No newline at end of file | ||
diff --git a/src/config.rs b/src/config.rs index d095178..025e020 100644 --- a/src/config.rs +++ b/src/config.rs | |||
| @@ -71,6 +71,18 @@ pub struct Config { | |||
| 71 | /// Database backend type | 71 | /// Database backend type |
| 72 | #[arg(long, env = "NGIT_DATABASE_BACKEND", value_enum, default_value_t = DatabaseBackend::Lmdb)] | 72 | #[arg(long, env = "NGIT_DATABASE_BACKEND", value_enum, default_value_t = DatabaseBackend::Lmdb)] |
| 73 | pub database_backend: DatabaseBackend, | 73 | pub database_backend: DatabaseBackend, |
| 74 | |||
| 75 | /// Enable Prometheus metrics endpoint | ||
| 76 | #[arg(long, env = "NGIT_METRICS_ENABLED", default_value_t = true)] | ||
| 77 | pub metrics_enabled: bool, | ||
| 78 | |||
| 79 | /// Connections per IP before flagging as potential abuse in metrics (display only, no rate limiting) | ||
| 80 | #[arg(long = "metrics-connection-per-ip-abuse-threshold", env = "NGIT_METRICS_CONNECTION_PER_IP_ABUSE_THRESHOLD", default_value_t = 10)] | ||
| 81 | pub metrics_connection_per_ip_abuse_threshold: u32, | ||
| 82 | |||
| 83 | /// Number of top bandwidth repos to track in metrics | ||
| 84 | #[arg(long = "metrics-top-n-repos", env = "NGIT_METRICS_TOP_N_REPOS", default_value_t = 10)] | ||
| 85 | pub metrics_top_n_repos: usize, | ||
| 74 | } | 86 | } |
| 75 | 87 | ||
| 76 | impl Config { | 88 | impl Config { |
| @@ -123,6 +135,9 @@ impl Config { | |||
| 123 | relay_data_path: "./test_data/relay".to_string(), | 135 | relay_data_path: "./test_data/relay".to_string(), |
| 124 | bind_address: "127.0.0.1:8080".to_string(), | 136 | bind_address: "127.0.0.1:8080".to_string(), |
| 125 | database_backend: DatabaseBackend::Memory, | 137 | database_backend: DatabaseBackend::Memory, |
| 138 | metrics_enabled: true, | ||
| 139 | metrics_connection_per_ip_abuse_threshold: 10, | ||
| 140 | metrics_top_n_repos: 10, | ||
| 126 | } | 141 | } |
| 127 | } | 142 | } |
| 128 | } | 143 | } |
| @@ -202,4 +217,25 @@ mod tests { | |||
| 202 | }; | 217 | }; |
| 203 | assert!(config.owner_npub.is_none()); | 218 | assert!(config.owner_npub.is_none()); |
| 204 | } | 219 | } |
| 220 | |||
| 221 | #[test] | ||
| 222 | fn test_metrics_config_defaults() { | ||
| 223 | let config = Config::for_testing(); | ||
| 224 | assert!(config.metrics_enabled); | ||
| 225 | assert_eq!(config.metrics_connection_per_ip_abuse_threshold, 10); | ||
| 226 | assert_eq!(config.metrics_top_n_repos, 10); | ||
| 227 | } | ||
| 228 | |||
| 229 | #[test] | ||
| 230 | fn test_metrics_config_custom_values() { | ||
| 231 | let config = Config { | ||
| 232 | metrics_enabled: false, | ||
| 233 | metrics_connection_per_ip_abuse_threshold: 50, | ||
| 234 | metrics_top_n_repos: 25, | ||
| 235 | ..Config::for_testing() | ||
| 236 | }; | ||
| 237 | assert!(!config.metrics_enabled); | ||
| 238 | assert_eq!(config.metrics_connection_per_ip_abuse_threshold, 50); | ||
| 239 | assert_eq!(config.metrics_top_n_repos, 25); | ||
| 240 | } | ||
| 205 | } | 241 | } |
diff --git a/src/http/mod.rs b/src/http/mod.rs index 8b1f687..f584e03 100644 --- a/src/http/mod.rs +++ b/src/http/mod.rs | |||
| @@ -7,6 +7,7 @@ pub mod nip11; | |||
| 7 | use std::future::Future; | 7 | use std::future::Future; |
| 8 | use std::net::SocketAddr; | 8 | use std::net::SocketAddr; |
| 9 | use std::pin::Pin; | 9 | use std::pin::Pin; |
| 10 | use std::sync::Arc; | ||
| 10 | 11 | ||
| 11 | use base64::Engine; | 12 | use base64::Engine; |
| 12 | use http_body_util::{BodyExt, Full}; | 13 | use http_body_util::{BodyExt, Full}; |
| @@ -24,6 +25,7 @@ use tokio::net::TcpListener; | |||
| 24 | 25 | ||
| 25 | use crate::config::Config; | 26 | use crate::config::Config; |
| 26 | use crate::git; | 27 | use crate::git; |
| 28 | use crate::metrics::Metrics; | ||
| 27 | use crate::nostr::builder::SharedDatabase; | 29 | use crate::nostr::builder::SharedDatabase; |
| 28 | 30 | ||
| 29 | /// CORS headers required by GRASP-01 specification (lines 40-47) | 31 | /// CORS headers required by GRASP-01 specification (lines 40-47) |
| @@ -90,6 +92,8 @@ struct HttpService { | |||
| 90 | remote: SocketAddr, | 92 | remote: SocketAddr, |
| 91 | /// Database reference for direct queries (e.g., push authorization) | 93 | /// Database reference for direct queries (e.g., push authorization) |
| 92 | database: SharedDatabase, | 94 | database: SharedDatabase, |
| 95 | /// Optional metrics for Prometheus endpoint | ||
| 96 | metrics: Option<Arc<Metrics>>, | ||
| 93 | } | 97 | } |
| 94 | 98 | ||
| 95 | impl HttpService { | 99 | impl HttpService { |
| @@ -98,12 +102,14 @@ impl HttpService { | |||
| 98 | config: Config, | 102 | config: Config, |
| 99 | remote: SocketAddr, | 103 | remote: SocketAddr, |
| 100 | database: SharedDatabase, | 104 | database: SharedDatabase, |
| 105 | metrics: Option<Arc<Metrics>>, | ||
| 101 | ) -> Self { | 106 | ) -> Self { |
| 102 | Self { | 107 | Self { |
| 103 | relay, | 108 | relay, |
| 104 | config, | 109 | config, |
| 105 | remote, | 110 | remote, |
| 106 | database, | 111 | database, |
| 112 | metrics, | ||
| 107 | } | 113 | } |
| 108 | } | 114 | } |
| 109 | } | 115 | } |
| @@ -150,6 +156,7 @@ impl Service<Request<Incoming>> for HttpService { | |||
| 150 | ); | 156 | ); |
| 151 | 157 | ||
| 152 | let repo_path = git::resolve_repo_path(&git_data_path, &npub, &identifier); | 158 | let repo_path = git::resolve_repo_path(&git_data_path, &npub, &identifier); |
| 159 | let metrics_clone = self.metrics.clone(); | ||
| 153 | 160 | ||
| 154 | return Box::pin(async move { | 161 | return Box::pin(async move { |
| 155 | // Collect request body once before the match statement | 162 | // Collect request body once before the match statement |
| @@ -170,14 +177,31 @@ impl Service<Request<Incoming>> for HttpService { | |||
| 170 | .and_then(git::protocol::GitService::from_query_param); | 177 | .and_then(git::protocol::GitService::from_query_param); |
| 171 | 178 | ||
| 172 | match service { | 179 | match service { |
| 173 | Some(svc) => git::handlers::handle_info_refs(repo_path, svc).await, | 180 | Some(svc) => { |
| 181 | let result = git::handlers::handle_info_refs(repo_path, svc).await; | ||
| 182 | // Track operation | ||
| 183 | if let Some(ref m) = metrics_clone { | ||
| 184 | let status = if result.is_ok() { "success" } else { "error" }; | ||
| 185 | let operation = match svc { | ||
| 186 | git::protocol::GitService::UploadPack => "fetch", | ||
| 187 | git::protocol::GitService::ReceivePack => "push", | ||
| 188 | }; | ||
| 189 | m.record_git_operation(operation, status); | ||
| 190 | } | ||
| 191 | result | ||
| 192 | } | ||
| 174 | None => Err(git::handlers::GitError::RepositoryNotFound), | 193 | None => Err(git::handlers::GitError::RepositoryNotFound), |
| 175 | } | 194 | } |
| 176 | } | 195 | } |
| 177 | 196 | ||
| 178 | // POST /git-upload-pack (clone/fetch) | 197 | // POST /git-upload-pack (clone/fetch) |
| 179 | (m, "git-upload-pack") if m == Method::POST => { | 198 | (m, "git-upload-pack") if m == Method::POST => { |
| 180 | git::handlers::handle_upload_pack(repo_path, body_bytes).await | 199 | let result = git::handlers::handle_upload_pack(repo_path, body_bytes).await; |
| 200 | if let Some(ref m) = metrics_clone { | ||
| 201 | let status = if result.is_ok() { "success" } else { "error" }; | ||
| 202 | m.record_git_operation("clone", status); | ||
| 203 | } | ||
| 204 | result | ||
| 181 | } | 205 | } |
| 182 | 206 | ||
| 183 | // POST /git-receive-pack (push) - with GRASP authorization via database | 207 | // POST /git-receive-pack (push) - with GRASP authorization via database |
| @@ -187,6 +211,10 @@ impl Service<Request<Incoming>> for HttpService { | |||
| 187 | Ok(pk) => pk.to_hex(), | 211 | Ok(pk) => pk.to_hex(), |
| 188 | Err(e) => { | 212 | Err(e) => { |
| 189 | tracing::warn!("Invalid npub in URL {}: {}", npub, e); | 213 | tracing::warn!("Invalid npub in URL {}: {}", npub, e); |
| 214 | // Track failed push due to invalid npub | ||
| 215 | if let Some(ref m) = metrics_clone { | ||
| 216 | m.record_git_operation("push", "error"); | ||
| 217 | } | ||
| 190 | return Ok(add_cors_headers(Response::builder()) | 218 | return Ok(add_cors_headers(Response::builder()) |
| 191 | .status(hyper::StatusCode::BAD_REQUEST) | 219 | .status(hyper::StatusCode::BAD_REQUEST) |
| 192 | .body(Full::new(Bytes::from(format!("Invalid npub: {}", e)))) | 220 | .body(Full::new(Bytes::from(format!("Invalid npub: {}", e)))) |
| @@ -194,14 +222,21 @@ impl Service<Request<Incoming>> for HttpService { | |||
| 194 | } | 222 | } |
| 195 | }; | 223 | }; |
| 196 | 224 | ||
| 197 | git::handlers::handle_receive_pack( | 225 | let result = git::handlers::handle_receive_pack( |
| 198 | repo_path, | 226 | repo_path, |
| 199 | body_bytes.clone(), | 227 | body_bytes.clone(), |
| 200 | Some(database.clone()), | 228 | Some(database.clone()), |
| 201 | &identifier, | 229 | &identifier, |
| 202 | &owner_pubkey_hex, | 230 | &owner_pubkey_hex, |
| 203 | ) | 231 | ) |
| 204 | .await | 232 | .await; |
| 233 | |||
| 234 | if let Some(ref m) = metrics_clone { | ||
| 235 | let status = if result.is_ok() { "success" } else { "error" }; | ||
| 236 | m.record_git_operation("push", status); | ||
| 237 | } | ||
| 238 | |||
| 239 | result | ||
| 205 | } | 240 | } |
| 206 | 241 | ||
| 207 | _ => Err(git::handlers::GitError::RepositoryNotFound), | 242 | _ => Err(git::handlers::GitError::RepositoryNotFound), |
| @@ -333,17 +368,27 @@ impl Service<Request<Incoming>> for HttpService { | |||
| 333 | 368 | ||
| 334 | let addr = self.remote; | 369 | let addr = self.remote; |
| 335 | let relay = self.relay.clone(); | 370 | let relay = self.relay.clone(); |
| 371 | let metrics_clone = self.metrics.clone(); | ||
| 336 | 372 | ||
| 337 | tokio::spawn(async move { | 373 | tokio::spawn(async move { |
| 338 | match hyper::upgrade::on(req).await { | 374 | match hyper::upgrade::on(req).await { |
| 339 | Ok(upgraded) => { | 375 | Ok(upgraded) => { |
| 340 | tracing::info!("WebSocket connection established from {}", addr); | 376 | tracing::info!("WebSocket connection established from {}", addr); |
| 377 | // Track connection | ||
| 378 | if let Some(ref m) = metrics_clone { | ||
| 379 | m.connection_tracker().on_connect(addr.ip()); | ||
| 380 | m.record_websocket_connection(); | ||
| 381 | } | ||
| 341 | if let Err(e) = | 382 | if let Err(e) = |
| 342 | relay.take_connection(TokioIo::new(upgraded), addr).await | 383 | relay.take_connection(TokioIo::new(upgraded), addr).await |
| 343 | { | 384 | { |
| 344 | tracing::error!("Relay error for {}: {}", addr, e); | 385 | tracing::error!("Relay error for {}: {}", addr, e); |
| 345 | } | 386 | } |
| 346 | tracing::info!("WebSocket connection closed for {}", addr); | 387 | tracing::info!("WebSocket connection closed for {}", addr); |
| 388 | // Untrack connection | ||
| 389 | if let Some(ref m) = metrics_clone { | ||
| 390 | m.connection_tracker().on_disconnect(addr.ip()); | ||
| 391 | } | ||
| 347 | } | 392 | } |
| 348 | Err(e) => tracing::error!("Upgrade error: {}", e), | 393 | Err(e) => tracing::error!("Upgrade error: {}", e), |
| 349 | } | 394 | } |
| @@ -361,6 +406,33 @@ impl Service<Request<Incoming>> for HttpService { | |||
| 361 | } | 406 | } |
| 362 | } | 407 | } |
| 363 | 408 | ||
| 409 | // Serve Prometheus metrics if enabled | ||
| 410 | if path == "/metrics" { | ||
| 411 | if let Some(ref metrics) = self.metrics { | ||
| 412 | let metrics = metrics.clone(); | ||
| 413 | return Box::pin(async move { | ||
| 414 | let output = metrics.render(); | ||
| 415 | Ok( | ||
| 416 | add_cors_headers(Response::builder().header("server", "ngit-grasp")) | ||
| 417 | .status(200) | ||
| 418 | .header("content-type", "text/plain; version=0.0.4; charset=utf-8") | ||
| 419 | .body(Full::new(Bytes::from(output))) | ||
| 420 | .unwrap(), | ||
| 421 | ) | ||
| 422 | }); | ||
| 423 | } else { | ||
| 424 | // Metrics disabled | ||
| 425 | return Box::pin(async move { | ||
| 426 | Ok( | ||
| 427 | add_cors_headers(Response::builder().header("server", "ngit-grasp")) | ||
| 428 | .status(404) | ||
| 429 | .body(Full::new(Bytes::from("Metrics disabled"))) | ||
| 430 | .unwrap(), | ||
| 431 | ) | ||
| 432 | }); | ||
| 433 | } | ||
| 434 | } | ||
| 435 | |||
| 364 | // Serve static icon at /icon.png | 436 | // Serve static icon at /icon.png |
| 365 | if path == "/icon.png" { | 437 | if path == "/icon.png" { |
| 366 | return Box::pin(async move { | 438 | return Box::pin(async move { |
| @@ -419,10 +491,12 @@ fn derive_accept_key(request_key: &[u8]) -> String { | |||
| 419 | /// * `config` - Server configuration | 491 | /// * `config` - Server configuration |
| 420 | /// * `relay` - The LocalRelay for WebSocket connections | 492 | /// * `relay` - The LocalRelay for WebSocket connections |
| 421 | /// * `database` - The database for direct queries (e.g., push authorization) | 493 | /// * `database` - The database for direct queries (e.g., push authorization) |
| 494 | /// * `metrics` - Optional metrics for Prometheus endpoint | ||
| 422 | pub async fn run_server( | 495 | pub async fn run_server( |
| 423 | config: Config, | 496 | config: Config, |
| 424 | relay: LocalRelay, | 497 | relay: LocalRelay, |
| 425 | database: SharedDatabase, | 498 | database: SharedDatabase, |
| 499 | metrics: Option<Arc<Metrics>>, | ||
| 426 | ) -> anyhow::Result<()> { | 500 | ) -> anyhow::Result<()> { |
| 427 | let bind_addr: SocketAddr = config.bind_address.parse()?; | 501 | let bind_addr: SocketAddr = config.bind_address.parse()?; |
| 428 | 502 | ||
| @@ -435,7 +509,7 @@ pub async fn run_server( | |||
| 435 | loop { | 509 | loop { |
| 436 | let (socket, addr) = listener.accept().await?; | 510 | let (socket, addr) = listener.accept().await?; |
| 437 | let io = TokioIo::new(socket); | 511 | let io = TokioIo::new(socket); |
| 438 | let service = HttpService::new(relay.clone(), config.clone(), addr, database.clone()); | 512 | let service = HttpService::new(relay.clone(), config.clone(), addr, database.clone(), metrics.clone()); |
| 439 | 513 | ||
| 440 | tokio::spawn(async move { | 514 | tokio::spawn(async move { |
| 441 | if let Err(e) = http1::Builder::new() | 515 | if let Err(e) = http1::Builder::new() |
diff --git a/src/http/nip11.rs b/src/http/nip11.rs index 1ed80de..1723601 100644 --- a/src/http/nip11.rs +++ b/src/http/nip11.rs | |||
| @@ -102,6 +102,9 @@ mod tests { | |||
| 102 | relay_data_path: "./data/relay".to_string(), | 102 | relay_data_path: "./data/relay".to_string(), |
| 103 | bind_address: "127.0.0.1:8080".to_string(), | 103 | bind_address: "127.0.0.1:8080".to_string(), |
| 104 | database_backend: crate::config::DatabaseBackend::Memory, | 104 | database_backend: crate::config::DatabaseBackend::Memory, |
| 105 | metrics_enabled: true, | ||
| 106 | metrics_connection_per_ip_abuse_threshold: 10, | ||
| 107 | metrics_top_n_repos: 10, | ||
| 105 | }; | 108 | }; |
| 106 | 109 | ||
| 107 | let doc = RelayInformationDocument::from_config(&config); | 110 | let doc = RelayInformationDocument::from_config(&config); |
| @@ -133,6 +136,9 @@ mod tests { | |||
| 133 | relay_data_path: "./data/relay".to_string(), | 136 | relay_data_path: "./data/relay".to_string(), |
| 134 | bind_address: "127.0.0.1:8080".to_string(), | 137 | bind_address: "127.0.0.1:8080".to_string(), |
| 135 | database_backend: crate::config::DatabaseBackend::Memory, | 138 | database_backend: crate::config::DatabaseBackend::Memory, |
| 139 | metrics_enabled: true, | ||
| 140 | metrics_connection_per_ip_abuse_threshold: 10, | ||
| 141 | metrics_top_n_repos: 10, | ||
| 136 | }; | 142 | }; |
| 137 | 143 | ||
| 138 | let doc = RelayInformationDocument::from_config(&config); | 144 | let doc = RelayInformationDocument::from_config(&config); |
| @@ -1,4 +1,5 @@ | |||
| 1 | pub mod config; | 1 | pub mod config; |
| 2 | pub mod git; | 2 | pub mod git; |
| 3 | pub mod http; | 3 | pub mod http; |
| 4 | pub mod metrics; | ||
| 4 | pub mod nostr; | 5 | pub mod nostr; |
diff --git a/src/main.rs b/src/main.rs index f80e920..9200cc2 100644 --- a/src/main.rs +++ b/src/main.rs | |||
| @@ -1,10 +1,14 @@ | |||
| 1 | use std::sync::Arc; | ||
| 2 | |||
| 1 | use anyhow::Result; | 3 | use anyhow::Result; |
| 2 | use tracing::{info, Level}; | 4 | use tracing::{info, Level}; |
| 3 | use tracing_subscriber::FmtSubscriber; | 5 | use tracing_subscriber::FmtSubscriber; |
| 4 | 6 | ||
| 5 | use ngit_grasp::{ | 7 | use ngit_grasp::{ |
| 6 | config::{Config, DatabaseBackend}, | 8 | config::{Config, DatabaseBackend}, |
| 7 | http, nostr, | 9 | http, |
| 10 | metrics::Metrics, | ||
| 11 | nostr, | ||
| 8 | }; | 12 | }; |
| 9 | 13 | ||
| 10 | #[tokio::main] | 14 | #[tokio::main] |
| @@ -29,6 +33,15 @@ async fn main() -> Result<()> { | |||
| 29 | } | 33 | } |
| 30 | info!("Database backend: {}", config.database_backend); | 34 | info!("Database backend: {}", config.database_backend); |
| 31 | 35 | ||
| 36 | // Initialize metrics if enabled | ||
| 37 | let metrics = if config.metrics_enabled { | ||
| 38 | info!("Metrics enabled on /metrics endpoint"); | ||
| 39 | Some(Arc::new(Metrics::new(config.metrics_connection_per_ip_abuse_threshold))) | ||
| 40 | } else { | ||
| 41 | info!("Metrics disabled"); | ||
| 42 | None | ||
| 43 | }; | ||
| 44 | |||
| 32 | // Create Nostr relay with NIP-34 validation | 45 | // Create Nostr relay with NIP-34 validation |
| 33 | // Returns both the relay and database for direct queries in handlers | 46 | // Returns both the relay and database for direct queries in handlers |
| 34 | if let Ok(relay_with_db) = nostr::builder::create_relay(&config) { | 47 | if let Ok(relay_with_db) = nostr::builder::create_relay(&config) { |
| @@ -39,7 +52,7 @@ async fn main() -> Result<()> { | |||
| 39 | 52 | ||
| 40 | // Start HTTP server with integrated relay and database | 53 | // Start HTTP server with integrated relay and database |
| 41 | info!("Starting HTTP server on {}", config.bind_address); | 54 | info!("Starting HTTP server on {}", config.bind_address); |
| 42 | http::run_server(config, relay_with_db.relay, relay_with_db.database).await?; | 55 | http::run_server(config, relay_with_db.relay, relay_with_db.database, metrics).await?; |
| 43 | } | 56 | } |
| 44 | 57 | ||
| 45 | Ok(()) | 58 | Ok(()) |
diff --git a/src/metrics/bandwidth.rs b/src/metrics/bandwidth.rs new file mode 100644 index 0000000..d2c53e8 --- /dev/null +++ b/src/metrics/bandwidth.rs | |||
| @@ -0,0 +1,301 @@ | |||
| 1 | //! Repository bandwidth tracking with cardinality control. | ||
| 2 | //! | ||
| 3 | //! This module tracks bandwidth per repository but only exposes the top N | ||
| 4 | //! repositories to Prometheus to prevent cardinality explosion with many repos. | ||
| 5 | //! | ||
| 6 | //! # Cardinality Control | ||
| 7 | //! | ||
| 8 | //! - All per-repo bandwidth is tracked internally in a `DashMap<RepoId, u64>` | ||
| 9 | //! - Every 60 seconds, the top 10 are calculated and exposed to Prometheus | ||
| 10 | //! - Previous repo labels are cleared before setting new ones | ||
| 11 | //! - Prometheus only ever sees ~10 label values, keeping cardinality low | ||
| 12 | |||
| 13 | use std::sync::atomic::{AtomicU64, Ordering}; | ||
| 14 | use std::time::{Duration, Instant}; | ||
| 15 | |||
| 16 | use dashmap::DashMap; | ||
| 17 | use prometheus::{GaugeVec, Opts, Registry}; | ||
| 18 | |||
| 19 | /// Default number of top repositories to expose in metrics | ||
| 20 | const DEFAULT_TOP_N: usize = 10; | ||
| 21 | |||
| 22 | /// Default refresh interval for top-N calculation (60 seconds) | ||
| 23 | const DEFAULT_REFRESH_INTERVAL: Duration = Duration::from_secs(60); | ||
| 24 | |||
| 25 | /// Tracks bandwidth per repository with top-N exposure to Prometheus. | ||
| 26 | /// | ||
| 27 | /// # Design | ||
| 28 | /// | ||
| 29 | /// All repositories are tracked internally for accurate total bandwidth, | ||
| 30 | /// but only the top N by bytes transferred are exposed to Prometheus. | ||
| 31 | /// This prevents cardinality explosion when hosting thousands of repositories. | ||
| 32 | /// | ||
| 33 | /// # Thread Safety | ||
| 34 | /// | ||
| 35 | /// Uses `DashMap` for lock-free concurrent access and atomics for | ||
| 36 | /// the refresh timestamp. | ||
| 37 | pub struct BandwidthTracker { | ||
| 38 | /// Internal: tracks ALL repos (memory only, not exposed) | ||
| 39 | all_repos: DashMap<String, u64>, | ||
| 40 | |||
| 41 | /// Exposed to Prometheus: only top N repos | ||
| 42 | top_repos_gauge: GaugeVec, | ||
| 43 | |||
| 44 | /// Last refresh timestamp (stored as nanos since some epoch) | ||
| 45 | last_refresh_nanos: AtomicU64, | ||
| 46 | |||
| 47 | /// Instant when the tracker was created (for relative timing) | ||
| 48 | start_instant: Instant, | ||
| 49 | |||
| 50 | /// Number of top repos to expose | ||
| 51 | top_n: usize, | ||
| 52 | |||
| 53 | /// Refresh interval | ||
| 54 | refresh_interval: Duration, | ||
| 55 | } | ||
| 56 | |||
| 57 | impl BandwidthTracker { | ||
| 58 | /// Creates a new BandwidthTracker and registers metrics with Prometheus. | ||
| 59 | /// | ||
| 60 | /// Uses default settings: | ||
| 61 | /// - Top 10 repositories exposed | ||
| 62 | /// - 60 second refresh interval | ||
| 63 | pub fn new(registry: &Registry) -> Self { | ||
| 64 | Self::with_config(registry, DEFAULT_TOP_N, DEFAULT_REFRESH_INTERVAL) | ||
| 65 | } | ||
| 66 | |||
| 67 | /// Creates a new BandwidthTracker with custom configuration. | ||
| 68 | /// | ||
| 69 | /// # Arguments | ||
| 70 | /// | ||
| 71 | /// * `registry` - Prometheus registry to register metrics with | ||
| 72 | /// * `top_n` - Number of top repositories to expose in metrics | ||
| 73 | /// * `refresh_interval` - How often to recalculate the top-N list | ||
| 74 | pub fn with_config(registry: &Registry, top_n: usize, refresh_interval: Duration) -> Self { | ||
| 75 | let top_repos_gauge = GaugeVec::new( | ||
| 76 | Opts::new( | ||
| 77 | "ngit_git_top_repos_bytes", | ||
| 78 | "Top repositories by bandwidth (refreshed periodically)", | ||
| 79 | ), | ||
| 80 | &["repo"], | ||
| 81 | ) | ||
| 82 | .unwrap(); | ||
| 83 | registry.register(Box::new(top_repos_gauge.clone())).unwrap(); | ||
| 84 | |||
| 85 | Self { | ||
| 86 | all_repos: DashMap::new(), | ||
| 87 | top_repos_gauge, | ||
| 88 | last_refresh_nanos: AtomicU64::new(0), | ||
| 89 | start_instant: Instant::now(), | ||
| 90 | top_n, | ||
| 91 | refresh_interval, | ||
| 92 | } | ||
| 93 | } | ||
| 94 | |||
| 95 | /// Records bytes transferred for a repository. | ||
| 96 | /// | ||
| 97 | /// # Arguments | ||
| 98 | /// | ||
| 99 | /// * `repo_id` - Repository identifier (e.g., npub or repo name) | ||
| 100 | /// * `bytes` - Number of bytes transferred | ||
| 101 | pub fn record_transfer(&self, repo_id: &str, bytes: u64) { | ||
| 102 | self.all_repos | ||
| 103 | .entry(repo_id.to_string()) | ||
| 104 | .and_modify(|v| *v = v.saturating_add(bytes)) | ||
| 105 | .or_insert(bytes); | ||
| 106 | } | ||
| 107 | |||
| 108 | /// Conditionally refreshes the top-N list if the refresh interval has elapsed. | ||
| 109 | /// | ||
| 110 | /// This method is designed to be called frequently (e.g., on every | ||
| 111 | /// `/metrics` request) without performance impact - it only does work | ||
| 112 | /// when the refresh interval has elapsed. | ||
| 113 | pub fn maybe_refresh_top_n(&self) { | ||
| 114 | let elapsed_nanos = self.start_instant.elapsed().as_nanos() as u64; | ||
| 115 | let last_refresh = self.last_refresh_nanos.load(Ordering::Relaxed); | ||
| 116 | let interval_nanos = self.refresh_interval.as_nanos() as u64; | ||
| 117 | |||
| 118 | // Check if enough time has passed since last refresh | ||
| 119 | if elapsed_nanos.saturating_sub(last_refresh) >= interval_nanos { | ||
| 120 | // Try to update the timestamp atomically to prevent concurrent refreshes | ||
| 121 | if self | ||
| 122 | .last_refresh_nanos | ||
| 123 | .compare_exchange(last_refresh, elapsed_nanos, Ordering::SeqCst, Ordering::Relaxed) | ||
| 124 | .is_ok() | ||
| 125 | { | ||
| 126 | self.refresh_top_n(); | ||
| 127 | } | ||
| 128 | } | ||
| 129 | } | ||
| 130 | |||
| 131 | /// Forces a refresh of the top-N list. | ||
| 132 | /// | ||
| 133 | /// This recalculates which repositories are in the top N by bandwidth | ||
| 134 | /// and updates the Prometheus gauges accordingly. | ||
| 135 | pub fn refresh_top_n(&self) { | ||
| 136 | // Collect all repo data | ||
| 137 | let mut sorted: Vec<_> = self | ||
| 138 | .all_repos | ||
| 139 | .iter() | ||
| 140 | .map(|r| (r.key().clone(), *r.value())) | ||
| 141 | .collect(); | ||
| 142 | |||
| 143 | // Sort by bytes descending | ||
| 144 | sorted.sort_by(|a, b| b.1.cmp(&a.1)); | ||
| 145 | |||
| 146 | // Clear old labels and set new top N | ||
| 147 | self.top_repos_gauge.reset(); | ||
| 148 | for (repo, bytes) in sorted.into_iter().take(self.top_n) { | ||
| 149 | self.top_repos_gauge | ||
| 150 | .with_label_values(&[&repo]) | ||
| 151 | .set(bytes as f64); | ||
| 152 | } | ||
| 153 | } | ||
| 154 | |||
| 155 | /// Returns the total bytes transferred for a specific repository. | ||
| 156 | /// | ||
| 157 | /// Returns `None` if the repository has not been seen. | ||
| 158 | pub fn get_repo_bytes(&self, repo_id: &str) -> Option<u64> { | ||
| 159 | self.all_repos.get(repo_id).map(|v| *v) | ||
| 160 | } | ||
| 161 | |||
| 162 | /// Returns the total bytes transferred across all repositories. | ||
| 163 | pub fn total_bytes(&self) -> u64 { | ||
| 164 | self.all_repos.iter().map(|r| *r.value()).sum() | ||
| 165 | } | ||
| 166 | |||
| 167 | /// Returns the number of repositories being tracked. | ||
| 168 | pub fn repo_count(&self) -> usize { | ||
| 169 | self.all_repos.len() | ||
| 170 | } | ||
| 171 | |||
| 172 | /// Returns the top N repositories by bandwidth. | ||
| 173 | /// | ||
| 174 | /// This is a snapshot and may not match the Prometheus gauges if | ||
| 175 | /// a refresh hasn't occurred recently. | ||
| 176 | pub fn get_top_repos(&self) -> Vec<(String, u64)> { | ||
| 177 | let mut sorted: Vec<_> = self | ||
| 178 | .all_repos | ||
| 179 | .iter() | ||
| 180 | .map(|r| (r.key().clone(), *r.value())) | ||
| 181 | .collect(); | ||
| 182 | |||
| 183 | sorted.sort_by(|a, b| b.1.cmp(&a.1)); | ||
| 184 | sorted.truncate(self.top_n); | ||
| 185 | sorted | ||
| 186 | } | ||
| 187 | } | ||
| 188 | |||
| 189 | #[cfg(test)] | ||
| 190 | mod tests { | ||
| 191 | use super::*; | ||
| 192 | |||
| 193 | fn test_registry() -> Registry { | ||
| 194 | Registry::new() | ||
| 195 | } | ||
| 196 | |||
| 197 | #[test] | ||
| 198 | fn test_bandwidth_tracking() { | ||
| 199 | let registry = test_registry(); | ||
| 200 | let tracker = BandwidthTracker::new(®istry); | ||
| 201 | |||
| 202 | // Record transfers | ||
| 203 | tracker.record_transfer("repo-a", 1000); | ||
| 204 | tracker.record_transfer("repo-b", 2000); | ||
| 205 | tracker.record_transfer("repo-a", 500); // Additional transfer to repo-a | ||
| 206 | |||
| 207 | assert_eq!(tracker.get_repo_bytes("repo-a"), Some(1500)); | ||
| 208 | assert_eq!(tracker.get_repo_bytes("repo-b"), Some(2000)); | ||
| 209 | assert_eq!(tracker.get_repo_bytes("repo-c"), None); | ||
| 210 | assert_eq!(tracker.total_bytes(), 3500); | ||
| 211 | assert_eq!(tracker.repo_count(), 2); | ||
| 212 | } | ||
| 213 | |||
| 214 | #[test] | ||
| 215 | fn test_top_n_repos() { | ||
| 216 | let registry = test_registry(); | ||
| 217 | let tracker = BandwidthTracker::with_config(®istry, 3, Duration::from_secs(60)); | ||
| 218 | |||
| 219 | // Create 5 repos with different bandwidth | ||
| 220 | tracker.record_transfer("repo-1", 100); | ||
| 221 | tracker.record_transfer("repo-2", 500); | ||
| 222 | tracker.record_transfer("repo-3", 200); | ||
| 223 | tracker.record_transfer("repo-4", 800); | ||
| 224 | tracker.record_transfer("repo-5", 300); | ||
| 225 | |||
| 226 | let top = tracker.get_top_repos(); | ||
| 227 | assert_eq!(top.len(), 3); | ||
| 228 | assert_eq!(top[0], ("repo-4".to_string(), 800)); | ||
| 229 | assert_eq!(top[1], ("repo-2".to_string(), 500)); | ||
| 230 | assert_eq!(top[2], ("repo-5".to_string(), 300)); | ||
| 231 | } | ||
| 232 | |||
| 233 | #[test] | ||
| 234 | fn test_refresh_updates_gauge() { | ||
| 235 | let registry = test_registry(); | ||
| 236 | let tracker = BandwidthTracker::new(®istry); | ||
| 237 | |||
| 238 | tracker.record_transfer("high-bandwidth-repo", 10_000_000); | ||
| 239 | tracker.record_transfer("low-bandwidth-repo", 1000); | ||
| 240 | |||
| 241 | // Force a refresh | ||
| 242 | tracker.refresh_top_n(); | ||
| 243 | |||
| 244 | // Verify the gauge values (we can't easily access them directly, | ||
| 245 | // but we can verify the tracker state is correct) | ||
| 246 | assert_eq!(tracker.repo_count(), 2); | ||
| 247 | assert_eq!(tracker.total_bytes(), 10_001_000); | ||
| 248 | } | ||
| 249 | |||
| 250 | #[test] | ||
| 251 | fn test_saturating_add() { | ||
| 252 | let registry = test_registry(); | ||
| 253 | let tracker = BandwidthTracker::new(®istry); | ||
| 254 | |||
| 255 | // Test that we don't overflow | ||
| 256 | tracker.record_transfer("huge-repo", u64::MAX - 100); | ||
| 257 | tracker.record_transfer("huge-repo", 200); | ||
| 258 | |||
| 259 | // Should saturate to MAX, not overflow | ||
| 260 | assert_eq!(tracker.get_repo_bytes("huge-repo"), Some(u64::MAX)); | ||
| 261 | } | ||
| 262 | |||
| 263 | #[test] | ||
| 264 | fn test_maybe_refresh_respects_interval() { | ||
| 265 | let registry = test_registry(); | ||
| 266 | // Use a very short interval for testing | ||
| 267 | let tracker = BandwidthTracker::with_config(®istry, 10, Duration::from_millis(10)); | ||
| 268 | |||
| 269 | tracker.record_transfer("repo-a", 1000); | ||
| 270 | |||
| 271 | // First call should trigger refresh (no previous refresh) | ||
| 272 | tracker.maybe_refresh_top_n(); | ||
| 273 | |||
| 274 | // Add more data | ||
| 275 | tracker.record_transfer("repo-b", 2000); | ||
| 276 | |||
| 277 | // Immediate second call should NOT trigger refresh | ||
| 278 | let count_before = tracker.repo_count(); | ||
| 279 | tracker.maybe_refresh_top_n(); | ||
| 280 | assert_eq!(tracker.repo_count(), count_before); | ||
| 281 | |||
| 282 | // Wait for interval to pass | ||
| 283 | std::thread::sleep(Duration::from_millis(15)); | ||
| 284 | |||
| 285 | // Now it should refresh | ||
| 286 | tracker.maybe_refresh_top_n(); | ||
| 287 | } | ||
| 288 | |||
| 289 | #[test] | ||
| 290 | fn test_empty_tracker() { | ||
| 291 | let registry = test_registry(); | ||
| 292 | let tracker = BandwidthTracker::new(®istry); | ||
| 293 | |||
| 294 | assert_eq!(tracker.total_bytes(), 0); | ||
| 295 | assert_eq!(tracker.repo_count(), 0); | ||
| 296 | assert!(tracker.get_top_repos().is_empty()); | ||
| 297 | |||
| 298 | // Refresh should not panic on empty data | ||
| 299 | tracker.refresh_top_n(); | ||
| 300 | } | ||
| 301 | } \ No newline at end of file | ||
diff --git a/src/metrics/connection.rs b/src/metrics/connection.rs new file mode 100644 index 0000000..6a7f406 --- /dev/null +++ b/src/metrics/connection.rs | |||
| @@ -0,0 +1,337 @@ | |||
| 1 | //! Connection tracking with privacy-preserving abuse detection. | ||
| 2 | //! | ||
| 3 | //! This module tracks WebSocket connections per IP address internally for abuse | ||
| 4 | //! detection, but NEVER exposes IP addresses in Prometheus metrics. Only aggregate | ||
| 5 | //! counts are exposed. | ||
| 6 | //! | ||
| 7 | //! # Privacy Model | ||
| 8 | //! | ||
| 9 | //! | Data | Location | Exposed? | | ||
| 10 | //! |------|----------|----------| | ||
| 11 | //! | Total connections | Prometheus | ✅ Yes | | ||
| 12 | //! | Unique IP count | Prometheus | ✅ Yes | | ||
| 13 | //! | Flagged abuser count | Prometheus | ✅ Yes | | ||
| 14 | //! | Actual IP addresses | Internal HashMap | ❌ No | | ||
| 15 | //! | IP + abuse flag | Logs (when flagged) | ⚠️ Logs only | | ||
| 16 | |||
| 17 | use std::net::IpAddr; | ||
| 18 | use std::time::Instant; | ||
| 19 | |||
| 20 | use dashmap::DashMap; | ||
| 21 | use prometheus::{IntGauge, Opts, Registry}; | ||
| 22 | use tracing::warn; | ||
| 23 | |||
| 24 | /// Information about connections from a specific IP address. | ||
| 25 | struct ConnectionInfo { | ||
| 26 | /// Number of active connections from this IP | ||
| 27 | count: u32, | ||
| 28 | /// When the first connection from this IP was established | ||
| 29 | first_seen: Instant, | ||
| 30 | /// Whether this IP has been flagged as potentially abusive | ||
| 31 | flagged_as_abuse: bool, | ||
| 32 | } | ||
| 33 | |||
| 34 | /// Tracks WebSocket connections per IP with abuse detection. | ||
| 35 | /// | ||
| 36 | /// # Thread Safety | ||
| 37 | /// | ||
| 38 | /// Uses `DashMap` for lock-free concurrent access, as connection tracking | ||
| 39 | /// happens across multiple tokio tasks. | ||
| 40 | /// | ||
| 41 | /// # Privacy | ||
| 42 | /// | ||
| 43 | /// IP addresses are stored internally only for abuse detection and are | ||
| 44 | /// NEVER exposed in Prometheus metrics. Only aggregate counts are exposed: | ||
| 45 | /// - Total active connections | ||
| 46 | /// - Number of unique IPs | ||
| 47 | /// - Number of IPs flagged as potential abusers | ||
| 48 | pub struct ConnectionTracker { | ||
| 49 | /// Active connections per IP (INTERNAL ONLY - never exposed to metrics) | ||
| 50 | connections: DashMap<IpAddr, ConnectionInfo>, | ||
| 51 | |||
| 52 | /// Threshold for abuse flagging (connections per IP) | ||
| 53 | abuse_threshold: u32, | ||
| 54 | |||
| 55 | /// Prometheus gauge: total active connections | ||
| 56 | active_connections: IntGauge, | ||
| 57 | |||
| 58 | /// Prometheus gauge: number of unique IPs connected | ||
| 59 | unique_ips: IntGauge, | ||
| 60 | |||
| 61 | /// Prometheus gauge: number of IPs flagged as potential abusers | ||
| 62 | flagged_abusers: IntGauge, | ||
| 63 | } | ||
| 64 | |||
| 65 | impl ConnectionTracker { | ||
| 66 | /// Creates a new ConnectionTracker and registers metrics with Prometheus. | ||
| 67 | /// | ||
| 68 | /// # Arguments | ||
| 69 | /// | ||
| 70 | /// * `abuse_threshold` - Number of connections from a single IP before flagging | ||
| 71 | /// * `registry` - Prometheus registry to register metrics with | ||
| 72 | pub fn new(abuse_threshold: u32, registry: &Registry) -> Self { | ||
| 73 | let active_connections = IntGauge::with_opts( | ||
| 74 | Opts::new( | ||
| 75 | "ngit_websocket_connections_active", | ||
| 76 | "Current active WebSocket connections", | ||
| 77 | ) | ||
| 78 | ).unwrap(); | ||
| 79 | registry.register(Box::new(active_connections.clone())).unwrap(); | ||
| 80 | |||
| 81 | let unique_ips = IntGauge::with_opts( | ||
| 82 | Opts::new( | ||
| 83 | "ngit_websocket_unique_ips", | ||
| 84 | "Number of unique IP addresses connected (NOT the IPs themselves)", | ||
| 85 | ) | ||
| 86 | ).unwrap(); | ||
| 87 | registry.register(Box::new(unique_ips.clone())).unwrap(); | ||
| 88 | |||
| 89 | let flagged_abusers = IntGauge::with_opts( | ||
| 90 | Opts::new( | ||
| 91 | "ngit_websocket_flagged_abusers", | ||
| 92 | "Number of IPs exceeding connection threshold", | ||
| 93 | ) | ||
| 94 | ).unwrap(); | ||
| 95 | registry.register(Box::new(flagged_abusers.clone())).unwrap(); | ||
| 96 | |||
| 97 | Self { | ||
| 98 | connections: DashMap::new(), | ||
| 99 | abuse_threshold, | ||
| 100 | active_connections, | ||
| 101 | unique_ips, | ||
| 102 | flagged_abusers, | ||
| 103 | } | ||
| 104 | } | ||
| 105 | |||
| 106 | /// Called when a new WebSocket connection is established. | ||
| 107 | /// | ||
| 108 | /// This method: | ||
| 109 | /// 1. Increments the connection count for this IP | ||
| 110 | /// 2. Checks if the IP has exceeded the abuse threshold | ||
| 111 | /// 3. Logs a warning if abuse is detected (IP is logged here only) | ||
| 112 | /// 4. Updates Prometheus metrics (aggregate counts only) | ||
| 113 | /// | ||
| 114 | /// # Privacy | ||
| 115 | /// | ||
| 116 | /// The IP address is logged only when abuse is detected. It is NEVER | ||
| 117 | /// exposed in Prometheus metrics. | ||
| 118 | pub fn on_connect(&self, ip: IpAddr) { | ||
| 119 | let mut is_new_ip = false; | ||
| 120 | let mut newly_flagged = false; | ||
| 121 | |||
| 122 | self.connections | ||
| 123 | .entry(ip) | ||
| 124 | .and_modify(|info| { | ||
| 125 | info.count += 1; | ||
| 126 | // Check if this connection pushes us over the threshold | ||
| 127 | if !info.flagged_as_abuse && info.count >= self.abuse_threshold { | ||
| 128 | info.flagged_as_abuse = true; | ||
| 129 | newly_flagged = true; | ||
| 130 | } | ||
| 131 | }) | ||
| 132 | .or_insert_with(|| { | ||
| 133 | is_new_ip = true; | ||
| 134 | ConnectionInfo { | ||
| 135 | count: 1, | ||
| 136 | first_seen: Instant::now(), | ||
| 137 | flagged_as_abuse: false, | ||
| 138 | } | ||
| 139 | }); | ||
| 140 | |||
| 141 | // Update Prometheus metrics (aggregate counts only) | ||
| 142 | self.active_connections.inc(); | ||
| 143 | |||
| 144 | if is_new_ip { | ||
| 145 | self.unique_ips.inc(); | ||
| 146 | } | ||
| 147 | |||
| 148 | if newly_flagged { | ||
| 149 | self.flagged_abusers.inc(); | ||
| 150 | // Log the abuse detection - IP is only exposed in logs, not metrics | ||
| 151 | warn!( | ||
| 152 | ip = %ip, | ||
| 153 | threshold = self.abuse_threshold, | ||
| 154 | "Potential abuse detected: IP exceeded connection threshold" | ||
| 155 | ); | ||
| 156 | } | ||
| 157 | } | ||
| 158 | |||
| 159 | /// Called when a WebSocket connection is closed. | ||
| 160 | /// | ||
| 161 | /// This method: | ||
| 162 | /// 1. Decrements the connection count for this IP | ||
| 163 | /// 2. Removes the IP from tracking if count reaches 0 | ||
| 164 | /// 3. Updates the abuse flag count if the IP was flagged | ||
| 165 | /// 4. Updates Prometheus metrics (aggregate counts only) | ||
| 166 | pub fn on_disconnect(&self, ip: IpAddr) { | ||
| 167 | let mut remove_entry = false; | ||
| 168 | let mut was_flagged = false; | ||
| 169 | let mut had_connection = false; | ||
| 170 | |||
| 171 | if let Some(mut entry) = self.connections.get_mut(&ip) { | ||
| 172 | had_connection = true; | ||
| 173 | entry.count = entry.count.saturating_sub(1); | ||
| 174 | if entry.count == 0 { | ||
| 175 | remove_entry = true; | ||
| 176 | was_flagged = entry.flagged_as_abuse; | ||
| 177 | } | ||
| 178 | } | ||
| 179 | |||
| 180 | // Remove the entry if count is 0 | ||
| 181 | if remove_entry { | ||
| 182 | self.connections.remove(&ip); | ||
| 183 | self.unique_ips.dec(); | ||
| 184 | if was_flagged { | ||
| 185 | self.flagged_abusers.dec(); | ||
| 186 | } | ||
| 187 | } | ||
| 188 | |||
| 189 | // Update total connections only if this IP had a tracked connection | ||
| 190 | if had_connection { | ||
| 191 | self.active_connections.dec(); | ||
| 192 | } | ||
| 193 | } | ||
| 194 | |||
| 195 | /// Returns the current number of active connections. | ||
| 196 | pub fn active_connections(&self) -> u64 { | ||
| 197 | self.active_connections.get() as u64 | ||
| 198 | } | ||
| 199 | |||
| 200 | /// Returns the current number of unique IPs. | ||
| 201 | pub fn unique_ip_count(&self) -> u64 { | ||
| 202 | self.unique_ips.get() as u64 | ||
| 203 | } | ||
| 204 | |||
| 205 | /// Returns the current number of flagged abusers. | ||
| 206 | pub fn flagged_abuser_count(&self) -> u64 { | ||
| 207 | self.flagged_abusers.get() as u64 | ||
| 208 | } | ||
| 209 | |||
| 210 | /// Returns the connection count for a specific IP (for internal use only). | ||
| 211 | /// | ||
| 212 | /// # Privacy | ||
| 213 | /// | ||
| 214 | /// This is an internal method. The returned data should NEVER be exposed | ||
| 215 | /// in metrics or logs without privacy consideration. | ||
| 216 | #[cfg(test)] | ||
| 217 | pub(crate) fn connection_count(&self, ip: &IpAddr) -> Option<u32> { | ||
| 218 | self.connections.get(ip).map(|info| info.count) | ||
| 219 | } | ||
| 220 | |||
| 221 | /// Returns whether an IP is flagged as abusive (for internal use only). | ||
| 222 | #[cfg(test)] | ||
| 223 | pub(crate) fn is_flagged(&self, ip: &IpAddr) -> bool { | ||
| 224 | self.connections | ||
| 225 | .get(ip) | ||
| 226 | .map(|info| info.flagged_as_abuse) | ||
| 227 | .unwrap_or(false) | ||
| 228 | } | ||
| 229 | } | ||
| 230 | |||
| 231 | #[cfg(test)] | ||
| 232 | mod tests { | ||
| 233 | use super::*; | ||
| 234 | use std::net::{Ipv4Addr, Ipv6Addr}; | ||
| 235 | |||
| 236 | fn test_registry() -> Registry { | ||
| 237 | Registry::new() | ||
| 238 | } | ||
| 239 | |||
| 240 | #[test] | ||
| 241 | fn test_connection_tracking() { | ||
| 242 | let registry = test_registry(); | ||
| 243 | let tracker = ConnectionTracker::new(5, ®istry); | ||
| 244 | let ip = IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1)); | ||
| 245 | |||
| 246 | // Connect | ||
| 247 | tracker.on_connect(ip); | ||
| 248 | assert_eq!(tracker.active_connections(), 1); | ||
| 249 | assert_eq!(tracker.unique_ip_count(), 1); | ||
| 250 | assert_eq!(tracker.connection_count(&ip), Some(1)); | ||
| 251 | |||
| 252 | // Connect again from same IP | ||
| 253 | tracker.on_connect(ip); | ||
| 254 | assert_eq!(tracker.active_connections(), 2); | ||
| 255 | assert_eq!(tracker.unique_ip_count(), 1); // Still 1 unique IP | ||
| 256 | assert_eq!(tracker.connection_count(&ip), Some(2)); | ||
| 257 | |||
| 258 | // Disconnect one | ||
| 259 | tracker.on_disconnect(ip); | ||
| 260 | assert_eq!(tracker.active_connections(), 1); | ||
| 261 | assert_eq!(tracker.unique_ip_count(), 1); | ||
| 262 | assert_eq!(tracker.connection_count(&ip), Some(1)); | ||
| 263 | |||
| 264 | // Disconnect last | ||
| 265 | tracker.on_disconnect(ip); | ||
| 266 | assert_eq!(tracker.active_connections(), 0); | ||
| 267 | assert_eq!(tracker.unique_ip_count(), 0); | ||
| 268 | assert_eq!(tracker.connection_count(&ip), None); | ||
| 269 | } | ||
| 270 | |||
| 271 | #[test] | ||
| 272 | fn test_multiple_ips() { | ||
| 273 | let registry = test_registry(); | ||
| 274 | let tracker = ConnectionTracker::new(5, ®istry); | ||
| 275 | let ip1 = IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1)); | ||
| 276 | let ip2 = IpAddr::V4(Ipv4Addr::new(192, 168, 1, 2)); | ||
| 277 | let ip3 = IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1)); | ||
| 278 | |||
| 279 | tracker.on_connect(ip1); | ||
| 280 | tracker.on_connect(ip2); | ||
| 281 | tracker.on_connect(ip3); | ||
| 282 | |||
| 283 | assert_eq!(tracker.active_connections(), 3); | ||
| 284 | assert_eq!(tracker.unique_ip_count(), 3); | ||
| 285 | |||
| 286 | tracker.on_disconnect(ip2); | ||
| 287 | assert_eq!(tracker.active_connections(), 2); | ||
| 288 | assert_eq!(tracker.unique_ip_count(), 2); | ||
| 289 | } | ||
| 290 | |||
| 291 | #[test] | ||
| 292 | fn test_abuse_detection() { | ||
| 293 | let registry = test_registry(); | ||
| 294 | let threshold = 3; | ||
| 295 | let tracker = ConnectionTracker::new(threshold, ®istry); | ||
| 296 | let abuser_ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)); | ||
| 297 | let normal_ip = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)); | ||
| 298 | |||
| 299 | // Normal user with 1 connection | ||
| 300 | tracker.on_connect(normal_ip); | ||
| 301 | assert!(!tracker.is_flagged(&normal_ip)); | ||
| 302 | assert_eq!(tracker.flagged_abuser_count(), 0); | ||
| 303 | |||
| 304 | // Abuser approaching threshold | ||
| 305 | tracker.on_connect(abuser_ip); | ||
| 306 | tracker.on_connect(abuser_ip); | ||
| 307 | assert!(!tracker.is_flagged(&abuser_ip)); | ||
| 308 | assert_eq!(tracker.flagged_abuser_count(), 0); | ||
| 309 | |||
| 310 | // Abuser hits threshold | ||
| 311 | tracker.on_connect(abuser_ip); | ||
| 312 | assert!(tracker.is_flagged(&abuser_ip)); | ||
| 313 | assert_eq!(tracker.flagged_abuser_count(), 1); | ||
| 314 | |||
| 315 | // Normal user still not flagged | ||
| 316 | assert!(!tracker.is_flagged(&normal_ip)); | ||
| 317 | |||
| 318 | // Abuser disconnects all - should be removed from flagged count | ||
| 319 | tracker.on_disconnect(abuser_ip); | ||
| 320 | tracker.on_disconnect(abuser_ip); | ||
| 321 | tracker.on_disconnect(abuser_ip); | ||
| 322 | assert_eq!(tracker.flagged_abuser_count(), 0); | ||
| 323 | assert_eq!(tracker.active_connections(), 1); // Only normal user remains | ||
| 324 | } | ||
| 325 | |||
| 326 | #[test] | ||
| 327 | fn test_disconnect_without_connect() { | ||
| 328 | let registry = test_registry(); | ||
| 329 | let tracker = ConnectionTracker::new(5, ®istry); | ||
| 330 | let ip = IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1)); | ||
| 331 | |||
| 332 | // Disconnect without connect should not panic or go negative | ||
| 333 | tracker.on_disconnect(ip); | ||
| 334 | assert_eq!(tracker.active_connections(), 0); | ||
| 335 | assert_eq!(tracker.unique_ip_count(), 0); | ||
| 336 | } | ||
| 337 | } \ No newline at end of file | ||
diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs new file mode 100644 index 0000000..4a4fe57 --- /dev/null +++ b/src/metrics/mod.rs | |||
| @@ -0,0 +1,469 @@ | |||
| 1 | //! Prometheus metrics for ngit-grasp relay. | ||
| 2 | //! | ||
| 3 | //! This module provides comprehensive monitoring metrics including: | ||
| 4 | //! - WebSocket connection tracking (with privacy-preserving IP aggregation) | ||
| 5 | //! - Git operation metrics (clone, fetch, push) | ||
| 6 | //! - Repository bandwidth tracking (top-N only for cardinality control) | ||
| 7 | //! - Nostr event metrics | ||
| 8 | //! | ||
| 9 | //! # Privacy | ||
| 10 | //! IP addresses are NEVER exposed in metrics. The `ConnectionTracker` maintains | ||
| 11 | //! per-IP counts internally only for abuse detection. Only aggregate counts | ||
| 12 | //! are exposed to Prometheus. | ||
| 13 | |||
| 14 | pub mod bandwidth; | ||
| 15 | pub mod connection; | ||
| 16 | |||
| 17 | use std::sync::Arc; | ||
| 18 | use std::time::Instant; | ||
| 19 | |||
| 20 | use lazy_static::lazy_static; | ||
| 21 | use prometheus::{ | ||
| 22 | Counter, CounterVec, Encoder, Gauge, GaugeVec, Histogram, HistogramOpts, HistogramVec, Opts, | ||
| 23 | Registry, TextEncoder, | ||
| 24 | }; | ||
| 25 | |||
| 26 | use bandwidth::BandwidthTracker; | ||
| 27 | use connection::ConnectionTracker; | ||
| 28 | |||
| 29 | lazy_static! { | ||
| 30 | /// Global Prometheus registry for ngit-grasp metrics | ||
| 31 | pub static ref REGISTRY: Registry = Registry::new(); | ||
| 32 | } | ||
| 33 | |||
| 34 | /// Central metrics collection for ngit-grasp relay. | ||
| 35 | /// | ||
| 36 | /// Thread-safe and designed for concurrent access from multiple tokio tasks. | ||
| 37 | #[derive(Clone)] | ||
| 38 | pub struct Metrics { | ||
| 39 | inner: Arc<MetricsInner>, | ||
| 40 | } | ||
| 41 | |||
| 42 | struct MetricsInner { | ||
| 43 | /// Connection tracking with abuse detection | ||
| 44 | pub connection_tracker: ConnectionTracker, | ||
| 45 | |||
| 46 | /// Repository bandwidth tracking (top-N only) | ||
| 47 | pub bandwidth_tracker: BandwidthTracker, | ||
| 48 | |||
| 49 | // === WebSocket Metrics === | ||
| 50 | /// Total WebSocket connections since startup | ||
| 51 | pub websocket_connections_total: Counter, | ||
| 52 | /// Connection duration histogram | ||
| 53 | pub websocket_connection_duration: Histogram, | ||
| 54 | /// Messages received by type (REQ, EVENT, CLOSE) | ||
| 55 | pub websocket_messages_received: CounterVec, | ||
| 56 | /// Messages sent by type (EVENT, EOSE, OK, NOTICE) | ||
| 57 | pub websocket_messages_sent: CounterVec, | ||
| 58 | |||
| 59 | // === Git Operation Metrics === | ||
| 60 | /// Git operations by type and status | ||
| 61 | pub git_operations_total: CounterVec, | ||
| 62 | /// Git operation duration histogram | ||
| 63 | pub git_operation_duration: HistogramVec, | ||
| 64 | /// Total bytes transferred | ||
| 65 | pub git_bytes_total: CounterVec, | ||
| 66 | /// Push authorization results | ||
| 67 | pub git_push_authorization: CounterVec, | ||
| 68 | |||
| 69 | // === Nostr Event Metrics === | ||
| 70 | /// Events received by kind | ||
| 71 | pub events_received_total: CounterVec, | ||
| 72 | /// Events successfully stored by kind | ||
| 73 | pub events_stored_total: CounterVec, | ||
| 74 | /// Events rejected by kind and reason | ||
| 75 | pub events_rejected_total: CounterVec, | ||
| 76 | |||
| 77 | // === Repository Metrics === | ||
| 78 | /// Total repositories hosted | ||
| 79 | pub repositories_total: Gauge, | ||
| 80 | |||
| 81 | // === System Health Metrics === | ||
| 82 | /// Server start time for uptime calculation | ||
| 83 | pub start_time: Instant, | ||
| 84 | /// Build information gauge | ||
| 85 | pub build_info: GaugeVec, | ||
| 86 | } | ||
| 87 | |||
| 88 | impl Metrics { | ||
| 89 | /// Creates a new Metrics instance and registers all metrics with Prometheus. | ||
| 90 | /// | ||
| 91 | /// # Arguments | ||
| 92 | /// * `abuse_threshold` - Number of connections from a single IP before flagging as abuse | ||
| 93 | pub fn new(abuse_threshold: u32) -> Self { | ||
| 94 | let inner = MetricsInner::new(abuse_threshold); | ||
| 95 | Self { | ||
| 96 | inner: Arc::new(inner), | ||
| 97 | } | ||
| 98 | } | ||
| 99 | |||
| 100 | /// Returns the connection tracker for WebSocket connection management. | ||
| 101 | pub fn connection_tracker(&self) -> &ConnectionTracker { | ||
| 102 | &self.inner.connection_tracker | ||
| 103 | } | ||
| 104 | |||
| 105 | /// Returns the bandwidth tracker for repository bandwidth tracking. | ||
| 106 | pub fn bandwidth_tracker(&self) -> &BandwidthTracker { | ||
| 107 | &self.inner.bandwidth_tracker | ||
| 108 | } | ||
| 109 | |||
| 110 | // === WebSocket Recording Methods === | ||
| 111 | |||
| 112 | /// Record a new WebSocket connection | ||
| 113 | pub fn record_websocket_connection(&self) { | ||
| 114 | self.inner.websocket_connections_total.inc(); | ||
| 115 | } | ||
| 116 | |||
| 117 | /// Start timing a WebSocket connection, returns timer that records on drop | ||
| 118 | pub fn start_connection_timer(&self) -> HistogramTimer { | ||
| 119 | HistogramTimer::new(self.inner.websocket_connection_duration.clone()) | ||
| 120 | } | ||
| 121 | |||
| 122 | /// Record a received WebSocket message | ||
| 123 | pub fn record_message_received(&self, msg_type: &str) { | ||
| 124 | self.inner | ||
| 125 | .websocket_messages_received | ||
| 126 | .with_label_values(&[msg_type]) | ||
| 127 | .inc(); | ||
| 128 | } | ||
| 129 | |||
| 130 | /// Record a sent WebSocket message | ||
| 131 | pub fn record_message_sent(&self, msg_type: &str) { | ||
| 132 | self.inner | ||
| 133 | .websocket_messages_sent | ||
| 134 | .with_label_values(&[msg_type]) | ||
| 135 | .inc(); | ||
| 136 | } | ||
| 137 | |||
| 138 | // === Git Operation Recording Methods === | ||
| 139 | |||
| 140 | /// Record a git operation completion | ||
| 141 | pub fn record_git_operation(&self, operation: &str, status: &str) { | ||
| 142 | self.inner | ||
| 143 | .git_operations_total | ||
| 144 | .with_label_values(&[operation, status]) | ||
| 145 | .inc(); | ||
| 146 | } | ||
| 147 | |||
| 148 | /// Start timing a git operation, returns a timer | ||
| 149 | pub fn start_git_operation_timer(&self, operation: &str) -> GitOperationTimer { | ||
| 150 | GitOperationTimer::new(self.inner.git_operation_duration.clone(), operation.to_string()) | ||
| 151 | } | ||
| 152 | |||
| 153 | /// Record bytes transferred for a git operation | ||
| 154 | pub fn record_git_bytes(&self, direction: &str, bytes: u64) { | ||
| 155 | self.inner | ||
| 156 | .git_bytes_total | ||
| 157 | .with_label_values(&[direction]) | ||
| 158 | .inc_by(bytes as f64); | ||
| 159 | } | ||
| 160 | |||
| 161 | /// Record a push authorization result | ||
| 162 | pub fn record_push_authorization(&self, result: &str) { | ||
| 163 | self.inner | ||
| 164 | .git_push_authorization | ||
| 165 | .with_label_values(&[result]) | ||
| 166 | .inc(); | ||
| 167 | } | ||
| 168 | |||
| 169 | // === Nostr Event Recording Methods === | ||
| 170 | |||
| 171 | /// Record a received Nostr event | ||
| 172 | pub fn record_event_received(&self, kind: u64) { | ||
| 173 | self.inner | ||
| 174 | .events_received_total | ||
| 175 | .with_label_values(&[&kind.to_string()]) | ||
| 176 | .inc(); | ||
| 177 | } | ||
| 178 | |||
| 179 | /// Record a stored Nostr event | ||
| 180 | pub fn record_event_stored(&self, kind: u64) { | ||
| 181 | self.inner | ||
| 182 | .events_stored_total | ||
| 183 | .with_label_values(&[&kind.to_string()]) | ||
| 184 | .inc(); | ||
| 185 | } | ||
| 186 | |||
| 187 | /// Record a rejected Nostr event | ||
| 188 | pub fn record_event_rejected(&self, kind: u64, reason: &str) { | ||
| 189 | self.inner | ||
| 190 | .events_rejected_total | ||
| 191 | .with_label_values(&[&kind.to_string(), reason]) | ||
| 192 | .inc(); | ||
| 193 | } | ||
| 194 | |||
| 195 | // === Repository Metrics === | ||
| 196 | |||
| 197 | /// Set the total number of repositories | ||
| 198 | pub fn set_repositories_total(&self, count: u64) { | ||
| 199 | self.inner.repositories_total.set(count as f64); | ||
| 200 | } | ||
| 201 | |||
| 202 | /// Increment the repository count | ||
| 203 | pub fn inc_repositories_total(&self) { | ||
| 204 | self.inner.repositories_total.inc(); | ||
| 205 | } | ||
| 206 | |||
| 207 | // === Rendering === | ||
| 208 | |||
| 209 | /// Render all metrics in Prometheus text format. | ||
| 210 | /// | ||
| 211 | /// This method: | ||
| 212 | /// 1. Refreshes the top-N bandwidth metrics if needed | ||
| 213 | /// 2. Updates uptime | ||
| 214 | /// 3. Gathers all metrics from the registry | ||
| 215 | /// 4. Encodes them in Prometheus text format | ||
| 216 | pub fn render(&self) -> String { | ||
| 217 | // Refresh top-N bandwidth repos if needed | ||
| 218 | self.inner.bandwidth_tracker.maybe_refresh_top_n(); | ||
| 219 | |||
| 220 | // Gather and encode metrics | ||
| 221 | let encoder = TextEncoder::new(); | ||
| 222 | let metric_families = REGISTRY.gather(); | ||
| 223 | let mut buffer = Vec::new(); | ||
| 224 | encoder.encode(&metric_families, &mut buffer).unwrap(); | ||
| 225 | |||
| 226 | // Add uptime as a comment (it's derived, not a registered metric) | ||
| 227 | let uptime = self.inner.start_time.elapsed().as_secs(); | ||
| 228 | let mut output = String::from_utf8(buffer).unwrap(); | ||
| 229 | output.push_str(&format!( | ||
| 230 | "\n# HELP ngit_uptime_seconds Seconds since server startup\n# TYPE ngit_uptime_seconds counter\nngit_uptime_seconds {}\n", | ||
| 231 | uptime | ||
| 232 | )); | ||
| 233 | |||
| 234 | output | ||
| 235 | } | ||
| 236 | |||
| 237 | /// Check if the system is under high load (for sync scheduling) | ||
| 238 | pub fn is_high_load(&self, threshold: u64) -> bool { | ||
| 239 | self.inner.connection_tracker.active_connections() > threshold | ||
| 240 | } | ||
| 241 | } | ||
| 242 | |||
| 243 | impl MetricsInner { | ||
| 244 | fn new(abuse_threshold: u32) -> Self { | ||
| 245 | // Create connection tracker | ||
| 246 | let connection_tracker = ConnectionTracker::new(abuse_threshold, ®ISTRY); | ||
| 247 | |||
| 248 | // Create bandwidth tracker | ||
| 249 | let bandwidth_tracker = BandwidthTracker::new(®ISTRY); | ||
| 250 | |||
| 251 | // WebSocket metrics | ||
| 252 | let websocket_connections_total = Counter::with_opts( | ||
| 253 | Opts::new( | ||
| 254 | "ngit_websocket_connections_total", | ||
| 255 | "Total WebSocket connections since startup", | ||
| 256 | ) | ||
| 257 | ).unwrap(); | ||
| 258 | REGISTRY.register(Box::new(websocket_connections_total.clone())).unwrap(); | ||
| 259 | |||
| 260 | let websocket_connection_duration = Histogram::with_opts( | ||
| 261 | HistogramOpts::new( | ||
| 262 | "ngit_websocket_connection_duration_seconds", | ||
| 263 | "Duration of WebSocket connections", | ||
| 264 | ) | ||
| 265 | .buckets(vec![1.0, 5.0, 15.0, 30.0, 60.0, 300.0, 900.0, 3600.0]), | ||
| 266 | ).unwrap(); | ||
| 267 | REGISTRY.register(Box::new(websocket_connection_duration.clone())).unwrap(); | ||
| 268 | |||
| 269 | let websocket_messages_received = CounterVec::new( | ||
| 270 | Opts::new( | ||
| 271 | "ngit_websocket_messages_received_total", | ||
| 272 | "WebSocket messages received by type", | ||
| 273 | ), | ||
| 274 | &["type"], | ||
| 275 | ).unwrap(); | ||
| 276 | REGISTRY.register(Box::new(websocket_messages_received.clone())).unwrap(); | ||
| 277 | |||
| 278 | let websocket_messages_sent = CounterVec::new( | ||
| 279 | Opts::new( | ||
| 280 | "ngit_websocket_messages_sent_total", | ||
| 281 | "WebSocket messages sent by type", | ||
| 282 | ), | ||
| 283 | &["type"], | ||
| 284 | ).unwrap(); | ||
| 285 | REGISTRY.register(Box::new(websocket_messages_sent.clone())).unwrap(); | ||
| 286 | |||
| 287 | // Git operation metrics | ||
| 288 | let git_operations_total = CounterVec::new( | ||
| 289 | Opts::new( | ||
| 290 | "ngit_git_operations_total", | ||
| 291 | "Git operations by type and status", | ||
| 292 | ), | ||
| 293 | &["operation", "status"], | ||
| 294 | ).unwrap(); | ||
| 295 | REGISTRY.register(Box::new(git_operations_total.clone())).unwrap(); | ||
| 296 | |||
| 297 | let git_operation_duration = HistogramVec::new( | ||
| 298 | HistogramOpts::new( | ||
| 299 | "ngit_git_operation_duration_seconds", | ||
| 300 | "Duration of git operations", | ||
| 301 | ) | ||
| 302 | .buckets(vec![0.1, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0]), | ||
| 303 | &["operation"], | ||
| 304 | ).unwrap(); | ||
| 305 | REGISTRY.register(Box::new(git_operation_duration.clone())).unwrap(); | ||
| 306 | |||
| 307 | let git_bytes_total = CounterVec::new( | ||
| 308 | Opts::new( | ||
| 309 | "ngit_git_bytes_total", | ||
| 310 | "Total bytes transferred for git operations", | ||
| 311 | ), | ||
| 312 | &["direction"], | ||
| 313 | ).unwrap(); | ||
| 314 | REGISTRY.register(Box::new(git_bytes_total.clone())).unwrap(); | ||
| 315 | |||
| 316 | let git_push_authorization = CounterVec::new( | ||
| 317 | Opts::new( | ||
| 318 | "ngit_git_push_authorization_total", | ||
| 319 | "Push authorization results", | ||
| 320 | ), | ||
| 321 | &["result"], | ||
| 322 | ).unwrap(); | ||
| 323 | REGISTRY.register(Box::new(git_push_authorization.clone())).unwrap(); | ||
| 324 | |||
| 325 | // Nostr event metrics | ||
| 326 | let events_received_total = CounterVec::new( | ||
| 327 | Opts::new( | ||
| 328 | "ngit_events_received_total", | ||
| 329 | "Nostr events received by kind", | ||
| 330 | ), | ||
| 331 | &["kind"], | ||
| 332 | ).unwrap(); | ||
| 333 | REGISTRY.register(Box::new(events_received_total.clone())).unwrap(); | ||
| 334 | |||
| 335 | let events_stored_total = CounterVec::new( | ||
| 336 | Opts::new( | ||
| 337 | "ngit_events_stored_total", | ||
| 338 | "Nostr events successfully stored by kind", | ||
| 339 | ), | ||
| 340 | &["kind"], | ||
| 341 | ).unwrap(); | ||
| 342 | REGISTRY.register(Box::new(events_stored_total.clone())).unwrap(); | ||
| 343 | |||
| 344 | let events_rejected_total = CounterVec::new( | ||
| 345 | Opts::new( | ||
| 346 | "ngit_events_rejected_total", | ||
| 347 | "Nostr events rejected by kind and reason", | ||
| 348 | ), | ||
| 349 | &["kind", "reason"], | ||
| 350 | ).unwrap(); | ||
| 351 | REGISTRY.register(Box::new(events_rejected_total.clone())).unwrap(); | ||
| 352 | |||
| 353 | // Repository metrics | ||
| 354 | let repositories_total = Gauge::with_opts( | ||
| 355 | Opts::new( | ||
| 356 | "ngit_repositories_total", | ||
| 357 | "Total repositories hosted", | ||
| 358 | ) | ||
| 359 | ).unwrap(); | ||
| 360 | REGISTRY.register(Box::new(repositories_total.clone())).unwrap(); | ||
| 361 | |||
| 362 | // Build info | ||
| 363 | let build_info = GaugeVec::new( | ||
| 364 | Opts::new( | ||
| 365 | "ngit_build_info", | ||
| 366 | "Build information", | ||
| 367 | ), | ||
| 368 | &["version", "commit"], | ||
| 369 | ).unwrap(); | ||
| 370 | REGISTRY.register(Box::new(build_info.clone())).unwrap(); | ||
| 371 | |||
| 372 | // Set build info gauge to 1 (it's just for labels) | ||
| 373 | build_info | ||
| 374 | .with_label_values(&[env!("CARGO_PKG_VERSION"), option_env!("GIT_HASH").unwrap_or("unknown")]) | ||
| 375 | .set(1.0); | ||
| 376 | |||
| 377 | Self { | ||
| 378 | connection_tracker, | ||
| 379 | bandwidth_tracker, | ||
| 380 | websocket_connections_total, | ||
| 381 | websocket_connection_duration, | ||
| 382 | websocket_messages_received, | ||
| 383 | websocket_messages_sent, | ||
| 384 | git_operations_total, | ||
| 385 | git_operation_duration, | ||
| 386 | git_bytes_total, | ||
| 387 | git_push_authorization, | ||
| 388 | events_received_total, | ||
| 389 | events_stored_total, | ||
| 390 | events_rejected_total, | ||
| 391 | repositories_total, | ||
| 392 | start_time: Instant::now(), | ||
| 393 | build_info, | ||
| 394 | } | ||
| 395 | } | ||
| 396 | } | ||
| 397 | |||
| 398 | /// Timer for tracking WebSocket connection duration. | ||
| 399 | /// Records the elapsed time when dropped. | ||
| 400 | pub struct HistogramTimer { | ||
| 401 | histogram: Histogram, | ||
| 402 | start: Instant, | ||
| 403 | } | ||
| 404 | |||
| 405 | impl HistogramTimer { | ||
| 406 | fn new(histogram: Histogram) -> Self { | ||
| 407 | Self { | ||
| 408 | histogram, | ||
| 409 | start: Instant::now(), | ||
| 410 | } | ||
| 411 | } | ||
| 412 | } | ||
| 413 | |||
| 414 | impl Drop for HistogramTimer { | ||
| 415 | fn drop(&mut self) { | ||
| 416 | let elapsed = self.start.elapsed().as_secs_f64(); | ||
| 417 | self.histogram.observe(elapsed); | ||
| 418 | } | ||
| 419 | } | ||
| 420 | |||
| 421 | /// Timer for tracking Git operation duration. | ||
| 422 | /// Records the elapsed time when dropped. | ||
| 423 | pub struct GitOperationTimer { | ||
| 424 | histogram_vec: HistogramVec, | ||
| 425 | operation: String, | ||
| 426 | start: Instant, | ||
| 427 | } | ||
| 428 | |||
| 429 | impl GitOperationTimer { | ||
| 430 | fn new(histogram_vec: HistogramVec, operation: String) -> Self { | ||
| 431 | Self { | ||
| 432 | histogram_vec, | ||
| 433 | operation, | ||
| 434 | start: Instant::now(), | ||
| 435 | } | ||
| 436 | } | ||
| 437 | } | ||
| 438 | |||
| 439 | impl Drop for GitOperationTimer { | ||
| 440 | fn drop(&mut self) { | ||
| 441 | let elapsed = self.start.elapsed().as_secs_f64(); | ||
| 442 | self.histogram_vec | ||
| 443 | .with_label_values(&[&self.operation]) | ||
| 444 | .observe(elapsed); | ||
| 445 | } | ||
| 446 | } | ||
| 447 | |||
| 448 | #[cfg(test)] | ||
| 449 | mod tests { | ||
| 450 | use super::*; | ||
| 451 | |||
| 452 | #[test] | ||
| 453 | fn test_metrics_creation() { | ||
| 454 | // Note: This test may fail if run with other tests due to global registry | ||
| 455 | // In production, consider using a test-specific registry | ||
| 456 | let metrics = Metrics::new(10); | ||
| 457 | |||
| 458 | // Test that we can record metrics without panicking | ||
| 459 | metrics.record_websocket_connection(); | ||
| 460 | metrics.record_message_received("REQ"); | ||
| 461 | metrics.record_message_sent("EVENT"); | ||
| 462 | metrics.record_git_operation("clone", "success"); | ||
| 463 | metrics.record_git_bytes("in", 1024); | ||
| 464 | metrics.record_event_received(1); | ||
| 465 | metrics.record_event_stored(1); | ||
| 466 | metrics.record_event_rejected(1, "invalid_signature"); | ||
| 467 | metrics.set_repositories_total(5); | ||
| 468 | } | ||
| 469 | } \ No newline at end of file | ||