From fd0c87c787d0626b3546fa571541c9c809711821 Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Thu, 4 Dec 2025 15:17:04 +0000 Subject: add prometheus metrics --- docs/explanation/monitoring-strategy.md | 462 ---------------------- docs/explanation/monitoring.md | 99 +++++ docs/grafana/ngit-grasp-dashboard.json | 675 ++++++++++++++++++++++++++++++++ docs/how-to/prometheus-setup.md | 178 +++++++++ 4 files changed, 952 insertions(+), 462 deletions(-) delete mode 100644 docs/explanation/monitoring-strategy.md create mode 100644 docs/explanation/monitoring.md create mode 100644 docs/grafana/ngit-grasp-dashboard.json create mode 100644 docs/how-to/prometheus-setup.md (limited to 'docs') diff --git a/docs/explanation/monitoring-strategy.md b/docs/explanation/monitoring-strategy.md deleted file mode 100644 index 4668305..0000000 --- a/docs/explanation/monitoring-strategy.md +++ /dev/null @@ -1,462 +0,0 @@ -# Monitoring Strategy - Design Document - -## Overview - -This document describes the logging and monitoring strategy for ngit-grasp, designed to help administrators: - -1. Monitor WebSocket connections per unique IP -2. Correlate resource spikes (memory, CPU) with usage patterns -3. Detect potential abuse (too many connections from single IP) -4. Support future load-based scheduling of background jobs (GRASP-02 sync) - -## Architecture - -```mermaid -flowchart TB - subgraph ngit-grasp - HTTP[HTTP Service] - WS[WebSocket Handler] - GIT[Git Handlers] - RELAY[Nostr Relay] - - subgraph Metrics Module - REG[Prometheus Registry] - CT[ConnectionTracker] - MC[Metric Counters] - end - - ME[/metrics endpoint] - end - - subgraph External - PROM[Prometheus Server] - GRAF[Grafana] - ADMIN[Admin Browser] - end - - HTTP --> ME - WS --> CT - WS --> MC - GIT --> MC - RELAY --> MC - - CT --> REG - MC --> REG - REG --> ME - - PROM -->|scrape /metrics| ME - GRAF -->|query| PROM - ADMIN -->|view dashboards| GRAF -``` - -## Metric Categories - -### 1. WebSocket Connection Metrics - -| Metric Name | Type | Labels | Description | -|------------|------|--------|-------------| -| `ngit_websocket_connections_total` | Counter | - | Total WebSocket connections since startup | -| `ngit_websocket_connections_active` | Gauge | - | Current active WebSocket connections | -| `ngit_websocket_unique_ips` | Gauge | - | Number of unique IP addresses connected (NOT the IPs themselves) | -| `ngit_websocket_flagged_abusers` | Gauge | - | Number of IPs exceeding connection threshold | -| `ngit_websocket_connection_duration_seconds` | Histogram | - | Duration of WebSocket connections | -| `ngit_websocket_messages_received_total` | Counter | `type` | Messages received (REQ, EVENT, CLOSE) | -| `ngit_websocket_messages_sent_total` | Counter | `type` | Messages sent (EVENT, EOSE, OK, NOTICE) | - -**Privacy Note:** IP addresses are NEVER exposed in metrics. The `ConnectionTracker` maintains per-IP counts internally only for abuse detection, logging warnings when thresholds are exceeded. - -### 2. Git Operation Metrics - -| Metric Name | Type | Labels | Description | -|------------|------|--------|-------------| -| `ngit_git_operations_total` | Counter | `operation`, `status` | Git operations (clone, fetch, push) | -| `ngit_git_operation_duration_seconds` | Histogram | `operation` | Duration of git operations | -| `ngit_git_bytes_total` | Counter | `direction` | Total bytes in/out for git operations | -| `ngit_git_push_authorization_total` | Counter | `result` | Push auth results (allowed, denied, error) | - -### 3. Top-N Repository Bandwidth Tracking - -To identify high-bandwidth repositories without creating cardinality explosion (which doesn't scale to 1000+ repos), we use a hybrid approach: - -| Metric Name | Type | Labels | Description | -|------------|------|--------|-------------| -| `ngit_git_top_repos_bytes` | Gauge | `repo` | Top 10 repositories by bandwidth (refreshed every 60s) | - -**How it works:** -- All per-repo bandwidth is tracked internally in a `HashMap` -- Every 60 seconds, the top 10 are calculated and exposed to Prometheus -- Previous repo labels are cleared before setting new ones -- Prometheus only ever sees ~10 label values, keeping cardinality low - -```rust -struct BandwidthTracker { - // Internal: tracks ALL repos (memory only, not exposed) - all_repos: DashMap, - - // Exposed to Prometheus: only top 10 - top_repos_gauge: GaugeVec, - - // Refresh interval - last_refresh: Instant, -} - -impl BandwidthTracker { - fn record_transfer(&self, repo_id: &str, bytes: u64) { - self.all_repos - .entry(repo_id.to_string()) - .and_modify(|v| *v += bytes) - .or_insert(bytes); - } - - fn maybe_refresh_top_n(&self) { - if self.last_refresh.elapsed() > Duration::from_secs(60) { - self.refresh_top_n(); - } - } - - fn refresh_top_n(&self) { - let mut sorted: Vec<_> = self.all_repos.iter() - .map(|r| (r.key().clone(), *r.value())) - .collect(); - sorted.sort_by(|a, b| b.1.cmp(&a.1)); - - // Clear old labels, set new top 10 - self.top_repos_gauge.reset(); - for (repo, bytes) in sorted.into_iter().take(10) { - self.top_repos_gauge - .with_label_values(&[&repo]) - .set(bytes as i64); - } - } -} -``` - -### 4. Nostr Event Metrics - -| Metric Name | Type | Labels | Description | -|------------|------|--------|-------------| -| `ngit_events_received_total` | Counter | `kind` | Events received by kind | -| `ngit_events_stored_total` | Counter | `kind` | Events successfully stored | -| `ngit_events_rejected_total` | Counter | `kind`, `reason` | Events rejected and why | - -### 5. Repository Metrics - -| Metric Name | Type | Labels | Description | -|------------|------|--------|-------------| -| `ngit_repositories_total` | Gauge | - | Total repositories hosted | - -### 6. System Health Metrics - -| Metric Name | Type | Labels | Description | -|------------|------|--------|-------------| -| `ngit_uptime_seconds` | Counter | - | Seconds since startup | -| `ngit_build_info` | Gauge | `version`, `commit` | Build information | - -### 7. Future: Sync Metrics (GRASP-02) - -| Metric Name | Type | Labels | Description | -|------------|------|--------|-------------| -| `ngit_sync_events_received_total` | Counter | `source` | Events from sync (live vs catchup) | -| `ngit_sync_relay_connections_active` | Gauge | - | Active outbound relay connections | -| `ngit_sync_catchup_gap_total` | Counter | - | Events found during catchup (sync failures) | - -## Connection Tracker Design - -The `ConnectionTracker` maintains per-IP connection counts internally for abuse detection. **IP addresses are never exposed in metrics** - only aggregate counts. - -```mermaid -flowchart LR - subgraph ConnectionTracker - HM[Internal: HashMap IP to Count] - TH[Abuse Threshold] - CNT[Exposed: Unique IP Count] - FLAG[Exposed: Abuse Flag Count] - end - - CONN[New Connection] --> CHECK{Count >= Threshold?} - CHECK -->|No| INC[Increment Count] - CHECK -->|Yes| FLAG_IT[Flag as Abuse] - FLAG_IT --> LOG[Log Warning - IP in log only] - FLAG_IT --> FLAG - - DISC[Disconnection] --> DEC[Decrement Count] - DEC --> CLEAN{Count == 0?} - CLEAN -->|Yes| RM[Remove from Map] - - HM --> CNT -``` - -### Data Structure - -```rust -pub struct ConnectionTracker { - /// Active connections per IP (INTERNAL ONLY - never exposed to metrics) - connections: DashMap, - /// Threshold for abuse flagging - abuse_threshold: u32, - /// Prometheus gauges (aggregate counts only, no IPs) - active_connections: IntGauge, // Total connections - unique_ips: IntGauge, // len() of HashMap - flagged_abusers: IntGauge, // Count where flagged_as_abuse == true -} - -struct ConnectionInfo { - count: u32, - first_seen: Instant, - flagged_as_abuse: bool, -} -``` - -### What Gets Exposed vs Internal - -| Data | Location | Exposed? | -|------|----------|----------| -| Total connections | Prometheus | ✅ Yes | -| Unique IP count | Prometheus | ✅ Yes | -| Flagged abuser count | Prometheus | ✅ Yes | -| Actual IP addresses | Internal HashMap | ❌ No | -| IP + abuse flag | Logs (when flagged) | ⚠️ Logs only | - -### Thread Safety - -Using `DashMap` for lock-free concurrent access, as connection tracking happens across multiple tokio tasks. - -## /metrics Endpoint - -The `/metrics` endpoint returns Prometheus text format: - -``` -# HELP ngit_websocket_connections_active Current active WebSocket connections -# TYPE ngit_websocket_connections_active gauge -ngit_websocket_connections_active 23 - -# HELP ngit_websocket_connections_by_ip Active connections per IP -# TYPE ngit_websocket_connections_by_ip gauge -ngit_websocket_connections_by_ip{ip="192.168.1.100"} 2 -ngit_websocket_connections_by_ip{ip="10.0.0.50"} 5 - -# HELP ngit_git_operations_total Git operations by type and status -# TYPE ngit_git_operations_total counter -ngit_git_operations_total{operation="clone",status="success"} 1247 -ngit_git_operations_total{operation="push",status="denied"} 12 -``` - -## Integration Points - -### HTTP Service Integration - -In [`src/http/mod.rs`](../../src/http/mod.rs): - -```rust -// Add to HttpService -struct HttpService { - // ... existing fields ... - metrics: Arc, -} - -// Add /metrics route handling -if path == "/metrics" { - let metrics_output = self.metrics.render(); - return Ok(Response::builder() - .status(200) - .header("content-type", "text/plain; version=0.0.4") - .body(Full::new(Bytes::from(metrics_output))) - .unwrap()); -} -``` - -### WebSocket Connection Tracking - -In the WebSocket upgrade handler: - -```rust -// On connection -let ip = addr.ip(); -metrics.connection_tracker.on_connect(ip); - -// Spawn connection handler -tokio::spawn(async move { - // ... handle connection ... - // On disconnect - metrics.connection_tracker.on_disconnect(ip); -}); -``` - -### Git Handler Integration - -In [`src/git/handlers.rs`](../../src/git/handlers.rs): - -```rust -// Wrap git operations with metrics -let timer = metrics.git_operation_duration.start_timer(); -let result = git::handlers::handle_upload_pack(repo_path, body_bytes).await; -timer.observe_duration(); - -metrics.git_operations_total - .with_label_values(&["clone", result_status]) - .inc(); -``` - -## Configuration - -New configuration options in [`src/config.rs`](../../src/config.rs): - -| Option | CLI Flag | Environment Variable | Default | Description | -|--------|----------|---------------------|---------|-------------| -| Metrics enabled | `--metrics-enabled` | `NGIT_METRICS_ENABLED` | `true` | Enable /metrics endpoint | -| Abuse threshold | `--abuse-threshold` | `NGIT_ABUSE_THRESHOLD` | `10` | Max connections per IP before flagging | -| Metrics path | `--metrics-path` | `NGIT_METRICS_PATH` | `/metrics` | Path for metrics endpoint | - -## Crate Dependencies - -Add to `Cargo.toml`: - -```toml -# Metrics -prometheus = "0.13" -dashmap = "5" # Lock-free concurrent HashMap -lazy_static = "1.4" # For static metric registration -``` - -## Module Structure - -``` -src/ -├── metrics/ -│ ├── mod.rs # Module exports, Metrics struct -│ ├── connection.rs # ConnectionTracker implementation -│ ├── definitions.rs # Metric definitions (lazy_static!) -│ └── render.rs # Prometheus format rendering -├── http/ -│ └── mod.rs # Add /metrics route -└── ... -``` - -## Grafana Dashboard - -A pre-built Grafana dashboard will be provided at `docs/grafana/ngit-grasp-dashboard.json` with panels for: - -1. **Overview Row** - - Active connections (gauge) - - Requests per second (graph) - - Git operations per minute (graph) - -2. **Connections Row** - - Active connections over time - - Connections by IP (top 10) - - Flagged abuse IPs (table) - -3. **Git Operations Row** - - Clone/fetch/push rates - - Push authorization results (pie chart) - - Operation duration percentiles - -4. **Events Row** - - Events received by kind - - Events rejected by reason - - Active subscriptions - -## Deployment: Prometheus on NixOS - -Example NixOS configuration for Prometheus: - -```nix -services.prometheus = { - enable = true; - scrapeConfigs = [ - { - job_name = "ngit-grasp"; - static_configs = [{ - targets = [ "localhost:8080" ]; # ngit-grasp bind address - }]; - scrape_interval = "15s"; - metrics_path = "/metrics"; - } - ]; -}; - -services.grafana = { - enable = true; - settings.server.http_port = 3000; - provision.datasources.settings.datasources = [{ - name = "Prometheus"; - type = "prometheus"; - url = "http://localhost:9090"; - }]; -}; -``` - -## Future: Load-Based Sync Scheduling - -The metrics infrastructure enables future load-based scheduling for GRASP-02 sync jobs: - -```mermaid -flowchart TD - SYNC[Sync Manager] --> CHECK{Check Load} - CHECK --> MET[Query Metrics] - MET --> CPU{CPU > 80%?} - CPU -->|Yes| DELAY[Delay 5 min] - CPU -->|No| CONN{Connections > N?} - CONN -->|Yes| DELAY - CONN -->|No| RUN[Run Sync Job] - DELAY --> CHECK -``` - -The `Metrics` struct will expose a method for checking load: - -```rust -impl Metrics { - /// Check if system is under high load - pub fn is_high_load(&self) -> bool { - let active = self.websocket_connections_active.get(); - active > self.config.high_load_threshold - } -} -``` - -## Future Enhancement: Loki for Detailed Logging - -For detailed per-repository investigation at scale, consider adding **Loki** (log aggregation) in a future iteration: - -```rust -// Structured logging with tracing -tracing::info!( - repo = %repo_id, - npub = %npub, - bytes = bytes_transferred, - operation = "clone", - duration_ms = elapsed.as_millis(), - "git_transfer_complete" -); -``` - -Loki query examples: -```logql -# Find all transfers > 10MB -{job="ngit-grasp"} |= "git_transfer_complete" | json | bytes > 10000000 - -# Sum bytes by repo in last hour -sum by (repo) ( - {job="ngit-grasp"} |= "git_transfer_complete" | json | unwrap bytes -) -``` - -This pairs with Prometheus for long-term trends while enabling ad-hoc deep dives. - -## Privacy Considerations - -- IP addresses are stored only in memory (not logged to disk by default) -- Per-IP metrics can be disabled via configuration -- Consider IP anonymization for GDPR compliance if needed - -## Summary - -| Component | Purpose | -|-----------|---------| -| `Metrics` struct | Central registry and access point | -| `ConnectionTracker` | Per-IP tracking with abuse detection | -| `/metrics` endpoint | Prometheus scraping interface | -| Grafana dashboard | Visualization and analysis | -| NixOS config | Easy deployment for operators | - -This strategy provides comprehensive observability without requiring a separate database - Prometheus handles all time-series storage and Grafana provides the visualization layer. \ No newline at end of file diff --git a/docs/explanation/monitoring.md b/docs/explanation/monitoring.md new file mode 100644 index 0000000..3b1b1ac --- /dev/null +++ b/docs/explanation/monitoring.md @@ -0,0 +1,99 @@ +# Monitoring + +ngit-grasp exposes Prometheus metrics at `/metrics` for monitoring WebSocket connections, Git operations, Nostr events, and system health. + +## Architecture + +```mermaid +flowchart TB + subgraph ngit-grasp + HTTP[HTTP Service] + WS[WebSocket Handler] + GIT[Git Handlers] + RELAY[Nostr Relay] + + subgraph Metrics Module + REG[Prometheus Registry] + CT[ConnectionTracker] + MC[Metric Counters] + end + + ME[/metrics endpoint] + end + + subgraph External + PROM[Prometheus Server] + GRAF[Grafana] + ADMIN[Admin Browser] + end + + HTTP --> ME + WS --> CT + WS --> MC + GIT --> MC + RELAY --> MC + + CT --> REG + MC --> REG + REG --> ME + + PROM -->|scrape /metrics| ME + GRAF -->|query| PROM + ADMIN -->|view dashboards| GRAF +``` + +## Configuration + +| Option | CLI Flag | Environment Variable | Default | Description | +|--------|----------|---------------------|---------|-------------| +| Metrics enabled | `--metrics-enabled` | `NGIT_METRICS_ENABLED` | `true` | Enable /metrics endpoint | +| Abuse threshold | `--abuse-threshold` | `NGIT_ABUSE_THRESHOLD` | `10` | Max connections per IP before flagging | +| Top N repos | `--top-n-repos` | `NGIT_TOP_N_REPOS` | `10` | Number of top bandwidth repos to track | + +## Privacy Model + +IP addresses are **never exposed in Prometheus metrics**. The connection tracker maintains per-IP counts internally only for abuse detection: + +| Data | Exposed in Metrics? | +|------|---------------------| +| Total connections | ✅ Yes | +| Unique IP count | ✅ Yes | +| Flagged abuser count | ✅ Yes | +| Actual IP addresses | ❌ No (internal only) | +| IP + abuse flag | ⚠️ Logs only (when flagged) | + +When an IP exceeds the abuse threshold, a warning is logged but the IP is never exposed via Prometheus. + +## Deployment + +See [Prometheus Setup Guide](../how-to/prometheus-setup.md) for NixOS configuration and Grafana dashboard provisioning. + +## Future: Load-Based Sync Scheduling (GRASP-02) + +The metrics infrastructure enables future load-based scheduling for GRASP-02 sync jobs: + +```mermaid +flowchart TD + SYNC[Sync Manager] --> CHECK{Check Load} + CHECK --> MET[Query Metrics] + MET --> CONN{Connections > N?} + CONN -->|Yes| DELAY[Delay 5 min] + CONN -->|No| RUN[Run Sync Job] + DELAY --> CHECK +``` + +## Future: Loki for Detailed Logging + +For detailed per-repository investigation at scale, consider adding **Loki** (log aggregation): + +- Structured logging with tracing crate already in place +- Loki queries enable ad-hoc deep dives (e.g., find all transfers > 10MB) +- Pairs with Prometheus for long-term trends + +## Future: Sync Metrics (GRASP-02) + +When GRASP-02 proactive sync is implemented, additional metrics will track: + +- Events received from sync (live vs catchup) +- Active outbound relay connections +- Catchup gap (events found during catchup indicating sync failures) \ No newline at end of file diff --git a/docs/grafana/ngit-grasp-dashboard.json b/docs/grafana/ngit-grasp-dashboard.json new file mode 100644 index 0000000..bd1b6fe --- /dev/null +++ b/docs/grafana/ngit-grasp-dashboard.json @@ -0,0 +1,675 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "title": "Overview", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 50 }, + { "color": "red", "value": 100 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "ngit_websocket_connections_active", + "legendFormat": "Active", + "refId": "A" + } + ], + "title": "Active Connections", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "ngit_websocket_unique_ips", + "legendFormat": "Unique IPs", + "refId": "A" + } + ], + "title": "Unique IPs", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "ngit_websocket_flagged_abusers", + "legendFormat": "Flagged", + "refId": "A" + } + ], + "title": "Flagged Abusers", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "blue", "value": null }] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "ngit_repositories_total", + "legendFormat": "Repos", + "refId": "A" + } + ], + "title": "Total Repositories", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "unit": "s" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "ngit_uptime_seconds", + "legendFormat": "Uptime", + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "purple", "value": null }] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name" + }, + "targets": [ + { + "expr": "ngit_build_info", + "legendFormat": "{{version}}", + "refId": "A" + } + ], + "title": "Version", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "id": 10, + "title": "WebSocket Connections", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [] }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "id": 11, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "expr": "ngit_websocket_connections_active", + "legendFormat": "Active Connections", + "refId": "A" + }, + { + "expr": "ngit_websocket_unique_ips", + "legendFormat": "Unique IPs", + "refId": "B" + } + ], + "title": "Connections Over Time", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [] }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "id": 12, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "expr": "rate(ngit_websocket_messages_received_total[5m])", + "legendFormat": "Received: {{type}}", + "refId": "A" + }, + { + "expr": "rate(ngit_websocket_messages_sent_total[5m])", + "legendFormat": "Sent: {{type}}", + "refId": "B" + } + ], + "title": "Message Rate (5m)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, + "mappings": [], + "unit": "s" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, + "id": 13, + "options": { + "legend": { "displayMode": "list", "placement": "right", "showLegend": true }, + "pieType": "pie", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, rate(ngit_websocket_connection_duration_seconds_bucket[1h]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(ngit_websocket_connection_duration_seconds_bucket[1h]))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, rate(ngit_websocket_connection_duration_seconds_bucket[1h]))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Connection Duration Percentiles", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, + "id": 20, + "title": "Git Operations", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [] }, + "unit": "ops" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, + "id": 21, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "expr": "rate(ngit_git_operations_total{status=\"success\"}[5m])", + "legendFormat": "{{operation}} (success)", + "refId": "A" + }, + { + "expr": "rate(ngit_git_operations_total{status=\"error\"}[5m])", + "legendFormat": "{{operation}} (error)", + "refId": "B" + } + ], + "title": "Git Operations Rate (5m)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [] }, + "unit": "bytes" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 }, + "id": 22, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "expr": "rate(ngit_git_bytes_total[5m])", + "legendFormat": "{{direction}}", + "refId": "A" + } + ], + "title": "Git Bandwidth (5m)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, + "mappings": [], + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "denied" }, + "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "allowed" }, + "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + } + ] + }, + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 31 }, + "id": 23, + "options": { + "legend": { "displayMode": "list", "placement": "right", "showLegend": true }, + "pieType": "pie", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "expr": "increase(ngit_git_push_authorization_total[24h])", + "legendFormat": "{{result}}", + "refId": "A" + } + ], + "title": "Push Authorization (24h)", + "type": "piechart" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "bytes" + } + }, + "gridPos": { "h": 8, "w": 18, "x": 6, "y": 31 }, + "id": 24, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showUnfilled": true, + "valueMode": "color" + }, + "targets": [ + { + "expr": "topk(10, ngit_git_top_repos_bytes)", + "legendFormat": "{{repo}}", + "refId": "A" + } + ], + "title": "Top Repositories by Bandwidth", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 }, + "id": 30, + "title": "Nostr Events", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [] }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, + "id": 31, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "expr": "rate(ngit_events_received_total[5m])", + "legendFormat": "Kind {{kind}}", + "refId": "A" + } + ], + "title": "Events Received by Kind (5m)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [] }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, + "id": 32, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "expr": "rate(ngit_events_stored_total[5m])", + "legendFormat": "Stored: Kind {{kind}}", + "refId": "A" + }, + { + "expr": "rate(ngit_events_rejected_total[5m])", + "legendFormat": "Rejected: {{reason}}", + "refId": "B" + } + ], + "title": "Events Stored vs Rejected (5m)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": ["ngit-grasp", "nostr", "git"], + "templating": { + "list": [ + { + "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "ngit-grasp", + "uid": "ngit-grasp", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/docs/how-to/prometheus-setup.md b/docs/how-to/prometheus-setup.md new file mode 100644 index 0000000..741255b --- /dev/null +++ b/docs/how-to/prometheus-setup.md @@ -0,0 +1,178 @@ +# Prometheus and Grafana Setup + +This guide shows how to configure Prometheus and Grafana to monitor ngit-grasp. + +## Prerequisites + +- ngit-grasp running with metrics enabled (default: `--metrics-enabled true`) +- Prometheus server +- Grafana (optional, for dashboards) + +## Verify Metrics Endpoint + +First, verify that ngit-grasp is exposing metrics: + +```bash +curl http://localhost:8080/metrics +``` + +You should see Prometheus-formatted metrics like: + +``` +# HELP ngit_websocket_connections_active Current active WebSocket connections +# TYPE ngit_websocket_connections_active gauge +ngit_websocket_connections_active 5 + +# HELP ngit_git_operations_total Git operations by type and status +# TYPE ngit_git_operations_total counter +ngit_git_operations_total{operation="clone",status="success"} 42 +``` + +## NixOS Configuration + +### Prometheus + +Add ngit-grasp as a scrape target: + +```nix +services.prometheus = { + enable = true; + scrapeConfigs = [ + { + job_name = "ngit-grasp"; + static_configs = [{ + targets = [ "localhost:8080" ]; # ngit-grasp bind address + }]; + scrape_interval = "15s"; + metrics_path = "/metrics"; + } + ]; +}; +``` + +### Grafana with Prometheus Datasource + +```nix +services.grafana = { + enable = true; + settings.server.http_port = 3000; + + provision.datasources.settings.datasources = [{ + name = "Prometheus"; + type = "prometheus"; + url = "http://localhost:9090"; + isDefault = true; + }]; + + # Optional: provision the ngit-grasp dashboard + provision.dashboards.settings.providers = [{ + name = "ngit-grasp"; + options.path = "/path/to/ngit-grasp/docs/grafana"; + }]; +}; +``` + +## Docker Compose Configuration + +For non-NixOS deployments: + +```yaml +version: '3.8' +services: + prometheus: + image: prom/prometheus:latest + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + + grafana: + image: grafana/grafana:latest + ports: + - "3000:3000" + volumes: + - ./docs/grafana:/var/lib/grafana/dashboards + environment: + - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/ngit-grasp-dashboard.json +``` + +With `prometheus.yml`: + +```yaml +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'ngit-grasp' + static_configs: + - targets: ['host.docker.internal:8080'] # or your ngit-grasp host + metrics_path: /metrics +``` + +## Import Dashboard + +1. Open Grafana at `http://localhost:3000` +2. Go to **Dashboards** → **Import** +3. Upload `docs/grafana/ngit-grasp-dashboard.json` +4. Select your Prometheus datasource +5. Click **Import** + +## Key Metrics to Monitor + +### Connection Health +- `ngit_websocket_connections_active` - Current active connections +- `ngit_websocket_unique_ips` - Number of unique client IPs +- `ngit_websocket_flagged_abusers` - IPs exceeding connection threshold + +### Git Operations +- `ngit_git_operations_total` - Operations by type (clone/fetch/push) and status +- `ngit_git_bytes_total` - Bandwidth by direction (in/out) +- `ngit_git_top_repos_bytes` - Top N repositories by bandwidth + +### Nostr Events +- `ngit_events_received_total` - Events received by kind +- `ngit_events_stored_total` - Events successfully stored +- `ngit_events_rejected_total` - Events rejected by reason + +### System +- `ngit_uptime_seconds` - Server uptime +- `ngit_build_info` - Version and commit info +- `ngit_repositories_total` - Total hosted repositories + +## Example Alerts + +Add to your Prometheus alerting rules: + +```yaml +groups: + - name: ngit-grasp + rules: + - alert: HighConnectionCount + expr: ngit_websocket_connections_active > 100 + for: 5m + labels: + severity: warning + annotations: + summary: "High number of WebSocket connections" + + - alert: AbusiveIPs + expr: ngit_websocket_flagged_abusers > 0 + for: 1m + labels: + severity: warning + annotations: + summary: "{{ $value }} IPs flagged for excessive connections" + + - alert: PushAuthorizationFailures + expr: rate(ngit_git_operations_total{operation="push",status="denied"}[5m]) > 0.1 + for: 5m + labels: + severity: info + annotations: + summary: "Elevated push authorization failures" +``` + +## See Also + +- [Monitoring Overview](../explanation/monitoring.md) - Architecture and design +- [Configuration Reference](../reference/configuration.md) - All config options \ No newline at end of file -- cgit v1.2.3