diff options
| -rw-r--r-- | .env.example | 3 | ||||
| -rw-r--r-- | README.md | 1 | ||||
| -rw-r--r-- | docs/explanation/architecture.md | 2 | ||||
| -rw-r--r-- | docs/explanation/comparison.md | 10 | ||||
| -rw-r--r-- | docs/explanation/deletion-requests.md | 2 | ||||
| -rw-r--r-- | docs/how-to/deploy.md | 4 | ||||
| -rw-r--r-- | docs/learnings/grasp-01-implementation.md | 4 | ||||
| -rw-r--r-- | docs/reference/configuration.md | 28 | ||||
| -rw-r--r-- | nix/module.nix | 3 | ||||
| -rw-r--r-- | src/config.rs | 4 | ||||
| -rw-r--r-- | src/nostr/builder.rs | 10 |
11 files changed, 24 insertions, 47 deletions
diff --git a/.env.example b/.env.example index 01854f4..43f8f6f 100644 --- a/.env.example +++ b/.env.example | |||
| @@ -67,11 +67,10 @@ | |||
| 67 | 67 | ||
| 68 | # Database backend for Nostr events | 68 | # Database backend for Nostr events |
| 69 | # CLI: --database-backend <backend> | 69 | # CLI: --database-backend <backend> |
| 70 | # Options: lmdb, memory, nostrdb | 70 | # Options: lmdb, memory |
| 71 | # Default: lmdb | 71 | # Default: lmdb |
| 72 | # - lmdb: LMDB backend (persistent, general purpose) - RECOMMENDED | 72 | # - lmdb: LMDB backend (persistent, general purpose) - RECOMMENDED |
| 73 | # - memory: In-memory database (fastest, no persistence, uses temp dirs) | 73 | # - memory: In-memory database (fastest, no persistence, uses temp dirs) |
| 74 | # - nostrdb: NostrDB backend (persistent, Nostr-optimized) [Not yet implemented] | ||
| 75 | # | 74 | # |
| 76 | # Note: When using 'memory' backend, git_data_path and relay_data_path | 75 | # Note: When using 'memory' backend, git_data_path and relay_data_path |
| 77 | # are automatically set to temporary directories for ephemeral testing. | 76 | # are automatically set to temporary directories for ephemeral testing. |
| @@ -477,7 +477,6 @@ NGIT_RELAY_OWNER_NSEC=nsec1... ngit-grasp --domain relay.example.com | |||
| 477 | 477 | ||
| 478 | - `lmdb`: LMDB backend (default, persistent, general purpose) | 478 | - `lmdb`: LMDB backend (default, persistent, general purpose) |
| 479 | - `memory`: In-memory database (fastest, no persistence - uses temp directories) | 479 | - `memory`: In-memory database (fastest, no persistence - uses temp directories) |
| 480 | - `nostrdb`: NostrDB backend (persistent, optimized for Nostr) [Not yet implemented] | ||
| 481 | 480 | ||
| 482 | > **Note:** When using the `memory` backend, git data are automatically stored in temporary directories for ephemeral testing. | 481 | > **Note:** When using the `memory` backend, git data are automatically stored in temporary directories for ephemeral testing. |
| 483 | 482 | ||
diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md index e101425..e0a57e5 100644 --- a/docs/explanation/architecture.md +++ b/docs/explanation/architecture.md | |||
| @@ -101,7 +101,7 @@ After examining both the reference implementation and HTTP server options, we ha | |||
| 101 | - Initialize configuration from environment (clap + dotenvy) | 101 | - Initialize configuration from environment (clap + dotenvy) |
| 102 | - Set up Hyper HTTP server with request routing | 102 | - Set up Hyper HTTP server with request routing |
| 103 | - Initialize Nostr relay builder with custom [`Nip34WritePolicy`](src/nostr/builder.rs:51) | 103 | - Initialize Nostr relay builder with custom [`Nip34WritePolicy`](src/nostr/builder.rs:51) |
| 104 | - Set up shared storage (LMDB, NostrDB, or Memory) | 104 | - Set up shared storage (LMDB or Memory) |
| 105 | - Handle WebSocket upgrades for Nostr relay | 105 | - Handle WebSocket upgrades for Nostr relay |
| 106 | - Handle graceful shutdown | 106 | - Handle graceful shutdown |
| 107 | 107 | ||
diff --git a/docs/explanation/comparison.md b/docs/explanation/comparison.md index 090e0eb..315f091 100644 --- a/docs/explanation/comparison.md +++ b/docs/explanation/comparison.md | |||
| @@ -11,7 +11,7 @@ This document compares ngit-grasp (this project) with ngit-relay (the reference | |||
| 11 | | **Git Protocol** | git-http-backend (C via fcgiwrap) | HTTP layer in Rust + git subprocess | | 11 | | **Git Protocol** | git-http-backend (C via fcgiwrap) | HTTP layer in Rust + git subprocess | |
| 12 | | **Authorization** | Pre-receive Git hook | Inline HTTP handler validation | | 12 | | **Authorization** | Pre-receive Git hook | Inline HTTP handler validation | |
| 13 | | **Nostr Relay** | Khatru (Go library) | nostr-relay-builder (Rust library) | | 13 | | **Nostr Relay** | Khatru (Go library) | nostr-relay-builder (Rust library) | |
| 14 | | **Event Store** | Badger (Go KV database) | LMDB or NostrDB (Rust) | | 14 | | **Event Store** | Badger (Go KV database) | LMDB (Rust) | |
| 15 | | **Proactive Sync** | Git-only (polls DB + fetches from git servers) | Nostr event sync + git sync (event-driven) | | 15 | | **Proactive Sync** | Git-only (polls DB + fetches from git servers) | Nostr event sync + git sync (event-driven) | |
| 16 | | **Process Management** | supervisord (4 processes) | Single tokio runtime | | 16 | | **Process Management** | supervisord (4 processes) | Single tokio runtime | |
| 17 | | **Packaging** | Docker with supervisord | Single static binary or Docker | | 17 | | **Packaging** | Docker with supervisord | Single static binary or Docker | |
| @@ -78,7 +78,7 @@ This document compares ngit-grasp (this project) with ngit-relay (the reference | |||
| 78 | │ │ │ │ builder library) │ │ | 78 | │ │ │ │ builder library) │ │ |
| 79 | │ │ - info/refs │ │ - NIP-34 Policy │ │ | 79 | │ │ - info/refs │ │ - NIP-34 Policy │ │ |
| 80 | │ │ - upload-pk │◀─────┤ (inline query) │ │ | 80 | │ │ - upload-pk │◀─────┤ (inline query) │ │ |
| 81 | │ │ - receive-pk │ auth │ - LMDB/NostrDB │ │ | 81 | │ │ - receive-pk │ auth │ - LMDB/Memory │ │ |
| 82 | │ │ + inline │ check│ - WebSocket │ │ | 82 | │ │ + inline │ check│ - WebSocket │ │ |
| 83 | │ │ validation │ │ - NIP-11 endpoint │ │ | 83 | │ │ validation │ │ - NIP-11 endpoint │ │ |
| 84 | │ └──────┬───────┘ └──────────┬─────────┘ │ | 84 | │ └──────┬───────┘ └──────────┬─────────┘ │ |
| @@ -100,7 +100,7 @@ This document compares ngit-grasp (this project) with ngit-relay (the reference | |||
| 100 | │ │ | 100 | │ │ |
| 101 | │ ┌──────────────────────────────────────────────────┐ │ | 101 | │ ┌──────────────────────────────────────────────────┐ │ |
| 102 | │ │ Shared State (Arc<T>) │ │ | 102 | │ │ Shared State (Arc<T>) │ │ |
| 103 | │ │ - Database (LMDB/NostrDB/Memory) │ │ | 103 | │ │ - Database (LMDB/Memory) │ │ |
| 104 | │ │ - Purgatory (DashMap - concurrent queue) │ │ | 104 | │ │ - Purgatory (DashMap - concurrent queue) │ │ |
| 105 | │ │ - Metrics (Prometheus) │ │ | 105 | │ │ - Metrics (Prometheus) │ │ |
| 106 | │ └──────────────────────────────────────────────────┘ │ | 106 | │ └──────────────────────────────────────────────────┘ │ |
| @@ -161,7 +161,7 @@ This is why ngit-grasp has ~13x more code - the majority is implementing GRASP-0 | |||
| 161 | | Feature | ngit-relay | ngit-grasp | | 161 | | Feature | ngit-relay | ngit-grasp | |
| 162 | |---------|-----------|-----------| | 162 | |---------|-----------|-----------| |
| 163 | | **Implementation** | Khatru (Go library) | nostr-relay-builder (Rust library) | | 163 | | **Implementation** | Khatru (Go library) | nostr-relay-builder (Rust library) | |
| 164 | | **Database** | Badger (Go KV store) | LMDB or NostrDB (Rust) | | 164 | | **Database** | Badger (Go KV store) | LMDB (Rust) | |
| 165 | | **Process** | Separate process on :3334 | Integrated (same binary) | | 165 | | **Process** | Separate process on :3334 | Integrated (same binary) | |
| 166 | | **Policies** | Go functions in `policies.go` | Rust traits (modular sub-policies) | | 166 | | **Policies** | Go functions in `policies.go` | Rust traits (modular sub-policies) | |
| 167 | | **Event Validation** | Single function with branches | 4 separate policy modules | | 167 | | **Event Validation** | Single function with branches | 4 separate policy modules | |
| @@ -287,7 +287,7 @@ For users of ngit-relay, migration to ngit-grasp involves: | |||
| 287 | 287 | ||
| 288 | ### Data Migration | 288 | ### Data Migration |
| 289 | 289 | ||
| 290 | 1. **Events**: Export from Badger → Import to LMDB/NostrDB | 290 | 1. **Events**: Export from Badger → Import to LMDB |
| 291 | - No direct migration tool yet (would need to be built) | 291 | - No direct migration tool yet (would need to be built) |
| 292 | - Alternative: Use proactive sync to re-fetch from other relays | 292 | - Alternative: Use proactive sync to re-fetch from other relays |
| 293 | 2. **Git Repositories**: Direct copy (same structure) | 293 | 2. **Git Repositories**: Direct copy (same structure) |
diff --git a/docs/explanation/deletion-requests.md b/docs/explanation/deletion-requests.md index 7660774..4b4cd88 100644 --- a/docs/explanation/deletion-requests.md +++ b/docs/explanation/deletion-requests.md | |||
| @@ -28,7 +28,7 @@ The deletion system uses three separate data stores: | |||
| 28 | ┌─────────────────────────────────────────────────────────┐ | 28 | ┌─────────────────────────────────────────────────────────┐ |
| 29 | │ Main Database │ | 29 | │ Main Database │ |
| 30 | │ (Live events - actively served) │ | 30 | │ (Live events - actively served) │ |
| 31 | │ LMDB/NostrDB/Memory backend │ | 31 | │ LMDB/Memory backend │ |
| 32 | └─────────────────────────────────────────────────────────┘ | 32 | └─────────────────────────────────────────────────────────┘ |
| 33 | ↓ deletion request | 33 | ↓ deletion request |
| 34 | ┌─────────────────────────────────────────────────────────┐ | 34 | ┌─────────────────────────────────────────────────────────┐ |
diff --git a/docs/how-to/deploy.md b/docs/how-to/deploy.md index 157e5f8..9117fe2 100644 --- a/docs/how-to/deploy.md +++ b/docs/how-to/deploy.md | |||
| @@ -233,7 +233,7 @@ git ls-remote https://ngit.example.com/<npub>/<repo>.git | |||
| 233 | 233 | ||
| 234 | ### Storage | 234 | ### Storage |
| 235 | - `dataDir` - Base directory for data (default: /var/lib/ngit-grasp-{name}) | 235 | - `dataDir` - Base directory for data (default: /var/lib/ngit-grasp-{name}) |
| 236 | - `databaseBackend` - "lmdb" | "nostr-db" | "memory" (default: "lmdb") | 236 | - `databaseBackend` - "lmdb" | "memory" (default: "lmdb") |
| 237 | 237 | ||
| 238 | ### Identity | 238 | ### Identity |
| 239 | - `relayName` - Relay name for NIP-11 (default: "{domain} grasp relay") | 239 | - `relayName` - Relay name for NIP-11 (default: "{domain} grasp relay") |
| @@ -363,7 +363,7 @@ curl http://localhost:8082/metrics | |||
| 363 | **Tune configuration:** | 363 | **Tune configuration:** |
| 364 | - Reduce `metricsTopNRepos` | 364 | - Reduce `metricsTopNRepos` |
| 365 | - Increase `syncMaxBackoffSecs` | 365 | - Increase `syncMaxBackoffSecs` |
| 366 | - Switch to `databaseBackend = "nostr-db"` for better performance | 366 | - Tune `syncMaxBackoffSecs` for your network conditions |
| 367 | 367 | ||
| 368 | --- | 368 | --- |
| 369 | 369 | ||
diff --git a/docs/learnings/grasp-01-implementation.md b/docs/learnings/grasp-01-implementation.md index 27124af..f893d78 100644 --- a/docs/learnings/grasp-01-implementation.md +++ b/docs/learnings/grasp-01-implementation.md | |||
| @@ -20,7 +20,7 @@ | |||
| 20 | |-----------|--------|-----------| | 20 | |-----------|--------|-----------| |
| 21 | | HTTP Server | Hyper (not actix-web) | Better control over WebSocket upgrade handling | | 21 | | HTTP Server | Hyper (not actix-web) | Better control over WebSocket upgrade handling | |
| 22 | | Nostr Relay | nostr-relay-builder | Mature, well-tested, supports custom policies | | 22 | | Nostr Relay | nostr-relay-builder | Mature, well-tested, supports custom policies | |
| 23 | | Database | LMDB (default), NostrDB, Memory | LMDB for production, Memory for testing | | 23 | | Database | LMDB (default), Memory | LMDB for production, Memory for testing | |
| 24 | | Configuration | clap + dotenvy | CLI flags > env vars > .env > defaults | | 24 | | Configuration | clap + dotenvy | CLI flags > env vars > .env > defaults | |
| 25 | 25 | ||
| 26 | --- | 26 | --- |
| @@ -125,7 +125,7 @@ The decision to validate pushes **before** spawning git-receive-pack worked extr | |||
| 125 | Using rust-nostr's relay builder was the right call: | 125 | Using rust-nostr's relay builder was the right call: |
| 126 | - Handles NIP-01 protocol correctly | 126 | - Handles NIP-01 protocol correctly |
| 127 | - Custom `WritePolicy` trait for our validation | 127 | - Custom `WritePolicy` trait for our validation |
| 128 | - Database abstraction (LMDB, NostrDB, Memory) | 128 | - Database abstraction (LMDB, Memory) |
| 129 | - Active maintenance and updates | 129 | - Active maintenance and updates |
| 130 | 130 | ||
| 131 | ### 3. Separate Audit Tool | 131 | ### 3. Separate Audit Tool |
diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index b09b20f..a05aeee 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md | |||
| @@ -245,44 +245,38 @@ NGIT_RELAY_DATA_PATH=/mnt/ssd/relay-data | |||
| 245 | #### `NGIT_DATABASE_BACKEND` | 245 | #### `NGIT_DATABASE_BACKEND` |
| 246 | 246 | ||
| 247 | **Description:** Database backend type for storing Nostr events | 247 | **Description:** Database backend type for storing Nostr events |
| 248 | **Type:** String (enum: memory, nostrdb, lmdb) | 248 | **Type:** String (enum: memory, lmdb) |
| 249 | **Default:** `memory` | 249 | **Default:** `lmdb` |
| 250 | **Required:** No | 250 | **Required:** No |
| 251 | 251 | ||
| 252 | **Valid Values:** | 252 | **Valid Values:** |
| 253 | 253 | ||
| 254 | - `memory` - In-memory database (default, fastest, no persistence) | 254 | - `lmdb` - LMDB backend (persistent, general purpose) |
| 255 | - `nostrdb` - NostrDB backend (persistent, optimized for Nostr) [Not yet implemented] | 255 | - `memory` - In-memory database (fastest, no persistence) |
| 256 | - `lmdb` - LMDB backend (persistent, general purpose) [Not yet implemented] | ||
| 257 | 256 | ||
| 258 | **Examples:** | 257 | **Examples:** |
| 259 | 258 | ||
| 260 | ```bash | 259 | ```bash |
| 261 | # Development (default, no persistence) | 260 | # Production (default, persistent) |
| 262 | NGIT_DATABASE_BACKEND=memory | ||
| 263 | |||
| 264 | # Production with NostrDB (when implemented) | ||
| 265 | NGIT_DATABASE_BACKEND=nostrdb | ||
| 266 | |||
| 267 | # Production with LMDB (when implemented) | ||
| 268 | NGIT_DATABASE_BACKEND=lmdb | 261 | NGIT_DATABASE_BACKEND=lmdb |
| 262 | |||
| 263 | # Development/testing (no persistence) | ||
| 264 | NGIT_DATABASE_BACKEND=memory | ||
| 269 | ``` | 265 | ``` |
| 270 | 266 | ||
| 271 | **Comparison:** | 267 | **Comparison:** |
| 272 | 268 | ||
| 273 | | Backend | Persistence | Performance | Use Case | | 269 | | Backend | Persistence | Performance | Use Case | |
| 274 | | ------- | ----------- | ----------- | ---------------------------- | | 270 | | ------- | ----------- | ----------- | ---------------------------- | |
| 275 | | memory | No | Fastest | Development, testing | | ||
| 276 | | nostrdb | Yes | High | Production (Nostr-optimized) | | ||
| 277 | | lmdb | Yes | High | Production (general purpose) | | 271 | | lmdb | Yes | High | Production (general purpose) | |
| 272 | | memory | No | Fastest | Development, testing | | ||
| 278 | 273 | ||
| 279 | **Notes:** | 274 | **Notes:** |
| 280 | 275 | ||
| 281 | - `memory` backend loses all data on restart | 276 | - `memory` backend loses all data on restart |
| 282 | - NostrDB and LMDB backends will use `NGIT_RELAY_DATA_PATH` for storage | 277 | - `lmdb` backend uses `NGIT_RELAY_DATA_PATH` for storage |
| 283 | - NostrDB and LMDB are planned features, not yet available | ||
| 284 | - Default `memory` backend suitable for development and testing only | 278 | - Default `memory` backend suitable for development and testing only |
| 285 | - Production deployments should use persistent backends when available | 279 | - Production deployments should use `lmdb` |
| 286 | 280 | ||
| 287 | --- | 281 | --- |
| 288 | 282 | ||
diff --git a/nix/module.nix b/nix/module.nix index 89d58de..7354ab6 100644 --- a/nix/module.nix +++ b/nix/module.nix | |||
| @@ -97,12 +97,11 @@ let | |||
| 97 | }; | 97 | }; |
| 98 | 98 | ||
| 99 | databaseBackend = mkOption { | 99 | databaseBackend = mkOption { |
| 100 | type = types.enum [ "lmdb" "nostr-db" "memory" ]; | 100 | type = types.enum [ "lmdb" "memory" ]; |
| 101 | default = "lmdb"; | 101 | default = "lmdb"; |
| 102 | description = '' | 102 | description = '' |
| 103 | Database backend type: | 103 | Database backend type: |
| 104 | - lmdb: LMDB backend (persistent, general purpose) | 104 | - lmdb: LMDB backend (persistent, general purpose) |
| 105 | - nostr-db: NostrDB backend (persistent, optimized for Nostr) | ||
| 106 | - memory: In-memory database (fastest, no persistence) | 105 | - memory: In-memory database (fastest, no persistence) |
| 107 | ''; | 106 | ''; |
| 108 | }; | 107 | }; |
diff --git a/src/config.rs b/src/config.rs index dd7b1e3..5c9303c 100644 --- a/src/config.rs +++ b/src/config.rs | |||
| @@ -280,8 +280,6 @@ pub enum DatabaseBackend { | |||
| 280 | /// LMDB backend (persistent, general purpose) | 280 | /// LMDB backend (persistent, general purpose) |
| 281 | #[default] | 281 | #[default] |
| 282 | Lmdb, | 282 | Lmdb, |
| 283 | /// NostrDB backend (persistent, optimized for Nostr) | ||
| 284 | NostrDb, | ||
| 285 | /// In-memory database (fastest, no persistence - uses temp directory for git data) | 283 | /// In-memory database (fastest, no persistence - uses temp directory for git data) |
| 286 | Memory, | 284 | Memory, |
| 287 | } | 285 | } |
| @@ -290,7 +288,6 @@ impl std::fmt::Display for DatabaseBackend { | |||
| 290 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { | 288 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { |
| 291 | match self { | 289 | match self { |
| 292 | Self::Memory => write!(f, "memory"), | 290 | Self::Memory => write!(f, "memory"), |
| 293 | Self::NostrDb => write!(f, "nostrdb"), | ||
| 294 | Self::Lmdb => write!(f, "lmdb"), | 291 | Self::Lmdb => write!(f, "lmdb"), |
| 295 | } | 292 | } |
| 296 | } | 293 | } |
| @@ -821,7 +818,6 @@ mod tests { | |||
| 821 | #[test] | 818 | #[test] |
| 822 | fn test_database_backend_display() { | 819 | fn test_database_backend_display() { |
| 823 | assert_eq!(DatabaseBackend::Memory.to_string(), "memory"); | 820 | assert_eq!(DatabaseBackend::Memory.to_string(), "memory"); |
| 824 | assert_eq!(DatabaseBackend::NostrDb.to_string(), "nostrdb"); | ||
| 825 | assert_eq!(DatabaseBackend::Lmdb.to_string(), "lmdb"); | 821 | assert_eq!(DatabaseBackend::Lmdb.to_string(), "lmdb"); |
| 826 | } | 822 | } |
| 827 | 823 | ||
diff --git a/src/nostr/builder.rs b/src/nostr/builder.rs index 7a05348..a0088e1 100644 --- a/src/nostr/builder.rs +++ b/src/nostr/builder.rs | |||
| @@ -689,16 +689,6 @@ pub async fn create_relay( | |||
| 689 | max_events: Some(NonZeroUsize::new(100_000).unwrap()), | 689 | max_events: Some(NonZeroUsize::new(100_000).unwrap()), |
| 690 | })) | 690 | })) |
| 691 | } | 691 | } |
| 692 | DatabaseBackend::NostrDb => { | ||
| 693 | tracing::info!("Using NostrDB backend at: {}", db_path.display()); | ||
| 694 | // TODO: Implement NostrDB backend once nostr-relay-builder supports it | ||
| 695 | // For now, fall back to memory database | ||
| 696 | tracing::warn!("NostrDB backend not yet implemented, using in-memory database"); | ||
| 697 | Arc::new(MemoryDatabase::with_opts(MemoryDatabaseOptions { | ||
| 698 | events: true, | ||
| 699 | max_events: Some(NonZeroUsize::new(100_000).unwrap()), | ||
| 700 | })) | ||
| 701 | } | ||
| 702 | DatabaseBackend::Lmdb => { | 692 | DatabaseBackend::Lmdb => { |
| 703 | tracing::info!("Using LMDB backend at: {}", db_path.display()); | 693 | tracing::info!("Using LMDB backend at: {}", db_path.display()); |
| 704 | // Ensure the database directory exists | 694 | // Ensure the database directory exists |