diff options
| -rw-r--r-- | Cargo.lock | 78 | ||||
| -rw-r--r-- | PLAN.md | 119 | ||||
| -rw-r--r-- | src/git_mirror.rs | 13 | ||||
| -rw-r--r-- | src/main.rs | 16 |
4 files changed, 127 insertions, 99 deletions
| @@ -148,6 +148,58 @@ source = "registry+https://github.com/rust-lang/crates.io-index" | |||
| 148 | checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" | 148 | checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" |
| 149 | 149 | ||
| 150 | [[package]] | 150 | [[package]] |
| 151 | name = "axum" | ||
| 152 | version = "0.8.9" | ||
| 153 | source = "registry+https://github.com/rust-lang/crates.io-index" | ||
| 154 | checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" | ||
| 155 | dependencies = [ | ||
| 156 | "axum-core", | ||
| 157 | "bytes", | ||
| 158 | "form_urlencoded", | ||
| 159 | "futures-util", | ||
| 160 | "http", | ||
| 161 | "http-body", | ||
| 162 | "http-body-util", | ||
| 163 | "hyper", | ||
| 164 | "hyper-util", | ||
| 165 | "itoa", | ||
| 166 | "matchit", | ||
| 167 | "memchr", | ||
| 168 | "mime", | ||
| 169 | "percent-encoding", | ||
| 170 | "pin-project-lite", | ||
| 171 | "serde_core", | ||
| 172 | "serde_json", | ||
| 173 | "serde_path_to_error", | ||
| 174 | "serde_urlencoded", | ||
| 175 | "sync_wrapper", | ||
| 176 | "tokio", | ||
| 177 | "tower", | ||
| 178 | "tower-layer", | ||
| 179 | "tower-service", | ||
| 180 | "tracing", | ||
| 181 | ] | ||
| 182 | |||
| 183 | [[package]] | ||
| 184 | name = "axum-core" | ||
| 185 | version = "0.5.6" | ||
| 186 | source = "registry+https://github.com/rust-lang/crates.io-index" | ||
| 187 | checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" | ||
| 188 | dependencies = [ | ||
| 189 | "bytes", | ||
| 190 | "futures-core", | ||
| 191 | "http", | ||
| 192 | "http-body", | ||
| 193 | "http-body-util", | ||
| 194 | "mime", | ||
| 195 | "pin-project-lite", | ||
| 196 | "sync_wrapper", | ||
| 197 | "tower-layer", | ||
| 198 | "tower-service", | ||
| 199 | "tracing", | ||
| 200 | ] | ||
| 201 | |||
| 202 | [[package]] | ||
| 151 | name = "base64" | 203 | name = "base64" |
| 152 | version = "0.22.1" | 204 | version = "0.22.1" |
| 153 | source = "registry+https://github.com/rust-lang/crates.io-index" | 205 | source = "registry+https://github.com/rust-lang/crates.io-index" |
| @@ -785,6 +837,7 @@ name = "grasp-mirror" | |||
| 785 | version = "0.1.0" | 837 | version = "0.1.0" |
| 786 | dependencies = [ | 838 | dependencies = [ |
| 787 | "anyhow", | 839 | "anyhow", |
| 840 | "axum", | ||
| 788 | "clap", | 841 | "clap", |
| 789 | "dirs", | 842 | "dirs", |
| 790 | "dotenvy", | 843 | "dotenvy", |
| @@ -935,6 +988,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" | |||
| 935 | checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" | 988 | checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" |
| 936 | 989 | ||
| 937 | [[package]] | 990 | [[package]] |
| 991 | name = "httpdate" | ||
| 992 | version = "1.0.3" | ||
| 993 | source = "registry+https://github.com/rust-lang/crates.io-index" | ||
| 994 | checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" | ||
| 995 | |||
| 996 | [[package]] | ||
| 938 | name = "hyper" | 997 | name = "hyper" |
| 939 | version = "1.9.0" | 998 | version = "1.9.0" |
| 940 | source = "registry+https://github.com/rust-lang/crates.io-index" | 999 | source = "registry+https://github.com/rust-lang/crates.io-index" |
| @@ -948,6 +1007,7 @@ dependencies = [ | |||
| 948 | "http", | 1007 | "http", |
| 949 | "http-body", | 1008 | "http-body", |
| 950 | "httparse", | 1009 | "httparse", |
| 1010 | "httpdate", | ||
| 951 | "itoa", | 1011 | "itoa", |
| 952 | "pin-project-lite", | 1012 | "pin-project-lite", |
| 953 | "smallvec", | 1013 | "smallvec", |
| @@ -1321,6 +1381,12 @@ dependencies = [ | |||
| 1321 | ] | 1381 | ] |
| 1322 | 1382 | ||
| 1323 | [[package]] | 1383 | [[package]] |
| 1384 | name = "matchit" | ||
| 1385 | version = "0.8.4" | ||
| 1386 | source = "registry+https://github.com/rust-lang/crates.io-index" | ||
| 1387 | checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" | ||
| 1388 | |||
| 1389 | [[package]] | ||
| 1324 | name = "md-5" | 1390 | name = "md-5" |
| 1325 | version = "0.10.6" | 1391 | version = "0.10.6" |
| 1326 | source = "registry+https://github.com/rust-lang/crates.io-index" | 1392 | source = "registry+https://github.com/rust-lang/crates.io-index" |
| @@ -2114,6 +2180,17 @@ dependencies = [ | |||
| 2114 | ] | 2180 | ] |
| 2115 | 2181 | ||
| 2116 | [[package]] | 2182 | [[package]] |
| 2183 | name = "serde_path_to_error" | ||
| 2184 | version = "0.1.20" | ||
| 2185 | source = "registry+https://github.com/rust-lang/crates.io-index" | ||
| 2186 | checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" | ||
| 2187 | dependencies = [ | ||
| 2188 | "itoa", | ||
| 2189 | "serde", | ||
| 2190 | "serde_core", | ||
| 2191 | ] | ||
| 2192 | |||
| 2193 | [[package]] | ||
| 2117 | name = "serde_spanned" | 2194 | name = "serde_spanned" |
| 2118 | version = "0.6.9" | 2195 | version = "0.6.9" |
| 2119 | source = "registry+https://github.com/rust-lang/crates.io-index" | 2196 | source = "registry+https://github.com/rust-lang/crates.io-index" |
| @@ -2745,6 +2822,7 @@ dependencies = [ | |||
| 2745 | "tokio", | 2822 | "tokio", |
| 2746 | "tower-layer", | 2823 | "tower-layer", |
| 2747 | "tower-service", | 2824 | "tower-service", |
| 2825 | "tracing", | ||
| 2748 | ] | 2826 | ] |
| 2749 | 2827 | ||
| 2750 | [[package]] | 2828 | [[package]] |
| @@ -1,94 +1,25 @@ | |||
| 1 | # GRASP Mirror Daemon — Implementation Plan | 1 | # grasp-mirror Bugfix Plan |
| 2 | 2 | ||
| 3 | A Rust daemon that discovers repos from watched npubs and mirrors git data + Nostr events across all known GRASP servers for redundancy. | 3 | ## Bugs Identified from VPS Logs |
| 4 | 4 | ||
| 5 | ## Phase 1: Core (MVP) | 5 | ### Bug 1: `remote 'push_target' already exists` |
| 6 | 6 | - **Location**: `src/git_mirror.rs:137` | |
| 7 | ### Infrastructure | 7 | - **Cause**: `repo.remote("push_target", target_url)` creates a new remote every cycle. On subsequent cycles, the bare repo already has a `push_target` remote, so `git2` returns `Exists (-4)`. |
| 8 | 8 | - **Fix**: Use `repo.find_remote("push_target")` first. If it exists, call `repo.remote_set_url()` to update the URL. If not, create it. | |
| 9 | - [x] Cargo.toml with dependencies | 9 | |
| 10 | - [x] Project directory structure | 10 | ### Bug 2: Single repo failure aborts entire cycle |
| 11 | - [x] PLAN.md (this file) | 11 | - **Location**: `src/main.rs:mirror_cycle()` — the `mirror.mirror_repo_to_servers()` call uses `?` which propagates the error and aborts the loop. |
| 12 | - [ ] config.example.toml | 12 | - **Cause**: When one repo (e.g. `market`) fails to clone from any server, the `?` operator returns early, skipping the remaining ~87 repos. |
| 13 | - [ ] .env.example | 13 | - **Fix**: Replace `?` with a `match` that logs the error and continues to the next repo. Only the `discover_repos_from_relays` and `persist_discovered_repos` calls should be fatal (if relay queries fail, nothing works). |
| 14 | - [ ] SQLite migration schema | 14 | |
| 15 | 15 | ## Checklist | |
| 16 | ### Source Modules | 16 | |
| 17 | 17 | - [ ] Fix `push_mirror` in `src/git_mirror.rs` — reuse existing `push_target` remote | |
| 18 | - [ ] `config.rs` — TOML config + .env loading, npub parsing, server list | 18 | - [ ] Fix `mirror_cycle` in `src/main.rs` — make per-repo errors non-fatal |
| 19 | - [ ] `db.rs` — SQLite state tracking (repos, sync status, event dedup) | 19 | - [ ] Rebuild release binary (`cargo build --release`) |
| 20 | - [ ] `health.rs` — NIP-11 GRASP server verification and health checks | 20 | - [ ] Push updated source to all 9 GRASP servers (`git push origin master`) |
| 21 | - [ ] `discovery.rs` — Relay subscriptions for kind:30617/10317/30618, repo parsing | 21 | - [ ] Redeploy via Ansible (`ansible-playbook playbooks/30-grasp-mirror.yml`) |
| 22 | - [ ] `git_mirror.rs` — Bare clone + `git push --mirror` to missing GRASP servers | 22 | - [ ] Verify: `grasp-mirror status` shows repos tracked and sync records |
| 23 | - [ ] `nostr_mirror.rs` — Forward Nostr events to all GRASP server relays | 23 | - [ ] Verify: `journalctl -u grasp-mirror` shows successful mirror cycles without `already exists` errors |
| 24 | - [ ] `signing.rs` — Optional: update announcements to include mirrored servers | 24 | - [ ] Verify: spot-check `git ls-remote` against a few GRASP servers for repos from all 3 npubs |
| 25 | - [ ] `main.rs` — CLI entry point, daemon loop, signal handling | 25 | - [ ] Verify: `https://git.orangesync.tech/api/mirror-health` returns `"status": "ok"` |
| 26 | |||
| 27 | ## Phase 2: Resilience | ||
| 28 | |||
| 29 | - [ ] Continuous state sync (watch kind:30618, push new refs to missing servers) | ||
| 30 | - [ ] Periodic NIP-11 health checks on all known GRASP servers | ||
| 31 | - [ ] Auto-discover new GRASP servers from indexer kind:10317 events | ||
| 32 | - [ ] Retry with exponential backoff for failed mirrors | ||
| 33 | - [ ] SIGHUP hot-reload of npub list from .env | ||
| 34 | |||
| 35 | ## Phase 3: Advanced | ||
| 36 | |||
| 37 | - [ ] Per-npub / per-repo server allowlists and blocklists | ||
| 38 | - [ ] Prometheus metrics endpoint | ||
| 39 | - [ ] CLI status subcommand (show mirror health per repo/server) | ||
| 40 | - [ ] Web dashboard | ||
| 41 | |||
| 42 | ## Known GRASP Servers | ||
| 43 | |||
| 44 | | Server | Software | GRASP Support | | ||
| 45 | |--------|----------|---------------| | ||
| 46 | | `relay.ngit.dev` | ngit-grasp 1.1.0 | 01/02/06 | | ||
| 47 | | `gitnostr.com` | ngit-grasp 1.0.2 | 01/02/06 | | ||
| 48 | | `git.orangesync.tech` | ngit-grasp 0.1.0 | 01/02/05 | | ||
| 49 | | `ngit.danconwaydev.com` | ngit-grasp 1.0.2 | 01/02/06 | | ||
| 50 | | `git.shakespeare.diy` | ngit-grasp 1.0.1 | 01/02 | | ||
| 51 | | `git.upleb.uk` | ngit-grasp 1.0.2 | 01/02 | | ||
| 52 | | `grasp.budabit.club` | ngit-grasp 1.0.2 | 01/02 | | ||
| 53 | | `git.uid.ovh` | ngit-grasp 1.0.2 | 01/02 | | ||
| 54 | | `git.vps.satsnode.xyz` | ngit-grasp 1.0.1 | 01/02 | | ||
| 55 | |||
| 56 | ## Architecture | ||
| 57 | |||
| 58 | ``` | ||
| 59 | ┌─────────────────────────────────────────────────────┐ | ||
| 60 | │ grasp-mirror │ | ||
| 61 | │ │ | ||
| 62 | │ ┌────────────────┐ ┌─────────────────────────┐ │ | ||
| 63 | │ │ Discovery │ │ Mirror Engine │ │ | ||
| 64 | │ │ ──────────── │ │ ──────────────────── │ │ | ||
| 65 | │ │ • Index relay │────▶│ For each repo: │ │ | ||
| 66 | │ │ • GRASP relays│ │ 1. git clone --bare │ │ | ||
| 67 | │ │ • kind:30617 │ │ from any source │ │ | ||
| 68 | │ │ • kind:10317 │ │ 2. git push --mirror │ │ | ||
| 69 | │ │ • kind:30618 │ │ to missing servers │ │ | ||
| 70 | │ │ │ │ 3. Republish nostr │ │ | ||
| 71 | │ └────────────────┘ │ events to all relays │ │ | ||
| 72 | │ └─────────────────────────┘ │ | ||
| 73 | │ ┌────────────────┐ ┌─────────────────────────┐ │ | ||
| 74 | │ │ Config │ │ State DB (SQLite) │ │ | ||
| 75 | │ │ ──────────── │ │ ──────────────────── │ │ | ||
| 76 | │ │ config.toml │ │ • repos seen │ │ | ||
| 77 | │ │ .env (npubs) │ │ • last sync per server │ │ | ||
| 78 | │ │ [optional] │ │ • sync status/errors │ │ | ||
| 79 | │ │ nsec for │ │ • event ids processed │ │ | ||
| 80 | │ │ signing │ └─────────────────────────┘ │ | ||
| 81 | │ └────────────────┘ │ | ||
| 82 | └─────────────────────────────────────────────────────┘ | ||
| 83 | ``` | ||
| 84 | |||
| 85 | ## Data Flow | ||
| 86 | |||
| 87 | 1. **Discovery**: Subscribe to index relay for kind:30617 from watched npubs | ||
| 88 | 2. **Parse**: Extract repo identifier, clone URLs, relay URLs from each announcement | ||
| 89 | 3. **Diff**: Compare clone URLs against known GRASP servers to find missing mirrors | ||
| 90 | 4. **Mirror Git**: | ||
| 91 | - `git clone --bare` from any available clone URL | ||
| 92 | - For each missing GRASP server: publish announcement to its relay (triggers repo creation), then `git push --mirror` | ||
| 93 | 5. **Mirror Nostr**: Forward all related events (issues, comments, state) to all GRASP server relays | ||
| 94 | 6. **Track**: Record mirror status in SQLite, skip already-mirrored repos on restart | ||
diff --git a/src/git_mirror.rs b/src/git_mirror.rs index 47c0442..6866de3 100644 --- a/src/git_mirror.rs +++ b/src/git_mirror.rs | |||
| @@ -134,7 +134,18 @@ impl GitMirror { | |||
| 134 | let repo = git2::Repository::open(repo_path) | 134 | let repo = git2::Repository::open(repo_path) |
| 135 | .with_context(|| format!("failed to open bare repo at {:?}", repo_path))?; | 135 | .with_context(|| format!("failed to open bare repo at {:?}", repo_path))?; |
| 136 | 136 | ||
| 137 | let mut remote = repo.remote("push_target", target_url)?; | 137 | let remote_name = "push_target"; |
| 138 | |||
| 139 | match repo.find_remote(remote_name) { | ||
| 140 | Ok(_) => { | ||
| 141 | repo.remote_set_url(remote_name, target_url)?; | ||
| 142 | } | ||
| 143 | Err(_) => { | ||
| 144 | repo.remote(remote_name, target_url)?; | ||
| 145 | } | ||
| 146 | } | ||
| 147 | |||
| 148 | let mut remote = repo.find_remote(remote_name)?; | ||
| 138 | 149 | ||
| 139 | let mut callbacks = RemoteCallbacks::new(); | 150 | let mut callbacks = RemoteCallbacks::new(); |
| 140 | callbacks.credentials(|_url, _username, _allowed| git2::Cred::default()); | 151 | callbacks.credentials(|_url, _username, _allowed| git2::Cred::default()); |
diff --git a/src/main.rs b/src/main.rs index 494342c..3fcd27b 100644 --- a/src/main.rs +++ b/src/main.rs | |||
| @@ -190,13 +190,21 @@ async fn mirror_cycle( | |||
| 190 | "mirroring to missing servers" | 190 | "mirroring to missing servers" |
| 191 | ); | 191 | ); |
| 192 | 192 | ||
| 193 | mirror.mirror_repo_to_servers(db, repo, &missing).await?; | 193 | if let Err(e) = mirror.mirror_repo_to_servers(db, repo, &missing).await { |
| 194 | nostr_mirror.forward_repo_events(db, repo, servers).await?; | 194 | tracing::error!(identifier = %repo.identifier, error = %e, "git mirror failed for repo, continuing"); |
| 195 | } | ||
| 196 | |||
| 197 | if let Err(e) = nostr_mirror.forward_repo_events(db, repo, servers).await { | ||
| 198 | tracing::error!(identifier = %repo.identifier, error = %e, "nostr mirror failed for repo, continuing"); | ||
| 199 | } | ||
| 195 | } | 200 | } |
| 196 | 201 | ||
| 197 | nostr_mirror | 202 | if let Err(e) = nostr_mirror |
| 198 | .sync_all_events(db, &config.npubs, servers) | 203 | .sync_all_events(db, &config.npubs, servers) |
| 199 | .await?; | 204 | .await |
| 205 | { | ||
| 206 | tracing::error!(error = %e, "event sync failed"); | ||
| 207 | } | ||
| 200 | 208 | ||
| 201 | tracing::info!("mirror cycle complete"); | 209 | tracing::info!("mirror cycle complete"); |
| 202 | Ok(()) | 210 | Ok(()) |