From 7b1b36b8c7e448d1d170c8c6e1f88bb766163fbb Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 26 May 2026 20:38:49 +0530 Subject: Fix push_target remote reuse and non-fatal per-repo errors - git_mirror: reuse existing push_target remote via remote_set_url instead of failing on 'remote already exists' on subsequent cycles - mirror_cycle: catch per-repo errors instead of propagating with ? so one failed clone doesn't abort the remaining 87 repos - Add PLAN.md with bug tracking checklist --- Cargo.lock | 78 +++++++++++++++++++++++++++++++++++ PLAN.md | 119 ++++++++++++------------------------------------------ src/git_mirror.rs | 13 +++++- src/main.rs | 16 ++++++-- 4 files changed, 127 insertions(+), 99 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6fb8ee4..3630a59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,6 +147,58 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "base64" version = "0.22.1" @@ -785,6 +837,7 @@ name = "grasp-mirror" version = "0.1.0" dependencies = [ "anyhow", + "axum", "clap", "dirs", "dotenvy", @@ -934,6 +987,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "hyper" version = "1.9.0" @@ -948,6 +1007,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1320,6 +1380,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "md-5" version = "0.10.6" @@ -2113,6 +2179,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_spanned" version = "0.6.9" @@ -2745,6 +2822,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] diff --git a/PLAN.md b/PLAN.md index 2664d51..ab1ea9f 100644 --- a/PLAN.md +++ b/PLAN.md @@ -1,94 +1,25 @@ -# GRASP Mirror Daemon — Implementation Plan - -A Rust daemon that discovers repos from watched npubs and mirrors git data + Nostr events across all known GRASP servers for redundancy. - -## Phase 1: Core (MVP) - -### Infrastructure - -- [x] Cargo.toml with dependencies -- [x] Project directory structure -- [x] PLAN.md (this file) -- [ ] config.example.toml -- [ ] .env.example -- [ ] SQLite migration schema - -### Source Modules - -- [ ] `config.rs` — TOML config + .env loading, npub parsing, server list -- [ ] `db.rs` — SQLite state tracking (repos, sync status, event dedup) -- [ ] `health.rs` — NIP-11 GRASP server verification and health checks -- [ ] `discovery.rs` — Relay subscriptions for kind:30617/10317/30618, repo parsing -- [ ] `git_mirror.rs` — Bare clone + `git push --mirror` to missing GRASP servers -- [ ] `nostr_mirror.rs` — Forward Nostr events to all GRASP server relays -- [ ] `signing.rs` — Optional: update announcements to include mirrored servers -- [ ] `main.rs` — CLI entry point, daemon loop, signal handling - -## Phase 2: Resilience - -- [ ] Continuous state sync (watch kind:30618, push new refs to missing servers) -- [ ] Periodic NIP-11 health checks on all known GRASP servers -- [ ] Auto-discover new GRASP servers from indexer kind:10317 events -- [ ] Retry with exponential backoff for failed mirrors -- [ ] SIGHUP hot-reload of npub list from .env - -## Phase 3: Advanced - -- [ ] Per-npub / per-repo server allowlists and blocklists -- [ ] Prometheus metrics endpoint -- [ ] CLI status subcommand (show mirror health per repo/server) -- [ ] Web dashboard - -## Known GRASP Servers - -| Server | Software | GRASP Support | -|--------|----------|---------------| -| `relay.ngit.dev` | ngit-grasp 1.1.0 | 01/02/06 | -| `gitnostr.com` | ngit-grasp 1.0.2 | 01/02/06 | -| `git.orangesync.tech` | ngit-grasp 0.1.0 | 01/02/05 | -| `ngit.danconwaydev.com` | ngit-grasp 1.0.2 | 01/02/06 | -| `git.shakespeare.diy` | ngit-grasp 1.0.1 | 01/02 | -| `git.upleb.uk` | ngit-grasp 1.0.2 | 01/02 | -| `grasp.budabit.club` | ngit-grasp 1.0.2 | 01/02 | -| `git.uid.ovh` | ngit-grasp 1.0.2 | 01/02 | -| `git.vps.satsnode.xyz` | ngit-grasp 1.0.1 | 01/02 | - -## Architecture - -``` -┌─────────────────────────────────────────────────────┐ -│ grasp-mirror │ -│ │ -│ ┌────────────────┐ ┌─────────────────────────┐ │ -│ │ Discovery │ │ Mirror Engine │ │ -│ │ ──────────── │ │ ──────────────────── │ │ -│ │ • Index relay │────▶│ For each repo: │ │ -│ │ • GRASP relays│ │ 1. git clone --bare │ │ -│ │ • kind:30617 │ │ from any source │ │ -│ │ • kind:10317 │ │ 2. git push --mirror │ │ -│ │ • kind:30618 │ │ to missing servers │ │ -│ │ │ │ 3. Republish nostr │ │ -│ └────────────────┘ │ events to all relays │ │ -│ └─────────────────────────┘ │ -│ ┌────────────────┐ ┌─────────────────────────┐ │ -│ │ Config │ │ State DB (SQLite) │ │ -│ │ ──────────── │ │ ──────────────────── │ │ -│ │ config.toml │ │ • repos seen │ │ -│ │ .env (npubs) │ │ • last sync per server │ │ -│ │ [optional] │ │ • sync status/errors │ │ -│ │ nsec for │ │ • event ids processed │ │ -│ │ signing │ └─────────────────────────┘ │ -│ └────────────────┘ │ -└─────────────────────────────────────────────────────┘ -``` - -## Data Flow - -1. **Discovery**: Subscribe to index relay for kind:30617 from watched npubs -2. **Parse**: Extract repo identifier, clone URLs, relay URLs from each announcement -3. **Diff**: Compare clone URLs against known GRASP servers to find missing mirrors -4. **Mirror Git**: - - `git clone --bare` from any available clone URL - - For each missing GRASP server: publish announcement to its relay (triggers repo creation), then `git push --mirror` -5. **Mirror Nostr**: Forward all related events (issues, comments, state) to all GRASP server relays -6. **Track**: Record mirror status in SQLite, skip already-mirrored repos on restart +# grasp-mirror Bugfix Plan + +## Bugs Identified from VPS Logs + +### Bug 1: `remote 'push_target' already exists` +- **Location**: `src/git_mirror.rs:137` +- **Cause**: `repo.remote("push_target", target_url)` creates a new remote every cycle. On subsequent cycles, the bare repo already has a `push_target` remote, so `git2` returns `Exists (-4)`. +- **Fix**: Use `repo.find_remote("push_target")` first. If it exists, call `repo.remote_set_url()` to update the URL. If not, create it. + +### Bug 2: Single repo failure aborts entire cycle +- **Location**: `src/main.rs:mirror_cycle()` — the `mirror.mirror_repo_to_servers()` call uses `?` which propagates the error and aborts the loop. +- **Cause**: When one repo (e.g. `market`) fails to clone from any server, the `?` operator returns early, skipping the remaining ~87 repos. +- **Fix**: Replace `?` with a `match` that logs the error and continues to the next repo. Only the `discover_repos_from_relays` and `persist_discovered_repos` calls should be fatal (if relay queries fail, nothing works). + +## Checklist + +- [ ] Fix `push_mirror` in `src/git_mirror.rs` — reuse existing `push_target` remote +- [ ] Fix `mirror_cycle` in `src/main.rs` — make per-repo errors non-fatal +- [ ] Rebuild release binary (`cargo build --release`) +- [ ] Push updated source to all 9 GRASP servers (`git push origin master`) +- [ ] Redeploy via Ansible (`ansible-playbook playbooks/30-grasp-mirror.yml`) +- [ ] Verify: `grasp-mirror status` shows repos tracked and sync records +- [ ] Verify: `journalctl -u grasp-mirror` shows successful mirror cycles without `already exists` errors +- [ ] Verify: spot-check `git ls-remote` against a few GRASP servers for repos from all 3 npubs +- [ ] Verify: `https://git.orangesync.tech/api/mirror-health` returns `"status": "ok"` diff --git a/src/git_mirror.rs b/src/git_mirror.rs index 47c0442..6866de3 100644 --- a/src/git_mirror.rs +++ b/src/git_mirror.rs @@ -134,7 +134,18 @@ impl GitMirror { let repo = git2::Repository::open(repo_path) .with_context(|| format!("failed to open bare repo at {:?}", repo_path))?; - let mut remote = repo.remote("push_target", target_url)?; + let remote_name = "push_target"; + + match repo.find_remote(remote_name) { + Ok(_) => { + repo.remote_set_url(remote_name, target_url)?; + } + Err(_) => { + repo.remote(remote_name, target_url)?; + } + } + + let mut remote = repo.find_remote(remote_name)?; let mut callbacks = RemoteCallbacks::new(); callbacks.credentials(|_url, _username, _allowed| git2::Cred::default()); diff --git a/src/main.rs b/src/main.rs index 494342c..3fcd27b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -190,13 +190,21 @@ async fn mirror_cycle( "mirroring to missing servers" ); - mirror.mirror_repo_to_servers(db, repo, &missing).await?; - nostr_mirror.forward_repo_events(db, repo, servers).await?; + if let Err(e) = mirror.mirror_repo_to_servers(db, repo, &missing).await { + tracing::error!(identifier = %repo.identifier, error = %e, "git mirror failed for repo, continuing"); + } + + if let Err(e) = nostr_mirror.forward_repo_events(db, repo, servers).await { + tracing::error!(identifier = %repo.identifier, error = %e, "nostr mirror failed for repo, continuing"); + } } - nostr_mirror + if let Err(e) = nostr_mirror .sync_all_events(db, &config.npubs, servers) - .await?; + .await + { + tracing::error!(error = %e, "event sync failed"); + } tracing::info!("mirror cycle complete"); Ok(()) -- cgit v1.2.3