From 79d2d01923cf66217fd5aa3a0be5ee9ea2000872 Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Fri, 23 Jan 2026 12:15:12 +0000 Subject: Generalize migration guide for any GRASP implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename guide: migrate-ngit-relay-to-ngit-grasp.md → migrate-to-ngit-grasp.md - Remove ngit-relay and relay.ngit.dev specific references - Use generic terminology: source/target relay, current implementation - Add Compatibility section explaining requirements - Update examples to be implementation-agnostic - Update script comments to reference GRASP relay (not ngit-relay) - Update README.md to link to the new guide Scripts already work with any GRASP implementation via parameters. --- docs/how-to/README.md | 14 +- docs/how-to/migrate-ngit-relay-to-ngit-grasp.md | 395 -------------------- docs/how-to/migrate-to-ngit-grasp.md | 412 +++++++++++++++++++++ docs/how-to/migration-scripts/01-fetch-events.sh | 4 +- docs/how-to/migration-scripts/10-check-git-sync.sh | 20 +- docs/how-to/migration-scripts/20-categorize.sh | 4 +- docs/how-to/migration-scripts/21-compare-relays.sh | 4 +- .../migration-scripts/30-extract-parse-failures.sh | 10 +- .../31-extract-purgatory-expiry.sh | 10 +- .../migration-scripts/40-classify-actions.sh | 4 +- .../migration-scripts/run-migration-analysis.sh | 14 +- 11 files changed, 454 insertions(+), 437 deletions(-) delete mode 100644 docs/how-to/migrate-ngit-relay-to-ngit-grasp.md create mode 100644 docs/how-to/migrate-to-ngit-grasp.md (limited to 'docs') diff --git a/docs/how-to/README.md b/docs/how-to/README.md index ba58c08..f755be1 100644 --- a/docs/how-to/README.md +++ b/docs/how-to/README.md @@ -110,15 +110,15 @@ How-to guides are **recipes** that show you how to solve specific problems or ac --- -### Migrate from ngit-relay -**Status:** 🔜 Planned +### [Migrate to ngit-grasp](migrate-to-ngit-grasp.md) +**Status:** ✅ Available -**Problem:** Switch from reference implementation +**Problem:** Switch from another GRASP implementation **You'll learn:** -- Export data from ngit-relay -- Import to ngit-grasp -- Update repository URLs -- Verify migration +- Analyze existing relay data +- Identify repositories needing attention +- Run migration analysis scripts +- Plan and execute cutover --- diff --git a/docs/how-to/migrate-ngit-relay-to-ngit-grasp.md b/docs/how-to/migrate-ngit-relay-to-ngit-grasp.md deleted file mode 100644 index 975eb4c..0000000 --- a/docs/how-to/migrate-ngit-relay-to-ngit-grasp.md +++ /dev/null @@ -1,395 +0,0 @@ -# Migrate ngit-relay to ngit-grasp - -This guide walks you through migrating a production ngit-relay instance to ngit-grasp. The process involves analyzing your existing data to identify repositories that need attention before switching over. - -## Quick Start - -Run the migration analysis with a single command: - -```bash -# Basic analysis (fetches events, compares relays) -./docs/how-to/migration-scripts/run-migration-analysis.sh \ - --prod-relay wss://relay.ngit.dev \ - --archive-relay wss://archive.relay.ngit.dev - -# Full analysis (includes git sync check - run on VPS) -./docs/how-to/migration-scripts/run-migration-analysis.sh \ - --prod-relay wss://relay.ngit.dev \ - --archive-relay wss://archive.relay.ngit.dev \ - --prod-git /var/lib/ngit-relay/git \ - --archive-git /var/lib/ngit-relay-archive/git \ - --service ngit-grasp.service -``` - -The script produces three output files: -- `results/no-action-required.txt` - Repos ready for migration -- `results/action-required.txt` - Repos needing intervention -- `results/manual-investigation.txt` - Repos needing human review - -See [Running the Analysis](#running-the-analysis) for detailed options. - -## Prerequisites - -### Required Tools - -- **nak** - Nostr Army Knife for fetching events ([install](https://github.com/fiatjaf/nak)) -- **jq** - JSON processing (install via package manager) - -### For Full Analysis (VPS) - -- SSH access to the VPS running ngit-relay -- Read access to git data directories -- Access to systemd journal (for log extraction) - -### Verify Installation - -```bash -# Check required tools -nak --version -jq --version - -# Check optional tools (for VPS phases) -journalctl --version -``` - -## Migration Overview - -The migration process has three stages: - -### Stage 1: Deploy Archive Instance - -Deploy ngit-grasp alongside your production ngit-relay: - -1. Configure ngit-grasp with: - - `domain` set to `.internal` (temporary) - - `archiveService` set to your production domain - - Running on a different port - -2. Let it sync for ~1 hour to gather all events and git data - -### Stage 2: Analyze Data - -Run the migration analysis to identify: -- Repositories successfully migrated (no action needed) -- Repositories with incomplete data (need investigation) -- Repositories with parse failures (may need re-announcement) - -### Stage 3: Switch Over - -Once all issues are resolved: -1. Set `domain` to your production URL -2. Disable archive mode -3. Update your reverse proxy to point to ngit-grasp - -## Running the Analysis - -### Basic Usage - -```bash -# Preview what will happen (dry run) -./run-migration-analysis.sh \ - --prod-relay wss://relay.ngit.dev \ - --archive-relay wss://archive.relay.ngit.dev \ - --dry-run - -# Run the analysis -./run-migration-analysis.sh \ - --prod-relay wss://relay.ngit.dev \ - --archive-relay wss://archive.relay.ngit.dev -``` - -### Full Analysis on VPS - -```bash -./run-migration-analysis.sh \ - --prod-relay wss://relay.ngit.dev \ - --archive-relay wss://archive.relay.ngit.dev \ - --prod-git /var/lib/ngit-relay/git \ - --archive-git /var/lib/ngit-relay-archive/git \ - --service ngit-grasp.service -``` - -### Phase Control - -Skip or run specific phases: - -```bash -# Skip Phase 2 (use cached git sync data) -./run-migration-analysis.sh ... --skip-phase-2 - -# Run only Phase 1 (fetch events) -./run-migration-analysis.sh ... --only-phase-1 - -# Resume from Phase 3 (using existing data) -./run-migration-analysis.sh ... --from-phase-3 --output work/migration-analysis-20260122-1430 -``` - -### All Options - -| Option | Description | -|--------|-------------| -| `--prod-relay ` | Production relay WebSocket URL (required) | -| `--archive-relay ` | Archive relay WebSocket URL (required) | -| `--prod-git ` | Git base directory for prod (enables Phase 2) | -| `--archive-git ` | Git base directory for archive (enables Phase 2) | -| `--service ` | Systemd service name (enables Phase 4) | -| `--output ` | Output directory (default: auto-generated) | -| `--skip-phase-N` | Skip phase N (1-5) | -| `--only-phase-N` | Run only phase N | -| `--from-phase-N` | Start from phase N | -| `--dry-run` | Show what would be executed | -| `--continue-on-error` | Continue even if a phase fails | - -## Understanding Results - -### Summary File - -The `results/summary.txt` file provides an overview: - -``` -## Overview - -| Category | Count | Percentage | -|----------|-------|------------| -| No Action Required | 450 | 85.7% | -| Action Required | 52 | 9.9% | -| Manual Investigation | 23 | 4.4% | -``` - -### No Action Required - -Repositories in `no-action-required.txt` are ready for migration: - -``` -myrepo | npub1abc... | complete in both prod and archive -oldrepo | npub1def... | deleted by user -testrepo | npub1ghi... | empty/blank in both (user never pushed) -``` - -**Common reasons:** -- `complete in both prod and archive` - Successfully migrated -- `deleted by user` - User requested deletion (kind 5 event) -- `empty/blank in both` - No git data was ever pushed -- `purgatory expired` - System already handled the timeout - -### Action Required - -Repositories in `action-required.txt` need intervention: - -``` -myrepo | npub1abc... | complete in prod, missing from archive | trigger re-sync or investigate -otherrepo | npub1def... | incomplete in both (prod=cat3, archive=cat2) | investigate git data source -``` - -**Common actions:** -- **Re-sync needed**: Trigger the archive to re-fetch from the source -- **Wait for sync**: Archive sync may still be in progress -- **Investigate git source**: Original git data may be incomplete -- **Fix parse failure**: Event format issue, may need re-announcement - -### Manual Investigation - -Repositories in `manual-investigation.txt` have unusual states: - -``` -weirdrepo | npub1abc... | in archive (cat1) but not in prod | may be new announcement or deleted from prod -conflictrepo | npub1def... | complete in prod, missing from archive, parse failure logged | investigate parse failure -``` - -These require human judgment to determine the correct action. - -## Troubleshooting - -### "nak not found" - -Install nak from https://github.com/fiatjaf/nak: - -```bash -# Using Go -go install github.com/fiatjaf/nak@latest - -# Or download binary from releases -``` - -### "Permission denied" on git directories - -Run with sudo or ensure your user has read access: - -```bash -# Check permissions -ls -la /var/lib/ngit-relay/git - -# Run with sudo if needed -sudo ./run-migration-analysis.sh ... -``` - -### Phase 2 takes too long - -The git sync check processes each repository individually (~20 minutes total). To speed up iteration: - -1. Run Phase 2 once and save the output -2. Use `--skip-phase-2` for subsequent runs -3. Use `--from-phase-3` to re-run classification with existing data - -### No parse failures found - -This is expected if: -- ngit-grasp logging improvements aren't deployed yet -- No events actually failed to parse - -The analysis will continue without log data. - -### Event counts are multiples of 250 - -This suggests pagination may have failed. The scripts use `--paginate` by default, but if you see exactly 250, 500, 750 events, verify the relay is responding correctly. - -## Architecture - -### Analysis Phases - -The analysis is split into 5 modular phases: - -| Phase | Name | Time | Location | Description | -|-------|------|------|----------|-------------| -| 1 | Fetch Events | ~30s each | Local | Fetch events from both relays | -| 2 | Git Sync Check | ~20 min each | VPS | Compare state events to git data | -| 3 | Categorize & Compare | <1s | Local | Categorize and compare results | -| 4 | Extract Logs | <30s | VPS | Extract parse failures and purgatory expiry | -| 5 | Final Classification | <5s | Local | Combine all data into actionable results | - -### Phase Flow Diagram - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ PHASE 1: Fetch Events (~30s, local) │ -│ Fetches kind 30618 (state), 30617 (announcements), 5 (deletion) │ -│ Run twice: once for prod, once for archive │ -└─────────────────────────────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────────────────────────────┐ -│ PHASE 2: Git Sync Check (~20 mins, VPS required) │ -│ Compares state event refs to actual git data on disk │ -│ Categorizes into: complete, empty, partial, no-match │ -└─────────────────────────────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────────────────────────────┐ -│ PHASE 3: Categorize & Compare (fast, local) │ -│ Compares prod vs archive categories │ -│ Identifies gaps and sync issues │ -└─────────────────────────────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────────────────────────────┐ -│ PHASE 4: Log-Based Categories (VPS required) │ -│ Extracts [PARSE_FAIL] and [PURGATORY_EXPIRED] from logs │ -│ Provides context for why repos failed to sync │ -└─────────────────────────────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────────────────────────────┐ -│ PHASE 5: Final Classification (fast, local) │ -│ Combines all data sources │ -│ Outputs: no-action, action-required, manual-investigation │ -└─────────────────────────────────────────────────────────────────┘ -``` - -### Git Sync Categories - -Phase 2 categorizes repositories into 4 categories: - -| Category | Description | Meaning | -|----------|-------------|---------| -| 1 | Complete Match | All refs in state event match git data | -| 2 | Empty/Blank | No git data available | -| 3 | Partial Match | Some refs match, some don't | -| 4 | No Match | Git data exists but refs don't match | - -### Output Directory Structure - -``` -work/migration-analysis-YYYYMMDD-HHMM/ -├── prod/ -│ ├── raw/ -│ │ ├── state-events.json # Phase 1 -│ │ ├── announcements.json # Phase 1 -│ │ └── deletions.json # Phase 1 -│ ├── git-sync-status.tsv # Phase 2 -│ └── category*.txt # Phase 2/3 -├── archive/ -│ └── (same structure as prod) -├── comparison/ -│ ├── complete-in-both.txt # Phase 3 -│ ├── complete-prod-missing-archive.txt -│ ├── complete-prod-incomplete-archive.txt -│ ├── incomplete-in-both.txt -│ ├── in-archive-not-prod.txt -│ └── summary.txt -├── logs/ -│ ├── parse-failures.txt # Phase 4 -│ └── purgatory-expired.txt # Phase 4 -└── results/ - ├── no-action-required.txt # Phase 5 - ├── action-required.txt # Phase 5 - ├── manual-investigation.txt # Phase 5 - └── summary.txt # Phase 5 -``` - -## Key Differences: ngit-relay vs ngit-grasp - -Understanding these differences helps explain why some repositories need attention: - -| Aspect | ngit-relay | ngit-grasp | -|--------|------------|------------| -| Git data validation | Accepts commits/tags referenced in state event | Requires all git data to reproduce state | -| PR refs cleanup | Doesn't clear `refs/nostr/` | Properly manages PR refs | -| Parse failures | Silently ignores | Logs structured `[PARSE_FAIL]` entries | -| Sync timeout | No timeout | Purgatory expires after configurable period | - -## Next Steps - -After running the analysis: - -1. **Review the summary** - Check `results/summary.txt` for the overview -2. **Address action items** - Work through `results/action-required.txt` -3. **Investigate edge cases** - Review `results/manual-investigation.txt` -4. **Re-run analysis** - After fixing issues, re-run to verify -5. **Plan cutover** - Schedule the switch when all issues are resolved - -### When to Re-run - -Re-run the analysis when: -- Archive sync has had time to complete -- You've fixed parse failures or re-announced events -- You want to verify fixes before cutover - -```bash -# Re-run with existing Phase 2 data (faster) -./run-migration-analysis.sh ... --skip-phase-2 --output work/migration-analysis-20260122-1430 -``` - -## Individual Scripts - -For advanced usage, you can run individual phase scripts: - -```bash -# Phase 1: Fetch events -./migration-scripts/01-fetch-events.sh wss://relay.ngit.dev output/prod - -# Phase 2: Git sync check -./migration-scripts/10-check-git-sync.sh output/prod/raw/state-events.json /var/lib/ngit-relay/git output/prod --categorize - -# Phase 3a: Categorize -./migration-scripts/20-categorize.sh output/prod/git-sync-status.tsv output/prod - -# Phase 3b: Compare relays -./migration-scripts/21-compare-relays.sh output/prod output/archive output/comparison - -# Phase 4a: Extract parse failures -./migration-scripts/30-extract-parse-failures.sh ngit-grasp.service output/logs - -# Phase 4b: Extract purgatory expiry -./migration-scripts/31-extract-purgatory-expiry.sh ngit-grasp.service output/logs - -# Phase 5: Final classification -./migration-scripts/40-classify-actions.sh work/migration-analysis-20260122-1430 -``` - -Each script has detailed help available with `--help` or by reading the script header. diff --git a/docs/how-to/migrate-to-ngit-grasp.md b/docs/how-to/migrate-to-ngit-grasp.md new file mode 100644 index 0000000..f4dff86 --- /dev/null +++ b/docs/how-to/migrate-to-ngit-grasp.md @@ -0,0 +1,412 @@ +# Migrate to ngit-grasp from another GRASP implementation + +This guide walks you through migrating a production GRASP relay to ngit-grasp. The process involves analyzing your existing data to identify repositories that need attention before switching over. + +## Compatibility + +This migration process works with any GRASP implementation that: + +- Stores git data in the `/.git` directory structure +- Uses standard GRASP events (kind 30617 announcements, kind 30618 state, kind 5 deletions) +- Exposes a Nostr relay WebSocket endpoint + +**Known compatible implementations:** +- ngit-relay (reference implementation) +- ngit-grasp (when migrating between instances or from archive mode) +- Other GRASP-compliant relays following the specification + +The migration scripts analyze Nostr events and git data directly, making them implementation-agnostic. + +## Quick Start + +Run the migration analysis with a single command: + +```bash +# Basic analysis (fetches events, compares relays) +./docs/how-to/migration-scripts/run-migration-analysis.sh \ + --prod-relay wss://source-relay.example.com \ + --archive-relay wss://target-relay.example.com + +# Full analysis (includes git sync check - run on VPS) +./docs/how-to/migration-scripts/run-migration-analysis.sh \ + --prod-relay wss://source-relay.example.com \ + --archive-relay wss://target-relay.example.com \ + --prod-git /var/lib/grasp-relay/git \ + --archive-git /var/lib/ngit-grasp/git \ + --service ngit-grasp.service +``` + +The script produces three output files: +- `results/no-action-required.txt` - Repos ready for migration +- `results/action-required.txt` - Repos needing intervention +- `results/manual-investigation.txt` - Repos needing human review + +See [Running the Analysis](#running-the-analysis) for detailed options. + +## Prerequisites + +### Required Tools + +- **nak** - Nostr Army Knife for fetching events ([install](https://github.com/fiatjaf/nak)) +- **jq** - JSON processing (install via package manager) + +### For Full Analysis (VPS) + +- SSH access to the VPS running your source relay +- Read access to git data directories +- Access to systemd journal (for log extraction) + +### Verify Installation + +```bash +# Check required tools +nak --version +jq --version + +# Check optional tools (for VPS phases) +journalctl --version +``` + +## Migration Overview + +The migration process has three stages: + +### Stage 1: Deploy Archive Instance + +Deploy ngit-grasp alongside your production relay: + +1. Configure ngit-grasp with: + - `domain` set to `.internal` (temporary) + - `archiveService` set to your production domain + - Running on a different port + +2. Let it sync for ~1 hour to gather all events and git data + +### Stage 2: Analyze Data + +Run the migration analysis to identify: +- Repositories successfully migrated (no action needed) +- Repositories with incomplete data (need investigation) +- Repositories with parse failures (may need re-announcement) + +### Stage 3: Switch Over + +Once all issues are resolved: +1. Set `domain` to your production URL +2. Disable archive mode +3. Update your reverse proxy to point to ngit-grasp + +## Running the Analysis + +### Basic Usage + +```bash +# Preview what will happen (dry run) +./run-migration-analysis.sh \ + --prod-relay wss://source-relay.example.com \ + --archive-relay wss://target-relay.example.com \ + --dry-run + +# Run the analysis +./run-migration-analysis.sh \ + --prod-relay wss://source-relay.example.com \ + --archive-relay wss://target-relay.example.com +``` + +### Full Analysis on VPS + +```bash +./run-migration-analysis.sh \ + --prod-relay wss://source-relay.example.com \ + --archive-relay wss://target-relay.example.com \ + --prod-git /var/lib/grasp-relay/git \ + --archive-git /var/lib/ngit-grasp/git \ + --service ngit-grasp.service +``` + +### Phase Control + +Skip or run specific phases: + +```bash +# Skip Phase 2 (use cached git sync data) +./run-migration-analysis.sh ... --skip-phase-2 + +# Run only Phase 1 (fetch events) +./run-migration-analysis.sh ... --only-phase-1 + +# Resume from Phase 3 (using existing data) +./run-migration-analysis.sh ... --from-phase-3 --output work/migration-analysis-20260122-1430 +``` + +### All Options + +| Option | Description | +|--------|-------------| +| `--prod-relay ` | Source relay WebSocket URL (required) | +| `--archive-relay ` | Target relay WebSocket URL (required) | +| `--prod-git ` | Git base directory for prod (enables Phase 2) | +| `--archive-git ` | Git base directory for archive (enables Phase 2) | +| `--service ` | Systemd service name (enables Phase 4) | +| `--output ` | Output directory (default: auto-generated) | +| `--skip-phase-N` | Skip phase N (1-5) | +| `--only-phase-N` | Run only phase N | +| `--from-phase-N` | Start from phase N | +| `--dry-run` | Show what would be executed | +| `--continue-on-error` | Continue even if a phase fails | + +## Understanding Results + +### Summary File + +The `results/summary.txt` file provides an overview: + +``` +## Overview + +| Category | Count | Percentage | +|----------|-------|------------| +| No Action Required | 450 | 85.7% | +| Action Required | 52 | 9.9% | +| Manual Investigation | 23 | 4.4% | +``` + +### No Action Required + +Repositories in `no-action-required.txt` are ready for migration: + +``` +myrepo | npub1abc... | complete in both prod and archive +oldrepo | npub1def... | deleted by user +testrepo | npub1ghi... | empty/blank in both (user never pushed) +``` + +**Common reasons:** +- `complete in both prod and archive` - Successfully migrated +- `deleted by user` - User requested deletion (kind 5 event) +- `empty/blank in both` - No git data was ever pushed +- `purgatory expired` - System already handled the timeout + +### Action Required + +Repositories in `action-required.txt` need intervention: + +``` +myrepo | npub1abc... | complete in prod, missing from archive | trigger re-sync or investigate +otherrepo | npub1def... | incomplete in both (prod=cat3, archive=cat2) | investigate git data source +``` + +**Common actions:** +- **Re-sync needed**: Trigger the archive to re-fetch from the source +- **Wait for sync**: Archive sync may still be in progress +- **Investigate git source**: Original git data may be incomplete +- **Fix parse failure**: Event format issue, may need re-announcement + +### Manual Investigation + +Repositories in `manual-investigation.txt` have unusual states: + +``` +weirdrepo | npub1abc... | in archive (cat1) but not in prod | may be new announcement or deleted from prod +conflictrepo | npub1def... | complete in prod, missing from archive, parse failure logged | investigate parse failure +``` + +These require human judgment to determine the correct action. + +## Troubleshooting + +### "nak not found" + +Install nak from https://github.com/fiatjaf/nak: + +```bash +# Using Go +go install github.com/fiatjaf/nak@latest + +# Or download binary from releases +``` + +### "Permission denied" on git directories + +Run with sudo or ensure your user has read access: + +```bash +# Check permissions +ls -la /var/lib/grasp-relay/git + +# Run with sudo if needed +sudo ./run-migration-analysis.sh ... +``` + +### Phase 2 takes too long + +The git sync check processes each repository individually (~20 minutes total). To speed up iteration: + +1. Run Phase 2 once and save the output +2. Use `--skip-phase-2` for subsequent runs +3. Use `--from-phase-3` to re-run classification with existing data + +### No parse failures found + +This is expected if: +- ngit-grasp logging improvements aren't deployed yet +- No events actually failed to parse + +The analysis will continue without log data. + +### Event counts are multiples of 250 + +This suggests pagination may have failed. The scripts use `--paginate` by default, but if you see exactly 250, 500, 750 events, verify the relay is responding correctly. + +## Architecture + +### Analysis Phases + +The analysis is split into 5 modular phases: + +| Phase | Name | Time | Location | Description | +|-------|------|------|----------|-------------| +| 1 | Fetch Events | ~30s each | Local | Fetch events from both relays | +| 2 | Git Sync Check | ~20 min each | VPS | Compare state events to git data | +| 3 | Categorize & Compare | <1s | Local | Categorize and compare results | +| 4 | Extract Logs | <30s | VPS | Extract parse failures and purgatory expiry | +| 5 | Final Classification | <5s | Local | Combine all data into actionable results | + +### Phase Flow Diagram + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ PHASE 1: Fetch Events (~30s, local) │ +│ Fetches kind 30618 (state), 30617 (announcements), 5 (deletion) │ +│ Run twice: once for prod, once for archive │ +└─────────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ PHASE 2: Git Sync Check (~20 mins, VPS required) │ +│ Compares state event refs to actual git data on disk │ +│ Categorizes into: complete, empty, partial, no-match │ +└─────────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ PHASE 3: Categorize & Compare (fast, local) │ +│ Compares prod vs archive categories │ +│ Identifies gaps and sync issues │ +└─────────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ PHASE 4: Log-Based Categories (VPS required) │ +│ Extracts [PARSE_FAIL] and [PURGATORY_EXPIRED] from logs │ +│ Provides context for why repos failed to sync │ +└─────────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ PHASE 5: Final Classification (fast, local) │ +│ Combines all data sources │ +│ Outputs: no-action, action-required, manual-investigation │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Git Sync Categories + +Phase 2 categorizes repositories into 4 categories: + +| Category | Description | Meaning | +|----------|-------------|---------| +| 1 | Complete Match | All refs in state event match git data | +| 2 | Empty/Blank | No git data available | +| 3 | Partial Match | Some refs match, some don't | +| 4 | No Match | Git data exists but refs don't match | + +### Output Directory Structure + +``` +work/migration-analysis-YYYYMMDD-HHMM/ +├── prod/ +│ ├── raw/ +│ │ ├── state-events.json # Phase 1 +│ │ ├── announcements.json # Phase 1 +│ │ └── deletions.json # Phase 1 +│ ├── git-sync-status.tsv # Phase 2 +│ └── category*.txt # Phase 2/3 +├── archive/ +│ └── (same structure as prod) +├── comparison/ +│ ├── complete-in-both.txt # Phase 3 +│ ├── complete-prod-missing-archive.txt +│ ├── complete-prod-incomplete-archive.txt +│ ├── incomplete-in-both.txt +│ ├── in-archive-not-prod.txt +│ └── summary.txt +├── logs/ +│ ├── parse-failures.txt # Phase 4 +│ └── purgatory-expired.txt # Phase 4 +└── results/ + ├── no-action-required.txt # Phase 5 + ├── action-required.txt # Phase 5 + ├── manual-investigation.txt # Phase 5 + └── summary.txt # Phase 5 +``` + +## Why Migration May Require Attention + +Different GRASP implementations may handle edge cases differently. ngit-grasp has stricter validation and better observability, which can surface issues that were previously hidden: + +| Aspect | Typical Source Relay | ngit-grasp | +|--------|---------------------|------------| +| Git data validation | May accept partial data | Requires all git data to reproduce state | +| PR refs cleanup | May not clear `refs/nostr/` | Properly manages PR refs | +| Parse failures | May silently ignore | Logs structured `[PARSE_FAIL]` entries | +| Sync timeout | May have no timeout | Purgatory expires after configurable period | + +These differences explain why some repositories may need attention during migration - ngit-grasp's stricter validation catches issues that other implementations may have silently accepted. + +## Next Steps + +After running the analysis: + +1. **Review the summary** - Check `results/summary.txt` for the overview +2. **Address action items** - Work through `results/action-required.txt` +3. **Investigate edge cases** - Review `results/manual-investigation.txt` +4. **Re-run analysis** - After fixing issues, re-run to verify +5. **Plan cutover** - Schedule the switch when all issues are resolved + +### When to Re-run + +Re-run the analysis when: +- Archive sync has had time to complete +- You've fixed parse failures or re-announced events +- You want to verify fixes before cutover + +```bash +# Re-run with existing Phase 2 data (faster) +./run-migration-analysis.sh ... --skip-phase-2 --output work/migration-analysis-20260122-1430 +``` + +## Individual Scripts + +For advanced usage, you can run individual phase scripts: + +```bash +# Phase 1: Fetch events +./migration-scripts/01-fetch-events.sh wss://source-relay.example.com output/prod + +# Phase 2: Git sync check +./migration-scripts/10-check-git-sync.sh output/prod/raw/state-events.json /var/lib/grasp-relay/git output/prod --categorize + +# Phase 3a: Categorize +./migration-scripts/20-categorize.sh output/prod/git-sync-status.tsv output/prod + +# Phase 3b: Compare relays +./migration-scripts/21-compare-relays.sh output/prod output/archive output/comparison + +# Phase 4a: Extract parse failures +./migration-scripts/30-extract-parse-failures.sh ngit-grasp.service output/logs + +# Phase 4b: Extract purgatory expiry +./migration-scripts/31-extract-purgatory-expiry.sh ngit-grasp.service output/logs + +# Phase 5: Final classification +./migration-scripts/40-classify-actions.sh work/migration-analysis-20260122-1430 +``` + +Each script has detailed help available with `--help` or by reading the script header. diff --git a/docs/how-to/migration-scripts/01-fetch-events.sh b/docs/how-to/migration-scripts/01-fetch-events.sh index 6870659..e0d6f26 100755 --- a/docs/how-to/migration-scripts/01-fetch-events.sh +++ b/docs/how-to/migration-scripts/01-fetch-events.sh @@ -2,7 +2,7 @@ # # 01-fetch-events.sh - Fetch nostr events from a relay for migration analysis # -# PHASE 1 of the ngit-relay to ngit-grasp migration analysis pipeline. +# PHASE 1 of the GRASP relay to ngit-grasp migration analysis pipeline. # Fetches kind 30618 (state), 30617 (announcement), and 5 (deletion) events. # # USAGE: @@ -41,7 +41,7 @@ # - Run Phase 1 and Phase 2 back-to-back for accurate snapshot # # SEE ALSO: -# docs/how-to/migrate-ngit-relay-to-ngit-grasp.md - Full migration guide +# docs/how-to/migrate-to-ngit-grasp.md - Full migration guide # set -euo pipefail diff --git a/docs/how-to/migration-scripts/10-check-git-sync.sh b/docs/how-to/migration-scripts/10-check-git-sync.sh index 493d50a..1f2ce04 100755 --- a/docs/how-to/migration-scripts/10-check-git-sync.sh +++ b/docs/how-to/migration-scripts/10-check-git-sync.sh @@ -2,21 +2,21 @@ # # 10-check-git-sync.sh - Compare state events to actual git data on disk # -# PHASE 2 of the ngit-relay to ngit-grasp migration analysis pipeline. +# PHASE 2 of the GRASP relay to ngit-grasp migration analysis pipeline. # Compares kind 30618 state events against actual git refs on disk. # # USAGE: # ./10-check-git-sync.sh [--categorize] # # EXAMPLES: -# # Check prod relay against prod git data -# ./10-check-git-sync.sh output/prod/raw/state-events.json /var/lib/ngit-relay/git output/prod +# # Check source relay against source git data +# ./10-check-git-sync.sh output/prod/raw/state-events.json /var/lib/grasp-relay/git output/prod # -# # Check archive relay against archive git data -# ./10-check-git-sync.sh output/archive/raw/state-events.json /var/lib/ngit-relay-archive/git output/archive +# # Check target relay against target git data +# ./10-check-git-sync.sh output/archive/raw/state-events.json /var/lib/ngit-grasp/git output/archive # # # Check and categorize in one step (convenience mode) -# ./10-check-git-sync.sh output/prod/raw/state-events.json /var/lib/ngit-relay/git output/prod --categorize +# ./10-check-git-sync.sh output/prod/raw/state-events.json /var/lib/grasp-relay/git output/prod --categorize # # INPUT: # state-events.json - JSONL file from Phase 1 (01-fetch-events.sh) @@ -53,7 +53,7 @@ # - Handles packed refs (git show-ref) and loose refs # # SEE ALSO: -# docs/how-to/migrate-ngit-relay-to-ngit-grasp.md - Full migration guide +# docs/how-to/migrate-to-ngit-grasp.md - Full migration guide # 01-fetch-events.sh - Phase 1 script that produces input for this script # 20-categorize.sh - Phase 3a script that consumes output from this script # @@ -101,13 +101,13 @@ usage() { echo "" echo "Arguments:" echo " state-events.json JSONL file from Phase 1 (kind 30618 events)" - echo " git-base-dir Base directory for git repos (e.g., /var/lib/ngit-relay/git)" + echo " git-base-dir Base directory for git repos (e.g., /var/lib/grasp-relay/git)" echo " output-dir Directory to store output files" echo " --categorize Optional: also output category files (like Phase 3)" echo "" echo "Examples:" - echo " $0 output/prod/raw/state-events.json /var/lib/ngit-relay/git output/prod" - echo " $0 output/archive/raw/state-events.json /var/lib/ngit-relay-archive/git output/archive" + echo " $0 output/prod/raw/state-events.json /var/lib/grasp-relay/git output/prod" + echo " $0 output/archive/raw/state-events.json /var/lib/ngit-grasp/git output/archive" echo "" echo "Output:" echo " git-sync-status.tsv - TSV with: repo, npub, state_refs, git_refs, matches, reason" diff --git a/docs/how-to/migration-scripts/20-categorize.sh b/docs/how-to/migration-scripts/20-categorize.sh index f47eb55..b38dc00 100755 --- a/docs/how-to/migration-scripts/20-categorize.sh +++ b/docs/how-to/migration-scripts/20-categorize.sh @@ -2,7 +2,7 @@ # # 20-categorize.sh - Categorize git sync status into 4 categories # -# PHASE 3a of the ngit-relay to ngit-grasp migration analysis pipeline. +# PHASE 3a of the GRASP relay to ngit-grasp migration analysis pipeline. # Takes git-sync-status.tsv from Phase 2 and categorizes into 4 files. # # USAGE: @@ -39,7 +39,7 @@ # RUNTIME: < 1 second (local processing only) # # SEE ALSO: -# docs/how-to/migrate-ngit-relay-to-ngit-grasp.md - Full migration guide +# docs/how-to/migrate-to-ngit-grasp.md - Full migration guide # 10-check-git-sync.sh - Phase 2 script that produces input for this script # diff --git a/docs/how-to/migration-scripts/21-compare-relays.sh b/docs/how-to/migration-scripts/21-compare-relays.sh index 6b40dc8..b9c0d30 100755 --- a/docs/how-to/migration-scripts/21-compare-relays.sh +++ b/docs/how-to/migration-scripts/21-compare-relays.sh @@ -2,7 +2,7 @@ # # 21-compare-relays.sh - Compare prod vs archive category files to find gaps # -# PHASE 3b of the ngit-relay to ngit-grasp migration analysis pipeline. +# PHASE 3b of the GRASP relay to ngit-grasp migration analysis pipeline. # Compares categorized output from prod and archive to identify: # - Repos complete in prod but missing/incomplete in archive # - Repos in archive but not in prod @@ -39,7 +39,7 @@ # RUNTIME: < 1 second (local processing only) # # SEE ALSO: -# docs/how-to/migrate-ngit-relay-to-ngit-grasp.md - Full migration guide +# docs/how-to/migrate-to-ngit-grasp.md - Full migration guide # 20-categorize.sh - Phase 3a script that produces input for this script # diff --git a/docs/how-to/migration-scripts/30-extract-parse-failures.sh b/docs/how-to/migration-scripts/30-extract-parse-failures.sh index 753fd3e..bc2049a 100755 --- a/docs/how-to/migration-scripts/30-extract-parse-failures.sh +++ b/docs/how-to/migration-scripts/30-extract-parse-failures.sh @@ -2,7 +2,7 @@ # # 30-extract-parse-failures.sh - Extract parse failure events from systemd logs # -# PHASE 4a of the ngit-relay to ngit-grasp migration analysis pipeline. +# PHASE 4a of the GRASP relay to ngit-grasp migration analysis pipeline. # Extracts structured [PARSE_FAIL] log entries from journalctl. # # USAGE: @@ -42,7 +42,7 @@ # [PARSE_FAIL] log entries. Until those are implemented, this script will # find no matching entries (which is handled gracefully). # -# See: docs/how-to/migrate-ngit-relay-to-ngit-grasp.md (Dependencies section) +# See: docs/how-to/migrate-to-ngit-grasp.md (Dependencies section) # # Expected Rust logging code: # tracing::warn!( @@ -59,7 +59,7 @@ # RUNTIME: Depends on log volume, typically < 30 seconds # # SEE ALSO: -# docs/how-to/migrate-ngit-relay-to-ngit-grasp.md - Full migration guide +# docs/how-to/migrate-to-ngit-grasp.md - Full migration guide # 31-extract-purgatory-expiry.sh - Companion script for purgatory expiry logs # @@ -231,7 +231,7 @@ main() { if [[ "$sample_count" -eq 0 ]]; then log_warn "No [PARSE_FAIL] entries found in logs." log_warn "This is expected if ngit-grasp logging improvements are not yet deployed." - log_warn "See: docs/how-to/migrate-ngit-relay-to-ngit-grasp.md (Dependencies section)" + log_warn "See: docs/how-to/migrate-to-ngit-grasp.md (Dependencies section)" fi exit 0 @@ -259,7 +259,7 @@ main() { log_warn "" log_warn " [PARSE_FAIL] kind=30618 event_id=abc123 reason=\"...\" repo=myrepo npub=npub1..." log_warn "" - log_warn "See: docs/how-to/migrate-ngit-relay-to-ngit-grasp.md (Dependencies section)" + log_warn "See: docs/how-to/migrate-to-ngit-grasp.md (Dependencies section)" log_warn "" # Create empty output file with header comment diff --git a/docs/how-to/migration-scripts/31-extract-purgatory-expiry.sh b/docs/how-to/migration-scripts/31-extract-purgatory-expiry.sh index 38b2ca3..8cadad9 100755 --- a/docs/how-to/migration-scripts/31-extract-purgatory-expiry.sh +++ b/docs/how-to/migration-scripts/31-extract-purgatory-expiry.sh @@ -2,7 +2,7 @@ # # 31-extract-purgatory-expiry.sh - Extract purgatory expiry events from systemd logs # -# PHASE 4b of the ngit-relay to ngit-grasp migration analysis pipeline. +# PHASE 4b of the GRASP relay to ngit-grasp migration analysis pipeline. # Extracts structured [PURGATORY_EXPIRED] log entries from journalctl. # # USAGE: @@ -53,7 +53,7 @@ # [PURGATORY_EXPIRED] log entries. Until those are implemented, this script # will find no matching entries (which is handled gracefully). # -# See: docs/how-to/migrate-ngit-relay-to-ngit-grasp.md (Dependencies section) +# See: docs/how-to/migrate-to-ngit-grasp.md (Dependencies section) # # Expected Rust logging code: # tracing::warn!( @@ -70,7 +70,7 @@ # RUNTIME: Depends on log volume, typically < 30 seconds # # SEE ALSO: -# docs/how-to/migrate-ngit-relay-to-ngit-grasp.md - Full migration guide +# docs/how-to/migrate-to-ngit-grasp.md - Full migration guide # 30-extract-parse-failures.sh - Companion script for parse failure logs # @@ -238,7 +238,7 @@ main() { if [[ "$sample_count" -eq 0 ]]; then log_warn "No [PURGATORY_EXPIRED] entries found in logs." log_warn "This is expected if ngit-grasp logging improvements are not yet deployed." - log_warn "See: docs/how-to/migrate-ngit-relay-to-ngit-grasp.md (Dependencies section)" + log_warn "See: docs/how-to/migrate-to-ngit-grasp.md (Dependencies section)" fi exit 0 @@ -266,7 +266,7 @@ main() { log_warn "" log_warn " [PURGATORY_EXPIRED] repo=myrepo npub=npub1... reason=\"...\"" log_warn "" - log_warn "See: docs/how-to/migrate-ngit-relay-to-ngit-grasp.md (Dependencies section)" + log_warn "See: docs/how-to/migrate-to-ngit-grasp.md (Dependencies section)" log_warn "" # Create empty output file with header comment diff --git a/docs/how-to/migration-scripts/40-classify-actions.sh b/docs/how-to/migration-scripts/40-classify-actions.sh index 9fc718f..1706e47 100755 --- a/docs/how-to/migration-scripts/40-classify-actions.sh +++ b/docs/how-to/migration-scripts/40-classify-actions.sh @@ -2,7 +2,7 @@ # # 40-classify-actions.sh - Final classification of repos for migration action # -# PHASE 5 of the ngit-relay to ngit-grasp migration analysis pipeline. +# PHASE 5 of the GRASP relay to ngit-grasp migration analysis pipeline. # Combines all data sources from previous phases to produce actionable results. # # USAGE: @@ -82,7 +82,7 @@ # RUNTIME: < 5 seconds (local processing only) # # SEE ALSO: -# docs/how-to/migrate-ngit-relay-to-ngit-grasp.md - Full migration guide +# docs/how-to/migrate-to-ngit-grasp.md - Full migration guide # 01-fetch-events.sh - Phase 1 (fetch events) # 10-check-git-sync.sh - Phase 2 (git sync check) # 20-categorize.sh, 21-compare-relays.sh - Phase 3 (categorize and compare) diff --git a/docs/how-to/migration-scripts/run-migration-analysis.sh b/docs/how-to/migration-scripts/run-migration-analysis.sh index 2ba25ab..84aec2a 100755 --- a/docs/how-to/migration-scripts/run-migration-analysis.sh +++ b/docs/how-to/migration-scripts/run-migration-analysis.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# run-migration-analysis.sh - Orchestrate the complete ngit-relay to ngit-grasp migration analysis +# run-migration-analysis.sh - Orchestrate the complete GRASP relay to ngit-grasp migration analysis # # This script runs all 5 phases of the migration analysis pipeline in sequence, # with proper error handling, progress reporting, and timing information. @@ -13,8 +13,8 @@ # ./run-migration-analysis.sh \ # --prod-relay wss://relay.ngit.dev \ # --archive-relay wss://archive.relay.ngit.dev \ -# --prod-git /var/lib/ngit-relay/git \ -# --archive-git /var/lib/ngit-relay-archive/git +# --prod-git /var/lib/grasp-relay/git \ +# --archive-git /var/lib/ngit-grasp/git # # USAGE: # ./run-migration-analysis.sh [options] @@ -64,12 +64,12 @@ # ./run-migration-analysis.sh \ # --prod-relay wss://relay.ngit.dev \ # --archive-relay wss://archive.relay.ngit.dev \ -# --prod-git /var/lib/ngit-relay/git \ -# --archive-git /var/lib/ngit-relay-archive/git \ +# --prod-git /var/lib/grasp-relay/git \ +# --archive-git /var/lib/ngit-grasp/git \ # --service ngit-grasp.service # # SEE ALSO: -# docs/how-to/migrate-ngit-relay-to-ngit-grasp.md - Full migration guide +# docs/how-to/migrate-to-ngit-grasp.md - Full migration guide # set -euo pipefail @@ -658,7 +658,7 @@ display_summary() { main() { parse_args "$@" - log_header "ngit-relay to ngit-grasp Migration Analysis" + log_header "GRASP Relay to ngit-grasp Migration Analysis" validate_args check_prerequisites -- cgit v1.2.3