diff options
Diffstat (limited to 'docs/archive/2026-01-relay-ngit-dev-migration/scripts/01-fetch-events.sh')
| -rwxr-xr-x | docs/archive/2026-01-relay-ngit-dev-migration/scripts/01-fetch-events.sh | 206 |
1 files changed, 206 insertions, 0 deletions
diff --git a/docs/archive/2026-01-relay-ngit-dev-migration/scripts/01-fetch-events.sh b/docs/archive/2026-01-relay-ngit-dev-migration/scripts/01-fetch-events.sh new file mode 100755 index 0000000..e0d6f26 --- /dev/null +++ b/docs/archive/2026-01-relay-ngit-dev-migration/scripts/01-fetch-events.sh | |||
| @@ -0,0 +1,206 @@ | |||
| 1 | #!/usr/bin/env bash | ||
| 2 | # | ||
| 3 | # 01-fetch-events.sh - Fetch nostr events from a relay for migration analysis | ||
| 4 | # | ||
| 5 | # PHASE 1 of the GRASP relay to ngit-grasp migration analysis pipeline. | ||
| 6 | # Fetches kind 30618 (state), 30617 (announcement), and 5 (deletion) events. | ||
| 7 | # | ||
| 8 | # USAGE: | ||
| 9 | # ./01-fetch-events.sh <relay-url> <output-dir> | ||
| 10 | # | ||
| 11 | # EXAMPLES: | ||
| 12 | # # Fetch from production relay | ||
| 13 | # ./01-fetch-events.sh wss://relay.ngit.dev output/prod | ||
| 14 | # | ||
| 15 | # # Fetch from archive relay | ||
| 16 | # ./01-fetch-events.sh wss://archive.relay.ngit.dev output/archive | ||
| 17 | # | ||
| 18 | # # Full migration analysis setup | ||
| 19 | # mkdir -p work/migration-analysis-$(date +%Y%m%d-%H%M) | ||
| 20 | # ./01-fetch-events.sh wss://relay.ngit.dev work/migration-analysis-*/prod | ||
| 21 | # ./01-fetch-events.sh wss://archive.relay.ngit.dev work/migration-analysis-*/archive | ||
| 22 | # | ||
| 23 | # OUTPUT: | ||
| 24 | # <output-dir>/raw/state-events.json - kind 30618 events (one per line, JSONL) | ||
| 25 | # <output-dir>/raw/announcements.json - kind 30617 events (one per line, JSONL) | ||
| 26 | # <output-dir>/raw/deletions.json - kind 5 events (one per line, JSONL) | ||
| 27 | # | ||
| 28 | # OUTPUT FORMAT: | ||
| 29 | # Each file contains one JSON event per line (JSONL format). | ||
| 30 | # Events are the raw nostr event objects as returned by the relay. | ||
| 31 | # | ||
| 32 | # PREREQUISITES: | ||
| 33 | # - nak (Nostr Army Knife) - https://github.com/fiatjaf/nak | ||
| 34 | # - jq (for counting/validation) | ||
| 35 | # | ||
| 36 | # RUNTIME: ~30 seconds per relay (depends on network and event count) | ||
| 37 | # | ||
| 38 | # NOTES: | ||
| 39 | # - Uses --paginate to ensure all events are fetched (not just first page) | ||
| 40 | # - If event counts are exact multiples of 250, pagination may have failed | ||
| 41 | # - Run Phase 1 and Phase 2 back-to-back for accurate snapshot | ||
| 42 | # | ||
| 43 | # SEE ALSO: | ||
| 44 | # docs/how-to/migrate-to-ngit-grasp.md - Full migration guide | ||
| 45 | # | ||
| 46 | |||
| 47 | set -euo pipefail | ||
| 48 | |||
| 49 | # Colors for output (disabled if not a terminal) | ||
| 50 | if [[ -t 1 ]]; then | ||
| 51 | RED='\033[0;31m' | ||
| 52 | GREEN='\033[0;32m' | ||
| 53 | YELLOW='\033[0;33m' | ||
| 54 | BLUE='\033[0;34m' | ||
| 55 | NC='\033[0m' # No Color | ||
| 56 | else | ||
| 57 | RED='' | ||
| 58 | GREEN='' | ||
| 59 | YELLOW='' | ||
| 60 | BLUE='' | ||
| 61 | NC='' | ||
| 62 | fi | ||
| 63 | |||
| 64 | log_info() { | ||
| 65 | echo -e "${BLUE}[INFO]${NC} $*" >&2 | ||
| 66 | } | ||
| 67 | |||
| 68 | log_success() { | ||
| 69 | echo -e "${GREEN}[OK]${NC} $*" >&2 | ||
| 70 | } | ||
| 71 | |||
| 72 | log_warn() { | ||
| 73 | echo -e "${YELLOW}[WARN]${NC} $*" >&2 | ||
| 74 | } | ||
| 75 | |||
| 76 | log_error() { | ||
| 77 | echo -e "${RED}[ERROR]${NC} $*" >&2 | ||
| 78 | } | ||
| 79 | |||
| 80 | usage() { | ||
| 81 | echo "Usage: $0 <relay-url> <output-dir>" | ||
| 82 | echo "" | ||
| 83 | echo "Arguments:" | ||
| 84 | echo " relay-url WebSocket URL of the relay (e.g., wss://relay.ngit.dev)" | ||
| 85 | echo " output-dir Directory to store fetched events (e.g., output/prod)" | ||
| 86 | echo "" | ||
| 87 | echo "Examples:" | ||
| 88 | echo " $0 wss://relay.ngit.dev output/prod" | ||
| 89 | echo " $0 wss://archive.relay.ngit.dev output/archive" | ||
| 90 | exit 1 | ||
| 91 | } | ||
| 92 | |||
| 93 | # Check prerequisites | ||
| 94 | check_prerequisites() { | ||
| 95 | local missing=0 | ||
| 96 | |||
| 97 | if ! command -v nak &> /dev/null; then | ||
| 98 | log_error "nak not found. Install from: https://github.com/fiatjaf/nak" | ||
| 99 | missing=1 | ||
| 100 | fi | ||
| 101 | |||
| 102 | if ! command -v jq &> /dev/null; then | ||
| 103 | log_error "jq not found. Install with your package manager." | ||
| 104 | missing=1 | ||
| 105 | fi | ||
| 106 | |||
| 107 | if [[ $missing -eq 1 ]]; then | ||
| 108 | exit 1 | ||
| 109 | fi | ||
| 110 | } | ||
| 111 | |||
| 112 | # Fetch events of a specific kind | ||
| 113 | # Args: $1=relay, $2=kind, $3=output_file, $4=description | ||
| 114 | fetch_kind() { | ||
| 115 | local relay="$1" | ||
| 116 | local kind="$2" | ||
| 117 | local output_file="$3" | ||
| 118 | local description="$4" | ||
| 119 | |||
| 120 | log_info "Fetching $description (kind $kind) from $relay..." | ||
| 121 | |||
| 122 | local start_time | ||
| 123 | start_time=$(date +%s) | ||
| 124 | |||
| 125 | # Use --paginate to ensure we get all events, not just first page | ||
| 126 | # nak outputs one event per line (JSONL format) | ||
| 127 | if ! nak req -k "$kind" --paginate "$relay" > "$output_file" 2>/dev/null; then | ||
| 128 | log_error "Failed to fetch $description from $relay" | ||
| 129 | return 1 | ||
| 130 | fi | ||
| 131 | |||
| 132 | local end_time | ||
| 133 | end_time=$(date +%s) | ||
| 134 | local duration=$((end_time - start_time)) | ||
| 135 | |||
| 136 | # Count events | ||
| 137 | local count | ||
| 138 | count=$(wc -l < "$output_file" | tr -d ' ') | ||
| 139 | |||
| 140 | # Warn if count is suspicious (exact multiple of 250 suggests pagination issue) | ||
| 141 | if [[ $count -gt 0 ]] && [[ $((count % 250)) -eq 0 ]]; then | ||
| 142 | log_warn "$description count ($count) is exact multiple of 250 - pagination may have failed!" | ||
| 143 | fi | ||
| 144 | |||
| 145 | log_success "Fetched $count $description in ${duration}s -> $output_file" | ||
| 146 | |||
| 147 | echo "$count" | ||
| 148 | } | ||
| 149 | |||
| 150 | # Main | ||
| 151 | main() { | ||
| 152 | if [[ $# -ne 2 ]]; then | ||
| 153 | usage | ||
| 154 | fi | ||
| 155 | |||
| 156 | local relay="$1" | ||
| 157 | local output_dir="$2" | ||
| 158 | |||
| 159 | # Validate relay URL | ||
| 160 | if [[ ! "$relay" =~ ^wss?:// ]]; then | ||
| 161 | log_error "Invalid relay URL: $relay (must start with ws:// or wss://)" | ||
| 162 | exit 1 | ||
| 163 | fi | ||
| 164 | |||
| 165 | check_prerequisites | ||
| 166 | |||
| 167 | log_info "Starting event fetch from $relay" | ||
| 168 | log_info "Output directory: $output_dir" | ||
| 169 | |||
| 170 | # Create output directory structure | ||
| 171 | local raw_dir="$output_dir/raw" | ||
| 172 | mkdir -p "$raw_dir" | ||
| 173 | |||
| 174 | local total_start | ||
| 175 | total_start=$(date +%s) | ||
| 176 | |||
| 177 | # Fetch each event type | ||
| 178 | local state_count announcement_count deletion_count | ||
| 179 | |||
| 180 | state_count=$(fetch_kind "$relay" 30618 "$raw_dir/state-events.json" "state events") | ||
| 181 | announcement_count=$(fetch_kind "$relay" 30617 "$raw_dir/announcements.json" "announcements") | ||
| 182 | deletion_count=$(fetch_kind "$relay" 5 "$raw_dir/deletions.json" "deletion requests") | ||
| 183 | |||
| 184 | local total_end | ||
| 185 | total_end=$(date +%s) | ||
| 186 | local total_duration=$((total_end - total_start)) | ||
| 187 | |||
| 188 | # Summary | ||
| 189 | echo "" | ||
| 190 | log_info "=== Fetch Summary ===" | ||
| 191 | log_info "Relay: $relay" | ||
| 192 | log_info "Output: $output_dir" | ||
| 193 | log_info "State events (30618): $state_count" | ||
| 194 | log_info "Announcements (30617): $announcement_count" | ||
| 195 | log_info "Deletions (5): $deletion_count" | ||
| 196 | log_info "Total time: ${total_duration}s" | ||
| 197 | echo "" | ||
| 198 | |||
| 199 | # Output file listing for easy copy/paste | ||
| 200 | log_info "Output files:" | ||
| 201 | echo " $raw_dir/state-events.json" | ||
| 202 | echo " $raw_dir/announcements.json" | ||
| 203 | echo " $raw_dir/deletions.json" | ||
| 204 | } | ||
| 205 | |||
| 206 | main "$@" | ||