From 7536160c0ab1b64090ba9b5ab8ea6aef4747bb48 Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Fri, 23 Jan 2026 10:56:12 +0000 Subject: Add Phase 1 migration script to fetch events from relay - Fetches kind 30618 (state), 30617 (announcement), 5 (deletion) events - Uses nak req --paginate for complete event retrieval - Outputs JSONL format for downstream processing - Includes error handling and timing information --- docs/how-to/migration-scripts/01-fetch-events.sh | 206 +++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100755 docs/how-to/migration-scripts/01-fetch-events.sh (limited to 'docs/how-to/migration-scripts') diff --git a/docs/how-to/migration-scripts/01-fetch-events.sh b/docs/how-to/migration-scripts/01-fetch-events.sh new file mode 100755 index 0000000..6870659 --- /dev/null +++ b/docs/how-to/migration-scripts/01-fetch-events.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# +# 01-fetch-events.sh - Fetch nostr events from a relay for migration analysis +# +# PHASE 1 of the ngit-relay to ngit-grasp migration analysis pipeline. +# Fetches kind 30618 (state), 30617 (announcement), and 5 (deletion) events. +# +# USAGE: +# ./01-fetch-events.sh +# +# EXAMPLES: +# # Fetch from production relay +# ./01-fetch-events.sh wss://relay.ngit.dev output/prod +# +# # Fetch from archive relay +# ./01-fetch-events.sh wss://archive.relay.ngit.dev output/archive +# +# # Full migration analysis setup +# mkdir -p work/migration-analysis-$(date +%Y%m%d-%H%M) +# ./01-fetch-events.sh wss://relay.ngit.dev work/migration-analysis-*/prod +# ./01-fetch-events.sh wss://archive.relay.ngit.dev work/migration-analysis-*/archive +# +# OUTPUT: +# /raw/state-events.json - kind 30618 events (one per line, JSONL) +# /raw/announcements.json - kind 30617 events (one per line, JSONL) +# /raw/deletions.json - kind 5 events (one per line, JSONL) +# +# OUTPUT FORMAT: +# Each file contains one JSON event per line (JSONL format). +# Events are the raw nostr event objects as returned by the relay. +# +# PREREQUISITES: +# - nak (Nostr Army Knife) - https://github.com/fiatjaf/nak +# - jq (for counting/validation) +# +# RUNTIME: ~30 seconds per relay (depends on network and event count) +# +# NOTES: +# - Uses --paginate to ensure all events are fetched (not just first page) +# - If event counts are exact multiples of 250, pagination may have failed +# - Run Phase 1 and Phase 2 back-to-back for accurate snapshot +# +# SEE ALSO: +# docs/how-to/migrate-ngit-relay-to-ngit-grasp.md - Full migration guide +# + +set -euo pipefail + +# Colors for output (disabled if not a terminal) +if [[ -t 1 ]]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[0;33m' + BLUE='\033[0;34m' + NC='\033[0m' # No Color +else + RED='' + GREEN='' + YELLOW='' + BLUE='' + NC='' +fi + +log_info() { + echo -e "${BLUE}[INFO]${NC} $*" >&2 +} + +log_success() { + echo -e "${GREEN}[OK]${NC} $*" >&2 +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $*" >&2 +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $*" >&2 +} + +usage() { + echo "Usage: $0 " + echo "" + echo "Arguments:" + echo " relay-url WebSocket URL of the relay (e.g., wss://relay.ngit.dev)" + echo " output-dir Directory to store fetched events (e.g., output/prod)" + echo "" + echo "Examples:" + echo " $0 wss://relay.ngit.dev output/prod" + echo " $0 wss://archive.relay.ngit.dev output/archive" + exit 1 +} + +# Check prerequisites +check_prerequisites() { + local missing=0 + + if ! command -v nak &> /dev/null; then + log_error "nak not found. Install from: https://github.com/fiatjaf/nak" + missing=1 + fi + + if ! command -v jq &> /dev/null; then + log_error "jq not found. Install with your package manager." + missing=1 + fi + + if [[ $missing -eq 1 ]]; then + exit 1 + fi +} + +# Fetch events of a specific kind +# Args: $1=relay, $2=kind, $3=output_file, $4=description +fetch_kind() { + local relay="$1" + local kind="$2" + local output_file="$3" + local description="$4" + + log_info "Fetching $description (kind $kind) from $relay..." + + local start_time + start_time=$(date +%s) + + # Use --paginate to ensure we get all events, not just first page + # nak outputs one event per line (JSONL format) + if ! nak req -k "$kind" --paginate "$relay" > "$output_file" 2>/dev/null; then + log_error "Failed to fetch $description from $relay" + return 1 + fi + + local end_time + end_time=$(date +%s) + local duration=$((end_time - start_time)) + + # Count events + local count + count=$(wc -l < "$output_file" | tr -d ' ') + + # Warn if count is suspicious (exact multiple of 250 suggests pagination issue) + if [[ $count -gt 0 ]] && [[ $((count % 250)) -eq 0 ]]; then + log_warn "$description count ($count) is exact multiple of 250 - pagination may have failed!" + fi + + log_success "Fetched $count $description in ${duration}s -> $output_file" + + echo "$count" +} + +# Main +main() { + if [[ $# -ne 2 ]]; then + usage + fi + + local relay="$1" + local output_dir="$2" + + # Validate relay URL + if [[ ! "$relay" =~ ^wss?:// ]]; then + log_error "Invalid relay URL: $relay (must start with ws:// or wss://)" + exit 1 + fi + + check_prerequisites + + log_info "Starting event fetch from $relay" + log_info "Output directory: $output_dir" + + # Create output directory structure + local raw_dir="$output_dir/raw" + mkdir -p "$raw_dir" + + local total_start + total_start=$(date +%s) + + # Fetch each event type + local state_count announcement_count deletion_count + + state_count=$(fetch_kind "$relay" 30618 "$raw_dir/state-events.json" "state events") + announcement_count=$(fetch_kind "$relay" 30617 "$raw_dir/announcements.json" "announcements") + deletion_count=$(fetch_kind "$relay" 5 "$raw_dir/deletions.json" "deletion requests") + + local total_end + total_end=$(date +%s) + local total_duration=$((total_end - total_start)) + + # Summary + echo "" + log_info "=== Fetch Summary ===" + log_info "Relay: $relay" + log_info "Output: $output_dir" + log_info "State events (30618): $state_count" + log_info "Announcements (30617): $announcement_count" + log_info "Deletions (5): $deletion_count" + log_info "Total time: ${total_duration}s" + echo "" + + # Output file listing for easy copy/paste + log_info "Output files:" + echo " $raw_dir/state-events.json" + echo " $raw_dir/announcements.json" + echo " $raw_dir/deletions.json" +} + +main "$@" -- cgit v1.2.3