From 28cc7820953efeafb2bc4d41ebcf3d682da86711 Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Fri, 23 Jan 2026 11:16:50 +0000 Subject: Add Phase 4 migration scripts for log extraction - 30-extract-parse-failures.sh: Extracts parse failure events from logs - 31-extract-purgatory-expiry.sh: Extracts purgatory expiry events from logs - Both support time range filtering (--since, --until) - Includes dry-run mode for testing - Gracefully handles missing logs with dependency notes - TSV output format for Phase 5 consumption - Ready for when structured logging is implemented in ngit-grasp --- .../migration-scripts/30-extract-parse-failures.sh | 328 +++++++++++++++++++ .../31-extract-purgatory-expiry.sh | 346 +++++++++++++++++++++ 2 files changed, 674 insertions(+) create mode 100755 docs/how-to/migration-scripts/30-extract-parse-failures.sh create mode 100755 docs/how-to/migration-scripts/31-extract-purgatory-expiry.sh (limited to 'docs/how-to/migration-scripts') diff --git a/docs/how-to/migration-scripts/30-extract-parse-failures.sh b/docs/how-to/migration-scripts/30-extract-parse-failures.sh new file mode 100755 index 0000000..753fd3e --- /dev/null +++ b/docs/how-to/migration-scripts/30-extract-parse-failures.sh @@ -0,0 +1,328 @@ +#!/usr/bin/env bash +# +# 30-extract-parse-failures.sh - Extract parse failure events from systemd logs +# +# PHASE 4a of the ngit-relay to ngit-grasp migration analysis pipeline. +# Extracts structured [PARSE_FAIL] log entries from journalctl. +# +# USAGE: +# ./30-extract-parse-failures.sh [options] +# +# EXAMPLES: +# # Extract from ngit-grasp service (last 30 days, default) +# ./30-extract-parse-failures.sh ngit-grasp.service output/logs +# +# # Extract with custom time range +# ./30-extract-parse-failures.sh ngit-grasp.service output/logs --since "2026-01-01" +# +# # Extract from specific time window +# ./30-extract-parse-failures.sh ngit-grasp.service output/logs --since "2026-01-15" --until "2026-01-22" +# +# OPTIONS: +# --since Start date for log extraction (default: 30 days ago) +# --until End date for log extraction (default: now) +# --dry-run Show what would be extracted without writing files +# +# OUTPUT: +# /parse-failures.txt +# +# OUTPUT FORMAT (TSV): +# reponpubkindevent_idreason +# +# EXPECTED LOG FORMAT: +# The script looks for structured log entries in this format: +# +# 2026-01-22T10:30:45Z ngit-grasp[1234]: [PARSE_FAIL] kind=30618 event_id=abc123... reason="invalid refs format" repo=myrepo npub=npub1... +# +# Required fields: kind, event_id, reason +# Optional fields: repo, npub (may not be available if parsing failed early) +# +# DEPENDENCY: +# This script requires logging improvements in ngit-grasp to emit structured +# [PARSE_FAIL] log entries. Until those are implemented, this script will +# find no matching entries (which is handled gracefully). +# +# See: docs/how-to/migrate-ngit-relay-to-ngit-grasp.md (Dependencies section) +# +# Expected Rust logging code: +# tracing::warn!( +# target: "migration", +# "[PARSE_FAIL] kind={} event_id={} reason=\"{}\" repo={} npub={}", +# event.kind, event.id, reason, identifier, npub +# ); +# +# PREREQUISITES: +# - journalctl (systemd) +# - grep, awk (standard Unix tools) +# - Access to systemd journal (may require sudo or journal group membership) +# +# RUNTIME: Depends on log volume, typically < 30 seconds +# +# SEE ALSO: +# docs/how-to/migrate-ngit-relay-to-ngit-grasp.md - Full migration guide +# 31-extract-purgatory-expiry.sh - Companion script for purgatory expiry logs +# + +set -euo pipefail + +# Colors for output (disabled if not a terminal) +if [[ -t 1 ]]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[0;33m' + BLUE='\033[0;34m' + NC='\033[0m' +else + RED='' + GREEN='' + YELLOW='' + BLUE='' + NC='' +fi + +log_info() { + echo -e "${BLUE}[INFO]${NC} $*" >&2 +} + +log_success() { + echo -e "${GREEN}[OK]${NC} $*" >&2 +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $*" >&2 +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $*" >&2 +} + +usage() { + echo "Usage: $0 [options]" + echo "" + echo "Arguments:" + echo " service-name Systemd service name (e.g., ngit-grasp.service)" + echo " output-dir Directory to store extracted log data" + echo "" + echo "Options:" + echo " --since Start date (default: 30 days ago)" + echo " --until End date (default: now)" + echo " --dry-run Show what would be extracted without writing" + echo "" + echo "Examples:" + echo " $0 ngit-grasp.service output/logs" + echo " $0 ngit-grasp.service output/logs --since '2026-01-01'" + echo " $0 ngit-grasp.service output/logs --since '2026-01-15' --until '2026-01-22'" + echo "" + echo "Expected log format:" + echo " [PARSE_FAIL] kind=30618 event_id=abc123 reason=\"...\" repo=myrepo npub=npub1..." + exit 1 +} + +# Parse a single log line and extract fields +# Input: log line containing [PARSE_FAIL] +# Output: TSV line: reponpubkindevent_idreason +parse_log_line() { + local line="$1" + + # Extract fields using grep -oP (Perl regex) or awk + # Fields: kind, event_id, reason, repo (optional), npub (optional) + + local kind event_id reason repo npub + + # Extract kind=VALUE + kind=$(echo "$line" | grep -oP 'kind=\K[0-9]+' || echo "") + + # Extract event_id=VALUE (hex string, possibly truncated with ...) + event_id=$(echo "$line" | grep -oP 'event_id=\K[a-f0-9]+' || echo "") + + # Extract reason="VALUE" (quoted string) + reason=$(echo "$line" | grep -oP 'reason="\K[^"]*' || echo "") + + # Extract repo=VALUE (optional, unquoted identifier) + repo=$(echo "$line" | grep -oP 'repo=\K[^ ]+' || echo "") + + # Extract npub=VALUE (optional, npub1... format) + npub=$(echo "$line" | grep -oP 'npub=\K[^ ]+' || echo "") + + # Only output if we have the required fields + if [[ -n "$kind" && -n "$event_id" && -n "$reason" ]]; then + printf '%s\t%s\t%s\t%s\t%s\n' "$repo" "$npub" "$kind" "$event_id" "$reason" + fi +} + +# Main +main() { + if [[ $# -lt 2 ]]; then + usage + fi + + local service="$1" + local output_dir="$2" + shift 2 + + # Default time range: last 30 days + local since_date + since_date=$(date -d "30 days ago" "+%Y-%m-%d" 2>/dev/null || date -v-30d "+%Y-%m-%d" 2>/dev/null || echo "") + local until_date="" + local dry_run=false + + # Parse options + while [[ $# -gt 0 ]]; do + case "$1" in + --since) + since_date="$2" + shift 2 + ;; + --until) + until_date="$2" + shift 2 + ;; + --dry-run) + dry_run=true + shift + ;; + *) + log_error "Unknown option: $1" + usage + ;; + esac + done + + # Validate service name + if [[ ! "$service" =~ \.service$ ]]; then + service="${service}.service" + fi + + log_info "Extracting parse failures from systemd logs" + log_info "Service: $service" + log_info "Output: $output_dir" + log_info "Time range: ${since_date:-beginning} to ${until_date:-now}" + + # Check if journalctl is available + if ! command -v journalctl &> /dev/null; then + log_error "journalctl not found. This script requires systemd." + exit 1 + fi + + # Build journalctl command + local journal_cmd="journalctl -u $service --no-pager -o short-iso" + + if [[ -n "$since_date" ]]; then + journal_cmd="$journal_cmd --since '$since_date'" + fi + + if [[ -n "$until_date" ]]; then + journal_cmd="$journal_cmd --until '$until_date'" + fi + + log_info "Running: $journal_cmd | grep '\\[PARSE_FAIL\\]'" + + if [[ "$dry_run" == true ]]; then + log_info "[DRY RUN] Would extract to: $output_dir/parse-failures.txt" + + # Show sample of what would be extracted + log_info "Checking for matching log entries..." + local sample_count + sample_count=$(eval "$journal_cmd" 2>/dev/null | grep -c '\[PARSE_FAIL\]' || echo "0") + sample_count="${sample_count//[^0-9]/}" # Strip non-numeric characters + sample_count="${sample_count:-0}" + log_info "Found $sample_count matching log entries" + + if [[ "$sample_count" -eq 0 ]]; then + log_warn "No [PARSE_FAIL] entries found in logs." + log_warn "This is expected if ngit-grasp logging improvements are not yet deployed." + log_warn "See: docs/how-to/migrate-ngit-relay-to-ngit-grasp.md (Dependencies section)" + fi + + exit 0 + fi + + # Create output directory + mkdir -p "$output_dir" + + local output_file="$output_dir/parse-failures.txt" + local temp_file + temp_file=$(mktemp) + + # Extract and parse log entries + log_info "Extracting log entries..." + + # Get raw log lines containing [PARSE_FAIL] + local raw_lines + raw_lines=$(eval "$journal_cmd" 2>/dev/null | grep '\[PARSE_FAIL\]' || true) + + if [[ -z "$raw_lines" ]]; then + log_warn "No [PARSE_FAIL] entries found in logs." + log_warn "" + log_warn "This is expected if ngit-grasp logging improvements are not yet deployed." + log_warn "The structured log format required by this script:" + log_warn "" + log_warn " [PARSE_FAIL] kind=30618 event_id=abc123 reason=\"...\" repo=myrepo npub=npub1..." + log_warn "" + log_warn "See: docs/how-to/migrate-ngit-relay-to-ngit-grasp.md (Dependencies section)" + log_warn "" + + # Create empty output file with header comment + { + echo "# Parse failures extracted from $service" + echo "# Time range: ${since_date:-beginning} to ${until_date:-now}" + echo "# Extracted: $(date -Iseconds)" + echo "# Format: reponpubkindevent_idreason" + echo "#" + echo "# NOTE: No [PARSE_FAIL] entries found." + echo "# This is expected if ngit-grasp logging improvements are not yet deployed." + } > "$output_file" + + log_info "Created empty output file: $output_file" + exit 0 + fi + + # Write header + { + echo "# Parse failures extracted from $service" + echo "# Time range: ${since_date:-beginning} to ${until_date:-now}" + echo "# Extracted: $(date -Iseconds)" + echo "# Format: reponpubkindevent_idreason" + } > "$output_file" + + # Parse each line + local count=0 + while IFS= read -r line; do + local parsed + parsed=$(parse_log_line "$line") + if [[ -n "$parsed" ]]; then + echo "$parsed" >> "$output_file" + ((count++)) + fi + done <<< "$raw_lines" + + rm -f "$temp_file" + + # Summary + echo "" + log_info "=== Extraction Summary ===" + log_info "Service: $service" + log_info "Time range: ${since_date:-beginning} to ${until_date:-now}" + log_success "Extracted $count parse failure entries" + echo "" + log_info "Output file: $output_file" + + if [[ $count -gt 0 ]]; then + echo "" + log_info "Sample entries (first 5):" + tail -n +5 "$output_file" | head -5 | while IFS=$'\t' read -r repo npub kind event_id reason; do + echo " kind=$kind repo=$repo reason=\"$reason\"" + done + fi + + # Breakdown by kind + if [[ $count -gt 0 ]]; then + echo "" + log_info "Breakdown by event kind:" + tail -n +5 "$output_file" | awk -F'\t' '{print $3}' | sort | uniq -c | sort -rn | while read -r cnt kind; do + echo " kind $kind: $cnt failures" + done + fi +} + +main "$@" diff --git a/docs/how-to/migration-scripts/31-extract-purgatory-expiry.sh b/docs/how-to/migration-scripts/31-extract-purgatory-expiry.sh new file mode 100755 index 0000000..38b2ca3 --- /dev/null +++ b/docs/how-to/migration-scripts/31-extract-purgatory-expiry.sh @@ -0,0 +1,346 @@ +#!/usr/bin/env bash +# +# 31-extract-purgatory-expiry.sh - Extract purgatory expiry events from systemd logs +# +# PHASE 4b of the ngit-relay to ngit-grasp migration analysis pipeline. +# Extracts structured [PURGATORY_EXPIRED] log entries from journalctl. +# +# USAGE: +# ./31-extract-purgatory-expiry.sh [options] +# +# EXAMPLES: +# # Extract from ngit-grasp service (last 30 days, default) +# ./31-extract-purgatory-expiry.sh ngit-grasp.service output/logs +# +# # Extract with custom time range +# ./31-extract-purgatory-expiry.sh ngit-grasp.service output/logs --since "2026-01-01" +# +# # Extract from specific time window +# ./31-extract-purgatory-expiry.sh ngit-grasp.service output/logs --since "2026-01-15" --until "2026-01-22" +# +# OPTIONS: +# --since Start date for log extraction (default: 30 days ago) +# --until End date for log extraction (default: now) +# --dry-run Show what would be extracted without writing files +# +# OUTPUT: +# /purgatory-expired.txt +# +# OUTPUT FORMAT (TSV): +# reponpubtimestampreason +# +# EXPECTED LOG FORMAT: +# The script looks for structured log entries in this format: +# +# 2026-01-22T10:30:45Z ngit-grasp[1234]: [PURGATORY_EXPIRED] repo=myrepo npub=npub1... reason="clone URL unreachable after 7 days" +# +# Required fields: repo, npub +# Optional fields: reason (explains why purgatory expired) +# +# BACKGROUND: +# "Purgatory" is the state where ngit-grasp has received an announcement event +# but cannot yet sync the git data (e.g., clone URL unreachable, git server down). +# After a configurable timeout (default 7 days), the repository is marked as +# expired and removed from purgatory. +# +# Purgatory expiry during migration analysis indicates repositories that: +# - Had valid announcements on the production relay +# - Could not be synced to the archive relay +# - May need manual intervention or investigation +# +# DEPENDENCY: +# This script requires logging improvements in ngit-grasp to emit structured +# [PURGATORY_EXPIRED] log entries. Until those are implemented, this script +# will find no matching entries (which is handled gracefully). +# +# See: docs/how-to/migrate-ngit-relay-to-ngit-grasp.md (Dependencies section) +# +# Expected Rust logging code: +# tracing::warn!( +# target: "migration", +# "[PURGATORY_EXPIRED] repo={} npub={} reason=\"{}\"", +# identifier, npub, reason +# ); +# +# PREREQUISITES: +# - journalctl (systemd) +# - grep, awk (standard Unix tools) +# - Access to systemd journal (may require sudo or journal group membership) +# +# RUNTIME: Depends on log volume, typically < 30 seconds +# +# SEE ALSO: +# docs/how-to/migrate-ngit-relay-to-ngit-grasp.md - Full migration guide +# 30-extract-parse-failures.sh - Companion script for parse failure logs +# + +set -euo pipefail + +# Colors for output (disabled if not a terminal) +if [[ -t 1 ]]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[0;33m' + BLUE='\033[0;34m' + NC='\033[0m' +else + RED='' + GREEN='' + YELLOW='' + BLUE='' + NC='' +fi + +log_info() { + echo -e "${BLUE}[INFO]${NC} $*" >&2 +} + +log_success() { + echo -e "${GREEN}[OK]${NC} $*" >&2 +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $*" >&2 +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $*" >&2 +} + +usage() { + echo "Usage: $0 [options]" + echo "" + echo "Arguments:" + echo " service-name Systemd service name (e.g., ngit-grasp.service)" + echo " output-dir Directory to store extracted log data" + echo "" + echo "Options:" + echo " --since Start date (default: 30 days ago)" + echo " --until End date (default: now)" + echo " --dry-run Show what would be extracted without writing" + echo "" + echo "Examples:" + echo " $0 ngit-grasp.service output/logs" + echo " $0 ngit-grasp.service output/logs --since '2026-01-01'" + echo " $0 ngit-grasp.service output/logs --since '2026-01-15' --until '2026-01-22'" + echo "" + echo "Expected log format:" + echo " [PURGATORY_EXPIRED] repo=myrepo npub=npub1... reason=\"...\"" + exit 1 +} + +# Parse a single log line and extract fields +# Input: log line containing [PURGATORY_EXPIRED] +# Output: TSV line: reponpubtimestampreason +parse_log_line() { + local line="$1" + + # Extract timestamp from the beginning of the log line + # Format: 2026-01-22T10:30:45+0000 or similar ISO format + local timestamp repo npub reason + + # Extract ISO timestamp from beginning of line + timestamp=$(echo "$line" | grep -oP '^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}' || echo "") + + # Extract repo=VALUE (unquoted identifier) + repo=$(echo "$line" | grep -oP 'repo=\K[^ ]+' || echo "") + + # Extract npub=VALUE (npub1... format) + npub=$(echo "$line" | grep -oP 'npub=\K[^ ]+' || echo "") + + # Extract reason="VALUE" (quoted string, optional) + reason=$(echo "$line" | grep -oP 'reason="\K[^"]*' || echo "") + + # Only output if we have the required fields + if [[ -n "$repo" && -n "$npub" ]]; then + printf '%s\t%s\t%s\t%s\n' "$repo" "$npub" "$timestamp" "$reason" + fi +} + +# Main +main() { + if [[ $# -lt 2 ]]; then + usage + fi + + local service="$1" + local output_dir="$2" + shift 2 + + # Default time range: last 30 days + local since_date + since_date=$(date -d "30 days ago" "+%Y-%m-%d" 2>/dev/null || date -v-30d "+%Y-%m-%d" 2>/dev/null || echo "") + local until_date="" + local dry_run=false + + # Parse options + while [[ $# -gt 0 ]]; do + case "$1" in + --since) + since_date="$2" + shift 2 + ;; + --until) + until_date="$2" + shift 2 + ;; + --dry-run) + dry_run=true + shift + ;; + *) + log_error "Unknown option: $1" + usage + ;; + esac + done + + # Validate service name + if [[ ! "$service" =~ \.service$ ]]; then + service="${service}.service" + fi + + log_info "Extracting purgatory expiry events from systemd logs" + log_info "Service: $service" + log_info "Output: $output_dir" + log_info "Time range: ${since_date:-beginning} to ${until_date:-now}" + + # Check if journalctl is available + if ! command -v journalctl &> /dev/null; then + log_error "journalctl not found. This script requires systemd." + exit 1 + fi + + # Build journalctl command + local journal_cmd="journalctl -u $service --no-pager -o short-iso" + + if [[ -n "$since_date" ]]; then + journal_cmd="$journal_cmd --since '$since_date'" + fi + + if [[ -n "$until_date" ]]; then + journal_cmd="$journal_cmd --until '$until_date'" + fi + + log_info "Running: $journal_cmd | grep '\\[PURGATORY_EXPIRED\\]'" + + if [[ "$dry_run" == true ]]; then + log_info "[DRY RUN] Would extract to: $output_dir/purgatory-expired.txt" + + # Show sample of what would be extracted + log_info "Checking for matching log entries..." + local sample_count + sample_count=$(eval "$journal_cmd" 2>/dev/null | grep -c '\[PURGATORY_EXPIRED\]' || echo "0") + sample_count="${sample_count//[^0-9]/}" # Strip non-numeric characters + sample_count="${sample_count:-0}" + log_info "Found $sample_count matching log entries" + + if [[ "$sample_count" -eq 0 ]]; then + log_warn "No [PURGATORY_EXPIRED] entries found in logs." + log_warn "This is expected if ngit-grasp logging improvements are not yet deployed." + log_warn "See: docs/how-to/migrate-ngit-relay-to-ngit-grasp.md (Dependencies section)" + fi + + exit 0 + fi + + # Create output directory + mkdir -p "$output_dir" + + local output_file="$output_dir/purgatory-expired.txt" + local temp_file + temp_file=$(mktemp) + + # Extract and parse log entries + log_info "Extracting log entries..." + + # Get raw log lines containing [PURGATORY_EXPIRED] + local raw_lines + raw_lines=$(eval "$journal_cmd" 2>/dev/null | grep '\[PURGATORY_EXPIRED\]' || true) + + if [[ -z "$raw_lines" ]]; then + log_warn "No [PURGATORY_EXPIRED] entries found in logs." + log_warn "" + log_warn "This is expected if ngit-grasp logging improvements are not yet deployed." + log_warn "The structured log format required by this script:" + log_warn "" + log_warn " [PURGATORY_EXPIRED] repo=myrepo npub=npub1... reason=\"...\"" + log_warn "" + log_warn "See: docs/how-to/migrate-ngit-relay-to-ngit-grasp.md (Dependencies section)" + log_warn "" + + # Create empty output file with header comment + { + echo "# Purgatory expiry events extracted from $service" + echo "# Time range: ${since_date:-beginning} to ${until_date:-now}" + echo "# Extracted: $(date -Iseconds)" + echo "# Format: reponpubtimestampreason" + echo "#" + echo "# NOTE: No [PURGATORY_EXPIRED] entries found." + echo "# This is expected if ngit-grasp logging improvements are not yet deployed." + } > "$output_file" + + log_info "Created empty output file: $output_file" + exit 0 + fi + + # Write header + { + echo "# Purgatory expiry events extracted from $service" + echo "# Time range: ${since_date:-beginning} to ${until_date:-now}" + echo "# Extracted: $(date -Iseconds)" + echo "# Format: reponpubtimestampreason" + } > "$output_file" + + # Parse each line + local count=0 + while IFS= read -r line; do + local parsed + parsed=$(parse_log_line "$line") + if [[ -n "$parsed" ]]; then + echo "$parsed" >> "$output_file" + ((count++)) + fi + done <<< "$raw_lines" + + rm -f "$temp_file" + + # Summary + echo "" + log_info "=== Extraction Summary ===" + log_info "Service: $service" + log_info "Time range: ${since_date:-beginning} to ${until_date:-now}" + log_success "Extracted $count purgatory expiry entries" + echo "" + log_info "Output file: $output_file" + + if [[ $count -gt 0 ]]; then + echo "" + log_info "Sample entries (first 5):" + tail -n +5 "$output_file" | head -5 | while IFS=$'\t' read -r repo npub timestamp reason; do + echo " repo=$repo npub=${npub:0:20}... timestamp=$timestamp" + done + fi + + # Show unique repos affected + if [[ $count -gt 0 ]]; then + echo "" + local unique_repos + unique_repos=$(tail -n +5 "$output_file" | awk -F'\t' '{print $1}' | sort -u | wc -l) + log_info "Unique repositories affected: $unique_repos" + + echo "" + log_info "Repositories with purgatory expiry:" + tail -n +5 "$output_file" | awk -F'\t' '{print $1}' | sort | uniq -c | sort -rn | head -10 | while read -r cnt repo; do + echo " $repo: $cnt expiry events" + done + + local total_repos + total_repos=$(tail -n +5 "$output_file" | awk -F'\t' '{print $1}' | sort -u | wc -l) + if [[ $total_repos -gt 10 ]]; then + echo " ... and $((total_repos - 10)) more repositories" + fi + fi +} + +main "$@" -- cgit v1.2.3