diff options
Diffstat (limited to 'grasp-audit/src/probe.rs')
| -rw-r--r-- | grasp-audit/src/probe.rs | 122 |
1 files changed, 94 insertions, 28 deletions
diff --git a/grasp-audit/src/probe.rs b/grasp-audit/src/probe.rs index 0dca74c..4d8f31f 100644 --- a/grasp-audit/src/probe.rs +++ b/grasp-audit/src/probe.rs | |||
| @@ -124,26 +124,6 @@ impl ProbeReport { | |||
| 124 | } | 124 | } |
| 125 | } | 125 | } |
| 126 | 126 | ||
| 127 | impl ProbeReport { | ||
| 128 | /// Build a synthetic report for when the overall probe timeout fires. | ||
| 129 | pub fn overall_timeout(relay_url: &str, duration_ms: u64) -> Self { | ||
| 130 | ProbeReport { | ||
| 131 | relay_url: relay_url.to_string(), | ||
| 132 | timestamp: now_iso8601(), | ||
| 133 | all_passed: false, | ||
| 134 | total_duration_ms: duration_ms, | ||
| 135 | checks: vec![ProbeCheck { | ||
| 136 | name: "overall_timeout", | ||
| 137 | passed: false, | ||
| 138 | skipped: false, | ||
| 139 | duration_ms, | ||
| 140 | detail: None, | ||
| 141 | error: Some("probe exceeded overall timeout".to_string()), | ||
| 142 | }], | ||
| 143 | } | ||
| 144 | } | ||
| 145 | } | ||
| 146 | |||
| 147 | // ============================================================ | 127 | // ============================================================ |
| 148 | // Helpers | 128 | // Helpers |
| 149 | // ============================================================ | 129 | // ============================================================ |
| @@ -197,6 +177,19 @@ fn now_iso8601() -> String { | |||
| 197 | // Main probe function | 177 | // Main probe function |
| 198 | // ============================================================ | 178 | // ============================================================ |
| 199 | 179 | ||
| 180 | /// All check names in the order they may appear, used to fill skipped entries | ||
| 181 | /// when the overall deadline fires. | ||
| 182 | const ALL_CHECK_NAMES: &[&str] = &[ | ||
| 183 | "connect_websocket", | ||
| 184 | "nip11_fetch", | ||
| 185 | "publish_events", | ||
| 186 | "git_repo_initialised", | ||
| 187 | "git_push", | ||
| 188 | "serves_latest_announcement", | ||
| 189 | "git_fetch_refs", | ||
| 190 | "git_refs_match_state", | ||
| 191 | ]; | ||
| 192 | |||
| 200 | /// Run a probe against a GRASP relay and return a full report. | 193 | /// Run a probe against a GRASP relay and return a full report. |
| 201 | /// | 194 | /// |
| 202 | /// # Arguments | 195 | /// # Arguments |
| @@ -204,16 +197,52 @@ fn now_iso8601() -> String { | |||
| 204 | /// * `keys` - Optional keypair to use; `None` generates fresh keys | 197 | /// * `keys` - Optional keypair to use; `None` generates fresh keys |
| 205 | /// * `read_only` - When `true`, skip write steps and only check existing repos | 198 | /// * `read_only` - When `true`, skip write steps and only check existing repos |
| 206 | /// * `timeout_secs` - Per-step timeout in seconds | 199 | /// * `timeout_secs` - Per-step timeout in seconds |
| 200 | /// * `overall_secs` - Hard cap on total probe duration; remaining checks are | ||
| 201 | /// marked skipped with reason "overall timeout" if the deadline fires | ||
| 207 | pub async fn run_probe( | 202 | pub async fn run_probe( |
| 208 | relay_url: &str, | 203 | relay_url: &str, |
| 209 | keys: Option<Keys>, | 204 | keys: Option<Keys>, |
| 210 | read_only: bool, | 205 | read_only: bool, |
| 211 | timeout_secs: u64, | 206 | timeout_secs: u64, |
| 207 | overall_secs: u64, | ||
| 212 | ) -> ProbeReport { | 208 | ) -> ProbeReport { |
| 213 | let total_start = Instant::now(); | 209 | let total_start = Instant::now(); |
| 210 | let deadline = total_start + Duration::from_secs(overall_secs); | ||
| 214 | let timestamp = now_iso8601(); | 211 | let timestamp = now_iso8601(); |
| 215 | let mut checks: Vec<ProbeCheck> = Vec::new(); | 212 | let mut checks: Vec<ProbeCheck> = Vec::new(); |
| 216 | 213 | ||
| 214 | /// Fill all check names not yet present in `checks` as skipped with the | ||
| 215 | /// given reason, then return a finished ProbeReport. | ||
| 216 | macro_rules! deadline_return { | ||
| 217 | ($relay_url:expr, $timestamp:expr, $total_start:expr, $checks:expr, $timed_out_name:expr) => {{ | ||
| 218 | // Mark the step that hit the deadline as a timeout failure | ||
| 219 | $checks.push(ProbeCheck { | ||
| 220 | name: $timed_out_name, | ||
| 221 | passed: false, | ||
| 222 | skipped: false, | ||
| 223 | duration_ms: $total_start.elapsed().as_millis() as u64, | ||
| 224 | detail: None, | ||
| 225 | error: Some("overall timeout".to_string()), | ||
| 226 | }); | ||
| 227 | // Skip all subsequent checks | ||
| 228 | let already: std::collections::HashSet<&str> = | ||
| 229 | $checks.iter().map(|c| c.name).collect(); | ||
| 230 | for name in ALL_CHECK_NAMES { | ||
| 231 | if !already.contains(name) { | ||
| 232 | $checks.push(skipped(name, "overall timeout")); | ||
| 233 | } | ||
| 234 | } | ||
| 235 | let all_passed = $checks.iter().all(|c| c.passed || c.skipped); | ||
| 236 | return ProbeReport { | ||
| 237 | relay_url: $relay_url.to_string(), | ||
| 238 | timestamp: $timestamp, | ||
| 239 | all_passed, | ||
| 240 | total_duration_ms: $total_start.elapsed().as_millis() as u64, | ||
| 241 | checks: $checks, | ||
| 242 | }; | ||
| 243 | }}; | ||
| 244 | } | ||
| 245 | |||
| 217 | // ============================================================ | 246 | // ============================================================ |
| 218 | // PREPARE (offline) | 247 | // PREPARE (offline) |
| 219 | // ============================================================ | 248 | // ============================================================ |
| @@ -317,8 +346,16 @@ pub async fn run_probe( | |||
| 317 | // ============================================================ | 346 | // ============================================================ |
| 318 | // Step 1: connect_websocket | 347 | // Step 1: connect_websocket |
| 319 | // ============================================================ | 348 | // ============================================================ |
| 349 | if Instant::now() >= deadline { | ||
| 350 | deadline_return!(relay_url, timestamp, total_start, checks, "connect_websocket"); | ||
| 351 | } | ||
| 320 | let step1_start = Instant::now(); | 352 | let step1_start = Instant::now(); |
| 321 | let client_result = AuditClient::new_with_keys(relay_url, config.clone(), keys.clone()).await; | 353 | let client_result = tokio::time::timeout( |
| 354 | deadline.saturating_duration_since(Instant::now()), | ||
| 355 | AuditClient::new_with_keys(relay_url, config.clone(), keys.clone()), | ||
| 356 | ) | ||
| 357 | .await | ||
| 358 | .unwrap_or_else(|_| Err(anyhow::anyhow!("overall timeout"))); | ||
| 322 | let step1_ms = step1_start.elapsed().as_millis() as u64; | 359 | let step1_ms = step1_start.elapsed().as_millis() as u64; |
| 323 | 360 | ||
| 324 | let client = match client_result { | 361 | let client = match client_result { |
| @@ -367,10 +404,13 @@ pub async fn run_probe( | |||
| 367 | // Step 2: nip11_fetch (independent — always runs if step 1 passed) | 404 | // Step 2: nip11_fetch (independent — always runs if step 1 passed) |
| 368 | // ============================================================ | 405 | // ============================================================ |
| 369 | { | 406 | { |
| 407 | if Instant::now() >= deadline { | ||
| 408 | deadline_return!(relay_url, timestamp, total_start, checks, "nip11_fetch"); | ||
| 409 | } | ||
| 370 | let step2_start = Instant::now(); | 410 | let step2_start = Instant::now(); |
| 371 | let http_client = reqwest::Client::new(); | 411 | let http_client = reqwest::Client::new(); |
| 372 | let nip11_result = tokio::time::timeout( | 412 | let nip11_result = tokio::time::timeout( |
| 373 | Duration::from_secs(timeout_secs), | 413 | deadline.saturating_duration_since(Instant::now()).min(Duration::from_secs(timeout_secs)), |
| 374 | http_client | 414 | http_client |
| 375 | .get(&http_base) | 415 | .get(&http_base) |
| 376 | .header("Accept", "application/nostr+json") | 416 | .header("Accept", "application/nostr+json") |
| @@ -434,6 +474,10 @@ pub async fn run_probe( | |||
| 434 | // ============================================================ | 474 | // ============================================================ |
| 435 | let mut write_succeeded = false; | 475 | let mut write_succeeded = false; |
| 436 | 476 | ||
| 477 | if Instant::now() >= deadline { | ||
| 478 | deadline_return!(relay_url, timestamp, total_start, checks, "publish_events"); | ||
| 479 | } | ||
| 480 | |||
| 437 | if read_only { | 481 | if read_only { |
| 438 | checks.push(skipped("publish_events", "read-only mode")); | 482 | checks.push(skipped("publish_events", "read-only mode")); |
| 439 | checks.push(skipped("git_repo_initialised", "read-only mode")); | 483 | checks.push(skipped("git_repo_initialised", "read-only mode")); |
| @@ -492,14 +536,18 @@ pub async fn run_probe( | |||
| 492 | // Step 4: git_repo_initialised (requires step 3) | 536 | // Step 4: git_repo_initialised (requires step 3) |
| 493 | // ============================================================ | 537 | // ============================================================ |
| 494 | if write_succeeded { | 538 | if write_succeeded { |
| 539 | if Instant::now() >= deadline { | ||
| 540 | deadline_return!(relay_url, timestamp, total_start, checks, "git_repo_initialised"); | ||
| 541 | } | ||
| 495 | let step4_start = Instant::now(); | 542 | let step4_start = Instant::now(); |
| 496 | let poll_url = format!("{}/info/refs?service=git-upload-pack", clone_url); | 543 | let poll_url = format!("{}/info/refs?service=git-upload-pack", clone_url); |
| 497 | let http_client = reqwest::Client::new(); | 544 | let http_client = reqwest::Client::new(); |
| 498 | let deadline = Instant::now() + Duration::from_secs(15); | 545 | // Cap the poll deadline at both 15s and the overall deadline |
| 546 | let poll_deadline = (Instant::now() + Duration::from_secs(15)).min(deadline); | ||
| 499 | let mut repo_ready = false; | 547 | let mut repo_ready = false; |
| 500 | 548 | ||
| 501 | loop { | 549 | loop { |
| 502 | if Instant::now() >= deadline { | 550 | if Instant::now() >= poll_deadline { |
| 503 | break; | 551 | break; |
| 504 | } | 552 | } |
| 505 | match http_client.get(&poll_url).send().await { | 553 | match http_client.get(&poll_url).send().await { |
| @@ -541,6 +589,9 @@ pub async fn run_probe( | |||
| 541 | // Step 5: git_push (requires step 4) | 589 | // Step 5: git_push (requires step 4) |
| 542 | // ============================================================ | 590 | // ============================================================ |
| 543 | if write_succeeded { | 591 | if write_succeeded { |
| 592 | if Instant::now() >= deadline { | ||
| 593 | deadline_return!(relay_url, timestamp, total_start, checks, "git_push"); | ||
| 594 | } | ||
| 544 | let step5_start = Instant::now(); | 595 | let step5_start = Instant::now(); |
| 545 | let push_result = try_push(&local_repo_path); | 596 | let push_result = try_push(&local_repo_path); |
| 546 | let step5_ms = step5_start.elapsed().as_millis() as u64; | 597 | let step5_ms = step5_start.elapsed().as_millis() as u64; |
| @@ -619,12 +670,15 @@ pub async fn run_probe( | |||
| 619 | if write_succeeded { | 670 | if write_succeeded { |
| 620 | // ---- Write path ---- | 671 | // ---- Write path ---- |
| 621 | // Step 6a: git_fetch_refs — just verify the endpoint returns 200 | 672 | // Step 6a: git_fetch_refs — just verify the endpoint returns 200 |
| 673 | if Instant::now() >= deadline { | ||
| 674 | deadline_return!(relay_url, timestamp, total_start, checks, "git_fetch_refs"); | ||
| 675 | } | ||
| 622 | let refs_url = format!("{}/info/refs?service=git-upload-pack", clone_url); | 676 | let refs_url = format!("{}/info/refs?service=git-upload-pack", clone_url); |
| 623 | let http_client = reqwest::Client::new(); | 677 | let http_client = reqwest::Client::new(); |
| 624 | 678 | ||
| 625 | let step6_start = Instant::now(); | 679 | let step6_start = Instant::now(); |
| 626 | let refs_result = tokio::time::timeout( | 680 | let refs_result = tokio::time::timeout( |
| 627 | Duration::from_secs(timeout_secs), | 681 | deadline.saturating_duration_since(Instant::now()).min(Duration::from_secs(timeout_secs)), |
| 628 | http_client.get(&refs_url).send(), | 682 | http_client.get(&refs_url).send(), |
| 629 | ) | 683 | ) |
| 630 | .await; | 684 | .await; |
| @@ -744,10 +798,16 @@ pub async fn run_probe( | |||
| 744 | // ---- Fallback path: find any existing kind 30617, check refs readable ---- | 798 | // ---- Fallback path: find any existing kind 30617, check refs readable ---- |
| 745 | 799 | ||
| 746 | // In read-only mode: first check that at least one announcement exists | 800 | // In read-only mode: first check that at least one announcement exists |
| 801 | if Instant::now() >= deadline { | ||
| 802 | deadline_return!(relay_url, timestamp, total_start, checks, "serves_latest_announcement"); | ||
| 803 | } | ||
| 747 | let filter = Filter::new().kind(Kind::GitRepoAnnouncement).limit(1); | 804 | let filter = Filter::new().kind(Kind::GitRepoAnnouncement).limit(1); |
| 748 | let existing = client | 805 | let existing = client |
| 749 | .client() | 806 | .client() |
| 750 | .fetch_events(filter, Duration::from_secs(5)) | 807 | .fetch_events( |
| 808 | filter, | ||
| 809 | deadline.saturating_duration_since(Instant::now()).min(Duration::from_secs(5)), | ||
| 810 | ) | ||
| 751 | .await | 811 | .await |
| 752 | .unwrap_or_default(); | 812 | .unwrap_or_default(); |
| 753 | 813 | ||
| @@ -819,11 +879,14 @@ pub async fn run_probe( | |||
| 819 | format!("{}/{}/{}.git", http_base, ann_npub, ann_id) | 879 | format!("{}/{}/{}.git", http_base, ann_npub, ann_id) |
| 820 | }); | 880 | }); |
| 821 | 881 | ||
| 882 | if Instant::now() >= deadline { | ||
| 883 | deadline_return!(relay_url, timestamp, total_start, checks, "git_fetch_refs"); | ||
| 884 | } | ||
| 822 | let step6_start = Instant::now(); | 885 | let step6_start = Instant::now(); |
| 823 | let refs_url = format!("{}/info/refs?service=git-upload-pack", fetch_url); | 886 | let refs_url = format!("{}/info/refs?service=git-upload-pack", fetch_url); |
| 824 | let http_client = reqwest::Client::new(); | 887 | let http_client = reqwest::Client::new(); |
| 825 | let refs_result = tokio::time::timeout( | 888 | let refs_result = tokio::time::timeout( |
| 826 | Duration::from_secs(timeout_secs), | 889 | deadline.saturating_duration_since(Instant::now()).min(Duration::from_secs(timeout_secs)), |
| 827 | http_client.get(&refs_url).send(), | 890 | http_client.get(&refs_url).send(), |
| 828 | ) | 891 | ) |
| 829 | .await; | 892 | .await; |
| @@ -905,7 +968,10 @@ pub async fn run_probe( | |||
| 905 | ); | 968 | ); |
| 906 | let state_events = client | 969 | let state_events = client |
| 907 | .client() | 970 | .client() |
| 908 | .fetch_events(state_filter, Duration::from_secs(5)) | 971 | .fetch_events( |
| 972 | state_filter, | ||
| 973 | deadline.saturating_duration_since(Instant::now()).min(Duration::from_secs(5)), | ||
| 974 | ) | ||
| 909 | .await | 975 | .await |
| 910 | .unwrap_or_default(); | 976 | .unwrap_or_default(); |
| 911 | 977 | ||