From 0550b3229f35ef3ee125bac47d85bbd08d1250b1 Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Wed, 7 Jan 2026 21:33:22 +0000 Subject: test: add WIP SmartHttpServer to test --- tests/common/git_server.rs | 707 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 689 insertions(+), 18 deletions(-) (limited to 'tests/common/git_server.rs') diff --git a/tests/common/git_server.rs b/tests/common/git_server.rs index b225084..6121adc 100644 --- a/tests/common/git_server.rs +++ b/tests/common/git_server.rs @@ -1,13 +1,21 @@ -//! Simple HTTP Git Server for Testing +//! HTTP Git Servers for Testing //! -//! Provides a dumb HTTP server for serving git repositories in integration tests. -//! This server serves static files from a bare git repository, enabling `git fetch` -//! operations without requiring a full git HTTP backend. +//! This module provides two git server implementations for integration tests: +//! +//! ## `SimpleGitServer` (Dumb HTTP Protocol) +//! +//! Serves static files from a bare git repository using Git's "dumb HTTP" protocol. +//! This is lightweight but does NOT support shallow fetches (`git fetch --depth=1`). +//! +//! ## `SmartGitServer` (Smart HTTP Protocol) +//! +//! Implements the Git Smart HTTP protocol by spawning `git upload-pack` subprocesses. +//! This supports all git fetch operations including shallow fetches. //! //! # Usage //! //! ```ignore -//! use common::SimpleGitServer; +//! use common::{SimpleGitServer, SmartGitServer}; //! //! #[tokio::test] //! async fn test_git_fetch() { @@ -15,12 +23,12 @@ //! let temp_dir = tempfile::tempdir().unwrap(); //! create_test_repo_with_commit(temp_dir.path(), CommitVariant::StateTest).unwrap(); //! -//! // Start the server -//! let server = SimpleGitServer::start(temp_dir.path()).await; +//! // Use SmartGitServer for full protocol support (including shallow fetches) +//! let server = SmartGitServer::start(temp_dir.path()).await; //! //! // Git operations work against server.url() //! let output = Command::new("git") -//! .args(["ls-remote", server.url()]) +//! .args(["clone", "--depth=1", server.url(), "/tmp/clone"]) //! .output() //! .unwrap(); //! assert!(output.status.success()); @@ -30,17 +38,13 @@ //! } //! ``` //! -//! # How It Works +//! # When to Use Which //! -//! Git's "dumb HTTP" protocol just needs static file access to: -//! - `info/refs` - List of refs (generated by `git update-server-info`) -//! - `objects/info/packs` - List of pack files -//! - `objects/pack/*` - Pack files -//! - `objects/??/*` - Loose objects +//! - **SimpleGitServer**: Fast, lightweight, good for basic `git fetch` without depth limits +//! - **SmartGitServer**: Full protocol support, required for `--depth=1` shallow fetches //! -//! The server creates a bare clone of the source repository, runs -//! `git update-server-info` to generate the required metadata files, -//! and serves them over HTTP. +//! The purgatory sync system uses `git fetch --depth=1`, so tests involving purgatory +//! sync should use `SmartGitServer`. use std::net::SocketAddr; use std::path::{Path, PathBuf}; @@ -138,11 +142,14 @@ impl SimpleGitServer { .expect("Failed to bind to address"); let handle = tokio::spawn(async move { + println!("[SmartGitServer] Server loop started on port {}", port); + eprintln!("[SmartGitServer] Server loop started on port {}", port); loop { tokio::select! { accept_result = listener.accept() => { match accept_result { - Ok((stream, _)) => { + Ok((stream, addr)) => { + eprintln!("[SmartGitServer] Accepted connection from {}", addr); let repo_path = Arc::clone(&repo_path); let io = TokioIo::new(stream); @@ -485,3 +492,667 @@ mod tests { assert!(!is_safe_path(Path::new("/tmp/repo/../../etc/passwd"), repo_path)); } } + +// ============================================================================= +// SmartGitServer - Git Smart HTTP Protocol Server +// ============================================================================= + +/// Smart HTTP server for serving git repositories with full protocol support. +/// +/// Unlike `SimpleGitServer` which uses the "dumb HTTP" protocol (static files), +/// this server implements the Git Smart HTTP protocol by spawning `git upload-pack` +/// subprocesses. This enables: +/// +/// - Shallow clones (`git clone --depth=1`) +/// - Shallow fetches (`git fetch --depth=1`) +/// - Full protocol negotiation +/// +/// This is required for testing purgatory sync, which uses `git fetch --depth=1`. +pub struct SmartGitServer { + /// Shutdown signal sender + shutdown_tx: Option>, + /// Server task handle + handle: Option>, + /// Server URL (http://127.0.0.1:) + url: String, + /// Server port + #[allow(dead_code)] + port: u16, + /// Temporary directory containing the bare repository + /// Kept alive for the lifetime of the server + _temp_dir: tempfile::TempDir, +} + +impl SmartGitServer { + /// Start a smart HTTP git server serving the given repository. + /// + /// Creates a bare clone of the source repository and starts an HTTP server + /// that implements the Git Smart HTTP protocol. + /// + /// # Arguments + /// * `source_repo` - Path to the source git repository (can be non-bare) + /// + /// # Returns + /// A `SmartGitServer` instance with the server running + /// + /// # Panics + /// Panics if the git operations fail or the server cannot start + pub async fn start(source_repo: &Path) -> Self { + println!("[SmartGitServer::start] Creating temp dir"); + // 1. Create temp directory for bare repo + let temp_dir = tempfile::tempdir().expect("Failed to create temp dir for git server"); + let bare_repo_path = temp_dir.path().join("repo.git"); + + println!("[SmartGitServer::start] Cloning bare repo from {:?}", source_repo); + // 2. Create bare clone + let output = Command::new("git") + .args(["clone", "--bare"]) + .arg(source_repo) + .arg(&bare_repo_path) + .output() + .expect("Failed to run git clone --bare"); + + if !output.status.success() { + panic!( + "git clone --bare failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + println!("[SmartGitServer::start] Bare clone created"); + + // 3. Find a free port + println!("[SmartGitServer::start] Finding free port"); + let port = find_free_port(); + println!("[SmartGitServer::start] Found port {}", port); + let addr: SocketAddr = ([127, 0, 0, 1], port).into(); + + // 4. Create shutdown channel + let (shutdown_tx, mut shutdown_rx) = oneshot::channel::<()>(); + + println!("[SmartGitServer::start] Binding to {}", addr); + // 5. Start the HTTP server + let repo_path = Arc::new(bare_repo_path); + let listener = TcpListener::bind(addr) + .await + .expect("Failed to bind to address"); + println!("[SmartGitServer::start] Listener bound successfully"); + + let handle = tokio::spawn(async move { + eprintln!("[SmartGitServer] Server loop started, waiting for connections..."); + loop { + tokio::select! { + accept_result = listener.accept() => { + match accept_result { + Ok((stream, addr)) => { + eprintln!("[SmartGitServer] Accepted connection from {}", addr); + let repo_path = Arc::clone(&repo_path); + let io = TokioIo::new(stream); + + tokio::spawn(async move { + eprintln!("[SmartGitServer] Spawning handler for connection"); + let service = service_fn(move |req| { + let repo_path = Arc::clone(&repo_path); + async move { handle_smart_request(req, &repo_path).await } + }); + + eprintln!("[SmartGitServer] About to serve_connection"); + if let Err(e) = http1::Builder::new() + .serve_connection(io, service) + .await + { + // Connection errors are expected when client disconnects + if !e.to_string().contains("connection") { + eprintln!("SmartGitServer connection error: {}", e); + } + } + eprintln!("[SmartGitServer] Connection handler finished"); + }); + } + Err(e) => { + eprintln!("SmartGitServer accept error: {}", e); + } + } + } + _ = &mut shutdown_rx => { + // Shutdown signal received + eprintln!("[SmartGitServer] Shutdown signal received"); + break; + } + } + } + eprintln!("[SmartGitServer] Server loop exited"); + }); + + let url = format!("http://127.0.0.1:{}", port); + + println!("[SmartGitServer::start] Waiting for server to be ready on port {}", port); + // 6. Wait for server to be ready + wait_for_server_ready(port).await; + println!("[SmartGitServer::start] Server is ready!"); + + Self { + shutdown_tx: Some(shutdown_tx), + handle: Some(handle), + url, + port, + _temp_dir: temp_dir, + } + } + + /// Get the server URL. + /// + /// Returns the HTTP URL where the git repository is served. + /// Can be used directly with `git clone`, `git fetch`, or `git ls-remote`. + pub fn url(&self) -> &str { + &self.url + } + + /// Stop the server. + /// + /// Sends a shutdown signal and waits for the server to stop. + /// The temporary directory is cleaned up when the server is dropped. + pub async fn stop(mut self) { + // Send shutdown signal + if let Some(tx) = self.shutdown_tx.take() { + let _ = tx.send(()); + } + + // Wait for server task to complete + if let Some(handle) = self.handle.take() { + let _ = handle.await; + } + } +} + +impl Drop for SmartGitServer { + fn drop(&mut self) { + // Send shutdown signal if not already sent + if let Some(tx) = self.shutdown_tx.take() { + let _ = tx.send(()); + } + // Note: We can't await the handle in drop, but the temp_dir cleanup + // will happen automatically when _temp_dir is dropped + } +} + +/// Handle an HTTP request using the Git Smart HTTP protocol. +async fn handle_smart_request( + req: Request, + repo_path: &Path, +) -> Result>, hyper::Error> { + let path = req.uri().path(); + let query = req.uri().query().unwrap_or(""); + let method = req.method(); + + println!("[SmartGitServer] {} {} query={}", method, path, query); + eprintln!("[SmartGitServer] {} {} query={}", method, path, query); + + // Extract Git-Protocol header (for protocol version 2) + // We need to clone it to avoid borrowing issues when moving req + let git_protocol = req + .headers() + .get("Git-Protocol") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()); + + if let Some(ref proto) = git_protocol { + eprintln!("[SmartGitServer] Git-Protocol: {}", proto); + } + + // Route: GET /info/refs?service=git-upload-pack + if method == hyper::Method::GET && path.ends_with("/info/refs") { + // Parse service from query string + let service = query + .split('&') + .find_map(|param| { + let mut parts = param.splitn(2, '='); + match (parts.next(), parts.next()) { + (Some("service"), Some(svc)) => Some(svc), + _ => None, + } + }); + + match service { + Some("git-upload-pack") => { + eprintln!("[SmartGitServer] Handling info/refs for upload-pack"); + return handle_info_refs_upload_pack(repo_path, git_protocol.as_deref()).await; + } + Some("git-receive-pack") => { + // We only support upload-pack for testing (fetch/clone) + return Ok(Response::builder() + .status(StatusCode::FORBIDDEN) + .body(Full::new(Bytes::from("receive-pack not supported"))) + .unwrap()); + } + _ => { + return Ok(Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(Full::new(Bytes::from("Missing or invalid service parameter"))) + .unwrap()); + } + } + } + + // Route: POST /git-upload-pack + if method == hyper::Method::POST && path.ends_with("/git-upload-pack") { + eprintln!("[SmartGitServer] Handling POST /git-upload-pack"); + return handle_upload_pack(req, repo_path, git_protocol.as_deref()).await; + } + + // Not found + Ok(Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Full::new(Bytes::from("Not Found"))) + .unwrap()) +} + +/// Handle GET /info/refs?service=git-upload-pack +/// +/// This advertises the repository's refs to the client using the smart protocol. +async fn handle_info_refs_upload_pack( + repo_path: &Path, + git_protocol_version: Option<&str>, +) -> Result>, hyper::Error> { + use std::process::Stdio; + use tokio::process::Command as TokioCommand; + use tokio::io::AsyncReadExt; + + // Spawn git upload-pack --advertise-refs + let mut cmd = TokioCommand::new("git"); + cmd.arg("-c") + .arg("uploadpack.allowReachableSHA1InWant=true") + .arg("-c") + .arg("uploadpack.allowTipSHA1InWant=true") + .arg("upload-pack") + .arg("--advertise-refs") + .arg("--stateless-rpc"); + + // Set GIT_PROTOCOL environment variable if version 2 is requested + if let Some(version) = git_protocol_version { + cmd.env("GIT_PROTOCOL", version); + } + + cmd.arg(repo_path) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let mut child = match cmd.spawn() + { + Ok(child) => child, + Err(e) => { + eprintln!("Failed to spawn git upload-pack: {}", e); + return Ok(Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Full::new(Bytes::from("Failed to spawn git process"))) + .unwrap()); + } + }; + + // Read stdout + let mut output = Vec::new(); + if let Some(mut stdout) = child.stdout.take() { + if let Err(e) = stdout.read_to_end(&mut output).await { + eprintln!("Failed to read git output: {}", e); + } + } + + // Wait for process + let status = child.wait().await; + if let Ok(s) = &status { + if !s.success() { + eprintln!("git upload-pack --advertise-refs failed"); + } + } + + // Build response with pkt-line header + // Format: pkt-line("# service=git-upload-pack\n") + flush + git output + let mut response_body = Vec::new(); + + // First line: service advertisement + let service_line = "# service=git-upload-pack\n"; + let len = service_line.len() + 4; + response_body.extend_from_slice(format!("{:04x}", len).as_bytes()); + response_body.extend_from_slice(service_line.as_bytes()); + + // Flush packet + response_body.extend_from_slice(b"0000"); + + // Then the git output + response_body.extend_from_slice(&output); + + Ok(Response::builder() + .status(StatusCode::OK) + .header("Content-Type", "application/x-git-upload-pack-advertisement") + .header("Cache-Control", "no-cache") + .body(Full::new(Bytes::from(response_body))) + .unwrap()) +} + +/// Handle POST /git-upload-pack +/// +/// This handles the actual fetch negotiation and pack data transfer. +async fn handle_upload_pack( + req: Request, + repo_path: &Path, + git_protocol_version: Option<&str>, +) -> Result>, hyper::Error> { + use http_body_util::BodyExt; + use std::process::Stdio; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::process::Command as TokioCommand; + + // Read request body + let body_bytes = req.collect().await?.to_bytes(); + + // Spawn git upload-pack + let mut cmd = TokioCommand::new("git"); + cmd.arg("-c") + .arg("uploadpack.allowReachableSHA1InWant=true") + .arg("-c") + .arg("uploadpack.allowTipSHA1InWant=true") + .arg("upload-pack") + .arg("--stateless-rpc"); + + // Set GIT_PROTOCOL environment variable if version 2 is requested + if let Some(version) = git_protocol_version { + cmd.env("GIT_PROTOCOL", version); + } + + cmd.arg(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let mut child = match cmd.spawn() + { + Ok(child) => child, + Err(e) => { + eprintln!("Failed to spawn git upload-pack: {}", e); + return Ok(Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Full::new(Bytes::from("Failed to spawn git process"))) + .unwrap()); + } + }; + + // Write request body to stdin + if let Some(mut stdin) = child.stdin.take() { + if let Err(e) = stdin.write_all(&body_bytes).await { + eprintln!("Failed to write to git stdin: {}", e); + } + // Close stdin to signal end of input + drop(stdin); + } + + // Read stdout + let mut output = Vec::new(); + if let Some(mut stdout) = child.stdout.take() { + if let Err(e) = stdout.read_to_end(&mut output).await { + eprintln!("Failed to read git output: {}", e); + } + } + + // Read stderr for debugging + let mut stderr_output = Vec::new(); + if let Some(mut stderr) = child.stderr.take() { + let _ = stderr.read_to_end(&mut stderr_output).await; + } + + // Wait for process + let status = child.wait().await; + if let Ok(s) = &status { + if !s.success() { + let stderr_str = String::from_utf8_lossy(&stderr_output); + eprintln!("git upload-pack failed: {}", stderr_str); + } + } + + Ok(Response::builder() + .status(StatusCode::OK) + .header("Content-Type", "application/x-git-upload-pack-result") + .header("Cache-Control", "no-cache") + .body(Full::new(Bytes::from(output))) + .unwrap()) +} + +#[cfg(test)] +mod smart_git_server_tests { + use super::*; + use crate::common::purgatory_helpers::{create_test_repo_with_commit, CommitVariant}; + + #[tokio::test] + async fn test_smart_git_server_starts_and_stops() { + // Create a test repo + let temp_dir = tempfile::tempdir().expect("Failed to create temp dir"); + create_test_repo_with_commit(temp_dir.path(), CommitVariant::StateTest) + .expect("Failed to create test repo"); + + // Start server + let server = SmartGitServer::start(temp_dir.path()).await; + + // Verify URL is set + assert!(server.url().starts_with("http://127.0.0.1:")); + + // Stop server + server.stop().await; + } + + #[tokio::test] + async fn test_smart_git_server_info_refs() { + // Create a test repo + let temp_dir = tempfile::tempdir().expect("Failed to create temp dir"); + create_test_repo_with_commit(temp_dir.path(), CommitVariant::StateTest) + .expect("Failed to create test repo"); + + // Start server + let server = SmartGitServer::start(temp_dir.path()).await; + + // Fetch info/refs with service parameter + let info_refs_url = format!("{}/info/refs?service=git-upload-pack", server.url()); + let response = reqwest::get(&info_refs_url) + .await + .expect("Failed to fetch info/refs"); + + assert!( + response.status().is_success(), + "info/refs should be accessible" + ); + + // Check content type + let content_type = response + .headers() + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + assert!( + content_type.contains("application/x-git-upload-pack-advertisement"), + "Content-Type should be git-upload-pack-advertisement, got: {}", + content_type + ); + + let body = response.bytes().await.expect("Failed to read response body"); + + // Should start with service advertisement pkt-line + let body_str = String::from_utf8_lossy(&body); + assert!( + body_str.contains("# service=git-upload-pack"), + "Response should contain service advertisement, got: {}", + body_str + ); + + server.stop().await; + } + + #[tokio::test] + async fn test_smart_git_server_ls_remote() { + println!("[TEST] Starting test_smart_git_server_ls_remote"); + eprintln!("[TEST] Starting test_smart_git_server_ls_remote"); + + // Create a test repo + let temp_dir = tempfile::tempdir().expect("Failed to create temp dir"); + println!("[TEST] Created temp dir"); + let commit_hash = create_test_repo_with_commit(temp_dir.path(), CommitVariant::StateTest) + .expect("Failed to create test repo"); + println!("[TEST] Created test repo with commit {}", commit_hash); + + // Start server + println!("[TEST] About to start SmartGitServer"); + let server = SmartGitServer::start(temp_dir.path()).await; + println!("[TEST] Server started at {}", server.url()); + + // Give the server loop task a chance to print + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + // Run git ls-remote against the server + println!("[TEST] Running git ls-remote {}", server.url()); + let output = Command::new("git") + .args(["ls-remote", server.url()]) + .output() + .expect("Failed to run git ls-remote"); + println!("[TEST] git ls-remote completed"); + + assert!( + output.status.success(), + "git ls-remote should succeed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let stdout = String::from_utf8_lossy(&output.stdout); + + // Should list the main branch with the correct commit + assert!( + stdout.contains(&commit_hash), + "ls-remote output should contain commit {}, got: {}", + commit_hash, + stdout + ); + assert!( + stdout.contains("refs/heads/main"), + "ls-remote output should contain refs/heads/main, got: {}", + stdout + ); + + server.stop().await; + } + + #[tokio::test] + async fn test_smart_git_server_fetch() { + // Create a source repo with a commit + let source_dir = tempfile::tempdir().expect("Failed to create source dir"); + let commit_hash = create_test_repo_with_commit(source_dir.path(), CommitVariant::StateTest) + .expect("Failed to create test repo"); + + // Start server serving the source repo + let server = SmartGitServer::start(source_dir.path()).await; + + // Create a destination repo to fetch into + let dest_dir = tempfile::tempdir().expect("Failed to create dest dir"); + + // Initialize empty repo + let output = Command::new("git") + .args(["init"]) + .current_dir(dest_dir.path()) + .output() + .expect("Failed to init dest repo"); + assert!(output.status.success()); + + // Add the server as a remote + let output = Command::new("git") + .args(["remote", "add", "origin", server.url()]) + .current_dir(dest_dir.path()) + .output() + .expect("Failed to add remote"); + assert!(output.status.success()); + + // Fetch from the server + let output = Command::new("git") + .args(["fetch", "origin"]) + .current_dir(dest_dir.path()) + .output() + .expect("Failed to fetch"); + + assert!( + output.status.success(), + "git fetch should succeed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + // Verify the commit was fetched + let output = Command::new("git") + .args(["rev-parse", "origin/main"]) + .current_dir(dest_dir.path()) + .output() + .expect("Failed to rev-parse"); + + assert!(output.status.success()); + let fetched_commit = String::from_utf8_lossy(&output.stdout).trim().to_string(); + assert_eq!( + fetched_commit, commit_hash, + "Fetched commit should match source commit" + ); + + server.stop().await; + } + + #[tokio::test] + async fn test_smart_git_server_shallow_fetch() { + // This is the KEY test - shallow fetch requires smart HTTP protocol + + // Create a source repo with a commit + let source_dir = tempfile::tempdir().expect("Failed to create source dir"); + let commit_hash = create_test_repo_with_commit(source_dir.path(), CommitVariant::StateTest) + .expect("Failed to create test repo"); + + // Start server serving the source repo + let server = SmartGitServer::start(source_dir.path()).await; + + // Create a destination repo to fetch into + let dest_dir = tempfile::tempdir().expect("Failed to create dest dir"); + + // Initialize empty repo + let output = Command::new("git") + .args(["init"]) + .current_dir(dest_dir.path()) + .output() + .expect("Failed to init dest repo"); + assert!(output.status.success()); + + // Add the server as a remote + let output = Command::new("git") + .args(["remote", "add", "origin", server.url()]) + .current_dir(dest_dir.path()) + .output() + .expect("Failed to add remote"); + assert!(output.status.success()); + + // Shallow fetch from the server - THIS IS WHAT PURGATORY SYNC USES + let output = Command::new("git") + .args(["fetch", "--depth=1", "origin", &commit_hash]) + .current_dir(dest_dir.path()) + .output() + .expect("Failed to fetch"); + + assert!( + output.status.success(), + "git fetch --depth=1 should succeed with smart HTTP: {}", + String::from_utf8_lossy(&output.stderr) + ); + + // Verify the commit was fetched + let output = Command::new("git") + .args(["cat-file", "-t", &commit_hash]) + .current_dir(dest_dir.path()) + .output() + .expect("Failed to cat-file"); + + assert!( + output.status.success(), + "Commit should exist after shallow fetch" + ); + let object_type = String::from_utf8_lossy(&output.stdout).trim().to_string(); + assert_eq!(object_type, "commit", "Object should be a commit"); + + server.stop().await; + } +} -- cgit v1.2.3