Merge branch 'add-ngit-sync-cmd'

12 files changed, 662 insertions, 476 deletions

diff --git a/src/bin/git_remote_nostr/fetch.rs b/src/bin/git_remote_nostr/fetch.rs
index f3d4362..221d964 100644
--- a/src/bin/git_remote_nostr/fetch.rs
+++ b/src/bin/git_remote_nostr/fetch.rs
@@ -19,15 +19,15 @@ use ngit::{
     git_events::{KIND_PULL_REQUEST, KIND_PULL_REQUEST_UPDATE, tag_value},
     login::get_curent_user,
     repo_ref::{RepoRef, is_grasp_server},
+    utils::{
+        Direction, find_proposal_and_patches_by_branch_name, get_oids_from_fetch_batch,
+        get_open_or_draft_proposals, get_read_protocols_to_try, join_with_and,
+        set_protocol_preference,
+    },
 };
 use nostr::nips::nip19;
 use nostr_sdk::{Event, ToBech32};
 
-use crate::utils::{
-    Direction, find_proposal_and_patches_by_branch_name, get_oids_from_fetch_batch,
-    get_open_or_draft_proposals, get_read_protocols_to_try, join_with_and, set_protocol_preference,
-};
-
 pub async fn run_fetch(
     git_repo: &Repo,
     repo_ref: &RepoRef,
diff --git a/src/bin/git_remote_nostr/list.rs b/src/bin/git_remote_nostr/list.rs
index 7bdf170..df98c12 100644
--- a/src/bin/git_remote_nostr/list.rs
+++ b/src/bin/git_remote_nostr/list.rs
@@ -1,30 +1,22 @@
-use core::str;
 use std::collections::HashMap;
 
-use anyhow::{Context, Result, anyhow};
-use auth_git2::GitAuthenticator;
+use anyhow::{Context, Result};
 use client::get_state_from_cache;
 use git::RepoActions;
 use ngit::{
     client,
-    git::{
-        self,
-        nostr_url::{CloneUrl, NostrUrlDecoded, ServerProtocol},
-    },
+    git::{self},
     git_events::{KIND_PULL_REQUEST, KIND_PULL_REQUEST_UPDATE, event_to_cover_letter, tag_value},
+    list::{get_ahead_behind, list_from_remotes},
     login::get_curent_user,
-    repo_ref::{self, is_grasp_server},
+    repo_ref::{self},
+    utils::{get_open_or_draft_proposals, get_short_git_server_name},
 };
-use nostr_sdk::hashes::sha1::Hash as Sha1Hash;
 use repo_ref::RepoRef;
 
 use crate::{
     fetch::{fetch_from_git_server, make_commits_for_proposal},
     git::Repo,
-    utils::{
-        Direction, get_open_or_draft_proposals, get_read_protocols_to_try,
-        get_short_git_server_name, join_with_and, set_protocol_preference,
-    },
 };
 
 pub async fn run_list(
@@ -210,148 +202,3 @@ async fn get_open_and_draft_proposals_state(
     }
     Ok(state)
 }
-
-pub fn list_from_remotes(
-    term: &console::Term,
-    git_repo: &Repo,
-    git_servers: &Vec<String>,
-    decoded_nostr_url: &NostrUrlDecoded,
-    grasp_servers: &[String],
-) -> HashMap<String, (HashMap<String, String>, bool)> {
-    let mut remote_states = HashMap::new();
-    let mut errors = HashMap::new();
-    for url in git_servers {
-        let is_grasp_server = is_grasp_server(url, grasp_servers);
-        match list_from_remote(term, git_repo, url, decoded_nostr_url, is_grasp_server) {
-            Err(error) => {
-                errors.insert(url, error);
-            }
-            Ok(state) => {
-                remote_states.insert(url.to_string(), (state, is_grasp_server));
-            }
-        }
-    }
-    remote_states
-}
-
-pub fn list_from_remote(
-    term: &console::Term,
-    git_repo: &Repo,
-    git_server_url: &str,
-    decoded_nostr_url: &NostrUrlDecoded,
-    is_grasp_server: bool,
-) -> Result<HashMap<String, String>> {
-    let server_url = git_server_url.parse::<CloneUrl>()?;
-    let protocols_to_attempt =
-        get_read_protocols_to_try(git_repo, &server_url, decoded_nostr_url, is_grasp_server);
-
-    let mut failed_protocols = vec![];
-    let mut remote_state: Option<HashMap<String, String>> = None;
-
-    for protocol in &protocols_to_attempt {
-        term.write_line(
-            format!(
-                "fetching {} ref list over {protocol}...",
-                server_url.short_name(),
-            )
-            .as_str(),
-        )?;
-
-        let formatted_url = server_url.format_as(protocol, &decoded_nostr_url.user)?;
-        let res = list_from_remote_url(
-            git_repo,
-            &formatted_url,
-            [ServerProtocol::UnauthHttps, ServerProtocol::UnauthHttp].contains(protocol),
-            term,
-        );
-
-        match res {
-            Ok(state) => {
-                remote_state = Some(state);
-                term.clear_last_lines(1)?;
-                if !failed_protocols.is_empty() {
-                    term.write_line(
-                        format!(
-                            "list: succeeded over {protocol} from {}",
-                            server_url.short_name(),
-                        )
-                        .as_str(),
-                    )?;
-                    let _ =
-                        set_protocol_preference(git_repo, protocol, &server_url, &Direction::Fetch);
-                }
-                break;
-            }
-            Err(error) => {
-                term.clear_last_lines(1)?;
-                term.write_line(
-                    format!("list: {formatted_url} failed over {protocol}: {error}").as_str(),
-                )?;
-                failed_protocols.push(protocol);
-            }
-        }
-    }
-    if let Some(remote_state) = remote_state {
-        if failed_protocols.is_empty() {
-            term.clear_last_lines(1)?;
-        }
-        Ok(remote_state)
-    } else {
-        let error = anyhow!(
-            "{} failed over {}{}",
-            server_url.short_name(),
-            join_with_and(&failed_protocols),
-            if decoded_nostr_url.protocol.is_some() {
-                " and nostr url contains protocol override so no other protocols were attempted"
-            } else {
-                ""
-            },
-        );
-        term.write_line(format!("list: {error}").as_str())?;
-        Err(error)
-    }
-}
-
-fn list_from_remote_url(
-    git_repo: &Repo,
-    git_server_remote_url: &str,
-    dont_authenticate: bool,
-    term: &console::Term,
-) -> Result<HashMap<String, String>> {
-    let git_config = git_repo.git_repo.config()?;
-
-    let mut git_server_remote = git_repo.git_repo.remote_anonymous(git_server_remote_url)?;
-    // authentication may be required
-    let auth = GitAuthenticator::default();
-    let mut remote_callbacks = git2::RemoteCallbacks::new();
-    if !dont_authenticate {
-        remote_callbacks.credentials(auth.credentials(&git_config));
-    }
-    term.write_line("list: connecting...")?;
-    git_server_remote.connect_auth(git2::Direction::Fetch, Some(remote_callbacks), None)?;
-    term.clear_last_lines(1)?;
-    let mut state = HashMap::new();
-    for head in git_server_remote.list()? {
-        if let Some(symbolic_reference) = head.symref_target() {
-            state.insert(
-                head.name().to_string(),
-                format!("ref: {symbolic_reference}"),
-            );
-        // ignore dereferenced tags
-        } else if !head.name().to_string().ends_with("^{}") {
-            state.insert(head.name().to_string(), head.oid().to_string());
-        }
-    }
-    git_server_remote.disconnect()?;
-    Ok(state)
-}
-
-fn get_ahead_behind(
-    git_repo: &Repo,
-    base_ref_or_oid: &str,
-    latest_ref_or_oid: &str,
-) -> Result<(Vec<Sha1Hash>, Vec<Sha1Hash>)> {
-    let base = git_repo.get_commit_or_tip_of_reference(base_ref_or_oid)?;
-    let latest = git_repo.get_commit_or_tip_of_reference(latest_ref_or_oid)?;
-    git_repo.get_commits_ahead_behind(&base, &latest)
-}
diff --git a/src/bin/git_remote_nostr/main.rs b/src/bin/git_remote_nostr/main.rs
index cf47a30..aac626b 100644
--- a/src/bin/git_remote_nostr/main.rs
+++ b/src/bin/git_remote_nostr/main.rs
@@ -18,16 +18,15 @@ use ngit::{
     client::{self, Params},
     git::{self, utils::set_git_timeout},
     login::existing::load_existing_login,
+    utils::read_line,
 };
 use nostr::nips::nip19::Nip19Coordinate;
-use utils::read_line;
 
 use crate::{client::Client, git::Repo};
 
 mod fetch;
 mod list;
 mod push;
-mod utils;
 
 #[tokio::main]
 async fn main() -> Result<()> {
diff --git a/src/bin/git_remote_nostr/push.rs b/src/bin/git_remote_nostr/push.rs
index 909a0ab..4e0d760 100644
--- a/src/bin/git_remote_nostr/push.rs
+++ b/src/bin/git_remote_nostr/push.rs
@@ -2,12 +2,10 @@ use core::str;
 use std::{
     collections::{HashMap, HashSet},
     io::Stdin,
-    sync::{Arc, Mutex},
-    time::Instant,
+    sync::Arc,
 };
 
-use anyhow::{Context, Result, anyhow, bail};
-use auth_git2::GitAuthenticator;
+use anyhow::{Context, Result, bail};
 use client::{get_events_from_local_cache, get_state_from_cache, send_events, sign_event};
 use console::Term;
 use git::{RepoActions, sha1_to_oid};
@@ -17,17 +15,18 @@ use git_events::{
 };
 use git2::{Oid, Repository};
 use ngit::{
-    cli_interactor::count_lines_per_msg_vec,
     client::{self, get_event_from_cache_by_id, sign_draft_event},
-    git::{
-        self,
-        nostr_url::{CloneUrl, NostrUrlDecoded},
-        oid_to_shorthand_string,
-    },
+    git::{self, nostr_url::NostrUrlDecoded},
     git_events::{self, KIND_PULL_REQUEST, event_to_cover_letter, get_event_root},
+    list::list_from_remotes,
     login::{self, user::UserRef},
+    push::{push_to_remote, push_to_remote_url},
     repo_ref::{self, get_repo_config_from_yaml, is_grasp_server, normalize_grasp_server_url},
     repo_state,
+    utils::{
+        find_proposal_and_patches_by_branch_name, get_all_proposals, get_remote_name_by_url,
+        get_short_git_server_name, read_line,
+    },
 };
 use nostr::{event::UnsignedEvent, nips::nip10::Marker};
 use nostr_sdk::{
@@ -37,16 +36,7 @@ use nostr_sdk::{
 use repo_ref::RepoRef;
 use repo_state::RepoState;
 
-use crate::{
-    client::Client,
-    git::Repo,
-    list::list_from_remotes,
-    utils::{
-        Direction, find_proposal_and_patches_by_branch_name, get_all_proposals,
-        get_remote_name_by_url, get_short_git_server_name, get_write_protocols_to_try,
-        join_with_and, read_line, set_protocol_preference,
-    },
-};
+use crate::{client::Client, git::Repo};
 
 #[allow(clippy::too_many_lines)]
 #[allow(clippy::type_complexity)]
@@ -590,295 +580,6 @@ async fn generate_patches_or_pr_event_or_pr_updates(
     Ok(events)
 }
 
-fn push_to_remote(
-    git_repo: &Repo,
-    git_server_url: &str,
-    decoded_nostr_url: &NostrUrlDecoded,
-    remote_refspecs: &[String],
-    term: &Term,
-    is_grasp_server: bool,
-) -> Result<()> {
-    let server_url = git_server_url.parse::<CloneUrl>()?;
-    let protocols_to_attempt =
-        get_write_protocols_to_try(git_repo, &server_url, decoded_nostr_url, is_grasp_server);
-
-    let mut failed_protocols = vec![];
-    let mut success = false;
-
-    for protocol in &protocols_to_attempt {
-        term.write_line(format!("push: {} over {protocol}...", server_url.short_name(),).as_str())?;
-
-        let formatted_url = server_url.format_as(protocol, &decoded_nostr_url.user)?;
-
-        if let Err(error) = push_to_remote_url(git_repo, &formatted_url, remote_refspecs, term) {
-            term.write_line(
-                format!("push: {formatted_url} failed over {protocol}: {error}").as_str(),
-            )?;
-            failed_protocols.push(protocol);
-        } else {
-            success = true;
-            if !failed_protocols.is_empty() {
-                term.write_line(format!("push: succeeded over {protocol}").as_str())?;
-                let _ = set_protocol_preference(git_repo, protocol, &server_url, &Direction::Push);
-            }
-            break;
-        }
-    }
-    if success {
-        Ok(())
-    } else {
-        let error = anyhow!(
-            "{} failed over {}{}",
-            server_url.short_name(),
-            join_with_and(&failed_protocols),
-            if decoded_nostr_url.protocol.is_some() {
-                " and nostr url contains protocol override so no other protocols were attempted"
-            } else {
-                ""
-            },
-        );
-        term.write_line(format!("push: {error}").as_str())?;
-        Err(error)
-    }
-}
-
-fn push_to_remote_url(
-    git_repo: &Repo,
-    git_server_url: &str,
-    remote_refspecs: &[String],
-    term: &Term,
-) -> Result<()> {
-    let git_config = git_repo.git_repo.config()?;
-    let mut git_server_remote = git_repo.git_repo.remote_anonymous(git_server_url)?;
-    let auth = GitAuthenticator::default();
-    let mut push_options = git2::PushOptions::new();
-    let mut remote_callbacks = git2::RemoteCallbacks::new();
-    let push_reporter = Arc::new(Mutex::new(PushReporter::new(term)));
-
-    remote_callbacks.credentials(auth.credentials(&git_config));
-
-    remote_callbacks.push_update_reference({
-        let push_reporter = Arc::clone(&push_reporter);
-        move |name, error| {
-            let mut reporter = push_reporter.lock().unwrap();
-            if let Some(error) = error {
-                let existing_lines = reporter.count_all_existing_lines();
-                reporter.update_reference_errors.push(format!(
-                    "WARNING: {} failed to push {name} error: {error}",
-                    get_short_git_server_name(git_repo, git_server_url),
-                ));
-                reporter.write_all(existing_lines);
-            }
-            Ok(())
-        }
-    });
-
-    remote_callbacks.push_negotiation({
-        let push_reporter = Arc::clone(&push_reporter);
-        move |updates| {
-            let mut reporter = push_reporter.lock().unwrap();
-            let existing_lines = reporter.count_all_existing_lines();
-
-            for update in updates {
-                let dst_refname = update
-                    .dst_refname()
-                    .unwrap_or("")
-                    .replace("refs/heads/", "")
-                    .replace("refs/tags/", "tags/");
-                let msg = if update.dst().is_zero() {
-                    format!("push: - [delete]          {dst_refname}")
-                } else if update.src().is_zero() {
-                    if update.dst_refname().unwrap_or("").contains("refs/tags") {
-                        format!("push: * [new tag]         {dst_refname}")
-                    } else {
-                        format!("push: * [new branch]      {dst_refname}")
-                    }
-                } else {
-                    let force = remote_refspecs
-                        .iter()
-                        .any(|r| r.contains(&dst_refname) && r.contains('+'));
-                    format!(
-                        "push: {} {}..{}  {} -> {dst_refname}",
-                        if force { "+" } else { " " },
-                        oid_to_shorthand_string(update.src()).unwrap(),
-                        oid_to_shorthand_string(update.dst()).unwrap(),
-                        update
-                            .src_refname()
-                            .unwrap_or("")
-                            .replace("refs/heads/", "")
-                            .replace("refs/tags/", "tags/"),
-                    )
-                };
-                // other possibilities will result in push to fail but better reporting is
-                // needed:
-                // deleting a non-existant branch:
-                // ! [remote rejected] <old-branch-name> -> <old-branch-name> (not found)
-                // adding a branch that already exists:
-                // ! [remote rejected] <new-branch-name> -> <new-branch-name> (already exists)
-                // pushing without non-fast-forward without force:
-                // ! [rejected]        <branch-name> -> <branch-name> (non-fast-forward)
-                reporter.negotiation.push(msg);
-            }
-            reporter.write_all(existing_lines);
-            Ok(())
-        }
-    });
-
-    remote_callbacks.push_transfer_progress({
-        let push_reporter = Arc::clone(&push_reporter);
-        #[allow(clippy::cast_precision_loss)]
-        move |current, total, bytes| {
-            let mut reporter = push_reporter.lock().unwrap();
-            reporter.process_transfer_progress_update(current, total, bytes);
-        }
-    });
-
-    remote_callbacks.sideband_progress({
-        let push_reporter = Arc::clone(&push_reporter);
-        move |data| {
-            let mut reporter = push_reporter.lock().unwrap();
-            reporter.process_remote_msg(data);
-            true
-        }
-    });
-    push_options.remote_callbacks(remote_callbacks);
-    git_server_remote.push(remote_refspecs, Some(&mut push_options))?;
-    let _ = git_server_remote.disconnect();
-    Ok(())
-}
-
-#[allow(clippy::cast_precision_loss)]
-#[allow(clippy::float_cmp)]
-#[allow(clippy::needless_pass_by_value)]
-fn report_on_transfer_progress(
-    current: usize,
-    total: usize,
-    bytes: usize,
-    start_time: &Instant,
-    end_time: Option<&Instant>,
-) -> Option<String> {
-    if total == 0 {
-        return None;
-    }
-    let percentage = ((current as f64 / total as f64) * 100.0)
-        // always round down because 100% complete is misleading when its not complete
-        .floor();
-    let (size, unit) = if bytes as f64 >= (1024.0 * 1024.0) {
-        (bytes as f64 / (1024.0 * 1024.0), "MiB")
-    } else {
-        (bytes as f64 / 1024.0, "KiB")
-    };
-    let speed = {
-        let duration = if let Some(end_time) = end_time {
-            (*end_time - *start_time).as_millis() as f64
-        } else {
-            start_time.elapsed().as_millis() as f64
-        };
-
-        if duration > 0.0 {
-            (bytes as f64 / (1024.0 * 1024.0)) / (duration / 1000.0) // Convert bytes to MiB and milliseconds to seconds
-        } else {
-            0.0
-        }
-    };
-
-    Some(format!(
-        "push: Writing objects: {percentage}% ({current}/{total}) {size:.2} {unit}  | {speed:.2} MiB/s{}",
-        if current == total { ", done." } else { "" },
-    ))
-}
-
-struct PushReporter<'a> {
-    remote_msgs: Vec<String>,
-    negotiation: Vec<String>,
-    transfer_progress_msgs: Vec<String>,
-    update_reference_errors: Vec<String>,
-    term: &'a console::Term,
-    start_time: Option<Instant>,
-    end_time: Option<Instant>,
-}
-impl<'a> PushReporter<'a> {
-    fn new(term: &'a console::Term) -> Self {
-        Self {
-            remote_msgs: vec![],
-            negotiation: vec![],
-            transfer_progress_msgs: vec![],
-            update_reference_errors: vec![],
-            term,
-            start_time: None,
-            end_time: None,
-        }
-    }
-    fn write_all(&self, lines_to_clear: usize) {
-        let _ = self.term.clear_last_lines(lines_to_clear);
-        for msg in &self.remote_msgs {
-            let _ = self.term.write_line(format!("remote: {msg}").as_str());
-        }
-        for msg in &self.negotiation {
-            let _ = self.term.write_line(msg);
-        }
-        for msg in &self.transfer_progress_msgs {
-            let _ = self.term.write_line(msg);
-        }
-        for msg in &self.update_reference_errors {
-            let _ = self.term.write_line(msg);
-        }
-    }
-
-    fn count_all_existing_lines(&self) -> usize {
-        let width = self.term.size().1;
-        count_lines_per_msg_vec(width, &self.remote_msgs, "remote: ".len())
-            + count_lines_per_msg_vec(width, &self.negotiation, 0)
-            + count_lines_per_msg_vec(width, &self.transfer_progress_msgs, 0)
-            + count_lines_per_msg_vec(width, &self.update_reference_errors, 0)
-    }
-    fn process_remote_msg(&mut self, data: &[u8]) {
-        if let Ok(data) = str::from_utf8(data) {
-            let data = data
-                .split(['\n', '\r'])
-                .map(str::trim)
-                .filter(|line| !line.trim().is_empty())
-                .collect::<Vec<&str>>();
-            for data in data {
-                let existing_lines = self.count_all_existing_lines();
-                let msg = data.to_string();
-                if let Some(last) = self.remote_msgs.last() {
-                    if (last.contains('%') && !last.contains("100%"))
-                        || last == &msg.replace(", done.", "")
-                    {
-                        self.remote_msgs.pop();
-                        self.remote_msgs.push(msg);
-                    } else {
-                        self.remote_msgs.push(msg);
-                    }
-                } else {
-                    self.remote_msgs.push(msg);
-                }
-                self.write_all(existing_lines);
-            }
-        }
-    }
-    fn process_transfer_progress_update(&mut self, current: usize, total: usize, bytes: usize) {
-        if self.start_time.is_none() {
-            self.start_time = Some(Instant::now());
-        }
-        if let Some(report) = report_on_transfer_progress(
-            current,
-            total,
-            bytes,
-            &self.start_time.unwrap(),
-            self.end_time.as_ref(),
-        ) {
-            let existing_lines = self.count_all_existing_lines();
-            if report.contains("100%") {
-                self.end_time = Some(Instant::now());
-            }
-            self.transfer_progress_msgs = vec![report];
-            self.write_all(existing_lines);
-        }
-    }
-}
-
 type HashMapUrlRefspecs = HashMap<String, Vec<String>>;
 
 #[allow(clippy::too_many_lines)]
diff --git a/src/bin/ngit/cli.rs b/src/bin/ngit/cli.rs
index 843c904..76874c3 100644
--- a/src/bin/ngit/cli.rs
+++ b/src/bin/ngit/cli.rs
@@ -96,6 +96,9 @@ pub enum Commands {
     Send(sub_commands::send::SubCommandArgs),
     /// list PRs; checkout, apply or download selected
     List,
+    /// update repo git servers to reflect nostr state (add, update or delete
+    /// remote refs)
+    Sync(sub_commands::sync::SubCommandArgs),
     /// login, logout or export keys
     Account(AccountSubCommandArgs),
 }
diff --git a/src/bin/ngit/main.rs b/src/bin/ngit/main.rs
index f896e97..f07203a 100644
--- a/src/bin/ngit/main.rs
+++ b/src/bin/ngit/main.rs
@@ -32,6 +32,7 @@ async fn main() -> Result<()> {
             Commands::Init(args) => sub_commands::init::launch(&cli, args).await,
             Commands::List => sub_commands::list::launch().await,
             Commands::Send(args) => sub_commands::send::launch(&cli, args, false).await,
+            Commands::Sync(args) => sub_commands::sync::launch(args).await,
         }
     } else {
         // Handle the case where no command is provided
diff --git a/src/bin/ngit/sub_commands/mod.rs b/src/bin/ngit/sub_commands/mod.rs
index b59b88f..b2e7c9a 100644
--- a/src/bin/ngit/sub_commands/mod.rs
+++ b/src/bin/ngit/sub_commands/mod.rs
@@ -4,3 +4,4 @@ pub mod list;
 pub mod login;
 pub mod logout;
 pub mod send;
+pub mod sync;
diff --git a/src/bin/ngit/sub_commands/sync.rs b/src/bin/ngit/sub_commands/sync.rs
new file mode 100644
index 0000000..c1a3484
--- /dev/null
+++ b/src/bin/ngit/sub_commands/sync.rs
@@ -0,0 +1,161 @@
+use anyhow::{Context, Result};
+use ngit::{
+    client::{
+        Client, Connect, Params, fetching_with_report, get_repo_ref_from_cache,
+        get_state_from_cache,
+    },
+    git::{Repo, RepoActions},
+    list::{get_ahead_behind, list_from_remotes},
+    push::push_to_remote,
+    repo_ref::get_repo_coordinates_when_remote_unknown,
+    utils::get_short_git_server_name,
+};
+
+#[derive(Debug, clap::Args)]
+pub struct SubCommandArgs {
+    /// force push updates and delete refs from non-grasp git servers
+    #[arg(long, action)]
+    force: bool,
+}
+
+#[allow(clippy::too_many_lines)]
+pub async fn launch(args: &SubCommandArgs) -> Result<()> {
+    let git_repo = Repo::discover().context("failed to find a git repository")?;
+    let git_repo_path = git_repo.get_path()?;
+
+    let client = Client::new(Params::with_git_config_relay_defaults(&Some(&git_repo)));
+
+    let (nostr_remote_name, decoded_nostr_url) = git_repo
+        .get_first_nostr_remote_when_in_ngit_binary()
+        .await.context("failed to list git remotes")?
+        .context("no `nostr://` remote detected. `ngit sync` must be run from a repo with a nostr remote")?;
+
+    let repo_coordinate = get_repo_coordinates_when_remote_unknown(&git_repo, &client).await?;
+
+    let _ = fetching_with_report(git_repo_path, &client, &repo_coordinate).await?;
+
+    // TODO push announcement event, then state event to grasps
+
+    let repo_ref = get_repo_ref_from_cache(Some(git_repo_path), &repo_coordinate).await?;
+
+    let nostr_state = get_state_from_cache(Some(git_repo_path), &repo_ref).await?;
+
+    let term = console::Term::stderr();
+
+    let remote_states = list_from_remotes(
+        &term,
+        &git_repo,
+        &repo_ref.git_server,
+        &repo_ref.to_nostr_git_url(&None),
+        &repo_ref.grasp_servers(),
+    );
+
+    for (url, (remote_state, is_grasp_server)) in &remote_states {
+        let remote_name = get_short_git_server_name(&git_repo, url);
+        let mut refspecs = vec![];
+        // delete ref from remote
+        let mut not_deleted = vec![];
+        for remote_ref_name in remote_state.keys() {
+            if (!remote_ref_name.starts_with("refs/heads/pr/")
+                && (remote_ref_name.starts_with("refs/heads/")
+                    || remote_ref_name.starts_with("refs/tags/")))
+                && !nostr_state
+                    .state
+                    .keys()
+                    .any(|nostr_ref| nostr_ref.eq(remote_ref_name))
+            {
+                if *is_grasp_server || args.force {
+                    // delete branches / tags not on nostr
+                    refspecs.push(format!(":{remote_ref_name}"));
+                } else {
+                    not_deleted.push(remote_ref_name);
+                }
+            }
+        }
+        // add or update ref on remote
+        let mut not_updated = vec![];
+        for nostr_ref_name in nostr_state.state.keys() {
+            if nostr_ref_name.starts_with("refs/heads/")
+                || nostr_ref_name.starts_with("refs/tags/")
+                || !nostr_ref_name.starts_with("refs/heads/pr/")
+            {
+                // ensure nostr_state only supports refs/heads and refs/tags/
+                // and not refs/heads/prs/*
+            } else if let Some(remote_ref_value) = remote_state.get(nostr_ref_name) {
+                // update ref
+                let force_required = {
+                    if let Ok((ahead, _)) =
+                        get_ahead_behind(&git_repo, nostr_ref_name, remote_ref_value)
+                    {
+                        !ahead.is_empty()
+                    } else {
+                        true
+                    }
+                };
+                if nostr_state
+                    .state
+                    .get(nostr_ref_name)
+                    .is_none_or(|nostr_ref_value| nostr_ref_value.eq(remote_ref_value))
+                {
+                    // no action if ref in sync
+                } else if remote_ref_value.starts_with("ref ") && !(args.force || *is_grasp_server)
+                {
+                    // dont try and sync push symbolic refs
+                } else if !force_required {
+                    refspecs.push(format!(
+                        "refs/remotes/{nostr_remote_name}/{nostr_ref_name}:{nostr_ref_name}",
+                    ));
+                } else if *is_grasp_server || args.force {
+                    refspecs.push(format!(
+                        "+refs/remotes/{nostr_remote_name}/{nostr_ref_name}:{nostr_ref_name}",
+                    ));
+                } else {
+                    not_updated.push(nostr_ref_name);
+                }
+            } else {
+                // add missing refs
+                refspecs.push(format!(
+                    "refs/remotes/{nostr_remote_name}/{nostr_ref_name}:{nostr_ref_name}",
+                ));
+            }
+        }
+
+        if refspecs.is_empty() {
+            if !not_updated.is_empty() || !not_deleted.is_empty() {
+                term.write_line(&format!("{remote_name} in sync excluding"))?;
+            } else {
+                term.write_line(&format!("{remote_name} already in sync"))?;
+            }
+            // report already in sync
+        } else if let Err(error) = push_to_remote(
+            &git_repo,
+            url,
+            &decoded_nostr_url,
+            &refspecs,
+            &term,
+            *is_grasp_server,
+        ) {
+            term.write_line(&format!(
+                "error pushing updates to {remote_name}: error: {error}"
+            ))?;
+        } else if *is_grasp_server || args.force {
+            term.write_line(&format!("{remote_name} sync completed"))?;
+            // TODO we only know if there was an error but not if it
+            // rejected any updates
+        } else {
+            // we should report on refs not force pushed
+            term.write_line(&format!("{remote_name} sync completed"))?;
+        }
+        for name in &not_deleted {
+            term.write_line(&format!("  - {name} not deleted"))?;
+        }
+        for name in &not_updated {
+            term.write_line(&format!("  - {name} not updated due to conflicts"))?;
+        }
+        if !not_updated.is_empty() || !not_deleted.is_empty() {
+            term.write_line("run `ngit sync --force` to delete refs or overwrite conflicts and potentially lose work")?;
+        }
+    }
+
+    Ok(())
+}
diff --git a/src/lib/list.rs b/src/lib/list.rs
new file mode 100644
index 0000000..b940546
--- /dev/null
+++ b/src/lib/list.rs
@@ -0,0 +1,159 @@
+use std::collections::HashMap;
+
+use anyhow::{Result, anyhow};
+use auth_git2::GitAuthenticator;
+use nostr::hashes::sha1::Hash as Sha1Hash;
+
+use crate::{
+    git::{
+        Repo, RepoActions,
+        nostr_url::{CloneUrl, NostrUrlDecoded, ServerProtocol},
+    },
+    repo_ref::is_grasp_server,
+    utils::{Direction, get_read_protocols_to_try, join_with_and, set_protocol_preference},
+};
+
+pub fn list_from_remotes(
+    term: &console::Term,
+    git_repo: &Repo,
+    git_servers: &Vec<String>,
+    decoded_nostr_url: &NostrUrlDecoded,
+    grasp_servers: &[String],
+) -> HashMap<String, (HashMap<String, String>, bool)> {
+    let mut remote_states = HashMap::new();
+    let mut errors = HashMap::new();
+    for url in git_servers {
+        let is_grasp_server = is_grasp_server(url, grasp_servers);
+        match list_from_remote(term, git_repo, url, decoded_nostr_url, is_grasp_server) {
+            Err(error) => {
+                errors.insert(url, error);
+            }
+            Ok(state) => {
+                remote_states.insert(url.to_string(), (state, is_grasp_server));
+            }
+        }
+    }
+    remote_states
+}
+
+pub fn list_from_remote(
+    term: &console::Term,
+    git_repo: &Repo,
+    git_server_url: &str,
+    decoded_nostr_url: &NostrUrlDecoded,
+    is_grasp_server: bool,
+) -> Result<HashMap<String, String>> {
+    let server_url = git_server_url.parse::<CloneUrl>()?;
+    let protocols_to_attempt =
+        get_read_protocols_to_try(git_repo, &server_url, decoded_nostr_url, is_grasp_server);
+
+    let mut failed_protocols = vec![];
+    let mut remote_state: Option<HashMap<String, String>> = None;
+
+    for protocol in &protocols_to_attempt {
+        term.write_line(
+            format!(
+                "fetching {} ref list over {protocol}...",
+                server_url.short_name(),
+            )
+            .as_str(),
+        )?;
+
+        let formatted_url = server_url.format_as(protocol, &decoded_nostr_url.user)?;
+        let res = list_from_remote_url(
+            git_repo,
+            &formatted_url,
+            [ServerProtocol::UnauthHttps, ServerProtocol::UnauthHttp].contains(protocol),
+            term,
+        );
+
+        match res {
+            Ok(state) => {
+                remote_state = Some(state);
+                term.clear_last_lines(1)?;
+                if !failed_protocols.is_empty() {
+                    term.write_line(
+                        format!(
+                            "list: succeeded over {protocol} from {}",
+                            server_url.short_name(),
+                        )
+                        .as_str(),
+                    )?;
+                    let _ =
+                        set_protocol_preference(git_repo, protocol, &server_url, &Direction::Fetch);
+                }
+                break;
+            }
+            Err(error) => {
+                term.clear_last_lines(1)?;
+                term.write_line(
+                    format!("list: {formatted_url} failed over {protocol}: {error}").as_str(),
+                )?;
+                failed_protocols.push(protocol);
+            }
+        }
+    }
+    if let Some(remote_state) = remote_state {
+        if failed_protocols.is_empty() {
+            term.clear_last_lines(1)?;
+        }
+        Ok(remote_state)
+    } else {
+        let error = anyhow!(
+            "{} failed over {}{}",
+            server_url.short_name(),
+            join_with_and(&failed_protocols),
+            if decoded_nostr_url.protocol.is_some() {
+                " and nostr url contains protocol override so no other protocols were attempted"
+            } else {
+                ""
+            },
+        );
+        term.write_line(format!("list: {error}").as_str())?;
+        Err(error)
+    }
+}
+
+fn list_from_remote_url(
+    git_repo: &Repo,
+    git_server_remote_url: &str,
+    dont_authenticate: bool,
+    term: &console::Term,
+) -> Result<HashMap<String, String>> {
+    let git_config = git_repo.git_repo.config()?;
+
+    let mut git_server_remote = git_repo.git_repo.remote_anonymous(git_server_remote_url)?;
+    // authentication may be required
+    let auth = GitAuthenticator::default();
+    let mut remote_callbacks = git2::RemoteCallbacks::new();
+    if !dont_authenticate {
+        remote_callbacks.credentials(auth.credentials(&git_config));
+    }
+    term.write_line("list: connecting...")?;
+    git_server_remote.connect_auth(git2::Direction::Fetch, Some(remote_callbacks), None)?;
+    term.clear_last_lines(1)?;
+    let mut state = HashMap::new();
+    for head in git_server_remote.list()? {
+        if let Some(symbolic_reference) = head.symref_target() {
+            state.insert(
+                head.name().to_string(),
+                format!("ref: {symbolic_reference}"),
+            );
+        // ignore dereferenced tags
+        } else if !head.name().to_string().ends_with("^{}") {
+            state.insert(head.name().to_string(), head.oid().to_string());
+        }
+    }
+    git_server_remote.disconnect()?;
+    Ok(state)
+}
+
+pub fn get_ahead_behind(
+    git_repo: &Repo,
+    base_ref_or_oid: &str,
+    latest_ref_or_oid: &str,
+) -> Result<(Vec<Sha1Hash>, Vec<Sha1Hash>)> {
+    let base = git_repo.get_commit_or_tip_of_reference(base_ref_or_oid)?;
+    let latest = git_repo.get_commit_or_tip_of_reference(latest_ref_or_oid)?;
+    git_repo.get_commits_ahead_behind(&base, &latest)
+}
diff --git a/src/lib/mod.rs b/src/lib/mod.rs
index 7c7bd6a..265dd6b 100644
--- a/src/lib/mod.rs
+++ b/src/lib/mod.rs
@@ -2,9 +2,12 @@ pub mod cli_interactor;
 pub mod client;
 pub mod git;
 pub mod git_events;
+pub mod list;
 pub mod login;
+pub mod push;
 pub mod repo_ref;
 pub mod repo_state;
+pub mod utils;
 
 use anyhow::{Result, anyhow};
 use directories::ProjectDirs;
diff --git a/src/lib/push.rs b/src/lib/push.rs
new file mode 100644
index 0000000..0d0ec93
--- /dev/null
+++ b/src/lib/push.rs
@@ -0,0 +1,310 @@
+use std::{
+    sync::{Arc, Mutex},
+    time::Instant,
+};
+
+use anyhow::{Result, anyhow};
+use auth_git2::GitAuthenticator;
+use console::Term;
+
+use crate::{
+    cli_interactor::count_lines_per_msg_vec,
+    git::{
+        Repo,
+        nostr_url::{CloneUrl, NostrUrlDecoded},
+        oid_to_shorthand_string,
+    },
+    utils::{
+        Direction, get_short_git_server_name, get_write_protocols_to_try, join_with_and,
+        set_protocol_preference,
+    },
+};
+
+pub fn push_to_remote(
+    git_repo: &Repo,
+    git_server_url: &str,
+    decoded_nostr_url: &NostrUrlDecoded,
+    remote_refspecs: &[String],
+    term: &Term,
+    is_grasp_server: bool,
+) -> Result<()> {
+    let server_url = git_server_url.parse::<CloneUrl>()?;
+    let protocols_to_attempt =
+        get_write_protocols_to_try(git_repo, &server_url, decoded_nostr_url, is_grasp_server);
+
+    let mut failed_protocols = vec![];
+    let mut success = false;
+
+    for protocol in &protocols_to_attempt {
+        term.write_line(format!("push: {} over {protocol}...", server_url.short_name(),).as_str())?;
+
+        let formatted_url = server_url.format_as(protocol, &decoded_nostr_url.user)?;
+
+        if let Err(error) = push_to_remote_url(git_repo, &formatted_url, remote_refspecs, term) {
+            term.write_line(
+                format!("push: {formatted_url} failed over {protocol}: {error}").as_str(),
+            )?;
+            failed_protocols.push(protocol);
+        } else {
+            success = true;
+            if !failed_protocols.is_empty() {
+                term.write_line(format!("push: succeeded over {protocol}").as_str())?;
+                let _ = set_protocol_preference(git_repo, protocol, &server_url, &Direction::Push);
+            }
+            break;
+        }
+    }
+    if success {
+        Ok(())
+    } else {
+        let error = anyhow!(
+            "{} failed over {}{}",
+            server_url.short_name(),
+            join_with_and(&failed_protocols),
+            if decoded_nostr_url.protocol.is_some() {
+                " and nostr url contains protocol override so no other protocols were attempted"
+            } else {
+                ""
+            },
+        );
+        term.write_line(format!("push: {error}").as_str())?;
+        Err(error)
+    }
+}
+
+pub fn push_to_remote_url(
+    git_repo: &Repo,
+    git_server_url: &str,
+    remote_refspecs: &[String],
+    term: &Term,
+) -> Result<()> {
+    let git_config = git_repo.git_repo.config()?;
+    let mut git_server_remote = git_repo.git_repo.remote_anonymous(git_server_url)?;
+    let auth = GitAuthenticator::default();
+    let mut push_options = git2::PushOptions::new();
+    let mut remote_callbacks = git2::RemoteCallbacks::new();
+    let push_reporter = Arc::new(Mutex::new(PushReporter::new(term)));
+
+    remote_callbacks.credentials(auth.credentials(&git_config));
+
+    remote_callbacks.push_update_reference({
+        let push_reporter = Arc::clone(&push_reporter);
+        move |name, error| {
+            let mut reporter = push_reporter.lock().unwrap();
+            if let Some(error) = error {
+                let existing_lines = reporter.count_all_existing_lines();
+                reporter.update_reference_errors.push(format!(
+                    "WARNING: {} failed to push {name} error: {error}",
+                    get_short_git_server_name(git_repo, git_server_url),
+                ));
+                reporter.write_all(existing_lines);
+            }
+            Ok(())
+        }
+    });
+
+    remote_callbacks.push_negotiation({
+        let push_reporter = Arc::clone(&push_reporter);
+        move |updates| {
+            let mut reporter = push_reporter.lock().unwrap();
+            let existing_lines = reporter.count_all_existing_lines();
+
+            for update in updates {
+                let dst_refname = update
+                    .dst_refname()
+                    .unwrap_or("")
+                    .replace("refs/heads/", "")
+                    .replace("refs/tags/", "tags/");
+                let msg = if update.dst().is_zero() {
+                    format!("push: - [delete]          {dst_refname}")
+                } else if update.src().is_zero() {
+                    if update.dst_refname().unwrap_or("").contains("refs/tags") {
+                        format!("push: * [new tag]         {dst_refname}")
+                    } else {
+                        format!("push: * [new branch]      {dst_refname}")
+                    }
+                } else {
+                    let force = remote_refspecs
+                        .iter()
+                        .any(|r| r.contains(&dst_refname) && r.contains('+'));
+                    format!(
+                        "push: {} {}..{}  {} -> {dst_refname}",
+                        if force { "+" } else { " " },
+                        oid_to_shorthand_string(update.src()).unwrap(),
+                        oid_to_shorthand_string(update.dst()).unwrap(),
+                        update
+                            .src_refname()
+                            .unwrap_or("")
+                            .replace("refs/heads/", "")
+                            .replace("refs/tags/", "tags/"),
+                    )
+                };
+                // other possibilities will result in push to fail but better reporting is
+                // needed:
+                // deleting a non-existant branch:
+                // ! [remote rejected] <old-branch-name> -> <old-branch-name> (not found)
+                // adding a branch that already exists:
+                // ! [remote rejected] <new-branch-name> -> <new-branch-name> (already exists)
+                // pushing without non-fast-forward without force:
+                // ! [rejected]        <branch-name> -> <branch-name> (non-fast-forward)
+                reporter.negotiation.push(msg);
+            }
+            reporter.write_all(existing_lines);
+            Ok(())
+        }
+    });
+
+    remote_callbacks.push_transfer_progress({
+        let push_reporter = Arc::clone(&push_reporter);
+        #[allow(clippy::cast_precision_loss)]
+        move |current, total, bytes| {
+            let mut reporter = push_reporter.lock().unwrap();
+            reporter.process_transfer_progress_update(current, total, bytes);
+        }
+    });
+
+    remote_callbacks.sideband_progress({
+        let push_reporter = Arc::clone(&push_reporter);
+        move |data| {
+            let mut reporter = push_reporter.lock().unwrap();
+            reporter.process_remote_msg(data);
+            true
+        }
+    });
+    push_options.remote_callbacks(remote_callbacks);
+    git_server_remote.push(remote_refspecs, Some(&mut push_options))?;
+    let _ = git_server_remote.disconnect();
+    Ok(())
+}
+
+#[allow(clippy::cast_precision_loss)]
+#[allow(clippy::float_cmp)]
+#[allow(clippy::needless_pass_by_value)]
+fn report_on_transfer_progress(
+    current: usize,
+    total: usize,
+    bytes: usize,
+    start_time: &Instant,
+    end_time: Option<&Instant>,
+) -> Option<String> {
+    if total == 0 {
+        return None;
+    }
+    let percentage = ((current as f64 / total as f64) * 100.0)
+        // always round down because 100% complete is misleading when its not complete
+        .floor();
+    let (size, unit) = if bytes as f64 >= (1024.0 * 1024.0) {
+        (bytes as f64 / (1024.0 * 1024.0), "MiB")
+    } else {
+        (bytes as f64 / 1024.0, "KiB")
+    };
+    let speed = {
+        let duration = if let Some(end_time) = end_time {
+            (*end_time - *start_time).as_millis() as f64
+        } else {
+            start_time.elapsed().as_millis() as f64
+        };
+
+        if duration > 0.0 {
+            (bytes as f64 / (1024.0 * 1024.0)) / (duration / 1000.0) // Convert bytes to MiB and milliseconds to seconds
+        } else {
+            0.0
+        }
+    };
+
+    Some(format!(
+        "push: Writing objects: {percentage}% ({current}/{total}) {size:.2} {unit}  | {speed:.2} MiB/s{}",
+        if current == total { ", done." } else { "" },
+    ))
+}
+
+pub struct PushReporter<'a> {
+    remote_msgs: Vec<String>,
+    negotiation: Vec<String>,
+    transfer_progress_msgs: Vec<String>,
+    update_reference_errors: Vec<String>,
+    term: &'a console::Term,
+    start_time: Option<Instant>,
+    end_time: Option<Instant>,
+}
+impl<'a> PushReporter<'a> {
+    fn new(term: &'a console::Term) -> Self {
+        Self {
+            remote_msgs: vec![],
+            negotiation: vec![],
+            transfer_progress_msgs: vec![],
+            update_reference_errors: vec![],
+            term,
+            start_time: None,
+            end_time: None,
+        }
+    }
+    fn write_all(&self, lines_to_clear: usize) {
+        let _ = self.term.clear_last_lines(lines_to_clear);
+        for msg in &self.remote_msgs {
+            let _ = self.term.write_line(format!("remote: {msg}").as_str());
+        }
+        for msg in &self.negotiation {
+            let _ = self.term.write_line(msg);
+        }
+        for msg in &self.transfer_progress_msgs {
+            let _ = self.term.write_line(msg);
+        }
+        for msg in &self.update_reference_errors {
+            let _ = self.term.write_line(msg);
+        }
+    }
+
+    fn count_all_existing_lines(&self) -> usize {
+        let width = self.term.size().1;
+        count_lines_per_msg_vec(width, &self.remote_msgs, "remote: ".len())
+            + count_lines_per_msg_vec(width, &self.negotiation, 0)
+            + count_lines_per_msg_vec(width, &self.transfer_progress_msgs, 0)
+            + count_lines_per_msg_vec(width, &self.update_reference_errors, 0)
+    }
+    fn process_remote_msg(&mut self, data: &[u8]) {
+        if let Ok(data) = str::from_utf8(data) {
+            let data = data
+                .split(['\n', '\r'])
+                .map(str::trim)
+                .filter(|line| !line.trim().is_empty())
+                .collect::<Vec<&str>>();
+            for data in data {
+                let existing_lines = self.count_all_existing_lines();
+                let msg = data.to_string();
+                if let Some(last) = self.remote_msgs.last() {
+                    if (last.contains('%') && !last.contains("100%"))
+                        || last == &msg.replace(", done.", "")
+                    {
+                        self.remote_msgs.pop();
+                        self.remote_msgs.push(msg);
+                    } else {
+                        self.remote_msgs.push(msg);
+                    }
+                } else {
+                    self.remote_msgs.push(msg);
+                }
+                self.write_all(existing_lines);
+            }
+        }
+    }
+    fn process_transfer_progress_update(&mut self, current: usize, total: usize, bytes: usize) {
+        if self.start_time.is_none() {
+            self.start_time = Some(Instant::now());
+        }
+        if let Some(report) = report_on_transfer_progress(
+            current,
+            total,
+            bytes,
+            &self.start_time.unwrap(),
+            self.end_time.as_ref(),
+        ) {
+            let existing_lines = self.count_all_existing_lines();
+            if report.contains("100%") {
+                self.end_time = Some(Instant::now());
+            }
+            self.transfer_progress_msgs = vec![report];
+            self.write_all(existing_lines);
+        }
+    }
+}
diff --git a/src/bin/git_remote_nostr/utils.rs b/src/lib/utils.rs
index 2cb85bf..431757f 100644
--- a/src/bin/git_remote_nostr/utils.rs
+++ b/src/lib/utils.rs
@@ -8,7 +8,9 @@ use std::{
 
 use anyhow::{Context, Result, bail};
 use git2::Repository;
-use ngit::{
+use nostr_sdk::{Event, EventId, Kind, PublicKey, Url};
+
+use crate::{
     client::{
         get_all_proposal_patch_pr_pr_update_events_from_cache, get_events_from_local_cache,
         get_proposals_and_revisions_from_cache,
@@ -23,7 +25,6 @@ use ngit::{
     },
     repo_ref::RepoRef,
 };
-use nostr_sdk::{Event, EventId, Kind, PublicKey, Url};
 
 pub fn get_short_git_server_name(git_repo: &Repo, url: &str) -> std::string::String {
     if let Ok(name) = get_remote_name_by_url(&git_repo.git_repo, url) {