upleb.uk

Public git repos — served from a NIP-34 GRASP relay at git.upleb.uk

summaryrefslogtreecommitdiff
path: root/src/bin/git_remote_nostr
diff options
context:
space:
mode:
Diffstat (limited to 'src/bin/git_remote_nostr')
-rw-r--r--src/bin/git_remote_nostr/fetch.rs526
-rw-r--r--src/bin/git_remote_nostr/list.rs6
2 files changed, 10 insertions, 522 deletions
diff --git a/src/bin/git_remote_nostr/fetch.rs b/src/bin/git_remote_nostr/fetch.rs
index 2cc87da..b0d6cfe 100644
--- a/src/bin/git_remote_nostr/fetch.rs
+++ b/src/bin/git_remote_nostr/fetch.rs
@@ -2,27 +2,21 @@ use core::str;
2use std::{ 2use std::{
3 collections::{HashMap, HashSet}, 3 collections::{HashMap, HashSet},
4 io::Stdin, 4 io::Stdin,
5 sync::{Arc, Mutex},
6 time::Instant,
7}; 5};
8 6
9use anyhow::{Context, Result, anyhow, bail}; 7use anyhow::{Context, Result, bail};
10use auth_git2::GitAuthenticator;
11use git2::{Progress, Repository};
12use ngit::{ 8use ngit::{
13 cli_interactor::count_lines_per_msg_vec, 9 fetch::fetch_from_git_server,
14 git::{ 10 git::{Repo, RepoActions},
15 Repo, RepoActions, 11 git_events::{
16 nostr_url::{CloneUrl, NostrUrlDecoded, ServerProtocol}, 12 KIND_PULL_REQUEST, KIND_PULL_REQUEST_UPDATE,
17 utils::check_ssh_keys, 13 identify_clone_urls_for_oids_from_pr_pr_update_events, tag_value,
18 }, 14 },
19 git_events::{KIND_PULL_REQUEST, KIND_PULL_REQUEST_UPDATE, tag_value},
20 login::get_curent_user, 15 login::get_curent_user,
21 repo_ref::{RepoRef, is_grasp_server_in_list}, 16 repo_ref::{RepoRef, is_grasp_server_in_list},
22 utils::{ 17 utils::{
23 Direction, find_proposal_and_patches_by_branch_name, get_oids_from_fetch_batch, 18 find_proposal_and_patches_by_branch_name, get_oids_from_fetch_batch,
24 get_open_or_draft_proposals, get_read_protocols_to_try, join_with_and, 19 get_open_or_draft_proposals,
25 set_protocol_preference,
26 }, 20 },
27}; 21};
28use nostr::nips::nip19; 22use nostr::nips::nip19;
@@ -126,36 +120,6 @@ pub async fn run_fetch(
126 Ok(()) 120 Ok(())
127} 121}
128 122
129async fn identify_clone_urls_for_oids_from_pr_pr_update_events(
130 oids: Vec<&String>,
131 git_repo: &Repo,
132 repo_ref: &RepoRef,
133) -> Result<HashMap<String, Vec<String>>> {
134 let mut map: HashMap<String, Vec<String>> = HashMap::new();
135
136 let open_and_draft_proposals = get_open_or_draft_proposals(git_repo, repo_ref).await?;
137
138 for (_, (_, events)) in open_and_draft_proposals {
139 for event in events {
140 if [KIND_PULL_REQUEST, KIND_PULL_REQUEST_UPDATE].contains(&event.kind) {
141 if let Ok(c) = tag_value(&event, "c") {
142 if oids.contains(&&c) {
143 for tag in event.tags.as_slice() {
144 if tag.kind().eq(&nostr::event::TagKind::Clone) {
145 for clone_url in tag.as_slice().iter().skip(1) {
146 map.entry(c.clone()).or_default().push(clone_url.clone());
147 }
148 }
149 }
150 }
151 }
152 }
153 }
154 }
155
156 Ok(map)
157}
158
159pub fn make_commits_for_proposal( 123pub fn make_commits_for_proposal(
160 git_repo: &Repo, 124 git_repo: &Repo,
161 repo_ref: &RepoRef, 125 repo_ref: &RepoRef,
@@ -235,477 +199,3 @@ async fn fetch_open_or_draft_proposals_from_patches(
235 } 199 }
236 Ok(()) 200 Ok(())
237} 201}
238
239pub fn fetch_from_git_server(
240 git_repo: &Repo,
241 oids: &[String],
242 git_server_url: &str,
243 decoded_nostr_url: &NostrUrlDecoded,
244 term: &console::Term,
245 is_grasp_server: bool,
246) -> Result<()> {
247 let already_have_oids = oids
248 .iter()
249 .all(|oid| git_repo.does_commit_exist(oid).is_ok_and(|outcome| outcome));
250 if already_have_oids {
251 return Ok(());
252 }
253
254 let server_url = git_server_url.parse::<CloneUrl>()?;
255
256 let protocols_to_attempt =
257 get_read_protocols_to_try(git_repo, &server_url, decoded_nostr_url, is_grasp_server);
258
259 let mut failed_protocols = vec![];
260 let mut success = false;
261 for protocol in &protocols_to_attempt {
262 term.write_line(
263 format!("fetching {} over {protocol}...", server_url.short_name(),).as_str(),
264 )?;
265
266 let formatted_url = server_url.format_as(protocol, &decoded_nostr_url.user)?;
267 let res = fetch_from_git_server_url(
268 &git_repo.git_repo,
269 oids,
270 &formatted_url,
271 [ServerProtocol::UnauthHttps, ServerProtocol::UnauthHttp].contains(protocol),
272 term,
273 );
274 if let Err(error) = res {
275 term.write_line(
276 format!("fetch: {formatted_url} failed over {protocol}: {error}").as_str(),
277 )?;
278 failed_protocols.push(protocol);
279 } else {
280 success = true;
281 if !failed_protocols.is_empty() {
282 term.write_line(format!("fetch: succeeded over {protocol}").as_str())?;
283 let _ = set_protocol_preference(git_repo, protocol, &server_url, &Direction::Push);
284 }
285 break;
286 }
287 }
288 if success {
289 Ok(())
290 } else {
291 let error = anyhow!(
292 "{} failed over {}{}",
293 server_url.short_name(),
294 join_with_and(&failed_protocols),
295 if decoded_nostr_url.protocol.is_some() {
296 " and nostr url contains protocol override so no other protocols were attempted"
297 } else {
298 ""
299 },
300 );
301 term.write_line(format!("fetch: {error}").as_str())?;
302 Err(error)
303 }
304}
305
306#[allow(clippy::cast_precision_loss)]
307#[allow(clippy::float_cmp)]
308#[allow(clippy::needless_pass_by_value)]
309fn report_on_transfer_progress(
310 progress_stats: &Progress<'_>,
311 start_time: &Instant,
312 end_time: Option<&Instant>,
313) -> Vec<String> {
314 let mut report = vec![];
315 let total = progress_stats.total_objects() as f64;
316 if total == 0.0 {
317 return report;
318 }
319 let received = progress_stats.received_objects() as f64;
320 let percentage = ((received / total) * 100.0)
321 // always round down because 100% complete is misleading when its not complete
322 .floor();
323
324 let received_bytes = progress_stats.received_bytes() as f64;
325
326 let (size, unit) = if received_bytes >= (1024.0 * 1024.0) {
327 (received_bytes / (1024.0 * 1024.0), "MiB")
328 } else {
329 (received_bytes / 1024.0, "KiB")
330 };
331
332 let speed = {
333 let duration = if let Some(end_time) = end_time {
334 (*end_time - *start_time).as_millis() as f64
335 } else {
336 start_time.elapsed().as_millis() as f64
337 };
338
339 if duration > 0.0 {
340 (received_bytes / (1024.0 * 1024.0)) / (duration / 1000.0) // Convert bytes to MiB and milliseconds to seconds
341 } else {
342 0.0
343 }
344 };
345
346 // Format the output for receiving objects
347 report.push(format!(
348 "Receiving objects: {percentage}% ({received}/{total}) {size:.2} {unit} | {speed:.2} MiB/s{}",
349 if received == total {
350 ", done."
351 } else { ""},
352 ));
353 if received == total {
354 let indexed_deltas = progress_stats.indexed_deltas() as f64;
355 let total_deltas = progress_stats.total_deltas() as f64;
356 let percentage = ((indexed_deltas / total_deltas) * 100.0)
357 // always round down because 100% complete is misleading when its not complete
358 .floor();
359 if total_deltas > 0.0 {
360 report.push(format!(
361 "Resolving deltas: {percentage}% ({indexed_deltas}/{total_deltas}){}",
362 if indexed_deltas == total_deltas {
363 ", done."
364 } else {
365 ""
366 },
367 ));
368 }
369 }
370 report
371}
372
373struct FetchReporter<'a> {
374 remote_msgs: Vec<String>,
375 transfer_progress_msgs: Vec<String>,
376 term: &'a console::Term,
377 start_time: Option<Instant>,
378 end_time: Option<Instant>,
379}
380impl<'a> FetchReporter<'a> {
381 fn new(term: &'a console::Term) -> Self {
382 Self {
383 remote_msgs: vec![],
384 transfer_progress_msgs: vec![],
385 term,
386 start_time: None,
387 end_time: None,
388 }
389 }
390 fn write_all(&self, lines_to_clear: usize) {
391 let _ = self.term.clear_last_lines(lines_to_clear);
392 for msg in &self.remote_msgs {
393 let _ = self.term.write_line(format!("remote: {msg}").as_str());
394 }
395 for msg in &self.transfer_progress_msgs {
396 let _ = self.term.write_line(msg);
397 }
398 }
399 fn count_all_existing_lines(&self) -> usize {
400 let width = self.term.size().1;
401 count_lines_per_msg_vec(width, &self.remote_msgs, "remote: ".len())
402 + count_lines_per_msg_vec(width, &self.transfer_progress_msgs, 0)
403 }
404 fn just_write_transfer_progress(&self, lines_to_clear: usize) {
405 let _ = self.term.clear_last_lines(lines_to_clear);
406 for msg in &self.transfer_progress_msgs {
407 let _ = self.term.write_line(msg);
408 }
409 }
410 fn just_count_transfer_progress(&self) -> usize {
411 let width = self.term.size().1;
412 count_lines_per_msg_vec(width, &self.transfer_progress_msgs, 0)
413 }
414 fn process_remote_msg(&mut self, data: &[u8]) {
415 if let Ok(data) = str::from_utf8(data) {
416 let data = data
417 .split(['\n', '\r'])
418 .map(str::trim)
419 .filter(|line| !line.trim().is_empty())
420 .collect::<Vec<&str>>();
421 for data in data {
422 let existing_lines = self.count_all_existing_lines();
423 let msg = data.to_string();
424 if let Some(last) = self.remote_msgs.last() {
425 // if previous line begins with x but doesnt finish with y then its part of the
426 // same msg
427 if (last.starts_with("Enume") && !last.ends_with(", done."))
428 || ((last.starts_with("Compre") || last.starts_with("Count"))
429 && !last.contains(')'))
430 {
431 let last = self.remote_msgs.pop().unwrap();
432 self.remote_msgs.push(format!("{last}{msg}"));
433 // if previous msg contains % and its not 100% then it
434 // should be overwritten
435 } else if (last.contains('%') && !last.contains("100%"))
436 // but also if the next message is identical with "", done." appended
437 || last == &msg.replace(", done.", "")
438 {
439 self.remote_msgs.pop();
440 self.remote_msgs.push(msg);
441 } else {
442 self.remote_msgs.push(msg);
443 }
444 } else {
445 self.remote_msgs.push(msg);
446 }
447 self.write_all(existing_lines);
448 }
449 }
450 }
451 fn process_transfer_progress_update(&mut self, progress_stats: &git2::Progress<'_>) {
452 if self.start_time.is_none() {
453 self.start_time = Some(Instant::now());
454 }
455 let existing_lines = self.just_count_transfer_progress();
456 let updated = report_on_transfer_progress(
457 progress_stats,
458 &self.start_time.unwrap(),
459 self.end_time.as_ref(),
460 );
461 if self.transfer_progress_msgs.len() <= updated.len() {
462 if self.end_time.is_none() && updated.first().is_some_and(|f| f.contains("100%")) {
463 self.end_time = Some(Instant::now());
464 }
465 // once "Resolving Deltas" is complete, deltas get reset to 0 and it stops
466 // reporting on it so we want to keep the old report
467 self.transfer_progress_msgs = updated;
468 }
469 self.just_write_transfer_progress(existing_lines);
470 }
471}
472
473fn fetch_from_git_server_url(
474 git_repo: &Repository,
475 oids: &[String],
476 git_server_url: &str,
477 dont_authenticate: bool,
478 term: &console::Term,
479) -> Result<()> {
480 if git_server_url.parse::<CloneUrl>()?.protocol() == ServerProtocol::Ssh && !check_ssh_keys() {
481 bail!("no ssh keys found");
482 }
483 let git_config = git_repo.config()?;
484 let mut git_server_remote = git_repo.remote_anonymous(git_server_url)?;
485 let auth = GitAuthenticator::default();
486 let mut fetch_options = git2::FetchOptions::new();
487 let mut remote_callbacks = git2::RemoteCallbacks::new();
488 let fetch_reporter = Arc::new(Mutex::new(FetchReporter::new(term)));
489 remote_callbacks.sideband_progress({
490 let fetch_reporter = Arc::clone(&fetch_reporter);
491 move |data| {
492 let mut reporter = fetch_reporter.lock().unwrap();
493 reporter.process_remote_msg(data);
494 true
495 }
496 });
497 remote_callbacks.transfer_progress({
498 let fetch_reporter = Arc::clone(&fetch_reporter);
499 move |stats| {
500 let mut reporter = fetch_reporter.lock().unwrap();
501 reporter.process_transfer_progress_update(&stats);
502 true
503 }
504 });
505
506 if !dont_authenticate {
507 remote_callbacks.credentials(auth.credentials(&git_config));
508 }
509 fetch_options.remote_callbacks(remote_callbacks);
510
511 git_server_remote.download(oids, Some(&mut fetch_options))?;
512
513 git_server_remote.disconnect()?;
514 Ok(())
515}
516
517#[cfg(test)]
518mod tests {
519
520 use super::*;
521
522 fn pass_through_fetch_reporter_proces_remote_msg(msgs: Vec<&str>) -> Vec<String> {
523 let term = console::Term::stdout();
524 let mut reporter = FetchReporter::new(&term);
525 for msg in msgs {
526 reporter.process_remote_msg(msg.as_bytes());
527 }
528 reporter.remote_msgs
529 }
530
531 #[test]
532 fn logs_single_msg() {
533 assert_eq!(
534 pass_through_fetch_reporter_proces_remote_msg(vec![
535 "Enumerating objects: 23716, done.",
536 ]),
537 vec!["Enumerating objects: 23716, done."]
538 );
539 }
540
541 #[test]
542 fn logs_multiple_msgs() {
543 assert_eq!(
544 pass_through_fetch_reporter_proces_remote_msg(vec![
545 "Enumerating objects: 23716, done.",
546 "Counting objects: 0% (1/2195)",
547 ]),
548 vec![
549 "Enumerating objects: 23716, done.",
550 "Counting objects: 0% (1/2195)",
551 ]
552 );
553 }
554
555 mod ignores {
556 use super::*;
557
558 #[test]
559 fn empty_msgs() {
560 assert_eq!(
561 pass_through_fetch_reporter_proces_remote_msg(vec![
562 "Enumerating objects: 23716, done.",
563 "",
564 "Counting objects: 0% (1/2195)",
565 "",
566 ]),
567 vec![
568 "Enumerating objects: 23716, done.",
569 "Counting objects: 0% (1/2195)",
570 ]
571 );
572 }
573
574 #[test]
575 fn whitespace_msgs() {
576 assert_eq!(
577 pass_through_fetch_reporter_proces_remote_msg(vec![
578 "Enumerating objects: 23716, done.",
579 " ",
580 "Counting objects: 0% (1/2195)",
581 " \r\n \r",
582 ]),
583 vec![
584 "Enumerating objects: 23716, done.",
585 "Counting objects: 0% (1/2195)",
586 ]
587 );
588 }
589 }
590
591 mod splits {
592 use super::*;
593
594 #[test]
595 fn multiple_lines_in_single_msg() {
596 assert_eq!(
597 pass_through_fetch_reporter_proces_remote_msg(vec![
598 "Enumerating objects: 23716, done.\r\nCounting objects: 0% (1/2195)",
599 "",
600 ]),
601 vec![
602 "Enumerating objects: 23716, done.",
603 "Counting objects: 0% (1/2195)",
604 ]
605 );
606 }
607 }
608
609 mod joins_lines_sent_over_multiple_msgs {
610 use super::*;
611
612 #[test]
613 fn enumerating() {
614 assert_eq!(
615 pass_through_fetch_reporter_proces_remote_msg(vec![
616 "Enumerat",
617 "ing objec",
618 "ts: 23716, done.",
619 "Counting objects: 0% (1/2195)",
620 ]),
621 vec![
622 "Enumerating objects: 23716, done.",
623 "Counting objects: 0% (1/2195)",
624 ]
625 );
626 }
627 #[test]
628 fn counting() {
629 assert_eq!(
630 pass_through_fetch_reporter_proces_remote_msg(vec![
631 "Enumerating objects: 23716, done.",
632 "Counting obj",
633 "ects: 0% (1/2195)",
634 "Count",
635 "ing objects: 1% (22/",
636 "2195)",
637 ]),
638 vec![
639 "Enumerating objects: 23716, done.",
640 "Counting objects: 1% (22/2195)",
641 ]
642 );
643 }
644 #[test]
645 fn compressing() {
646 assert_eq!(
647 pass_through_fetch_reporter_proces_remote_msg(vec![
648 "Compress",
649 "ing obj",
650 "ect",
651 "s: 0% (1/56",
652 "0)"
653 ]),
654 vec!["Compressing objects: 0% (1/560)"]
655 );
656 }
657 }
658
659 #[test]
660 fn msgs_with_pc_and_not_100pc_are_replaced() {
661 assert_eq!(
662 pass_through_fetch_reporter_proces_remote_msg(vec![
663 "Enumerating objects: 23716, done.",
664 "Counting objects: 0% (1/2195)",
665 "Counting objects: 1% (22/2195)",
666 ]),
667 vec![
668 "Enumerating objects: 23716, done.",
669 "Counting objects: 1% (22/2195)",
670 ]
671 );
672 }
673 mod msgs_with_pc_100pc_are_not_replaced {
674 use super::*;
675
676 #[test]
677 fn when_next_msg_is_not_identical_but_with_done() {
678 assert_eq!(
679 pass_through_fetch_reporter_proces_remote_msg(vec![
680 "Enumerating objects: 23716, done.",
681 "Counting objects: 0% (1/2195)",
682 "Counting objects: 1% (22/2195)",
683 "Counting objects: 100% (2195/2195)",
684 "Compressing objects: 0% (1/560)"
685 ]),
686 vec![
687 "Enumerating objects: 23716, done.",
688 "Counting objects: 100% (2195/2195)",
689 "Compressing objects: 0% (1/560)"
690 ]
691 );
692 }
693
694 #[test]
695 fn but_is_when_next_msg_is_identical_but_with_done_appended() {
696 assert_eq!(
697 pass_through_fetch_reporter_proces_remote_msg(vec![
698 "Enumerating objects: 23716, done.",
699 "Counting objects: 0% (1/2195)",
700 "Counting objects: 1% (22/2195)",
701 "Counting objects: 100% (2195/2195)",
702 "Counting objects: 100% (2195/2195), done.",
703 ]),
704 vec![
705 "Enumerating objects: 23716, done.",
706 "Counting objects: 100% (2195/2195), done.",
707 ]
708 );
709 }
710 }
711}
diff --git a/src/bin/git_remote_nostr/list.rs b/src/bin/git_remote_nostr/list.rs
index f5eaee4..be83991 100644
--- a/src/bin/git_remote_nostr/list.rs
+++ b/src/bin/git_remote_nostr/list.rs
@@ -5,6 +5,7 @@ use client::get_state_from_cache;
5use git::RepoActions; 5use git::RepoActions;
6use ngit::{ 6use ngit::{
7 client, 7 client,
8 fetch::fetch_from_git_server,
8 git::{self}, 9 git::{self},
9 git_events::{KIND_PULL_REQUEST, KIND_PULL_REQUEST_UPDATE, event_to_cover_letter, tag_value}, 10 git_events::{KIND_PULL_REQUEST, KIND_PULL_REQUEST_UPDATE, event_to_cover_letter, tag_value},
10 list::{get_ahead_behind, list_from_remotes}, 11 list::{get_ahead_behind, list_from_remotes},
@@ -14,10 +15,7 @@ use ngit::{
14}; 15};
15use repo_ref::RepoRef; 16use repo_ref::RepoRef;
16 17
17use crate::{ 18use crate::{fetch::make_commits_for_proposal, git::Repo};
18 fetch::{fetch_from_git_server, make_commits_for_proposal},
19 git::Repo,
20};
21 19
22pub async fn run_list( 20pub async fn run_list(
23 git_repo: &Repo, 21 git_repo: &Repo,
generated by cgit v1.2.3 (git 2.39.1) at 2026-06-05 02:37:28 +0000