diff options
| author | DanConwayDev <DanConwayDev@protonmail.com> | 2026-02-23 15:41:32 +0000 |
|---|---|---|
| committer | DanConwayDev <DanConwayDev@protonmail.com> | 2026-02-23 15:41:32 +0000 |
| commit | c54ce061d6d278cce8362d5af085808ca60c239b (patch) | |
| tree | ec967d6195d9f7ec4f061449596611afe3a0950f /src/git/sync.rs | |
| parent | e0ad39a489b3398f8208713bf728db0cb11475b0 (diff) | |
| parent | 113928aa84894ea8f65c247d9987527e792b32a9 (diff) | |
feat: announcement purgatory
Extends purgatory to hold repository announcements until git data arrives,
preventing empty repositories from being served to clients.
When an announcement is received, a bare repo is created immediately and the
announcement is held in purgatory. It is only promoted and served once a git
push confirms real content exists. If no push arrives before expiry, the bare
repo is deleted and the announcement is silently discarded.
Key behaviours:
- Soft expiry: announcements are hidden from clients but kept alive while git
pushes are in progress, reviving on successful push
- Expiry is extended when a matching state event or git push is observed
- NIP-09 deletion events remove announcements from purgatory
- Purgatory state (announcements, state events, PR events, expired set) is
persisted to disk on graceful shutdown and restored on startup, with elapsed
downtime subtracted from expiry deadlines
- Purgatory announcements drive StateOnly sync in the sync system so state
events are fetched from listed relays before promotion
- SyncLevel added to RepoSyncIndex to distinguish purgatory repos (StateOnly)
from promoted repos (Full L2+L3 sync)
Diffstat (limited to 'src/git/sync.rs')
| -rw-r--r-- | src/git/sync.rs | 234 |
1 files changed, 230 insertions, 4 deletions
diff --git a/src/git/sync.rs b/src/git/sync.rs index b1a9b49..c24d16b 100644 --- a/src/git/sync.rs +++ b/src/git/sync.rs | |||
| @@ -32,17 +32,20 @@ | |||
| 32 | use std::collections::{HashMap, HashSet}; | 32 | use std::collections::{HashMap, HashSet}; |
| 33 | use std::path::Path; | 33 | use std::path::Path; |
| 34 | use std::process::Command; | 34 | use std::process::Command; |
| 35 | use std::sync::Arc; | ||
| 35 | use tracing::{debug, info, warn}; | 36 | use tracing::{debug, info, warn}; |
| 36 | 37 | ||
| 37 | use nostr_sdk::Event; | 38 | use nostr_sdk::Event; |
| 38 | 39 | ||
| 39 | use crate::git::authorization::{ | 40 | use crate::git::authorization::{ |
| 40 | collect_authorized_maintainers, fetch_repository_data, RepositoryData, | 41 | collect_authorized_maintainers, fetch_repository_data, fetch_repository_data_with_purgatory, |
| 42 | RepositoryData, | ||
| 41 | }; | 43 | }; |
| 42 | use crate::git::{self, oid_exists}; | 44 | use crate::git::{self, oid_exists}; |
| 43 | use crate::nostr::builder::SharedDatabase; | 45 | use crate::nostr::builder::{Nip34WritePolicy, SharedDatabase}; |
| 44 | use crate::nostr::events::RepositoryState; | 46 | use crate::nostr::events::RepositoryState; |
| 45 | use crate::purgatory::{can_apply_state, Purgatory}; | 47 | use crate::purgatory::{can_apply_state, Purgatory}; |
| 48 | use crate::sync::rejected_index::RejectedEventsIndex; | ||
| 46 | 49 | ||
| 47 | /// Result of processing newly available git data. | 50 | /// Result of processing newly available git data. |
| 48 | /// | 51 | /// |
| @@ -51,6 +54,8 @@ use crate::purgatory::{can_apply_state, Purgatory}; | |||
| 51 | /// or from purgatory sync fetching OIDs from remote servers). | 54 | /// or from purgatory sync fetching OIDs from remote servers). |
| 52 | #[derive(Debug, Default, Clone)] | 55 | #[derive(Debug, Default, Clone)] |
| 53 | pub struct ProcessResult { | 56 | pub struct ProcessResult { |
| 57 | /// Number of announcements released from purgatory | ||
| 58 | pub announcements_released: usize, | ||
| 54 | /// Number of state events released from purgatory | 59 | /// Number of state events released from purgatory |
| 55 | pub states_released: usize, | 60 | pub states_released: usize, |
| 56 | /// Number of PR events released from purgatory | 61 | /// Number of PR events released from purgatory |
| @@ -70,11 +75,12 @@ pub struct ProcessResult { | |||
| 70 | impl ProcessResult { | 75 | impl ProcessResult { |
| 71 | /// Check if any events were released | 76 | /// Check if any events were released |
| 72 | pub fn released_any(&self) -> bool { | 77 | pub fn released_any(&self) -> bool { |
| 73 | self.states_released > 0 || self.prs_released > 0 | 78 | self.announcements_released > 0 || self.states_released > 0 || self.prs_released > 0 |
| 74 | } | 79 | } |
| 75 | 80 | ||
| 76 | /// Merge another ProcessResult into this one | 81 | /// Merge another ProcessResult into this one |
| 77 | pub fn merge(&mut self, other: ProcessResult) { | 82 | pub fn merge(&mut self, other: ProcessResult) { |
| 83 | self.announcements_released += other.announcements_released; | ||
| 78 | self.states_released += other.states_released; | 84 | self.states_released += other.states_released; |
| 79 | self.prs_released += other.prs_released; | 85 | self.prs_released += other.prs_released; |
| 80 | self.repos_synced += other.repos_synced; | 86 | self.repos_synced += other.repos_synced; |
| @@ -815,6 +821,8 @@ pub async fn process_newly_available_git_data( | |||
| 815 | local_relay: Option<&nostr_relay_builder::LocalRelay>, | 821 | local_relay: Option<&nostr_relay_builder::LocalRelay>, |
| 816 | purgatory: &Purgatory, | 822 | purgatory: &Purgatory, |
| 817 | git_data_path: &Path, | 823 | git_data_path: &Path, |
| 824 | write_policy: Option<&Nip34WritePolicy>, | ||
| 825 | rejected_events_index: Option<&Arc<RejectedEventsIndex>>, | ||
| 818 | ) -> anyhow::Result<ProcessResult> { | 826 | ) -> anyhow::Result<ProcessResult> { |
| 819 | let mut result = ProcessResult::default(); | 827 | let mut result = ProcessResult::default(); |
| 820 | 828 | ||
| @@ -836,6 +844,20 @@ pub async fn process_newly_available_git_data( | |||
| 836 | "Processing newly available git data" | 844 | "Processing newly available git data" |
| 837 | ); | 845 | ); |
| 838 | 846 | ||
| 847 | // Process announcements from purgatory | ||
| 848 | let announcement_result = process_purgatory_announcements( | ||
| 849 | &identifier, | ||
| 850 | source_repo_path, | ||
| 851 | database, | ||
| 852 | local_relay, | ||
| 853 | purgatory, | ||
| 854 | git_data_path, | ||
| 855 | write_policy, | ||
| 856 | rejected_events_index, | ||
| 857 | ) | ||
| 858 | .await; | ||
| 859 | result.merge(announcement_result); | ||
| 860 | |||
| 839 | // Process state events from purgatory | 861 | // Process state events from purgatory |
| 840 | let state_result = process_purgatory_state_events( | 862 | let state_result = process_purgatory_state_events( |
| 841 | &identifier, | 863 | &identifier, |
| @@ -863,6 +885,7 @@ pub async fn process_newly_available_git_data( | |||
| 863 | if result.released_any() { | 885 | if result.released_any() { |
| 864 | info!( | 886 | info!( |
| 865 | identifier = %identifier, | 887 | identifier = %identifier, |
| 888 | announcements_released = result.announcements_released, | ||
| 866 | states_released = result.states_released, | 889 | states_released = result.states_released, |
| 867 | prs_released = result.prs_released, | 890 | prs_released = result.prs_released, |
| 868 | repos_synced = result.repos_synced, | 891 | repos_synced = result.repos_synced, |
| @@ -907,7 +930,10 @@ async fn process_purgatory_state_events( | |||
| 907 | ); | 930 | ); |
| 908 | 931 | ||
| 909 | // Fetch repository data once for all state events | 932 | // Fetch repository data once for all state events |
| 910 | let mut db_repo_data = match fetch_repository_data(database, identifier).await { | 933 | // IMPORTANT: Use fetch_repository_data_with_purgatory to include announcements |
| 934 | // that may still be in purgatory (not yet promoted). This ensures authorization | ||
| 935 | // works correctly even if the announcement promotion happens in the same batch. | ||
| 936 | let mut db_repo_data = match fetch_repository_data_with_purgatory(database, purgatory, identifier).await { | ||
| 911 | Ok(data) => data, | 937 | Ok(data) => data, |
| 912 | Err(e) => { | 938 | Err(e) => { |
| 913 | warn!( | 939 | warn!( |
| @@ -1151,6 +1177,9 @@ async fn process_purgatory_pr_events( | |||
| 1151 | ); | 1177 | ); |
| 1152 | 1178 | ||
| 1153 | // Fetch repository data for syncing | 1179 | // Fetch repository data for syncing |
| 1180 | // NOTE: Only fetch from database, NOT purgatory. PR events should only be | ||
| 1181 | // released from purgatory when the announcement has been promoted (validated). | ||
| 1182 | // This ensures we don't accept PR events for announcements that fail validation. | ||
| 1154 | let db_repo_data = match fetch_repository_data(database, identifier).await { | 1183 | let db_repo_data = match fetch_repository_data(database, identifier).await { |
| 1155 | Ok(data) => data, | 1184 | Ok(data) => data, |
| 1156 | Err(e) => { | 1185 | Err(e) => { |
| @@ -1250,6 +1279,195 @@ async fn process_purgatory_pr_events( | |||
| 1250 | result | 1279 | result |
| 1251 | } | 1280 | } |
| 1252 | 1281 | ||
| 1282 | /// Process announcements from purgatory that can now be promoted. | ||
| 1283 | /// | ||
| 1284 | /// When git data arrives for a repository, any announcements in purgatory | ||
| 1285 | /// for that repository should be promoted to the database and served to clients. | ||
| 1286 | /// | ||
| 1287 | /// When `write_policy` and `rejected_events_index` are provided (git push path), | ||
| 1288 | /// any maintainer announcements sitting in the hot cache are re-processed immediately | ||
| 1289 | /// after the owner announcement is promoted, so they don't wait for the next sync cycle. | ||
| 1290 | async fn process_purgatory_announcements( | ||
| 1291 | identifier: &str, | ||
| 1292 | source_repo_path: &Path, | ||
| 1293 | database: &SharedDatabase, | ||
| 1294 | local_relay: Option<&nostr_relay_builder::LocalRelay>, | ||
| 1295 | purgatory: &Purgatory, | ||
| 1296 | git_data_path: &Path, | ||
| 1297 | write_policy: Option<&Nip34WritePolicy>, | ||
| 1298 | rejected_events_index: Option<&Arc<RejectedEventsIndex>>, | ||
| 1299 | ) -> ProcessResult { | ||
| 1300 | let mut result = ProcessResult::default(); | ||
| 1301 | |||
| 1302 | // Extract owner pubkey from the source repo path | ||
| 1303 | let owner_pubkey = match extract_owner_from_repo_path(source_repo_path, git_data_path) { | ||
| 1304 | Some(npub) => npub, | ||
| 1305 | None => { | ||
| 1306 | debug!( | ||
| 1307 | identifier = %identifier, | ||
| 1308 | "Could not extract owner from repo path" | ||
| 1309 | ); | ||
| 1310 | return result; | ||
| 1311 | } | ||
| 1312 | }; | ||
| 1313 | |||
| 1314 | // Parse the npub back to PublicKey | ||
| 1315 | let owner = match nostr_sdk::PublicKey::parse(&owner_pubkey) { | ||
| 1316 | Ok(pk) => pk, | ||
| 1317 | Err(e) => { | ||
| 1318 | warn!( | ||
| 1319 | identifier = %identifier, | ||
| 1320 | owner_pubkey = %owner_pubkey, | ||
| 1321 | error = %e, | ||
| 1322 | "Failed to parse owner pubkey" | ||
| 1323 | ); | ||
| 1324 | result.errors.push(format!("Failed to parse owner pubkey: {}", e)); | ||
| 1325 | return result; | ||
| 1326 | } | ||
| 1327 | }; | ||
| 1328 | |||
| 1329 | // Check if there's an announcement in purgatory for this owner and identifier | ||
| 1330 | let announcement_event = purgatory.promote_announcement(&owner, identifier); | ||
| 1331 | |||
| 1332 | if let Some(event) = announcement_event { | ||
| 1333 | // Save to database | ||
| 1334 | match database.save_event(&event).await { | ||
| 1335 | Ok(_) => { | ||
| 1336 | info!( | ||
| 1337 | identifier = %identifier, | ||
| 1338 | event_id = %event.id, | ||
| 1339 | "Promoted announcement from purgatory to database" | ||
| 1340 | ); | ||
| 1341 | |||
| 1342 | // Notify WebSocket subscribers | ||
| 1343 | if let Some(relay) = local_relay { | ||
| 1344 | if relay.notify_event(event.clone()) { | ||
| 1345 | debug!( | ||
| 1346 | identifier = %identifier, | ||
| 1347 | event_id = %event.id, | ||
| 1348 | "Broadcast announcement event to WebSocket listeners" | ||
| 1349 | ); | ||
| 1350 | } | ||
| 1351 | } | ||
| 1352 | |||
| 1353 | result.announcements_released += 1; | ||
| 1354 | |||
| 1355 | // Re-process any maintainer announcements sitting in the hot cache. | ||
| 1356 | // | ||
| 1357 | // When an owner announcement is promoted from purgatory via a git push, | ||
| 1358 | // maintainer announcements that arrived earlier (via relay sync) may have | ||
| 1359 | // been rejected and stored in the hot cache because the owner announcement | ||
| 1360 | // didn't exist in the DB yet. Now that the owner announcement is saved, | ||
| 1361 | // we must invalidate and re-process those cached events immediately. | ||
| 1362 | // | ||
| 1363 | // This only applies on the git push path (write_policy + rejected_events_index | ||
| 1364 | // are Some). The purgatory sync path already handles this via | ||
| 1365 | // SyncManager::process_event_static. | ||
| 1366 | if let (Some(wp), Some(rei), Some(relay)) = | ||
| 1367 | (write_policy, rejected_events_index, local_relay) | ||
| 1368 | { | ||
| 1369 | use crate::nostr::events::RepositoryAnnouncement; | ||
| 1370 | use nostr_relay_builder::prelude::{WritePolicy, WritePolicyResult}; | ||
| 1371 | use std::net::{IpAddr, Ipv4Addr, SocketAddr}; | ||
| 1372 | |||
| 1373 | if let Ok(announcement) = RepositoryAnnouncement::from_event(event.clone()) { | ||
| 1374 | if !announcement.maintainers.is_empty() { | ||
| 1375 | debug!( | ||
| 1376 | identifier = %identifier, | ||
| 1377 | event_id = %event.id, | ||
| 1378 | maintainer_count = announcement.maintainers.len(), | ||
| 1379 | "Owner announcement promoted via git push, checking hot cache for rejected maintainer announcements" | ||
| 1380 | ); | ||
| 1381 | |||
| 1382 | for maintainer_hex in &announcement.maintainers { | ||
| 1383 | match nostr_sdk::PublicKey::from_hex(maintainer_hex) { | ||
| 1384 | Ok(maintainer_pubkey) => { | ||
| 1385 | let (removed, hot_events) = rei.invalidate_and_get( | ||
| 1386 | &maintainer_pubkey, | ||
| 1387 | &announcement.identifier, | ||
| 1388 | Some(crate::sync::rejected_index::EventType::Announcement), | ||
| 1389 | ); | ||
| 1390 | |||
| 1391 | if removed > 0 { | ||
| 1392 | info!( | ||
| 1393 | maintainer = %maintainer_hex, | ||
| 1394 | identifier = %announcement.identifier, | ||
| 1395 | removed_from_cold_index = removed, | ||
| 1396 | hot_cache_events = hot_events.len(), | ||
| 1397 | "Invalidated rejected maintainer announcements after git push promotion" | ||
| 1398 | ); | ||
| 1399 | } | ||
| 1400 | |||
| 1401 | // Re-process events from hot cache | ||
| 1402 | let dummy_addr = SocketAddr::new( | ||
| 1403 | IpAddr::V4(Ipv4Addr::LOCALHOST), | ||
| 1404 | 0, | ||
| 1405 | ); | ||
| 1406 | for hot_event in hot_events { | ||
| 1407 | info!( | ||
| 1408 | event_id = %hot_event.id, | ||
| 1409 | maintainer = %maintainer_hex, | ||
| 1410 | identifier = %announcement.identifier, | ||
| 1411 | "Re-processing maintainer announcement from hot cache after git push promotion" | ||
| 1412 | ); | ||
| 1413 | match wp.admit_event(&hot_event, &dummy_addr).await { | ||
| 1414 | WritePolicyResult::Accept => { | ||
| 1415 | match database.save_event(&hot_event).await { | ||
| 1416 | Ok(_) => { | ||
| 1417 | relay.notify_event(hot_event.clone()); | ||
| 1418 | info!( | ||
| 1419 | event_id = %hot_event.id, | ||
| 1420 | "Maintainer announcement accepted and saved on re-processing" | ||
| 1421 | ); | ||
| 1422 | } | ||
| 1423 | Err(e) => { | ||
| 1424 | warn!( | ||
| 1425 | event_id = %hot_event.id, | ||
| 1426 | error = %e, | ||
| 1427 | "Failed to save re-processed maintainer announcement" | ||
| 1428 | ); | ||
| 1429 | } | ||
| 1430 | } | ||
| 1431 | } | ||
| 1432 | _ => { | ||
| 1433 | warn!( | ||
| 1434 | event_id = %hot_event.id, | ||
| 1435 | "Maintainer announcement still rejected on re-processing" | ||
| 1436 | ); | ||
| 1437 | } | ||
| 1438 | } | ||
| 1439 | } | ||
| 1440 | } | ||
| 1441 | Err(e) => { | ||
| 1442 | warn!( | ||
| 1443 | maintainer_hex = %maintainer_hex, | ||
| 1444 | error = %e, | ||
| 1445 | "Invalid maintainer public key in promoted announcement" | ||
| 1446 | ); | ||
| 1447 | } | ||
| 1448 | } | ||
| 1449 | } | ||
| 1450 | } | ||
| 1451 | } | ||
| 1452 | } | ||
| 1453 | } | ||
| 1454 | Err(e) => { | ||
| 1455 | warn!( | ||
| 1456 | identifier = %identifier, | ||
| 1457 | event_id = %event.id, | ||
| 1458 | error = %e, | ||
| 1459 | "Failed to save announcement to database" | ||
| 1460 | ); | ||
| 1461 | result | ||
| 1462 | .errors | ||
| 1463 | .push(format!("Failed to save announcement: {}", e)); | ||
| 1464 | } | ||
| 1465 | } | ||
| 1466 | } | ||
| 1467 | |||
| 1468 | result | ||
| 1469 | } | ||
| 1470 | |||
| 1253 | /// Extract owner pubkey from a repository path. | 1471 | /// Extract owner pubkey from a repository path. |
| 1254 | /// | 1472 | /// |
| 1255 | /// Given a path like `{git_data_path}/{npub}/{identifier}.git`, extracts the npub. | 1473 | /// Given a path like `{git_data_path}/{npub}/{identifier}.git`, extracts the npub. |
| @@ -1271,6 +1489,7 @@ mod tests { | |||
| 1271 | #[test] | 1489 | #[test] |
| 1272 | fn test_process_result_default() { | 1490 | fn test_process_result_default() { |
| 1273 | let result = ProcessResult::default(); | 1491 | let result = ProcessResult::default(); |
| 1492 | assert_eq!(result.announcements_released, 0); | ||
| 1274 | assert_eq!(result.states_released, 0); | 1493 | assert_eq!(result.states_released, 0); |
| 1275 | assert_eq!(result.prs_released, 0); | 1494 | assert_eq!(result.prs_released, 0); |
| 1276 | assert_eq!(result.repos_synced, 0); | 1495 | assert_eq!(result.repos_synced, 0); |
| @@ -1282,6 +1501,10 @@ mod tests { | |||
| 1282 | let mut result = ProcessResult::default(); | 1501 | let mut result = ProcessResult::default(); |
| 1283 | assert!(!result.released_any()); | 1502 | assert!(!result.released_any()); |
| 1284 | 1503 | ||
| 1504 | result.announcements_released = 1; | ||
| 1505 | assert!(result.released_any()); | ||
| 1506 | |||
| 1507 | result.announcements_released = 0; | ||
| 1285 | result.states_released = 1; | 1508 | result.states_released = 1; |
| 1286 | assert!(result.released_any()); | 1509 | assert!(result.released_any()); |
| 1287 | 1510 | ||
| @@ -1293,6 +1516,7 @@ mod tests { | |||
| 1293 | #[test] | 1516 | #[test] |
| 1294 | fn test_process_result_merge() { | 1517 | fn test_process_result_merge() { |
| 1295 | let mut result1 = ProcessResult { | 1518 | let mut result1 = ProcessResult { |
| 1519 | announcements_released: 0, | ||
| 1296 | states_released: 1, | 1520 | states_released: 1, |
| 1297 | prs_released: 2, | 1521 | prs_released: 2, |
| 1298 | repos_synced: 3, | 1522 | repos_synced: 3, |
| @@ -1303,6 +1527,7 @@ mod tests { | |||
| 1303 | }; | 1527 | }; |
| 1304 | 1528 | ||
| 1305 | let result2 = ProcessResult { | 1529 | let result2 = ProcessResult { |
| 1530 | announcements_released: 5, | ||
| 1306 | states_released: 10, | 1531 | states_released: 10, |
| 1307 | prs_released: 20, | 1532 | prs_released: 20, |
| 1308 | repos_synced: 30, | 1533 | repos_synced: 30, |
| @@ -1314,6 +1539,7 @@ mod tests { | |||
| 1314 | 1539 | ||
| 1315 | result1.merge(result2); | 1540 | result1.merge(result2); |
| 1316 | 1541 | ||
| 1542 | assert_eq!(result1.announcements_released, 5); | ||
| 1317 | assert_eq!(result1.states_released, 11); | 1543 | assert_eq!(result1.states_released, 11); |
| 1318 | assert_eq!(result1.prs_released, 22); | 1544 | assert_eq!(result1.prs_released, 22); |
| 1319 | assert_eq!(result1.repos_synced, 33); | 1545 | assert_eq!(result1.repos_synced, 33); |