diff options
| author | DanConwayDev <DanConwayDev@protonmail.com> | 2026-01-09 16:27:38 +0000 |
|---|---|---|
| committer | DanConwayDev <DanConwayDev@protonmail.com> | 2026-01-09 16:27:38 +0000 |
| commit | 895359aeb6746b98ff82944e4fca503f4a6e5439 (patch) | |
| tree | ea01bf45433282a365748dd3ba102879946d2426 /src/sync/metrics.rs | |
| parent | 83d29a446d96f87e5c947faf49fb33f18db4fc17 (diff) | |
feat(sync): add cleanup loops and metrics for rejected events index
Add automatic cleanup and Prometheus metrics for the two-tier rejected
events index that caches rejected announcements for re-processing.
Cleanup loops:
- Hot cache: Every 60 seconds (events expire after 2 minutes)
- Cold index: Every 24 hours (metadata expires after 7 days)
- Background task with graceful shutdown support
New Prometheus metrics (7):
- Gauges: hot_cache_current, cold_index_current
- Counters: hits, misses, hot_expired, cold_expired, invalidated
This completes the maintainer announcement re-processing feature,
reducing wait time from 24 hours to <1 second when a maintainer's
announcement arrives before the repository owner's announcement.
Memory is bounded through automatic cleanup, and comprehensive metrics
enable monitoring of hit rates, memory usage, and cleanup effectiveness.
Changes:
- src/sync/metrics.rs: Added 7 metrics with recording methods
- src/sync/rejected_index.rs: Added optional metrics support
- src/sync/mod.rs: Added cleanup background task
Tests: 248 library tests passing, 3 integration tests passing
Diffstat (limited to 'src/sync/metrics.rs')
| -rw-r--r-- | src/sync/metrics.rs | 124 |
1 files changed, 124 insertions, 0 deletions
diff --git a/src/sync/metrics.rs b/src/sync/metrics.rs index 7907d8e..a175210 100644 --- a/src/sync/metrics.rs +++ b/src/sync/metrics.rs | |||
| @@ -40,6 +40,22 @@ pub struct SyncMetrics { | |||
| 40 | relays_connected_total: IntGauge, | 40 | relays_connected_total: IntGauge, |
| 41 | /// Relays marked as dead | 41 | /// Relays marked as dead |
| 42 | relays_dead_total: IntGauge, | 42 | relays_dead_total: IntGauge, |
| 43 | |||
| 44 | // === Rejected Announcements Index Metrics === | ||
| 45 | /// Current number of entries in hot cache | ||
| 46 | rejected_announcements_hot_cache_current: IntGauge, | ||
| 47 | /// Total hot cache hits (events re-processed from cache) | ||
| 48 | rejected_announcements_hot_cache_hits_total: IntCounter, | ||
| 49 | /// Total hot cache misses (events not in cache) | ||
| 50 | rejected_announcements_hot_cache_misses_total: IntCounter, | ||
| 51 | /// Total expired entries removed from hot cache | ||
| 52 | rejected_announcements_hot_cache_expired_total: IntCounter, | ||
| 53 | /// Current number of entries in cold index | ||
| 54 | rejected_announcements_cold_index_current: IntGauge, | ||
| 55 | /// Total cold index entries expired and removed | ||
| 56 | rejected_announcements_cold_index_expired_total: IntCounter, | ||
| 57 | /// Total invalidations (maintainer announcements invalidated) | ||
| 58 | rejected_announcements_invalidated_total: IntCounter, | ||
| 43 | } | 59 | } |
| 44 | 60 | ||
| 45 | impl SyncMetrics { | 61 | impl SyncMetrics { |
| @@ -113,6 +129,49 @@ impl SyncMetrics { | |||
| 113 | ))?; | 129 | ))?; |
| 114 | registry.register(Box::new(relays_dead_total.clone()))?; | 130 | registry.register(Box::new(relays_dead_total.clone()))?; |
| 115 | 131 | ||
| 132 | // Rejected announcements metrics | ||
| 133 | let rejected_announcements_hot_cache_current = IntGauge::with_opts(Opts::new( | ||
| 134 | "ngit_sync_rejected_announcements_hot_cache_current", | ||
| 135 | "Current number of entries in hot cache (full events, 2 min expiry)", | ||
| 136 | ))?; | ||
| 137 | registry.register(Box::new(rejected_announcements_hot_cache_current.clone()))?; | ||
| 138 | |||
| 139 | let rejected_announcements_hot_cache_hits_total = IntCounter::with_opts(Opts::new( | ||
| 140 | "ngit_sync_rejected_announcements_hot_cache_hits_total", | ||
| 141 | "Total hot cache hits (events re-processed from cache)", | ||
| 142 | ))?; | ||
| 143 | registry.register(Box::new(rejected_announcements_hot_cache_hits_total.clone()))?; | ||
| 144 | |||
| 145 | let rejected_announcements_hot_cache_misses_total = IntCounter::with_opts(Opts::new( | ||
| 146 | "ngit_sync_rejected_announcements_hot_cache_misses_total", | ||
| 147 | "Total hot cache misses (events not in cache when invalidated)", | ||
| 148 | ))?; | ||
| 149 | registry.register(Box::new(rejected_announcements_hot_cache_misses_total.clone()))?; | ||
| 150 | |||
| 151 | let rejected_announcements_hot_cache_expired_total = IntCounter::with_opts(Opts::new( | ||
| 152 | "ngit_sync_rejected_announcements_hot_cache_expired_total", | ||
| 153 | "Total expired entries removed from hot cache", | ||
| 154 | ))?; | ||
| 155 | registry.register(Box::new(rejected_announcements_hot_cache_expired_total.clone()))?; | ||
| 156 | |||
| 157 | let rejected_announcements_cold_index_current = IntGauge::with_opts(Opts::new( | ||
| 158 | "ngit_sync_rejected_announcements_cold_index_current", | ||
| 159 | "Current number of entries in cold index (metadata only, 7 day expiry)", | ||
| 160 | ))?; | ||
| 161 | registry.register(Box::new(rejected_announcements_cold_index_current.clone()))?; | ||
| 162 | |||
| 163 | let rejected_announcements_cold_index_expired_total = IntCounter::with_opts(Opts::new( | ||
| 164 | "ngit_sync_rejected_announcements_cold_index_expired_total", | ||
| 165 | "Total expired entries removed from cold index", | ||
| 166 | ))?; | ||
| 167 | registry.register(Box::new(rejected_announcements_cold_index_expired_total.clone()))?; | ||
| 168 | |||
| 169 | let rejected_announcements_invalidated_total = IntCounter::with_opts(Opts::new( | ||
| 170 | "ngit_sync_rejected_announcements_invalidated_total", | ||
| 171 | "Total invalidations (maintainer announcements invalidated when owner accepted)", | ||
| 172 | ))?; | ||
| 173 | registry.register(Box::new(rejected_announcements_invalidated_total.clone()))?; | ||
| 174 | |||
| 116 | Ok(Self { | 175 | Ok(Self { |
| 117 | relay_connected, | 176 | relay_connected, |
| 118 | connection_attempts_total, | 177 | connection_attempts_total, |
| @@ -122,6 +181,13 @@ impl SyncMetrics { | |||
| 122 | relays_tracked_total, | 181 | relays_tracked_total, |
| 123 | relays_connected_total, | 182 | relays_connected_total, |
| 124 | relays_dead_total, | 183 | relays_dead_total, |
| 184 | rejected_announcements_hot_cache_current, | ||
| 185 | rejected_announcements_hot_cache_hits_total, | ||
| 186 | rejected_announcements_hot_cache_misses_total, | ||
| 187 | rejected_announcements_hot_cache_expired_total, | ||
| 188 | rejected_announcements_cold_index_current, | ||
| 189 | rejected_announcements_cold_index_expired_total, | ||
| 190 | rejected_announcements_invalidated_total, | ||
| 125 | }) | 191 | }) |
| 126 | } | 192 | } |
| 127 | 193 | ||
| @@ -293,6 +359,43 @@ impl SyncMetrics { | |||
| 293 | pub fn get_dead_count(&self) -> i64 { | 359 | pub fn get_dead_count(&self) -> i64 { |
| 294 | self.relays_dead_total.get() | 360 | self.relays_dead_total.get() |
| 295 | } | 361 | } |
| 362 | |||
| 363 | // === Rejected Announcements Recording Methods === | ||
| 364 | |||
| 365 | /// Update hot cache current size gauge. | ||
| 366 | pub fn update_hot_cache_size(&self, size: usize) { | ||
| 367 | self.rejected_announcements_hot_cache_current.set(size as i64); | ||
| 368 | } | ||
| 369 | |||
| 370 | /// Record hot cache hit (event re-processed from cache). | ||
| 371 | pub fn record_hot_cache_hit(&self) { | ||
| 372 | self.rejected_announcements_hot_cache_hits_total.inc(); | ||
| 373 | } | ||
| 374 | |||
| 375 | /// Record hot cache miss (event not in cache when invalidated). | ||
| 376 | pub fn record_hot_cache_miss(&self) { | ||
| 377 | self.rejected_announcements_hot_cache_misses_total.inc(); | ||
| 378 | } | ||
| 379 | |||
| 380 | /// Record hot cache expired entries. | ||
| 381 | pub fn record_hot_cache_expired(&self, count: usize) { | ||
| 382 | self.rejected_announcements_hot_cache_expired_total.inc_by(count as u64); | ||
| 383 | } | ||
| 384 | |||
| 385 | /// Update cold index current size gauge. | ||
| 386 | pub fn update_cold_index_size(&self, size: usize) { | ||
| 387 | self.rejected_announcements_cold_index_current.set(size as i64); | ||
| 388 | } | ||
| 389 | |||
| 390 | /// Record cold index expired entries. | ||
| 391 | pub fn record_cold_index_expired(&self, count: usize) { | ||
| 392 | self.rejected_announcements_cold_index_expired_total.inc_by(count as u64); | ||
| 393 | } | ||
| 394 | |||
| 395 | /// Record invalidation (maintainer announcement invalidated). | ||
| 396 | pub fn record_invalidation(&self, count: usize) { | ||
| 397 | self.rejected_announcements_invalidated_total.inc_by(count as u64); | ||
| 398 | } | ||
| 296 | } | 399 | } |
| 297 | 400 | ||
| 298 | #[cfg(test)] | 401 | #[cfg(test)] |
| @@ -395,4 +498,25 @@ mod tests { | |||
| 395 | let metrics2 = SyncMetrics::register(®istry); | 498 | let metrics2 = SyncMetrics::register(®istry); |
| 396 | assert!(metrics2.is_err()); | 499 | assert!(metrics2.is_err()); |
| 397 | } | 500 | } |
| 501 | |||
| 502 | #[test] | ||
| 503 | fn test_rejected_announcements_metrics() { | ||
| 504 | let registry = create_test_registry(); | ||
| 505 | let metrics = SyncMetrics::register(®istry).unwrap(); | ||
| 506 | |||
| 507 | // Test hot cache metrics | ||
| 508 | metrics.update_hot_cache_size(10); | ||
| 509 | metrics.record_hot_cache_hit(); | ||
| 510 | metrics.record_hot_cache_hit(); | ||
| 511 | metrics.record_hot_cache_miss(); | ||
| 512 | metrics.record_hot_cache_expired(5); | ||
| 513 | |||
| 514 | // Test cold index metrics | ||
| 515 | metrics.update_cold_index_size(100); | ||
| 516 | metrics.record_cold_index_expired(10); | ||
| 517 | |||
| 518 | // Test invalidation metrics | ||
| 519 | metrics.record_invalidation(3); | ||
| 520 | metrics.record_invalidation(2); | ||
| 521 | } | ||
| 398 | } | 522 | } |