max / makenotwork
13 files changed,
+127 insertions,
-155 deletions
| @@ -43,22 +43,26 @@ pub(crate) enum HistoryKind { | |||
| 43 | 43 | }, | |
| 44 | 44 | } | |
| 45 | 45 | ||
| 46 | + | /// Resolve an optional `--target` filter to the target names to act on. | |
| 47 | + | /// `None` selects every configured target; an unknown name is an error rather | |
| 48 | + | /// than a bare `process::exit`, so every command reports failure the same way. | |
| 49 | + | fn resolve_targets(config: &Config, target: Option<&str>) -> Result<Vec<String>> { | |
| 50 | + | match target { | |
| 51 | + | Some(t) if config.get_target(t).is_none() => { | |
| 52 | + | Err(PomError::Config(format!("Unknown target: {t}"))) | |
| 53 | + | } | |
| 54 | + | Some(t) => Ok(vec![t.to_string()]), | |
| 55 | + | None => Ok(config.target_names()), | |
| 56 | + | } | |
| 57 | + | } | |
| 58 | + | ||
| 46 | 59 | pub(crate) async fn cmd_health( | |
| 47 | 60 | pool: &sqlx::SqlitePool, | |
| 48 | 61 | config: &Config, | |
| 49 | 62 | target: Option<&str>, | |
| 50 | 63 | json: bool, | |
| 51 | 64 | ) -> Result<()> { | |
| 52 | - | let targets: Vec<String> = match target { | |
| 53 | - | Some(t) => { | |
| 54 | - | if config.get_target(t).is_none() { | |
| 55 | - | eprintln!("Unknown target: {t}"); | |
| 56 | - | std::process::exit(1); | |
| 57 | - | } | |
| 58 | - | vec![t.to_string()] | |
| 59 | - | } | |
| 60 | - | None => config.target_names(), | |
| 61 | - | }; | |
| 65 | + | let targets = resolve_targets(config, target)?; | |
| 62 | 66 | ||
| 63 | 67 | let mut snapshots = Vec::new(); | |
| 64 | 68 | ||
| @@ -182,16 +186,7 @@ pub(crate) async fn cmd_dns( | |||
| 182 | 186 | target: Option<&str>, | |
| 183 | 187 | json: bool, | |
| 184 | 188 | ) -> Result<()> { | |
| 185 | - | let targets: Vec<String> = match target { | |
| 186 | - | Some(t) => { | |
| 187 | - | if config.get_target(t).is_none() { | |
| 188 | - | eprintln!("Unknown target: {t}"); | |
| 189 | - | std::process::exit(1); | |
| 190 | - | } | |
| 191 | - | vec![t.to_string()] | |
| 192 | - | } | |
| 193 | - | None => config.target_names(), | |
| 194 | - | }; | |
| 189 | + | let targets = resolve_targets(config, target)?; | |
| 195 | 190 | ||
| 196 | 191 | let mut all_dns_results = Vec::new(); | |
| 197 | 192 | let mut all_whois_results = Vec::new(); |
| @@ -10,6 +10,7 @@ use pom::error::Result; | |||
| 10 | 10 | use pom::peer; | |
| 11 | 11 | ||
| 12 | 12 | use super::tasks; | |
| 13 | + | use super::tasks::CheckInterval; | |
| 13 | 14 | ||
| 14 | 15 | pub(crate) async fn cmd_serve( | |
| 15 | 16 | pool: &sqlx::SqlitePool, | |
| @@ -96,13 +97,8 @@ pub(crate) async fn cmd_serve( | |||
| 96 | 97 | let cancel = token.clone(); | |
| 97 | 98 | handles.push(tokio::spawn(async move { | |
| 98 | 99 | const MAX_ATTEMPTS: i64 = 8; // ~ several hours of backoff before dead-letter | |
| 99 | - | let mut interval = tokio::time::interval(std::time::Duration::from_secs(30)); | |
| 100 | - | interval.tick().await; // consume the immediate first tick | |
| 101 | - | loop { | |
| 102 | - | tokio::select! { | |
| 103 | - | _ = cancel.cancelled() => break, | |
| 104 | - | _ = interval.tick() => {} | |
| 105 | - | } | |
| 100 | + | let mut ticks = CheckInterval::new(30, cancel); | |
| 101 | + | while ticks.next().await { | |
| 106 | 102 | let due = db::due_pending_alerts(&pool, 50).await.unwrap_or_default(); | |
| 107 | 103 | for p in due { | |
| 108 | 104 | if retry_alerter.retry_pending(&p).await { |
| @@ -8,6 +8,8 @@ use pom::checks::backup; | |||
| 8 | 8 | use pom::config::Config; | |
| 9 | 9 | use pom::db; | |
| 10 | 10 | ||
| 11 | + | use super::{configured_targets, CheckInterval}; | |
| 12 | + | ||
| 11 | 13 | pub(crate) fn spawn_backup_tasks( | |
| 12 | 14 | config: &Config, | |
| 13 | 15 | pool: &sqlx::SqlitePool, | |
| @@ -16,8 +18,7 @@ pub(crate) fn spawn_backup_tasks( | |||
| 16 | 18 | ) -> Vec<JoinHandle<()>> { | |
| 17 | 19 | let mut handles = Vec::new(); | |
| 18 | 20 | ||
| 19 | - | for name in config.target_names() { | |
| 20 | - | let target_config = config.get_target(&name).unwrap().clone(); | |
| 21 | + | for (name, target_config) in configured_targets(config) { | |
| 21 | 22 | if let Some(backup_config) = target_config.backups { | |
| 22 | 23 | let pool = pool.clone(); | |
| 23 | 24 | let name = name.clone(); | |
| @@ -32,16 +33,8 @@ pub(crate) fn spawn_backup_tasks( | |||
| 32 | 33 | ); | |
| 33 | 34 | ||
| 34 | 35 | handles.push(tokio::spawn(async move { | |
| 35 | - | let mut interval = tokio::time::interval( | |
| 36 | - | std::time::Duration::from_secs(interval_secs), | |
| 37 | - | ); | |
| 38 | - | interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); | |
| 39 | - | interval.tick().await; // consume immediate first tick | |
| 40 | - | loop { | |
| 41 | - | tokio::select! { | |
| 42 | - | _ = cancel.cancelled() => break, | |
| 43 | - | _ = interval.tick() => {} | |
| 44 | - | } | |
| 36 | + | let mut ticks = CheckInterval::new(interval_secs, cancel); | |
| 37 | + | while ticks.next().await { | |
| 45 | 38 | for database in &backup_config.databases { | |
| 46 | 39 | let result = backup::check_backup( | |
| 47 | 40 | &name, |
| @@ -8,6 +8,8 @@ use pom::checks::cors; | |||
| 8 | 8 | use pom::config::Config; | |
| 9 | 9 | use pom::db; | |
| 10 | 10 | ||
| 11 | + | use super::{configured_targets, CheckInterval}; | |
| 12 | + | ||
| 11 | 13 | pub(crate) fn spawn_cors_tasks( | |
| 12 | 14 | config: &Config, | |
| 13 | 15 | pool: &sqlx::SqlitePool, | |
| @@ -17,8 +19,7 @@ pub(crate) fn spawn_cors_tasks( | |||
| 17 | 19 | let interval_secs = config.serve.cors_check_interval_secs; | |
| 18 | 20 | let mut handles = Vec::new(); | |
| 19 | 21 | ||
| 20 | - | for name in config.target_names() { | |
| 21 | - | let target_config = config.get_target(&name).unwrap().clone(); | |
| 22 | + | for (name, target_config) in configured_targets(config) { | |
| 22 | 23 | if target_config.cors.is_empty() { | |
| 23 | 24 | continue; | |
| 24 | 25 | } | |
| @@ -32,18 +33,10 @@ pub(crate) fn spawn_cors_tasks( | |||
| 32 | 33 | info!("{name}: CORS check every {interval_secs}s ({n} endpoints)"); | |
| 33 | 34 | ||
| 34 | 35 | handles.push(tokio::spawn(async move { | |
| 35 | - | let mut interval = tokio::time::interval( | |
| 36 | - | std::time::Duration::from_secs(interval_secs), | |
| 37 | - | ); | |
| 38 | - | interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); | |
| 36 | + | let mut ticks = CheckInterval::new(interval_secs, cancel); | |
| 39 | 37 | let mut prev_failed: std::collections::HashSet<String> = std::collections::HashSet::new(); | |
| 40 | 38 | ||
| 41 | - | interval.tick().await; // consume immediate first tick | |
| 42 | - | loop { | |
| 43 | - | tokio::select! { | |
| 44 | - | _ = cancel.cancelled() => break, | |
| 45 | - | _ = interval.tick() => {} | |
| 46 | - | } | |
| 39 | + | while ticks.next().await { | |
| 47 | 40 | let results = cors::check_cors(&name, &cors_checks).await; | |
| 48 | 41 | ||
| 49 | 42 | for result in &results { |
| @@ -8,6 +8,8 @@ use pom::checks::dns; | |||
| 8 | 8 | use pom::config::Config; | |
| 9 | 9 | use pom::db; | |
| 10 | 10 | ||
| 11 | + | use super::{configured_targets, CheckInterval}; | |
| 12 | + | ||
| 11 | 13 | pub(crate) fn spawn_dns_tasks( | |
| 12 | 14 | config: &Config, | |
| 13 | 15 | pool: &sqlx::SqlitePool, | |
| @@ -17,8 +19,7 @@ pub(crate) fn spawn_dns_tasks( | |||
| 17 | 19 | let dns_interval_secs = config.serve.dns_check_interval_secs; | |
| 18 | 20 | let mut handles = Vec::new(); | |
| 19 | 21 | ||
| 20 | - | for name in config.target_names() { | |
| 21 | - | let target_config = config.get_target(&name).unwrap().clone(); | |
| 22 | + | for (name, target_config) in configured_targets(config) { | |
| 22 | 23 | if target_config.dns.is_empty() { | |
| 23 | 24 | continue; | |
| 24 | 25 | } | |
| @@ -43,18 +44,10 @@ pub(crate) fn spawn_dns_tasks( | |||
| 43 | 44 | Err(e) => tracing::error!("{name}: failed to prune stale DNS: {e}"), | |
| 44 | 45 | } | |
| 45 | 46 | ||
| 46 | - | let mut interval = tokio::time::interval( | |
| 47 | - | std::time::Duration::from_secs(dns_interval_secs), | |
| 48 | - | ); | |
| 49 | - | interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); | |
| 47 | + | let mut ticks = CheckInterval::new(dns_interval_secs, cancel); | |
| 50 | 48 | let mut prev_mismatched: std::collections::HashSet<(String, pom::types::DnsRecordType)> = std::collections::HashSet::new(); | |
| 51 | 49 | ||
| 52 | - | interval.tick().await; // consume immediate first tick | |
| 53 | - | loop { | |
| 54 | - | tokio::select! { | |
| 55 | - | _ = cancel.cancelled() => break, | |
| 56 | - | _ = interval.tick() => {} | |
| 57 | - | } | |
| 50 | + | while ticks.next().await { | |
| 58 | 51 | let results = dns::check_dns(&name, &dns_records).await; | |
| 59 | 52 | ||
| 60 | 53 | for result in &results { |
| @@ -12,6 +12,7 @@ use pom::types::{HealthStatus, LatencyStats}; | |||
| 12 | 12 | ||
| 13 | 13 | use super::super::incident::{incident_action, IncidentAction}; | |
| 14 | 14 | use super::super::transition::TransitionGate; | |
| 15 | + | use super::{configured_targets, CheckInterval}; | |
| 15 | 16 | ||
| 16 | 17 | pub(crate) fn spawn_health_tasks( | |
| 17 | 18 | config: &Config, | |
| @@ -23,8 +24,7 @@ pub(crate) fn spawn_health_tasks( | |||
| 23 | 24 | let confirmations = config.serve.confirmations; | |
| 24 | 25 | let mut handles = Vec::new(); | |
| 25 | 26 | ||
| 26 | - | for name in config.target_names() { | |
| 27 | - | let target_config = config.get_target(&name).unwrap().clone(); | |
| 27 | + | for (name, target_config) in configured_targets(config) { | |
| 28 | 28 | if let Some(health_config) = target_config.health { | |
| 29 | 29 | let interval_secs = health_config.interval_secs.unwrap_or(default_interval); | |
| 30 | 30 | let pool = pool.clone(); | |
| @@ -37,10 +37,7 @@ pub(crate) fn spawn_health_tasks( | |||
| 37 | 37 | info!("{name}: health check every {interval_secs}s"); | |
| 38 | 38 | ||
| 39 | 39 | handles.push(tokio::spawn(async move { | |
| 40 | - | let mut interval = tokio::time::interval( | |
| 41 | - | std::time::Duration::from_secs(interval_secs), | |
| 42 | - | ); | |
| 43 | - | interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); | |
| 40 | + | let mut ticks = CheckInterval::new(interval_secs, cancel); | |
| 44 | 41 | let expect = health_config.expect.as_ref(); | |
| 45 | 42 | let mut in_drift = false; | |
| 46 | 43 | // Seed the N-of-M transition gate from the last stored status so a | |
| @@ -52,12 +49,7 @@ pub(crate) fn spawn_health_tasks( | |||
| 52 | 49 | .flatten() | |
| 53 | 50 | .map(|s| s.status); | |
| 54 | 51 | let mut gate = TransitionGate::seeded(seed, confirmations); | |
| 55 | - | interval.tick().await; // consume immediate first tick | |
| 56 | - | loop { | |
| 57 | - | tokio::select! { | |
| 58 | - | _ = cancel.cancelled() => break, | |
| 59 | - | _ = interval.tick() => {} | |
| 60 | - | } | |
| 52 | + | while ticks.next().await { | |
| 61 | 53 | let snapshot = http::check_health(&name, &health_config, expect).await; | |
| 62 | 54 | info!("{}: {} ({}ms)", name, snapshot.status, snapshot.response_time_ms); | |
| 63 | 55 | if let Err(e) = db::insert_health_check(&pool, &snapshot).await { | |
| @@ -146,8 +138,7 @@ pub(crate) fn spawn_health_tasks( | |||
| 146 | 138 | } | |
| 147 | 139 | ||
| 148 | 140 | // SSH banner checks — stored as health check entries with target name "{name}:ssh" | |
| 149 | - | for name in config.target_names() { | |
| 150 | - | let target_config = config.get_target(&name).unwrap().clone(); | |
| 141 | + | for (name, target_config) in configured_targets(config) { | |
| 151 | 142 | if let Some(ssh_config) = target_config.ssh_banner { | |
| 152 | 143 | let interval_secs = default_interval; | |
| 153 | 144 | let pool = pool.clone(); | |
| @@ -159,22 +150,14 @@ pub(crate) fn spawn_health_tasks( | |||
| 159 | 150 | info!("{check_name}: SSH banner check every {interval_secs}s"); | |
| 160 | 151 | ||
| 161 | 152 | handles.push(tokio::spawn(async move { | |
| 162 | - | let mut interval = tokio::time::interval( | |
| 163 | - | std::time::Duration::from_secs(interval_secs), | |
| 164 | - | ); | |
| 165 | - | interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); | |
| 153 | + | let mut ticks = CheckInterval::new(interval_secs, cancel); | |
| 166 | 154 | let seed = db::get_latest_health(&pool, &check_name) | |
| 167 | 155 | .await | |
| 168 | 156 | .ok() | |
| 169 | 157 | .flatten() | |
| 170 | 158 | .map(|s| s.status); | |
| 171 | 159 | let mut gate = TransitionGate::seeded(seed, confirmations); | |
| 172 | - | interval.tick().await; | |
| 173 | - | loop { | |
| 174 | - | tokio::select! { | |
| 175 | - | _ = cancel.cancelled() => break, | |
| 176 | - | _ = interval.tick() => {} | |
| 177 | - | } | |
| 160 | + | while ticks.next().await { | |
| 178 | 161 | let snapshot = ssh_banner::check_ssh_banner(&check_name, &ssh_config).await; | |
| 179 | 162 | info!("{}: {} ({}ms)", check_name, snapshot.status, snapshot.response_time_ms); | |
| 180 | 163 | if let Err(e) = db::insert_health_check(&pool, &snapshot).await { |
| @@ -8,6 +8,8 @@ use pom::config::Config; | |||
| 8 | 8 | use pom::db; | |
| 9 | 9 | use pom::types::HealthStatus; | |
| 10 | 10 | ||
| 11 | + | use super::CheckInterval; | |
| 12 | + | ||
| 11 | 13 | pub(crate) fn spawn_meta_alert_task( | |
| 12 | 14 | config: &Config, | |
| 13 | 15 | pool: &sqlx::SqlitePool, | |
| @@ -32,11 +34,7 @@ pub(crate) fn spawn_meta_alert_task( | |||
| 32 | 34 | info!("Meta-alert: monitoring-offline check every {meta_interval_secs}s ({} targets)", health_target_names.len()); | |
| 33 | 35 | ||
| 34 | 36 | Some(tokio::spawn(async move { | |
| 35 | - | let mut interval = tokio::time::interval( | |
| 36 | - | std::time::Duration::from_secs(meta_interval_secs), | |
| 37 | - | ); | |
| 38 | - | interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); | |
| 39 | - | interval.tick().await; // consume immediate first tick | |
| 37 | + | let mut ticks = CheckInterval::new(meta_interval_secs, cancel); | |
| 40 | 38 | // Seed from the ledger so a restart while monitoring is already-offline | |
| 41 | 39 | // doesn't re-fire the alert: was-down iff the latest monitoring alert is | |
| 42 | 40 | // the offline (not the recovery) one. | |
| @@ -55,11 +53,7 @@ pub(crate) fn spawn_meta_alert_task( | |||
| 55 | 53 | // not running, not that the target is healthy. | |
| 56 | 54 | let staleness_threshold = chrono::Duration::seconds((meta_interval_secs * 3) as i64); | |
| 57 | 55 | ||
| 58 | - | loop { | |
| 59 | - | tokio::select! { | |
| 60 | - | _ = cancel.cancelled() => break, | |
| 61 | - | _ = interval.tick() => {} | |
| 62 | - | } | |
| 56 | + | while ticks.next().await { | |
| 63 | 57 | ||
| 64 | 58 | let now = chrono::Utc::now(); | |
| 65 | 59 | let mut all_down = true; |
| @@ -11,6 +11,65 @@ mod scan_pipeline; | |||
| 11 | 11 | mod tls; | |
| 12 | 12 | mod whois; | |
| 13 | 13 | ||
| 14 | + | use tokio_util::sync::CancellationToken; | |
| 15 | + | ||
| 16 | + | use pom::config::{Config, TargetConfig}; | |
| 17 | + | ||
| 18 | + | /// The shared cadence of every background check: a fixed interval that skips its | |
| 19 | + | /// immediate first tick, delays rather than bursts after a missed one, and yields | |
| 20 | + | /// to cancellation without waiting out the remaining interval. | |
| 21 | + | /// | |
| 22 | + | /// This hands the caller a ticker instead of taking the check body as a callback. | |
| 23 | + | /// A callback would have to borrow `pool`/`name`/the target config out of its own | |
| 24 | + | /// captures across an await, which needs `AsyncFnMut::CallRefFuture` to name and | |
| 25 | + | /// bound as `Send` (unstable). Driving the loop from the caller sidesteps that | |
| 26 | + | /// entirely: every borrow stays local to the caller's own async block. | |
| 27 | + | pub(crate) struct CheckInterval { | |
| 28 | + | interval: tokio::time::Interval, | |
| 29 | + | cancel: CancellationToken, | |
| 30 | + | startup_tick_pending: bool, | |
| 31 | + | } | |
| 32 | + | ||
| 33 | + | impl CheckInterval { | |
| 34 | + | pub(crate) fn new(interval_secs: u64, cancel: CancellationToken) -> Self { | |
| 35 | + | let mut interval = | |
| 36 | + | tokio::time::interval(std::time::Duration::from_secs(interval_secs)); | |
| 37 | + | interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); | |
| 38 | + | Self { interval, cancel, startup_tick_pending: true } | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | /// Wait for the next tick. Returns `false` once cancelled, so the call site | |
| 42 | + | /// reads `while ticks.next().await { ... }`. | |
| 43 | + | /// | |
| 44 | + | /// The first call consumes the immediate startup tick, so a check fires one | |
| 45 | + | /// full interval after boot rather than the instant the process starts. | |
| 46 | + | pub(crate) async fn next(&mut self) -> bool { | |
| 47 | + | if self.startup_tick_pending { | |
| 48 | + | self.startup_tick_pending = false; | |
| 49 | + | self.interval.tick().await; | |
| 50 | + | } | |
| 51 | + | tokio::select! { | |
| 52 | + | _ = self.cancel.cancelled() => false, | |
| 53 | + | _ = self.interval.tick() => true, | |
| 54 | + | } | |
| 55 | + | } | |
| 56 | + | } | |
| 57 | + | ||
| 58 | + | /// Every configured target paired with its config, in `target_names` order. | |
| 59 | + | /// | |
| 60 | + | /// The spawners all opened with the same `for name in config.target_names()` / | |
| 61 | + | /// `get_target(&name).unwrap().clone()` pair; the `unwrap` is only sound because | |
| 62 | + | /// the name came from the map itself, which this keeps in one place. | |
| 63 | + | pub(crate) fn configured_targets(config: &Config) -> impl Iterator<Item = (String, TargetConfig)> + use<'_> { | |
| 64 | + | config | |
| 65 | + | .target_names() | |
| 66 | + | .into_iter() | |
| 67 | + | .map(|name| { | |
| 68 | + | let target = config.get_target(&name).expect("name came from target_names").clone(); | |
| 69 | + | (name, target) | |
| 70 | + | }) | |
| 71 | + | } | |
| 72 | + | ||
| 14 | 73 | pub(crate) use backup::spawn_backup_tasks; | |
| 15 | 74 | pub(crate) use cors::spawn_cors_tasks; | |
| 16 | 75 | pub(crate) use dns::spawn_dns_tasks; |
| @@ -5,6 +5,8 @@ use tracing::info; | |||
| 5 | 5 | ||
| 6 | 6 | use pom::db; | |
| 7 | 7 | ||
| 8 | + | use super::CheckInterval; | |
| 9 | + | ||
| 8 | 10 | pub(crate) fn spawn_prune_task( | |
| 9 | 11 | pool: &sqlx::SqlitePool, | |
| 10 | 12 | prune_days: i64, | |
| @@ -14,16 +16,8 @@ pub(crate) fn spawn_prune_task( | |||
| 14 | 16 | let cancel = cancel.clone(); | |
| 15 | 17 | ||
| 16 | 18 | tokio::spawn(async move { | |
| 17 | - | let mut interval = tokio::time::interval( | |
| 18 | - | std::time::Duration::from_secs(86400), | |
| 19 | - | ); | |
| 20 | - | interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); | |
| 21 | - | interval.tick().await; // consume immediate first tick | |
| 22 | - | loop { | |
| 23 | - | tokio::select! { | |
| 24 | - | _ = cancel.cancelled() => break, | |
| 25 | - | _ = interval.tick() => {} | |
| 26 | - | } | |
| 19 | + | let mut ticks = CheckInterval::new(86400, cancel); | |
| 20 | + | while ticks.next().await { | |
| 27 | 21 | match db::prune_old_records(&pool, prune_days).await { | |
| 28 | 22 | Ok(r) => info!("Pruned {} health checks, {} test runs, {} test details, {} peer heartbeats, {} alerts, {} TLS checks, {} incidents, {} route checks, {} DNS checks, {} WHOIS checks, {} backup checks", r.health, r.tests, r.test_details, r.heartbeats, r.alerts, r.tls, r.incidents, r.routes, r.dns, r.whois, r.backups), | |
| 29 | 23 | Err(e) => tracing::error!("Prune failed: {e}"), |
| @@ -8,6 +8,8 @@ use pom::checks::routes; | |||
| 8 | 8 | use pom::config::Config; | |
| 9 | 9 | use pom::db; | |
| 10 | 10 | ||
| 11 | + | use super::{configured_targets, CheckInterval}; | |
| 12 | + | ||
| 11 | 13 | pub(crate) fn spawn_route_tasks( | |
| 12 | 14 | config: &Config, | |
| 13 | 15 | pool: &sqlx::SqlitePool, | |
| @@ -17,8 +19,7 @@ pub(crate) fn spawn_route_tasks( | |||
| 17 | 19 | let route_interval = config.serve.route_check_interval_secs; | |
| 18 | 20 | let mut handles = Vec::new(); | |
| 19 | 21 | ||
| 20 | - | for name in config.target_names() { | |
| 21 | - | let target_config = config.get_target(&name).unwrap().clone(); | |
| 22 | + | for (name, target_config) in configured_targets(config) { | |
| 22 | 23 | if target_config.expected_routes.is_empty() { | |
| 23 | 24 | continue; | |
| 24 | 25 | } | |
| @@ -42,22 +43,14 @@ pub(crate) fn spawn_route_tasks( | |||
| 42 | 43 | Err(e) => tracing::error!("{name}: failed to prune stale routes: {e}"), | |
| 43 | 44 | } | |
| 44 | 45 | ||
| 45 | - | let mut interval = tokio::time::interval( | |
| 46 | - | std::time::Duration::from_secs(route_interval), | |
| 47 | - | ); | |
| 48 | - | interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); | |
| 46 | + | let mut ticks = CheckInterval::new(route_interval, cancel); | |
| 49 | 47 | let client = reqwest::Client::builder() | |
| 50 | 48 | .redirect(reqwest::redirect::Policy::none()) | |
| 51 | 49 | .build() | |
| 52 | 50 | .unwrap_or_default(); | |
| 53 | 51 | let mut prev_failed: std::collections::HashSet<String> = std::collections::HashSet::new(); | |
| 54 | 52 | ||
| 55 | - | interval.tick().await; // consume immediate first tick | |
| 56 | - | loop { | |
| 57 | - | tokio::select! { | |
| 58 | - | _ = cancel.cancelled() => break, | |
| 59 | - | _ = interval.tick() => {} | |
| 60 | - | } | |
| 53 | + | while ticks.next().await { | |
| 61 | 54 | let results = routes::check_routes(&client, &name, &base_url, &route_paths, timeout).await; | |
| 62 | 55 | ||
| 63 | 56 | for result in &results { |
| @@ -10,6 +10,8 @@ use pom::checks::scan_pipeline; | |||
| 10 | 10 | use pom::config::Config; | |
| 11 | 11 | use pom::db; | |
| 12 | 12 | ||
| 13 | + | use super::{configured_targets, CheckInterval}; | |
| 14 | + | ||
| 13 | 15 | pub(crate) fn spawn_scan_pipeline_tasks( | |
| 14 | 16 | config: &Config, | |
| 15 | 17 | pool: &sqlx::SqlitePool, | |
| @@ -18,8 +20,7 @@ pub(crate) fn spawn_scan_pipeline_tasks( | |||
| 18 | 20 | ) -> Vec<JoinHandle<()>> { | |
| 19 | 21 | let mut handles = Vec::new(); | |
| 20 | 22 | ||
| 21 | - | for name in config.target_names() { | |
| 22 | - | let target_config = config.get_target(&name).unwrap().clone(); | |
| 23 | + | for (name, target_config) in configured_targets(config) { | |
| 23 | 24 | let Some(sp_config) = target_config.scan_pipeline else { continue }; | |
| 24 | 25 | ||
| 25 | 26 | let name = name.clone(); | |
| @@ -34,11 +35,7 @@ pub(crate) fn spawn_scan_pipeline_tasks( | |||
| 34 | 35 | ); | |
| 35 | 36 | ||
| 36 | 37 | handles.push(tokio::spawn(async move { | |
| 37 | - | let mut interval = tokio::time::interval( | |
| 38 | - | std::time::Duration::from_secs(sp_config.interval_secs), | |
| 39 | - | ); | |
| 40 | - | interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); | |
| 41 | - | interval.tick().await; // consume immediate first tick | |
| 38 | + | let mut ticks = CheckInterval::new(sp_config.interval_secs, cancel); | |
| 42 | 39 | ||
| 43 | 40 | // Track previous status so we only fire alerts on transitions. Seed | |
| 44 | 41 | // from the ledger so a restart while the pipeline is already-degraded | |
| @@ -52,11 +49,7 @@ pub(crate) fn spawn_scan_pipeline_tasks( | |||
| 52 | 49 | .filter(|a| a.alert_type == "scan_pipeline_degraded") | |
| 53 | 50 | .and_then(|a| a.to_status); | |
| 54 | 51 | ||
| 55 | - | loop { | |
| 56 | - | tokio::select! { | |
| 57 | - | _ = cancel.cancelled() => break, | |
| 58 | - | _ = interval.tick() => {} | |
| 59 | - | } | |
| 52 | + | while ticks.next().await { | |
| 60 | 53 | ||
| 61 | 54 | let result = scan_pipeline::check_scan_pipeline( | |
| 62 | 55 | &name, &sp_config.base_url, sp_config.timeout_secs, |
| @@ -8,6 +8,8 @@ use pom::checks::tls; | |||
| 8 | 8 | use pom::config::Config; | |
| 9 | 9 | use pom::db; | |
| 10 | 10 | ||
| 11 | + | use super::{configured_targets, CheckInterval}; | |
| 12 | + | ||
| 11 | 13 | pub(crate) fn spawn_tls_tasks( | |
| 12 | 14 | config: &Config, | |
| 13 | 15 | pool: &sqlx::SqlitePool, | |
| @@ -17,8 +19,7 @@ pub(crate) fn spawn_tls_tasks( | |||
| 17 | 19 | let tls_interval_secs = config.serve.tls_check_interval_secs; | |
| 18 | 20 | let mut handles = Vec::new(); | |
| 19 | 21 | ||
| 20 | - | for name in config.target_names() { | |
| 21 | - | let target_config = config.get_target(&name).unwrap().clone(); | |
| 22 | + | for (name, target_config) in configured_targets(config) { | |
| 22 | 23 | if let Some(tls_config) = target_config.tls { | |
| 23 | 24 | let pool = pool.clone(); | |
| 24 | 25 | let name = name.clone(); | |
| @@ -30,16 +31,8 @@ pub(crate) fn spawn_tls_tasks( | |||
| 30 | 31 | info!("{name}: TLS check every {tls_interval_secs}s (host={})", tls_config.host); | |
| 31 | 32 | ||
| 32 | 33 | handles.push(tokio::spawn(async move { | |
| 33 | - | let mut interval = tokio::time::interval( | |
| 34 | - | std::time::Duration::from_secs(tls_interval_secs), | |
| 35 | - | ); | |
| 36 | - | interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); | |
| 37 | - | interval.tick().await; // consume immediate first tick | |
| 38 | - | loop { | |
| 39 | - | tokio::select! { | |
| 40 | - | _ = cancel.cancelled() => break, | |
| 41 | - | _ = interval.tick() => {} | |
| 42 | - | } | |
| 34 | + | let mut ticks = CheckInterval::new(tls_interval_secs, cancel); | |
| 35 | + | while ticks.next().await { | |
| 43 | 36 | let previous = db::get_latest_tls_check(&pool, &name).await.ok().flatten(); | |
| 44 | 37 | let status = tls::check_tls(&name, &tls_config).await; | |
| 45 | 38 | info!("{}: TLS {} — {}d remaining", name, if status.valid { "valid" } else { "invalid" }, status.days_remaining); |
| @@ -8,6 +8,8 @@ use pom::checks::whois; | |||
| 8 | 8 | use pom::config::Config; | |
| 9 | 9 | use pom::db; | |
| 10 | 10 | ||
| 11 | + | use super::{configured_targets, CheckInterval}; | |
| 12 | + | ||
| 11 | 13 | pub(crate) fn spawn_whois_tasks( | |
| 12 | 14 | config: &Config, | |
| 13 | 15 | pool: &sqlx::SqlitePool, | |
| @@ -17,8 +19,7 @@ pub(crate) fn spawn_whois_tasks( | |||
| 17 | 19 | let whois_interval_secs = config.serve.whois_check_interval_secs; | |
| 18 | 20 | let mut handles = Vec::new(); | |
| 19 | 21 | ||
| 20 | - | for name in config.target_names() { | |
| 21 | - | let target_config = config.get_target(&name).unwrap().clone(); | |
| 22 | + | for (name, target_config) in configured_targets(config) { | |
| 22 | 23 | let Some(whois_config) = target_config.whois else { continue }; | |
| 23 | 24 | let label = target_config.label.clone(); | |
| 24 | 25 | let pool = pool.clone(); | |
| @@ -29,17 +30,9 @@ pub(crate) fn spawn_whois_tasks( | |||
| 29 | 30 | info!("{name}: WHOIS check every {whois_interval_secs}s (domain={})", whois_config.domain); | |
| 30 | 31 | ||
| 31 | 32 | handles.push(tokio::spawn(async move { | |
| 32 | - | let mut interval = tokio::time::interval( | |
| 33 | - | std::time::Duration::from_secs(whois_interval_secs), | |
| 34 | - | ); | |
| 35 | - | interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); | |
| 33 | + | let mut ticks = CheckInterval::new(whois_interval_secs, cancel); | |
| 36 | 34 | ||
| 37 | - | interval.tick().await; // consume immediate first tick | |
| 38 | - | loop { | |
| 39 | - | tokio::select! { | |
| 40 | - | _ = cancel.cancelled() => break, | |
| 41 | - | _ = interval.tick() => {} | |
| 42 | - | } | |
| 35 | + | while ticks.next().await { | |
| 43 | 36 | let result = whois::check_whois(&name, &whois_config).await; | |
| 44 | 37 | info!("{}: WHOIS {} — {:?} days remaining", name, whois_config.domain, result.days_remaining); | |
| 45 | 38 |