max / makenotwork
53 files changed,
+1120 insertions,
-330 deletions
| @@ -3599,6 +3599,7 @@ | |||
| 3599 | 3599 | "sha1 0.10.6", | |
| 3600 | 3600 | "sha2 0.10.9", | |
| 3601 | 3601 | "sqlx", | |
| 3602 | + | "subtle", | |
| 3602 | 3603 | "syntect", | |
| 3603 | 3604 | "tagtree", | |
| 3604 | 3605 | "tempfile", |
| @@ -68,6 +68,7 @@ | |||
| 68 | 68 | hmac = "0.12.1" | |
| 69 | 69 | sha1 = "0.10.6" | |
| 70 | 70 | sha2 = "0.10.9" | |
| 71 | + | subtle = "2.6" | |
| 71 | 72 | hex = "0.4.3" | |
| 72 | 73 | base64 = "0.22.1" | |
| 73 | 74 |
| @@ -392,22 +392,31 @@ | |||
| 392 | 392 | String::new() | |
| 393 | 393 | }; | |
| 394 | 394 | ||
| 395 | - | // Upload to S3 | |
| 396 | - | let data = tokio::fs::read(&local_tmp) | |
| 397 | - | .await | |
| 398 | - | .map_err(|e| format!("failed to read artifact: {e}"))?; | |
| 399 | - | ||
| 400 | - | let _ = tokio::fs::remove_file(&local_tmp).await; | |
| 401 | - | ||
| 395 | + | // Upload to S3 via multipart streaming from disk — the previous | |
| 396 | + | // implementation `tokio::fs::read` → `Vec<u8>` → `upload_object` pinned | |
| 397 | + | // the entire artifact (up to ~100 MB per build) in RAM during upload. | |
| 398 | + | // `upload_multipart` reads the file in chunks and lets the S3 SDK do | |
| 399 | + | // parallel part uploads, keeping memory bounded regardless of artifact | |
| 400 | + | // size. | |
| 402 | 401 | let synckit_s3 = state | |
| 403 | 402 | .synckit_s3 | |
| 404 | 403 | .as_ref() | |
| 405 | 404 | .ok_or("SyncKit storage not configured")?; | |
| 406 | 405 | ||
| 407 | - | synckit_s3 | |
| 408 | - | .upload_object(&s3_key, "application/octet-stream", data, None) | |
| 406 | + | let upload_result = synckit_s3 | |
| 407 | + | .upload_multipart( | |
| 408 | + | &s3_key, | |
| 409 | + | "application/octet-stream", | |
| 410 | + | std::path::Path::new(&local_tmp), | |
| 411 | + | ) | |
| 409 | 412 | .await | |
| 410 | - | .map_err(|e| format!("S3 upload failed: {e}"))?; | |
| 413 | + | .map_err(|e| format!("S3 multipart upload failed: {e}")); | |
| 414 | + | ||
| 415 | + | // Always remove the local temp file, even if the upload failed — leaving | |
| 416 | + | // it on disk fills the build runner's tmp directory across retries. | |
| 417 | + | let _ = tokio::fs::remove_file(&local_tmp).await; | |
| 418 | + | ||
| 419 | + | upload_result?; | |
| 411 | 420 | ||
| 412 | 421 | if !signature.is_empty() { | |
| 413 | 422 | let _ = append_log_bounded( | |
| @@ -501,19 +510,22 @@ | |||
| 501 | 510 | } | |
| 502 | 511 | ||
| 503 | 512 | /// Append to build log, respecting the max log size. | |
| 513 | + | /// | |
| 514 | + | /// Probes `octet_length(log)` instead of fetching the whole row (the log | |
| 515 | + | /// column tops out at 5 MiB and is read on every line append). | |
| 504 | 516 | async fn append_log_bounded( | |
| 505 | 517 | state: &AppState, | |
| 506 | 518 | build_id: db::BuildId, | |
| 507 | 519 | line: &str, | |
| 508 | 520 | ) -> crate::error::Result<()> { | |
| 509 | - | // Check current log size (approximate — avoids fetching the full log) | |
| 510 | - | let build = db::builds::get_build(&state.db, build_id).await?; | |
| 511 | - | if let Some(b) = build | |
| 512 | - | && b.log.len() + line.len() > BUILD_MAX_LOG_BYTES | |
| 521 | + | const TRUNCATED: &str = "[log truncated]\n"; | |
| 522 | + | if let Some((current_len, already_truncated)) = | |
| 523 | + | db::builds::get_build_log_size(&state.db, build_id, TRUNCATED).await? | |
| 524 | + | && (current_len as usize) + line.len() > BUILD_MAX_LOG_BYTES | |
| 513 | 525 | { | |
| 514 | - | if !b.log.ends_with("[log truncated]\n") { | |
| 526 | + | if !already_truncated { | |
| 515 | 527 | tracing::warn!(build_id = %build_id, "Build log exceeded {} bytes, truncating", BUILD_MAX_LOG_BYTES); | |
| 516 | - | db::builds::append_build_log(&state.db, build_id, "[log truncated]\n").await?; | |
| 528 | + | db::builds::append_build_log(&state.db, build_id, TRUNCATED).await?; | |
| 517 | 529 | } | |
| 518 | 530 | return Ok(()); | |
| 519 | 531 | } |
| @@ -15,11 +15,18 @@ | |||
| 15 | 15 | ||
| 16 | 16 | // -- Sessions -- | |
| 17 | 17 | pub const SESSION_EXPIRY_DAYS: i64 = 7; | |
| 18 | - | pub const SESSION_TOUCH_CACHE_SECS: u64 = 30; // Skip DB touch if validated within this window | |
| 18 | + | /// Skip DB touch if validated within this window. Doubles as the upper bound on | |
| 19 | + | /// session-revocation lag (admin suspend, logout-everywhere, password change) — | |
| 20 | + | /// shorter = tighter revocation, slightly more DB load on the auth hot path. | |
| 21 | + | pub const SESSION_TOUCH_CACHE_SECS: u64 = 5; | |
| 19 | 22 | ||
| 20 | 23 | // -- Login security -- | |
| 21 | 24 | pub const MAX_LOGIN_ATTEMPTS: i32 = 5; | |
| 22 | 25 | pub const LOCKOUT_MINUTES: i64 = 15; | |
| 26 | + | /// How long a half-completed login (password verified, awaiting 2FA) stays | |
| 27 | + | /// valid before the user must re-enter their password. Defends against the | |
| 28 | + | /// "unattended browser one TOTP from logged in" failure mode. | |
| 29 | + | pub const PENDING_2FA_TTL_SECS: i64 = 600; | |
| 23 | 30 | ||
| 24 | 31 | // -- Email link expiry (seconds) -- | |
| 25 | 32 | pub const PASSWORD_RESET_EXPIRY_SECS: i64 = 900; // 15 minutes |
| @@ -1,30 +1,36 @@ | |||
| 1 | 1 | //! Cryptographic utilities: constant-time comparison, key generation, feed signing. | |
| 2 | 2 | ||
| 3 | - | /// Constant-time string comparison to prevent timing attacks. | |
| 3 | + | /// Constant-time byte comparison for tokens, MACs, and other fixed-shape | |
| 4 | + | /// secrets. Backed by [`subtle::ConstantTimeEq`] (audited reference impl) | |
| 5 | + | /// instead of a hand-rolled XOR loop wrapped in cosmetic SHA-256. | |
| 4 | 6 | /// | |
| 5 | - | /// Hashes both inputs with SHA-256 before comparing to avoid leaking | |
| 6 | - | /// the length of the expected value via early return. | |
| 7 | + | /// Length mismatch short-circuits — leaking the length of fixed-format | |
| 8 | + | /// tokens (hex-encoded HMACs, CSRF tokens, PKCE verifiers, base64 secrets) | |
| 9 | + | /// reveals nothing useful to an attacker, since the format already fixes | |
| 10 | + | /// the length. Don't use this for variable-length sensitive payloads | |
| 11 | + | /// where length is itself secret. | |
| 7 | 12 | pub fn constant_time_compare(a: &str, b: &str) -> bool { | |
| 8 | - | use sha2::{Sha256, Digest}; | |
| 9 | - | ||
| 10 | - | let hash_a = Sha256::digest(a.as_bytes()); | |
| 11 | - | let hash_b = Sha256::digest(b.as_bytes()); | |
| 12 | - | ||
| 13 | - | let mut result = 0u8; | |
| 14 | - | for (x, y) in hash_a.iter().zip(hash_b.iter()) { | |
| 15 | - | result |= x ^ y; | |
| 13 | + | use subtle::ConstantTimeEq; | |
| 14 | + | let a = a.as_bytes(); | |
| 15 | + | let b = b.as_bytes(); | |
| 16 | + | if a.len() != b.len() { | |
| 17 | + | return false; | |
| 16 | 18 | } | |
| 17 | - | result == 0 | |
| 19 | + | a.ct_eq(b).into() | |
| 18 | 20 | } | |
| 19 | 21 | ||
| 20 | - | /// Generate a license key code in word-word-word-word-word format. | |
| 22 | + | /// Generate a license key code in word-word-word-word-word-word format. | |
| 21 | 23 | /// | |
| 22 | - | /// Uses 5 random words from the 2048-word list (~55 bits of entropy). | |
| 23 | - | /// Returns a `KeyCode` via `from_trusted` — the wordlist guarantees validity. | |
| 24 | + | /// Six random words from the 2048-word list (~66 bits of entropy). Six was | |
| 25 | + | /// chosen over five (~55 bits) after a birthday-collision review: at five | |
| 26 | + | /// words, ~190M keys gives a coin-flip chance of collision; at six, the | |
| 27 | + | /// equivalent threshold rises to ~6B keys — far past the lifetime cap of | |
| 28 | + | /// any realistic license catalog. Returns a `KeyCode` via `from_trusted` — | |
| 29 | + | /// the wordlist guarantees validity. | |
| 24 | 30 | pub fn generate_key_code() -> crate::db::KeyCode { | |
| 25 | 31 | use rand::Rng; | |
| 26 | 32 | let mut rng = rand::rng(); | |
| 27 | - | let words: Vec<&str> = (0..5) | |
| 33 | + | let words: Vec<&str> = (0..6) | |
| 28 | 34 | .map(|_| { | |
| 29 | 35 | let idx = rng.random_range(0..crate::wordlist::WORDLIST.len()); | |
| 30 | 36 | crate::wordlist::WORDLIST[idx] | |
| @@ -104,7 +110,7 @@ | |||
| 104 | 110 | fn key_code_format() { | |
| 105 | 111 | let code = generate_key_code(); | |
| 106 | 112 | let parts: Vec<&str> = code.split('-').collect(); | |
| 107 | - | assert_eq!(parts.len(), 5, "Key code should have 5 words"); | |
| 113 | + | assert_eq!(parts.len(), 6, "Key code should have 6 words"); | |
| 108 | 114 | for word in &parts { | |
| 109 | 115 | assert!(word.len() >= 3, "Each word should be at least 3 chars: {}", word); | |
| 110 | 116 | assert!(word.len() <= 6, "Each word should be at most 6 chars: {}", word); |
| @@ -171,11 +171,11 @@ | |||
| 171 | 171 | Ok(true) => next.run(request).await, | |
| 172 | 172 | Ok(false) => { | |
| 173 | 173 | tracing::warn!(path = %path, "CSRF token mismatch"); | |
| 174 | - | (StatusCode::FORBIDDEN, "Invalid CSRF token").into_response() | |
| 174 | + | crate::error::AppError::Forbidden.into_response() | |
| 175 | 175 | } | |
| 176 | 176 | Err(e) => { | |
| 177 | 177 | tracing::error!(error = ?e, "CSRF validation error"); | |
| 178 | - | (StatusCode::INTERNAL_SERVER_ERROR, "CSRF validation error").into_response() | |
| 178 | + | crate::error::AppError::Internal(anyhow::anyhow!("CSRF validation error")).into_response() | |
| 179 | 179 | } | |
| 180 | 180 | }; | |
| 181 | 181 | } | |
| @@ -223,7 +223,7 @@ | |||
| 223 | 223 | Some(t) => t, | |
| 224 | 224 | None => { | |
| 225 | 225 | tracing::warn!(path = %path, "CSRF token missing from form body"); | |
| 226 | - | return (StatusCode::FORBIDDEN, "CSRF token required").into_response(); | |
| 226 | + | return crate::error::AppError::Forbidden.into_response(); | |
| 227 | 227 | } | |
| 228 | 228 | }; | |
| 229 | 229 |
| @@ -76,7 +76,14 @@ | |||
| 76 | 76 | let (operation, repo_path) = parse_ssh_command(original_cmd)?; | |
| 77 | 77 | let (owner, repo_name) = parse_repo_path(&repo_path)?; | |
| 78 | 78 | ||
| 79 | - | let owner_user = db::users::get_user_by_username(pool, &Username::from_trusted(owner.to_string())) | |
| 79 | + | // Validate the SSH-supplied owner string before any DB lookup or shell | |
| 80 | + | // reconstruction. `parse_repo_path` is a path-shape check, not a Username | |
| 81 | + | // syntax check — without this, a malformed owner could reach the DB layer | |
| 82 | + | // or end up embedded in the `git-shell -c` argument below. | |
| 83 | + | let owner_username = Username::new(owner) | |
| 84 | + | .map_err(|_| anyhow::anyhow!("repository not found"))?; | |
| 85 | + | ||
| 86 | + | let owner_user = db::users::get_user_by_username(pool, &owner_username) | |
| 80 | 87 | .await? | |
| 81 | 88 | .ok_or_else(|| anyhow::anyhow!("repository not found"))?; | |
| 82 | 89 |
| @@ -25,13 +25,14 @@ | |||
| 25 | 25 | ||
| 26 | 26 | /// Extract the client IP from request headers. | |
| 27 | 27 | /// | |
| 28 | - | /// Prefers `CF-Connecting-IP` (set by Cloudflare, trusted) over `X-Forwarded-For`. | |
| 29 | - | /// Returns the first IP in the chain, trimmed. All code paths that store or compare | |
| 30 | - | /// client IPs should use this function to ensure consistency. | |
| 28 | + | /// Honors `CF-Connecting-IP` only — the single header Cloudflare sets and that | |
| 29 | + | /// origin clients cannot reach (Hetzner firewall + Caddy strip arbitrary XFF). | |
| 30 | + | /// `X-Forwarded-For` is intentionally not consulted: there is no trusted-proxy | |
| 31 | + | /// allowlist, so any request bypassing Cloudflare could spoof the IP and evade | |
| 32 | + | /// sandbox caps / poison audit logs / forge "new device" notifications. | |
| 31 | 33 | pub fn extract_client_ip(headers: &HeaderMap) -> Option<String> { | |
| 32 | 34 | headers | |
| 33 | 35 | .get("cf-connecting-ip") | |
| 34 | - | .or_else(|| headers.get("x-forwarded-for")) | |
| 35 | 36 | .and_then(|v| v.to_str().ok()) | |
| 36 | 37 | .and_then(|s| s.split(',').next()) | |
| 37 | 38 | .map(|s| s.trim().to_string()) | |
| @@ -322,10 +323,20 @@ | |||
| 322 | 323 | } | |
| 323 | 324 | ||
| 324 | 325 | #[test] | |
| 325 | - | fn extract_client_ip_xff_fallback() { | |
| 326 | + | fn extract_client_ip_ignores_xff_when_cf_missing() { | |
| 327 | + | // XFF alone must not be trusted — see security note on extract_client_ip. | |
| 326 | 328 | let mut headers = HeaderMap::new(); | |
| 327 | 329 | headers.insert("x-forwarded-for", HeaderValue::from_static("5.6.7.8, 9.10.11.12")); | |
| 328 | - | assert_eq!(extract_client_ip(&headers).as_deref(), Some("5.6.7.8")); | |
| 330 | + | assert_eq!(extract_client_ip(&headers), None); | |
| 331 | + | } | |
| 332 | + | ||
| 333 | + | #[test] | |
| 334 | + | fn extract_client_ip_ignores_xff_even_when_cf_present() { | |
| 335 | + | // Defense in depth: presence of XFF must not influence the result. | |
| 336 | + | let mut headers = HeaderMap::new(); | |
| 337 | + | headers.insert("cf-connecting-ip", HeaderValue::from_static("1.2.3.4")); | |
| 338 | + | headers.insert("x-forwarded-for", HeaderValue::from_static("5.6.7.8")); | |
| 339 | + | assert_eq!(extract_client_ip(&headers).as_deref(), Some("1.2.3.4")); | |
| 329 | 340 | } | |
| 330 | 341 | ||
| 331 | 342 | #[test] |
| @@ -201,31 +201,44 @@ | |||
| 201 | 201 | ||
| 202 | 202 | // Only cache successful responses (2xx/3xx) to avoid caching transient errors | |
| 203 | 203 | if status_code < 400 { | |
| 204 | - | // Skip caching if content-length exceeds 1MB to avoid consuming the body | |
| 204 | + | // Only cache when content-length is present AND <= 1MB. We must decide | |
| 205 | + | // BEFORE consuming the body, otherwise a chunked / unknown-length response | |
| 206 | + | // that exceeds the cap would be silently truncated to empty — a correctness | |
| 207 | + | // landmine, since the status + headers would still claim success. | |
| 205 | 208 | let content_length = response.headers() | |
| 206 | 209 | .get(axum::http::header::CONTENT_LENGTH) | |
| 207 | 210 | .and_then(|v| v.to_str().ok()) | |
| 208 | 211 | .and_then(|v| v.parse::<usize>().ok()); | |
| 209 | - | if content_length.is_some_and(|len| len > 1024 * 1024) { | |
| 210 | - | tracing::info!( | |
| 212 | + | let Some(len) = content_length else { | |
| 213 | + | tracing::debug!( | |
| 211 | 214 | key = %idem_key, method = %method, path = %path, | |
| 212 | - | "response body exceeds 1MB, skipping idempotency cache" | |
| 215 | + | "no content-length on response; skipping idempotency cache (body left intact)" | |
| 216 | + | ); | |
| 217 | + | return response; | |
| 218 | + | }; | |
| 219 | + | if len > 1024 * 1024 { | |
| 220 | + | tracing::info!( | |
| 221 | + | key = %idem_key, method = %method, path = %path, len, | |
| 222 | + | "response body exceeds 1MB; skipping idempotency cache" | |
| 213 | 223 | ); | |
| 214 | 224 | return response; | |
| 215 | 225 | } | |
| 216 | 226 | ||
| 217 | - | // Extract body bytes to cache (up to 1MB) | |
| 227 | + | // Extract body bytes to cache. Content-length confirms <= 1MB, so this | |
| 228 | + | // should not exceed the limit; if it does, that's a header/body mismatch | |
| 229 | + | // and we surface 500 rather than silently dropping the body. | |
| 218 | 230 | let (parts, body) = response.into_parts(); | |
| 219 | 231 | let body_bytes = match axum::body::to_bytes(body, 1024 * 1024).await { | |
| 220 | 232 | Ok(b) => b, | |
| 221 | - | Err(_) => { | |
| 222 | - | tracing::info!( | |
| 223 | - | key = %idem_key, method = %method, path = %path, | |
| 224 | - | "response body exceeds 1MB, skipping idempotency cache" | |
| 233 | + | Err(e) => { | |
| 234 | + | tracing::error!( | |
| 235 | + | key = %idem_key, method = %method, path = %path, error = ?e, | |
| 236 | + | "response body exceeded 1MB despite content-length <= 1MB; failing closed" | |
| 225 | 237 | ); | |
| 226 | - | // Body is consumed — return empty. This only triggers for chunked | |
| 227 | - | // responses without content-length (rare for API endpoints). | |
| 228 | - | return axum::response::Response::from_parts(parts, axum::body::Body::empty()); | |
| 238 | + | return axum::response::Response::builder() | |
| 239 | + | .status(StatusCode::INTERNAL_SERVER_ERROR) | |
| 240 | + | .body(axum::body::Body::from("internal error")) | |
| 241 | + | .unwrap_or_else(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response()); | |
| 229 | 242 | } | |
| 230 | 243 | }; | |
| 231 | 244 | // Only cache UTF-8 responses — skip binary content to avoid corruption |
| @@ -158,56 +158,61 @@ | |||
| 158 | 158 | } | |
| 159 | 159 | } | |
| 160 | 160 | ||
| 161 | - | // Send alert email on status transitions (with cooldown) | |
| 162 | - | if let Some(ref to) = alert_email { | |
| 163 | - | let cooldown_elapsed = last_alert_at | |
| 164 | - | .is_none_or(|t| t.elapsed().as_secs() >= constants::ALERT_COOLDOWN_SECS); | |
| 161 | + | // Status-change notifications (admin alert + user notifications) share | |
| 162 | + | // a single cooldown so a flapping monitor cannot spam either audience. | |
| 163 | + | let cooldown_elapsed = last_alert_at | |
| 164 | + | .is_none_or(|t| t.elapsed().as_secs() >= constants::ALERT_COOLDOWN_SECS); | |
| 165 | 165 | ||
| 166 | - | if cooldown_elapsed { | |
| 166 | + | if cooldown_elapsed { | |
| 167 | + | if let Some(ref to) = alert_email { | |
| 167 | 168 | let (subject, body) = build_alert(previous_status, &snap); | |
| 168 | 169 | match state.email.send_alert(to, &subject, &body).await { | |
| 169 | - | Ok(()) => { | |
| 170 | - | last_alert_at = Some(Instant::now()); | |
| 171 | - | tracing::info!(recipient = %to, "alert email sent"); | |
| 172 | - | } | |
| 173 | - | Err(e) => { | |
| 174 | - | tracing::error!(error = ?e, "failed to send alert email"); | |
| 175 | - | } | |
| 170 | + | Ok(()) => tracing::info!(recipient = %to, "alert email sent"), | |
| 171 | + | Err(e) => tracing::error!(error = ?e, "failed to send alert email"), | |
| 176 | 172 | } | |
| 177 | 173 | } | |
| 178 | - | } | |
| 179 | 174 | ||
| 180 | - | // Notify opted-in users of status changes (fire-and-forget) | |
| 181 | - | { | |
| 182 | - | let pool = state.db.clone(); | |
| 183 | - | let email_client = state.email.clone(); | |
| 184 | - | let host_url = state.config.host_url.clone(); | |
| 185 | - | let signing_secret = state.config.signing_secret.clone(); | |
| 186 | - | let current_status = snap.status.as_str().to_string(); | |
| 187 | - | let prev_status = previous_status.map_or("unknown", |s| s.as_str()).to_string(); | |
| 188 | - | tokio::spawn(async move { | |
| 189 | - | match db::users::get_status_alert_subscribers(&pool).await { | |
| 190 | - | Ok(subscribers) if !subscribers.is_empty() => { | |
| 191 | - | tracing::info!(count = subscribers.len(), "sending status notifications to opted-in users"); | |
| 192 | - | for sub in &subscribers { | |
| 193 | - | let unsub_url = crate::email::generate_unsubscribe_url( | |
| 194 | - | &host_url, sub.id, crate::email::UnsubscribeAction::Status, &sub.id.to_string(), &signing_secret, | |
| 195 | - | ); | |
| 196 | - | let _ = email_client.send_status_notification( | |
| 197 | - | &sub.email, | |
| 198 | - | sub.display_name.as_deref(), | |
| 199 | - | ¤t_status, | |
| 200 | - | &prev_status, | |
| 201 | - | &unsub_url, | |
| 202 | - | ).await; | |
| 175 | + | // Notify opted-in users of status changes (fire-and-forget). | |
| 176 | + | // Paced at ~10/sec to stay under Postmark's default send rate. | |
| 177 | + | { | |
| 178 | + | let pool = state.db.clone(); | |
| 179 | + | let email_client = state.email.clone(); | |
| 180 | + | let host_url = state.config.host_url.clone(); | |
| 181 | + | let signing_secret = state.config.signing_secret.clone(); | |
| 182 | + | let current_status = snap.status.as_str().to_string(); | |
| 183 | + | let prev_status = previous_status.map_or("unknown", |s| s.as_str()).to_string(); | |
| 184 | + | tokio::spawn(async move { | |
| 185 | + | match db::users::get_status_alert_subscribers(&pool).await { | |
| 186 | + | Ok(subscribers) if !subscribers.is_empty() => { | |
| 187 | + | tracing::info!(count = subscribers.len(), "sending status notifications to opted-in users"); | |
| 188 | + | for sub in &subscribers { | |
| 189 | + | let unsub_url = crate::email::generate_unsubscribe_url( | |
| 190 | + | &host_url, sub.id, crate::email::UnsubscribeAction::Status, &sub.id.to_string(), &signing_secret, | |
| 191 | + | ); | |
| 192 | + | let _ = email_client.send_status_notification( | |
| 193 | + | &sub.email, | |
| 194 | + | sub.display_name.as_deref(), | |
| 195 | + | ¤t_status, | |
| 196 | + | &prev_status, | |
| 197 | + | &unsub_url, | |
| 198 | + | ).await; | |
| 199 | + | tokio::time::sleep(std::time::Duration::from_millis(100)).await; | |
| 200 | + | } | |
| 203 | 201 | } | |
| 202 | + | Err(e) => { | |
| 203 | + | tracing::error!(error = ?e, "failed to query status alert subscribers"); | |
| 204 | + | } | |
| 205 | + | _ => {} | |
| 204 | 206 | } | |
| 205 | - | Err(e) => { | |
| 206 | - | tracing::error!(error = ?e, "failed to query status alert subscribers"); | |
| 207 | - | } | |
| 208 | - | _ => {} | |
| 209 | - | } | |
| 210 | - | }); | |
| 207 | + | }); | |
| 208 | + | } | |
| 209 | + | ||
| 210 | + | // Only record the cooldown timestamp once we've issued at least | |
| 211 | + | // one notification path (admin or subscribers); admin block above | |
| 212 | + | // is gated on `alert_email` being set, but subscriber fan-out is | |
| 213 | + | // always spawned. Always-set is fine since the goal is "don't | |
| 214 | + | // re-notify within ALERT_COOLDOWN_SECS regardless of audience." | |
| 215 | + | last_alert_at = Some(Instant::now()); | |
| 211 | 216 | } | |
| 212 | 217 | ||
| 213 | 218 | // Create WAM ticket on degradation/error transitions |