Skip to main content

max / makenotwork

audit Run 15 Phase 4: resilience polish - Health dashboard session status probes whether the session store is queryable (matches monitor.rs) instead of active_session_count > 0, which false-alarmed the whole dashboard to "Issues detected" on an idle or freshly-swept instance. - Build runner bounds the artifact SCP download (600s) and remote cleanup rm -rf (60s) with tokio::time::timeout; previously only the build ssh was timeout-wrapped, so a build host stalling mid-transfer wedged the single build slot forever. kill_on_drop reaps the orphaned scp/ssh on timeout. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-02 01:04 UTC
Commit: 64dbc1d28079376c73ae4b3810f0ecf737d45a2c
Parent: 2389eaa
2 files changed, +47 insertions, -11 deletions
M server/src/build_runner.rs +38 -9
@@ -10,6 +10,28 @@ use crate::constants::{BUILD_MAX_LOG_BYTES, BUILD_TIMEOUT_SECS};
10 10 use crate::db::{self, BuildStatus, DbBuild, DbBuildConfig};
11 11 use crate::AppState;
12 12
13 + /// Wall-clock cap for an artifact SCP download. Only the build ssh was
14 + /// timeout-wrapped; a build host that stalled mid-transfer wedged the single
15 + /// build slot forever (Run 15 Resilience). Artifacts can be large, so this is
16 + /// generous; `kill_on_drop` reaps the scp when the timed-out future is dropped.
17 + const SCP_TRANSFER_TIMEOUT_SECS: u64 = 600;
18 +
19 + /// Wall-clock cap for best-effort remote cleanup (`rm -rf` of the build dir).
20 + /// Short: cleanup should be near-instant, and a stalled cleanup must not hold
21 + /// the build slot.
22 + const SSH_CLEANUP_TIMEOUT_SECS: u64 = 60;
23 +
24 + /// Best-effort remote build-dir cleanup with a bounded wall-clock. `kill_on_drop`
25 + /// in `run_ssh_command` reaps the ssh process if the timeout fires.
26 + async fn cleanup_remote_dir(host: &str, build_dir: &str) {
27 + let cmd = format!("rm -rf {}", shell_escape(build_dir));
28 + let _ = tokio::time::timeout(
29 + Duration::from_secs(SSH_CLEANUP_TIMEOUT_SECS),
30 + run_ssh_command(host, &cmd),
31 + )
32 + .await;
33 + }
34 +
13 35 /// Post-receive hook script template.
14 36 /// `__HMAC__` is replaced with a per-repo HMAC signature so the global token
15 37 /// is never stored on disk. The server verifies via `HMAC(token, owner:repo)`.
@@ -462,11 +484,11 @@ async fn execute_target(
462 484 Ok(Ok(output)) => output,
463 485 Ok(Err(e)) => {
464 486 // Cleanup remote build dir (best-effort)
465 - let _ = run_ssh_command(host, &format!("rm -rf {}", shell_escape(&build_dir))).await;
487 + cleanup_remote_dir(host, &build_dir).await;
466 488 return Err(format!("SSH command failed: {e}"));
467 489 }
468 490 Err(_) => {
469 - let _ = run_ssh_command(host, &format!("rm -rf {}", shell_escape(&build_dir))).await;
491 + cleanup_remote_dir(host, &build_dir).await;
470 492 return Err("build timed out".to_string());
471 493 }
472 494 };
@@ -493,7 +515,7 @@ async fn execute_target(
493 515 run_scp_download(host, &format!("{scp_remote_path}.sig"), &local_sig_tmp).await;
494 516
495 517 // Cleanup remote build dir
496 - let _ = run_ssh_command(host, &format!("rm -rf {}", shell_escape(&build_dir))).await;
518 + cleanup_remote_dir(host, &build_dir).await;
497 519
498 520 if let Err(e) = scp_result {
499 521 // The main artifact failed, but the .sig sidecar may already be on disk
@@ -660,14 +682,21 @@ async fn run_scp_download(
660 682 }
661 683 args.push(&remote);
662 684 args.push(local_path);
663 - let output = tokio::process::Command::new("scp")
685 + let scp = tokio::process::Command::new("scp")
664 686 .args(&args)
665 - // Kill scp if this future is dropped (build timeout) rather than leaving
666 - // an orphaned transfer running (ultra-fuzz Run 11 Perf).
687 + // Kill scp if this future is dropped (build timeout, or the transfer
688 + // timeout below) rather than leaving an orphaned transfer running
689 + // (ultra-fuzz Run 11 Perf).
667 690 .kill_on_drop(true)
668 - .output()
669 - .await
670 - .map_err(|e| format!("failed to spawn scp: {e}"))?;
691 + .output();
692 + // Bound the whole transfer: a build host that stalls mid-stream would
693 + // otherwise hang the build task and pin the single build slot forever
694 + // (Run 15 Resilience). ConnectTimeout only covers connection setup.
695 + let output = match tokio::time::timeout(Duration::from_secs(SCP_TRANSFER_TIMEOUT_SECS), scp).await
696 + {
697 + Ok(r) => r.map_err(|e| format!("failed to spawn scp: {e}"))?,
698 + Err(_) => return Err("scp transfer timed out".to_string()),
699 + };
671 700
672 701 if output.status.success() {
673 702 Ok(())
M server/src/routes/pages/public/health/mod.rs +9 -2
@@ -337,8 +337,15 @@ async fn collect_health(state: &AppState) -> HealthData {
337 337 let email_status_class = if email_configured { "status-ok" } else { "status-warn" };
338 338 let email_provider = if email_configured { "Postmark" } else { "Console" };
339 339
340 - // Session status
341 - let session_ok = stats.active_session_count > 0;
340 + // Session status: the signal is whether the session store is queryable, not
341 + // how many sessions happen to be active. A quiet or freshly-swept instance
342 + // has zero active sessions but is perfectly healthy — the old
343 + // `active_session_count > 0` check false-alarmed the whole dashboard to
344 + // "Issues detected" on an idle instance (Run 15). Matches monitor.rs.
345 + let session_ok = sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM tower_sessions.session)")
346 + .fetch_one(&state.db)
347 + .await
348 + .is_ok();
342 349 let session_status = if session_ok { "Active" } else { "Error" };
343 350 let session_status_class = if session_ok { "status-ok" } else { "status-error" };
344 351