max / makenotwork

observability: scan verdict, duration, and queue-depth metrics (audit Run 20 Phase 8) The scan pipeline exposed only the ClamAV degraded-hold counter, so quarantine/ hold/error rates, scan latency, and backlog were invisible to dashboards despite being security-critical. - scan_verdicts_total{verdict} and scan_duration_seconds recorded per job in the worker (including the pipeline-error path). - scan_queue_jobs{state} gauges (pending/running/stuck) sampled in the monitor loop alongside the other stat collectors. All graphable at /metrics rather than only via the unauthenticated health JSON. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: Max Johnson <me@maxj.phd> · 2026-07-03 07:55 UTC

Commit: 0df7c958f011ed20bf1d2f636980197b050352dc

Parent: 1b37fb1

3 files changed, +46 insertions, -0 deletions

M server/src/metrics.rs +29

			@@ -309,6 +309,35 @@ pub fn record_clamav_degraded_hold() {
309	309		counter!("clamav_degraded_hold_total").increment(1);
310	310		}
311	311
	312	+	/// Record the terminal verdict of a completed file scan, keyed by verdict
	313	+	/// (`clean`, `quarantined`, `held_for_review`, `error`). Before Run 20 the scan
	314	+	/// pipeline exposed only the degraded-hold counter, so quarantine/hold rates and
	315	+	/// scanner error rates were invisible to dashboards despite being security-
	316	+	/// critical signals.
	317	+	pub fn record_scan_verdict(verdict: &'static str) {
	318	+	counter!("scan_verdicts_total", "verdict" => verdict).increment(1);
	319	+	}
	320	+
	321	+	/// Record wall-clock duration of a completed file scan (seconds), so a slow
	322	+	/// scanner (clamd backpressure, large-archive walk) is graphable rather than
	323	+	/// only inferable from queue depth.
	324	+	pub fn record_scan_duration(seconds: f64) {
	325	+	histogram!("scan_duration_seconds").record(seconds);
	326	+	}
	327	+
	328	+	/// Sample the scan-job queue depth (pending / running / stuck) as gauges.
	329	+	/// Called from the periodic monitor loop alongside the other stat collectors so
	330	+	/// a stalled worker or a scan backlog is observable at `/metrics`, not just via
	331	+	/// the unauthenticated health JSON.
	332	+	pub async fn record_scan_queue_stats(pool: &sqlx::PgPool) {
	333	+	let pending = crate::db::scan_jobs::queued_count(pool).await.unwrap_or(0);
	334	+	let running = crate::db::scan_jobs::running_count(pool).await.unwrap_or(0);
	335	+	let stuck = crate::db::scan_jobs::stuck_count(pool, 300).await.unwrap_or(0);
	336	+	gauge!("scan_queue_jobs", "state" => "pending").set(pending as f64);
	337	+	gauge!("scan_queue_jobs", "state" => "running").set(running as f64);
	338	+	gauge!("scan_queue_jobs", "state" => "stuck").set(stuck as f64);
	339	+	}
	340	+
312	341		/// Axum middleware that implements idempotency keys for POST endpoints.
313	342		///
314	343		/// If the request includes an `Idempotency-Key` header and the user is

M server/src/monitor.rs +1

			@@ -191,6 +191,7 @@ async fn run_monitor_loop(state: AppState, mut shutdown_rx: watch::Receiver<()>)
191	191		if should_refresh {
192	192		crate::metrics::record_storage_fill_stats(&state.db).await;
193	193		crate::metrics::record_custom_pages_stats(&state.db).await;
	194	+	crate::metrics::record_scan_queue_stats(&state.db).await;
194	195		}
195	196
196	197		// Log status changes. Skip the bootstrap None->Operational transition

M server/src/scanning/worker.rs +16

			@@ -205,6 +205,7 @@ async fn process_job(ctx: &WorkerContext, job: ScanJob) -> Result<(), Box<dyn st
205	205		let kind = job.typed_kind().ok_or_else(\|\| format!("unknown target_kind: {}", job.target_kind))?;
206	206		let file_type = job.typed_file_type().ok_or_else(\|\| format!("unknown file_type: {}", job.file_type))?;
207	207		let target_id = job.target_id;
	208	+	let started = std::time::Instant::now();
208	209
209	210		// Mark target as Scanning while the worker is running (only entities with
210	211		// a scan_status column). This is a visible signal in the admin dashboard
			@@ -218,11 +219,26 @@ async fn process_job(ctx: &WorkerContext, job: ScanJob) -> Result<(), Box<dyn st
218	219		// HeldForReview so admins see it on the dashboard and decide
219	220		// whether to delete the orphan record.
220	221		update_entity_status(&ctx.db, kind, target_id, FileScanStatus::HeldForReview).await.ok();
	222	+	crate::metrics::record_scan_verdict("error");
	223	+	crate::metrics::record_scan_duration(started.elapsed().as_secs_f64());
221	224		return Err(e);
222	225		}
223	226		};
224	227		update_entity_status(&ctx.db, kind, target_id, entity_status).await?;
225	228
	229	+	// Verdict + duration metrics (Run 20 Observability): quarantine/hold/error
	230	+	// rates and scan latency are now graphable at /metrics.
	231	+	let verdict_label = match entity_status {
	232	+	FileScanStatus::Clean => "clean",
	233	+	FileScanStatus::Quarantined => "quarantined",
	234	+	FileScanStatus::HeldForReview => "held_for_review",
	235	+	FileScanStatus::Pending => "pending",
	236	+	FileScanStatus::Scanning => "scanning",
	237	+	FileScanStatus::Error => "error",
	238	+	};
	239	+	crate::metrics::record_scan_verdict(verdict_label);
	240	+	crate::metrics::record_scan_duration(started.elapsed().as_secs_f64());
	241	+
226	242		db::scan_jobs::mark_done(&ctx.db, job_id).await?;
227	243		Ok(())
228	244		}