| 7 |
7 |
|
//!
|
| 8 |
8 |
|
//! See `docs/scan-pipeline-audit.md` § 4.4 for the architecture.
|
| 9 |
9 |
|
|
| 10 |
|
- |
use std::sync::atomic::{AtomicBool, Ordering};
|
| 11 |
10 |
|
use std::sync::Arc;
|
| 12 |
11 |
|
use std::time::Duration;
|
| 13 |
12 |
|
|
| 40 |
39 |
|
pub pipeline: Arc<ScanPipeline>,
|
| 41 |
40 |
|
pub scan_semaphore: Arc<Semaphore>,
|
| 42 |
41 |
|
pub wam: Option<WamClient>,
|
| 43 |
|
- |
/// Runtime ClamAV liveness, maintained by [`spawn_clamav_health_probe`].
|
| 44 |
|
- |
/// `false` once the periodic probe finds clamd unreachable; a
|
| 45 |
|
- |
/// configured-but-erroring clamav scan then holds otherwise-clean uploads
|
| 46 |
|
- |
/// for review instead of passing them. `true` when clamav is healthy or
|
| 47 |
|
- |
/// not configured (so the overlay is a no-op without a probe). The boot
|
| 48 |
|
- |
/// gate only proves clamd was up at startup; this closes the post-boot
|
| 49 |
|
- |
/// "clamd died, FailOpen passes everything" window.
|
| 50 |
|
- |
pub clamav_healthy: Arc<AtomicBool>,
|
| 51 |
42 |
|
/// Cloudflare edge-cache purger, `None` when `CF_API_TOKEN`/`CF_ZONE_ID`
|
| 52 |
43 |
|
/// aren't configured. On quarantine we delete the origin object and, if a
|
| 53 |
44 |
|
/// purger is present, evict its `cdn_base_url`-prefixed URL from the edge so
|
| 71 |
62 |
|
///
|
| 72 |
63 |
|
/// Otherwise the pipeline returned `Clean`: a trusted uploader normally passes;
|
| 73 |
64 |
|
/// an untrusted one always routes to admin review. The ClamAV degraded-mode
|
| 74 |
|
- |
/// overlay adds one exception — if the runtime probe has found clamd DOWN *and*
|
| 75 |
|
- |
/// this scan's clamav (transport) layer errored (FailOpen made `final_status`
|
| 76 |
|
- |
/// skip it → Clean), even a trusted upload is held rather than passed on zero AV
|
| 77 |
|
- |
/// coverage. A single transient error while the probe still reports clamd
|
| 78 |
|
- |
/// healthy does NOT trip this — only a sustained outage the probe has observed.
|
|
65 |
+ |
/// overlay adds one exception — the clamav layer is `FailOpen` (a transport
|
|
66 |
+ |
/// error makes `final_status` skip it → Clean), but accepting a trusted upload
|
|
67 |
+ |
/// on zero AV coverage is the one fail-open we refuse, so ANY clamav error holds
|
|
68 |
+ |
/// the file for admin review rather than passing it (Run 9 Sec-S1: fail closed on
|
|
69 |
+ |
/// the first error instead of waiting for a runtime probe to observe a sustained
|
|
70 |
+ |
/// outage — the probe-lag window is gone).
|
| 79 |
71 |
|
fn resolve_pass_status(
|
| 80 |
72 |
|
pipeline_status: FileScanStatus,
|
| 81 |
73 |
|
is_trusted: bool,
|
| 82 |
|
- |
clamav_down: bool,
|
| 83 |
74 |
|
layers: &[LayerResult],
|
| 84 |
75 |
|
) -> FileScanStatus {
|
| 85 |
76 |
|
if pipeline_status == FileScanStatus::HeldForReview {
|
| 86 |
77 |
|
return FileScanStatus::HeldForReview;
|
| 87 |
78 |
|
}
|
| 88 |
|
- |
let clamav_errored = layers
|
| 89 |
|
- |
.iter()
|
| 90 |
|
- |
.any(|l| l.layer == "clamav" && l.verdict == LayerVerdict::Error);
|
| 91 |
|
- |
if is_trusted && !(clamav_down && clamav_errored) {
|
|
79 |
+ |
if is_trusted && !clamav_layer_errored(layers) {
|
| 92 |
80 |
|
FileScanStatus::Clean
|
| 93 |
81 |
|
} else {
|
| 94 |
82 |
|
FileScanStatus::HeldForReview
|
| 95 |
83 |
|
}
|
| 96 |
84 |
|
}
|
| 97 |
85 |
|
|
| 98 |
|
- |
/// Periodically ping clamd and maintain the shared liveness flag consumed by
|
| 99 |
|
- |
/// [`resolve_pass_status`]. Spawned once at startup when clamav is configured.
|
| 100 |
|
- |
/// Observes `shutdown_rx` for graceful exit.
|
| 101 |
|
- |
pub fn spawn_clamav_health_probe(
|
| 102 |
|
- |
socket: String,
|
| 103 |
|
- |
healthy: Arc<AtomicBool>,
|
| 104 |
|
- |
mut shutdown_rx: tokio::sync::watch::Receiver<()>,
|
| 105 |
|
- |
) {
|
| 106 |
|
- |
tokio::spawn(async move {
|
| 107 |
|
- |
let mut interval =
|
| 108 |
|
- |
tokio::time::interval(Duration::from_secs(constants::SCAN_CLAMAV_HEALTH_PROBE_SECS));
|
| 109 |
|
- |
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
| 110 |
|
- |
loop {
|
| 111 |
|
- |
tokio::select! {
|
| 112 |
|
- |
_ = interval.tick() => {
|
| 113 |
|
- |
let ok = super::clamav::ping(&socket).await.is_ok();
|
| 114 |
|
- |
let prev = healthy.swap(ok, Ordering::Relaxed);
|
| 115 |
|
- |
if prev != ok {
|
| 116 |
|
- |
if ok {
|
| 117 |
|
- |
tracing::warn!("clamav health probe: clamd RECOVERED — scans resume normal FailOpen posture");
|
| 118 |
|
- |
} else {
|
| 119 |
|
- |
tracing::error!("clamav health probe: clamd UNREACHABLE — trusted uploads whose clamav layer errors are now held for review");
|
| 120 |
|
- |
}
|
| 121 |
|
- |
}
|
| 122 |
|
- |
}
|
| 123 |
|
- |
_ = shutdown_rx.changed() => break,
|
| 124 |
|
- |
}
|
| 125 |
|
- |
}
|
| 126 |
|
- |
});
|
|
86 |
+ |
/// True if the clamav layer reported a transport/scan error on this file. Under
|
|
87 |
+ |
/// the layer's `FailOpen` policy `final_status` skips such an error, so the trust
|
|
88 |
+ |
/// overlay re-reads it to fail closed for trusted uploads.
|
|
89 |
+ |
fn clamav_layer_errored(layers: &[LayerResult]) -> bool {
|
|
90 |
+ |
layers
|
|
91 |
+ |
.iter()
|
|
92 |
+ |
.any(|l| l.layer == "clamav" && l.verdict == LayerVerdict::Error)
|
| 127 |
93 |
|
}
|
| 128 |
94 |
|
|
| 129 |
95 |
|
/// Spawn `n` scan workers on the current tokio runtime. Each worker drains
|
| 431 |
397 |
|
|
| 432 |
398 |
|
// Pipeline returned Clean or HeldForReview. Apply the uploader-trust
|
| 433 |
399 |
|
// overlay (untrusted users always route to admin review) plus the ClamAV
|
| 434 |
|
- |
// degraded-mode overlay (a trusted upload is held when clamd is known-down
|
| 435 |
|
- |
// and this scan's clamav layer errored). See `resolve_pass_status`.
|
|
400 |
+ |
// degraded-mode overlay (a trusted upload whose clamav layer errored is held
|
|
401 |
+ |
// rather than passed on reduced AV coverage). See `resolve_pass_status`.
|
| 436 |
402 |
|
let is_trusted = db::users::is_upload_trusted(&ctx.db, job.user_id).await?;
|
| 437 |
|
- |
let clamav_down = !ctx.clamav_healthy.load(Ordering::Relaxed);
|
| 438 |
|
- |
let status = resolve_pass_status(result.status, is_trusted, clamav_down, &result.layers);
|
| 439 |
|
- |
|
| 440 |
|
- |
// Visibility for the narrow fail-open window: a trusted upload passes Clean
|
| 441 |
|
- |
// while its clamav layer errored but the health probe has NOT yet observed
|
| 442 |
|
- |
// clamd down. resolve_pass_status deliberately lets this through (a single
|
| 443 |
|
- |
// transient error must not hold every trusted upload), but it is the one
|
| 444 |
|
- |
// path where a file is accepted on reduced AV coverage without a hold — so
|
| 445 |
|
- |
// an operator must be able to see it rather than have it pass silently
|
| 446 |
|
- |
// (ultra-fuzz Run 4 M-Sec4).
|
| 447 |
|
- |
if clamav_fail_open_occurred(status, is_trusted, clamav_down, &result.layers) {
|
|
403 |
+ |
let status = resolve_pass_status(result.status, is_trusted, &result.layers);
|
|
404 |
+ |
|
|
405 |
+ |
// Visibility: an otherwise-clean trusted upload that we held *because* its
|
|
406 |
+ |
// clamav layer errored. We now fail closed on the first such error (Run 9
|
|
407 |
+ |
// Sec-S1), so this is no longer a silent acceptance — but a clamd outage that
|
|
408 |
+ |
// starts holding trusted uploads is still operationally important, so surface
|
|
409 |
+ |
// it via a metric (scraped) and a WAM ticket (active alert).
|
|
410 |
+ |
if clamav_degraded_hold_occurred(result.status, is_trusted, &result.layers) {
|
| 448 |
411 |
|
tracing::warn!(
|
| 449 |
412 |
|
s3_key = %job.s3_key,
|
| 450 |
413 |
|
user_id = %job.user_id,
|
| 451 |
|
- |
"clamav layer errored while the health probe still reports clamd up; \
|
| 452 |
|
- |
trusted upload passed on reduced AV coverage (fail-open window)"
|
|
414 |
+ |
"clamav layer errored on a trusted upload; held for review on reduced AV coverage"
|
| 453 |
415 |
|
);
|
| 454 |
|
- |
// Metric (scraped) + WAM ticket (active alert): close R6-Sec-L1's repeated
|
| 455 |
|
- |
// "log-only" finding so the reduced-coverage window is operationally visible.
|
| 456 |
|
- |
crate::metrics::record_clamav_fail_open();
|
|
416 |
+ |
crate::metrics::record_clamav_degraded_hold();
|
| 457 |
417 |
|
if let Some(wam) = ctx.wam.clone() {
|
| 458 |
418 |
|
let body = format!(
|
| 459 |
|
- |
"A trusted upload passed Clean while the clamav layer errored and the \
|
| 460 |
|
- |
health probe had not yet observed clamd down (fail-open window).\n\n\
|
|
419 |
+ |
"A trusted upload was held for review because its clamav layer errored \
|
|
420 |
+ |
(reduced AV coverage — clamd may be unreachable).\n\n\
|
| 461 |
421 |
|
s3_key: {}\nuser_id: {}",
|
| 462 |
422 |
|
job.s3_key, job.user_id
|
| 463 |
423 |
|
);
|
| 464 |
424 |
|
wam.create_ticket(
|
| 465 |
|
- |
"ClamAV fail-open: trusted upload accepted on reduced AV coverage",
|
|
425 |
+ |
"ClamAV degraded: trusted upload held on reduced AV coverage",
|
| 466 |
426 |
|
Some(&body),
|
| 467 |
427 |
|
"medium",
|
| 468 |
|
- |
"clamav-fail-open",
|
|
428 |
+ |
"clamav-degraded-hold",
|
| 469 |
429 |
|
Some(&job.s3_key),
|
| 470 |
430 |
|
)
|
| 471 |
431 |
|
.await;
|
| 475 |
435 |
|
Ok(status)
|
| 476 |
436 |
|
}
|
| 477 |
437 |
|
|
| 478 |
|
- |
/// The one fail-open acceptance path: a trusted upload resolved `Clean` while its
|
| 479 |
|
- |
/// clamav layer errored but the health probe had not yet observed clamd down.
|
| 480 |
|
- |
/// Extracted as a pure predicate so the condition is unit-tested rather than
|
| 481 |
|
- |
/// living only inline (ultra-fuzz Run 6 R6-Sec-L1).
|
| 482 |
|
- |
fn clamav_fail_open_occurred(
|
| 483 |
|
- |
status: FileScanStatus,
|
|
438 |
+ |
/// True when an otherwise-clean trusted upload was held *because* its clamav layer
|
|
439 |
+ |
/// errored (the file would have passed but for the AV-coverage gap). Extracted as
|
|
440 |
+ |
/// a pure predicate so the condition is unit-tested rather than living only inline.
|
|
441 |
+ |
fn clamav_degraded_hold_occurred(
|
|
442 |
+ |
pipeline_status: FileScanStatus,
|
| 484 |
443 |
|
is_trusted: bool,
|
| 485 |
|
- |
clamav_down: bool,
|
| 486 |
444 |
|
layers: &[crate::scanning::LayerResult],
|
| 487 |
445 |
|
) -> bool {
|
| 488 |
|
- |
status == FileScanStatus::Clean
|
| 489 |
|
- |
&& is_trusted
|
| 490 |
|
- |
&& !clamav_down
|
| 491 |
|
- |
&& layers
|
| 492 |
|
- |
.iter()
|
| 493 |
|
- |
.any(|l| l.layer == "clamav" && l.verdict == LayerVerdict::Error)
|
|
446 |
+ |
pipeline_status == FileScanStatus::Clean && is_trusted && clamav_layer_errored(layers)
|
| 494 |
447 |
|
}
|
| 495 |
448 |
|
|
| 496 |
449 |
|
/// Update the per-entity `scan_status` column for the kinds that have one.
|
| 541 |
494 |
|
const CLEAN: FileScanStatus = FileScanStatus::Clean;
|
| 542 |
495 |
|
|
| 543 |
496 |
|
#[test]
|
| 544 |
|
- |
fn trusted_passes_when_clamav_healthy() {
|
|
497 |
+ |
fn trusted_passes_when_clamav_clean() {
|
| 545 |
498 |
|
let layers = [layer("clamav", LayerVerdict::Pass)];
|
| 546 |
|
- |
assert_eq!(resolve_pass_status(CLEAN, true, false, &layers), FileScanStatus::Clean);
|
|
499 |
+ |
assert_eq!(resolve_pass_status(CLEAN, true, &layers), FileScanStatus::Clean);
|
| 547 |
500 |
|
}
|
| 548 |
501 |
|
|
| 549 |
502 |
|
#[test]
|
| 550 |
|
- |
fn trusted_held_when_clamav_down_and_errored() {
|
| 551 |
|
- |
// The vulnerability window: clamd died post-boot (probe says down) AND
|
| 552 |
|
- |
// this scan's clamav (transport) layer errored (FailOpen would otherwise
|
| 553 |
|
- |
// skip it). final_status saw FailOpen → Clean; the overlay holds it.
|
|
503 |
+ |
fn trusted_held_on_first_clamav_error() {
|
|
504 |
+ |
// Sec-S1 (Run 9): fail closed on the FIRST clamav error. The clamav layer
|
|
505 |
+ |
// is FailOpen (final_status skipped the error → Clean), but a trusted
|
|
506 |
+ |
// upload on zero AV coverage is held for review immediately — no waiting
|
|
507 |
+ |
// for a runtime probe to observe a sustained outage.
|
| 554 |
508 |
|
let layers = [layer("clamav", LayerVerdict::Error)];
|
| 555 |
509 |
|
assert_eq!(
|
| 556 |
|
- |
resolve_pass_status(CLEAN, true, true, &layers),
|
|
510 |
+ |
resolve_pass_status(CLEAN, true, &layers),
|
| 557 |
511 |
|
FileScanStatus::HeldForReview
|
| 558 |
512 |
|
);
|
| 559 |
513 |
|
}
|
| 560 |
514 |
|
|
| 561 |
515 |
|
#[test]
|
| 562 |
|
- |
fn trusted_passes_when_down_flag_set_but_clamav_did_not_error() {
|
| 563 |
|
- |
// Probe flag stale/racing but the clamav layer actually passed — not a
|
| 564 |
|
- |
// coverage gap, so don't hold.
|
| 565 |
|
- |
let layers = [layer("clamav", LayerVerdict::Pass)];
|
| 566 |
|
- |
assert_eq!(resolve_pass_status(CLEAN, true, true, &layers), FileScanStatus::Clean);
|
| 567 |
|
- |
}
|
| 568 |
|
- |
|
| 569 |
|
- |
// ── clamav_fail_open_occurred (R6-Sec-L1 observability predicate) ──
|
| 570 |
|
- |
|
| 571 |
|
- |
#[test]
|
| 572 |
|
- |
fn fail_open_detected_for_trusted_errored_while_probe_up() {
|
| 573 |
|
- |
let layers = [layer("clamav", LayerVerdict::Error)];
|
| 574 |
|
- |
assert!(clamav_fail_open_occurred(CLEAN, true, false, &layers));
|
| 575 |
|
- |
}
|
| 576 |
|
- |
|
| 577 |
|
- |
#[test]
|
| 578 |
|
- |
fn fail_open_not_detected_for_untrusted_upload() {
|
| 579 |
|
- |
// Untrusted uploads are held on a clamav error, never fail-open.
|
| 580 |
|
- |
let layers = [layer("clamav", LayerVerdict::Error)];
|
| 581 |
|
- |
assert!(!clamav_fail_open_occurred(CLEAN, false, false, &layers));
|
| 582 |
|
- |
}
|
| 583 |
|
- |
|
| 584 |
|
- |
#[test]
|
| 585 |
|
- |
fn fail_open_not_detected_when_probe_already_down() {
|
| 586 |
|
- |
// resolve_pass_status already held this case; it is not a silent window.
|
| 587 |
|
- |
let layers = [layer("clamav", LayerVerdict::Error)];
|
| 588 |
|
- |
assert!(!clamav_fail_open_occurred(CLEAN, true, true, &layers));
|
| 589 |
|
- |
}
|
| 590 |
|
- |
|
| 591 |
|
- |
#[test]
|
| 592 |
|
- |
fn fail_open_not_detected_when_clamav_passed() {
|
| 593 |
|
- |
let layers = [layer("clamav", LayerVerdict::Pass)];
|
| 594 |
|
- |
assert!(!clamav_fail_open_occurred(CLEAN, true, false, &layers));
|
| 595 |
|
- |
}
|
| 596 |
|
- |
|
| 597 |
|
- |
#[test]
|
| 598 |
|
- |
fn fail_open_not_detected_when_pipeline_not_clean() {
|
| 599 |
|
- |
let layers = [layer("clamav", LayerVerdict::Error)];
|
| 600 |
|
- |
assert!(!clamav_fail_open_occurred(
|
| 601 |
|
- |
FileScanStatus::HeldForReview,
|
| 602 |
|
- |
true,
|
| 603 |
|
- |
false,
|
| 604 |
|
- |
&layers
|
| 605 |
|
- |
));
|
| 606 |
|
- |
}
|
| 607 |
|
- |
|
| 608 |
|
- |
#[test]
|
| 609 |
|
- |
fn trusted_passes_on_transient_error_while_probe_healthy() {
|
| 610 |
|
- |
// Single transient clamav error but the probe still reports healthy —
|
| 611 |
|
- |
// don't flood the review queue on a blip.
|
| 612 |
|
- |
let layers = [layer("clamav", LayerVerdict::Error)];
|
| 613 |
|
- |
assert_eq!(resolve_pass_status(CLEAN, true, false, &layers), FileScanStatus::Clean);
|
| 614 |
|
- |
}
|
| 615 |
|
- |
|
| 616 |
|
- |
#[test]
|
| 617 |
516 |
|
fn untrusted_always_held() {
|
| 618 |
517 |
|
let layers = [layer("clamav", LayerVerdict::Pass)];
|
| 619 |
518 |
|
assert_eq!(
|
| 620 |
|
- |
resolve_pass_status(CLEAN, false, false, &layers),
|
|
519 |
+ |
resolve_pass_status(CLEAN, false, &layers),
|
| 621 |
520 |
|
FileScanStatus::HeldForReview
|
| 622 |
521 |
|
);
|
| 623 |
522 |
|
}
|
| 625 |
524 |
|
#[test]
|
| 626 |
525 |
|
fn trusted_held_when_pipeline_already_held_not_downgraded() {
|
| 627 |
526 |
|
// CHRONIC S1: a reachable-but-incomplete clamav scan (or any FailClosed
|
| 628 |
|
- |
// layer error) makes final_status = HeldForReview. A trusted uploader on
|
| 629 |
|
- |
// a HEALTHY clamd must NOT have that downgraded to Clean — the prior code
|
| 630 |
|
- |
// ignored the pipeline status and would have passed it.
|
|
527 |
+ |
// layer error) makes final_status = HeldForReview. A trusted uploader must
|
|
528 |
+ |
// NOT have that downgraded to Clean.
|
| 631 |
529 |
|
let layers = [layer("clamav_incomplete", LayerVerdict::Error)];
|
| 632 |
530 |
|
assert_eq!(
|
| 633 |
|
- |
resolve_pass_status(FileScanStatus::HeldForReview, true, false, &layers),
|
|
531 |
+ |
resolve_pass_status(FileScanStatus::HeldForReview, true, &layers),
|
| 634 |
532 |
|
FileScanStatus::HeldForReview
|
| 635 |
533 |
|
);
|
| 636 |
534 |
|
}
|
| 642 |
540 |
|
// uploaders too.
|
| 643 |
541 |
|
let layers = [layer("scan_size_limit", LayerVerdict::Error)];
|
| 644 |
542 |
|
assert_eq!(
|
| 645 |
|
- |
resolve_pass_status(FileScanStatus::HeldForReview, true, false, &layers),
|
|
543 |
+ |
resolve_pass_status(FileScanStatus::HeldForReview, true, &layers),
|
| 646 |
544 |
|
FileScanStatus::HeldForReview
|
| 647 |
545 |
|
);
|
| 648 |
546 |
|
}
|
|
547 |
+ |
|
|
548 |
+ |
// ── clamav_degraded_hold_occurred (observability predicate) ──
|
|
549 |
+ |
|
|
550 |
+ |
#[test]
|
|
551 |
+ |
fn degraded_hold_detected_for_trusted_clamav_error() {
|
|
552 |
+ |
let layers = [layer("clamav", LayerVerdict::Error)];
|
|
553 |
+ |
assert!(clamav_degraded_hold_occurred(CLEAN, true, &layers));
|
|
554 |
+ |
}
|
|
555 |
+ |
|
|
556 |
+ |
#[test]
|
|
557 |
+ |
fn degraded_hold_not_for_untrusted_upload() {
|
|
558 |
+ |
// Untrusted uploads are always held regardless of clamav; the degraded
|
|
559 |
+ |
// metric tracks only the trusted-upload coverage-gap case.
|
|
560 |
+ |
let layers = [layer("clamav", LayerVerdict::Error)];
|
|
561 |
+ |
assert!(!clamav_degraded_hold_occurred(CLEAN, false, &layers));
|
|
562 |
+ |
}
|
|
563 |
+ |
|
|
564 |
+ |
#[test]
|
|
565 |
+ |
fn degraded_hold_not_when_clamav_passed() {
|
|
566 |
+ |
let layers = [layer("clamav", LayerVerdict::Pass)];
|
|
567 |
+ |
assert!(!clamav_degraded_hold_occurred(CLEAN, true, &layers));
|
|
568 |
+ |
}
|
|
569 |
+ |
|
|
570 |
+ |
#[test]
|
|
571 |
+ |
fn degraded_hold_not_when_pipeline_not_clean() {
|
|
572 |
+ |
// A file the pipeline already held (FailClosed) wasn't held *because* of
|
|
573 |
+ |
// the clamav coverage gap, so it is not a degraded-hold event.
|
|
574 |
+ |
let layers = [layer("clamav", LayerVerdict::Error)];
|
|
575 |
+ |
assert!(!clamav_degraded_hold_occurred(
|
|
576 |
+ |
FileScanStatus::HeldForReview,
|
|
577 |
+ |
true,
|
|
578 |
+ |
&layers
|
|
579 |
+ |
));
|
|
580 |
+ |
}
|
| 649 |
581 |
|
}
|
| 650 |
582 |
|
|