Skip to main content

max / makenotwork

server: drain scheduler + bg pool concurrently within one shutdown budget The two post-serve drains now run via tokio::join! under a single SHUTDOWN_POOL_DRAIN_SECS (5s) window instead of sequentially (>=5s + 5s could reach 10s and let the hard-exit truncate the second). Hard-exit reconciled to request(10) + pool(5) = 15s with a compile-time invariant assert. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-05 15:50 UTC
Commit: 6349845e43709ba751a5af3f14ff537ebdbec82a
Parent: ef761e8
1 file changed, +52 insertions, -30 deletions
M server/src/main.rs +52 -30
@@ -493,9 +493,11 @@ async fn main() {
493 493 .expect("Failed to bind to address");
494 494
495 495 // Use into_make_service_with_connect_info to provide peer IP for rate limiting.
496 - // After the shutdown signal fires, in-flight requests get a 10-second window
497 - // to complete before the process force-exits. This prevents hung connections
498 - // from blocking deployment restarts indefinitely.
496 + // After the shutdown signal fires, in-flight requests drain, then the
497 + // scheduler + background pool drain concurrently; the hard-exit timer
498 + // (SHUTDOWN_HARD_EXIT_SECS, armed at signal receipt) is the backstop that
499 + // frees a hung restart. The budget is reconciled so the pool drain fits
500 + // before the hard exit (see the shutdown constants below).
499 501 axum::serve(
500 502 listener,
501 503 app.into_make_service_with_connect_info::<std::net::SocketAddr>(),
@@ -506,38 +508,56 @@ async fn main() {
506 508
507 509 drop(shutdown_tx); // signal monitor, scheduler, and scan workers to stop
508 510
509 - // Drain the scheduler's in-flight tick before returning. Its jobs
510 - // (platform-credit settlement, pending-S3-deletion retries, webhook
511 - // re-drives) are engineered idempotent, so a truncated tick is recoverable —
512 - // but awaiting a bounded window lets an in-progress tick finish cleanly
513 - // instead of being cut off by the armed hard-exit deadline. If it overruns,
514 - // we proceed and rely on idempotency as the backstop.
515 - match tokio::time::timeout(Duration::from_secs(5), scheduler_handle).await {
516 - Ok(Ok(())) => tracing::info!("scheduler drained"),
517 - Ok(Err(e)) => tracing::warn!(error = ?e, "scheduler task ended abnormally on shutdown"),
511 + // Drain the scheduler's in-flight tick (platform-credit settlement,
512 + // pending-S3-deletion retries, webhook re-drives) and the fire-and-forget
513 + // background pool (queued emails / cache purges / MT thread creations)
514 + // together. Both are engineered idempotent, so a truncated drain is
515 + // recoverable — but awaiting a bounded window lets in-progress work finish
516 + // cleanly. They run CONCURRENTLY under one shared budget so the pool drain
517 + // isn't starved by a slow scheduler tick before the hard-exit fires: the old
518 + // sequential ≤5s + ≤5s could reach 10s and let the hard-exit truncate the
519 + // second drain (audit Run 23 F4). If the shared window overruns we proceed
520 + // and rely on idempotency as the backstop.
521 + let drains = async { tokio::join!(scheduler_handle, bg_handle) };
522 + match tokio::time::timeout(Duration::from_secs(SHUTDOWN_POOL_DRAIN_SECS), drains).await {
523 + Ok((scheduler_res, bg_res)) => {
524 + match scheduler_res {
525 + Ok(()) => tracing::info!("scheduler drained"),
526 + Err(e) => tracing::warn!(error = ?e, "scheduler task ended abnormally on shutdown"),
527 + }
528 + match bg_res {
529 + Ok(()) => tracing::info!("background pool drained"),
530 + Err(e) => {
531 + tracing::warn!(error = ?e, "background pool task ended abnormally on shutdown")
532 + }
533 + }
534 + }
518 535 Err(_) => tracing::warn!(
519 - "scheduler did not drain within 5s of shutdown; proceeding (jobs are idempotent)"
536 + "scheduler + background pool did not drain within {SHUTDOWN_POOL_DRAIN_SECS}s of \
537 + shutdown; proceeding (work is idempotent)"
520 538 ),
521 539 }
522 540
523 - // Drain the fire-and-forget background pool: it runs the queued emails /
524 - // cache purges / MT thread creations before the armed hard-exit deadline,
525 - // instead of abandoning them mid-flight (audit Run 17 Resilience). Bounded
526 - // like the scheduler; if it overruns we proceed to the hard exit.
527 - match tokio::time::timeout(Duration::from_secs(5), bg_handle).await {
528 - Ok(Ok(())) => tracing::info!("background pool drained"),
529 - Ok(Err(e)) => tracing::warn!(error = ?e, "background pool task ended abnormally on shutdown"),
530 - Err(_) => {
531 - tracing::warn!("background pool did not drain within 5s of shutdown; proceeding")
532 - }
533 - }
534 -
535 541 tracing::info!("Server shut down gracefully");
536 542 }
537 543
544 + /// Shutdown budget, reconciled so the background-pool drain can't be truncated
545 + /// by the hard-exit deadline (audit Run 23 F4). The hard-exit timer arms at
546 + /// signal receipt and must outlast the in-flight request drain PLUS the
547 + /// post-serve drain window — otherwise the later drain is cut off mid-flight.
548 + /// The two post-serve drains run *concurrently* (see `join!` below), so they
549 + /// share the pool-drain window rather than summing to twice it.
550 + /// Invariant: HARD_EXIT >= REQUEST_DRAIN + POOL_DRAIN.
551 + const SHUTDOWN_REQUEST_DRAIN_SECS: u64 = 10;
552 + const SHUTDOWN_POOL_DRAIN_SECS: u64 = 5;
553 + const SHUTDOWN_HARD_EXIT_SECS: u64 = SHUTDOWN_REQUEST_DRAIN_SECS + SHUTDOWN_POOL_DRAIN_SECS;
554 + const _: () =
555 + assert!(SHUTDOWN_HARD_EXIT_SECS >= SHUTDOWN_REQUEST_DRAIN_SECS + SHUTDOWN_POOL_DRAIN_SECS);
556 +
538 557 /// Wait for SIGINT (Ctrl-C) or SIGTERM, then return to trigger graceful shutdown.
539 - /// In-flight requests are allowed up to 10 seconds to complete. After that,
540 - /// the process force-exits to prevent hung connections from blocking restarts.
558 + /// In-flight requests get up to `SHUTDOWN_REQUEST_DRAIN_SECS` to complete, then
559 + /// the post-serve drains get `SHUTDOWN_POOL_DRAIN_SECS`; the hard-exit timer
560 + /// (`SHUTDOWN_HARD_EXIT_SECS`) is the backstop that frees a hung restart.
541 561 async fn shutdown_signal() {
542 562 use tokio::signal;
543 563
@@ -563,10 +583,12 @@ async fn shutdown_signal() {
563 583 () = terminate => tracing::info!("Received SIGTERM, starting graceful shutdown"),
564 584 }
565 585
566 - // Spawn a hard deadline: if graceful drain takes longer than 10s, force exit
586 + // Spawn the hard-exit backstop: if the full graceful drain (request drain +
587 + // concurrent pool drain) overruns its reconciled budget, force exit so a hung
588 + // connection can't block a deployment restart indefinitely.
567 589 tokio::spawn(async {
568 - tokio::time::sleep(Duration::from_secs(10)).await;
569 - eprintln!("Graceful shutdown timed out after 10s, forcing exit");
590 + tokio::time::sleep(Duration::from_secs(SHUTDOWN_HARD_EXIT_SECS)).await;
591 + eprintln!("Graceful shutdown timed out after {SHUTDOWN_HARD_EXIT_SECS}s, forcing exit");
570 592 std::process::exit(1);
571 593 });
572 594 }