max / makenotwork
1 file changed,
+40 insertions,
-8 deletions
| @@ -498,13 +498,39 @@ async fn main() { | |||
| 498 | 498 | // (SHUTDOWN_HARD_EXIT_SECS, armed at signal receipt) is the backstop that | |
| 499 | 499 | // frees a hung restart. The budget is reconciled so the pool drain fits | |
| 500 | 500 | // before the hard exit (see the shutdown constants below). | |
| 501 | - | axum::serve( | |
| 501 | + | // Bound the in-flight request drain. `with_graceful_shutdown` alone waits for | |
| 502 | + | // in-flight requests with NO deadline, so a single hung request stalls | |
| 503 | + | // shutdown until the hard-exit kills the process — taking the post-serve pool | |
| 504 | + | // drain with it (audit Run 24 A4). Fire a one-shot the instant the signal | |
| 505 | + | // arrives, then give the drain at most SHUTDOWN_REQUEST_DRAIN_SECS from that | |
| 506 | + | // point before abandoning it and proceeding to the pool drain. (The timer must | |
| 507 | + | // start at signal time, not server-start time, hence the one-shot.) | |
| 508 | + | let (signal_tx, signal_rx) = tokio::sync::oneshot::channel::<()>(); | |
| 509 | + | let signal_tx = std::sync::Mutex::new(Some(signal_tx)); | |
| 510 | + | let serve = axum::serve( | |
| 502 | 511 | listener, | |
| 503 | 512 | app.into_make_service_with_connect_info::<std::net::SocketAddr>(), | |
| 504 | 513 | ) | |
| 505 | - | .with_graceful_shutdown(shutdown_signal()) | |
| 506 | - | .await | |
| 507 | - | .expect("Server error"); | |
| 514 | + | .with_graceful_shutdown(async move { | |
| 515 | + | shutdown_signal().await; | |
| 516 | + | if let Some(tx) = signal_tx.lock().unwrap().take() { | |
| 517 | + | let _ = tx.send(()); | |
| 518 | + | } | |
| 519 | + | }) | |
| 520 | + | .into_future(); | |
| 521 | + | tokio::pin!(serve); | |
| 522 | + | ||
| 523 | + | let drain_deadline = async { | |
| 524 | + | let _ = signal_rx.await; // resolves only once the signal fires | |
| 525 | + | tokio::time::sleep(Duration::from_secs(SHUTDOWN_REQUEST_DRAIN_SECS)).await; | |
| 526 | + | }; | |
| 527 | + | tokio::select! { | |
| 528 | + | r = &mut serve => r.expect("Server error"), | |
| 529 | + | () = drain_deadline => tracing::warn!( | |
| 530 | + | "in-flight request drain exceeded {SHUTDOWN_REQUEST_DRAIN_SECS}s; \ | |
| 531 | + | abandoning it and proceeding to the pool drain" | |
| 532 | + | ), | |
| 533 | + | } | |
| 508 | 534 | ||
| 509 | 535 | drop(shutdown_tx); // signal monitor, scheduler, and scan workers to stop | |
| 510 | 536 | ||
| @@ -547,12 +573,18 @@ async fn main() { | |||
| 547 | 573 | /// post-serve drain window — otherwise the later drain is cut off mid-flight. | |
| 548 | 574 | /// The two post-serve drains run *concurrently* (see `join!` below), so they | |
| 549 | 575 | /// share the pool-drain window rather than summing to twice it. | |
| 550 | - | /// Invariant: HARD_EXIT >= REQUEST_DRAIN + POOL_DRAIN. | |
| 551 | 576 | const SHUTDOWN_REQUEST_DRAIN_SECS: u64 = 10; | |
| 552 | 577 | const SHUTDOWN_POOL_DRAIN_SECS: u64 = 5; | |
| 553 | - | const SHUTDOWN_HARD_EXIT_SECS: u64 = SHUTDOWN_REQUEST_DRAIN_SECS + SHUTDOWN_POOL_DRAIN_SECS; | |
| 554 | - | const _: () = | |
| 555 | - | assert!(SHUTDOWN_HARD_EXIT_SECS >= SHUTDOWN_REQUEST_DRAIN_SECS + SHUTDOWN_POOL_DRAIN_SECS); | |
| 578 | + | /// Hard-exit backstop, defined as its own ceiling (not the sum of the two | |
| 579 | + | /// drains) with margin above them — the process is force-killed only if BOTH | |
| 580 | + | /// bounded drains overrun their budgets. The margin also makes the invariant | |
| 581 | + | /// below a real check rather than the tautology it was when this equalled the | |
| 582 | + | /// sum (audit Run 24 A4). | |
| 583 | + | const SHUTDOWN_HARD_EXIT_SECS: u64 = 20; | |
| 584 | + | const _: () = assert!( | |
| 585 | + | SHUTDOWN_HARD_EXIT_SECS > SHUTDOWN_REQUEST_DRAIN_SECS + SHUTDOWN_POOL_DRAIN_SECS, | |
| 586 | + | "hard-exit must outlast the bounded request + pool drains, with margin" | |
| 587 | + | ); | |
| 556 | 588 | ||
| 557 | 589 | /// Wait for SIGINT (Ctrl-C) or SIGTERM, then return to trigger graceful shutdown. | |
| 558 | 590 | /// In-flight requests get up to `SHUTDOWN_REQUEST_DRAIN_SECS` to complete, then |