Skip to main content

max / makenotwork

35.8 KB · 833 lines History Blame Raw
1 //! Application entry point, tracing, config, pool, then hands off to `build_app`.
2
3 use axum::http::Request;
4 use sqlx::ConnectOptions;
5 use sqlx::postgres::PgPoolOptions;
6 use std::time::Duration;
7 use tower_http::request_id::{MakeRequestUuid, PropagateRequestIdLayer, SetRequestIdLayer};
8 use tower_http::trace::TraceLayer;
9 use tower_sessions::cookie::SameSite;
10 use tower_sessions::cookie::time::Duration as CookieDuration;
11 use tower_sessions::{Expiry, SessionManagerLayer};
12 use tower_sessions_sqlx_store::PostgresStore;
13 use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt};
14
15 use makenotwork::config::Config;
16 use makenotwork::constants;
17 use makenotwork::email::{EmailClient, EmailConfig};
18 use makenotwork::payments::StripeClient;
19 use makenotwork::scanning::ScanPipeline;
20 use makenotwork::storage::S3Client;
21 use makenotwork::{AppState, AppStorage, build_app};
22 use webauthn_rs::WebauthnBuilder;
23
24 #[tokio::main]
25 async fn main() {
26 dotenvy::dotenv().ok();
27
28 // Before any TLS client is built; see the function's own docs for why this
29 // cannot be left to rustls to work out.
30 makenotwork::crypto::install_default_crypto_provider();
31
32 // JSON in release, human-readable in dev
33 tracing_subscriber::registry()
34 .with(
35 tracing_subscriber::EnvFilter::try_from_default_env()
36 .unwrap_or_else(|_| makenotwork::DEFAULT_LOG_FILTER.into()),
37 )
38 .with(if cfg!(debug_assertions) {
39 tracing_subscriber::fmt::layer().boxed()
40 } else {
41 tracing_subscriber::fmt::layer().json().boxed()
42 })
43 .init();
44
45 // Sando boot-smoke gate spawns the binary with SANDO_BOOT_SMOKE=1 to
46 // verify it loads + links + survives. The real init loads
47 // assumptions.toml and other runtime files that don't exist in the
48 // sando build workspace. Short-circuit to a tiny axum server that
49 // proves the binary, tokio runtime, axum, and TCP bind all work,
50 // then idles until the gate kills it.
51 if std::env::var("SANDO_BOOT_SMOKE").is_ok() {
52 // Sando passes SANDO_BOOT_SMOKE_PORT so the gate can actually probe
53 // GET /health (readiness), not just check the process stays up. A bad
54 // value or unset falls back to an ephemeral port (liveness-only, the
55 // historical behavior) rather than failing the smoke for a config typo.
56 let port = std::env::var("SANDO_BOOT_SMOKE_PORT")
57 .ok()
58 .and_then(|p| p.parse::<u16>().ok())
59 .unwrap_or(0);
60 tracing::info!(port, "SANDO_BOOT_SMOKE=1; running minimal smoke server");
61 let app = axum::Router::new().route("/health", axum::routing::get(|| async { "ok" }));
62 let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, port))
63 .await
64 .expect("smoke: bind 127.0.0.1");
65 axum::serve(listener, app).await.expect("smoke: serve");
66 return;
67 }
68
69 // Config-only validation mode. Loads Config::from_env() with the process
70 // environment and exits, no DB connection, no migrations, no socket bind,
71 // so a required var missing on a target (e.g. CDN_BASE_URL) is caught
72 // *before* a deploy swaps the `current` symlink and restarts into a crash
73 // loop. Sando's pre-swap config-drift guard runs `MNW_CHECK_CONFIG=1
74 // <binary>` on the target with its env sourced; also handy by hand. Prints a
75 // stable sentinel line so a caller can distinguish a genuine config error
76 // (exit 1) from a binary too old to support this mode (which would ignore
77 // the var and fall through to normal startup).
78 if std::env::var("MNW_CHECK_CONFIG").is_ok() {
79 match Config::from_env() {
80 Ok(_) => {
81 println!("MNW_CONFIG_CHECK: ok");
82 std::process::exit(0);
83 }
84 Err(e) => {
85 eprintln!("MNW_CONFIG_CHECK: error: {e}");
86 std::process::exit(1);
87 }
88 }
89 }
90
91 // Docs integrity check mode. Loads the doc corpus with the exact production
92 // config and reports broken internal links, then exits: no DB, no socket,
93 // so it is cheap. Sando's `code_smoke` gate runs `MNW_CHECK_DOCS=1 <binary>`
94 // as its first step, before the throwaway-DB boot, so a rotted internal link
95 // fails the pipeline early rather than serving a live 404 in prod. Broken
96 // links fail the check; slug collisions are reported but do not fail it
97 // (pre-existing collisions shouldn't red the pipeline). Prints a stable
98 // sentinel line, like MNW_CHECK_CONFIG.
99 if std::env::var("MNW_CHECK_DOCS").is_ok() {
100 let assumptions = match makenotwork::site_docs::load_assumptions() {
101 Ok(a) => a,
102 Err(e) => {
103 eprintln!("MNW_CHECK_DOCS: error: {e}");
104 std::process::exit(1);
105 }
106 };
107 let docs = makenotwork::site_docs::build_doc_loader(assumptions);
108 let broken = docs.broken_links();
109 for b in broken {
110 eprintln!(
111 " broken link: {} -> {} (no page serves this slug)",
112 b.source_slug, b.target_slug
113 );
114 }
115 for c in docs.collisions() {
116 eprintln!(
117 " slug collision: {} ({} displaced by {})",
118 c.slug, c.displaced_section, c.winning_section
119 );
120 }
121 if broken.is_empty() {
122 println!(
123 "MNW_CHECK_DOCS: ok ({} collision(s) reported)",
124 docs.collisions().len()
125 );
126 std::process::exit(0);
127 }
128 println!("MNW_CHECK_DOCS: {} broken link(s)", broken.len());
129 std::process::exit(1);
130 }
131
132 let config = Config::from_env().expect("Failed to load configuration");
133 tracing::info!("Configuration loaded");
134
135 // Create database connection pool with health checks and lifecycle limits.
136 // - test_before_acquire: validates connections before use (catches stale/broken conns)
137 // - max_lifetime: rotates connections to prevent long-lived session issues
138 // - idle_timeout: prunes idle connections to free server resources
139 // - min_connections: keeps warm connections ready for immediate use
140 // - log_slow_statements: logs queries exceeding 100ms at WARN level
141 let connect_options: sqlx::postgres::PgConnectOptions =
142 config.database_url.parse().expect("Invalid DATABASE_URL");
143 let connect_options = connect_options
144 .log_statements(log::LevelFilter::Trace)
145 .log_slow_statements(log::LevelFilter::Warn, Duration::from_millis(100));
146
147 // Bound every query on every pooled connection: `statement_timeout` caps
148 // run time so a wedged query can't pin its connection forever (25 of those
149 // would exhaust the pool with no recovery, acquire_timeout only bounds
150 // getting a connection, not running one), and `lock_timeout` fails a
151 // lock-contended query fast instead of blocking indefinitely.
152 let statement_timeout_ms = constants::DB_STATEMENT_TIMEOUT_SECS * 1000;
153 let lock_timeout_ms = constants::DB_LOCK_TIMEOUT_SECS * 1000;
154 let db = PgPoolOptions::new()
155 .max_connections(constants::DB_POOL_MAX_CONNECTIONS)
156 .min_connections(constants::DB_POOL_MIN_CONNECTIONS)
157 .acquire_timeout(Duration::from_secs(constants::DB_ACQUIRE_TIMEOUT_SECS))
158 .max_lifetime(Duration::from_secs(constants::DB_MAX_LIFETIME_SECS))
159 .idle_timeout(Duration::from_secs(constants::DB_IDLE_TIMEOUT_SECS))
160 .test_before_acquire(true)
161 .after_connect(move |conn, _meta| {
162 Box::pin(async move {
163 use sqlx::Executor;
164 conn.execute(
165 format!(
166 "SET statement_timeout = {statement_timeout_ms}; SET lock_timeout = {lock_timeout_ms}"
167 )
168 .as_str(),
169 )
170 .await?;
171 Ok(())
172 })
173 })
174 .connect_with(connect_options)
175 .await
176 .expect("Failed to connect to database");
177
178 tracing::info!("Database connected");
179
180 // Run migrations, exit cleanly (code 2) on failure instead of panicking.
181 // This prevents systemd from crash-looping on migration errors (e.g. version
182 // conflicts after a bad deploy). WAM ticket alerts the operator.
183 //
184 // `sqlx::migrate!` embeds every migration as one array literal, so the lint
185 // measures the whole `migrations/` directory. It crossed 16KB when the
186 // mailing-list tables landed, which is why this is new and why it is not a
187 // finding: the array is built once at startup and the alternative is
188 // runtime migration discovery, which is strictly worse for a service that
189 // must not start against the wrong schema.
190 #[allow(
191 clippy::large_stack_arrays,
192 reason = "sqlx::migrate! embeds the whole directory"
193 )]
194 if let Err(e) = sqlx::migrate!("./migrations").run(&db).await {
195 tracing::error!(error = %e, "Migration failed, exiting without restart");
196
197 // Best-effort WAM alert (DB is up, WAM may be reachable)
198 if let Ok(wam_url) = std::env::var("WAM_URL") {
199 let body = format!("Migration error on startup:\n{e}");
200 let mut req = reqwest::Client::new()
201 .post(format!("{wam_url}/tickets"))
202 .json(&serde_json::json!({
203 "title": "Migration failure, server not starting",
204 "body": body,
205 "priority": "critical",
206 "source": "migration-failure",
207 }))
208 .timeout(std::time::Duration::from_secs(5));
209 if let Ok(token) = std::env::var("WAM_TOKEN")
210 && !token.trim().is_empty()
211 {
212 req = req.bearer_auth(token.trim());
213 }
214 let _ = req.send().await;
215 }
216
217 // Exit code 2, systemd configured not to restart on this code
218 std::process::exit(2);
219 }
220
221 tracing::info!("Migrations complete");
222
223 // Example-marketplace seed for staging (testnot.work). Runs against the
224 // already-migrated DB, then exits, never during normal boot. Layered guards
225 // (opt-in env, host allowlist, no-real-users) make it refuse against prod.
226 // See `makenotwork::seed` and `_private/docs/mnw/testnot-example-seed.md`.
227 if std::env::args().any(|a| a == "--seed-examples") {
228 let opts = makenotwork::seed::SeedOptions::from_env(&config.host_url);
229
230 // Build the storage handles the media phase needs (main + public/CDN
231 // buckets). Both are `None` when S3 is unconfigured, the media phase then
232 // no-ops and items stay hidden. Done here, inside the one-shot seed block,
233 // so normal boot (which builds these later) is untouched.
234 async fn seed_client(
235 cfg: Option<&makenotwork::config::StorageConfig>,
236 host_url: &str,
237 ) -> Option<std::sync::Arc<dyn makenotwork::storage::StorageBackend>> {
238 let cfg = cfg?;
239 match S3Client::new(cfg, host_url).await {
240 Ok(client) => Some(std::sync::Arc::new(client) as _),
241 Err(e) => {
242 tracing::warn!(error = ?e, "seed: failed to init S3 storage; media skipped");
243 None
244 }
245 }
246 }
247 let media = makenotwork::seed::SeedMedia {
248 s3: seed_client(config.storage.as_ref(), &config.host_url).await,
249 public_s3: seed_client(config.public_storage.as_ref(), &config.host_url).await,
250 cdn_base_url: Some(config.cdn_base_url.clone()),
251 assets: makenotwork::seed::manifest::ResolvedAssets::default(),
252 };
253
254 // Fetch the curated public-domain media before touching the database, so
255 // a manifest that cannot be resolved leaves the existing catalog alone.
256 let media = match media.with_manifest().await {
257 Ok(media) => media,
258 Err(e) => {
259 tracing::error!(error = %e, "example seed: media manifest could not be resolved");
260 std::process::exit(1);
261 }
262 };
263
264 match makenotwork::seed::run(&db, &opts, &media).await {
265 Ok(()) => {
266 tracing::info!("example seed complete");
267 std::process::exit(0);
268 }
269 Err(e) => {
270 tracing::error!(error = %e, "example seed refused or failed");
271 std::process::exit(1);
272 }
273 }
274 }
275
276 // Create PostgreSQL-backed session store (persists across restarts)
277 let session_store = PostgresStore::new(db.clone());
278 session_store
279 .migrate()
280 .await
281 .expect("Failed to migrate session store");
282
283 // Continuously delete expired rows from the tower-sessions table. A session
284 // row is minted on every anonymous page render, so without this the table
285 // grows without bound under bot/crawler traffic and is read on every
286 // request. (The app's own `user_sessions` table is pruned by the daily
287 // scheduler; this covers the tower-sessions store, which the scheduler does
288 // not own.) Runs hourly on a background task.
289 {
290 use tower_sessions::ExpiredDeletion;
291 let deletion_store = session_store.clone();
292 tokio::task::spawn(async move {
293 if let Err(e) = deletion_store
294 .continuously_delete_expired(tokio::time::Duration::from_hours(1))
295 .await
296 {
297 tracing::error!(error = ?e, "tower-sessions expired-deletion task exited");
298 }
299 });
300 }
301
302 // In release mode, require HTTPS for session cookies (override with INSECURE_COOKIES=1 for staging)
303 let secure_cookies =
304 !cfg!(debug_assertions) && std::env::var("INSECURE_COOKIES").unwrap_or_default() != "1";
305 if secure_cookies {
306 tracing::info!("Session cookies configured for HTTPS (secure=true)");
307 } else if !cfg!(debug_assertions) {
308 tracing::warn!("Session cookies set to insecure (INSECURE_COOKIES=1)");
309 }
310
311 let session_layer = SessionManagerLayer::new(session_store)
312 .with_secure(secure_cookies)
313 .with_http_only(true)
314 .with_same_site(SameSite::Lax) // Lax allows session on top-level navigations (OAuth redirects)
315 .with_expiry(Expiry::OnInactivity(CookieDuration::days(
316 constants::SESSION_EXPIRY_DAYS,
317 )));
318
319 tracing::info!("PostgreSQL session store initialized");
320
321 // Initialize S3 client if storage is configured
322 let s3: Option<std::sync::Arc<dyn makenotwork::storage::StorageBackend>> = if let Some(
323 ref storage_config,
324 ) = config.storage
325 {
326 match S3Client::new(storage_config, &config.host_url).await {
327 Ok(client) => {
328 tracing::info!(bucket = %storage_config.bucket, "S3 storage initialized");
329 Some(std::sync::Arc::new(client))
330 }
331 Err(e) => {
332 tracing::warn!(error = ?e, "failed to initialize S3 storage, file uploads unavailable");
333 None
334 }
335 }
336 } else {
337 tracing::info!("S3 storage not configured. File uploads will be unavailable.");
338 None
339 };
340
341 // Initialize SyncKit blob S3 client if configured (separate bucket)
342 let synckit_s3: Option<std::sync::Arc<dyn makenotwork::storage::StorageBackend>> = if let Some(
343 ref synckit_storage_config,
344 ) =
345 config.synckit_storage
346 {
347 match S3Client::new(synckit_storage_config, &config.host_url).await {
348 Ok(client) => {
349 tracing::info!(bucket = %synckit_storage_config.bucket, "SyncKit blob storage initialized");
350 Some(std::sync::Arc::new(client))
351 }
352 Err(e) => {
353 tracing::warn!(error = ?e, "Failed to initialize SyncKit blob storage");
354 None
355 }
356 }
357 } else {
358 tracing::info!("SyncKit blob storage not configured");
359 None
360 };
361
362 // Initialize public (CDN-served) bucket client if configured. Holds only
363 // promoted image content; the scan worker copies Clean covers/gallery here.
364 let public_s3: Option<std::sync::Arc<dyn makenotwork::storage::StorageBackend>> = if let Some(
365 ref public_storage_config,
366 ) =
367 config.public_storage
368 {
369 match S3Client::new(public_storage_config, &config.host_url).await {
370 Ok(client) => {
371 tracing::info!(bucket = %public_storage_config.bucket, "Public S3 bucket initialized");
372 Some(std::sync::Arc::new(client))
373 }
374 Err(e) => {
375 tracing::warn!(error = ?e, "Failed to initialize public S3 bucket");
376 None
377 }
378 }
379 } else {
380 tracing::info!("Public S3 bucket not configured");
381 None
382 };
383
384 // Initialize Stripe client if configured
385 let stripe: Option<std::sync::Arc<dyn makenotwork::payments::PaymentProvider>> = if let Some(
386 ref stripe_config,
387 ) =
388 config.stripe
389 {
390 match StripeClient::new(stripe_config) {
391 Ok(client) => {
392 tracing::info!("Stripe payments initialized");
393 Some(std::sync::Arc::new(client)
394 as std::sync::Arc<dyn makenotwork::payments::PaymentProvider>)
395 }
396 Err(e) => {
397 // Invalid client config is a boot invariant violation, exit
398 // without restart rather than run with payments silently broken.
399 tracing::error!(error = %e, "Failed to build Stripe client, exiting without restart");
400 std::process::exit(2);
401 }
402 }
403 } else {
404 tracing::info!("Stripe not configured. Payments will be unavailable.");
405 None
406 };
407
408 // Initialize email client (logs in dev mode when POSTMARK_TOKEN is not set)
409 let email = EmailClient::new(EmailConfig::from_env(), Some(db.clone()));
410
411 // Load business assumptions (single source of truth for figures in the docs).
412 // Validation failure aborts startup, we don't want to serve stale numbers.
413 let assumptions = makenotwork::site_docs::load_assumptions().unwrap_or_else(|e| panic!("{e}"));
414
415 // Load documentation pages from disk. The same builder backs the DB-free
416 // MNW_CHECK_DOCS integrity check above, so what boots and what is checked
417 // are the same corpus under the same config.
418 let docs = std::sync::Arc::new(makenotwork::site_docs::build_doc_loader(
419 assumptions.clone(),
420 ));
421
422 // Initialize file scanning pipeline (optional). If scanning is configured,
423 // assert at least one AV layer is actually live, otherwise the FailOpen
424 // policy on ClamAV silently passes everything as Clean and the deploy
425 // looks healthy while shipping zero real coverage. Refuse to boot.
426 let scanner = if let Some(ref scan_config) = config.scan {
427 match ScanPipeline::new(scan_config) {
428 Ok(s) => match s.assert_live().await {
429 Ok(()) => {
430 tracing::info!("File scanning enabled");
431 Some(std::sync::Arc::new(s))
432 }
433 Err(e) => panic!("Scanning configured but no live AV layer: {e}"),
434 },
435 Err(e) => {
436 tracing::warn!(error = %e, "File scanning disabled");
437 None
438 }
439 }
440 } else {
441 tracing::info!("File scanning not configured");
442 None
443 };
444
445 // Initialize WebAuthn from HOST_URL (passkey / passwordless login)
446 let rp_origin = url::Url::parse(&config.host_url).expect("HOST_URL is not a valid URL");
447 let rp_id = rp_origin
448 .host_str()
449 .expect("HOST_URL has no host")
450 .to_string();
451 let webauthn = std::sync::Arc::new(
452 WebauthnBuilder::new(&rp_id, &rp_origin)
453 .expect("Failed to create WebauthnBuilder")
454 .rp_name("Makenotwork")
455 .build()
456 .expect("Failed to build Webauthn"),
457 );
458 tracing::info!("WebAuthn initialized (rp_id={rp_id})");
459
460 // Initialize syntax highlighter if git repos path is configured
461 let syntax = if config.build.git_repos_path.is_some() {
462 tracing::info!(path = ?config.build.git_repos_path, "Git source browser enabled");
463 Some(std::sync::Arc::new(
464 makenotwork::git::SyntaxHighlighter::new(),
465 ))
466 } else {
467 tracing::info!("Git source browser not configured (GIT_REPOS_PATH unset)");
468 None
469 };
470
471 // Construct MT client when both mt_base_url and internal_shared_secret are set
472 let mt_client = match (
473 &config.integrations.mt_base_url,
474 &config.integrations.internal_shared_secret,
475 ) {
476 (Some(base_url), Some(secret)) => {
477 tracing::info!(base_url = %base_url, "MT integration enabled");
478 Some(makenotwork::mt_client::MtClient::new(
479 base_url.clone(),
480 secret.clone(),
481 ))
482 }
483 _ => {
484 tracing::info!(
485 "MT integration not configured (need MT_BASE_URL + INTERNAL_SHARED_SECRET)"
486 );
487 None
488 }
489 };
490
491 // WAM ticket manager client (tailnet-only, for operational alerts). The
492 // optional WAM_TOKEN is the shared secret WAM enforces when set.
493 let wam = config.integrations.wam_url.as_ref().map(|url| {
494 tracing::info!(url = %url, "WAM integration enabled");
495 makenotwork::wam_client::WamClient::new(url.clone(), std::env::var("WAM_TOKEN").ok())
496 });
497
498 // Warm custom domain cache
499 let domain_cache = std::sync::Arc::new(dashmap::DashMap::new());
500 match makenotwork::db::custom_domains::get_all_verified_domains(&db).await {
501 Ok(domains) => {
502 for d in &domains {
503 domain_cache.insert(d.domain.clone(), d.user_id);
504 }
505 if !domains.is_empty() {
506 tracing::info!(count = domains.len(), "Custom domain cache warmed");
507 }
508 }
509 Err(e) => {
510 tracing::warn!(error = ?e, "Failed to warm custom domain cache");
511 }
512 }
513
514 // Shutdown broadcast: dropped at graceful-shutdown time to signal the
515 // background pool, monitor, scheduler, and scan workers to drain and stop.
516 let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(());
517 let page_view_tx = makenotwork::db::page_views::spawn_batcher(db.clone());
518 let (bg, bg_handle) = makenotwork::background::spawn_pool(shutdown_tx.subscribe());
519
520 // Derive pricing from the loaded assumptions. Installing the process-global
521 // TierPrices is an explicit, ordered step here (not buried in AppState
522 // construction): CreatorTier accessors (price_cents, max_file_bytes,
523 // max_storage_bytes) read the global, so it must be installed before any code
524 // path constructs a CreatorTier and calls one of them.
525 let tier_prices = makenotwork::tier_prices::TierPrices::from_assumptions(&assumptions);
526 tier_prices.clone().install_global();
527 let runway_config = makenotwork::tier_prices::RunwayConfig::from_assumptions(&assumptions);
528 let fee_calculator = makenotwork::fee_calculator::FeeCalculator::load(
529 makenotwork::site_docs::assumptions_path(),
530 );
531
532 let state = AppState::build(makenotwork::AppStateParts {
533 db,
534 config: config.clone(),
535 storage: AppStorage {
536 s3,
537 synckit_s3,
538 public_s3,
539 },
540 stripe,
541 email,
542 docs,
543 tier_prices,
544 runway_config,
545 fee_calculator,
546 scanner,
547 webauthn,
548 syntax,
549 mt_client,
550 wam,
551 domain_cache,
552 metrics_handle: Some(makenotwork::metrics::init()),
553 page_view_tx,
554 bg,
555 });
556
557 // Log active features at startup
558 tracing::info!(
559 s3 = state.storage.s3.is_some(),
560 synckit_s3 = state.storage.synckit_s3.is_some(),
561 stripe = state.stripe.is_some(),
562 scanner = state.scanner.is_some(),
563 mt = state.mt_client.is_some(),
564 wam = state.wam.is_some(),
565 git = state.config.build.git_repos_path.is_some(),
566 "Active features"
567 );
568
569 // Start background health monitor and scheduler
570 let _monitor_handle =
571 makenotwork::monitor::spawn_monitor(axum::extract::FromRef::from_ref(&state), shutdown_rx);
572 let scheduler_shutdown_rx = shutdown_tx.subscribe();
573 let scheduler_handle =
574 makenotwork::scheduler::spawn_scheduler(state.clone(), scheduler_shutdown_rx);
575
576 // Per-instance (not scheduler-gated, so every instance in a rolling deploy
577 // agrees): decides whether the footer links to /changelog at all.
578 let _changelog_handle =
579 makenotwork::changelog::spawn_refresher(state.db.clone(), shutdown_tx.subscribe());
580
581 // Start scan worker pool. Only meaningful if a scanner is configured;
582 // otherwise enqueue_scan_for never enqueues (trust-gate fast path).
583 if let (Some(scanner), Some(s3_for_workers)) = (state.scanner.clone(), state.storage.s3.clone())
584 {
585 let scan_ctx = std::sync::Arc::new(makenotwork::scanning::worker::WorkerContext {
586 db: state.db.clone(),
587 s3: s3_for_workers,
588 pipeline: scanner,
589 scan_semaphore: state.limiters.scan_semaphore.clone(),
590 wam: state.wam.clone(),
591 bg: state.bg.clone(),
592 cloudflare: makenotwork::cloudflare::CloudflarePurger::from_env(),
593 cdn_base_url: std::sync::Arc::from(state.config.cdn_base_url.as_str()),
594 synckit_s3: state.storage.synckit_s3.clone(),
595 public_s3: state.storage.public_s3.clone(),
596 config: state.config.clone(),
597 });
598 let worker_count = makenotwork::constants::SCAN_WORKER_COUNT;
599 let worker_shutdown_rx = shutdown_tx.subscribe();
600 makenotwork::scanning::worker::spawn_pool(worker_count, &scan_ctx, worker_shutdown_rx);
601 tracing::info!(worker_count, "scan worker pool started");
602
603 let report = makenotwork::scanning::spool::reap_all(std::path::Path::new(
604 makenotwork::constants::SCAN_SPOOL_DIR,
605 ));
606 if report.deleted > 0 || report.errors > 0 {
607 tracing::info!(
608 deleted = report.deleted,
609 errors = report.errors,
610 "scan spool startup reaper completed"
611 );
612 }
613 }
614
615 // Security signals need somewhere to send an alert. Before the router, so
616 // nothing the server answers is counted against an uninstalled sink.
617 makenotwork::security_signals::install(state.db.clone(), state.email.clone());
618
619 // Build router (shared with integration tests via lib.rs)
620 let app = build_app(&state, session_layer)
621 // Outside every per-route limiter, so a 429 the governor returned is
622 // seen here. Counting only; the alert, if any, is spawned off the
623 // request path.
624 .layer(axum::middleware::from_fn(
625 |req: Request<axum::body::Body>, next: axum::middleware::Next| async move {
626 // Read the header rather than calling `extract_client_ip`: that
627 // helper counts absences to warn about a missing Cloudflare
628 // proxy, and a per-response call would double every count.
629 let ip = req
630 .headers()
631 .get("cf-connecting-ip")
632 .and_then(|v| v.to_str().ok())
633 .and_then(|s| s.split(',').next())
634 .map(|s| s.trim().to_string())
635 .filter(|s| !s.is_empty());
636 let response = next.run(req).await;
637 makenotwork::security_signals::note_response(
638 response.status().as_u16(),
639 ip.as_deref(),
640 );
641 response
642 },
643 ))
644 // Request ID: propagate → trace → set (Axum applies inside-out)
645 .layer(PropagateRequestIdLayer::x_request_id())
646 .layer(
647 TraceLayer::new_for_http()
648 .make_span_with(|request: &Request<_>| {
649 let request_id = request
650 .headers()
651 .get("x-request-id")
652 .and_then(|v| v.to_str().ok())
653 .unwrap_or("-");
654
655 tracing::info_span!(
656 "request",
657 method = %request.method(),
658 uri = %request.uri(),
659 request_id = %request_id,
660 user_id = tracing::field::Empty,
661 )
662 })
663 .on_response(
664 |response: &axum::http::Response<_>,
665 latency: Duration,
666 _span: &tracing::Span| {
667 let status = response.status().as_u16();
668 if status >= 500 {
669 tracing::error!(
670 status,
671 latency_ms = latency.as_millis() as u64,
672 "response"
673 );
674 } else if status >= 400 {
675 tracing::warn!(
676 status,
677 latency_ms = latency.as_millis() as u64,
678 "response"
679 );
680 } else {
681 tracing::debug!(
682 status,
683 latency_ms = latency.as_millis() as u64,
684 "response"
685 );
686 }
687 },
688 ),
689 )
690 .layer(SetRequestIdLayer::x_request_id(MakeRequestUuid));
691
692 let addr = config.socket_addr();
693 tracing::info!(addr = %addr, "listening");
694
695 let listener = tokio::net::TcpListener::bind(addr)
696 .await
697 .expect("Failed to bind to address");
698
699 // Use into_make_service_with_connect_info to provide peer IP for rate limiting.
700 // After the shutdown signal fires, in-flight requests drain, then the
701 // scheduler + background pool drain concurrently; the hard-exit timer
702 // (SHUTDOWN_HARD_EXIT_SECS, armed at signal receipt) is the backstop that
703 // frees a hung restart. The budget is reconciled so the pool drain fits
704 // before the hard exit (see the shutdown constants below).
705 // Bound the in-flight request drain. `with_graceful_shutdown` alone waits for
706 // in-flight requests with NO deadline, so a single hung request stalls
707 // shutdown until the hard-exit kills the process, taking the post-serve pool
708 // drain with it (audit Run 24 A4). Fire a one-shot the instant the signal
709 // arrives, then give the drain at most SHUTDOWN_REQUEST_DRAIN_SECS from that
710 // point before abandoning it and proceeding to the pool drain. (The timer must
711 // start at signal time, not server-start time, hence the one-shot.)
712 let (signal_tx, signal_rx) = tokio::sync::oneshot::channel::<()>();
713 let signal_tx = std::sync::Mutex::new(Some(signal_tx));
714 let serve = axum::serve(
715 listener,
716 app.into_make_service_with_connect_info::<std::net::SocketAddr>(),
717 )
718 .with_graceful_shutdown(async move {
719 shutdown_signal().await;
720 if let Some(tx) = signal_tx.lock().unwrap().take() {
721 let _ = tx.send(());
722 }
723 })
724 .into_future();
725 tokio::pin!(serve);
726
727 let drain_deadline = async {
728 let _ = signal_rx.await; // resolves only once the signal fires
729 tokio::time::sleep(Duration::from_secs(SHUTDOWN_REQUEST_DRAIN_SECS)).await;
730 };
731 tokio::select! {
732 r = &mut serve => r.expect("Server error"),
733 () = drain_deadline => tracing::warn!(
734 "in-flight request drain exceeded {SHUTDOWN_REQUEST_DRAIN_SECS}s; \
735 abandoning it and proceeding to the pool drain"
736 ),
737 }
738
739 drop(shutdown_tx); // signal monitor, scheduler, and scan workers to stop
740
741 // Drain the scheduler's in-flight tick (platform-credit settlement,
742 // pending-S3-deletion retries, webhook re-drives) and the fire-and-forget
743 // background pool (queued emails / cache purges / MT thread creations)
744 // together. Both are engineered idempotent, so a truncated drain is
745 // recoverable, but awaiting a bounded window lets in-progress work finish
746 // cleanly. They run CONCURRENTLY under one shared budget so the pool drain
747 // isn't starved by a slow scheduler tick before the hard-exit fires: the old
748 // sequential ≤5s + ≤5s could reach 10s and let the hard-exit truncate the
749 // second drain (audit Run 23 F4). If the shared window overruns we proceed
750 // and rely on idempotency as the backstop.
751 let drains = async { tokio::join!(scheduler_handle, bg_handle) };
752 match tokio::time::timeout(Duration::from_secs(SHUTDOWN_POOL_DRAIN_SECS), drains).await {
753 Ok((scheduler_res, bg_res)) => {
754 match scheduler_res {
755 Ok(()) => tracing::info!("scheduler drained"),
756 Err(e) => tracing::warn!(error = ?e, "scheduler task ended abnormally on shutdown"),
757 }
758 match bg_res {
759 Ok(()) => tracing::info!("background pool drained"),
760 Err(e) => {
761 tracing::warn!(error = ?e, "background pool task ended abnormally on shutdown");
762 }
763 }
764 }
765 Err(_) => tracing::warn!(
766 "scheduler + background pool did not drain within {SHUTDOWN_POOL_DRAIN_SECS}s of \
767 shutdown; proceeding (work is idempotent)"
768 ),
769 }
770
771 tracing::info!("Server shut down gracefully");
772 }
773
774 /// Filesystem path to the business-assumptions TOML, `ASSUMPTIONS_PATH` or its
775 /// default. Single source for the default so every reader agrees.
776 /// Shutdown budget, reconciled so the background-pool drain can't be truncated
777 /// by the hard-exit deadline (audit Run 23 F4). The hard-exit timer arms at
778 /// signal receipt and must outlast the in-flight request drain PLUS the
779 /// post-serve drain window, otherwise the later drain is cut off mid-flight.
780 /// The two post-serve drains run *concurrently* (see `join!` below), so they
781 /// share the pool-drain window rather than summing to twice it.
782 const SHUTDOWN_REQUEST_DRAIN_SECS: u64 = 10;
783 const SHUTDOWN_POOL_DRAIN_SECS: u64 = 5;
784 /// Hard-exit backstop, defined as its own ceiling (not the sum of the two
785 /// drains) with margin above them, the process is force-killed only if BOTH
786 /// bounded drains overrun their budgets. The margin also makes the invariant
787 /// below a real check rather than the tautology it was when this equalled the
788 /// sum (audit Run 24 A4).
789 const SHUTDOWN_HARD_EXIT_SECS: u64 = 20;
790 const _: () = assert!(
791 SHUTDOWN_HARD_EXIT_SECS > SHUTDOWN_REQUEST_DRAIN_SECS + SHUTDOWN_POOL_DRAIN_SECS,
792 "hard-exit must outlast the bounded request + pool drains, with margin"
793 );
794
795 /// Wait for SIGINT (Ctrl-C) or SIGTERM, then return to trigger graceful shutdown.
796 /// In-flight requests get up to `SHUTDOWN_REQUEST_DRAIN_SECS` to complete, then
797 /// the post-serve drains get `SHUTDOWN_POOL_DRAIN_SECS`; the hard-exit timer
798 /// (`SHUTDOWN_HARD_EXIT_SECS`) is the backstop that frees a hung restart.
799 async fn shutdown_signal() {
800 use tokio::signal;
801
802 let ctrl_c = async {
803 signal::ctrl_c()
804 .await
805 .expect("Failed to install Ctrl+C handler");
806 };
807
808 #[cfg(unix)]
809 let terminate = async {
810 signal::unix::signal(signal::unix::SignalKind::terminate())
811 .expect("Failed to install SIGTERM handler")
812 .recv()
813 .await;
814 };
815
816 #[cfg(not(unix))]
817 let terminate = std::future::pending::<()>();
818
819 tokio::select! {
820 () = ctrl_c => tracing::info!("Received SIGINT, starting graceful shutdown"),
821 () = terminate => tracing::info!("Received SIGTERM, starting graceful shutdown"),
822 }
823
824 // Spawn the hard-exit backstop: if the full graceful drain (request drain +
825 // concurrent pool drain) overruns its reconciled budget, force exit so a hung
826 // connection can't block a deployment restart indefinitely.
827 tokio::spawn(async {
828 tokio::time::sleep(Duration::from_secs(SHUTDOWN_HARD_EXIT_SECS)).await;
829 eprintln!("Graceful shutdown timed out after {SHUTDOWN_HARD_EXIT_SECS}s, forcing exit");
830 std::process::exit(1);
831 });
832 }
833