Skip to main content

max / makenotwork

Perf: shared HTTP client; bounded-parallel admin bulk actions - Add a shared helpers::HTTP_CLIENT (LazyLock, 10s default timeout) and point the per-request reqwest::Client::new() sites at it (SSO token exchange, MT summary on landing/dashboard/integrations, pwned-password range, CLI features, domain DNS-over-HTTPS). reqwest pools connections, so building one per request discarded the pool and re-ran TLS each call. Per-request .timeout() (e.g. the 3s dashboard MT check) still overrides. - admin bulk rescan/promote: process the held-review backlog with a bounded JoinSet (8 in-flight) instead of one sequential round-trip per row, so clearing a large queue isn't serial. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-15 23:43 UTC
Commit: 4d2702e08601fbd8ed49912cce5c7493b0254a51
Parent: 4e3b799
9 files changed, +95 insertions, -27 deletions
M server/src/auth.rs +1 -1
@@ -572,7 +572,7 @@ pub async fn check_password_breach(password: &str) -> Option<u64> {
572 572 let (prefix, suffix) = hash.split_at(5);
573 573
574 574 let url = format!("https://api.pwnedpasswords.com/range/{}", prefix);
575 - let response = match reqwest::Client::new()
575 + let response = match crate::helpers::HTTP_CLIENT
576 576 .get(&url)
577 577 .header("User-Agent", "MakeNotWork-Security-Check")
578 578 .header("Add-Padding", "true")
M server/src/helpers.rs +13
@@ -23,6 +23,19 @@ pub use crate::rate_limit::{
23 23 CloudflareIpKeyExtractor, SyncAppKeyExtractor,
24 24 };
25 25
26 + /// Shared outbound HTTP client. `reqwest::Client` is internally `Arc`-wrapped
27 + /// and pools connections, so it is meant to be built once and reused; a
28 + /// per-request `reqwest::Client::new()` throws away the connection pool and
29 + /// re-runs TLS setup every call. Carries a 10s default timeout as a backstop —
30 + /// call sites may still set a tighter per-request `.timeout()`, which wins.
31 + /// (Clients that need bespoke config — Postmark, PoM probe — keep their own.)
32 + pub static HTTP_CLIENT: std::sync::LazyLock<reqwest::Client> = std::sync::LazyLock::new(|| {
33 + reqwest::Client::builder()
34 + .timeout(std::time::Duration::from_secs(10))
35 + .build()
36 + .expect("build shared reqwest client")
37 + });
38 +
26 39 /// Escape HTML special characters, including `'` so the output is safe in
27 40 /// single-quoted attribute contexts as well as element text. Use whenever a
28 41 /// value is interpolated into a raw `format!`-built HTML string instead of an
M server/src/routes/admin/uploads.rs +75 -20
@@ -415,18 +415,40 @@ pub(super) async fn admin_bulk_rescan_held(
415 415 State(state): State<AppState>,
416 416 AdminUser(admin): AdminUser,
417 417 ) -> Result<Response> {
418 + // Clear the backlog with bounded concurrency instead of one sequential
419 + // round-trip per candidate (Run #21 Performance MODERATE). MAX_INFLIGHT
420 + // caps how many connections this admin action borrows from the pool at once.
421 + const MAX_INFLIGHT: usize = 8;
422 + let mut set: tokio::task::JoinSet<bool> = tokio::task::JoinSet::new();
418 423 let mut total = 0usize;
419 424
420 425 for cand in db::scan_jobs::rescan_candidates_versions(&state.db).await? {
421 - let id = VersionId::from_uuid(cand.version_id);
422 - if rescan_version_inner(&state, id, admin.id, AdminAction::BulkRescan).await.is_ok() {
423 - total += 1;
426 + if set.len() >= MAX_INFLIGHT && let Some(Ok(ok)) = set.join_next().await {
427 + total += ok as usize;
424 428 }
429 + let state = state.clone();
430 + let admin_id = admin.id;
431 + set.spawn(async move {
432 + rescan_version_inner(&state, VersionId::from_uuid(cand.version_id), admin_id, AdminAction::BulkRescan)
433 + .await
434 + .is_ok()
435 + });
425 436 }
426 437 for cand in db::scan_jobs::rescan_candidates_items(&state.db).await? {
427 - let id = ItemId::from_uuid(cand.item_id);
428 - if rescan_item_inner(&state, id, admin.id, AdminAction::BulkRescan).await.is_ok() {
429 - total += 1;
438 + if set.len() >= MAX_INFLIGHT && let Some(Ok(ok)) = set.join_next().await {
439 + total += ok as usize;
440 + }
441 + let state = state.clone();
442 + let admin_id = admin.id;
443 + set.spawn(async move {
444 + rescan_item_inner(&state, ItemId::from_uuid(cand.item_id), admin_id, AdminAction::BulkRescan)
445 + .await
446 + .is_ok()
447 + });
448 + }
449 + while let Some(res) = set.join_next().await {
450 + if let Ok(ok) = res {
451 + total += ok as usize;
430 452 }
431 453 }
432 454 tracing::info!(total, admin_id = %admin.id, "bulk rescan of held queue dispatched");
@@ -456,24 +478,57 @@ pub(super) async fn admin_bulk_promote_held(
456 478
457 479 let held_items = db::scanning::get_held_items(&state.db).await?;
458 480 let held_versions = db::scanning::get_held_versions(&state.db).await?;
481 +
482 + // Bounded-concurrency promote (Run #21 Performance MODERATE). The note is
483 + // shared across tasks via Arc; MAX_INFLIGHT caps borrowed connections.
484 + const MAX_INFLIGHT: usize = 8;
485 + let note: std::sync::Arc<str> = note.into();
486 + let mut set: tokio::task::JoinSet<bool> = tokio::task::JoinSet::new();
459 487 let mut total = 0usize;
460 488
461 - for v in &held_versions {
462 - if db::scanning::update_version_scan_status(&state.db, v.version_id, FileScanStatus::Clean).await.is_ok() {
463 - db::scan_admin_actions::log_version(
464 - &state.db, v.version_id, admin.id, AdminAction::BulkPromote,
465 - Some("held_for_review"), Some("clean"), Some(note),
466 - ).await.ok();
467 - total += 1;
489 + for v in held_versions {
490 + if set.len() >= MAX_INFLIGHT && let Some(Ok(ok)) = set.join_next().await {
491 + total += ok as usize;
468 492 }
493 + let state = state.clone();
494 + let note = note.clone();
495 + let admin_id = admin.id;
496 + let vid = v.version_id;
497 + set.spawn(async move {
498 + if db::scanning::update_version_scan_status(&state.db, vid, FileScanStatus::Clean).await.is_ok() {
499 + db::scan_admin_actions::log_version(
500 + &state.db, vid, admin_id, AdminAction::BulkPromote,
501 + Some("held_for_review"), Some("clean"), Some(&note),
502 + ).await.ok();
503 + true
504 + } else {
505 + false
506 + }
507 + });
508 + }
509 + for i in held_items {
510 + if set.len() >= MAX_INFLIGHT && let Some(Ok(ok)) = set.join_next().await {
511 + total += ok as usize;
512 + }
513 + let state = state.clone();
514 + let note = note.clone();
515 + let admin_id = admin.id;
516 + let iid = i.item_id;
517 + set.spawn(async move {
518 + if db::scanning::update_item_scan_status(&state.db, iid, FileScanStatus::Clean).await.is_ok() {
519 + db::scan_admin_actions::log_item(
520 + &state.db, iid, admin_id, AdminAction::BulkPromote,
521 + Some("held_for_review"), Some("clean"), Some(&note),
522 + ).await.ok();
523 + true
524 + } else {
525 + false
526 + }
527 + });
469 528 }
470 - for i in &held_items {
471 - if db::scanning::update_item_scan_status(&state.db, i.item_id, FileScanStatus::Clean).await.is_ok() {
472 - db::scan_admin_actions::log_item(
473 - &state.db, i.item_id, admin.id, AdminAction::BulkPromote,
474 - Some("held_for_review"), Some("clean"), Some(note),
475 - ).await.ok();
476 - total += 1;
529 + while let Some(res) = set.join_next().await {
530 + if let Ok(ok) = res {
531 + total += ok as usize;
477 532 }
478 533 }
479 534 tracing::warn!(total, admin_id = %admin.id, note = %note, "bulk promote of held queue executed");
M server/src/routes/api/domains.rs +1 -1
@@ -322,7 +322,7 @@ fn generate_verification_token() -> String {
322 322
323 323 /// Query TXT records via Cloudflare DNS-over-HTTPS.
324 324 async fn dns_lookup_txt(name: &str) -> Result<Vec<String>> {
325 - let client = reqwest::Client::new();
325 + let client = &*crate::helpers::HTTP_CLIENT;
326 326 let resp = client
327 327 .get("https://cloudflare-dns.com/dns-query")
328 328 .query(&[("name", name), ("type", "TXT")])
M server/src/routes/api/internal/cli_features.rs +1 -1
@@ -437,7 +437,7 @@ pub(super) async fn verify_domain(
437 437 "https://cloudflare-dns.com/dns-query?name={}&type=TXT",
438 438 lookup_name
439 439 );
440 - let resp = reqwest::Client::new()
440 + let resp = crate::helpers::HTTP_CLIENT
441 441 .get(&url)
442 442 .header("accept", "application/dns-json")
443 443 .timeout(std::time::Duration::from_secs(5))
M server/src/routes/pages/dashboard/main.rs +1 -1
@@ -115,7 +115,7 @@ pub(super) async fn dashboard(
115 115 // Check if user has MT forum memberships (for Forums tab visibility)
116 116 let has_mt_memberships = if let Some(ref mt_url) = state.config.mt_base_url {
117 117 let url = format!("{}/api/user/{}/summary", mt_url, session_user.id);
118 - match reqwest::Client::new()
118 + match crate::helpers::HTTP_CLIENT
119 119 .get(&url)
120 120 .timeout(std::time::Duration::from_secs(3))
121 121 .send()
M server/src/routes/pages/dashboard/tabs/user/integrations.rs +1 -1
@@ -27,7 +27,7 @@ pub(in crate::routes::pages::dashboard) async fn dashboard_tab_forums(
27 27
28 28 let url = format!("{}/api/user/{}/summary", mt_base_url, session_user.id);
29 29
30 - let resp = reqwest::Client::new()
30 + let resp = crate::helpers::HTTP_CLIENT
31 31 .get(&url)
32 32 .timeout(std::time::Duration::from_secs(5))
33 33 .send()
M server/src/routes/pages/public/landing.rs +1 -1
@@ -333,7 +333,7 @@ pub(super) async fn library_tab_communities(
333 333
334 334 let url = format!("{}/api/user/{}/summary", mt_base_url, user.id);
335 335
336 - let resp = reqwest::Client::new()
336 + let resp = crate::helpers::HTTP_CLIENT
337 337 .get(&url)
338 338 .timeout(std::time::Duration::from_secs(5))
339 339 .send()
M server/src/routes/sso.rs +1 -1
@@ -126,7 +126,7 @@ async fn sso_callback(
126 126
127 127 // Exchange the code at the provider's token endpoint.
128 128 let redirect_uri = format!("{}/sso/callback", state.config.host_url);
129 - let resp = reqwest::Client::new()
129 + let resp = crate::helpers::HTTP_CLIENT
130 130 .post(format!("{}/oauth/token", sso.provider_url))
131 131 .timeout(std::time::Duration::from_secs(10))
132 132 .form(&[