max / makenotwork
- Co-Authored-By
- Claude Opus 5 (1M context) <noreply@anthropic.com>
11 files changed,
+636 insertions,
-71 deletions
| @@ -60,6 +60,28 @@ | |||
| 60 | 60 | - `LOAD_VUS`: virtual users (default: 20) | |
| 61 | 61 | - `LOAD_DURATION_SECS`: duration (default: 30) | |
| 62 | 62 | - `LOAD_RAMP_SECS`: ramp-up (default: 5) | |
| 63 | + | - `LOAD_MIX`: scenario shares, e.g. `anon:25,buyer:15,creator:10,dash:50`. Must sum to 100. | |
| 64 | + | - `QUASI_SCREENS`: which screens serve from the description layer (`*` for all). | |
| 65 | + | - `LOAD_MT_LATENCY_MS`: how long the stub Multithreaded sleeps before answering (default: 0). | |
| 66 | + | - `LOAD_MT_MEMBERSHIPS`: rows the stub answers with (default: 8). | |
| 67 | + | - `LOAD_PROBE_MS`: blocking-pool probe interval (default: 50). | |
| 68 | + | ||
| 69 | + | The report carries two things beyond the endpoint table: | |
| 70 | + | ||
| 71 | + | - **`Rej`**, 4xx other than 429. Read it first. A rejected request is fast, so a | |
| 72 | + | route quietly answering 403 or 404 wins a latency comparison it never ran. | |
| 73 | + | - **Blocking-pool dispatch delay.** How long a fresh `spawn_blocking` waits to | |
| 74 | + | start, sampled through the run. The description layer's whole runtime cost is | |
| 75 | + | that quasi's router is sync, so every described request holds a pool thread; | |
| 76 | + | this is whether that pool still has headroom. `RuntimeMetrics` would say it | |
| 77 | + | directly but is behind `tokio_unstable`, which this tree does not build with. | |
| 78 | + | Read it with the `POST /join/step/account` mean, which is argon2 and the same | |
| 79 | + | fact from the other end. | |
| 80 | + | ||
| 81 | + | For comparing the description layer against Askama, use | |
| 82 | + | `scripts/load-conversion-ab.sh` rather than driving `QUASI_SCREENS` by hand: it | |
| 83 | + | runs the alternating-pairs protocol (Askama / described / described / Askama) at | |
| 84 | + | one fixed mix, so the four runs stay comparable. | |
| 63 | 85 | ||
| 64 | 86 | ### Health Tests (`cargo test --test health`) | |
| 65 | 87 |
| @@ -18,6 +18,18 @@ | |||
| 18 | 18 | pub db_acquire_timeout: Duration, | |
| 19 | 19 | /// Scenario distribution across VUs. | |
| 20 | 20 | pub scenario_mix: ScenarioMix, | |
| 21 | + | /// How long the Multithreaded stub sleeps before answering. | |
| 22 | + | /// | |
| 23 | + | /// The variable the forum-membership measurement turns. Those two screens | |
| 24 | + | /// are the only described ones whose work is an outbound call, and the | |
| 25 | + | /// described version holds a blocking-pool thread across it. Sweeping this | |
| 26 | + | /// is how the run answers "at what upstream latency does occupancy start to | |
| 27 | + | /// cost something" instead of "was it fine on the afternoon we looked". | |
| 28 | + | pub mt_latency: Duration, | |
| 29 | + | /// How many membership rows the stub answers with. | |
| 30 | + | pub mt_memberships: usize, | |
| 31 | + | /// How often the blocking-pool dispatch probe samples. | |
| 32 | + | pub probe_interval: Duration, | |
| 21 | 33 | } | |
| 22 | 34 | ||
| 23 | 35 | impl LoadConfig { | |
| @@ -36,6 +48,9 @@ | |||
| 36 | 48 | db_max_connections: 10, | |
| 37 | 49 | db_acquire_timeout: Duration::from_secs(3), | |
| 38 | 50 | scenario_mix: ScenarioMix::from_env(), | |
| 51 | + | mt_latency: Duration::from_millis(env_or("LOAD_MT_LATENCY_MS", 0)), | |
| 52 | + | mt_memberships: env_or("LOAD_MT_MEMBERSHIPS", 8), | |
| 53 | + | probe_interval: Duration::from_millis(env_or("LOAD_PROBE_MS", 50)), | |
| 39 | 54 | } | |
| 40 | 55 | } | |
| 41 | 56 | } |
| @@ -56,10 +56,13 @@ | |||
| 56 | 56 | // Per-endpoint stats | |
| 57 | 57 | let mut by_label: HashMap<String, Vec<Duration>> = HashMap::new(); | |
| 58 | 58 | let mut errors_by_label: HashMap<String, usize> = HashMap::new(); | |
| 59 | + | let mut rejected_by_label: HashMap<String, usize> = HashMap::new(); | |
| 59 | 60 | for m in metrics.iter() { | |
| 60 | 61 | by_label.entry(m.label.clone()).or_default().push(m.latency); | |
| 61 | 62 | if m.status.is_server_error() || m.status == StatusCode::TOO_MANY_REQUESTS { | |
| 62 | 63 | *errors_by_label.entry(m.label.clone()).or_default() += 1; | |
| 64 | + | } else if m.status.is_client_error() { | |
| 65 | + | *rejected_by_label.entry(m.label.clone()).or_default() += 1; | |
| 63 | 66 | } | |
| 64 | 67 | } | |
| 65 | 68 | ||
| @@ -69,6 +72,7 @@ | |||
| 69 | 72 | latencies.sort(); | |
| 70 | 73 | let count = latencies.len(); | |
| 71 | 74 | let errors = errors_by_label.get(&label).copied().unwrap_or(0); | |
| 75 | + | let rejected = rejected_by_label.get(&label).copied().unwrap_or(0); | |
| 72 | 76 | let min = latencies[0]; | |
| 73 | 77 | let max = latencies[count - 1]; | |
| 74 | 78 | let mean = latencies.iter().sum::<Duration>() / count as u32; | |
| @@ -80,6 +84,7 @@ | |||
| 80 | 84 | label, | |
| 81 | 85 | count, | |
| 82 | 86 | errors, | |
| 87 | + | rejected, | |
| 83 | 88 | min, | |
| 84 | 89 | max, | |
| 85 | 90 | mean, | |
| @@ -121,6 +126,15 @@ | |||
| 121 | 126 | pub label: String, | |
| 122 | 127 | pub count: usize, | |
| 123 | 128 | pub errors: usize, | |
| 129 | + | /// 4xx other than 429, which the error count already owns. | |
| 130 | + | /// | |
| 131 | + | /// Its own column because a rejected request is FAST, and a comparison | |
| 132 | + | /// between two renderings of one screen reads a route that quietly answers | |
| 133 | + | /// 403 or 404 as the winner. That is not hypothetical here: the described | |
| 134 | + | /// settings forums section refuses outright when Multithreaded is | |
| 135 | + | /// unconfigured, and the creator-only dashboard tabs refuse a plain signup. | |
| 136 | + | /// Both would have looked like sub-millisecond routes. | |
| 137 | + | pub rejected: usize, | |
| 124 | 138 | pub min: Duration, | |
| 125 | 139 | pub max: Duration, | |
| 126 | 140 | pub mean: Duration, | |
| @@ -165,20 +179,21 @@ | |||
| 165 | 179 | ||
| 166 | 180 | // Per-endpoint table | |
| 167 | 181 | println!( | |
| 168 | - | " {:<30} {:>6} {:>6} {:>8} {:>8} {:>8} {:>8} {:>8} {:>8}", | |
| 169 | - | "Endpoint", "Count", "Errors", "Min", "Max", "Mean", "p50", "p95", "p99" | |
| 182 | + | " {:<34} {:>6} {:>6} {:>6} {:>8} {:>8} {:>8} {:>8} {:>8} {:>8}", | |
| 183 | + | "Endpoint", "Count", "Errors", "Rej", "Min", "Max", "Mean", "p50", "p95", "p99" | |
| 170 | 184 | ); | |
| 171 | 185 | println!( | |
| 172 | - | " {:-<30} {:-<6} {:-<6} {:-<8} {:-<8} {:-<8} {:-<8} {:-<8} {:-<8}", | |
| 173 | - | "", "", "", "", "", "", "", "", "" | |
| 186 | + | " {:-<34} {:-<6} {:-<6} {:-<6} {:-<8} {:-<8} {:-<8} {:-<8} {:-<8} {:-<8}", | |
| 187 | + | "", "", "", "", "", "", "", "", "", "" | |
| 174 | 188 | ); | |
| 175 | 189 | ||
| 176 | 190 | for ep in &self.endpoint_stats { | |
| 177 | 191 | println!( | |
| 178 | - | " {:<30} {:>6} {:>6} {:>8} {:>8} {:>8} {:>8} {:>8} {:>8}", | |
| 192 | + | " {:<34} {:>6} {:>6} {:>6} {:>8} {:>8} {:>8} {:>8} {:>8} {:>8}", | |
| 179 | 193 | ep.label, | |
| 180 | 194 | ep.count, | |
| 181 | 195 | ep.errors, | |
| 196 | + | ep.rejected, | |
| 182 | 197 | format_dur(ep.min), | |
| 183 | 198 | format_dur(ep.max), | |
| 184 | 199 | format_dur(ep.mean), |
| @@ -1,8 +1,10 @@ | |||
| 1 | 1 | //! Load testing module: simulates concurrent virtual users against a shared | |
| 2 | 2 | //! app instance to measure performance under contention. | |
| 3 | 3 | ||
| 4 | + | mod blocking_probe; | |
| 4 | 5 | mod config; | |
| 5 | 6 | mod metrics; | |
| 7 | + | mod mt_stub; | |
| 6 | 8 | mod runner; | |
| 7 | 9 | mod scenarios; | |
| 8 | 10 | ||
| @@ -19,6 +21,16 @@ | |||
| 19 | 21 | println!(" Think time: {:?}", config.think_time); | |
| 20 | 22 | println!(" DB max connections: {}", config.db_max_connections); | |
| 21 | 23 | println!(" Scenario mix: {:?}", config.scenario_mix); | |
| 24 | + | // The described set is the independent variable of every conversion | |
| 25 | + | // measurement, so it is printed with the rest of the configuration rather | |
| 26 | + | // than left in the shell history of whoever ran it. An empty line here means | |
| 27 | + | // the run measured the Askama side. | |
| 28 | + | println!( | |
| 29 | + | " QUASI_SCREENS: {:?}", | |
| 30 | + | std::env::var("QUASI_SCREENS").unwrap_or_default() | |
| 31 | + | ); | |
| 32 | + | println!(" MT stub latency: {:?}", config.mt_latency); | |
| 33 | + | println!(" MT memberships: {}", config.mt_memberships); | |
| 22 | 34 | println!(); | |
| 23 | 35 | ||
| 24 | 36 | runner::run(config).await; |
| @@ -21,8 +21,10 @@ | |||
| 21 | 21 | use crate::harness::client::TestClient; | |
| 22 | 22 | use crate::harness::db::TestDb; | |
| 23 | 23 | ||
| 24 | + | use super::blocking_probe::BlockingProbe; | |
| 24 | 25 | use super::config::{LoadConfig, ScenarioType}; | |
| 25 | 26 | use super::metrics::MetricsCollector; | |
| 27 | + | use super::mt_stub::MtStub; | |
| 26 | 28 | use super::scenarios::{self, SeedData}; | |
| 27 | 29 | ||
| 28 | 30 | /// Run the full load test. | |
| @@ -30,6 +32,12 @@ | |||
| 30 | 32 | // 1. Database setup: TestDb for creation/migration/cleanup | |
| 31 | 33 | let test_db = TestDb::new().await; | |
| 32 | 34 | ||
| 35 | + | // A stand-in Multithreaded, up before the app so its address can be | |
| 36 | + | // configured. Always started, not only when latency is asked for: the two | |
| 37 | + | // forum screens refuse or empty out when `mt_base_url` is unset, and a run | |
| 38 | + | // that measures a refusal reads as a very fast route. | |
| 39 | + | let mt = MtStub::start(config.mt_latency, config.mt_memberships).await; | |
| 40 | + | ||
| 33 | 41 | // Production-sized pool against the same test database | |
| 34 | 42 | let pool = PgPoolOptions::new() | |
| 35 | 43 | .max_connections(config.db_max_connections) | |
| @@ -101,9 +109,15 @@ | |||
| 101 | 109 | founder_window_open: false, | |
| 102 | 110 | }, | |
| 103 | 111 | integrations: IntegrationsConfig { | |
| 104 | - | mt_base_url: None, | |
| 112 | + | // Both halves, or the forum screens are not exercised. This was | |
| 113 | + | // hardcoded `None`, which is why the two Multithreaded-backed | |
| 114 | + | // described screens had no load measurement at all: the described | |
| 115 | + | // library tab answered an empty list without making the call, and | |
| 116 | + | // the described settings section 404'd, so either would have | |
| 117 | + | // reported as the fastest route in the run. | |
| 118 | + | mt_base_url: Some(mt.base_url()), | |
| 105 | 119 | wam_url: None, | |
| 106 | - | internal_shared_secret: None, | |
| 120 | + | internal_shared_secret: Some("load-test-mt-secret".to_string()), | |
| 107 | 121 | cli_service_token: None, | |
| 108 | 122 | alerts_ingest_token: None, | |
| 109 | 123 | }, | |
| @@ -188,6 +202,9 @@ | |||
| 188 | 202 | ||
| 189 | 203 | // 5. Spawn VUs | |
| 190 | 204 | let metrics = MetricsCollector::new(); | |
| 205 | + | // Started after seeding, which is itself heavy enough to show as a stall and | |
| 206 | + | // is not part of what the run is measuring. | |
| 207 | + | let probe = BlockingProbe::start(config.probe_interval); | |
| 191 | 208 | let ramp_delay = if config.virtual_users > 1 { | |
| 192 | 209 | config.ramp_up / config.virtual_users | |
| 193 | 210 | } else { | |
| @@ -229,7 +246,7 @@ | |||
| 229 | 246 | scenarios::creator_flow(a, ip, deadline, think_time, m, p).await; | |
| 230 | 247 | } | |
| 231 | 248 | ScenarioType::DashboardSession => { | |
| 232 | - | scenarios::dashboard_session(a, ip, deadline, think_time, m).await; | |
| 249 | + | scenarios::dashboard_session(a, ip, deadline, think_time, m, p).await; | |
| 233 | 250 | } | |
| 234 | 251 | } | |
| 235 | 252 | }); | |
| @@ -247,8 +264,10 @@ | |||
| 247 | 264 | ||
| 248 | 265 | // 7. Report | |
| 249 | 266 | metrics.report().print(); | |
| 267 | + | probe.finish().print(); | |
| 250 | 268 | ||
| 251 | 269 | // 8. Cleanup (TestDb dropped here) | |
| 270 | + | drop(mt); | |
| 252 | 271 | drop(pool); | |
| 253 | 272 | drop(test_db); | |
| 254 | 273 | } |
| @@ -382,53 +382,104 @@ | |||
| 382 | 382 | } | |
| 383 | 383 | } | |
| 384 | 384 | ||
| 385 | - | /// Dashboard session: one-time signup, then loop through dashboard tabs. | |
| 385 | + | /// Every address a described screen can claim, paired with the tab label the | |
| 386 | + | /// report files it under. | |
| 387 | + | /// | |
| 388 | + | /// Each is served by Askama or by the description layer depending on | |
| 389 | + | /// `QUASI_SCREENS`, at the same address either way, which is what makes the | |
| 390 | + | /// alternating-pairs protocol a comparison rather than two unrelated runs. | |
| 391 | + | /// | |
| 392 | + | /// Each gets its OWN label. The undescribed tabs share one on purpose (they are | |
| 393 | + | /// one population); these are the population being compared, and folding them | |
| 394 | + | /// together would average away the number the run is after. S3 measured one of | |
| 395 | + | /// them, so its numbers say nothing about a mix where six are on at once and | |
| 396 | + | /// blocking-pool occupancy accumulates across them. | |
| 397 | + | const DESCRIBED: &[(&str, &str)] = &[ | |
| 398 | + | ("/dashboard/tabs/ssh-keys", "HTMX described:ssh-keys"), | |
| 399 | + | ("/dashboard/tabs/contacts", "HTMX described:buyer-contacts"), | |
| 400 | + | ("/dashboard/tabs/analytics", "HTMX described:analytics"), | |
| 401 | + | // The two Multithreaded-backed ones. Their work is an outbound HTTP call | |
| 402 | + | // rather than a query, so under the description layer they hold a blocking | |
| 403 | + | // thread for the upstream latency rather than for a sub-millisecond round | |
| 404 | + | // trip. That is the regime nothing had measured. | |
| 405 | + | ("/dashboard/tabs/forums", "HTMX described:forums"), | |
| 406 | + | ( | |
| 407 | + | "/library/tabs/communities", | |
| 408 | + | "HTMX described:library-communities", | |
| 409 | + | ), | |
| 410 | + | ("/library/tabs/contacts", "HTMX described:library-contacts"), | |
| 411 | + | ]; | |
| 412 | + | ||
| 413 | + | /// Dashboard session: one-time signup, then loop through dashboard and library | |
| 414 | + | /// tabs, described ones included. | |
| 415 | + | /// | |
| 416 | + | /// Takes the pool because two of the described screens are creator-only. A plain | |
| 417 | + | /// signup gets 403 on them, and a 403 is fast: without the grant this scenario | |
| 418 | + | /// would have reported the two creator screens as the quickest routes in the run | |
| 419 | + | /// on both sides of the comparison, and the `Rej` column exists to make that | |
| 420 | + | /// visible if it ever regresses. | |
| 386 | 421 | pub(super) async fn dashboard_session( | |
| 387 | 422 | app: Router, | |
| 388 | 423 | ip: String, | |
| 389 | 424 | deadline: Instant, | |
| 390 | 425 | think_time: Duration, | |
| 391 | 426 | metrics: MetricsCollector, | |
| 427 | + | pool: sqlx::PgPool, | |
| 392 | 428 | ) { | |
| 393 | - | let mut client = TestClient::new(app); | |
| 429 | + | let mut client = TestClient::new(app.clone()); | |
| 394 | 430 | client.set_forwarded_ip(&ip); | |
| 395 | 431 | ||
| 396 | 432 | let username = next_username("dash"); | |
| 397 | 433 | if !signup(&mut client, &username, &metrics).await { | |
| 398 | 434 | return; | |
| 399 | 435 | } | |
| 436 | + | ||
| 437 | + | // Grant creator, then re-login on a fresh client so the session carries it. | |
| 438 | + | // Same dance as the seed path, and for the same reason: `authenticate` | |
| 439 | + | // caches its touch for a few seconds, so the live session predates the grant. | |
| 440 | + | let granted = sqlx::query_scalar::<_, uuid::Uuid>("SELECT id FROM users WHERE username = $1") | |
| 441 | + | .bind(&username) | |
| 442 | + | .fetch_optional(&pool) | |
| 443 | + | .await | |
| 444 | + | .ok() | |
| 445 | + | .flatten(); | |
| 446 | + | if let Some(user_id) = granted { | |
| 447 | + | let _ = sqlx::query("UPDATE users SET can_create_projects = true WHERE id = $1") | |
| 448 | + | .bind(user_id) | |
| 449 | + | .execute(&pool) | |
| 450 | + | .await; | |
| 451 | + | let mut fresh = TestClient::new(app); | |
| 452 | + | fresh.set_forwarded_ip(&ip); | |
| 453 | + | if login(&mut fresh, &username, &metrics).await { | |
| 454 | + | client = fresh; | |
| 455 | + | } | |
| 456 | + | } | |
| 400 | 457 | sleep(think_time).await; | |
| 401 | 458 | ||
| 402 | - | // ssh-keys is the described screen (S3). It is in the mix rather than | |
| 403 | - | // measured alone because the question is what a described route does to the | |
| 404 | - | // rest of the server, not what it costs on an idle box: it holds a | |
| 405 | - | // blocking-pool thread for three database round trips while everything | |
| 406 | - | // else contends for the same pool. | |
| 407 | - | let tabs = [ | |
| 408 | - | "details", | |
| 409 | - | "payments", | |
| 410 | - | "projects", | |
| 411 | - | "creator", | |
| 412 | - | "promotions", | |
| 413 | - | "ssh-keys", | |
| 414 | - | ]; | |
| 459 | + | // The undescribed tabs the described ones are compared against. Each reports | |
| 460 | + | // under its own label rather than sharing one: the shared label was carrying | |
| 461 | + | // a `promotions` entry that has no route at all (a 404 on every cycle, | |
| 462 | + | // averaged into the baseline the described screens are measured against) and | |
| 463 | + | // a tab returning 500, and neither was visible while five tabs were one row. | |
| 464 | + | // The old aggregate is still recoverable by summing these. | |
| 465 | + | let plain_tabs = ["details", "payments", "projects", "creator"]; | |
| 415 | 466 | ||
| 416 | 467 | while Instant::now() < deadline { | |
| 417 | 468 | timed_get(&mut client, "/dashboard", "GET /dashboard", &metrics).await; | |
| 418 | 469 | sleep(think_time).await; | |
| 419 | 470 | ||
| 420 | - | for tab in &tabs { | |
| 471 | + | for tab in &plain_tabs { | |
| 421 | 472 | let url = format!("/dashboard/tabs/{tab}"); | |
| 422 | - | // ssh-keys reports under its own label. The others share one on | |
| 423 | - | // purpose (they are one population), but the described screen is | |
| 424 | - | // the population being compared, and folding it into the rest | |
| 425 | - | // would average away exactly the number S3 is after. | |
| 426 | - | let label = if *tab == "ssh-keys" { | |
| 427 | - | "HTMX /dashboard/tabs/ssh-keys" | |
| 428 | - | } else { | |
| 429 | - | "HTMX /dashboard/tabs/{tab}" | |
| 430 | - | }; | |
| 431 | - | timed_htmx_get(&mut client, &url, label, &metrics).await; | |
| 473 | + | let label = format!("HTMX /dashboard/tabs/{tab}"); | |
| 474 | + | timed_htmx_get(&mut client, &url, &label, &metrics).await; | |
| 475 | + | sleep(think_time).await; | |
| 476 | + | } | |
| 477 | + | ||
| 478 | + | timed_get(&mut client, "/library", "GET /library", &metrics).await; | |
| 479 | + | sleep(think_time).await; | |
| 480 | + | ||
| 481 | + | for (path, label) in DESCRIBED { | |
| 482 | + | timed_htmx_get(&mut client, path, label, &metrics).await; | |
| 432 | 483 | sleep(think_time).await; | |
| 433 | 484 | } | |
| 434 | 485 |
| @@ -451,3 +451,76 @@ | |||
| 451 | 451 | "an enforced creator is not swept twice" | |
| 452 | 452 | ); | |
| 453 | 453 | } | |
| 454 | + | ||
| 455 | + | // ── the empty-category clamp ──────────────────────────────────────────────── | |
| 456 | + | ||
| 457 | + | /// A creator who has uploaded nothing reads as zero across every category. | |
| 458 | + | /// | |
| 459 | + | /// Regression, 2026-08-14. Each category clamped its SUM with | |
| 460 | + | /// `LEAST(SUM(x), i64::MAX)`, and Postgres `LEAST` IGNORES NULL arguments rather | |
| 461 | + | /// than propagating them: over zero rows that is `i64::MAX`, not NULL, so the | |
| 462 | + | /// outer `COALESCE(..., 0)` never fired and the ceiling behaved as a floor. | |
| 463 | + | /// | |
| 464 | + | /// Every assertion here failed before the fix, and the first one failed by | |
| 465 | + | /// erroring rather than by returning a wrong number: `cover_bytes` adds item | |
| 466 | + | /// covers to project covers, so a creator with neither added `i64::MAX` to | |
| 467 | + | /// `i64::MAX` and Postgres answered `22003 bigint out of range`. That is the | |
| 468 | + | /// state every creator is in on the day they are granted, and it surfaced as a | |
| 469 | + | /// 500 on the whole `/dashboard/tabs/creator` route. | |
| 470 | + | #[tokio::test] | |
| 471 | + | async fn an_empty_creator_reads_as_zero_bytes_everywhere() { | |
| 472 | + | let db = TestDb::new().await; | |
| 473 | + | let user = seed_user(&db.pool, "emptycreator").await; | |
| 474 | + | sqlx::query("UPDATE users SET can_create_projects = true WHERE id = $1") | |
| 475 | + | .bind(user) | |
| 476 | + | .execute(&db.pool) | |
| 477 | + | .await | |
| 478 | + | .unwrap(); | |
| 479 | + | ||
| 480 | + | let breakdown = db::creator_tiers::get_storage_breakdown(&db.pool, user) | |
| 481 | + | .await | |
| 482 | + | .expect("an empty breakdown is a query, not an overflow"); | |
| 483 | + | ||
| 484 | + | assert_eq!(breakdown.audio_bytes, 0); | |
| 485 | + | assert_eq!(breakdown.cover_bytes, 0); | |
| 486 | + | assert_eq!(breakdown.download_bytes, 0); | |
| 487 | + | assert_eq!(breakdown.insertion_bytes, 0); | |
| 488 | + | assert_eq!(breakdown.video_bytes, 0); | |
| 489 | + | assert_eq!(breakdown.media_bytes, 0); | |
| 490 | + | assert_eq!(breakdown.gallery_bytes, 0); | |
| 491 | + | assert_eq!( | |
| 492 | + | breakdown.total_bytes, 0, | |
| 493 | + | "the total is the sum of the categories, and they are all empty" | |
| 494 | + | ); | |
| 495 | + | } | |
| 496 | + | ||
| 497 | + | /// The weekly drift-correction reconciles an empty creator to zero. | |
| 498 | + | /// | |
| 499 | + | /// The same clamp, in the other query that uses it. This one adds NINE clamped | |
| 500 | + | /// terms in SQL, so it did not merely record a wrong total: it errored out, and | |
| 501 | + | /// a recalc that cannot run is a counter that silently keeps whatever it had. | |
| 502 | + | #[tokio::test] | |
| 503 | + | async fn the_storage_recalc_reconciles_an_empty_creator_to_zero() { | |
| 504 | + | let db = TestDb::new().await; | |
| 505 | + | let user = seed_user(&db.pool, "recalccreator").await; | |
| 506 | + | // A non-zero starting counter, so reconciling to zero is a real correction | |
| 507 | + | // rather than a value that was already right. | |
| 508 | + | sqlx::query( | |
| 509 | + | "UPDATE users SET can_create_projects = true, storage_used_bytes = $2 WHERE id = $1", | |
| 510 | + | ) | |
| 511 | + | .bind(user) | |
| 512 | + | .bind(64 * MB) | |
| 513 | + | .execute(&db.pool) | |
| 514 | + | .await | |
| 515 | + | .unwrap(); | |
| 516 | + | ||
| 517 | + | db::creator_tiers::recalculate_all_storage_batch(&db.pool) | |
| 518 | + | .await | |
| 519 | + | .expect("the recalc must not overflow on creators who uploaded nothing"); | |
| 520 | + | ||
| 521 | + | assert_eq!( | |
| 522 | + | storage_used(&db.pool, user).await, | |
| 523 | + | 0, | |
| 524 | + | "an empty creator's counter reconciles to zero" | |
| 525 | + | ); | |
| 526 | + | } |
| @@ -210,54 +210,77 @@ | |||
| 210 | 210 | // so the dashboard total reconciles with `storage_used_bytes`. `cover_bytes` | |
| 211 | 211 | // folds item covers + project covers; `gallery_bytes` covers the item/project | |
| 212 | 212 | // image carousels (Run #18 Storage B1). | |
| 213 | + | // | |
| 214 | + | // COALESCE THE SUM BEFORE THE CLAMP, not after. `LEAST` ignores NULL | |
| 215 | + | // arguments rather than propagating them, so `LEAST(SUM(x), i64::MAX)` over | |
| 216 | + | // zero rows is `i64::MAX`, not NULL, and the outer COALESCE never fires. The | |
| 217 | + | // clamp meant to be a ceiling was acting as a floor on every empty category. | |
| 218 | + | // | |
| 219 | + | // It read as wrong storage totals in the general case and as a 500 on the | |
| 220 | + | // creator dashboard tab in the two-term ones: `cover_bytes` and | |
| 221 | + | // `gallery_bytes` add two clamped values, and a creator with neither item | |
| 222 | + | // covers nor project covers added i64::MAX to i64::MAX, which Postgres | |
| 223 | + | // answers with `22003 bigint out of range` from `int8pl`. That is every new | |
| 224 | + | // creator, so the tab 500'd for them from the moment they were granted. | |
| 225 | + | // Found 2026-08-14 by the load harness, which had been averaging it into a | |
| 226 | + | // five-tab aggregate; no test covered the route. | |
| 227 | + | // | |
| 228 | + | // The two-term sums add in NUMERIC and clamp once at the end, so the | |
| 229 | + | // addition itself cannot overflow either. | |
| 213 | 230 | let row: (i64, i64, i64, i64, i64, i64, i64) = sqlx::query_as( | |
| 214 | 231 | r" | |
| 215 | 232 | WITH audio_bytes AS ( | |
| 216 | - | SELECT COALESCE(GREATEST(0, LEAST(SUM(i.audio_file_size_bytes), 9223372036854775807))::BIGINT, 0) AS total | |
| 233 | + | SELECT COALESCE(GREATEST(0, LEAST(COALESCE(SUM(i.audio_file_size_bytes), 0), 9223372036854775807))::BIGINT, 0) AS total | |
| 217 | 234 | FROM items i JOIN projects p ON i.project_id = p.id | |
| 218 | 235 | WHERE p.user_id = $1 AND i.audio_file_size_bytes IS NOT NULL | |
| 219 | 236 | ), | |
| 220 | 237 | cover_bytes AS ( | |
| 221 | - | SELECT COALESCE(GREATEST(0, LEAST(SUM(i.cover_file_size_bytes), 9223372036854775807))::BIGINT, 0) | |
| 222 | - | + COALESCE(( | |
| 223 | - | SELECT GREATEST(0, LEAST(SUM(p.cover_image_size_bytes), 9223372036854775807))::BIGINT | |
| 224 | - | FROM projects p | |
| 225 | - | WHERE p.user_id = $1 AND p.cover_image_size_bytes IS NOT NULL | |
| 226 | - | ), 0) AS total | |
| 238 | + | SELECT LEAST( | |
| 239 | + | GREATEST(0, COALESCE(SUM(i.cover_file_size_bytes), 0))::NUMERIC | |
| 240 | + | + COALESCE(( | |
| 241 | + | SELECT GREATEST(0, COALESCE(SUM(p.cover_image_size_bytes), 0))::NUMERIC | |
| 242 | + | FROM projects p | |
| 243 | + | WHERE p.user_id = $1 AND p.cover_image_size_bytes IS NOT NULL | |
| 244 | + | ), 0), | |
| 245 | + | 9223372036854775807 | |
| 246 | + | )::BIGINT AS total | |
| 227 | 247 | FROM items i JOIN projects p ON i.project_id = p.id | |
| 228 | 248 | WHERE p.user_id = $1 AND i.cover_file_size_bytes IS NOT NULL | |
| 229 | 249 | ), | |
| 230 | 250 | version_bytes AS ( | |
| 231 | - | SELECT COALESCE(GREATEST(0, LEAST(SUM(v.file_size_bytes), 9223372036854775807))::BIGINT, 0) AS total | |
| 251 | + | SELECT COALESCE(GREATEST(0, LEAST(COALESCE(SUM(v.file_size_bytes), 0), 9223372036854775807))::BIGINT, 0) AS total | |
| 232 | 252 | FROM versions v | |
| 233 | 253 | JOIN items i ON v.item_id = i.id | |
| 234 | 254 | JOIN projects p ON i.project_id = p.id | |
| 235 | 255 | WHERE p.user_id = $1 AND v.file_size_bytes IS NOT NULL | |
| 236 | 256 | ), | |
| 237 | 257 | insertion_bytes AS ( | |
| 238 | - | SELECT COALESCE(GREATEST(0, LEAST(SUM(file_size), 9223372036854775807))::BIGINT, 0) AS total | |
| 258 | + | SELECT COALESCE(GREATEST(0, LEAST(COALESCE(SUM(file_size), 0), 9223372036854775807))::BIGINT, 0) AS total | |
| 239 | 259 | FROM content_insertions WHERE user_id = $1 | |
| 240 | 260 | ), | |
| 241 | 261 | video_bytes AS ( | |
| 242 | - | SELECT COALESCE(GREATEST(0, LEAST(SUM(i.video_file_size_bytes), 9223372036854775807))::BIGINT, 0) AS total | |
| 262 | + | SELECT COALESCE(GREATEST(0, LEAST(COALESCE(SUM(i.video_file_size_bytes), 0), 9223372036854775807))::BIGINT, 0) AS total | |
| 243 | 263 | FROM items i JOIN projects p ON i.project_id = p.id | |
| 244 | 264 | WHERE p.user_id = $1 AND i.video_file_size_bytes IS NOT NULL | |
| 245 | 265 | ), | |
| 246 | 266 | media_bytes AS ( | |
| 247 | - | SELECT COALESCE(GREATEST(0, LEAST(SUM(file_size_bytes), 9223372036854775807))::BIGINT, 0) AS total | |
| 267 | + | SELECT COALESCE(GREATEST(0, LEAST(COALESCE(SUM(file_size_bytes), 0), 9223372036854775807))::BIGINT, 0) AS total | |
| 248 | 268 | FROM media_files WHERE user_id = $1 | |
| 249 | 269 | ), | |
| 250 | 270 | gallery_bytes AS ( | |
| 251 | - | SELECT COALESCE(( | |
| 252 | - | SELECT GREATEST(0, LEAST(SUM(ii.file_size_bytes), 9223372036854775807))::BIGINT | |
| 271 | + | SELECT LEAST( | |
| 272 | + | COALESCE(( | |
| 273 | + | SELECT GREATEST(0, COALESCE(SUM(ii.file_size_bytes), 0))::NUMERIC | |
| 253 | 274 | FROM item_images ii JOIN items i ON ii.item_id = i.id JOIN projects p ON i.project_id = p.id | |
| 254 | 275 | WHERE p.user_id = $1 | |
| 255 | - | ), 0) | |
| 256 | - | + COALESCE(( | |
| 257 | - | SELECT GREATEST(0, LEAST(SUM(pi.file_size_bytes), 9223372036854775807))::BIGINT | |
| 276 | + | ), 0) | |
| 277 | + | + COALESCE(( | |
| 278 | + | SELECT GREATEST(0, COALESCE(SUM(pi.file_size_bytes), 0))::NUMERIC | |
| 258 | 279 | FROM project_images pi JOIN projects p ON pi.project_id = p.id | |
| 259 | 280 | WHERE p.user_id = $1 | |
| 260 | - | ), 0) AS total | |
| 281 | + | ), 0), | |
| 282 | + | 9223372036854775807 | |
| 283 | + | )::BIGINT AS total | |
| 261 | 284 | ) | |
| 262 | 285 | SELECT | |
| 263 | 286 | (SELECT total FROM audio_bytes), | |
| @@ -326,42 +349,50 @@ | |||
| 326 | 349 | UPDATE users SET storage_used_bytes = totals.total | |
| 327 | 350 | FROM ( | |
| 328 | 351 | SELECT u.id AS user_id, | |
| 329 | - | COALESCE(audio.total, 0) | |
| 330 | - | + COALESCE(cover.total, 0) | |
| 331 | - | + COALESCE(video.total, 0) | |
| 332 | - | + COALESCE(versions.total, 0) | |
| 333 | - | + COALESCE(insertions.total, 0) | |
| 334 | - | + COALESCE(media.total, 0) | |
| 335 | - | + COALESCE(project_cover.total, 0) | |
| 336 | - | + COALESCE(item_gallery.total, 0) | |
| 337 | - | + COALESCE(project_gallery.total, 0) AS total | |
| 352 | + | -- NUMERIC for the addition, clamped once at the end: nine | |
| 353 | + | -- BIGINT terms each capped at i64::MAX overflow `int8pl` long | |
| 354 | + | -- before the cap means anything. See the note on | |
| 355 | + | -- `get_storage_breakdown` for how that used to fire on every | |
| 356 | + | -- creator rather than only on an implausible one. | |
| 357 | + | LEAST( | |
| 358 | + | COALESCE(audio.total, 0)::NUMERIC | |
| 359 | + | + COALESCE(cover.total, 0)::NUMERIC | |
| 360 | + | + COALESCE(video.total, 0)::NUMERIC | |
| 361 | + | + COALESCE(versions.total, 0)::NUMERIC | |
| 362 | + | + COALESCE(insertions.total, 0)::NUMERIC | |
| 363 | + | + COALESCE(media.total, 0)::NUMERIC | |
| 364 | + | + COALESCE(project_cover.total, 0)::NUMERIC | |
| 365 | + | + COALESCE(item_gallery.total, 0)::NUMERIC | |
| 366 | + | + COALESCE(project_gallery.total, 0)::NUMERIC, | |
| 367 | + | 9223372036854775807 | |
| 368 | + | )::BIGINT AS total | |
| 338 | 369 | FROM users u | |
| 339 | 370 | LEFT JOIN LATERAL ( | |
| 340 | - | SELECT GREATEST(0, LEAST(SUM(i.audio_file_size_bytes), 9223372036854775807))::BIGINT AS total | |
| 371 | + | SELECT GREATEST(0, LEAST(COALESCE(SUM(i.audio_file_size_bytes), 0), 9223372036854775807))::BIGINT AS total | |
| 341 | 372 | FROM items i JOIN projects p ON i.project_id = p.id | |
| 342 | 373 | WHERE p.user_id = u.id AND i.audio_file_size_bytes IS NOT NULL | |
| 343 | 374 | ) audio ON true | |
| 344 | 375 | LEFT JOIN LATERAL ( | |
| 345 | - | SELECT GREATEST(0, LEAST(SUM(i.cover_file_size_bytes), 9223372036854775807))::BIGINT AS total | |
| 376 | + | SELECT GREATEST(0, LEAST(COALESCE(SUM(i.cover_file_size_bytes), 0), 9223372036854775807))::BIGINT AS total | |
| 346 | 377 | FROM items i JOIN projects p ON i.project_id = p.id | |
| 347 | 378 | WHERE p.user_id = u.id AND i.cover_file_size_bytes IS NOT NULL | |
| 348 | 379 | ) cover ON true | |
| 349 | 380 | LEFT JOIN LATERAL ( | |
| 350 | - | SELECT GREATEST(0, LEAST(SUM(i.video_file_size_bytes), 9223372036854775807))::BIGINT AS total | |
| 381 | + | SELECT GREATEST(0, LEAST(COALESCE(SUM(i.video_file_size_bytes), 0), 9223372036854775807))::BIGINT AS total | |
| 351 | 382 | FROM items i JOIN projects p ON i.project_id = p.id | |
| 352 | 383 | WHERE p.user_id = u.id AND i.video_file_size_bytes IS NOT NULL | |
| 353 | 384 | ) video ON true | |
| 354 | 385 | LEFT JOIN LATERAL ( | |
| 355 | - | SELECT GREATEST(0, LEAST(SUM(v.file_size_bytes), 9223372036854775807))::BIGINT AS total | |
| 386 | + | SELECT GREATEST(0, LEAST(COALESCE(SUM(v.file_size_bytes), 0), 9223372036854775807))::BIGINT AS total | |
| 356 | 387 | FROM versions v JOIN items i ON v.item_id = i.id JOIN projects p ON i.project_id = p.id | |
| 357 | 388 | WHERE p.user_id = u.id AND v.file_size_bytes IS NOT NULL | |
| 358 | 389 | ) versions ON true | |
| 359 | 390 | LEFT JOIN LATERAL ( | |
| 360 | - | SELECT GREATEST(0, LEAST(SUM(ci.file_size), 9223372036854775807))::BIGINT AS total | |
| 391 | + | SELECT GREATEST(0, LEAST(COALESCE(SUM(ci.file_size), 0), 9223372036854775807))::BIGINT AS total | |
| 361 | 392 | FROM content_insertions ci WHERE ci.user_id = u.id | |
| 362 | 393 | ) insertions ON true | |
| 363 | 394 | LEFT JOIN LATERAL ( | |
| 364 | - | SELECT GREATEST(0, LEAST(SUM(mf.file_size_bytes), 9223372036854775807))::BIGINT AS total | |
| 395 | + | SELECT GREATEST(0, LEAST(COALESCE(SUM(mf.file_size_bytes), 0), 9223372036854775807))::BIGINT AS total | |
| 365 | 396 | FROM media_files mf WHERE mf.user_id = u.id | |
| 366 | 397 | ) media ON true | |
| 367 | 398 | -- Project cover images charge storage at confirm but live in their own | |
| @@ -372,17 +403,17 @@ | |||
| 372 | 403 | -- gallery/project-cover charge and the counter oscillated week to week | |
| 373 | 404 | -- (Run #18 Storage B1). Reconcile them from the same rows the charge writes. | |
| 374 | 405 | LEFT JOIN LATERAL ( | |
| 375 | - | SELECT GREATEST(0, LEAST(SUM(p.cover_image_size_bytes), 9223372036854775807))::BIGINT AS total | |
| 406 | + | SELECT GREATEST(0, LEAST(COALESCE(SUM(p.cover_image_size_bytes), 0), 9223372036854775807))::BIGINT AS total | |
| 376 | 407 | FROM projects p | |
| 377 | 408 | WHERE p.user_id = u.id AND p.cover_image_size_bytes IS NOT NULL | |
| 378 | 409 | ) project_cover ON true | |
| 379 | 410 | LEFT JOIN LATERAL ( | |
| 380 | - | SELECT GREATEST(0, LEAST(SUM(ii.file_size_bytes), 9223372036854775807))::BIGINT AS total | |
| 411 | + | SELECT GREATEST(0, LEAST(COALESCE(SUM(ii.file_size_bytes), 0), 9223372036854775807))::BIGINT AS total | |
| 381 | 412 | FROM item_images ii JOIN items i ON ii.item_id = i.id JOIN projects p ON i.project_id = p.id | |
| 382 | 413 | WHERE p.user_id = u.id | |
| 383 | 414 | ) item_gallery ON true | |
| 384 | 415 | LEFT JOIN LATERAL ( | |
| 385 | - | SELECT GREATEST(0, LEAST(SUM(pi.file_size_bytes), 9223372036854775807))::BIGINT AS total | |
| 416 | + | SELECT GREATEST(0, LEAST(COALESCE(SUM(pi.file_size_bytes), 0), 9223372036854775807))::BIGINT AS total | |
| 386 | 417 | FROM project_images pi JOIN projects p ON pi.project_id = p.id | |
| 387 | 418 | WHERE p.user_id = u.id | |
| 388 | 419 | ) project_gallery ON true | |
| @@ -626,7 +657,7 @@ | |||
| 626 | 657 | pub async fn get_user_content_size(pool: &PgPool, user_id: UserId) -> Result<i64> { | |
| 627 | 658 | let version_size: i64 = sqlx::query_scalar( | |
| 628 | 659 | r" | |
| 629 | - | SELECT COALESCE(GREATEST(0, LEAST(SUM(v.file_size_bytes), 9223372036854775807))::BIGINT, 0) | |
| 660 | + | SELECT COALESCE(GREATEST(0, LEAST(COALESCE(SUM(v.file_size_bytes), 0), 9223372036854775807))::BIGINT, 0) | |
| 630 | 661 | FROM versions v | |
| 631 | 662 | JOIN items i ON v.item_id = i.id | |
| 632 | 663 | JOIN projects p ON i.project_id = p.id | |
| @@ -638,7 +669,7 @@ | |||
| 638 | 669 | .await?; | |
| 639 | 670 | ||
| 640 | 671 | let insertion_size: i64 = sqlx::query_scalar( | |
| 641 | - | "SELECT COALESCE(GREATEST(0, LEAST(SUM(file_size), 9223372036854775807))::BIGINT, 0) FROM content_insertions WHERE user_id = $1", | |
| 672 | + | "SELECT COALESCE(GREATEST(0, LEAST(COALESCE(SUM(file_size), 0), 9223372036854775807))::BIGINT, 0) FROM content_insertions WHERE user_id = $1", | |
| 642 | 673 | ) | |
| 643 | 674 | .bind(user_id) | |
| 644 | 675 | .fetch_one(pool) |
| @@ -1,0 +1,82 @@ | |||
| 1 | + | #!/usr/bin/env bash | |
| 2 | + | # The alternating-pairs protocol for measuring the description layer. | |
| 3 | + | # | |
| 4 | + | # S3 ran Askama / described / described / Askama by hand. Running it by hand is | |
| 5 | + | # how a measurement ends up not comparable with the one before it: the mix, the | |
| 6 | + | # VU count and the described set all have to be identical across the four runs, | |
| 7 | + | # and only the middle two differ in what serves the screens. | |
| 8 | + | # | |
| 9 | + | # Two things this answers that S3's numbers do not: | |
| 10 | + | # | |
| 11 | + | # ACCUMULATION. S3 measured ONE described screen in an otherwise-Askama mix, | |
| 12 | + | # and its verdict rests on database connection occupancy, which is exactly the | |
| 13 | + | # quantity that grows with the number of described screens. Six are converted. | |
| 14 | + | # Default here is all six. | |
| 15 | + | # | |
| 16 | + | # THE MULTITHREADED REGIME. The two forum screens' work is an outbound HTTP | |
| 17 | + | # call, not a query, and the described version holds a BLOCKING-POOL thread | |
| 18 | + | # across it where Askama pays the same latency on a runtime worker. Sweep | |
| 19 | + | # MT_LATENCY to find the upstream latency at which that starts to cost | |
| 20 | + | # something. Sizing the pool is a config line either way, so the output worth | |
| 21 | + | # having is the threshold, not a pass/fail. | |
| 22 | + | # | |
| 23 | + | # Usage: | |
| 24 | + | # scripts/load-conversion-ab.sh # all six screens, 0ms upstream | |
| 25 | + | # MT_LATENCY=500 scripts/load-conversion-ab.sh # slow Multithreaded | |
| 26 | + | # SCREENS=user_ssh_keys scripts/load-conversion-ab.sh # reproduce S3's shape | |
| 27 | + | # | |
| 28 | + | # Read the report's `Rej` column before reading anything else. A rejected request | |
| 29 | + | # is fast, so a route answering 403 or 404 wins a latency comparison it never ran. | |
| 30 | + | ||
| 31 | + | set -euo pipefail | |
| 32 | + | cd "$(dirname "$0")/.." | |
| 33 | + | ||
| 34 | + | # Half the virtual users on the dashboard is far above a real day. It is chosen | |
| 35 | + | # so the described routes see enough concurrency to say anything at all; a | |
| 36 | + | # production-shaped mix puts one user on them and one user reaches no contention. | |
| 37 | + | : "${MIX:=anon:25,buyer:15,creator:10,dash:50}" | |
| 38 | + | : "${VUS:=60}" | |
| 39 | + | : "${DURATION:=60}" | |
| 40 | + | : "${RAMP:=10}" | |
| 41 | + | : "${SCREENS:=*}" | |
| 42 | + | : "${MT_LATENCY:=0}" | |
| 43 | + | : "${OUT:=target/load-ab}" | |
| 44 | + | ||
| 45 | + | export TEST_DATABASE_URL="${TEST_DATABASE_URL:-postgres:///postgres}" | |
| 46 | + | export LOAD_VUS="$VUS" | |
| 47 | + | export LOAD_DURATION_SECS="$DURATION" | |
| 48 | + | export LOAD_RAMP_SECS="$RAMP" | |
| 49 | + | export LOAD_MIX="$MIX" | |
| 50 | + | export LOAD_MT_LATENCY_MS="$MT_LATENCY" | |
| 51 | + | ||
| 52 | + | mkdir -p "$OUT" | |
| 53 | + | ||
| 54 | + | # Built once, outside the timed runs: a cold compile inside run 1 would show up | |
| 55 | + | # as run 1 being slower than run 4, which is the shape the alternation exists to | |
| 56 | + | # cancel out. | |
| 57 | + | echo "Building the load binary..." | |
| 58 | + | cargo test --test load --no-run --quiet | |
| 59 | + | ||
| 60 | + | run() { | |
| 61 | + | local label="$1" | |
| 62 | + | local screens="$2" | |
| 63 | + | local path="$OUT/$label.txt" | |
| 64 | + | echo | |
| 65 | + | echo "=== $label (QUASI_SCREENS=${screens:-<none>}) ===" | |
| 66 | + | QUASI_SCREENS="$screens" \ | |
| 67 | + | cargo test --test load -- --ignored --nocapture 2>&1 | tee "$path" | | |
| 68 | + | sed -n '/LOAD TEST REPORT/,$p' | |
| 69 | + | } | |
| 70 | + | ||
| 71 | + | # Askama, described, described, Askama. The pairs are inner and the controls are | |
| 72 | + | # outer so a drift in the box over the run (thermal, page cache, another process | |
| 73 | + | # arriving) lands on both sides rather than on one. | |
| 74 | + | run 1-askama "" | |
| 75 | + | run 2-described "$SCREENS" | |
| 76 | + | run 3-described "$SCREENS" | |
| 77 | + | run 4-askama "" | |
| 78 | + | ||
| 79 | + | echo | |
| 80 | + | echo "Four reports under $OUT/. Compare 2+3 against 1+4, per endpoint label." | |
| 81 | + | echo "The described screens report under 'HTMX described:*' in every run: same" | |
| 82 | + | echo "address either way, so the rows line up and only what serves them changed." |
| @@ -1,0 +1,151 @@ | |||
| 1 | + | //! How long it takes to get onto the blocking pool, sampled through the run. | |
| 2 | + | //! | |
| 3 | + | //! The description layer's whole runtime cost is that quasi's router is sync, so | |
| 4 | + | //! quasi-axum dispatches every described request on `spawn_blocking` and the | |
| 5 | + | //! handler holds that thread across its database or upstream call. This server | |
| 6 | + | //! shares that pool with argon2 hashing, content exports and the file scanner, | |
| 7 | + | //! so the question a load run has to answer is whether described requests starve | |
| 8 | + | //! them. | |
| 9 | + | //! | |
| 10 | + | //! Throughput does not answer it and neither does per-endpoint latency: a pool | |
| 11 | + | //! at its limit shows up as everything getting slower together, which is | |
| 12 | + | //! indistinguishable from the box being busy. What separates them is DISPATCH | |
| 13 | + | //! DELAY — how long a fresh `spawn_blocking` waits before its closure starts | |
| 14 | + | //! running. On an unsaturated pool that is microseconds whatever the load; | |
| 15 | + | //! it only grows when every thread is occupied and tokio has to wait for one or | |
| 16 | + | //! spawn another. | |
| 17 | + | //! | |
| 18 | + | //! `RuntimeMetrics::num_blocking_threads` would say it directly, but it is | |
| 19 | + | //! behind `tokio_unstable` and this tree does not build with it. This measures | |
| 20 | + | //! the same saturation from the outside, with no cfg and no build flag: the | |
| 21 | + | //! probe is what a request arriving at that instant would have experienced. | |
| 22 | + | //! | |
| 23 | + | //! Read it alongside the argon2 signup mean (`POST /join/step/account`), which | |
| 24 | + | //! is the same fact seen from the other end: signup is the heaviest genuine | |
| 25 | + | //! blocking-pool user in the mix, so if described requests are crowding the pool | |
| 26 | + | //! both numbers move together. | |
| 27 | + | ||
| 28 | + | use std::sync::{Arc, Mutex}; | |
| 29 | + | use std::time::{Duration, Instant}; | |
| 30 | + | ||
| 31 | + | /// Samples dispatch delay until it is stopped. | |
| 32 | + | pub(super) struct BlockingProbe { | |
| 33 | + | samples: Arc<Mutex<Vec<Duration>>>, | |
| 34 | + | sampler: tokio::task::JoinHandle<()>, | |
| 35 | + | } | |
| 36 | + | ||
| 37 | + | impl BlockingProbe { | |
| 38 | + | /// Start sampling every `interval`. | |
| 39 | + | /// | |
| 40 | + | /// One in-flight sample at a time, on purpose: the probe measures the pool, | |
| 41 | + | /// and a probe that queued its own work would be measuring itself. | |
| 42 | + | pub(super) fn start(interval: Duration) -> Self { | |
| 43 | + | let samples = Arc::new(Mutex::new(Vec::new())); | |
| 44 | + | let into = Arc::clone(&samples); | |
| 45 | + | ||
| 46 | + | let sampler = tokio::spawn(async move { | |
| 47 | + | loop { | |
| 48 | + | let queued = Instant::now(); | |
| 49 | + | // The closure records how long it waited to BEGIN, not how long | |
| 50 | + | // it ran. It does nothing else, so the two are not confusable. | |
| 51 | + | let waited = tokio::task::spawn_blocking(move || queued.elapsed()).await; | |
| 52 | + | if let Ok(waited) = waited { | |
| 53 | + | into.lock().unwrap().push(waited); | |
| 54 | + | } | |
| 55 | + | tokio::time::sleep(interval).await; | |
| 56 | + | } | |
| 57 | + | }); | |
| 58 | + | ||
| 59 | + | BlockingProbe { samples, sampler } | |
| 60 | + | } | |
| 61 | + | ||
| 62 | + | /// Stop sampling and summarise. | |
| 63 | + | pub(super) fn finish(self) -> BlockingReport { | |
| 64 | + | self.sampler.abort(); | |
| 65 | + | let mut samples = std::mem::take(&mut *self.samples.lock().unwrap()); | |
| 66 | + | samples.sort(); | |
| 67 | + | BlockingReport::of(&samples) | |
| 68 | + | } | |
| 69 | + | } | |
| 70 | + | ||
| 71 | + | /// What the probe saw over one run. | |
| 72 | + | pub(super) struct BlockingReport { | |
| 73 | + | pub count: usize, | |
| 74 | + | pub p50: Duration, | |
| 75 | + | pub p95: Duration, | |
| 76 | + | pub p99: Duration, | |
| 77 | + | pub max: Duration, | |
| 78 | + | /// Samples that waited longer than [`STALL`]. The number that matters: on a | |
| 79 | + | /// pool with headroom it is zero, and it stops being zero before mean | |
| 80 | + | /// latency moves at all. | |
| 81 | + | pub stalls: usize, | |
| 82 | + | } | |
| 83 | + | ||
| 84 | + | /// The line above which a dispatch delay is a queue rather than scheduling | |
| 85 | + | /// noise. Dispatch onto an idle pool is single-digit microseconds; a millisecond | |
| 86 | + | /// means the sample waited for a thread. | |
| 87 | + | pub(super) const STALL: Duration = Duration::from_millis(1); | |
| 88 | + | ||
| 89 | + | impl BlockingReport { | |
| 90 | + | /// Print the probe's summary under the endpoint table. | |
| 91 | + | pub(super) fn print(&self) { | |
| 92 | + | println!(" Blocking-pool dispatch delay ({} samples):", self.count); | |
| 93 | + | if self.count == 0 { | |
| 94 | + | println!(" no samples"); | |
| 95 | + | println!(); | |
| 96 | + | return; | |
| 97 | + | } | |
| 98 | + | println!( | |
| 99 | + | " p50 {} p95 {} p99 {} max {}", | |
| 100 | + | format_dur(self.p50), | |
| 101 | + | format_dur(self.p95), | |
| 102 | + | format_dur(self.p99), | |
| 103 | + | format_dur(self.max), | |
| 104 | + | ); | |
| 105 | + | println!( | |
| 106 | + | " stalls over {}: {} ({:.1}%)", | |
| 107 | + | format_dur(STALL), | |
| 108 | + | self.stalls, | |
| 109 | + | self.stalls as f64 / self.count as f64 * 100.0, | |
| 110 | + | ); | |
| 111 | + | if self.stalls == 0 { | |
| 112 | + | println!(" the pool had headroom throughout"); | |
| 113 | + | } | |
| 114 | + | println!(); | |
| 115 | + | } | |
| 116 | + | ||
| 117 | + | fn of(sorted: &[Duration]) -> Self { | |
| 118 | + | if sorted.is_empty() { | |
| 119 | + | return BlockingReport { | |
| 120 | + | count: 0, | |
| 121 | + | p50: Duration::ZERO, | |
| 122 | + | p95: Duration::ZERO, | |
| 123 | + | p99: Duration::ZERO, | |
| 124 | + | max: Duration::ZERO, | |
| 125 | + | stalls: 0, | |
| 126 | + | }; | |
| 127 | + | } | |
| 128 | + | let at = |pct: f64| { | |
| 129 | + | let idx = ((sorted.len() as f64 * pct) as usize).min(sorted.len() - 1); | |
| 130 | + | sorted[idx] | |
| 131 | + | }; | |
| 132 | + | BlockingReport { | |
| 133 | + | count: sorted.len(), | |
| 134 | + | p50: at(0.50), | |
| 135 | + | p95: at(0.95), | |
| 136 | + | p99: at(0.99), | |
| 137 | + | max: sorted[sorted.len() - 1], | |
| 138 | + | stalls: sorted.iter().filter(|d| **d > STALL).count(), | |
| 139 | + | } | |
| 140 | + | } | |
| 141 | + | } | |
| 142 | + | ||
| 143 | + | /// Same rendering the endpoint table uses, so the two read against each other. | |
| 144 | + | fn format_dur(d: Duration) -> String { | |
| 145 | + | let us = d.as_micros(); | |
| 146 | + | if us >= 1000 { | |
| 147 | + | format!("{:.1}ms", us as f64 / 1000.0) | |
| 148 | + | } else { | |
| 149 | + | format!("{us}us") | |
| 150 | + | } | |
| 151 | + | } |
| @@ -1,0 +1,94 @@ | |||
| 1 | + | //! A stand-in Multithreaded, so the forum-membership screens can be put under | |
| 2 | + | //! load without a real instance. | |
| 3 | + | //! | |
| 4 | + | //! The two described forum screens (`src/quasi/forum_memberships.rs`) are the | |
| 5 | + | //! only ones whose work is an outbound HTTP call rather than a database round | |
| 6 | + | //! trip, and quasi's router is sync: the described version holds a | |
| 7 | + | //! BLOCKING-POOL thread for that whole call, where the Askama version pays the | |
| 8 | + | //! same latency on a runtime worker. Every runtime number about the description | |
| 9 | + | //! layer so far came from sub-millisecond sqlx queries, so nothing has measured | |
| 10 | + | //! that regime. | |
| 11 | + | //! | |
| 12 | + | //! What makes it measurable is that the upstream latency becomes the variable. | |
| 13 | + | //! This stub sleeps `LOAD_MT_LATENCY_MS` before answering, so a run can be | |
| 14 | + | //! repeated at 0ms, 50ms, 500ms and 5s and the question becomes the threshold at | |
| 15 | + | //! which occupancy starts to cost something, rather than a pass/fail against | |
| 16 | + | //! whatever a real instance happened to be doing that afternoon. | |
| 17 | + | //! | |
| 18 | + | //! It sleeps on the runtime rather than blocking, which is the point: the stub | |
| 19 | + | //! must not itself consume the pool being measured. | |
| 20 | + | ||
| 21 | + | use axum::Router; | |
| 22 | + | use axum::extract::Path; | |
| 23 | + | use axum::routing::get; | |
| 24 | + | use std::time::Duration; | |
| 25 | + | ||
| 26 | + | /// A running stub, alive until it is dropped. | |
| 27 | + | pub(super) struct MtStub { | |
| 28 | + | /// What to hand `IntegrationsConfig::mt_base_url`. | |
| 29 | + | base_url: String, | |
| 30 | + | server: tokio::task::JoinHandle<()>, | |
| 31 | + | } | |
| 32 | + | ||
| 33 | + | impl MtStub { | |
| 34 | + | /// Bind on an ephemeral port and start answering. | |
| 35 | + | /// | |
| 36 | + | /// `latency` is slept before every answer. `memberships` is how many rows | |
| 37 | + | /// come back, which decides how much describing the screen has to do once | |
| 38 | + | /// the call lands. | |
| 39 | + | pub(super) async fn start(latency: Duration, memberships: usize) -> Self { | |
| 40 | + | let app = Router::new().route( | |
| 41 | + | "/api/user/{user_id}/summary", | |
| 42 | + | get(move |Path(_user_id): Path<String>| async move { | |
| 43 | + | if !latency.is_zero() { | |
| 44 | + | tokio::time::sleep(latency).await; | |
| 45 | + | } | |
| 46 | + | axum::Json(summary(memberships)) | |
| 47 | + | }), | |
| 48 | + | ); | |
| 49 | + | ||
| 50 | + | let listener = tokio::net::TcpListener::bind("127.0.0.1:0") | |
| 51 | + | .await | |
| 52 | + | .expect("MT stub could not bind"); | |
| 53 | + | let addr = listener.local_addr().expect("MT stub has no address"); | |
| 54 | + | ||
| 55 | + | let server = tokio::spawn(async move { | |
| 56 | + | // Ignore the result: the runner drops the stub at the end of the | |
| 57 | + | // run and a shutdown error there is not a finding. | |
| 58 | + | let _ = axum::serve(listener, app).await; | |
| 59 | + | }); | |
| 60 | + | ||
| 61 | + | MtStub { | |
| 62 | + | base_url: format!("http://{addr}"), | |
| 63 | + | server, | |
| 64 | + | } | |
| 65 | + | } | |
| 66 | + | ||
| 67 | + | pub(super) fn base_url(&self) -> String { | |
| 68 | + | self.base_url.clone() | |
| 69 | + | } | |
| 70 | + | } | |
| 71 | + | ||
| 72 | + | impl Drop for MtStub { | |
| 73 | + | fn drop(&mut self) { | |
| 74 | + | self.server.abort(); | |
| 75 | + | } | |
| 76 | + | } | |
| 77 | + | ||
| 78 | + | /// The shape `fetch` in `src/quasi/forum_memberships.rs` reads, and the same | |
| 79 | + | /// shape the Askama handler in `routes::pages::public::landing` reads. Both | |
| 80 | + | /// sides of the comparison parse this, so a change to it moves both together. | |
| 81 | + | fn summary(memberships: usize) -> serde_json::Value { | |
| 82 | + | let rows: Vec<serde_json::Value> = (0..memberships) | |
| 83 | + | .map(|i| { | |
| 84 | + | serde_json::json!({ | |
| 85 | + | "community_slug": format!("community-{i}"), | |
| 86 | + | "community_name": format!("Community {i}"), | |
| 87 | + | "role": if i == 0 { "Moderator" } else { "Member" }, | |
| 88 | + | "post_count": 40 + i as i64, | |
| 89 | + | "joined_at": "2026-03-14T09:00:00Z", | |
| 90 | + | }) | |
| 91 | + | }) | |
| 92 | + | .collect(); | |
| 93 | + | serde_json::json!({ "memberships": rows }) | |
| 94 | + | } |