max / makenotwork

3.8 KB · 90 lines History Blame Raw

1	#!/usr/bin/env bash
2	# The alternating-pairs protocol for measuring the description layer.
3	#
4	# S3 ran Askama / described / described / Askama by hand. Running it by hand is
5	# how a measurement ends up not comparable with the one before it: the mix, the
6	# VU count and the described set all have to be identical across the four runs,
7	# and only the middle two differ in what serves the screens.
8	#
9	# Two things this answers that S3's numbers do not:
10	#
11	# ACCUMULATION. S3 measured ONE described screen in an otherwise-Askama mix,
12	# and its verdict rests on database connection occupancy, which is exactly the
13	# quantity that grows with the number of described screens. Six are converted.
14	# Default here is all six.
15	#
16	# THE MULTITHREADED REGIME. The two forum screens' work is an outbound HTTP
17	# call, not a query, and the described version holds a BLOCKING-POOL thread
18	# across it where Askama pays the same latency on a runtime worker. Sweep
19	# MT_LATENCY to find the upstream latency at which that starts to cost
20	# something. Sizing the pool is a config line either way, so the output worth
21	# having is the threshold, not a pass/fail.
22	#
23	# Usage:
24	# scripts/load-conversion-ab.sh # all six screens, 0ms upstream
25	# MT_LATENCY=500 scripts/load-conversion-ab.sh # slow Multithreaded
26	# SCREENS=user_ssh_keys scripts/load-conversion-ab.sh # reproduce S3's shape
27	#
28	# Read the report's `Rej` column before reading anything else. A rejected request
29	# is fast, so a route answering 403 or 404 wins a latency comparison it never ran.
30	#
31	# THE PROFILE IS RELEASE, and it is here rather than in shell history so a later
32	# reader knows which one produced the numbers. Render cost is the quantity being
33	# compared, and in debug it is dominated by unoptimized rendering, so a debug
34	# sweep measures rustc's -O0 output more than it measures the description layer.
35	# S3's section in the wiki does not record its profile, so treat any comparison
36	# against S3's figures as indicative rather than exact.
37
38	set -euo pipefail
39	cd "$(dirname "$0")/.."
40
41	# Half the virtual users on the dashboard is far above a real day. It is chosen
42	# so the described routes see enough concurrency to say anything at all; a
43	# production-shaped mix puts one user on them and one user reaches no contention.
44	: "${MIX:=anon:25,buyer:15,creator:10,dash:50}"
45	: "${VUS:=60}"
46	: "${DURATION:=60}"
47	: "${RAMP:=10}"
48	: "${SCREENS:=*}"
49	: "${MT_LATENCY:=0}"
50	: "${OUT:=target/load-ab}"
51
52	export TEST_DATABASE_URL="${TEST_DATABASE_URL:-postgres:///postgres}"
53	export LOAD_VUS="$VUS"
54	export LOAD_DURATION_SECS="$DURATION"
55	export LOAD_RAMP_SECS="$RAMP"
56	export LOAD_MIX="$MIX"
57	export LOAD_MT_LATENCY_MS="$MT_LATENCY"
58
59	mkdir -p "$OUT"
60
61	# Built once, outside the timed runs: a cold compile inside run 1 would show up
62	# as run 1 being slower than run 4, which is the shape the alternation exists to
63	# cancel out.
64	echo "Building the load binary (release)..."
65	cargo test --release --test load --no-run --quiet
66
67	run() {
68	local label="$1"
69	local screens="$2"
70	local path="$OUT/$label.txt"
71	echo
72	echo "=== $label (QUASI_SCREENS=${screens:-<none>}) ==="
73	QUASI_SCREENS="$screens" \
74	cargo test --release --test load -- --ignored --nocapture 2>&1 \| tee "$path" \|
75	sed -n '/LOAD TEST REPORT/,$p'
76	}
77
78	# Askama, described, described, Askama. The pairs are inner and the controls are
79	# outer so a drift in the box over the run (thermal, page cache, another process
80	# arriving) lands on both sides rather than on one.
81	run 1-askama ""
82	run 2-described "$SCREENS"
83	run 3-described "$SCREENS"
84	run 4-askama ""
85
86	echo
87	echo "Four reports under $OUT/. Compare 2+3 against 1+4, per endpoint label."
88	echo "The described screens report under 'HTMX described:*' in every run: same"
89	echo "address either way, so the rows line up and only what serves them changed."
90