max / makenotwork

6.7 KB · 159 lines History Blame Raw

1	#!/usr/bin/env bash
2	# The alternating-pairs protocol for measuring the description layer.
3	#
4	# WHAT THE TWO SIDES ARE, AND WHY THEY CHANGED
5	#
6	# This script used to serve one side through Askama and the other through the
7	# description layer, switching with `QUASI_SCREENS`. That switch was deleted in
8	# `64b33b26`: `quasi::mounts` is unconditional, the route tables register no
9	# Askama counterpart, and there is nothing left to revert to. Nothing reads the
10	# variable, so every run the old script made measured the same binary four times
11	# and reported a delta of zero -- which reads as "the conversion bought nothing"
12	# and means "the experiment did not run".
13	#
14	# So the two sides are two BUILDS, at two revisions. That is also the comparison
15	# there is now a question about: every described screen renders through quasi
16	# either way, and what differs is whether its markup is derived at build time
17	# and walked (the residual seam, quasicoherent `793d99dd`) or built as a `Node`
18	# and rendered per request.
19	#
20	# Both trees are built and run on astra, one after another, on the same box with
21	# the same database. fw13 is where the day's editing happens and a load
22	# measurement wants a quiet machine.
23	#
24	# READ THE CONTROLS FIRST. Not every described screen is on the seam --
25	# `ssh-keys` and `analytics` are not, for reasons filed against quasicoherent --
26	# so they report in both runs as themselves. A run where the seam screens move
27	# and the controls move with them measured the box, not the seam.
28	#
29	# Also read the report's `Rej` column before anything else. A rejected request is
30	# fast, so a route answering 403 or 404 wins a latency comparison it never ran.
31	#
32	# THE PROFILE IS RELEASE, and it is here rather than in shell history so a later
33	# reader knows which one produced the numbers. Render cost is the quantity being
34	# compared, and in debug it is dominated by unoptimized rendering.
35	#
36	# THE BOX IS aarch64. Absolute figures here are not comparable with the
37	# 2026-08-14 sweep's, whatever box that used; what is comparable is the
38	# before/after within one run of this script.
39	#
40	# Usage:
41	# scripts/load-conversion-ab.sh # HEAD against the pre-seam tree
42	# BEFORE_MNW=abc1234 BEFORE_QUASI=def5678 scripts/load-conversion-ab.sh
43	# MT_LATENCY=500 scripts/load-conversion-ab.sh # slow Multithreaded
44	#
45	# Sweep MT_LATENCY to find the upstream latency at which holding a blocking-pool
46	# thread across an outbound call starts to cost something: the two forum screens
47	# are the only described ones whose work is an HTTP call rather than a query.
48	# Sizing the pool is a config line either way, so the output worth having is the
49	# threshold, not a pass/fail.
50
51	set -euo pipefail
52	cd "$(dirname "$0")/.."
53
54	CODE="$(cd ../.. && pwd)"
55
56	# The tree before the seam took the screens this measures. `c554d7ed` is the
57	# last MNW commit with four screens on the seam rather than thirteen, and
58	# `b5a21fe` is the quasi it was written against.
59	: "${BEFORE_MNW:=c554d7ed}"
60	: "${BEFORE_QUASI:=b5a21fe}"
61	: "${HOST:=astra}"
62	: "${ROOT:=ab-seam}"
63	: "${MIX:=anon:25,buyer:15,creator:10,dash:50}"
64	: "${VUS:=60}"
65	: "${DURATION:=60}"
66	: "${RAMP:=10}"
67	: "${MT_LATENCY:=0}"
68	: "${OUT:=target/load-ab}"
69
70	AFTER_MNW="$(git -C "$CODE/MNW" rev-parse --short HEAD)"
71	AFTER_QUASI="$(git -C "$CODE/quasi" rev-parse --short HEAD)"
72	SHARED="$CODE/_private/infra/bootstrap/cargo-config.shared.toml"
73
74	echo "before: MNW $BEFORE_MNW quasi $BEFORE_QUASI"
75	echo "after: MNW $AFTER_MNW quasi $AFTER_QUASI"
76
77	# Pristine trees from git rather than rsynced working copies. The two sides
78	# differ by a revision and nothing else, and an uncommitted edit in either would
79	# make the comparison say something other than what it claims.
80	REMOTE_HOME="$(ssh "$HOST" 'echo $HOME')"
81	ssh "$HOST" "rm -rf ~/$ROOT && mkdir -p ~/$ROOT/{before,after}/{MNW,quasi,Libraries,.cargo}"
82
83	send() { git -C "$CODE/$1" archive "$2" \| ssh "$HOST" "tar -x -C ~/$ROOT/$3"; }
84	send MNW "$BEFORE_MNW" before/MNW
85	send quasi "$BEFORE_QUASI" before/quasi
86	send MNW "$AFTER_MNW" after/MNW
87	send quasi "$AFTER_QUASI" after/quasi
88
89	for side in before after; do
90	# The siblings the `[patch]` block redirects that neither revision touches.
91	for repo in synckit Libraries/docengine Libraries/quasi-type; do
92	rsync -aq --delete --exclude 'target/' --exclude '.git/' --exclude 'mutants.out/' \
93	"$CODE/$repo/" "$HOST:$ROOT/$side/$repo/"
94	done
95	# `[patch]` with absolute paths. The local pair resolves twice -- `include`
96	# against the symlink's directory, then the included file's paths against
97	# the parent of its own -- and neither survives being copied to another
98	# layout. `[build]` and `[profile]` are fw13's numbers.
99	awk -v root="$REMOTE_HOME/$ROOT/$side" '
100	/^\[/ { keep = ($0 ~ /^\[patch/) }
101	keep { gsub(/path = "\.\.\/\.\.\//, "path = \"" root "/"); print }
102	' "$SHARED" \| ssh "$HOST" "cat > ~/$ROOT/$side/.cargo/config.toml"
103	done
104
105	# A patch that silently did not apply is the one failure a green run cannot tell
106	# you about: the side would be built against the published quasi rather than the
107	# revision this script just shipped.
108	for side in before after; do
109	[ "$(ssh "$HOST" "grep -c 'path = \"/' ~/$ROOT/$side/.cargo/config.toml")" -gt 0 ] \|\|
110	{ echo "the $side cargo config carries no absolute patch paths" >&2; exit 1; }
111	done
112
113	ssh "$HOST" "cat > ~/$ROOT/run.sh" <<REMOTE
114	set -u
115	ROOT="\$HOME/$ROOT"
116	OUT="\$ROOT/out"
117	mkdir -p "\$OUT"
118	export PATH="\$HOME/.cargo/bin:\$PATH"
119	export TEST_DATABASE_URL="\${TEST_DATABASE_URL:-postgres:///postgres}"
120	export LOAD_VUS=$VUS
121	export LOAD_DURATION_SECS=$DURATION
122	export LOAD_RAMP_SECS=$RAMP
123	export LOAD_MIX="$MIX"
124	export LOAD_MT_LATENCY_MS=$MT_LATENCY
125
126	# Built outside the timed runs: a cold compile inside run 1 would show up as run
127	# 1 being slower than run 4, which is the shape the alternation cancels out.
128	for side in before after; do
129	echo "=== building \$side ==="
130	( cd "\$ROOT/\$side/MNW/server" && cargo test --release --test load --no-run --quiet ) \
131	> "\$OUT/build-\$side.log" 2>&1 \|\| { echo "build \$side FAILED"; exit 1; }
132	done
133
134	run() {
135	echo "=== run \$1 (\$2) ==="
136	( cd "\$ROOT/\$2/MNW/server" && cargo test --release --test load -- --ignored --nocapture ) \
137	> "\$OUT/\$1-\$2.txt" 2>&1
138	echo "run \$1 exit=\$?"
139	}
140
141	# before, after, after, before. The pairs are inner and the controls outer so a
142	# drift in the box over the run lands on both sides rather than on one.
143	run 1 before
144	run 2 after
145	run 3 after
146	run 4 before
147	echo DONE
148	REMOTE
149
150	echo "Running on $HOST. Four reports will land in ~/$ROOT/out/."
151	ssh "$HOST" "bash ~/$ROOT/run.sh"
152
153	mkdir -p "$OUT"
154	scp -q "$HOST:$ROOT/out/*.txt" "$OUT/" \|\| true
155	echo
156	echo "Reports under $OUT/. Compare 2+3 against 1+4, per endpoint label."
157	echo "The described screens report under 'HTMX described:*' in both, so the rows"
158	echo "line up and only what serves them changed."
159