max / makenotwork

6.5 KB · 141 lines History Blame Raw

1	#!/bin/bash
2	# Run the server's expensive suites on astra instead of on fw13.
3	#
4	# fw13 is 12 cores against 14 GB, and `~/Code/.cargo/config.toml` caps cargo at
5	# `jobs = 6` because a heavy crate costs rustc ~3 GB and the box swaps rather
6	# than compiles above that. astra is 96 cores and 125 GB, usually idle outside
7	# the 03:20 sweep. The server's integration suite is the one workload where that
8	# difference is worth a round trip: ~1,340 tests and a build of the largest crate
9	# in the tree.
10	#
11	# This is for the expensive runs only. A filtered local run beats this every
12	# time: `cargo test --test integration described_screens` answers in seconds,
13	# where anything here pays a sync plus a build. Reach for this when the question
14	# is "is the whole suite green", not "did my change work".
15	#
16	# WHAT IT SYNCS, AND WHY IT IS NOT JUST THE SERVER. The point of the script is
17	# testing work that is not committed yet -- astra's own checkout only ever has
18	# what has been pushed, which is exactly what made it useless for a day of
19	# editing quasi and the server together. So it rsyncs the working trees, and it
20	# has to send every repo the local `[patch]` block redirects, or astra silently
21	# builds the published quasi against your edited server and reports green on a
22	# combination that does not exist. That is worse than no script.
23	#
24	# CAVEAT THAT DOES NOT GO AWAY: astra is aarch64. Green here is not green on
25	# x86_64. It catches logic, not size, alignment or intrinsics assumptions.
26	#
27	# Usage:
28	# scripts/test-on-astra.sh # the integration suite
29	# scripts/test-on-astra.sh --lib # or any cargo test args
30	# scripts/test-on-astra.sh --test integration described_screens
31	#
32	# Environment:
33	# ASTRA_HOST ssh host (default: astra)
34	# ASTRA_DIR remote directory, under $HOME (default: mnw-test)
35	# ASTRA_THREADS test threads (default: 16, see the note below)
36
37	set -u
38	set -o pipefail
39
40	HOST="${ASTRA_HOST:-astra}"
41	DEST="${ASTRA_DIR:-mnw-test}"
42	# Postgres, not cores, is what bounds this suite: every test takes a
43	# `CREATE DATABASE ... TEMPLATE` clone, and astra's cluster is one instance like
44	# any other. Memory note `reference_astra_tests` records ~9/734 workflow tests
45	# hitting "connection slots reserved for SUPERUSER" under an unbounded run. So
46	# this is deliberately nowhere near 96, and raising it trades wall-clock for
47	# flakes that look like real failures.
48	THREADS="${ASTRA_THREADS:-16}"
49
50	ROOT="$(cd "$(dirname "$0")/../.." && pwd)" # ~/Code/MNW
51	CODE="$(cd "$ROOT/.." && pwd)" # ~/Code
52	CARGO_CONFIG="$CODE/.cargo/config.toml"
53
54	# Every repo that has to travel. MNW carries the server and the `shared/*` path
55	# dependencies; the rest are what `[patch]` redirects. Keep in step with
56	# `$CARGO_CONFIG` -- the check below fails the run if they drift.
57	REPOS=(MNW quasi synckit Libraries/docengine)
58
59	say() { printf '%s\n' "$*" >&2; }
60	die() { say "test-on-astra: $*"; exit 1; }
61
62	[ -f "$CARGO_CONFIG" ] \|\| die "no $CARGO_CONFIG; nothing says where the siblings are"
63
64	# Drift guard. A `[patch]` entry pointing outside the synced set would build on
65	# astra from the published crate instead of the working tree, and the run would
66	# be green about code nobody has.
67	missing=""
68	while read -r patched; do
69	covered=""
70	for repo in "${REPOS[@]}"; do
71	case "$patched" in "$repo"/*\|"$repo") covered=1; break;; esac
72	done
73	[ -n "$covered" ] \|\| missing="$missing $patched"
74	done < <(grep -oP '(?<=path = ")[^"]+' "$CARGO_CONFIG" \| sort -u)
75	[ -z "$missing" ] \|\| die "these patched paths are not in REPOS:$missing"
76
77	say "test-on-astra: syncing to $HOST:~/$DEST"
78	for repo in "${REPOS[@]}"; do
79	[ -d "$CODE/$repo" ] \|\| die "$CODE/$repo does not exist"
80	ssh "$HOST" "mkdir -p ~/$DEST/$(dirname "$repo")" \|\| die "cannot reach $HOST"
81	# --delete so a file removed locally is removed there; without it a deleted
82	# test keeps passing on astra forever. target/ and .git are excluded and
83	# therefore untouched by it, which is what keeps the remote build
84	# incremental across runs.
85	# `mutants.out/` is 218 MB of cargo-mutants detritus in synckit-client alone,
86	# which is twice everything else here put together.
87	rsync -a --delete \
88	--exclude 'target/' --exclude '.git/' --exclude 'node_modules/' \
89	--exclude 'mutants.out/' \
90	"$CODE/$repo/" "$HOST:$DEST/$repo/" \
91	\|\| die "rsync of $repo failed"
92	done
93
94	# The patch block, verbatim, because its paths are relative to the config file's
95	# own parent and the layout above mirrors ~/Code exactly. `[build]` and
96	# `[profile.*]` are dropped: `jobs = 6` is a statement about fw13's 14 GB and
97	# would leave 90 of astra's cores idle.
98	awk '
99	/^\[/ { keep = ($0 !~ /^\[build\]/ && $0 !~ /^\[profile/) }
100	keep { print }
101	' "$CARGO_CONFIG" \| ssh "$HOST" "mkdir -p ~/$DEST/.cargo && cat > ~/$DEST/.cargo/config.toml" \
102	\|\| die "could not write the remote cargo config"
103
104	# `--features fast-tests` is not an optimisation to argue about: it swaps argon2
105	# from production parameters (46 MiB, 2 iterations, ~600ms) to test ones (8 MiB,
106	# 1 iteration, ~10ms), and the suite performs ~874 hashes between signup, login
107	# and admin setup. That is ~9 CPU-minutes of key derivation proving nothing. It
108	# is the only thing the feature gates; see the note on `hash_password`.
109	ARGS=("$@")
110	[ ${#ARGS[@]} -gt 0 ] \|\| ARGS=(--test integration)
111
112	say "test-on-astra: building and running ($THREADS threads, aarch64)"
113	ssh "$HOST" bash -s -- "$DEST" "$THREADS" "${ARGS[@]}" <<'REMOTE'
114	set -u
115	DEST="$1"; shift
116	THREADS="$1"; shift
117
118	cd "$HOME/$DEST/MNW/server" \|\| exit 1
119	# rustup's toolchain, not the distro's: /usr/bin/rustc is older than the
120	# dependency tree's floor and fails with a resolver error that names a crate
121	# rather than the compiler.
122	export PATH="$HOME/.cargo/bin:$PATH"
123	# The default 1024 is exhausted by the per-test database pool.
124	ulimit -n 65536
125	# Socket form, no host: astra's postgres has no TCP listener, and the harness's
126	# `replace_db_name` splits on the last `/`, so a `?host=` query string is
127	# mangled rather than honoured.
128	export TEST_DATABASE_URL=postgres:///postgres
129	# Verify against the committed `.sqlx` metadata rather than a live database.
130	# astra's `makenotwork` has drifted from the schema in the tree, and pointing
131	# the compile-time macros at it fails the build with errors that read like code
132	# faults. Drift in the metadata is itself a finding.
133	export SQLX_OFFLINE=true
134
135	started=$(date +%s)
136	cargo test --features fast-tests "$@" -- --test-threads="$THREADS"
137	status=$?
138	echo "test-on-astra: $(( $(date +%s) - started ))s wall on astra, aarch64" >&2
139	exit $status
140	REMOTE
141