|
1 |
+ |
#!/bin/bash
|
|
2 |
+ |
# Run the server's expensive suites on astra instead of on fw13.
|
|
3 |
+ |
#
|
|
4 |
+ |
# fw13 is 12 cores against 14 GB, and `~/Code/.cargo/config.toml` caps cargo at
|
|
5 |
+ |
# `jobs = 6` because a heavy crate costs rustc ~3 GB and the box swaps rather
|
|
6 |
+ |
# than compiles above that. astra is 96 cores and 125 GB, usually idle outside
|
|
7 |
+ |
# the 03:20 sweep. The server's integration suite is the one workload where that
|
|
8 |
+ |
# difference is worth a round trip: ~1,340 tests and a build of the largest crate
|
|
9 |
+ |
# in the tree.
|
|
10 |
+ |
#
|
|
11 |
+ |
# **This is for the expensive runs only.** A filtered local run beats this every
|
|
12 |
+ |
# time: `cargo test --test integration described_screens` answers in seconds,
|
|
13 |
+ |
# where anything here pays a sync plus a build. Reach for this when the question
|
|
14 |
+ |
# is "is the whole suite green", not "did my change work".
|
|
15 |
+ |
#
|
|
16 |
+ |
# WHAT IT SYNCS, AND WHY IT IS NOT JUST THE SERVER. The point of the script is
|
|
17 |
+ |
# testing work that is *not committed yet* -- astra's own checkout only ever has
|
|
18 |
+ |
# what has been pushed, which is exactly what made it useless for a day of
|
|
19 |
+ |
# editing quasi and the server together. So it rsyncs the working trees, and it
|
|
20 |
+ |
# has to send every repo the local `[patch]` block redirects, or astra silently
|
|
21 |
+ |
# builds the published quasi against your edited server and reports green on a
|
|
22 |
+ |
# combination that does not exist. That is worse than no script.
|
|
23 |
+ |
#
|
|
24 |
+ |
# CAVEAT THAT DOES NOT GO AWAY: astra is aarch64. Green here is not green on
|
|
25 |
+ |
# x86_64. It catches logic, not size, alignment or intrinsics assumptions.
|
|
26 |
+ |
#
|
|
27 |
+ |
# Usage:
|
|
28 |
+ |
# scripts/test-on-astra.sh # the integration suite
|
|
29 |
+ |
# scripts/test-on-astra.sh --lib # or any cargo test args
|
|
30 |
+ |
# scripts/test-on-astra.sh --test integration described_screens
|
|
31 |
+ |
#
|
|
32 |
+ |
# Environment:
|
|
33 |
+ |
# ASTRA_HOST ssh host (default: astra)
|
|
34 |
+ |
# ASTRA_DIR remote directory, under $HOME (default: mnw-test)
|
|
35 |
+ |
# ASTRA_THREADS test threads (default: 16, see the note below)
|
|
36 |
+ |
|
|
37 |
+ |
set -u
|
|
38 |
+ |
set -o pipefail
|
|
39 |
+ |
|
|
40 |
+ |
HOST="${ASTRA_HOST:-astra}"
|
|
41 |
+ |
DEST="${ASTRA_DIR:-mnw-test}"
|
|
42 |
+ |
# Postgres, not cores, is what bounds this suite: every test takes a
|
|
43 |
+ |
# `CREATE DATABASE ... TEMPLATE` clone, and astra's cluster is one instance like
|
|
44 |
+ |
# any other. Memory note `reference_astra_tests` records ~9/734 workflow tests
|
|
45 |
+ |
# hitting "connection slots reserved for SUPERUSER" under an unbounded run. So
|
|
46 |
+ |
# this is deliberately nowhere near 96, and raising it trades wall-clock for
|
|
47 |
+ |
# flakes that look like real failures.
|
|
48 |
+ |
THREADS="${ASTRA_THREADS:-16}"
|
|
49 |
+ |
|
|
50 |
+ |
ROOT="$(cd "$(dirname "$0")/../.." && pwd)" # ~/Code/MNW
|
|
51 |
+ |
CODE="$(cd "$ROOT/.." && pwd)" # ~/Code
|
|
52 |
+ |
CARGO_CONFIG="$CODE/.cargo/config.toml"
|
|
53 |
+ |
|
|
54 |
+ |
# Every repo that has to travel. MNW carries the server and the `shared/*` path
|
|
55 |
+ |
# dependencies; the rest are what `[patch]` redirects. Keep in step with
|
|
56 |
+ |
# `$CARGO_CONFIG` -- the check below fails the run if they drift.
|
|
57 |
+ |
REPOS=(MNW quasi synckit Libraries/docengine)
|
|
58 |
+ |
|
|
59 |
+ |
say() { printf '%s\n' "$*" >&2; }
|
|
60 |
+ |
die() { say "test-on-astra: $*"; exit 1; }
|
|
61 |
+ |
|
|
62 |
+ |
[ -f "$CARGO_CONFIG" ] || die "no $CARGO_CONFIG; nothing says where the siblings are"
|
|
63 |
+ |
|
|
64 |
+ |
# Drift guard. A `[patch]` entry pointing outside the synced set would build on
|
|
65 |
+ |
# astra from the published crate instead of the working tree, and the run would
|
|
66 |
+ |
# be green about code nobody has.
|
|
67 |
+ |
missing=""
|
|
68 |
+ |
while read -r patched; do
|
|
69 |
+ |
covered=""
|
|
70 |
+ |
for repo in "${REPOS[@]}"; do
|
|
71 |
+ |
case "$patched" in "$repo"/*|"$repo") covered=1; break;; esac
|
|
72 |
+ |
done
|
|
73 |
+ |
[ -n "$covered" ] || missing="$missing $patched"
|
|
74 |
+ |
done < <(grep -oP '(?<=path = ")[^"]+' "$CARGO_CONFIG" | sort -u)
|
|
75 |
+ |
[ -z "$missing" ] || die "these patched paths are not in REPOS:$missing"
|
|
76 |
+ |
|
|
77 |
+ |
say "test-on-astra: syncing to $HOST:~/$DEST"
|
|
78 |
+ |
for repo in "${REPOS[@]}"; do
|
|
79 |
+ |
[ -d "$CODE/$repo" ] || die "$CODE/$repo does not exist"
|
|
80 |
+ |
ssh "$HOST" "mkdir -p ~/$DEST/$(dirname "$repo")" || die "cannot reach $HOST"
|
|
81 |
+ |
# --delete so a file removed locally is removed there; without it a deleted
|
|
82 |
+ |
# test keeps passing on astra forever. target/ and .git are excluded and
|
|
83 |
+ |
# therefore untouched by it, which is what keeps the remote build
|
|
84 |
+ |
# incremental across runs.
|
|
85 |
+ |
# `mutants.out/` is 218 MB of cargo-mutants detritus in synckit-client alone,
|
|
86 |
+ |
# which is twice everything else here put together.
|
|
87 |
+ |
rsync -a --delete \
|
|
88 |
+ |
--exclude 'target/' --exclude '.git/' --exclude 'node_modules/' \
|
|
89 |
+ |
--exclude 'mutants.out/' \
|
|
90 |
+ |
"$CODE/$repo/" "$HOST:$DEST/$repo/" \
|
|
91 |
+ |
|| die "rsync of $repo failed"
|
|
92 |
+ |
done
|
|
93 |
+ |
|
|
94 |
+ |
# The patch block, verbatim, because its paths are relative to the config file's
|
|
95 |
+ |
# own parent and the layout above mirrors ~/Code exactly. `[build]` and
|
|
96 |
+ |
# `[profile.*]` are dropped: `jobs = 6` is a statement about fw13's 14 GB and
|
|
97 |
+ |
# would leave 90 of astra's cores idle.
|
|
98 |
+ |
awk '
|
|
99 |
+ |
/^\[/ { keep = ($0 !~ /^\[build\]/ && $0 !~ /^\[profile/) }
|
|
100 |
+ |
keep { print }
|
|
101 |
+ |
' "$CARGO_CONFIG" | ssh "$HOST" "mkdir -p ~/$DEST/.cargo && cat > ~/$DEST/.cargo/config.toml" \
|
|
102 |
+ |
|| die "could not write the remote cargo config"
|
|
103 |
+ |
|
|
104 |
+ |
# `--features fast-tests` is not an optimisation to argue about: it swaps argon2
|
|
105 |
+ |
# from production parameters (46 MiB, 2 iterations, ~600ms) to test ones (8 MiB,
|
|
106 |
+ |
# 1 iteration, ~10ms), and the suite performs ~874 hashes between signup, login
|
|
107 |
+ |
# and admin setup. That is ~9 CPU-minutes of key derivation proving nothing. It
|
|
108 |
+ |
# is the only thing the feature gates; see the note on `hash_password`.
|
|
109 |
+ |
ARGS=("$@")
|
|
110 |
+ |
[ ${#ARGS[@]} -gt 0 ] || ARGS=(--test integration)
|
|
111 |
+ |
|
|
112 |
+ |
say "test-on-astra: building and running ($THREADS threads, aarch64)"
|
|
113 |
+ |
ssh "$HOST" bash -s -- "$DEST" "$THREADS" "${ARGS[@]}" <<'REMOTE'
|
|
114 |
+ |
set -u
|
|
115 |
+ |
DEST="$1"; shift
|
|
116 |
+ |
THREADS="$1"; shift
|
|
117 |
+ |
|
|
118 |
+ |
cd "$HOME/$DEST/MNW/server" || exit 1
|
|
119 |
+ |
# rustup's toolchain, not the distro's: /usr/bin/rustc is older than the
|
|
120 |
+ |
# dependency tree's floor and fails with a resolver error that names a crate
|
|
121 |
+ |
# rather than the compiler.
|
|
122 |
+ |
export PATH="$HOME/.cargo/bin:$PATH"
|
|
123 |
+ |
# The default 1024 is exhausted by the per-test database pool.
|
|
124 |
+ |
ulimit -n 65536
|
|
125 |
+ |
# Socket form, no host: astra's postgres has no TCP listener, and the harness's
|
|
126 |
+ |
# `replace_db_name` splits on the last `/`, so a `?host=` query string is
|
|
127 |
+ |
# mangled rather than honoured.
|
|
128 |
+ |
export TEST_DATABASE_URL=postgres:///postgres
|
|
129 |
+ |
# Verify against the committed `.sqlx` metadata rather than a live database.
|
|
130 |
+ |
# astra's `makenotwork` has drifted from the schema in the tree, and pointing
|
|
131 |
+ |
# the compile-time macros at it fails the build with errors that read like code
|
|
132 |
+ |
# faults. Drift in the metadata is itself a finding.
|
|
133 |
+ |
export SQLX_OFFLINE=true
|
|
134 |
+ |
|
|
135 |
+ |
started=$(date +%s)
|
|
136 |
+ |
cargo test --features fast-tests "$@" -- --test-threads="$THREADS"
|
|
137 |
+ |
status=$?
|
|
138 |
+ |
echo "test-on-astra: $(( $(date +%s) - started ))s wall on astra, aarch64" >&2
|
|
139 |
+ |
exit $status
|
|
140 |
+ |
REMOTE
|