#!/bin/bash
# Run the server's expensive suites on astra instead of on fw13.
#
# fw13 is 12 cores against 14 GB, and `~/Code/.cargo/config.toml` caps cargo at
# `jobs = 6` because a heavy crate costs rustc ~3 GB and the box swaps rather
# than compiles above that. astra is 96 cores and 125 GB, usually idle outside
# the 03:20 sweep. The server's integration suite is the one workload where that
# difference is worth a round trip: ~1,340 tests and a build of the largest crate
# in the tree.
#
# **This is for the expensive runs only.** A filtered local run beats this every
# time: `cargo test --test integration described_screens` answers in seconds,
# where anything here pays a sync plus a build. Reach for this when the question
# is "is the whole suite green", not "did my change work".
#
# WHAT IT SYNCS, AND WHY IT IS NOT JUST THE SERVER. The point of the script is
# testing work that is *not committed yet* -- astra's own checkout only ever has
# what has been pushed, which is exactly what made it useless for a day of
# editing quasi and the server together. So it rsyncs the working trees, and it
# has to send every repo the local `[patch]` block redirects, or astra silently
# builds the published quasi against your edited server and reports green on a
# combination that does not exist. That is worse than no script.
#
# CAVEAT THAT DOES NOT GO AWAY: astra is aarch64. Green here is not green on
# x86_64. It catches logic, not size, alignment or intrinsics assumptions.
#
# Usage:
#   scripts/test-on-astra.sh                          # the integration suite
#   scripts/test-on-astra.sh --lib                    # or any cargo test args
#   scripts/test-on-astra.sh --test integration described_screens
#
# Environment:
#   ASTRA_HOST        ssh host (default: astra)
#   ASTRA_DIR         remote directory, under $HOME (default: mnw-test)
#   ASTRA_THREADS     test threads (default: 16, see the note below)

set -u
set -o pipefail

HOST="${ASTRA_HOST:-astra}"
DEST="${ASTRA_DIR:-mnw-test}"
# Postgres, not cores, is what bounds this suite: every test takes a
# `CREATE DATABASE ... TEMPLATE` clone, and astra's cluster is one instance like
# any other. Memory note `reference_astra_tests` records ~9/734 workflow tests
# hitting "connection slots reserved for SUPERUSER" under an unbounded run. So
# this is deliberately nowhere near 96, and raising it trades wall-clock for
# flakes that look like real failures.
THREADS="${ASTRA_THREADS:-16}"

ROOT="$(cd "$(dirname "$0")/../.." && pwd)"   # ~/Code/MNW
CODE="$(cd "$ROOT/.." && pwd)"                # ~/Code
CARGO_CONFIG="$CODE/.cargo/config.toml"

# Every repo that has to travel. MNW carries the server and the `shared/*` path
# dependencies; the rest are what `[patch]` redirects. Keep in step with
# `$CARGO_CONFIG` -- the check below fails the run if they drift.
REPOS=(MNW quasi synckit Libraries/docengine Libraries/quasi-type)

say() { printf '%s\n' "$*" >&2; }
die() { say "test-on-astra: $*"; exit 1; }

[ -f "$CARGO_CONFIG" ] || die "no $CARGO_CONFIG; nothing says where the siblings are"

# Drift guard. A `[patch]` entry pointing outside the synced set would build on
# astra from the published crate instead of the working tree, and the run would
# be green about code nobody has.
missing=""
while read -r patched; do
    covered=""
    for repo in "${REPOS[@]}"; do
        case "$patched" in "$repo"/*|"$repo") covered=1; break;; esac
    done
    [ -n "$covered" ] || missing="$missing $patched"
done < <(grep -oP '(?<=path = ")[^"]+' "$CARGO_CONFIG" | sort -u)
[ -z "$missing" ] || die "these patched paths are not in REPOS:$missing"

say "test-on-astra: syncing to $HOST:~/$DEST"
for repo in "${REPOS[@]}"; do
    [ -d "$CODE/$repo" ] || die "$CODE/$repo does not exist"
    ssh "$HOST" "mkdir -p ~/$DEST/$(dirname "$repo")" || die "cannot reach $HOST"
    # --delete so a file removed locally is removed there; without it a deleted
    # test keeps passing on astra forever. target/ and .git are excluded and
    # therefore untouched by it, which is what keeps the remote build
    # incremental across runs.
    # `mutants.out/` is 218 MB of cargo-mutants detritus in synckit-client alone,
    # which is twice everything else here put together.
    rsync -a --delete \
        --exclude 'target/' --exclude '.git/' --exclude 'node_modules/' \
        --exclude 'mutants.out/' \
        "$CODE/$repo/" "$HOST:$DEST/$repo/" \
        || die "rsync of $repo failed"
done

# The patch block, verbatim, because its paths are relative to the config file's
# own parent and the layout above mirrors ~/Code exactly. `[build]` and
# `[profile.*]` are dropped: `jobs = 6` is a statement about fw13's 14 GB and
# would leave 90 of astra's cores idle.
awk '
    /^\[/ { keep = ($0 !~ /^\[build\]/ && $0 !~ /^\[profile/) }
    keep { print }
' "$CARGO_CONFIG" | ssh "$HOST" "mkdir -p ~/$DEST/.cargo && cat > ~/$DEST/.cargo/config.toml" \
    || die "could not write the remote cargo config"

# `--features fast-tests` is not an optimisation to argue about: it swaps argon2
# from production parameters (46 MiB, 2 iterations, ~600ms) to test ones (8 MiB,
# 1 iteration, ~10ms), and the suite performs ~874 hashes between signup, login
# and admin setup. That is ~9 CPU-minutes of key derivation proving nothing. It
# is the only thing the feature gates; see the note on `hash_password`.
ARGS=("$@")
[ ${#ARGS[@]} -gt 0 ] || ARGS=(--test integration)

say "test-on-astra: building and running ($THREADS threads, aarch64)"
ssh "$HOST" bash -s -- "$DEST" "$THREADS" "${ARGS[@]}" <<'REMOTE'
set -u
DEST="$1"; shift
THREADS="$1"; shift

cd "$HOME/$DEST/MNW/server" || exit 1
# rustup's toolchain, not the distro's: /usr/bin/rustc is older than the
# dependency tree's floor and fails with a resolver error that names a crate
# rather than the compiler.
export PATH="$HOME/.cargo/bin:$PATH"
# The default 1024 is exhausted by the per-test database pool.
ulimit -n 65536
# Socket form, no host: astra's postgres has no TCP listener, and the harness's
# `replace_db_name` splits on the last `/`, so a `?host=` query string is
# mangled rather than honoured.
export TEST_DATABASE_URL=postgres:///postgres
# Verify against the committed `.sqlx` metadata rather than a live database.
# astra's `makenotwork` has drifted from the schema in the tree, and pointing
# the compile-time macros at it fails the build with errors that read like code
# faults. Drift in the metadata is itself a finding.
export SQLX_OFFLINE=true

started=$(date +%s)
cargo test --features fast-tests "$@" -- --test-threads="$THREADS"
status=$?
echo "test-on-astra: $(( $(date +%s) - started ))s wall on astra, aarch64" >&2
exit $status
REMOTE