max / makenotwork

6.2 KB · 143 lines History Blame Raw

1	#!/bin/sh
2	# Test runner for PoM's scheduled suites on astra.
3	#
4	# pom.service calls this once a target's last run goes stale. Everything it
5	# touches lives under /var/lib/pom, the one path the hardened unit can write:
6	# the clones, the rust toolchain, and the build artifacts. That is deliberate.
7	# The daemon runs as `pom` with ProtectHome, so max's checkouts and max's
8	# toolchain are out of reach by design, and this script must not depend on them.
9	#
10	# The one exception is the git mirrors it fetches from, which are read-only.
11	#
12	# Usage: run-ci.sh <target> [filter]
13	# target one of: mnw mt go bb af sk
14	# filter optional cargo test filter, passed through by PoM
15	#
16	# Exit status is the suite's, which is what PoM records as pass/fail.
17
18	set -eu
19
20	STAGING="/var/lib/pom/staging"
21	MIRRORS="/home/max/git-mirrors"
22
23	# The pom user's own toolchain. /usr/bin/cargo is the system rustc, which
24	# trails what these dependency trees need.
25	export HOME="/var/lib/pom"
26	export RUSTUP_HOME="/var/lib/pom/.rustup"
27	export CARGO_HOME="/var/lib/pom/.cargo"
28	export PATH="$CARGO_HOME/bin:$PATH"
29
30	# Parallel test databases exhaust the default 1024 descriptors.
31	ulimit -n 65536 2>/dev/null \|\| true
32
33	# Peer auth on the socket means postgres wants an OS username, and sqlx cannot
34	# work one out inside the service sandbox: it falls back to the `whoami` crate's
35	# placeholder and every connection is refused as user "anonymous". libpq tools
36	# and sqlx both honour PGUSER, so say it once here.
37	export PGUSER=pom
38
39	# astra has 96 cores, and cargo's default job count is all of them. Ninety-six
40	# concurrent rustc processes on the MNW server workspace peaked past 16G and 20G
41	# of swap before the cgroup killed it. This is a memory cap wearing a job-count
42	# costume; the wall-clock cost is small next to thrashing.
43	#
44	# 8 rather than 16 because MemoryMax applies to the whole pom.service cgroup,
45	# not per invocation, and a sweep that finds several targets stale at once runs
46	# them concurrently under that one ceiling. Caches are warm in the steady state,
47	# so few crates actually rebuild and the job count barely shows up in wall clock.
48	export CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-8}"
49
50	# The mirrors are max's, this runs as pom, and git refuses a repo it considers
51	# foreign. Without this every fetch below fails and the clones silently test
52	# whatever commit they were created at, forever. Scoped to this process rather
53	# than written into a global gitconfig.
54	GIT_CONFIG_COUNT=1
55	GIT_CONFIG_KEY_0=safe.directory
56	GIT_CONFIG_VALUE_0='*'
57	export GIT_CONFIG_COUNT GIT_CONFIG_KEY_0 GIT_CONFIG_VALUE_0
58
59	target="${1:-}"
60	filter="${2:-}"
61
62	# The layout under $STAGING mirrors ~/Code on purpose, Apps/ included. Nothing
63	# in git records that tree, so a plain clone does not reproduce it, and the
64	# Tauri apps have a symlink that reaches across it:
65	# Apps/<app>/src-tauri/frontend/js/shared-updater.js
66	# -> ../../../../../MNW/shared/tauri-updater-ui/updater.js
67	# Flatten the tree and that dangles, and the app's build.rs panics on a read.
68	case "$target" in
69	mnw) dir="$STAGING/MNW/server" ;;
70	mt) dir="$STAGING/MNW/multithreaded" ;;
71	go) dir="$STAGING/Apps/goingson" ;;
72	bb) dir="$STAGING/Apps/balanced_breakfast" ;;
73	af) dir="$STAGING/Apps/audiofiles" ;;
74	sk) dir="$STAGING/synckit/synckit-client" ;;
75	*)
76	echo "run-ci.sh: unknown target '${target}' (want: mnw mt go bb af sk)" >&2
77	exit 2
78	;;
79	esac
80
81	if [ ! -d "$dir" ]; then
82	echo "run-ci.sh: $dir does not exist, clone it from $MIRRORS first" >&2
83	exit 2
84	fi
85
86	# Track the mirror so a run tests current main rather than whatever was cloned
87	# months ago. A failed fetch is not fatal: testing a slightly old tree beats
88	# reporting nothing, and the staleness clock keeps ticking either way.
89	repo_root=$(cd "$dir" && git rev-parse --show-toplevel)
90	if ! (cd "$repo_root" && git fetch --quiet origin && git reset --quiet --hard origin/HEAD); then
91	echo "run-ci.sh: warning, could not update $repo_root, testing the existing checkout" >&2
92	fi
93
94	cd "$dir"
95	echo "run-ci.sh: $target at $(git rev-parse --short HEAD) in $dir"
96
97	# Postgres on astra listens on a unix socket only, so the URLs carry no host.
98	# Both harnesses default to a TCP URL, which fails to connect here and fails
99	# every test in the suite within seconds rather than failing loudly once.
100	# Do not add ?host=... : the harnesses split these on the last '/'.
101	# Auth is peer on the socket, so the `pom` role needs LOGIN and CREATEDB.
102	case "$target" in
103	mnw)
104	# Its own database, not max's `makenotwork`: the sqlx macros are
105	# checked at compile time against whatever DATABASE_URL points at, so
106	# CI needs a schema it owns and can migrate. Compiling against the
107	# committed .sqlx cache instead is not an option worth taking, since
108	# the cache is hand-maintained by `cargo sqlx prepare` and goes stale
109	# silently the moment a migration lands without one.
110	DATABASE_URL='postgres:///pom_ci_makenotwork'
111	TEST_DATABASE_URL='postgres:///postgres'
112	export DATABASE_URL TEST_DATABASE_URL
113
114	psql "$TEST_DATABASE_URL" -tAc \
115	"SELECT 1 FROM pg_database WHERE datname = 'pom_ci_makenotwork'" \
116	\| grep -q 1 \|\| createdb pom_ci_makenotwork
117	sqlx migrate run --source migrations --database-url "$DATABASE_URL"
118	;;
119	mt)
120	# Two harnesses in one suite. Most tests use the hand-rolled `TestDb`,
121	# which reads TEST_DATABASE_URL. The `admin` and `bans` modules use
122	# `#[sqlx::test]`, which reads DATABASE_URL and manages its own throwaway
123	# databases off it. Set only the first and those 29 fail with
124	# "DATABASE_URL must be set", which reads like 29 broken tests rather
125	# than one missing variable.
126	DATABASE_URL='postgres:///pom_ci_mt'
127	TEST_DATABASE_URL='postgres:///postgres'
128	export DATABASE_URL TEST_DATABASE_URL
129
130	psql "$TEST_DATABASE_URL" -tAc \
131	"SELECT 1 FROM pg_database WHERE datname = 'pom_ci_mt'" \
132	\| grep -q 1 \|\| createdb pom_ci_mt
133	;;
134	esac
135
136	# --test-threads=4 rather than the default: at full parallelism the suites
137	# intermittently exhaust postgres connection slots on this box.
138	if [ -n "$filter" ]; then
139	exec cargo test --workspace "$filter" -- --test-threads=4
140	else
141	exec cargo test --workspace -- --test-threads=4
142	fi
143