Skip to main content

max / alloy

16.4 KB · 385 lines History Blame Raw
1 #!/usr/bin/env bash
2 #
3 # offline-first-boot.sh — install a machine, take its route away, and check
4 # that the first boot still lays the console down.
5 #
6 # THE REGRESSION THIS EXISTS FOR. alloy-layer-components.service installs the
7 # console and terminal from /usr/share/alloy/rpm, a file:// repo carried on the
8 # medium precisely so an offline install produces a working machine. It did
9 # not: rpm-ostree refreshes metadata for every ENABLED repo before it
10 # depsolves, and the image leaves Fedora's four enabled, so a first boot with
11 # no name resolution failed on a mirrorlist it did not need and left the
12 # machine with no console at all. Fixed in alloy@aa0dacd by fencing the
13 # transaction to the carried repo (usr/bin/alloy-layer-repos), and verified by
14 # hand — nothing in build/vmtest would have caught it coming back, which is
15 # what this closes.
16 #
17 # Red before aa0dacd, green after. The failing shape is specific: the unit
18 # fails, so it never reaches its `systemctl reboot`, so no RESET arrives and
19 # /usr/bin/alloy is absent afterwards.
20 #
21 # ## What it asserts, and why each one is here
22 #
23 # 1. the machine rebooted the unit's last act. Its absence IS the
24 # regression, and waiting for the event rather
25 # than for the clock is what makes a slow
26 # install distinguishable from a failed one.
27 # 2. /usr/bin/alloy exists the console got laid down. The done
28 # condition of the whole exercise.
29 # 3. the enabled repo set `alloy-layer-repos on` put /etc back, on the
30 # deployment the transaction staged. Fencing
31 # the repos and leaving them fenced would be a
32 # different defect with the same symptom
33 # later: `alloy pkg` finding nothing.
34 # 4. check-installed.sh folded in because it had never been run
35 # against an install from this harness either,
36 # and a scenario that boots a fresh machine is
37 # already holding everything it needs.
38 #
39 # ## How the route is taken away
40 #
41 # From the monitor, with QMP `set_link`, before the guest has booted at all.
42 # Not from inside: `ip link set enp0s2 down` travels over the ssh session it
43 # arrived on and kills it, which is why the hand run on 2026-08-25 needed a
44 # detached `setsid` script that brought the interface back up at the end. From
45 # out here the link is a property of the device and the guest gets no say.
46 #
47 # The link goes back up only after the reboot, and it cannot affect the result
48 # by then: the unit's `ConditionPathExists=!/usr/bin/alloy` is false on a
49 # machine that has a console, so the second boot does not run it. Bringing it
50 # up is how the assertions get in, and on the failing path it is how the
51 # evidence gets out.
52 #
53 # ## How the installer is driven
54 #
55 # Over the medium's own headless ssh installer, which is the route a person
56 # installing a screenless machine takes. `VMTEST_VIA=serial` takes GRUB's debug
57 # entry and its root shell instead, which is what this used while the ssh route
58 # could not install anything (GO alloy `2cf04f20`, fixed 2026-08-26 by running
59 # the wizard under run0).
60 #
61 # ## Requirements
62 #
63 # Everything build/vmtest/README.md lists, and an ISO whose baked pubkey matches
64 # the key given here:
65 #
66 # build/build-iso.sh --build-arg PROFILE=server --build-arg BROWSER=none \
67 # --build-arg ALLOY_SSH_KEY="$(cat ~/.ssh/id_ed25519.pub)"
68 #
69 # That key is doing two jobs: it is the installer session's only credential, and
70 # it is what the wizard puts in the NEW machine's authorized_keys, which is how
71 # the assertions get in afterwards. `VMTEST_VIA=serial` needs only the second.
72 #
73 # ## Use
74 #
75 # build/vmtest/offline-first-boot.sh
76 # build/vmtest/offline-first-boot.sh --key ~/.ssh/alloy_vm
77 # build/vmtest/offline-first-boot.sh --keep-state # reuse an un-booted install
78 #
79 # Exit codes follow check-installed.sh and check-rust-stage.sh:
80 #
81 # 0 the machine came up with a console after an offline first boot, with the
82 # ordinary repo set and a correctly labelled /etc.
83 # 1 it did not. What failed is printed, with the evidence gathered from the
84 # guest.
85 # 3 something about the run rather than about the machine: a missing tool, no
86 # ISO, no key, or an install that never got far enough to have a verdict.
87
88 set -Eeuo pipefail
89
90 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
91 REPO_ROOT="$(cd "$HERE/../.." && pwd)"
92 SCRATCH="${VM_STATE:-$HERE/state}"
93 export VM_STATE="$SCRATCH"
94
95 KEY="${VMTEST_KEY:-$HOME/.ssh/id_ed25519}"
96 ISO="${ALLOY_ISO:-$REPO_ROOT/output/install.iso}"
97 DISK_NAME="${VMTEST_DISK:-vda}"
98 HOSTNAME_="${VMTEST_HOSTNAME:-alloytest}"
99 USER_="${VMTEST_USER:-tester}"
100 PASSWORD="${VMTEST_PASSWORD:-alloytest}"
101 PORT="${VMTEST_PORT:-2222}"
102 KEEP_STATE=0
103 # Which way the wizard is reached. See install_drive.py's header; `ssh` needs a
104 # medium whose baked pubkey matches $KEY.
105 VIA="${VMTEST_VIA:-ssh}"
106
107 # The layering transaction is the long part, and it is the one whose absence is
108 # the finding. Generous, because a slow host must not read as a regression.
109 LAYER_TIMEOUT="${VMTEST_LAYER_TIMEOUT:-900}"
110 INSTALL_TIMEOUT="${VMTEST_INSTALL_TIMEOUT:-1800}"
111 SSH_TIMEOUT="${VMTEST_SSH_TIMEOUT:-300}"
112
113 QEMU_PID=""
114
115 die() { printf 'error: %s\n' "$*" >&2; exit 3; }
116 fail() { printf 'FAIL: %s\n' "$*" >&2; FAILED=$((FAILED + 1)); }
117 say() { printf '==> %s\n' "$*"; }
118 FAILED=0
119
120 while [ $# -gt 0 ]; do
121 case "$1" in
122 --key) KEY="${2:?--key needs a path}"; shift 2 ;;
123 --iso) ISO="${2:?--iso needs a path}"; shift 2 ;;
124 --disk) DISK_NAME="${2:?--disk needs a name}"; shift 2 ;;
125 # Reuse the disk in state/ instead of installing. For iterating on the
126 # second half without paying for the first every time — and only for a
127 # disk that has been installed and NOT yet first-booted, since the unit
128 # under test does nothing on a machine that already has a console. The
129 # verdict below says so rather than reading that as a regression.
130 --keep-state) KEEP_STATE=1; shift ;;
131 -h|--help) sed -n '2,80p' "$0"; exit 0 ;;
132 *) die "unknown argument: $1" ;;
133 esac
134 done
135
136 # ---- preflight ----
137
138 for tool in qemu-system-x86_64 swtpm swtpm_setup python3 ssh; do
139 command -v "$tool" >/dev/null 2>&1 || die "no $tool"
140 done
141 [ -f /usr/share/OVMF/OVMF_CODE_4M.fd ] || die "no OVMF at /usr/share/OVMF"
142 [ -f "$KEY" ] || die "no private key at $KEY"
143 [ -f "$KEY.pub" ] || die "no public key at $KEY.pub"
144 [ "$KEEP_STATE" = 1 ] || [ -f "$ISO" ] || die "no ISO at $ISO; build/build-iso.sh first"
145
146 SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
147 -o LogLevel=ERROR -o ConnectTimeout=10 -o BatchMode=yes
148 -p "$PORT" -i "$KEY")
149
150 qmp() { python3 "$HERE/qmp.py" "$@"; }
151
152 # THE INSTALLED MACHINE'S LOGIN SHELL IS NUSHELL (`/usr/bin/nu`), so a command
153 # handed to ssh is parsed by nushell and not by sh. This is not a detail: the
154 # obvious spellings fail, and two of them fail in ways that read as the machine
155 # being broken rather than as the command being wrong. Measured 2026-08-26:
156 #
157 # `... 2>&1` -> nu::parser::shell_outerr, "use 'out+err>'"
158 # `cat > /tmp/file` -> cat: '>': No such file or directory
159 # `scp file host:/tmp/` -> fails, because scp runs its far end through the
160 # login shell too
161 #
162 # So every remote command goes through a POSIX shell explicitly. `sh -s` with
163 # the script on stdin is the form to reach for: the script is then written once
164 # in its own syntax, with nothing to quote through nushell.
165 guest() { ssh "${SSH_OPTS[@]}" "$USER_@127.0.0.1" 'sh -s' "$@"; }
166
167 boot() {
168 # run-vm.sh exec's qemu, so this pid is qemu's and killing it is enough.
169 "$HERE/run-vm.sh" "$1" >"$SCRATCH/qemu-$1.log" 2>&1 &
170 QEMU_PID=$!
171 sleep 1
172 kill -0 "$QEMU_PID" 2>/dev/null || {
173 cat "$SCRATCH/qemu-$1.log" >&2
174 die "qemu died on '$1'"
175 }
176 }
177
178 shutdown_vm() {
179 [ -n "$QEMU_PID" ] || return 0
180 qmp cmd quit >/dev/null 2>&1 || true
181 # The socket goes away with the process, so `quit` returning nothing is the
182 # ordinary path rather than a failure. Wait, then insist.
183 for _ in $(seq 1 40); do
184 kill -0 "$QEMU_PID" 2>/dev/null || { QEMU_PID=""; return 0; }
185 sleep 0.25
186 done
187 kill -9 "$QEMU_PID" 2>/dev/null || true
188 wait "$QEMU_PID" 2>/dev/null || true
189 QEMU_PID=""
190 }
191
192 cleanup() {
193 [ -n "$QEMU_PID" ] && kill -9 "$QEMU_PID" 2>/dev/null
194 return 0
195 }
196 trap cleanup EXIT
197
198 # ---- phase 1: install ----
199
200 if [ "$KEEP_STATE" = 1 ]; then
201 [ -f "$SCRATCH/target.qcow2" ] || die "--keep-state, but there is no state/target.qcow2 to keep"
202 say "reusing the install in state/target.qcow2"
203 else
204 # Both together, always. Leaving the firmware variables behind boots the
205 # half-written disk to a grub prompt; build/vmtest/README.md records the
206 # afternoon that cost.
207 say "resetting state/"
208 rm -f "$SCRATCH/target.qcow2" "$SCRATCH/OVMF_VARS.fd"
209 rm -rf "$SCRATCH/tpm"
210
211 say "booting the installer medium"
212 ALLOY_ISO="$ISO" boot live
213
214 say "driving the wizard"
215 rc=0
216 python3 "$HERE/install_drive.py" \
217 --via "$VIA" --key "$KEY" --pubkey "$(cat "$KEY.pub")" --disk "$DISK_NAME" \
218 --hostname "$HOSTNAME_" --user "$USER_" --password "$PASSWORD" \
219 --install-timeout "$INSTALL_TIMEOUT" || rc=$?
220 shutdown_vm
221 [ "$rc" -eq 0 ] || {
222 [ "$rc" -eq 1 ] && { printf 'FAIL: the install itself failed, so the first boot was never reached\n' >&2; exit 1; }
223 die "the wizard could not be driven to a verdict"
224 }
225 say "installed"
226 fi
227
228 # ---- phase 2: the offline first boot ----
229
230 say "booting the installed machine with its link down"
231 boot installed
232 # Before the guest's firmware has handed over, let alone before userspace. The
233 # NIC reports no carrier for the whole of the boot that matters.
234 qmp cmd set_link '{"name": "nic0", "up": false}' >/dev/null
235
236 say "waiting up to ${LAYER_TIMEOUT}s for the machine to lay its components down and reboot"
237 REBOOTED=1
238 # `guest: true` rather than any RESET: a reset the host asked for and a reboot
239 # the guest performed are the same event with a different reason, and only the
240 # second one is the unit's last act.
241 qmp wait RESET "$LAYER_TIMEOUT" '{"guest": true}' >/dev/null 2>&1 || REBOOTED=0
242 [ "$REBOOTED" = 1 ] && say "it rebooted"
243
244 # ---- phase 3: read the result ----
245
246 # Only now, and it cannot change the answer: the unit's second condition is
247 # `ConditionPathExists=!/usr/bin/alloy`, so a machine that has a console does
248 # not run it again, and one that does not have a console failed before this.
249 say "putting the link back up"
250 qmp cmd set_link '{"name": "nic0", "up": true}' >/dev/null
251
252 say "waiting up to ${SSH_TIMEOUT}s for ssh"
253 reachable=0
254 end=$((SECONDS + SSH_TIMEOUT))
255 while [ "$SECONDS" -lt "$end" ]; do
256 if echo true | guest >/dev/null 2>&1; then reachable=1; break; fi
257 sleep 5
258 done
259 [ "$reachable" = 1 ] || {
260 shutdown_vm
261 die "the machine never answered ssh, so nothing can be read off it. \
262 state/serial-installed.log and state/qemu-installed.log are what there is."
263 }
264
265 # 2. The console got laid down, read together with 1 because the pair is what
266 # says which run this was. The unit's own conditions make "no reboot" mean two
267 # different things, and reporting them the same way is how a scenario run
268 # against the wrong disk gets filed as a regression.
269 CONSOLE=1
270 echo 'test -x /usr/bin/alloy' | guest >/dev/null 2>&1 || CONSOLE=0
271
272 if [ "$REBOOTED" = 0 ] && [ "$CONSOLE" = 1 ]; then
273 # ConditionPathExists=!/usr/bin/alloy was already false, so the unit was a
274 # no-op and nothing here was measured. Not a defect, and not a pass either.
275 shutdown_vm
276 die "this machine had already been first-booted: it has a console and never \
277 ran the unit. Reset state/ and install again, or drop --keep-state."
278 fi
279
280 if [ "$REBOOTED" = 0 ]; then
281 fail "the machine never rebooted, so alloy-layer-components.service did not finish"
282 fi
283 if [ "$CONSOLE" = 1 ]; then
284 say "/usr/bin/alloy: present ($(echo '/usr/bin/alloy --version' | guest 2>/dev/null | head -1))"
285 else
286 fail "/usr/bin/alloy is absent: the machine came up with no console"
287 printf '\nwhat the unit said:\n' >&2
288 echo 'journalctl -u alloy-layer-components.service --no-pager -o cat' \
289 | guest 2>&1 | tail -40 >&2 || true
290 fi
291
292 # 3. The repo set. Read from both /etc and the image's own /usr/etc, because
293 # `alloy-layer-repos on` restores one from the other and comparing them is the
294 # assertion in its own terms. The count is asserted separately: deriving the
295 # expectation from /usr/etc alone would follow a change to the image silently,
296 # and "alloy-local plus Fedora's four" is a number somebody decided.
297 # Fed on stdin rather than quoted into the ssh command line, so the awk program
298 # is written once in its own syntax instead of through two layers of shell
299 # quoting. A repo file can carry more than one section, so the id is tracked
300 # rather than taken from the filename.
301 enabled_repos() {
302 guest "$1" <<'AWKEOF' 2>/dev/null || true
303 awk '
304 /^\[/ { id = substr($0, 2, index($0, "]") - 2) }
305 /^enabled[ \t]*=[ \t]*1[ \t]*$/ { if (id != "") print id }
306 ' "$1"/*.repo | sort -u
307 AWKEOF
308 }
309 LIVE_REPOS="$(enabled_repos /etc/yum.repos.d)"
310 IMAGE_REPOS="$(enabled_repos /usr/etc/yum.repos.d)"
311
312 if [ -z "$LIVE_REPOS" ]; then
313 fail "no repo is enabled on the machine at all, so the fence was never lowered"
314 elif [ "$LIVE_REPOS" != "$IMAGE_REPOS" ]; then
315 fail "the enabled repo set does not match the image's own"
316 printf ' /etc: %s\n' "$(echo "$LIVE_REPOS" | paste -sd' ')" >&2
317 printf ' /usr/etc: %s\n' "$(echo "$IMAGE_REPOS" | paste -sd' ')" >&2
318 else
319 count="$(printf '%s\n' "$LIVE_REPOS" | wc -l)"
320 echo "$LIVE_REPOS" | grep -qx 'alloy-local' \
321 || fail "alloy-local is not enabled, so the carried repo is not reachable"
322 [ "$count" -eq 5 ] \
323 || fail "$count repos are enabled, not the five this expects (alloy-local plus Fedora's four): $(echo "$LIVE_REPOS" | paste -sd' ')"
324 [ "$count" -eq 5 ] && say "repos: $(echo "$LIVE_REPOS" | paste -sd' ')"
325 fi
326
327 # 4. And the labels, folded in because this is the only place an install from
328 # the wizard is standing still and reachable. Copied and then run rather than
329 # fed on stdin: sudo wants the password there.
330 say "running check-installed.sh on the guest"
331
332 # Delivered by writing it through a shell rather than with scp. scp needs the
333 # sftp subsystem at the far end and a working PATH there, which is a second
334 # thing to go wrong for no benefit; `cat >` needs a shell, which is already
335 # proven by every assertion above. Measured 2026-08-26: scp failed here and
336 # the run reported a machine defect for it.
337 #
338 # `sh -c` here rather than `sh -s`, because stdin is carrying the file. The
339 # string has no metacharacter nushell would take differently, which is what
340 # makes it safe to send one through.
341 #
342 # Two calls, not one, because the password and the script cannot share stdin.
343 # `sudo -S` reads a bufferful, not a line, so piping the password and then the
344 # script into `sudo -S bash -s` can have sudo swallow the top of the script.
345 CI_RAN=0
346 ci=0
347 if ! ssh "${SSH_OPTS[@]}" "$USER_@127.0.0.1" "sh -c 'cat > /tmp/check-installed.sh'" \
348 < "$REPO_ROOT/build/check-installed.sh"; then
349 printf 'note: could not write check-installed.sh to the guest (above)\n' >&2
350 else
351 CI_RAN=1
352 guest <<EOF || ci=$?
353 printf '%s\n' '$PASSWORD' | sudo -S -p '' bash /tmp/check-installed.sh
354 EOF
355 fi
356
357 if [ "$CI_RAN" = 0 ]; then
358 : # reported below, as a run problem rather than a machine one
359 else
360 case "$ci" in
361 0) say "check-installed: clean" ;;
362 1) fail "check-installed found a defect on the installed machine (above)" ;;
363 # 3 is check-installed's own "could not run" (not root, no restorecon, an
364 # install from the selinux=0 entry). Its verdict, and not this machine's.
365 *) printf 'note: check-installed could not run (exit %s)\n' "$ci"; CI_RAN=0 ;;
366 esac
367 fi
368
369 shutdown_vm
370
371 if [ "$FAILED" -ne 0 ]; then
372 printf '\n%s check(s) failed. The machine did not survive an offline first boot.\n' "$FAILED" >&2
373 exit 1
374 fi
375
376 # The three assertions this test is FOR have passed by here. The fold-in has
377 # not run, and calling that green would be claiming a check that did not
378 # happen, while calling it a failure would blame the machine for the harness.
379 if [ "$CI_RAN" = 0 ]; then
380 printf '\nThe offline first boot passed: a console was laid down and the repo set is the ordinary one.\n'
381 die "but check-installed.sh did not run, so the labels are unchecked"
382 fi
383
384 printf '\nAn offline first boot laid the console down, left the ordinary repo set, and labelled /etc correctly.\n'
385