#!/usr/bin/env bash
#
# offline-first-boot.sh — install a machine, take its route away, and check
# that the first boot still lays the console down.
#
# THE REGRESSION THIS EXISTS FOR. alloy-layer-components.service installs the
# console and terminal from /usr/share/alloy/rpm, a file:// repo carried on the
# medium precisely so an offline install produces a working machine. It did
# not: rpm-ostree refreshes metadata for every ENABLED repo before it
# depsolves, and the image leaves Fedora's four enabled, so a first boot with
# no name resolution failed on a mirrorlist it did not need and left the
# machine with no console at all. Fixed in alloy@aa0dacd by fencing the
# transaction to the carried repo (usr/bin/alloy-layer-repos), and verified by
# hand — nothing in build/vmtest would have caught it coming back, which is
# what this closes.
#
# Red before aa0dacd, green after. The failing shape is specific: the unit
# fails, so it never reaches its `systemctl reboot`, so no RESET arrives and
# /usr/bin/alloy is absent afterwards.
#
# ## What it asserts, and why each one is here
#
#   1. the machine rebooted        the unit's last act. Its absence IS the
#                                  regression, and waiting for the event rather
#                                  than for the clock is what makes a slow
#                                  install distinguishable from a failed one.
#   2. /usr/bin/alloy exists       the console got laid down. The done
#                                  condition of the whole exercise.
#   3. the enabled repo set        `alloy-layer-repos on` put /etc back, on the
#                                  deployment the transaction staged. Fencing
#                                  the repos and leaving them fenced would be a
#                                  different defect with the same symptom
#                                  later: `alloy pkg` finding nothing.
#   4. check-installed.sh          folded in because it had never been run
#                                  against an install from this harness either,
#                                  and a scenario that boots a fresh machine is
#                                  already holding everything it needs.
#
# ## How the route is taken away
#
# From the monitor, with QMP `set_link`, before the guest has booted at all.
# Not from inside: `ip link set enp0s2 down` travels over the ssh session it
# arrived on and kills it, which is why the hand run on 2026-08-25 needed a
# detached `setsid` script that brought the interface back up at the end. From
# out here the link is a property of the device and the guest gets no say.
#
# The link goes back up only after the reboot, and it cannot affect the result
# by then: the unit's `ConditionPathExists=!/usr/bin/alloy` is false on a
# machine that has a console, so the second boot does not run it. Bringing it
# up is how the assertions get in, and on the failing path it is how the
# evidence gets out.
#
# ## How the installer is driven
#
# Over the medium's own headless ssh installer, which is the route a person
# installing a screenless machine takes. `VMTEST_VIA=serial` takes GRUB's debug
# entry and its root shell instead, which is what this used while the ssh route
# could not install anything (GO alloy `2cf04f20`, fixed 2026-08-26 by running
# the wizard under run0).
#
# ## Requirements
#
# Everything build/vmtest/README.md lists, and an ISO whose baked pubkey matches
# the key given here:
#
#   build/build-iso.sh --build-arg PROFILE=server --build-arg BROWSER=none \
#     --build-arg ALLOY_SSH_KEY="$(cat ~/.ssh/id_ed25519.pub)"
#
# That key is doing two jobs: it is the installer session's only credential, and
# it is what the wizard puts in the NEW machine's authorized_keys, which is how
# the assertions get in afterwards. `VMTEST_VIA=serial` needs only the second.
#
# ## Use
#
#   build/vmtest/offline-first-boot.sh
#   build/vmtest/offline-first-boot.sh --key ~/.ssh/alloy_vm
#   build/vmtest/offline-first-boot.sh --keep-state   # reuse an un-booted install
#
# Exit codes follow check-installed.sh and check-rust-stage.sh:
#
#   0  the machine came up with a console after an offline first boot, with the
#      ordinary repo set and a correctly labelled /etc.
#   1  it did not. What failed is printed, with the evidence gathered from the
#      guest.
#   3  something about the run rather than about the machine: a missing tool, no
#      ISO, no key, or an install that never got far enough to have a verdict.

set -Eeuo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$HERE/../.." && pwd)"
SCRATCH="${VM_STATE:-$HERE/state}"
export VM_STATE="$SCRATCH"

KEY="${VMTEST_KEY:-$HOME/.ssh/id_ed25519}"
ISO="${ALLOY_ISO:-$REPO_ROOT/output/install.iso}"
DISK_NAME="${VMTEST_DISK:-vda}"
HOSTNAME_="${VMTEST_HOSTNAME:-alloytest}"
USER_="${VMTEST_USER:-tester}"
PASSWORD="${VMTEST_PASSWORD:-alloytest}"
PORT="${VMTEST_PORT:-2222}"
KEEP_STATE=0
# Which way the wizard is reached. See install_drive.py's header; `ssh` needs a
# medium whose baked pubkey matches $KEY.
VIA="${VMTEST_VIA:-ssh}"

# The layering transaction is the long part, and it is the one whose absence is
# the finding. Generous, because a slow host must not read as a regression.
LAYER_TIMEOUT="${VMTEST_LAYER_TIMEOUT:-900}"
INSTALL_TIMEOUT="${VMTEST_INSTALL_TIMEOUT:-1800}"
SSH_TIMEOUT="${VMTEST_SSH_TIMEOUT:-300}"

QEMU_PID=""

die()  { printf 'error: %s\n' "$*" >&2; exit 3; }
fail() { printf 'FAIL: %s\n' "$*" >&2; FAILED=$((FAILED + 1)); }
say()  { printf '==> %s\n' "$*"; }
FAILED=0

while [ $# -gt 0 ]; do
  case "$1" in
    --key)        KEY="${2:?--key needs a path}"; shift 2 ;;
    --iso)        ISO="${2:?--iso needs a path}"; shift 2 ;;
    --disk)       DISK_NAME="${2:?--disk needs a name}"; shift 2 ;;
    # Reuse the disk in state/ instead of installing. For iterating on the
    # second half without paying for the first every time — and only for a
    # disk that has been installed and NOT yet first-booted, since the unit
    # under test does nothing on a machine that already has a console. The
    # verdict below says so rather than reading that as a regression.
    --keep-state) KEEP_STATE=1; shift ;;
    -h|--help)    sed -n '2,80p' "$0"; exit 0 ;;
    *)            die "unknown argument: $1" ;;
  esac
done

# ---- preflight ----

for tool in qemu-system-x86_64 swtpm swtpm_setup python3 ssh; do
  command -v "$tool" >/dev/null 2>&1 || die "no $tool"
done
[ -f /usr/share/OVMF/OVMF_CODE_4M.fd ] || die "no OVMF at /usr/share/OVMF"
[ -f "$KEY" ] || die "no private key at $KEY"
[ -f "$KEY.pub" ] || die "no public key at $KEY.pub"
[ "$KEEP_STATE" = 1 ] || [ -f "$ISO" ] || die "no ISO at $ISO; build/build-iso.sh first"

SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
          -o LogLevel=ERROR -o ConnectTimeout=10 -o BatchMode=yes
          -p "$PORT" -i "$KEY")

qmp()  { python3 "$HERE/qmp.py" "$@"; }

# THE INSTALLED MACHINE'S LOGIN SHELL IS NUSHELL (`/usr/bin/nu`), so a command
# handed to ssh is parsed by nushell and not by sh. This is not a detail: the
# obvious spellings fail, and two of them fail in ways that read as the machine
# being broken rather than as the command being wrong. Measured 2026-08-26:
#
#   `... 2>&1`              -> nu::parser::shell_outerr, "use 'out+err>'"
#   `cat > /tmp/file`       -> cat: '>': No such file or directory
#   `scp file host:/tmp/`   -> fails, because scp runs its far end through the
#                              login shell too
#
# So every remote command goes through a POSIX shell explicitly. `sh -s` with
# the script on stdin is the form to reach for: the script is then written once
# in its own syntax, with nothing to quote through nushell.
guest() { ssh "${SSH_OPTS[@]}" "$USER_@127.0.0.1" 'sh -s' "$@"; }

boot() {
  # run-vm.sh exec's qemu, so this pid is qemu's and killing it is enough.
  "$HERE/run-vm.sh" "$1" >"$SCRATCH/qemu-$1.log" 2>&1 &
  QEMU_PID=$!
  sleep 1
  kill -0 "$QEMU_PID" 2>/dev/null || {
    cat "$SCRATCH/qemu-$1.log" >&2
    die "qemu died on '$1'"
  }
}

shutdown_vm() {
  [ -n "$QEMU_PID" ] || return 0
  qmp cmd quit >/dev/null 2>&1 || true
  # The socket goes away with the process, so `quit` returning nothing is the
  # ordinary path rather than a failure. Wait, then insist.
  for _ in $(seq 1 40); do
    kill -0 "$QEMU_PID" 2>/dev/null || { QEMU_PID=""; return 0; }
    sleep 0.25
  done
  kill -9 "$QEMU_PID" 2>/dev/null || true
  wait "$QEMU_PID" 2>/dev/null || true
  QEMU_PID=""
}

cleanup() {
  [ -n "$QEMU_PID" ] && kill -9 "$QEMU_PID" 2>/dev/null
  return 0
}
trap cleanup EXIT

# ---- phase 1: install ----

if [ "$KEEP_STATE" = 1 ]; then
  [ -f "$SCRATCH/target.qcow2" ] || die "--keep-state, but there is no state/target.qcow2 to keep"
  say "reusing the install in state/target.qcow2"
else
  # Both together, always. Leaving the firmware variables behind boots the
  # half-written disk to a grub prompt; build/vmtest/README.md records the
  # afternoon that cost.
  say "resetting state/"
  rm -f "$SCRATCH/target.qcow2" "$SCRATCH/OVMF_VARS.fd"
  rm -rf "$SCRATCH/tpm"

  say "booting the installer medium"
  ALLOY_ISO="$ISO" boot live

  say "driving the wizard"
  rc=0
  python3 "$HERE/install_drive.py" \
    --via "$VIA" --key "$KEY" --pubkey "$(cat "$KEY.pub")" --disk "$DISK_NAME" \
    --hostname "$HOSTNAME_" --user "$USER_" --password "$PASSWORD" \
    --install-timeout "$INSTALL_TIMEOUT" || rc=$?
  shutdown_vm
  [ "$rc" -eq 0 ] || {
    [ "$rc" -eq 1 ] && { printf 'FAIL: the install itself failed, so the first boot was never reached\n' >&2; exit 1; }
    die "the wizard could not be driven to a verdict"
  }
  say "installed"
fi

# ---- phase 2: the offline first boot ----

say "booting the installed machine with its link down"
boot installed
# Before the guest's firmware has handed over, let alone before userspace. The
# NIC reports no carrier for the whole of the boot that matters.
qmp cmd set_link '{"name": "nic0", "up": false}' >/dev/null

say "waiting up to ${LAYER_TIMEOUT}s for the machine to lay its components down and reboot"
REBOOTED=1
# `guest: true` rather than any RESET: a reset the host asked for and a reboot
# the guest performed are the same event with a different reason, and only the
# second one is the unit's last act.
qmp wait RESET "$LAYER_TIMEOUT" '{"guest": true}' >/dev/null 2>&1 || REBOOTED=0
[ "$REBOOTED" = 1 ] && say "it rebooted"

# ---- phase 3: read the result ----

# Only now, and it cannot change the answer: the unit's second condition is
# `ConditionPathExists=!/usr/bin/alloy`, so a machine that has a console does
# not run it again, and one that does not have a console failed before this.
say "putting the link back up"
qmp cmd set_link '{"name": "nic0", "up": true}' >/dev/null

say "waiting up to ${SSH_TIMEOUT}s for ssh"
reachable=0
end=$((SECONDS + SSH_TIMEOUT))
while [ "$SECONDS" -lt "$end" ]; do
  if echo true | guest >/dev/null 2>&1; then reachable=1; break; fi
  sleep 5
done
[ "$reachable" = 1 ] || {
  shutdown_vm
  die "the machine never answered ssh, so nothing can be read off it. \
state/serial-installed.log and state/qemu-installed.log are what there is."
}

# 2. The console got laid down, read together with 1 because the pair is what
# says which run this was. The unit's own conditions make "no reboot" mean two
# different things, and reporting them the same way is how a scenario run
# against the wrong disk gets filed as a regression.
CONSOLE=1
echo 'test -x /usr/bin/alloy' | guest >/dev/null 2>&1 || CONSOLE=0

if [ "$REBOOTED" = 0 ] && [ "$CONSOLE" = 1 ]; then
  # ConditionPathExists=!/usr/bin/alloy was already false, so the unit was a
  # no-op and nothing here was measured. Not a defect, and not a pass either.
  shutdown_vm
  die "this machine had already been first-booted: it has a console and never \
ran the unit. Reset state/ and install again, or drop --keep-state."
fi

if [ "$REBOOTED" = 0 ]; then
  fail "the machine never rebooted, so alloy-layer-components.service did not finish"
fi
if [ "$CONSOLE" = 1 ]; then
  say "/usr/bin/alloy: present ($(echo '/usr/bin/alloy --version' | guest 2>/dev/null | head -1))"
else
  fail "/usr/bin/alloy is absent: the machine came up with no console"
  printf '\nwhat the unit said:\n' >&2
  echo 'journalctl -u alloy-layer-components.service --no-pager -o cat' \
    | guest 2>&1 | tail -40 >&2 || true
fi

# 3. The repo set. Read from both /etc and the image's own /usr/etc, because
# `alloy-layer-repos on` restores one from the other and comparing them is the
# assertion in its own terms. The count is asserted separately: deriving the
# expectation from /usr/etc alone would follow a change to the image silently,
# and "alloy-local plus Fedora's four" is a number somebody decided.
# Fed on stdin rather than quoted into the ssh command line, so the awk program
# is written once in its own syntax instead of through two layers of shell
# quoting. A repo file can carry more than one section, so the id is tracked
# rather than taken from the filename.
enabled_repos() {
  guest "$1" <<'AWKEOF' 2>/dev/null || true
awk '
  /^\[/ { id = substr($0, 2, index($0, "]") - 2) }
  /^enabled[ \t]*=[ \t]*1[ \t]*$/ { if (id != "") print id }
' "$1"/*.repo | sort -u
AWKEOF
}
LIVE_REPOS="$(enabled_repos /etc/yum.repos.d)"
IMAGE_REPOS="$(enabled_repos /usr/etc/yum.repos.d)"

if [ -z "$LIVE_REPOS" ]; then
  fail "no repo is enabled on the machine at all, so the fence was never lowered"
elif [ "$LIVE_REPOS" != "$IMAGE_REPOS" ]; then
  fail "the enabled repo set does not match the image's own"
  printf '  /etc:     %s\n' "$(echo "$LIVE_REPOS" | paste -sd' ')" >&2
  printf '  /usr/etc: %s\n' "$(echo "$IMAGE_REPOS" | paste -sd' ')" >&2
else
  count="$(printf '%s\n' "$LIVE_REPOS" | wc -l)"
  echo "$LIVE_REPOS" | grep -qx 'alloy-local' \
    || fail "alloy-local is not enabled, so the carried repo is not reachable"
  [ "$count" -eq 5 ] \
    || fail "$count repos are enabled, not the five this expects (alloy-local plus Fedora's four): $(echo "$LIVE_REPOS" | paste -sd' ')"
  [ "$count" -eq 5 ] && say "repos: $(echo "$LIVE_REPOS" | paste -sd' ')"
fi

# 4. And the labels, folded in because this is the only place an install from
# the wizard is standing still and reachable. Copied and then run rather than
# fed on stdin: sudo wants the password there.
say "running check-installed.sh on the guest"

# Delivered by writing it through a shell rather than with scp. scp needs the
# sftp subsystem at the far end and a working PATH there, which is a second
# thing to go wrong for no benefit; `cat >` needs a shell, which is already
# proven by every assertion above. Measured 2026-08-26: scp failed here and
# the run reported a machine defect for it.
#
# `sh -c` here rather than `sh -s`, because stdin is carrying the file. The
# string has no metacharacter nushell would take differently, which is what
# makes it safe to send one through.
#
# Two calls, not one, because the password and the script cannot share stdin.
# `sudo -S` reads a bufferful, not a line, so piping the password and then the
# script into `sudo -S bash -s` can have sudo swallow the top of the script.
CI_RAN=0
ci=0
if ! ssh "${SSH_OPTS[@]}" "$USER_@127.0.0.1" "sh -c 'cat > /tmp/check-installed.sh'" \
     < "$REPO_ROOT/build/check-installed.sh"; then
  printf 'note: could not write check-installed.sh to the guest (above)\n' >&2
else
  CI_RAN=1
  guest <<EOF || ci=$?
printf '%s\n' '$PASSWORD' | sudo -S -p '' bash /tmp/check-installed.sh
EOF
fi

if [ "$CI_RAN" = 0 ]; then
  : # reported below, as a run problem rather than a machine one
else
  case "$ci" in
    0) say "check-installed: clean" ;;
    1) fail "check-installed found a defect on the installed machine (above)" ;;
    # 3 is check-installed's own "could not run" (not root, no restorecon, an
    # install from the selinux=0 entry). Its verdict, and not this machine's.
    *) printf 'note: check-installed could not run (exit %s)\n' "$ci"; CI_RAN=0 ;;
  esac
fi

shutdown_vm

if [ "$FAILED" -ne 0 ]; then
  printf '\n%s check(s) failed. The machine did not survive an offline first boot.\n' "$FAILED" >&2
  exit 1
fi

# The three assertions this test is FOR have passed by here. The fold-in has
# not run, and calling that green would be claiming a check that did not
# happen, while calling it a failure would blame the machine for the harness.
if [ "$CI_RAN" = 0 ]; then
  printf '\nThe offline first boot passed: a console was laid down and the repo set is the ordinary one.\n'
  die "but check-installed.sh did not run, so the labels are unchecked"
fi

printf '\nAn offline first boot laid the console down, left the ordinary repo set, and labelled /etc correctly.\n'