Skip to main content

max / alloy

Tell a base pin that moved from one that is gone --check reported both as STALE, so a tag advancing (normal, still pullable) looked like a digest 404ing (the image cannot be built at all). A check that is amber on ordinary business is one nobody reads on the night it matters. Only a dead pin exits 1 now. A failure to reach the registry exits 3 and says `error:`, which is not a claim about the pins either way; an ERR trap makes an unguarded abort say the same rather than exiting nonzero in silence.
Author: Max Johnson <me@maxj.phd> · 2026-08-07 01:51 UTC
Signed with PGP, not checked
Commit: 486f71f245bfb7912c416322181e48df4ec494e1
Parent: 1732826
1 file changed, +102 insertions, -27 deletions
M build/refresh-base-digests.sh +102 -27
@@ -20,12 +20,30 @@
20 20 #
21 21 # Usage:
22 22 # build/refresh-base-digests.sh # rewrite the Containerfile
23 - # build/refresh-base-digests.sh --check # exit 1 if the pins are stale
23 + # build/refresh-base-digests.sh --check # exit 1 if a pin is unpullable
24 24 #
25 - # --check is what a periodic reminder would run. It reports and changes
26 - # nothing, so it is safe to wire into anything.
25 + # --check reports and changes nothing, so it is safe to wire into anything. The
26 + # astra sweep runs it nightly (`base-pins` in sweep.toml).
27 + #
28 + # It separates two states that used to report identically as STALE and are not
29 + # the same thing at all:
30 + #
31 + # MOVED the tag now points somewhere else. The pin still resolves, the image
32 + # still builds, and moving it is a decision rather than maintenance.
33 + # Expected, and not a failure.
34 + # DEAD the pinned digest is gone from the registry. Nobody can build the
35 + # image, and a machine that already has the layers cannot tell. Alloy
36 + # publishes nothing and the install path is "build it yourself"
37 + # (docs/IMAGE.md), so this is the whole install path broken.
38 + #
39 + # Only DEAD exits nonzero, so a red cell means unbuildable rather than behind.
40 + # Both quay.io base digests were DEAD on 2026-08-06 and nothing noticed until a
41 + # build ran on a host with a cold container store.
42 + #
43 + # Anything else (no network, no token, a registry answering 500) exits 3 and says
44 + # `error:`, which is not a claim about the pins either way.
27 45
28 - set -euo pipefail
46 + set -Eeuo pipefail
29 47
30 48 REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
31 49 CONTAINERFILE="$REPO_ROOT/Containerfile"
@@ -40,11 +58,23 @@
40 58 REGISTRY="quay.io"
41 59 CHECK_ONLY=0
42 60
43 - die() { printf 'error: %s\n' "$*" >&2; exit 1; }
61 + # Exit 3, never 1: 1 is reserved for "a pin is dead", and a caller that cannot
62 + # tell a dead pin from a dead network learns the wrong thing from both.
63 + die() { printf 'error: %s\n' "$*" >&2; exit 3; }
64 +
65 + # Every unguarded failure lands here, so an aborted run says `error:` on stderr
66 + # like a deliberate one. Without it, a curl dying under `set -e` mid-substitution
67 + # would exit nonzero silently and read as a finding about the pins.
68 + #
69 + # A `die` inside a command substitution exits only that subshell, and the failed
70 + # assignment then trips this trap in every enclosing one. Status 3 is the marker
71 + # that something already said what went wrong, so it propagates without a second
72 + # and third guess at the same failure.
73 + trap 'rc=$?; [ "$rc" -eq 3 ] && exit 3; die "unexpected failure at line ${LINENO}"' ERR
44 74
45 75 case "${1:-}" in
46 76 --check) CHECK_ONLY=1 ;;
47 - -h|--help) sed -n '2,28p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
77 + -h|--help) sed -n '2,45p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
48 78 "") ;;
49 79 *) die "unknown argument: $1 (see --help)" ;;
50 80 esac
@@ -53,25 +83,53 @@
53 83 command -v python3 >/dev/null || die "python3 not found"
54 84 [ -f "$CONTAINERFILE" ] || die "no Containerfile at $CONTAINERFILE"
55 85
56 - # An anonymous pull token. quay.io serves public repos without credentials but
57 - # still wants a bearer token on the manifest endpoint.
58 - token_for() {
59 - curl -fsS "https://${REGISTRY}/v2/auth?service=${REGISTRY}&scope=repository:${1}:pull" \
60 - | python3 -c 'import sys, json; print(json.load(sys.stdin)["token"])'
61 - }
62 -
63 - # The index digest for a tag, asserted to be a real multi-arch index.
64 - #
65 86 # The Accept header decides what comes back: ask for the index types first or
66 87 # the registry answers with a single-arch manifest and the pin silently becomes
67 88 # amd64-only. That failure would not show up until someone built on astra.
89 + ACCEPT='Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json'
90 +
91 + # An anonymous pull token. quay.io serves public repos without credentials but
92 + # still wants a bearer token on the manifest endpoint.
93 + token_for() {
94 + local body
95 + body="$(curl -fsS "https://${REGISTRY}/v2/auth?service=${REGISTRY}&scope=repository:${1}:pull")" \
96 + || die "cannot reach ${REGISTRY} for a pull token on ${1}"
97 + printf '%s' "$body" | python3 -c 'import sys, json; print(json.load(sys.stdin)["token"])' \
98 + || die "${REGISTRY} returned no usable pull token for ${1}"
99 + }
100 +
101 + # Whether a reference still resolves at the registry: prints `alive`, `dead`, or
102 + # dies. This is the whole point of the check — a digest is immutable, so the only
103 + # thing that can happen to one is that it stops existing.
104 + #
105 + # Deliberately not `curl -f`: -f collapses every HTTP error into exit 22, and 404
106 + # (the pin is gone) has to stay distinguishable from 500 (the registry is having
107 + # a day). Only 200 and 404 are answers; anything else is an error about the run.
108 + manifest_state() {
109 + local repo="$1" ref="$2" token status
110 + token="$(token_for "$repo")"
111 +
112 + status="$(curl -sS -o /dev/null -w '%{http_code}' -I \
113 + -H "Authorization: Bearer ${token}" \
114 + -H "$ACCEPT" \
115 + "https://${REGISTRY}/v2/${repo}/manifests/${ref}")" \
116 + || die "cannot reach ${REGISTRY} for ${repo}"
117 +
118 + case "$status" in
119 + 200) printf 'alive' ;;
120 + 404) printf 'dead' ;;
121 + *) die "${REGISTRY}/${repo} answered ${status} for ${ref}, which is neither" ;;
122 + esac
123 + }
124 +
125 + # The index digest for a tag, asserted to be a real multi-arch index.
68 126 digest_for() {
69 127 local repo="$1" tag="$2" token body digest
70 128 token="$(token_for "$repo")"
71 129
72 130 digest="$(curl -fsSI \
73 131 -H "Authorization: Bearer ${token}" \
74 - -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \
132 + -H "$ACCEPT" \
75 133 "https://${REGISTRY}/v2/${repo}/manifests/${tag}" \
76 134 | grep -i '^docker-content-digest:' | tr -d '\r' | awk '{print $2}')"
77 135
@@ -80,7 +138,7 @@
80 138 # Prove it is an index over both build architectures before pinning it.
81 139 body="$(curl -fsS \
82 140 -H "Authorization: Bearer ${token}" \
83 - -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \
141 + -H "$ACCEPT" \
84 142 "https://${REGISTRY}/v2/${repo}/manifests/${digest}")"
85 143
86 144 printf '%s' "$body" | python3 -c '
@@ -95,7 +153,8 @@
95 153 printf '%s' "$digest"
96 154 }
97 155
98 - stale=0
156 + moved=0
157 + dead=0
99 158
100 159 for image in "${IMAGES[@]}"; do
101 160 repo="${image%:*}"
@@ -105,16 +164,23 @@
105 164 current="$(grep -oE "^FROM ${ref}@sha256:[0-9a-f]{64}" "$CONTAINERFILE" | head -1 | grep -oE 'sha256:[0-9a-f]{64}' || true)"
106 165 [ -n "$current" ] || die "no digest-pinned FROM line for ${ref} in the Containerfile"
107 166
167 + # The pin first, the tag second. Whether the tag advanced is interesting;
168 + # whether the thing we pinned can still be pulled is the finding.
169 + state="$(manifest_state "$repo" "$current")"
108 170 latest="$(digest_for "$repo" "$tag")"
109 171
110 - if [ "$current" = "$latest" ]; then
172 + if [ "$state" = "dead" ]; then
173 + dead=1
174 + printf 'DEAD %s\n pinned %s (gone from the registry)\n now %s\n' \
175 + "$ref" "$current" "$latest"
176 + elif [ "$current" = "$latest" ]; then
111 177 printf 'current %s\n %s\n' "$ref" "$current"
112 178 continue
179 + else
180 + moved=1
181 + printf 'moved %s\n was %s\n now %s\n' "$ref" "$current" "$latest"
113 182 fi
114 183
115 - stale=1
116 - printf 'STALE %s\n was %s\n now %s\n' "$ref" "$current" "$latest"
117 -
118 184 if [ "$CHECK_ONLY" -eq 0 ]; then
119 185 # Anchored at the start of the line and matching the full old digest, so
120 186 # this cannot touch a digest that happens to appear in a comment.
@@ -136,13 +202,22 @@
136 202 fi
137 203 done
138 204
139 - if [ "$CHECK_ONLY" -eq 1 ] && [ "$stale" -eq 1 ]; then
140 - echo
141 - echo "Pins are behind their tags. Run build/refresh-base-digests.sh to move them."
142 - exit 1
205 + if [ "$CHECK_ONLY" -eq 1 ]; then
206 + if [ "$dead" -eq 1 ]; then
207 + echo
208 + echo "A pinned base digest no longer exists. The image cannot be built from a"
209 + echo "cold container store until the pin moves: build/refresh-base-digests.sh"
210 + exit 1
211 + fi
212 + if [ "$moved" -eq 1 ]; then
213 + echo
214 + echo "Pins are behind their tags but still pullable. Moving them is a decision:"
215 + echo "it rebuilds everything, so run build/refresh-base-digests.sh when you mean to."
216 + fi
217 + exit 0
143 218 fi
144 219
145 - if [ "$stale" -eq 1 ]; then
220 + if [ "$moved" -eq 1 ] || [ "$dead" -eq 1 ]; then
146 221 echo
147 222 echo "Base images moved. Everything rebuilds from scratch on the next build."
148 223 fi