Skip to main content

max / makenotwork

9.8 KB · 243 lines History Blame Raw
1 #!/usr/bin/env bash
2 # Fail loudly when the testnot.work demo goes thin.
3 #
4 # Phase 5 of wiki `testnot-demo-content-plan`. The failure this exists for:
5 # nine of eleven items went invisible on testnot and nothing said so for weeks.
6 # The seed's own tests passed the whole time, because tests/workflows/seed_examples.rs
7 # runs against InMemoryStorage where media always attaches and no project-level
8 # paywall hides anything. The failure lived exactly in the gap between the test
9 # and the box, so more unit tests cannot close it. This runs against the live
10 # site instead.
11 #
12 # It checks thinness, not 500s. Every page here can return a cheerful 200 while
13 # showing nothing, and that is the state it is looking for.
14 #
15 # Run it after every reseed (mnw-testnot-seed.sh) and before any capture run
16 # that photographs this box. The landing carousel will carry screenshots taken
17 # from here, so a silent reseed that empties the catalog also silently
18 # invalidates the marketing images.
19 #
20 # ./mnw-testnot-smoke.sh # check testnot.work
21 # BASE=https://testnot.work ./mnw-testnot-smoke.sh
22 #
23 # Exit 0 = the demo is worth showing. Exit 1 = it is not; read the FAIL lines.
24 set -uo pipefail
25
26 BASE="${BASE:-https://testnot.work}"
27
28 # ── Sealed baselines ────────────────────────────────────────────────────────
29 #
30 # Same idea as tests/test_hygiene.rs: freeze what is true today and fail on a
31 # new violation, rather than demanding a clean sheet nobody will ever deliver.
32 # A number here may only move in the improving direction, and when it does, edit
33 # it. Never loosen one to make a run pass.
34
35 # Projects the demo must carry. Below this the catalog stops reading as a
36 # catalog and starts reading as a test fixture.
37 MIN_PROJECTS=5
38
39 # Items a visitor can actually see and click, summed across every storefront
40 # that is not paywalled. Twenty of twenty-five on 2026-08-07; the other five are
41 # behind the one deliberate subscription paywall (the Marginalia Reader).
42 #
43 # Was nine of eleven until the catalog was widened to five items per project, so
44 # that three-cards-in-a-five-wide-grid stopped reading as an empty shop.
45 MIN_VISIBLE_ITEMS=20
46
47 # Covers still served from the seed's generated placeholder (a 16x16 grayscale
48 # PNG, src/seed/media.rs). Zero as of 2026-08-07: src/seed/media-manifest.toml
49 # now names a real CC0 asset for every cover the seed attaches, so a placeholder
50 # appearing at all means either the manifest stopped resolving or storage did.
51 #
52 # It was sealed at five while the seed had never uploaded real art, on the
53 # reasoning that a check which is always red is a check nobody reads. That
54 # reason is spent: zero is now the true value, and the baseline may only move
55 # in the improving direction.
56 PLACEHOLDER_COVERS_BASELINE=0
57
58 # A cover at or under this is the placeholder or a broken upload, not artwork.
59 # The placeholder is 72 bytes; real cover art is orders of magnitude larger.
60 REAL_COVER_MIN_BYTES=2048
61
62 # A page under this rendered its chrome and nothing else.
63 MIN_PAGE_BYTES=3000
64
65 TMP="$(mktemp -d)"
66 trap 'rm -rf "$TMP"' EXIT
67
68 fails=0
69 warns=0
70 pass() { printf ' ok %s\n' "$*"; }
71 fail() { printf ' FAIL %s\n' "$*"; fails=$((fails + 1)); }
72 warn() { printf ' warn %s\n' "$*"; warns=$((warns + 1)); }
73 section() { printf '\n== %s\n' "$*"; }
74
75 # Fetch $1 into $TMP/body, echo the status code.
76 get() {
77 curl -sS --max-time 30 -o "$TMP/body" -w '%{http_code}' "$BASE$1" 2>/dev/null || echo 000
78 }
79
80 section "reachable"
81 code=$(get /)
82 if [ "$code" != "200" ]; then
83 fail "GET / returned $code; nothing else below is meaningful"
84 printf '\n%s\n' "smoke: FAILED ($fails)"
85 exit 1
86 fi
87 pass "GET / -> 200"
88
89 # ── The catalog ─────────────────────────────────────────────────────────────
90 #
91 # Discover is the index and each storefront is the page it points at. Checking
92 # them against each other is the whole trick: either alone can look healthy
93 # while disagreeing with the other, which is how the paywall bug hid.
94
95 section "catalog"
96 code=$(get /discover)
97 [ "$code" = "200" ] || fail "GET /discover returned $code"
98
99 # Discover's project rows carry both the slug and the item count it advertises.
100 grep -oE '<a href="/p/[a-z0-9-]+" class="table-row project-row">|<span class="row-items">[0-9]+</span>' "$TMP/body" |
101 sed -e 's/.*href="\/p\///' -e 's/" class.*//' -e 's/<span class="row-items">//' -e 's/<\/span>//' |
102 paste - - >"$TMP/advertised" 2>/dev/null
103
104 n_projects=$(wc -l <"$TMP/advertised")
105 if [ "$n_projects" -lt "$MIN_PROJECTS" ]; then
106 fail "discover lists $n_projects projects, floor is $MIN_PROJECTS"
107 else
108 pass "discover lists $n_projects projects"
109 fi
110
111 visible_total=0
112 : >"$TMP/covers"
113
114 while read -r slug advertised; do
115 [ -n "$slug" ] || continue
116 code=$(get "/p/$slug")
117 if [ "$code" != "200" ]; then
118 fail "/p/$slug returned $code"
119 continue
120 fi
121
122 rendered=$(grep -o 'class="item-card' "$TMP/body" | wc -l)
123 grep -oE 'src="https://[^"]*/image/[^"]*"' "$TMP/body" | sed -e 's/src="//' -e 's/"$//' >>"$TMP/covers"
124
125 if grep -q 'class="paywall-box"' "$TMP/body"; then
126 # A paywalled storefront shows no items by construction: the handler
127 # returns the paywall template before it reaches them. What it must
128 # still do is state the count, or it contradicts the discover card that
129 # sent the visitor here.
130 if [ "$rendered" -ne 0 ]; then
131 fail "/p/$slug is paywalled but rendered $rendered item cards"
132 elif grep -qE "(^|>)[^<]*\b$advertised items? included" "$TMP/body"; then
133 pass "/p/$slug paywalled, states its $advertised items"
134 else
135 fail "/p/$slug is paywalled and does not state its $advertised items; discover advertises them"
136 fi
137 else
138 visible_total=$((visible_total + rendered))
139 if [ "$rendered" -eq "$advertised" ]; then
140 pass "/p/$slug renders $rendered items, matching discover"
141 else
142 fail "/p/$slug renders $rendered items, discover advertises $advertised"
143 fi
144 fi
145 done <"$TMP/advertised"
146
147 if [ "$visible_total" -lt "$MIN_VISIBLE_ITEMS" ]; then
148 fail "$visible_total items visible across open storefronts, floor is $MIN_VISIBLE_ITEMS"
149 else
150 pass "$visible_total items visible across open storefronts"
151 fi
152
153 # ── Covers ──────────────────────────────────────────────────────────────────
154 #
155 # A cover URL that 404s still renders as a 200 page with a broken image, which
156 # no status-code check sees. Fetch each one.
157
158 section "covers"
159 sort -u "$TMP/covers" >"$TMP/covers.uniq"
160 n_covers=$(wc -l <"$TMP/covers.uniq")
161 [ "$n_covers" -gt 0 ] || fail "no cover images referenced by any storefront"
162
163 placeholders=0
164 while read -r url; do
165 [ -n "$url" ] || continue
166 read -r ccode csize ctype <<<"$(curl -sS --max-time 30 -o /dev/null \
167 -w '%{http_code} %{size_download} %{content_type}' "$url" 2>/dev/null || echo "000 0 -")"
168 short="${url##*/projects/}"
169 case "$ctype" in
170 image/*) is_image=1 ;;
171 *) is_image=0 ;;
172 esac
173 if [ "$ccode" != "200" ]; then
174 fail "cover $short returned $ccode"
175 elif [ "$is_image" -eq 0 ]; then
176 fail "cover $short served as ${ctype:--}, not an image"
177 elif [ "$csize" -lt "$REAL_COVER_MIN_BYTES" ]; then
178 placeholders=$((placeholders + 1))
179 fi
180 done <"$TMP/covers.uniq"
181
182 if [ "$placeholders" -gt "$PLACEHOLDER_COVERS_BASELINE" ]; then
183 fail "$placeholders placeholder covers, sealed baseline is $PLACEHOLDER_COVERS_BASELINE (a real cover reverted)"
184 elif [ "$placeholders" -eq "$PLACEHOLDER_COVERS_BASELINE" ] && [ "$placeholders" -gt 0 ]; then
185 warn "$placeholders of $n_covers covers are still the seed placeholder; at baseline, not a regression"
186 else
187 pass "$placeholders placeholder covers, down from a baseline of $PLACEHOLDER_COVERS_BASELINE. Lower PLACEHOLDER_COVERS_BASELINE to $placeholders"
188 fi
189
190 # ── The surfaces that are not the catalog ───────────────────────────────────
191 #
192 # Phase 4's list. Each is a place a curious visitor lands, and each can go
193 # blank without anything else noticing.
194
195 section "other surfaces"
196 for path in /discover?mode=items /pricing /fan-plus /docs; do
197 code=$(get "$path")
198 size=$(wc -c <"$TMP/body")
199 if [ "$code" != "200" ]; then
200 fail "$path returned $code"
201 elif [ "$size" -lt "$MIN_PAGE_BYTES" ]; then
202 fail "$path returned 200 but only ${size}B, which is chrome and no content"
203 else
204 pass "$path -> 200, ${size}B"
205 fi
206 done
207
208 section "blogs and feeds"
209 while read -r slug _; do
210 [ -n "$slug" ] || continue
211 code=$(get "/p/$slug/blog")
212 [ "$code" = "200" ] || fail "/p/$slug/blog returned $code"
213
214 # Note the path: the blog feed is feed.xml, not rss. `/p/{slug}/rss` is a
215 # different feed, of the project's items.
216 code=$(get "/p/$slug/blog/feed.xml")
217 posts=$(grep -c '<item>' "$TMP/body" 2>/dev/null || echo 0)
218 if [ "$code" != "200" ]; then
219 fail "/p/$slug/blog/feed.xml returned $code"
220 elif [ "$posts" -lt 1 ]; then
221 fail "/p/$slug/blog/feed.xml is an empty feed"
222 else
223 pass "/p/$slug blog + feed ($posts posts)"
224 fi
225 done <"$TMP/advertised"
226
227 # ── Deferred ────────────────────────────────────────────────────────────────
228
229 section "deferred"
230 code=$(get /library)
231 if [ "$code" = "401" ] || [ "$code" = "302" ] || [ "$code" = "303" ]; then
232 warn "/library is $code to anonymous, as expected. Checking its contents needs the demo buyer, which the seed creates when TESTNOT_BUYER_PASSWORD is set (GoingsOn 839a8e5a, option B). The carousel capture is what exercises it"
233 else
234 warn "/library returned $code to an anonymous visitor; expected an auth redirect. Worth a look"
235 fi
236
237 printf '\n'
238 if [ "$fails" -gt 0 ]; then
239 printf 'smoke: FAILED (%d failures, %d warnings)\n' "$fails" "$warns"
240 exit 1
241 fi
242 printf 'smoke: ok (%d warnings)\n' "$warns"
243