Skip to main content

max / makenotwork

test-fuzz: cover all 8 storage confirm failure branches via DB fault injection Trigger-based fault injection (no production test hooks): BEFORE UPDATE triggers on items/versions/projects, keyed on a sentinel in the staged key — 'faultlr' makes the guarded UPDATE affect zero rows (lost-race branch Ok(false)/Ok(0)), 'faulterr' makes it RAISE (commit-Err branch). Deterministic, no concurrency, scoped to each test's cloned DB and inert for any non-sentinel key. Each of the 8 tests (audio/version/project-image/item-image x lost-race/tx-error) asserts the two safety properties of a failed confirm: the storage credit rolls back (credit + CAS share one tx, handler returns without committing) and the staged key is orphan-enqueued with the handler's reason string. This closes the gap the CAS-predicate pins left open (predicate rejection vs. handler reaction). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-17 23:58 UTC
Commit: cf99d6659dc58c8abb9dd5d5384083373bb0652b
Parent: 752046e
1 file changed, +280 insertions, -0 deletions
M server/tests/workflows/storage.rs +280
@@ -1071,3 +1071,283 @@ async fn update_item_cover_guards_against_stale_confirm() {
1071 1071 assert_eq!(url.as_deref(), Some("u1"));
1072 1072 assert_eq!(sz, Some(1000));
1073 1073 }
1074 +
1075 + // ---------------------------------------------------------------------------
1076 + // Confirm failure-branch coverage via DB fault injection (test-fuzz Phase 2.2+)
1077 + //
1078 + // The CAS-guard pins above prove the predicate REJECTS a stale write. They do
1079 + // NOT exercise the handler's REACTION to a rejection — the storage-credit
1080 + // rollback (the credit and the CAS run in one tx; the handler returns without
1081 + // committing) and the orphan-enqueue of the staged key. Those branches only
1082 + // fire under a true concurrent confirm or a mid-tx DB error, neither of which a
1083 + // sequential test drives directly.
1084 + //
1085 + // We inject the fault at the DB layer with BEFORE UPDATE triggers keyed on a
1086 + // sentinel embedded in the uploaded key: a "faultlr" marker makes the guarded
1087 + // UPDATE affect zero rows (RETURN NULL) — the lost-race branch (Ok(false)/Ok(0));
1088 + // a "faulterr" marker makes it RAISE — the commit-Err branch. Both are fully
1089 + // deterministic, need no concurrency, and are scoped to this test's cloned DB,
1090 + // so no production code carries a test hook and no other test is affected (the
1091 + // triggers are inert for any key without the marker).
1092 + //
1093 + // Each test asserts the two safety properties of the failed confirm: storage is
1094 + // NOT credited (the tx rolled back) and the staged S3 key is orphan-enqueued
1095 + // with the handler's reason string (so the reaper cleans it and a blind delete
1096 + // never clobbers a live object).
1097 + // ---------------------------------------------------------------------------
1098 +
1099 + /// Install confirm-time fault triggers on items/versions/projects in this test's
1100 + /// cloned DB. A key containing `faultlr` skips the guarded UPDATE (zero rows ->
1101 + /// lost-race branch); a key containing `faulterr` raises (commit-Err branch).
1102 + async fn install_confirm_fault_triggers(pool: &sqlx::PgPool) {
1103 + let stmts = [
1104 + r#"CREATE OR REPLACE FUNCTION test_confirm_fault_items() RETURNS trigger AS $$
1105 + BEGIN
1106 + IF COALESCE(NEW.audio_s3_key,'') LIKE '%faulterr%' OR COALESCE(NEW.cover_s3_key,'') LIKE '%faulterr%' THEN
1107 + RAISE EXCEPTION 'injected confirm fault (items)';
1108 + END IF;
1109 + IF COALESCE(NEW.audio_s3_key,'') LIKE '%faultlr%' OR COALESCE(NEW.cover_s3_key,'') LIKE '%faultlr%' THEN
1110 + RETURN NULL;
1111 + END IF;
1112 + RETURN NEW;
1113 + END; $$ LANGUAGE plpgsql"#,
1114 + "DROP TRIGGER IF EXISTS test_confirm_fault_items ON items",
1115 + "CREATE TRIGGER test_confirm_fault_items BEFORE UPDATE ON items FOR EACH ROW EXECUTE FUNCTION test_confirm_fault_items()",
1116 + r#"CREATE OR REPLACE FUNCTION test_confirm_fault_versions() RETURNS trigger AS $$
1117 + BEGIN
1118 + IF COALESCE(NEW.s3_key,'') LIKE '%faulterr%' THEN RAISE EXCEPTION 'injected confirm fault (versions)'; END IF;
1119 + IF COALESCE(NEW.s3_key,'') LIKE '%faultlr%' THEN RETURN NULL; END IF;
1120 + RETURN NEW;
1121 + END; $$ LANGUAGE plpgsql"#,
1122 + "DROP TRIGGER IF EXISTS test_confirm_fault_versions ON versions",
1123 + "CREATE TRIGGER test_confirm_fault_versions BEFORE UPDATE ON versions FOR EACH ROW EXECUTE FUNCTION test_confirm_fault_versions()",
1124 + r#"CREATE OR REPLACE FUNCTION test_confirm_fault_projects() RETURNS trigger AS $$
1125 + BEGIN
1126 + IF COALESCE(NEW.cover_image_url,'') LIKE '%faulterr%' THEN RAISE EXCEPTION 'injected confirm fault (projects)'; END IF;
1127 + IF COALESCE(NEW.cover_image_url,'') LIKE '%faultlr%' THEN RETURN NULL; END IF;
1128 + RETURN NEW;
1129 + END; $$ LANGUAGE plpgsql"#,
1130 + "DROP TRIGGER IF EXISTS test_confirm_fault_projects ON projects",
1131 + "CREATE TRIGGER test_confirm_fault_projects BEFORE UPDATE ON projects FOR EACH ROW EXECUTE FUNCTION test_confirm_fault_projects()",
1132 + ];
1133 + for s in stmts {
1134 + sqlx::query(s).execute(pool).await.expect("install fault trigger");
1135 + }
1136 + }
1137 +
1138 + /// Assert a failed confirm rolled back the storage credit AND orphan-enqueued the
1139 + /// staged key with the expected reason.
1140 + async fn assert_uncredited_and_orphaned(
1141 + h: &TestHarness,
1142 + user_id: &str,
1143 + s3_key: &str,
1144 + before: i64,
1145 + expected_source: &str,
1146 + ) {
1147 + assert_eq!(
1148 + storage_used(h, user_id).await,
1149 + before,
1150 + "a failed confirm must roll back the storage credit"
1151 + );
1152 + let source: Option<String> =
1153 + sqlx::query_scalar("SELECT source FROM pending_s3_deletions WHERE s3_key = $1")
1154 + .bind(s3_key)
1155 + .fetch_optional(&h.db)
1156 + .await
1157 + .unwrap();
1158 + assert_eq!(
1159 + source.as_deref(),
1160 + Some(expected_source),
1161 + "a failed confirm must orphan-enqueue the staged key with the handler's reason"
1162 + );
1163 + }
1164 +
1165 + async fn presign_project_image(h: &mut TestHarness, project_id: &str, file_name: &str) -> String {
1166 + let resp = h
1167 + .client
1168 + .post_json(
1169 + "/api/projects/image/presign",
1170 + &json!({"project_id": project_id, "file_name": file_name, "content_type": "image/png"}).to_string(),
1171 + )
1172 + .await;
1173 + assert!(resp.status.is_success(), "project image presign failed: {}", resp.text);
1174 + let data: Value = resp.json();
1175 + data["s3_key"].as_str().unwrap().to_string()
1176 + }
1177 +
1178 + async fn presign_item_image(h: &mut TestHarness, item_id: &str, file_name: &str) -> String {
1179 + let resp = h
1180 + .client
1181 + .post_json(
1182 + "/api/items/image/presign",
1183 + &json!({"item_id": item_id, "file_name": file_name, "content_type": "image/png"}).to_string(),
1184 + )
1185 + .await;
1186 + assert!(resp.status.is_success(), "item image presign failed: {}", resp.text);
1187 + let data: Value = resp.json();
1188 + data["s3_key"].as_str().unwrap().to_string()
1189 + }
1190 +
1191 + // ---- uploads (item file) -------------------------------------------------
1192 +
1193 + #[tokio::test]
1194 + async fn confirm_audio_lost_race_rolls_back_and_orphans() {
1195 + let mut h = TestHarness::with_storage().await;
1196 + let (user_id, _proj, item_id) = setup_creator_with_item(&mut h, 0).await;
1197 + install_confirm_fault_triggers(&h.db).await;
1198 +
1199 + let s3_key = presign_audio(&mut h, &item_id, "faultlr.mp3").await;
1200 + h.storage.as_ref().unwrap().put(&s3_key, b"x".to_vec());
1201 + let before = storage_used(&h, &user_id).await;
1202 +
1203 + let resp = h
1204 + .client
1205 + .post_json("/api/upload/confirm", &json!({"item_id": item_id, "file_type": "audio", "s3_key": s3_key}).to_string())
1206 + .await;
1207 + assert_eq!(resp.status.as_u16(), 400, "lost-race confirm must 400: {}", resp.text);
1208 + assert_uncredited_and_orphaned(&h, &user_id, &s3_key, before, "item_upload_target_missing").await;
1209 + }
1210 +
1211 + #[tokio::test]
1212 + async fn confirm_audio_tx_error_rolls_back_and_orphans() {
1213 + let mut h = TestHarness::with_storage().await;
1214 + let (user_id, _proj, item_id) = setup_creator_with_item(&mut h, 0).await;
1215 + install_confirm_fault_triggers(&h.db).await;
1216 +
1217 + let s3_key = presign_audio(&mut h, &item_id, "faulterr.mp3").await;
1218 + h.storage.as_ref().unwrap().put(&s3_key, b"x".to_vec());
1219 + let before = storage_used(&h, &user_id).await;
1220 +
1221 + let resp = h
1222 + .client
1223 + .post_json("/api/upload/confirm", &json!({"item_id": item_id, "file_type": "audio", "s3_key": s3_key}).to_string())
1224 + .await;
1225 + assert!(resp.status.is_server_error(), "tx-error confirm must 5xx: {} {}", resp.status, resp.text);
1226 + assert_uncredited_and_orphaned(&h, &user_id, &s3_key, before, "item_confirm_failed").await;
1227 + }
1228 +
1229 + // ---- versions ------------------------------------------------------------
1230 +
1231 + async fn make_version(h: &mut TestHarness, item_id: &str) -> String {
1232 + let resp = h
1233 + .client
1234 + .post_json(&format!("/api/items/{}/versions", item_id), &json!({"version_number": "1.0.0"}).to_string())
1235 + .await;
1236 + assert!(resp.status.is_success(), "create version failed: {}", resp.text);
1237 + let v: Value = resp.json();
1238 + v["id"].as_str().unwrap().to_string()
1239 + }
1240 +
1241 + #[tokio::test]
1242 + async fn confirm_version_lost_race_rolls_back_and_orphans() {
1243 + let mut h = TestHarness::with_storage().await;
1244 + let (user_id, _proj, item_id) = setup_creator_with_item(&mut h, 0).await;
1245 + let version_id = make_version(&mut h, &item_id).await;
1246 + install_confirm_fault_triggers(&h.db).await;
1247 +
1248 + let s3_key = presign_version(&mut h, &version_id, "faultlr.zip").await;
1249 + h.storage.as_ref().unwrap().put(&s3_key, b"x".to_vec());
1250 + let before = storage_used(&h, &user_id).await;
1251 +
1252 + let resp = h
1253 + .client
1254 + .post_json(&format!("/api/versions/{}/upload/confirm", version_id), &json!({"s3_key": s3_key}).to_string())
1255 + .await;
1256 + assert_eq!(resp.status.as_u16(), 400, "lost-race version confirm must 400: {}", resp.text);
1257 + assert_uncredited_and_orphaned(&h, &user_id, &s3_key, before, "version_confirm_lost_race").await;
1258 + }
1259 +
1260 + #[tokio::test]
1261 + async fn confirm_version_tx_error_rolls_back_and_orphans() {
1262 + let mut h = TestHarness::with_storage().await;
1263 + let (user_id, _proj, item_id) = setup_creator_with_item(&mut h, 0).await;
1264 + let version_id = make_version(&mut h, &item_id).await;
1265 + install_confirm_fault_triggers(&h.db).await;
1266 +
1267 + let s3_key = presign_version(&mut h, &version_id, "faulterr.zip").await;
1268 + h.storage.as_ref().unwrap().put(&s3_key, b"x".to_vec());
1269 + let before = storage_used(&h, &user_id).await;
1270 +
1271 + let resp = h
1272 + .client
1273 + .post_json(&format!("/api/versions/{}/upload/confirm", version_id), &json!({"s3_key": s3_key}).to_string())
1274 + .await;
1275 + assert!(resp.status.is_server_error(), "tx-error version confirm must 5xx: {} {}", resp.status, resp.text);
1276 + assert_uncredited_and_orphaned(&h, &user_id, &s3_key, before, "version_confirm_failed").await;
1277 + }
1278 +
1279 + // ---- project cover image -------------------------------------------------
1280 +
1281 + #[tokio::test]
1282 + async fn confirm_project_image_lost_race_rolls_back_and_orphans() {
1283 + let mut h = TestHarness::with_storage().await;
1284 + let (user_id, project_id, _item) = setup_creator_with_item(&mut h, 0).await;
1285 + install_confirm_fault_triggers(&h.db).await;
1286 +
1287 + let s3_key = presign_project_image(&mut h, &project_id, "faultlr.png").await;
1288 + h.storage.as_ref().unwrap().put(&s3_key, b"x".to_vec());
1289 + let before = storage_used(&h, &user_id).await;
1290 +
1291 + let resp = h
1292 + .client
1293 + .post_json("/api/projects/image/confirm", &json!({"project_id": project_id, "s3_key": s3_key}).to_string())
1294 + .await;
1295 + assert_eq!(resp.status.as_u16(), 400, "lost-race project image confirm must 400: {}", resp.text);
1296 + assert_uncredited_and_orphaned(&h, &user_id, &s3_key, before, "project_image_update_failed").await;
1297 + }
1298 +
1299 + #[tokio::test]
1300 + async fn confirm_project_image_tx_error_rolls_back_and_orphans() {
1301 + let mut h = TestHarness::with_storage().await;
1302 + let (user_id, project_id, _item) = setup_creator_with_item(&mut h, 0).await;
1303 + install_confirm_fault_triggers(&h.db).await;
1304 +
1305 + let s3_key = presign_project_image(&mut h, &project_id, "faulterr.png").await;
1306 + h.storage.as_ref().unwrap().put(&s3_key, b"x".to_vec());
1307 + let before = storage_used(&h, &user_id).await;
1308 +
1309 + let resp = h
1310 + .client
1311 + .post_json("/api/projects/image/confirm", &json!({"project_id": project_id, "s3_key": s3_key}).to_string())
1312 + .await;
1313 + assert!(resp.status.is_server_error(), "tx-error project image confirm must 5xx: {} {}", resp.status, resp.text);
1314 + assert_uncredited_and_orphaned(&h, &user_id, &s3_key, before, "project_image_update_failed").await;
1315 + }
1316 +
1317 + // ---- item cover image ----------------------------------------------------
1318 +
1319 + #[tokio::test]
1320 + async fn confirm_item_image_lost_race_rolls_back_and_orphans() {
1321 + let mut h = TestHarness::with_storage().await;
1322 + let (user_id, _proj, item_id) = setup_creator_with_item(&mut h, 0).await;
1323 + install_confirm_fault_triggers(&h.db).await;
1324 +
1325 + let s3_key = presign_item_image(&mut h, &item_id, "faultlr.png").await;
1326 + h.storage.as_ref().unwrap().put(&s3_key, b"x".to_vec());
1327 + let before = storage_used(&h, &user_id).await;
1328 +
1329 + let resp = h
1330 + .client
1331 + .post_json("/api/items/image/confirm", &json!({"item_id": item_id, "s3_key": s3_key}).to_string())
1332 + .await;
1333 + assert_eq!(resp.status.as_u16(), 400, "lost-race item image confirm must 400: {}", resp.text);
1334 + assert_uncredited_and_orphaned(&h, &user_id, &s3_key, before, "item_image_update_failed").await;
1335 + }
1336 +
1337 + #[tokio::test]
1338 + async fn confirm_item_image_tx_error_rolls_back_and_orphans() {
1339 + let mut h = TestHarness::with_storage().await;
1340 + let (user_id, _proj, item_id) = setup_creator_with_item(&mut h, 0).await;
1341 + install_confirm_fault_triggers(&h.db).await;
1342 +
1343 + let s3_key = presign_item_image(&mut h, &item_id, "faulterr.png").await;
1344 + h.storage.as_ref().unwrap().put(&s3_key, b"x".to_vec());
1345 + let before = storage_used(&h, &user_id).await;
1346 +
1347 + let resp = h
1348 + .client
1349 + .post_json("/api/items/image/confirm", &json!({"item_id": item_id, "s3_key": s3_key}).to_string())
1350 + .await;
1351 + assert!(resp.status.is_server_error(), "tx-error item image confirm must 5xx: {} {}", resp.status, resp.text);
1352 + assert_uncredited_and_orphaned(&h, &user_id, &s3_key, before, "item_image_update_failed").await;
1353 + }