Skip to main content

max / makenotwork

30.4 KB · 808 lines History Blame Raw
1 //! Real public-domain / CC0 media for the example seed: the manifest that names
2 //! each asset, and the fetch step that turns it into bytes.
3 //!
4 //! The seed's other phases are pure data in `creators.rs`. Media cannot be,
5 //! because a real asset is a file somebody has to choose, license-check, and
6 //! host. So the choosing lives in `media-manifest.toml` (curated by hand, checked
7 //! into the repo, embedded in the binary) and everything downstream of it is
8 //! mechanical: fetch by URL, verify the digest, upload through the storage layer,
9 //! record the attribution.
10 //!
11 //! # Curation state is per asset, not per manifest
12 //!
13 //! An entry with no `url` is *declared but not yet curated*. It resolves to
14 //! nothing and [`super::media`] falls back to the generated placeholder for that
15 //! one slot. So the manifest doubles as the curation checklist: every id the seed
16 //! can use is listed, and the ones still carrying grey boxes are exactly the ones
17 //! with an empty `url`. Partial curation is a first-class state, and the box
18 //! stays seedable throughout.
19 //!
20 //! # Failure is loud, and it happens before any write
21 //!
22 //! A curated asset that will not fetch, or that fetches to the wrong bytes, fails
23 //! the whole seed. It does not silently degrade to the placeholder: that is the
24 //! failure mode `sando/deploy/mnw-testnot-smoke.sh` exists to catch, and a demo
25 //! box that quietly reverts to grey squares is worse than one that refuses to
26 //! reseed. Resolution runs to completion before the seed touches the database, so
27 //! a failure leaves the previous catalog standing.
28
29 use std::collections::HashMap;
30 use std::path::PathBuf;
31 use std::time::Duration;
32
33 use serde::Deserialize;
34
35 /// The curated manifest, embedded so a deployed binary carries it. Override with
36 /// [`MANIFEST_PATH_ENV`] when iterating locally.
37 const EMBEDDED_MANIFEST: &str = include_str!("media-manifest.toml");
38
39 /// Path to a manifest file to read instead of the embedded copy.
40 pub const MANIFEST_PATH_ENV: &str = "SEED_MEDIA_MANIFEST";
41
42 /// Directory holding fetched assets between runs. Defaults to
43 /// `{temp_dir}/mnw-seed-media`.
44 pub const CACHE_DIR_ENV: &str = "SEED_MEDIA_CACHE";
45
46 /// How long a single asset fetch may take.
47 const FETCH_TIMEOUT: Duration = Duration::from_mins(1);
48
49 /// Refuse an asset larger than this. Demo media, not a distribution channel; a
50 /// multi-gigabyte URL in the manifest is a mistake, not a big file.
51 const MAX_ASSET_BYTES: u64 = 64 * 1024 * 1024;
52
53 /// How many times to fetch one asset before believing a digest mismatch.
54 ///
55 /// A mismatch has two causes and they want opposite responses: the asset really
56 /// changed at the source (stop, re-check the licence), or this particular
57 /// response was not the asset (retry, and it comes back right). Build 60 on
58 /// 2026-08-19 was the second: seventeen pinned assets mismatched in one burst
59 /// and every one of them re-fetched byte-identical to its pin afterwards, from
60 /// the same machine. One retry separates the two cheaply.
61 const FETCH_ATTEMPTS: u32 = 3;
62
63 /// One curated file: where it comes from, what it is, and who to credit.
64 #[derive(Debug, Clone, Deserialize)]
65 pub struct Asset {
66 /// Stable id referenced from `creators.rs` (`ItemSpec::media`,
67 /// `ItemSpec::cover`, `ProjectSpec::cover`).
68 pub id: String,
69 /// Direct download URL. Absent = declared but not yet curated; the slot keeps
70 /// its generated placeholder.
71 #[serde(default)]
72 pub url: Option<String>,
73 /// Lowercase hex SHA-256 of the fetched bytes. Absent on a curated asset is
74 /// allowed but warned about: the run logs the digest it saw so it can be
75 /// pinned. Present and mismatched is a hard failure.
76 #[serde(default)]
77 pub sha256: Option<String>,
78 /// Content type to upload under (`audio/wav`, `image/jpeg`, ...).
79 pub media_type: String,
80 /// Filename to store the object as, and to show on the download.
81 pub filename: String,
82 /// SPDX-ish licence string. Public domain / CC0 only; see the manifest header.
83 pub license: String,
84 /// Human title of the work, for the credits page.
85 pub title: String,
86 /// Creator to credit. `None` for anonymous or corporate-anonymous works.
87 #[serde(default)]
88 pub author: Option<String>,
89 /// The page a visitor can reach to verify the licence claim. Not the direct
90 /// download URL, the landing page.
91 pub source: String,
92 }
93
94 impl Asset {
95 /// Whether this asset has been curated (has somewhere to fetch from).
96 fn is_curated(&self) -> bool {
97 self.url.as_deref().is_some_and(|u| !u.trim().is_empty())
98 }
99 }
100
101 /// Shape of the TOML file: a flat array of assets.
102 #[derive(Debug, Deserialize)]
103 struct ManifestFile {
104 #[serde(default)]
105 asset: Vec<Asset>,
106 }
107
108 /// Why the manifest could not be loaded, or an asset could not be resolved.
109 #[derive(Debug, thiserror::Error)]
110 pub enum ManifestError {
111 /// The override path was set but unreadable.
112 #[error("cannot read media manifest at {path}: {source}")]
113 Read {
114 path: PathBuf,
115 #[source]
116 source: std::io::Error,
117 },
118 /// The manifest is not valid TOML, or does not match the schema.
119 #[error("media manifest is not valid: {0}")]
120 Parse(#[from] toml::de::Error),
121 /// Two entries claim the same id, so a reference is ambiguous.
122 #[error("media manifest declares id {0:?} more than once")]
123 DuplicateId(String),
124 /// One or more curated assets failed to fetch or verify. Collected rather
125 /// than returned one at a time: a curation pass wants the whole list.
126 #[error("{} media asset(s) failed to resolve:\n{}", .0.len(), .0.join("\n"))]
127 Unresolved(Vec<String>),
128 }
129
130 /// The parsed manifest, indexed by asset id.
131 #[derive(Debug, Default)]
132 pub struct Manifest {
133 assets: Vec<Asset>,
134 }
135
136 impl Manifest {
137 /// Load the manifest: the file named by [`MANIFEST_PATH_ENV`] if set,
138 /// otherwise the copy embedded at build time.
139 pub fn load() -> Result<Self, ManifestError> {
140 match std::env::var(MANIFEST_PATH_ENV) {
141 Ok(path) if !path.trim().is_empty() => {
142 let path = PathBuf::from(path);
143 let text =
144 std::fs::read_to_string(&path).map_err(|source| ManifestError::Read {
145 path: path.clone(),
146 source,
147 })?;
148 tracing::info!(path = %path.display(), "example seed: using media manifest override");
149 Self::parse(&text)
150 }
151 _ => Self::parse(EMBEDDED_MANIFEST),
152 }
153 }
154
155 /// Parse and validate manifest text. Separated from [`Self::load`] so the
156 /// embedded manifest can be checked by a unit test with no filesystem.
157 pub fn parse(text: &str) -> Result<Self, ManifestError> {
158 let file: ManifestFile = toml::from_str(text)?;
159 let mut seen = std::collections::HashSet::with_capacity(file.asset.len());
160 for asset in &file.asset {
161 if !seen.insert(asset.id.as_str()) {
162 return Err(ManifestError::DuplicateId(asset.id.clone()));
163 }
164 }
165 Ok(Self { assets: file.asset })
166 }
167
168 /// Every declared id, curated or not. Used by the roster-coverage test.
169 pub fn ids(&self) -> impl Iterator<Item = &str> {
170 self.assets.iter().map(|a| a.id.as_str())
171 }
172
173 /// Fetch every curated asset, verify its digest, and return the resolved set.
174 ///
175 /// Uncurated entries are skipped (their slots keep the placeholder). Any
176 /// curated asset that fails is collected; the call returns
177 /// [`ManifestError::Unresolved`] listing all of them rather than stopping at
178 /// the first.
179 pub async fn resolve(&self) -> Result<ResolvedAssets, ManifestError> {
180 let curated: Vec<&Asset> = self.assets.iter().filter(|a| a.is_curated()).collect();
181 let total = self.assets.len();
182 if curated.is_empty() {
183 tracing::warn!(
184 declared = total,
185 "example seed: no media curated yet; every slot keeps its generated placeholder"
186 );
187 return Ok(ResolvedAssets::default());
188 }
189 tracing::info!(
190 curated = curated.len(),
191 declared = total,
192 "example seed: resolving curated media"
193 );
194
195 let cache = cache_dir();
196 if let Err(e) = std::fs::create_dir_all(&cache) {
197 tracing::warn!(dir = %cache.display(), error = ?e, "example seed: media cache unusable; fetching every asset fresh");
198 }
199
200 crate::crypto::install_default_crypto_provider();
201 let client = reqwest::Client::builder()
202 .timeout(FETCH_TIMEOUT)
203 // Wikimedia's User-Agent policy refuses generic agents outright, and
204 // some of the manifest is hosted there. Name the project and give a
205 // contact, which is what the policy asks for.
206 .user_agent("makenotwork-example-seed/1.0 (+https://makenot.work; info@makenot.work)")
207 .build()
208 .map_err(|e| ManifestError::Unresolved(vec![format!("http client: {e}")]))?;
209
210 let mut resolved = HashMap::with_capacity(curated.len());
211 let mut failures = Vec::new();
212 for asset in curated {
213 match fetch_asset(&client, &cache, asset).await {
214 Ok(bytes) => {
215 resolved.insert(asset.id.clone(), (asset.clone(), bytes));
216 }
217 Err(reason) => failures.push(format!(" {}: {reason}", asset.id)),
218 }
219 }
220
221 if !failures.is_empty() {
222 return Err(ManifestError::Unresolved(failures));
223 }
224 Ok(ResolvedAssets { assets: resolved })
225 }
226 }
227
228 /// Curated media, fetched and verified, ready to upload.
229 #[derive(Debug, Default)]
230 pub struct ResolvedAssets {
231 assets: HashMap<String, (Asset, Vec<u8>)>,
232 }
233
234 impl ResolvedAssets {
235 /// The asset behind an id, if it was curated and resolved.
236 pub fn get(&self, id: &str) -> Option<(&Asset, &[u8])> {
237 self.assets.get(id).map(|(a, b)| (a, b.as_slice()))
238 }
239
240 /// The asset behind an optional reference, so call sites can pass
241 /// `spec.cover` straight through.
242 pub fn lookup(&self, id: Option<&str>) -> Option<(&Asset, &[u8])> {
243 self.get(id?)
244 }
245
246 /// How many assets resolved.
247 pub fn len(&self) -> usize {
248 self.assets.len()
249 }
250
251 /// Whether nothing resolved (every slot is on its placeholder).
252 pub fn is_empty(&self) -> bool {
253 self.assets.is_empty()
254 }
255
256 /// A markdown credits list: one line per resolved asset, title linked to the
257 /// source page, with author and licence. Sorted by title so a reseed with the
258 /// same manifest produces the same page.
259 ///
260 /// Empty string when nothing resolved, so the caller can skip publishing.
261 pub fn attribution_markdown(&self) -> String {
262 if self.assets.is_empty() {
263 return String::new();
264 }
265 let mut lines: Vec<String> = self
266 .assets
267 .values()
268 .map(|(a, _)| {
269 let author = a
270 .author
271 .as_deref()
272 .map_or_else(String::new, |author| format!(" by {author}"));
273 format!("- [{}]({}){} — {}", a.title, a.source, author, a.license)
274 })
275 .collect();
276 lines.sort();
277 lines.dedup();
278 lines.join("\n")
279 }
280 }
281
282 /// Where fetched assets are cached between runs.
283 fn cache_dir() -> PathBuf {
284 match std::env::var(CACHE_DIR_ENV) {
285 Ok(dir) if !dir.trim().is_empty() => PathBuf::from(dir),
286 _ => std::env::temp_dir().join("mnw-seed-media"),
287 }
288 }
289
290 /// Fetch one asset, preferring the cache, and verify its digest.
291 ///
292 /// The cache is keyed by id, and a cached file is only trusted when the manifest
293 /// pins a digest and the file matches it. An unpinned asset is re-fetched every
294 /// run: without a digest there is nothing to tell a stale cache entry from a
295 /// current one.
296 async fn fetch_asset(
297 client: &reqwest::Client,
298 cache: &std::path::Path,
299 asset: &Asset,
300 ) -> Result<Vec<u8>, String> {
301 let cached = cache.join(&asset.id);
302 if let (Some(want), Ok(bytes)) = (asset.sha256.as_deref(), std::fs::read(&cached))
303 && digest_hex(&bytes).eq_ignore_ascii_case(want.trim())
304 {
305 tracing::debug!(id = %asset.id, "example seed: media cache hit");
306 return Ok(bytes);
307 }
308
309 let url = asset.url.as_deref().unwrap_or_default().trim();
310 let mut last = String::new();
311 for attempt in 1..=FETCH_ATTEMPTS {
312 match fetch_once(client, asset, url).await {
313 Ok(bytes) => {
314 if let Err(e) = std::fs::write(&cached, &bytes) {
315 tracing::debug!(id = %asset.id, error = ?e, "example seed: could not cache asset");
316 }
317 tracing::info!(id = %asset.id, bytes = bytes.len(), "example seed: fetched media asset");
318 return Ok(bytes);
319 }
320 Err(Fetch::Fatal(reason)) => return Err(reason),
321 Err(Fetch::Retryable(reason)) => {
322 tracing::warn!(
323 id = %asset.id,
324 attempt,
325 of = FETCH_ATTEMPTS,
326 reason = %reason,
327 "example seed: asset fetch failed; retrying"
328 );
329 last = reason;
330 }
331 }
332 }
333 Err(format!(
334 "{last} (unchanged over {FETCH_ATTEMPTS} attempts, so this is not a one-off bad response)"
335 ))
336 }
337
338 /// Why one attempt failed, and whether another attempt could do better.
339 enum Fetch {
340 /// Nothing about trying again would help: the URL is wrong, or the body is
341 /// over the ceiling.
342 Fatal(String),
343 /// The origin may answer differently next time — a transport error, a 5xx or
344 /// 429, a body that is not the asset, or bytes that miss the pinned digest.
345 Retryable(String),
346 }
347
348 /// One HTTP attempt at an asset, verified against the manifest.
349 async fn fetch_once(client: &reqwest::Client, asset: &Asset, url: &str) -> Result<Vec<u8>, Fetch> {
350 let response = client
351 .get(url)
352 .send()
353 .await
354 .map_err(|e| Fetch::Retryable(format!("fetching {url}: {e}")))?;
355 let status = response.status();
356 if !status.is_success() {
357 let reason = format!("fetching {url}: HTTP {status}");
358 // 5xx and 429 are the origin having a bad moment; a 404 or a 403 is a
359 // fact about the manifest and retrying only slows the failure down.
360 return Err(if status.is_server_error() || status.as_u16() == 429 {
361 Fetch::Retryable(reason)
362 } else {
363 Fetch::Fatal(reason)
364 });
365 }
366 // A CDN error page, a challenge, or a consent interstitial is a 200 with a
367 // non-empty body, and without this it reads as "the asset changed at the
368 // source" — which sends whoever is holding the red build off to re-check a
369 // licence that never moved.
370 //
371 // The test is deliberately "did we get a DOCUMENT where media was declared",
372 // not "does the type equal media_type". Content types for the same bytes
373 // legitimately vary — Wikimedia serves `application/ogg` for files this
374 // manifest declares `audio/ogg`, which is correct on both ends — so an
375 // equality check would fail good fetches. Nobody's JPEG is ever text/html.
376 if let Some(got) = response
377 .headers()
378 .get(reqwest::header::CONTENT_TYPE)
379 .and_then(|v| v.to_str().ok())
380 .map(|v| v.split(';').next().unwrap_or(v).trim().to_ascii_lowercase())
381 && is_document(&got)
382 && !is_document(&asset.media_type.to_ascii_lowercase())
383 {
384 return Err(Fetch::Retryable(format!(
385 "fetching {url}: the origin answered with {got} where the manifest \
386 declares {}, so this is a page about the asset rather than the \
387 asset. Its digest says nothing about whether the asset changed.",
388 asset.media_type
389 )));
390 }
391 // Refuse an oversized body before buffering it, when the server declares one.
392 if let Some(len) = response.content_length()
393 && len > MAX_ASSET_BYTES
394 {
395 return Err(Fetch::Fatal(format!(
396 "fetching {url}: {len} bytes exceeds the {MAX_ASSET_BYTES}-byte asset ceiling"
397 )));
398 }
399 let bytes = response
400 .bytes()
401 .await
402 .map_err(|e| Fetch::Retryable(format!("reading {url}: {e}")))?
403 .to_vec();
404 if bytes.len() as u64 > MAX_ASSET_BYTES {
405 return Err(Fetch::Fatal(format!(
406 "fetching {url}: {} bytes exceeds the {MAX_ASSET_BYTES}-byte asset ceiling",
407 bytes.len()
408 )));
409 }
410 if bytes.is_empty() {
411 return Err(Fetch::Retryable(format!("fetching {url}: empty body")));
412 }
413
414 let got = digest_hex(&bytes);
415 match asset.sha256.as_deref().map(str::trim) {
416 Some(want) if !want.is_empty() => {
417 if !got.eq_ignore_ascii_case(want) {
418 return Err(Fetch::Retryable(format!(
419 "digest mismatch for {url}: manifest pins {want}, fetched {got}. \
420 Either the asset changed at the source (re-check the licence \
421 before repinning) or this response was not the asset."
422 )));
423 }
424 }
425 _ => tracing::warn!(
426 id = %asset.id,
427 sha256 = %got,
428 "example seed: asset is unpinned; add sha256 = \"{got}\" to media-manifest.toml"
429 ),
430 }
431
432 Ok(bytes)
433 }
434
435 /// Whether a lowercased content type names a document rather than a media file.
436 ///
437 /// These are the shapes an origin answers with when it is telling you something
438 /// instead of giving you the bytes: an error page, a bot challenge, a JSON API
439 /// error. Anything else — including container types like `application/ogg` and
440 /// the `application/octet-stream` a plain file server falls back to — is
441 /// treated as media and left to the digest to judge.
442 fn is_document(content_type: &str) -> bool {
443 content_type.starts_with("text/")
444 || matches!(
445 content_type,
446 "application/json" | "application/xml" | "application/xhtml+xml"
447 )
448 }
449
450 /// Lowercase hex SHA-256.
451 fn digest_hex(bytes: &[u8]) -> String {
452 use sha2::{Digest, Sha256};
453 hex::encode(Sha256::digest(bytes))
454 }
455
456 #[cfg(test)]
457 mod tests {
458 use super::*;
459
460 #[test]
461 fn embedded_manifest_parses() {
462 Manifest::parse(EMBEDDED_MANIFEST).expect("the embedded manifest must always be valid");
463 }
464
465 /// A curated asset pointed at `url`, pinned to the digest of `body`.
466 fn pinned_asset(url: &str, body: &[u8]) -> Asset {
467 Asset {
468 id: "a".into(),
469 url: Some(url.into()),
470 sha256: Some(digest_hex(body)),
471 media_type: "image/jpeg".into(),
472 filename: "a.jpg".into(),
473 license: "CC0-1.0".into(),
474 title: "A".into(),
475 author: None,
476 source: "https://example.test/a".into(),
477 }
478 }
479
480 fn seed_client() -> reqwest::Client {
481 crate::crypto::install_default_crypto_provider();
482 reqwest::Client::builder().build().unwrap()
483 }
484
485 #[tokio::test]
486 async fn a_transient_bad_response_is_retried_rather_than_believed() {
487 use wiremock::matchers::{method, path};
488 use wiremock::{Mock, MockServer, ResponseTemplate};
489
490 // Sando build 60 (2026-08-19): seventeen pinned assets came back wrong in
491 // one burst and every one of them re-fetched byte-identical to its pin
492 // afterwards. Before this, the first bad response ended the build and the
493 // message sent whoever read it off to re-check a licence that never moved.
494 let server = MockServer::start().await;
495 let good = b"the real asset bytes";
496 Mock::given(method("GET"))
497 .and(path("/a.jpg"))
498 .respond_with(ResponseTemplate::new(200).set_body_bytes(b"a CDN error page".as_ref()))
499 .up_to_n_times(1)
500 .mount(&server)
501 .await;
502 Mock::given(method("GET"))
503 .and(path("/a.jpg"))
504 .respond_with(ResponseTemplate::new(200).set_body_bytes(good.as_ref()))
505 .mount(&server)
506 .await;
507
508 let url = format!("{}/a.jpg", server.uri());
509 let asset = pinned_asset(&url, good);
510 let cache = tempfile::tempdir().unwrap();
511
512 let bytes = fetch_asset(&seed_client(), cache.path(), &asset)
513 .await
514 .expect("the second attempt returns the pinned bytes");
515 assert_eq!(bytes, good);
516 }
517
518 #[tokio::test]
519 async fn a_body_that_is_not_the_declared_type_says_so_instead_of_blaming_the_pin() {
520 use wiremock::matchers::{method, path};
521 use wiremock::{Mock, MockServer, ResponseTemplate};
522
523 let server = MockServer::start().await;
524 Mock::given(method("GET"))
525 .and(path("/a.jpg"))
526 .respond_with(
527 ResponseTemplate::new(200)
528 .insert_header("content-type", "text/html")
529 .set_body_bytes(b"<html>are you a robot</html>".as_ref()),
530 )
531 .mount(&server)
532 .await;
533
534 let url = format!("{}/a.jpg", server.uri());
535 let asset = pinned_asset(&url, b"the real asset bytes");
536 let cache = tempfile::tempdir().unwrap();
537
538 let err = fetch_asset(&seed_client(), cache.path(), &asset)
539 .await
540 .expect_err("an interstitial is not the asset");
541 assert!(
542 err.contains("text/html") && err.contains("image/jpeg"),
543 "the error must name what came back instead: {err}"
544 );
545 assert!(
546 !err.contains("changed at the source"),
547 "a wrong content type is not evidence the asset changed: {err}"
548 );
549 }
550
551 #[tokio::test]
552 async fn a_container_type_that_differs_from_the_declared_one_is_still_the_asset() {
553 use wiremock::matchers::{method, path};
554 use wiremock::{Mock, MockServer, ResponseTemplate};
555
556 // Wikimedia serves `application/ogg` for the three .ogg files this
557 // manifest declares `audio/ogg`, and both are correct. An equality check
558 // on content type fails all three, which is how the first cut of the
559 // document guard was caught.
560 let server = MockServer::start().await;
561 let good = b"ogg bytes";
562 Mock::given(method("GET"))
563 .and(path("/a.ogg"))
564 .respond_with(
565 ResponseTemplate::new(200)
566 .insert_header("content-type", "application/ogg")
567 .set_body_bytes(good.as_ref()),
568 )
569 .mount(&server)
570 .await;
571
572 let url = format!("{}/a.ogg", server.uri());
573 let mut asset = pinned_asset(&url, good);
574 asset.media_type = "audio/ogg".into();
575 let cache = tempfile::tempdir().unwrap();
576
577 let bytes = fetch_asset(&seed_client(), cache.path(), &asset)
578 .await
579 .expect("application/ogg is an ogg file, not a page about one");
580 assert_eq!(bytes, good);
581 }
582
583 #[tokio::test]
584 async fn a_404_fails_once_rather_than_retrying() {
585 use wiremock::matchers::{method, path};
586 use wiremock::{Mock, MockServer, ResponseTemplate};
587
588 // A missing URL is a fact about the manifest. Retrying it only makes the
589 // build take three times as long to say the same thing.
590 let server = MockServer::start().await;
591 Mock::given(method("GET"))
592 .and(path("/a.jpg"))
593 .respond_with(ResponseTemplate::new(404))
594 .expect(1)
595 .mount(&server)
596 .await;
597
598 let url = format!("{}/a.jpg", server.uri());
599 let asset = pinned_asset(&url, b"the real asset bytes");
600 let cache = tempfile::tempdir().unwrap();
601
602 let err = fetch_asset(&seed_client(), cache.path(), &asset)
603 .await
604 .expect_err("a 404 is fatal");
605 assert!(err.contains("404"), "{err}");
606 // MockServer asserts the `expect(1)` on drop.
607 }
608
609 #[tokio::test]
610 async fn a_digest_that_never_matches_still_fails_after_the_retries() {
611 use wiremock::matchers::{method, path};
612 use wiremock::{Mock, MockServer, ResponseTemplate};
613
614 // The retry must not turn a genuine change at the source into a pass.
615 let server = MockServer::start().await;
616 Mock::given(method("GET"))
617 .and(path("/a.jpg"))
618 .respond_with(
619 ResponseTemplate::new(200)
620 .insert_header("content-type", "image/jpeg")
621 .set_body_bytes(b"different bytes every build".as_ref()),
622 )
623 .expect(u64::from(FETCH_ATTEMPTS))
624 .mount(&server)
625 .await;
626
627 let url = format!("{}/a.jpg", server.uri());
628 let asset = pinned_asset(&url, b"the real asset bytes");
629 let cache = tempfile::tempdir().unwrap();
630
631 let err = fetch_asset(&seed_client(), cache.path(), &asset)
632 .await
633 .expect_err("a real change must still fail the seed");
634 assert!(err.contains("digest mismatch"), "{err}");
635 assert!(
636 err.contains("re-check the licence"),
637 "a persistent mismatch is the case that wants the licence check: {err}"
638 );
639 }
640
641 #[test]
642 fn duplicate_ids_are_refused() {
643 let text = r#"
644 [[asset]]
645 id = "a"
646 media_type = "image/png"
647 filename = "a.png"
648 license = "CC0-1.0"
649 title = "A"
650 source = "https://example.test/a"
651
652 [[asset]]
653 id = "a"
654 media_type = "image/png"
655 filename = "b.png"
656 license = "CC0-1.0"
657 title = "B"
658 source = "https://example.test/b"
659 "#;
660 assert!(matches!(
661 Manifest::parse(text).unwrap_err(),
662 ManifestError::DuplicateId(id) if id == "a"
663 ));
664 }
665
666 #[test]
667 fn an_asset_without_a_url_is_uncurated() {
668 let text = r#"
669 [[asset]]
670 id = "a"
671 url = " "
672 media_type = "image/png"
673 filename = "a.png"
674 license = "CC0-1.0"
675 title = "A"
676 source = "https://example.test/a"
677 "#;
678 let manifest = Manifest::parse(text).unwrap();
679 assert!(!manifest.assets[0].is_curated());
680 }
681
682 #[tokio::test]
683 async fn an_uncurated_manifest_never_reaches_for_the_network() {
684 // Synthetic rather than the embedded manifest, which is now fully
685 // curated: this is a claim about the code path, and it should keep
686 // holding whatever the shipped manifest looks like.
687 let text = r#"
688 [[asset]]
689 id = "a"
690 media_type = "image/png"
691 filename = "a.png"
692 license = "CC0-1.0"
693 title = "A"
694 source = "https://example.test/a"
695 "#;
696 let resolved = Manifest::parse(text).unwrap().resolve().await.unwrap();
697 assert!(resolved.is_empty());
698 }
699
700 #[test]
701 fn every_shipped_asset_is_curated_and_pinned() {
702 // An unpinned asset still works, but it re-fetches every run and cannot
703 // detect the file changing at the source. The shipped manifest was
704 // curated in one pass with digests taken from the bytes that arrived,
705 // so anything unpinned here is an entry someone added without running
706 // it.
707 let manifest = Manifest::load().expect("manifest loads");
708 for asset in &manifest.assets {
709 assert!(asset.is_curated(), "{} has no url", asset.id);
710 let digest = asset.sha256.as_deref().unwrap_or_default().trim();
711 assert_eq!(digest.len(), 64, "{} is not pinned", asset.id);
712 assert!(
713 digest.chars().all(|c| c.is_ascii_hexdigit()),
714 "{} has a malformed digest",
715 asset.id
716 );
717 }
718 }
719
720 /// Fetch every shipped asset and check it against its pin.
721 ///
722 /// Ignored by default: it is ~90 MB over the network and depends on two
723 /// museums staying up, neither of which belongs in a normal test run. Run it
724 /// after editing the manifest, which is the moment it earns its cost:
725 ///
726 /// cargo test --lib seed::manifest -- --ignored --nocapture
727 #[tokio::test]
728 #[ignore = "network: fetches every asset in the manifest"]
729 async fn every_shipped_asset_actually_resolves() {
730 let manifest = Manifest::load().expect("manifest loads");
731 let declared = manifest.assets.len();
732 let resolved = manifest
733 .resolve()
734 .await
735 .expect("every asset should resolve");
736 assert_eq!(
737 resolved.len(),
738 declared,
739 "resolved {} of {declared} assets",
740 resolved.len()
741 );
742 assert!(!resolved.attribution_markdown().is_empty());
743 }
744
745 #[test]
746 fn every_shipped_asset_is_public_domain_and_verifiable() {
747 // Rule 1 and rule 2 of the manifest header, enforced rather than
748 // trusted. These files sit on a public box under a licence claim this
749 // repo makes; a CC-BY asset slipping in is a licensing problem, and a
750 // `source` that is not a reachable page makes the claim uncheckable.
751 let manifest = Manifest::load().expect("manifest loads");
752 for asset in &manifest.assets {
753 let licence = asset.license.to_ascii_lowercase();
754 assert!(
755 licence.contains("cc0") || licence.contains("public domain"),
756 "{} is licensed {:?}, which is not public domain or CC0",
757 asset.id,
758 asset.license
759 );
760 assert!(
761 asset.source.starts_with("https://"),
762 "{} has no https source page",
763 asset.id
764 );
765 assert!(
766 asset
767 .url
768 .as_deref()
769 .is_some_and(|u| u.starts_with("https://")),
770 "{} is not fetched over https",
771 asset.id
772 );
773 }
774 }
775
776 #[test]
777 fn attribution_lists_resolved_assets_only() {
778 let asset = Asset {
779 id: "x".into(),
780 url: Some("https://example.test/x.jpg".into()),
781 sha256: None,
782 media_type: "image/jpeg".into(),
783 filename: "x.jpg".into(),
784 license: "CC0-1.0".into(),
785 title: "A Study".into(),
786 author: Some("A. Person".into()),
787 source: "https://example.test/x".into(),
788 };
789 let mut assets = HashMap::new();
790 assets.insert("x".to_string(), (asset, vec![1, 2, 3]));
791 let resolved = ResolvedAssets { assets };
792 assert_eq!(
793 resolved.attribution_markdown(),
794 "- [A Study](https://example.test/x) by A. Person — CC0-1.0"
795 );
796 assert!(ResolvedAssets::default().attribution_markdown().is_empty());
797 }
798
799 #[test]
800 fn digest_is_lowercase_hex_sha256() {
801 // Known vector: SHA-256 of the empty string.
802 assert_eq!(
803 digest_hex(b""),
804 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
805 );
806 }
807 }
808