Skip to main content

max / makenotwork

30.3 KB · 807 lines History Blame Raw
1 //! Real public-domain / CC0 media for the example seed: the manifest that names
2 //! each asset, and the fetch step that turns it into bytes.
3 //!
4 //! The seed's other phases are pure data in `creators.rs`. Media cannot be,
5 //! because a real asset is a file somebody has to choose, license-check, and
6 //! host. So the choosing lives in `media-manifest.toml` (curated by hand, checked
7 //! into the repo, embedded in the binary) and everything downstream of it is
8 //! mechanical: fetch by URL, verify the digest, upload through the storage layer,
9 //! record the attribution.
10 //!
11 //! # Curation state is per asset, not per manifest
12 //!
13 //! An entry with no `url` is *declared but not yet curated*. It resolves to
14 //! nothing and [`super::media`] falls back to the generated placeholder for that
15 //! one slot. So the manifest doubles as the curation checklist: every id the seed
16 //! can use is listed, and the ones still carrying grey boxes are exactly the ones
17 //! with an empty `url`. Partial curation is a first-class state, and the box
18 //! stays seedable throughout.
19 //!
20 //! # Failure is loud, and it happens before any write
21 //!
22 //! A curated asset that will not fetch, or that fetches to the wrong bytes, fails
23 //! the whole seed. It does not silently degrade to the placeholder: that is the
24 //! failure mode `sando/deploy/mnw-testnot-smoke.sh` exists to catch, and a demo
25 //! box that quietly reverts to grey squares is worse than one that refuses to
26 //! reseed. Resolution runs to completion before the seed touches the database, so
27 //! a failure leaves the previous catalog standing.
28
29 use std::collections::HashMap;
30 use std::path::PathBuf;
31 use std::time::Duration;
32
33 use serde::Deserialize;
34
35 /// The curated manifest, embedded so a deployed binary carries it. Override with
36 /// [`MANIFEST_PATH_ENV`] when iterating locally.
37 const EMBEDDED_MANIFEST: &str = include_str!("media-manifest.toml");
38
39 /// Path to a manifest file to read instead of the embedded copy.
40 pub const MANIFEST_PATH_ENV: &str = "SEED_MEDIA_MANIFEST";
41
42 /// Directory holding fetched assets between runs. Defaults to
43 /// `{temp_dir}/mnw-seed-media`.
44 pub const CACHE_DIR_ENV: &str = "SEED_MEDIA_CACHE";
45
46 /// How long a single asset fetch may take.
47 const FETCH_TIMEOUT: Duration = Duration::from_mins(1);
48
49 /// Refuse an asset larger than this. Demo media, not a distribution channel; a
50 /// multi-gigabyte URL in the manifest is a mistake, not a big file.
51 const MAX_ASSET_BYTES: u64 = 64 * 1024 * 1024;
52
53 /// How many times to fetch one asset before believing a digest mismatch.
54 ///
55 /// A mismatch has two causes and they want opposite responses: the asset really
56 /// changed at the source (stop, re-check the licence), or this particular
57 /// response was not the asset (retry, and it comes back right). A burst of
58 /// mismatches that all re-fetch byte-identical is the second case. One retry
59 /// separates the two cheaply.
60 const FETCH_ATTEMPTS: u32 = 3;
61
62 /// One curated file: where it comes from, what it is, and who to credit.
63 #[derive(Debug, Clone, Deserialize)]
64 pub struct Asset {
65 /// Stable id referenced from `creators.rs` (`ItemSpec::media`,
66 /// `ItemSpec::cover`, `ProjectSpec::cover`).
67 pub id: String,
68 /// Direct download URL. Absent = declared but not yet curated; the slot keeps
69 /// its generated placeholder.
70 #[serde(default)]
71 pub url: Option<String>,
72 /// Lowercase hex SHA-256 of the fetched bytes. Absent on a curated asset is
73 /// allowed but warned about: the run logs the digest it saw so it can be
74 /// pinned. Present and mismatched is a hard failure.
75 #[serde(default)]
76 pub sha256: Option<String>,
77 /// Content type to upload under (`audio/wav`, `image/jpeg`, ...).
78 pub media_type: String,
79 /// Filename to store the object as, and to show on the download.
80 pub filename: String,
81 /// SPDX-ish licence string. Public domain / CC0 only; see the manifest header.
82 pub license: String,
83 /// Human title of the work, for the credits page.
84 pub title: String,
85 /// Creator to credit. `None` for anonymous or corporate-anonymous works.
86 #[serde(default)]
87 pub author: Option<String>,
88 /// The page a visitor can reach to verify the licence claim. Not the direct
89 /// download URL, the landing page.
90 pub source: String,
91 }
92
93 impl Asset {
94 /// Whether this asset has been curated (has somewhere to fetch from).
95 fn is_curated(&self) -> bool {
96 self.url.as_deref().is_some_and(|u| !u.trim().is_empty())
97 }
98 }
99
100 /// Shape of the TOML file: a flat array of assets.
101 #[derive(Debug, Deserialize)]
102 struct ManifestFile {
103 #[serde(default)]
104 asset: Vec<Asset>,
105 }
106
107 /// Why the manifest could not be loaded, or an asset could not be resolved.
108 #[derive(Debug, thiserror::Error)]
109 pub enum ManifestError {
110 /// The override path was set but unreadable.
111 #[error("cannot read media manifest at {path}: {source}")]
112 Read {
113 path: PathBuf,
114 #[source]
115 source: std::io::Error,
116 },
117 /// The manifest is not valid TOML, or does not match the schema.
118 #[error("media manifest is not valid: {0}")]
119 Parse(#[from] toml::de::Error),
120 /// Two entries claim the same id, so a reference is ambiguous.
121 #[error("media manifest declares id {0:?} more than once")]
122 DuplicateId(String),
123 /// One or more curated assets failed to fetch or verify. Collected rather
124 /// than returned one at a time: a curation pass wants the whole list.
125 #[error("{} media asset(s) failed to resolve:\n{}", .0.len(), .0.join("\n"))]
126 Unresolved(Vec<String>),
127 }
128
129 /// The parsed manifest, indexed by asset id.
130 #[derive(Debug, Default)]
131 pub struct Manifest {
132 assets: Vec<Asset>,
133 }
134
135 impl Manifest {
136 /// Load the manifest: the file named by [`MANIFEST_PATH_ENV`] if set,
137 /// otherwise the copy embedded at build time.
138 pub fn load() -> Result<Self, ManifestError> {
139 match std::env::var(MANIFEST_PATH_ENV) {
140 Ok(path) if !path.trim().is_empty() => {
141 let path = PathBuf::from(path);
142 let text =
143 std::fs::read_to_string(&path).map_err(|source| ManifestError::Read {
144 path: path.clone(),
145 source,
146 })?;
147 tracing::info!(path = %path.display(), "example seed: using media manifest override");
148 Self::parse(&text)
149 }
150 _ => Self::parse(EMBEDDED_MANIFEST),
151 }
152 }
153
154 /// Parse and validate manifest text. Separated from [`Self::load`] so the
155 /// embedded manifest can be checked by a unit test with no filesystem.
156 pub fn parse(text: &str) -> Result<Self, ManifestError> {
157 let file: ManifestFile = toml::from_str(text)?;
158 let mut seen = std::collections::HashSet::with_capacity(file.asset.len());
159 for asset in &file.asset {
160 if !seen.insert(asset.id.as_str()) {
161 return Err(ManifestError::DuplicateId(asset.id.clone()));
162 }
163 }
164 Ok(Self { assets: file.asset })
165 }
166
167 /// Every declared id, curated or not. Used by the roster-coverage test.
168 pub fn ids(&self) -> impl Iterator<Item = &str> {
169 self.assets.iter().map(|a| a.id.as_str())
170 }
171
172 /// Fetch every curated asset, verify its digest, and return the resolved set.
173 ///
174 /// Uncurated entries are skipped (their slots keep the placeholder). Any
175 /// curated asset that fails is collected; the call returns
176 /// [`ManifestError::Unresolved`] listing all of them rather than stopping at
177 /// the first.
178 pub async fn resolve(&self) -> Result<ResolvedAssets, ManifestError> {
179 let curated: Vec<&Asset> = self.assets.iter().filter(|a| a.is_curated()).collect();
180 let total = self.assets.len();
181 if curated.is_empty() {
182 tracing::warn!(
183 declared = total,
184 "example seed: no media curated yet; every slot keeps its generated placeholder"
185 );
186 return Ok(ResolvedAssets::default());
187 }
188 tracing::info!(
189 curated = curated.len(),
190 declared = total,
191 "example seed: resolving curated media"
192 );
193
194 let cache = cache_dir();
195 if let Err(e) = std::fs::create_dir_all(&cache) {
196 tracing::warn!(dir = %cache.display(), error = ?e, "example seed: media cache unusable; fetching every asset fresh");
197 }
198
199 crate::crypto::install_default_crypto_provider();
200 let client = reqwest::Client::builder()
201 .timeout(FETCH_TIMEOUT)
202 // Wikimedia's User-Agent policy refuses generic agents outright, and
203 // some of the manifest is hosted there. Name the project and give a
204 // contact, which is what the policy asks for.
205 .user_agent("makenotwork-example-seed/1.0 (+https://makenot.work; info@makenot.work)")
206 .build()
207 .map_err(|e| ManifestError::Unresolved(vec![format!("http client: {e}")]))?;
208
209 let mut resolved = HashMap::with_capacity(curated.len());
210 let mut failures = Vec::new();
211 for asset in curated {
212 match fetch_asset(&client, &cache, asset).await {
213 Ok(bytes) => {
214 resolved.insert(asset.id.clone(), (asset.clone(), bytes));
215 }
216 Err(reason) => failures.push(format!(" {}: {reason}", asset.id)),
217 }
218 }
219
220 if !failures.is_empty() {
221 return Err(ManifestError::Unresolved(failures));
222 }
223 Ok(ResolvedAssets { assets: resolved })
224 }
225 }
226
227 /// Curated media, fetched and verified, ready to upload.
228 #[derive(Debug, Default)]
229 pub struct ResolvedAssets {
230 assets: HashMap<String, (Asset, Vec<u8>)>,
231 }
232
233 impl ResolvedAssets {
234 /// The asset behind an id, if it was curated and resolved.
235 pub fn get(&self, id: &str) -> Option<(&Asset, &[u8])> {
236 self.assets.get(id).map(|(a, b)| (a, b.as_slice()))
237 }
238
239 /// The asset behind an optional reference, so call sites can pass
240 /// `spec.cover` straight through.
241 pub fn lookup(&self, id: Option<&str>) -> Option<(&Asset, &[u8])> {
242 self.get(id?)
243 }
244
245 /// How many assets resolved.
246 pub fn len(&self) -> usize {
247 self.assets.len()
248 }
249
250 /// Whether nothing resolved (every slot is on its placeholder).
251 pub fn is_empty(&self) -> bool {
252 self.assets.is_empty()
253 }
254
255 /// A markdown credits list: one line per resolved asset, title linked to the
256 /// source page, with author and licence. Sorted by title so a reseed with the
257 /// same manifest produces the same page.
258 ///
259 /// Empty string when nothing resolved, so the caller can skip publishing.
260 pub fn attribution_markdown(&self) -> String {
261 if self.assets.is_empty() {
262 return String::new();
263 }
264 let mut lines: Vec<String> = self
265 .assets
266 .values()
267 .map(|(a, _)| {
268 let author = a
269 .author
270 .as_deref()
271 .map_or_else(String::new, |author| format!(" by {author}"));
272 format!("- [{}]({}){} — {}", a.title, a.source, author, a.license)
273 })
274 .collect();
275 lines.sort();
276 lines.dedup();
277 lines.join("\n")
278 }
279 }
280
281 /// Where fetched assets are cached between runs.
282 fn cache_dir() -> PathBuf {
283 match std::env::var(CACHE_DIR_ENV) {
284 Ok(dir) if !dir.trim().is_empty() => PathBuf::from(dir),
285 _ => std::env::temp_dir().join("mnw-seed-media"),
286 }
287 }
288
289 /// Fetch one asset, preferring the cache, and verify its digest.
290 ///
291 /// The cache is keyed by id, and a cached file is only trusted when the manifest
292 /// pins a digest and the file matches it. An unpinned asset is re-fetched every
293 /// run: without a digest there is nothing to tell a stale cache entry from a
294 /// current one.
295 async fn fetch_asset(
296 client: &reqwest::Client,
297 cache: &std::path::Path,
298 asset: &Asset,
299 ) -> Result<Vec<u8>, String> {
300 let cached = cache.join(&asset.id);
301 if let (Some(want), Ok(bytes)) = (asset.sha256.as_deref(), std::fs::read(&cached))
302 && digest_hex(&bytes).eq_ignore_ascii_case(want.trim())
303 {
304 tracing::debug!(id = %asset.id, "example seed: media cache hit");
305 return Ok(bytes);
306 }
307
308 let url = asset.url.as_deref().unwrap_or_default().trim();
309 let mut last = String::new();
310 for attempt in 1..=FETCH_ATTEMPTS {
311 match fetch_once(client, asset, url).await {
312 Ok(bytes) => {
313 if let Err(e) = std::fs::write(&cached, &bytes) {
314 tracing::debug!(id = %asset.id, error = ?e, "example seed: could not cache asset");
315 }
316 tracing::info!(id = %asset.id, bytes = bytes.len(), "example seed: fetched media asset");
317 return Ok(bytes);
318 }
319 Err(Fetch::Fatal(reason)) => return Err(reason),
320 Err(Fetch::Retryable(reason)) => {
321 tracing::warn!(
322 id = %asset.id,
323 attempt,
324 of = FETCH_ATTEMPTS,
325 reason = %reason,
326 "example seed: asset fetch failed; retrying"
327 );
328 last = reason;
329 }
330 }
331 }
332 Err(format!(
333 "{last} (unchanged over {FETCH_ATTEMPTS} attempts, so this is not a one-off bad response)"
334 ))
335 }
336
337 /// Why one attempt failed, and whether another attempt could do better.
338 enum Fetch {
339 /// Nothing about trying again would help: the URL is wrong, or the body is
340 /// over the ceiling.
341 Fatal(String),
342 /// The origin may answer differently next time — a transport error, a 5xx or
343 /// 429, a body that is not the asset, or bytes that miss the pinned digest.
344 Retryable(String),
345 }
346
347 /// One HTTP attempt at an asset, verified against the manifest.
348 async fn fetch_once(client: &reqwest::Client, asset: &Asset, url: &str) -> Result<Vec<u8>, Fetch> {
349 let response = client
350 .get(url)
351 .send()
352 .await
353 .map_err(|e| Fetch::Retryable(format!("fetching {url}: {e}")))?;
354 let status = response.status();
355 if !status.is_success() {
356 let reason = format!("fetching {url}: HTTP {status}");
357 // 5xx and 429 are the origin having a bad moment; a 404 or a 403 is a
358 // fact about the manifest and retrying only slows the failure down.
359 return Err(if status.is_server_error() || status.as_u16() == 429 {
360 Fetch::Retryable(reason)
361 } else {
362 Fetch::Fatal(reason)
363 });
364 }
365 // A CDN error page, a challenge, or a consent interstitial is a 200 with a
366 // non-empty body, and without this it reads as "the asset changed at the
367 // source" — which sends whoever is holding the red build off to re-check a
368 // licence that never moved.
369 //
370 // The test is deliberately "did we get a DOCUMENT where media was declared",
371 // not "does the type equal media_type". Content types for the same bytes
372 // legitimately vary — Wikimedia serves `application/ogg` for files this
373 // manifest declares `audio/ogg`, which is correct on both ends — so an
374 // equality check would fail good fetches. Nobody's JPEG is ever text/html.
375 if let Some(got) = response
376 .headers()
377 .get(reqwest::header::CONTENT_TYPE)
378 .and_then(|v| v.to_str().ok())
379 .map(|v| v.split(';').next().unwrap_or(v).trim().to_ascii_lowercase())
380 && is_document(&got)
381 && !is_document(&asset.media_type.to_ascii_lowercase())
382 {
383 return Err(Fetch::Retryable(format!(
384 "fetching {url}: the origin answered with {got} where the manifest \
385 declares {}, so this is a page about the asset rather than the \
386 asset. Its digest says nothing about whether the asset changed.",
387 asset.media_type
388 )));
389 }
390 // Refuse an oversized body before buffering it, when the server declares one.
391 if let Some(len) = response.content_length()
392 && len > MAX_ASSET_BYTES
393 {
394 return Err(Fetch::Fatal(format!(
395 "fetching {url}: {len} bytes exceeds the {MAX_ASSET_BYTES}-byte asset ceiling"
396 )));
397 }
398 let bytes = response
399 .bytes()
400 .await
401 .map_err(|e| Fetch::Retryable(format!("reading {url}: {e}")))?
402 .to_vec();
403 if bytes.len() as u64 > MAX_ASSET_BYTES {
404 return Err(Fetch::Fatal(format!(
405 "fetching {url}: {} bytes exceeds the {MAX_ASSET_BYTES}-byte asset ceiling",
406 bytes.len()
407 )));
408 }
409 if bytes.is_empty() {
410 return Err(Fetch::Retryable(format!("fetching {url}: empty body")));
411 }
412
413 let got = digest_hex(&bytes);
414 match asset.sha256.as_deref().map(str::trim) {
415 Some(want) if !want.is_empty() => {
416 if !got.eq_ignore_ascii_case(want) {
417 return Err(Fetch::Retryable(format!(
418 "digest mismatch for {url}: manifest pins {want}, fetched {got}. \
419 Either the asset changed at the source (re-check the licence \
420 before repinning) or this response was not the asset."
421 )));
422 }
423 }
424 _ => tracing::warn!(
425 id = %asset.id,
426 sha256 = %got,
427 "example seed: asset is unpinned; add sha256 = \"{got}\" to media-manifest.toml"
428 ),
429 }
430
431 Ok(bytes)
432 }
433
434 /// Whether a lowercased content type names a document rather than a media file.
435 ///
436 /// These are the shapes an origin answers with when it is telling you something
437 /// instead of giving you the bytes: an error page, a bot challenge, a JSON API
438 /// error. Anything else — including container types like `application/ogg` and
439 /// the `application/octet-stream` a plain file server falls back to — is
440 /// treated as media and left to the digest to judge.
441 fn is_document(content_type: &str) -> bool {
442 content_type.starts_with("text/")
443 || matches!(
444 content_type,
445 "application/json" | "application/xml" | "application/xhtml+xml"
446 )
447 }
448
449 /// Lowercase hex SHA-256.
450 fn digest_hex(bytes: &[u8]) -> String {
451 use sha2::{Digest, Sha256};
452 hex::encode(Sha256::digest(bytes))
453 }
454
455 #[cfg(test)]
456 mod tests {
457 use super::*;
458
459 #[test]
460 fn embedded_manifest_parses() {
461 Manifest::parse(EMBEDDED_MANIFEST).expect("the embedded manifest must always be valid");
462 }
463
464 /// A curated asset pointed at `url`, pinned to the digest of `body`.
465 fn pinned_asset(url: &str, body: &[u8]) -> Asset {
466 Asset {
467 id: "a".into(),
468 url: Some(url.into()),
469 sha256: Some(digest_hex(body)),
470 media_type: "image/jpeg".into(),
471 filename: "a.jpg".into(),
472 license: "CC0-1.0".into(),
473 title: "A".into(),
474 author: None,
475 source: "https://example.test/a".into(),
476 }
477 }
478
479 fn seed_client() -> reqwest::Client {
480 crate::crypto::install_default_crypto_provider();
481 reqwest::Client::builder().build().unwrap()
482 }
483
484 #[tokio::test]
485 async fn a_transient_bad_response_is_retried_rather_than_believed() {
486 use wiremock::matchers::{method, path};
487 use wiremock::{Mock, MockServer, ResponseTemplate};
488
489 // Sando build 60 (2026-08-19): seventeen pinned assets came back wrong in
490 // one burst and every one of them re-fetched byte-identical to its pin
491 // afterwards. Before this, the first bad response ended the build and the
492 // message sent whoever read it off to re-check a licence that never moved.
493 let server = MockServer::start().await;
494 let good = b"the real asset bytes";
495 Mock::given(method("GET"))
496 .and(path("/a.jpg"))
497 .respond_with(ResponseTemplate::new(200).set_body_bytes(b"a CDN error page".as_ref()))
498 .up_to_n_times(1)
499 .mount(&server)
500 .await;
501 Mock::given(method("GET"))
502 .and(path("/a.jpg"))
503 .respond_with(ResponseTemplate::new(200).set_body_bytes(good.as_ref()))
504 .mount(&server)
505 .await;
506
507 let url = format!("{}/a.jpg", server.uri());
508 let asset = pinned_asset(&url, good);
509 let cache = tempfile::tempdir().unwrap();
510
511 let bytes = fetch_asset(&seed_client(), cache.path(), &asset)
512 .await
513 .expect("the second attempt returns the pinned bytes");
514 assert_eq!(bytes, good);
515 }
516
517 #[tokio::test]
518 async fn a_body_that_is_not_the_declared_type_says_so_instead_of_blaming_the_pin() {
519 use wiremock::matchers::{method, path};
520 use wiremock::{Mock, MockServer, ResponseTemplate};
521
522 let server = MockServer::start().await;
523 Mock::given(method("GET"))
524 .and(path("/a.jpg"))
525 .respond_with(
526 ResponseTemplate::new(200)
527 .insert_header("content-type", "text/html")
528 .set_body_bytes(b"<html>are you a robot</html>".as_ref()),
529 )
530 .mount(&server)
531 .await;
532
533 let url = format!("{}/a.jpg", server.uri());
534 let asset = pinned_asset(&url, b"the real asset bytes");
535 let cache = tempfile::tempdir().unwrap();
536
537 let err = fetch_asset(&seed_client(), cache.path(), &asset)
538 .await
539 .expect_err("an interstitial is not the asset");
540 assert!(
541 err.contains("text/html") && err.contains("image/jpeg"),
542 "the error must name what came back instead: {err}"
543 );
544 assert!(
545 !err.contains("changed at the source"),
546 "a wrong content type is not evidence the asset changed: {err}"
547 );
548 }
549
550 #[tokio::test]
551 async fn a_container_type_that_differs_from_the_declared_one_is_still_the_asset() {
552 use wiremock::matchers::{method, path};
553 use wiremock::{Mock, MockServer, ResponseTemplate};
554
555 // Wikimedia serves `application/ogg` for the three .ogg files this
556 // manifest declares `audio/ogg`, and both are correct. An equality check
557 // on content type fails all three, which is how the first cut of the
558 // document guard was caught.
559 let server = MockServer::start().await;
560 let good = b"ogg bytes";
561 Mock::given(method("GET"))
562 .and(path("/a.ogg"))
563 .respond_with(
564 ResponseTemplate::new(200)
565 .insert_header("content-type", "application/ogg")
566 .set_body_bytes(good.as_ref()),
567 )
568 .mount(&server)
569 .await;
570
571 let url = format!("{}/a.ogg", server.uri());
572 let mut asset = pinned_asset(&url, good);
573 asset.media_type = "audio/ogg".into();
574 let cache = tempfile::tempdir().unwrap();
575
576 let bytes = fetch_asset(&seed_client(), cache.path(), &asset)
577 .await
578 .expect("application/ogg is an ogg file, not a page about one");
579 assert_eq!(bytes, good);
580 }
581
582 #[tokio::test]
583 async fn a_404_fails_once_rather_than_retrying() {
584 use wiremock::matchers::{method, path};
585 use wiremock::{Mock, MockServer, ResponseTemplate};
586
587 // A missing URL is a fact about the manifest. Retrying it only makes the
588 // build take three times as long to say the same thing.
589 let server = MockServer::start().await;
590 Mock::given(method("GET"))
591 .and(path("/a.jpg"))
592 .respond_with(ResponseTemplate::new(404))
593 .expect(1)
594 .mount(&server)
595 .await;
596
597 let url = format!("{}/a.jpg", server.uri());
598 let asset = pinned_asset(&url, b"the real asset bytes");
599 let cache = tempfile::tempdir().unwrap();
600
601 let err = fetch_asset(&seed_client(), cache.path(), &asset)
602 .await
603 .expect_err("a 404 is fatal");
604 assert!(err.contains("404"), "{err}");
605 // MockServer asserts the `expect(1)` on drop.
606 }
607
608 #[tokio::test]
609 async fn a_digest_that_never_matches_still_fails_after_the_retries() {
610 use wiremock::matchers::{method, path};
611 use wiremock::{Mock, MockServer, ResponseTemplate};
612
613 // The retry must not turn a genuine change at the source into a pass.
614 let server = MockServer::start().await;
615 Mock::given(method("GET"))
616 .and(path("/a.jpg"))
617 .respond_with(
618 ResponseTemplate::new(200)
619 .insert_header("content-type", "image/jpeg")
620 .set_body_bytes(b"different bytes every build".as_ref()),
621 )
622 .expect(u64::from(FETCH_ATTEMPTS))
623 .mount(&server)
624 .await;
625
626 let url = format!("{}/a.jpg", server.uri());
627 let asset = pinned_asset(&url, b"the real asset bytes");
628 let cache = tempfile::tempdir().unwrap();
629
630 let err = fetch_asset(&seed_client(), cache.path(), &asset)
631 .await
632 .expect_err("a real change must still fail the seed");
633 assert!(err.contains("digest mismatch"), "{err}");
634 assert!(
635 err.contains("re-check the licence"),
636 "a persistent mismatch is the case that wants the licence check: {err}"
637 );
638 }
639
640 #[test]
641 fn duplicate_ids_are_refused() {
642 let text = r#"
643 [[asset]]
644 id = "a"
645 media_type = "image/png"
646 filename = "a.png"
647 license = "CC0-1.0"
648 title = "A"
649 source = "https://example.test/a"
650
651 [[asset]]
652 id = "a"
653 media_type = "image/png"
654 filename = "b.png"
655 license = "CC0-1.0"
656 title = "B"
657 source = "https://example.test/b"
658 "#;
659 assert!(matches!(
660 Manifest::parse(text).unwrap_err(),
661 ManifestError::DuplicateId(id) if id == "a"
662 ));
663 }
664
665 #[test]
666 fn an_asset_without_a_url_is_uncurated() {
667 let text = r#"
668 [[asset]]
669 id = "a"
670 url = " "
671 media_type = "image/png"
672 filename = "a.png"
673 license = "CC0-1.0"
674 title = "A"
675 source = "https://example.test/a"
676 "#;
677 let manifest = Manifest::parse(text).unwrap();
678 assert!(!manifest.assets[0].is_curated());
679 }
680
681 #[tokio::test]
682 async fn an_uncurated_manifest_never_reaches_for_the_network() {
683 // Synthetic rather than the embedded manifest, which is now fully
684 // curated: this is a claim about the code path, and it should keep
685 // holding whatever the shipped manifest looks like.
686 let text = r#"
687 [[asset]]
688 id = "a"
689 media_type = "image/png"
690 filename = "a.png"
691 license = "CC0-1.0"
692 title = "A"
693 source = "https://example.test/a"
694 "#;
695 let resolved = Manifest::parse(text).unwrap().resolve().await.unwrap();
696 assert!(resolved.is_empty());
697 }
698
699 #[test]
700 fn every_shipped_asset_is_curated_and_pinned() {
701 // An unpinned asset still works, but it re-fetches every run and cannot
702 // detect the file changing at the source. The shipped manifest was
703 // curated in one pass with digests taken from the bytes that arrived,
704 // so anything unpinned here is an entry someone added without running
705 // it.
706 let manifest = Manifest::load().expect("manifest loads");
707 for asset in &manifest.assets {
708 assert!(asset.is_curated(), "{} has no url", asset.id);
709 let digest = asset.sha256.as_deref().unwrap_or_default().trim();
710 assert_eq!(digest.len(), 64, "{} is not pinned", asset.id);
711 assert!(
712 digest.chars().all(|c| c.is_ascii_hexdigit()),
713 "{} has a malformed digest",
714 asset.id
715 );
716 }
717 }
718
719 /// Fetch every shipped asset and check it against its pin.
720 ///
721 /// Ignored by default: it is ~90 MB over the network and depends on two
722 /// museums staying up, neither of which belongs in a normal test run. Run it
723 /// after editing the manifest, which is the moment it earns its cost:
724 ///
725 /// cargo test --lib seed::manifest -- --ignored --nocapture
726 #[tokio::test]
727 #[ignore = "network: fetches every asset in the manifest"]
728 async fn every_shipped_asset_actually_resolves() {
729 let manifest = Manifest::load().expect("manifest loads");
730 let declared = manifest.assets.len();
731 let resolved = manifest
732 .resolve()
733 .await
734 .expect("every asset should resolve");
735 assert_eq!(
736 resolved.len(),
737 declared,
738 "resolved {} of {declared} assets",
739 resolved.len()
740 );
741 assert!(!resolved.attribution_markdown().is_empty());
742 }
743
744 #[test]
745 fn every_shipped_asset_is_public_domain_and_verifiable() {
746 // Rule 1 and rule 2 of the manifest header, enforced rather than
747 // trusted. These files sit on a public box under a licence claim this
748 // repo makes; a CC-BY asset slipping in is a licensing problem, and a
749 // `source` that is not a reachable page makes the claim uncheckable.
750 let manifest = Manifest::load().expect("manifest loads");
751 for asset in &manifest.assets {
752 let licence = asset.license.to_ascii_lowercase();
753 assert!(
754 licence.contains("cc0") || licence.contains("public domain"),
755 "{} is licensed {:?}, which is not public domain or CC0",
756 asset.id,
757 asset.license
758 );
759 assert!(
760 asset.source.starts_with("https://"),
761 "{} has no https source page",
762 asset.id
763 );
764 assert!(
765 asset
766 .url
767 .as_deref()
768 .is_some_and(|u| u.starts_with("https://")),
769 "{} is not fetched over https",
770 asset.id
771 );
772 }
773 }
774
775 #[test]
776 fn attribution_lists_resolved_assets_only() {
777 let asset = Asset {
778 id: "x".into(),
779 url: Some("https://example.test/x.jpg".into()),
780 sha256: None,
781 media_type: "image/jpeg".into(),
782 filename: "x.jpg".into(),
783 license: "CC0-1.0".into(),
784 title: "A Study".into(),
785 author: Some("A. Person".into()),
786 source: "https://example.test/x".into(),
787 };
788 let mut assets = HashMap::new();
789 assets.insert("x".to_string(), (asset, vec![1, 2, 3]));
790 let resolved = ResolvedAssets { assets };
791 assert_eq!(
792 resolved.attribution_markdown(),
793 "- [A Study](https://example.test/x) by A. Person — CC0-1.0"
794 );
795 assert!(ResolvedAssets::default().attribution_markdown().is_empty());
796 }
797
798 #[test]
799 fn digest_is_lowercase_hex_sha256() {
800 // Known vector: SHA-256 of the empty string.
801 assert_eq!(
802 digest_hex(b""),
803 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
804 );
805 }
806 }
807