Skip to main content

max / makenotwork

19.4 KB · 537 lines History Blame Raw
1 //! Real public-domain / CC0 media for the example seed: the manifest that names
2 //! each asset, and the fetch step that turns it into bytes.
3 //!
4 //! The seed's other phases are pure data in `creators.rs`. Media cannot be,
5 //! because a real asset is a file somebody has to choose, license-check, and
6 //! host. So the choosing lives in `media-manifest.toml` (curated by hand, checked
7 //! into the repo, embedded in the binary) and everything downstream of it is
8 //! mechanical: fetch by URL, verify the digest, upload through the storage layer,
9 //! record the attribution.
10 //!
11 //! # Curation state is per asset, not per manifest
12 //!
13 //! An entry with no `url` is *declared but not yet curated*. It resolves to
14 //! nothing and [`super::media`] falls back to the generated placeholder for that
15 //! one slot. So the manifest doubles as the curation checklist: every id the seed
16 //! can use is listed, and the ones still carrying grey boxes are exactly the ones
17 //! with an empty `url`. Partial curation is a first-class state, and the box
18 //! stays seedable throughout.
19 //!
20 //! # Failure is loud, and it happens before any write
21 //!
22 //! A curated asset that will not fetch, or that fetches to the wrong bytes, fails
23 //! the whole seed. It does not silently degrade to the placeholder: that is the
24 //! failure mode `sando/deploy/mnw-testnot-smoke.sh` exists to catch, and a demo
25 //! box that quietly reverts to grey squares is worse than one that refuses to
26 //! reseed. Resolution runs to completion before the seed touches the database, so
27 //! a failure leaves the previous catalog standing.
28
29 use std::collections::HashMap;
30 use std::path::PathBuf;
31 use std::time::Duration;
32
33 use serde::Deserialize;
34
35 /// The curated manifest, embedded so a deployed binary carries it. Override with
36 /// [`MANIFEST_PATH_ENV`] when iterating locally.
37 const EMBEDDED_MANIFEST: &str = include_str!("media-manifest.toml");
38
39 /// Path to a manifest file to read instead of the embedded copy.
40 pub const MANIFEST_PATH_ENV: &str = "SEED_MEDIA_MANIFEST";
41
42 /// Directory holding fetched assets between runs. Defaults to
43 /// `{temp_dir}/mnw-seed-media`.
44 pub const CACHE_DIR_ENV: &str = "SEED_MEDIA_CACHE";
45
46 /// How long a single asset fetch may take.
47 const FETCH_TIMEOUT: Duration = Duration::from_mins(1);
48
49 /// Refuse an asset larger than this. Demo media, not a distribution channel; a
50 /// multi-gigabyte URL in the manifest is a mistake, not a big file.
51 const MAX_ASSET_BYTES: u64 = 64 * 1024 * 1024;
52
53 /// One curated file: where it comes from, what it is, and who to credit.
54 #[derive(Debug, Clone, Deserialize)]
55 pub struct Asset {
56 /// Stable id referenced from `creators.rs` (`ItemSpec::media`,
57 /// `ItemSpec::cover`, `ProjectSpec::cover`).
58 pub id: String,
59 /// Direct download URL. Absent = declared but not yet curated; the slot keeps
60 /// its generated placeholder.
61 #[serde(default)]
62 pub url: Option<String>,
63 /// Lowercase hex SHA-256 of the fetched bytes. Absent on a curated asset is
64 /// allowed but warned about: the run logs the digest it saw so it can be
65 /// pinned. Present and mismatched is a hard failure.
66 #[serde(default)]
67 pub sha256: Option<String>,
68 /// Content type to upload under (`audio/wav`, `image/jpeg`, ...).
69 pub media_type: String,
70 /// Filename to store the object as, and to show on the download.
71 pub filename: String,
72 /// SPDX-ish licence string. Public domain / CC0 only; see the manifest header.
73 pub license: String,
74 /// Human title of the work, for the credits page.
75 pub title: String,
76 /// Creator to credit. `None` for anonymous or corporate-anonymous works.
77 #[serde(default)]
78 pub author: Option<String>,
79 /// The page a visitor can reach to verify the licence claim. Not the direct
80 /// download URL, the landing page.
81 pub source: String,
82 }
83
84 impl Asset {
85 /// Whether this asset has been curated (has somewhere to fetch from).
86 fn is_curated(&self) -> bool {
87 self.url.as_deref().is_some_and(|u| !u.trim().is_empty())
88 }
89 }
90
91 /// Shape of the TOML file: a flat array of assets.
92 #[derive(Debug, Deserialize)]
93 struct ManifestFile {
94 #[serde(default)]
95 asset: Vec<Asset>,
96 }
97
98 /// Why the manifest could not be loaded, or an asset could not be resolved.
99 #[derive(Debug, thiserror::Error)]
100 pub enum ManifestError {
101 /// The override path was set but unreadable.
102 #[error("cannot read media manifest at {path}: {source}")]
103 Read {
104 path: PathBuf,
105 #[source]
106 source: std::io::Error,
107 },
108 /// The manifest is not valid TOML, or does not match the schema.
109 #[error("media manifest is not valid: {0}")]
110 Parse(#[from] toml::de::Error),
111 /// Two entries claim the same id, so a reference is ambiguous.
112 #[error("media manifest declares id {0:?} more than once")]
113 DuplicateId(String),
114 /// One or more curated assets failed to fetch or verify. Collected rather
115 /// than returned one at a time: a curation pass wants the whole list.
116 #[error("{} media asset(s) failed to resolve:\n{}", .0.len(), .0.join("\n"))]
117 Unresolved(Vec<String>),
118 }
119
120 /// The parsed manifest, indexed by asset id.
121 #[derive(Debug, Default)]
122 pub struct Manifest {
123 assets: Vec<Asset>,
124 }
125
126 impl Manifest {
127 /// Load the manifest: the file named by [`MANIFEST_PATH_ENV`] if set,
128 /// otherwise the copy embedded at build time.
129 pub fn load() -> Result<Self, ManifestError> {
130 match std::env::var(MANIFEST_PATH_ENV) {
131 Ok(path) if !path.trim().is_empty() => {
132 let path = PathBuf::from(path);
133 let text =
134 std::fs::read_to_string(&path).map_err(|source| ManifestError::Read {
135 path: path.clone(),
136 source,
137 })?;
138 tracing::info!(path = %path.display(), "example seed: using media manifest override");
139 Self::parse(&text)
140 }
141 _ => Self::parse(EMBEDDED_MANIFEST),
142 }
143 }
144
145 /// Parse and validate manifest text. Separated from [`Self::load`] so the
146 /// embedded manifest can be checked by a unit test with no filesystem.
147 pub fn parse(text: &str) -> Result<Self, ManifestError> {
148 let file: ManifestFile = toml::from_str(text)?;
149 let mut seen = std::collections::HashSet::with_capacity(file.asset.len());
150 for asset in &file.asset {
151 if !seen.insert(asset.id.as_str()) {
152 return Err(ManifestError::DuplicateId(asset.id.clone()));
153 }
154 }
155 Ok(Self { assets: file.asset })
156 }
157
158 /// Every declared id, curated or not. Used by the roster-coverage test.
159 pub fn ids(&self) -> impl Iterator<Item = &str> {
160 self.assets.iter().map(|a| a.id.as_str())
161 }
162
163 /// Fetch every curated asset, verify its digest, and return the resolved set.
164 ///
165 /// Uncurated entries are skipped (their slots keep the placeholder). Any
166 /// curated asset that fails is collected; the call returns
167 /// [`ManifestError::Unresolved`] listing all of them rather than stopping at
168 /// the first.
169 pub async fn resolve(&self) -> Result<ResolvedAssets, ManifestError> {
170 let curated: Vec<&Asset> = self.assets.iter().filter(|a| a.is_curated()).collect();
171 let total = self.assets.len();
172 if curated.is_empty() {
173 tracing::warn!(
174 declared = total,
175 "example seed: no media curated yet; every slot keeps its generated placeholder"
176 );
177 return Ok(ResolvedAssets::default());
178 }
179 tracing::info!(
180 curated = curated.len(),
181 declared = total,
182 "example seed: resolving curated media"
183 );
184
185 let cache = cache_dir();
186 if let Err(e) = std::fs::create_dir_all(&cache) {
187 tracing::warn!(dir = %cache.display(), error = ?e, "example seed: media cache unusable; fetching every asset fresh");
188 }
189
190 crate::crypto::install_default_crypto_provider();
191 let client = reqwest::Client::builder()
192 .timeout(FETCH_TIMEOUT)
193 // Wikimedia's User-Agent policy refuses generic agents outright, and
194 // some of the manifest is hosted there. Name the project and give a
195 // contact, which is what the policy asks for.
196 .user_agent("makenotwork-example-seed/1.0 (+https://makenot.work; info@makenot.work)")
197 .build()
198 .map_err(|e| ManifestError::Unresolved(vec![format!("http client: {e}")]))?;
199
200 let mut resolved = HashMap::with_capacity(curated.len());
201 let mut failures = Vec::new();
202 for asset in curated {
203 match fetch_asset(&client, &cache, asset).await {
204 Ok(bytes) => {
205 resolved.insert(asset.id.clone(), (asset.clone(), bytes));
206 }
207 Err(reason) => failures.push(format!(" {}: {reason}", asset.id)),
208 }
209 }
210
211 if !failures.is_empty() {
212 return Err(ManifestError::Unresolved(failures));
213 }
214 Ok(ResolvedAssets { assets: resolved })
215 }
216 }
217
218 /// Curated media, fetched and verified, ready to upload.
219 #[derive(Debug, Default)]
220 pub struct ResolvedAssets {
221 assets: HashMap<String, (Asset, Vec<u8>)>,
222 }
223
224 impl ResolvedAssets {
225 /// The asset behind an id, if it was curated and resolved.
226 pub fn get(&self, id: &str) -> Option<(&Asset, &[u8])> {
227 self.assets.get(id).map(|(a, b)| (a, b.as_slice()))
228 }
229
230 /// The asset behind an optional reference, so call sites can pass
231 /// `spec.cover` straight through.
232 pub fn lookup(&self, id: Option<&str>) -> Option<(&Asset, &[u8])> {
233 self.get(id?)
234 }
235
236 /// How many assets resolved.
237 pub fn len(&self) -> usize {
238 self.assets.len()
239 }
240
241 /// Whether nothing resolved (every slot is on its placeholder).
242 pub fn is_empty(&self) -> bool {
243 self.assets.is_empty()
244 }
245
246 /// A markdown credits list: one line per resolved asset, title linked to the
247 /// source page, with author and licence. Sorted by title so a reseed with the
248 /// same manifest produces the same page.
249 ///
250 /// Empty string when nothing resolved, so the caller can skip publishing.
251 pub fn attribution_markdown(&self) -> String {
252 if self.assets.is_empty() {
253 return String::new();
254 }
255 let mut lines: Vec<String> = self
256 .assets
257 .values()
258 .map(|(a, _)| {
259 let author = a
260 .author
261 .as_deref()
262 .map_or_else(String::new, |author| format!(" by {author}"));
263 format!("- [{}]({}){} — {}", a.title, a.source, author, a.license)
264 })
265 .collect();
266 lines.sort();
267 lines.dedup();
268 lines.join("\n")
269 }
270 }
271
272 /// Where fetched assets are cached between runs.
273 fn cache_dir() -> PathBuf {
274 match std::env::var(CACHE_DIR_ENV) {
275 Ok(dir) if !dir.trim().is_empty() => PathBuf::from(dir),
276 _ => std::env::temp_dir().join("mnw-seed-media"),
277 }
278 }
279
280 /// Fetch one asset, preferring the cache, and verify its digest.
281 ///
282 /// The cache is keyed by id, and a cached file is only trusted when the manifest
283 /// pins a digest and the file matches it. An unpinned asset is re-fetched every
284 /// run: without a digest there is nothing to tell a stale cache entry from a
285 /// current one.
286 async fn fetch_asset(
287 client: &reqwest::Client,
288 cache: &std::path::Path,
289 asset: &Asset,
290 ) -> Result<Vec<u8>, String> {
291 let cached = cache.join(&asset.id);
292 if let (Some(want), Ok(bytes)) = (asset.sha256.as_deref(), std::fs::read(&cached))
293 && digest_hex(&bytes).eq_ignore_ascii_case(want.trim())
294 {
295 tracing::debug!(id = %asset.id, "example seed: media cache hit");
296 return Ok(bytes);
297 }
298
299 let url = asset.url.as_deref().unwrap_or_default().trim();
300 let response = client
301 .get(url)
302 .send()
303 .await
304 .map_err(|e| format!("fetching {url}: {e}"))?;
305 if !response.status().is_success() {
306 return Err(format!("fetching {url}: HTTP {}", response.status()));
307 }
308 // Refuse an oversized body before buffering it, when the server declares one.
309 if let Some(len) = response.content_length()
310 && len > MAX_ASSET_BYTES
311 {
312 return Err(format!(
313 "fetching {url}: {len} bytes exceeds the {MAX_ASSET_BYTES}-byte asset ceiling"
314 ));
315 }
316 let bytes = response
317 .bytes()
318 .await
319 .map_err(|e| format!("reading {url}: {e}"))?
320 .to_vec();
321 if bytes.len() as u64 > MAX_ASSET_BYTES {
322 return Err(format!(
323 "fetching {url}: {} bytes exceeds the {MAX_ASSET_BYTES}-byte asset ceiling",
324 bytes.len()
325 ));
326 }
327 if bytes.is_empty() {
328 return Err(format!("fetching {url}: empty body"));
329 }
330
331 let got = digest_hex(&bytes);
332 match asset.sha256.as_deref().map(str::trim) {
333 Some(want) if !want.is_empty() => {
334 if !got.eq_ignore_ascii_case(want) {
335 return Err(format!(
336 "digest mismatch for {url}: manifest pins {want}, fetched {got}. \
337 The asset changed at the source; re-check the licence before repinning."
338 ));
339 }
340 }
341 _ => tracing::warn!(
342 id = %asset.id,
343 sha256 = %got,
344 "example seed: asset is unpinned; add sha256 = \"{got}\" to media-manifest.toml"
345 ),
346 }
347
348 if let Err(e) = std::fs::write(&cached, &bytes) {
349 tracing::debug!(id = %asset.id, error = ?e, "example seed: could not cache asset");
350 }
351 tracing::info!(id = %asset.id, bytes = bytes.len(), "example seed: fetched media asset");
352 Ok(bytes)
353 }
354
355 /// Lowercase hex SHA-256.
356 fn digest_hex(bytes: &[u8]) -> String {
357 use sha2::{Digest, Sha256};
358 hex::encode(Sha256::digest(bytes))
359 }
360
361 #[cfg(test)]
362 mod tests {
363 use super::*;
364
365 #[test]
366 fn embedded_manifest_parses() {
367 Manifest::parse(EMBEDDED_MANIFEST).expect("the embedded manifest must always be valid");
368 }
369
370 #[test]
371 fn duplicate_ids_are_refused() {
372 let text = r#"
373 [[asset]]
374 id = "a"
375 media_type = "image/png"
376 filename = "a.png"
377 license = "CC0-1.0"
378 title = "A"
379 source = "https://example.test/a"
380
381 [[asset]]
382 id = "a"
383 media_type = "image/png"
384 filename = "b.png"
385 license = "CC0-1.0"
386 title = "B"
387 source = "https://example.test/b"
388 "#;
389 assert!(matches!(
390 Manifest::parse(text).unwrap_err(),
391 ManifestError::DuplicateId(id) if id == "a"
392 ));
393 }
394
395 #[test]
396 fn an_asset_without_a_url_is_uncurated() {
397 let text = r#"
398 [[asset]]
399 id = "a"
400 url = " "
401 media_type = "image/png"
402 filename = "a.png"
403 license = "CC0-1.0"
404 title = "A"
405 source = "https://example.test/a"
406 "#;
407 let manifest = Manifest::parse(text).unwrap();
408 assert!(!manifest.assets[0].is_curated());
409 }
410
411 #[tokio::test]
412 async fn an_uncurated_manifest_never_reaches_for_the_network() {
413 // Synthetic rather than the embedded manifest, which is now fully
414 // curated: this is a claim about the code path, and it should keep
415 // holding whatever the shipped manifest looks like.
416 let text = r#"
417 [[asset]]
418 id = "a"
419 media_type = "image/png"
420 filename = "a.png"
421 license = "CC0-1.0"
422 title = "A"
423 source = "https://example.test/a"
424 "#;
425 let resolved = Manifest::parse(text).unwrap().resolve().await.unwrap();
426 assert!(resolved.is_empty());
427 }
428
429 #[test]
430 fn every_shipped_asset_is_curated_and_pinned() {
431 // An unpinned asset still works, but it re-fetches every run and cannot
432 // detect the file changing at the source. The shipped manifest was
433 // curated in one pass with digests taken from the bytes that arrived,
434 // so anything unpinned here is an entry someone added without running
435 // it.
436 let manifest = Manifest::load().expect("manifest loads");
437 for asset in &manifest.assets {
438 assert!(asset.is_curated(), "{} has no url", asset.id);
439 let digest = asset.sha256.as_deref().unwrap_or_default().trim();
440 assert_eq!(digest.len(), 64, "{} is not pinned", asset.id);
441 assert!(
442 digest.chars().all(|c| c.is_ascii_hexdigit()),
443 "{} has a malformed digest",
444 asset.id
445 );
446 }
447 }
448
449 /// Fetch every shipped asset and check it against its pin.
450 ///
451 /// Ignored by default: it is ~90 MB over the network and depends on two
452 /// museums staying up, neither of which belongs in a normal test run. Run it
453 /// after editing the manifest, which is the moment it earns its cost:
454 ///
455 /// cargo test --lib seed::manifest -- --ignored --nocapture
456 #[tokio::test]
457 #[ignore = "network: fetches every asset in the manifest"]
458 async fn every_shipped_asset_actually_resolves() {
459 let manifest = Manifest::load().expect("manifest loads");
460 let declared = manifest.assets.len();
461 let resolved = manifest
462 .resolve()
463 .await
464 .expect("every asset should resolve");
465 assert_eq!(
466 resolved.len(),
467 declared,
468 "resolved {} of {declared} assets",
469 resolved.len()
470 );
471 assert!(!resolved.attribution_markdown().is_empty());
472 }
473
474 #[test]
475 fn every_shipped_asset_is_public_domain_and_verifiable() {
476 // Rule 1 and rule 2 of the manifest header, enforced rather than
477 // trusted. These files sit on a public box under a licence claim this
478 // repo makes; a CC-BY asset slipping in is a licensing problem, and a
479 // `source` that is not a reachable page makes the claim uncheckable.
480 let manifest = Manifest::load().expect("manifest loads");
481 for asset in &manifest.assets {
482 let licence = asset.license.to_ascii_lowercase();
483 assert!(
484 licence.contains("cc0") || licence.contains("public domain"),
485 "{} is licensed {:?}, which is not public domain or CC0",
486 asset.id,
487 asset.license
488 );
489 assert!(
490 asset.source.starts_with("https://"),
491 "{} has no https source page",
492 asset.id
493 );
494 assert!(
495 asset
496 .url
497 .as_deref()
498 .is_some_and(|u| u.starts_with("https://")),
499 "{} is not fetched over https",
500 asset.id
501 );
502 }
503 }
504
505 #[test]
506 fn attribution_lists_resolved_assets_only() {
507 let asset = Asset {
508 id: "x".into(),
509 url: Some("https://example.test/x.jpg".into()),
510 sha256: None,
511 media_type: "image/jpeg".into(),
512 filename: "x.jpg".into(),
513 license: "CC0-1.0".into(),
514 title: "A Study".into(),
515 author: Some("A. Person".into()),
516 source: "https://example.test/x".into(),
517 };
518 let mut assets = HashMap::new();
519 assets.insert("x".to_string(), (asset, vec![1, 2, 3]));
520 let resolved = ResolvedAssets { assets };
521 assert_eq!(
522 resolved.attribution_markdown(),
523 "- [A Study](https://example.test/x) by A. Person — CC0-1.0"
524 );
525 assert!(ResolvedAssets::default().attribution_markdown().is_empty());
526 }
527
528 #[test]
529 fn digest_is_lowercase_hex_sha256() {
530 // Known vector: SHA-256 of the empty string.
531 assert_eq!(
532 digest_hex(b""),
533 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
534 );
535 }
536 }
537