//! Real public-domain / CC0 media for the example seed: the manifest that names //! each asset, and the fetch step that turns it into bytes. //! //! The seed's other phases are pure data in `creators.rs`. Media cannot be, //! because a real asset is a file somebody has to choose, license-check, and //! host. So the choosing lives in `media-manifest.toml` (curated by hand, checked //! into the repo, embedded in the binary) and everything downstream of it is //! mechanical: fetch by URL, verify the digest, upload through the storage layer, //! record the attribution. //! //! # Curation state is per asset, not per manifest //! //! An entry with no `url` is *declared but not yet curated*. It resolves to //! nothing and [`super::media`] falls back to the generated placeholder for that //! one slot. So the manifest doubles as the curation checklist: every id the seed //! can use is listed, and the ones still carrying grey boxes are exactly the ones //! with an empty `url`. Partial curation is a first-class state, and the box //! stays seedable throughout. //! //! # Failure is loud, and it happens before any write //! //! A curated asset that will not fetch, or that fetches to the wrong bytes, fails //! the whole seed. It does not silently degrade to the placeholder: that is the //! failure mode `sando/deploy/mnw-testnot-smoke.sh` exists to catch, and a demo //! box that quietly reverts to grey squares is worse than one that refuses to //! reseed. Resolution runs to completion before the seed touches the database, so //! a failure leaves the previous catalog standing. use std::collections::HashMap; use std::path::PathBuf; use std::time::Duration; use serde::Deserialize; /// The curated manifest, embedded so a deployed binary carries it. Override with /// [`MANIFEST_PATH_ENV`] when iterating locally. const EMBEDDED_MANIFEST: &str = include_str!("media-manifest.toml"); /// Path to a manifest file to read instead of the embedded copy. pub const MANIFEST_PATH_ENV: &str = "SEED_MEDIA_MANIFEST"; /// Directory holding fetched assets between runs. Defaults to /// `{temp_dir}/mnw-seed-media`. pub const CACHE_DIR_ENV: &str = "SEED_MEDIA_CACHE"; /// How long a single asset fetch may take. const FETCH_TIMEOUT: Duration = Duration::from_mins(1); /// Refuse an asset larger than this. Demo media, not a distribution channel; a /// multi-gigabyte URL in the manifest is a mistake, not a big file. const MAX_ASSET_BYTES: u64 = 64 * 1024 * 1024; /// One curated file: where it comes from, what it is, and who to credit. #[derive(Debug, Clone, Deserialize)] pub struct Asset { /// Stable id referenced from `creators.rs` (`ItemSpec::media`, /// `ItemSpec::cover`, `ProjectSpec::cover`). pub id: String, /// Direct download URL. Absent = declared but not yet curated; the slot keeps /// its generated placeholder. #[serde(default)] pub url: Option, /// Lowercase hex SHA-256 of the fetched bytes. Absent on a curated asset is /// allowed but warned about: the run logs the digest it saw so it can be /// pinned. Present and mismatched is a hard failure. #[serde(default)] pub sha256: Option, /// Content type to upload under (`audio/wav`, `image/jpeg`, ...). pub media_type: String, /// Filename to store the object as, and to show on the download. pub filename: String, /// SPDX-ish licence string. Public domain / CC0 only; see the manifest header. pub license: String, /// Human title of the work, for the credits page. pub title: String, /// Creator to credit. `None` for anonymous or corporate-anonymous works. #[serde(default)] pub author: Option, /// The page a visitor can reach to verify the licence claim. Not the direct /// download URL, the landing page. pub source: String, } impl Asset { /// Whether this asset has been curated (has somewhere to fetch from). fn is_curated(&self) -> bool { self.url.as_deref().is_some_and(|u| !u.trim().is_empty()) } } /// Shape of the TOML file: a flat array of assets. #[derive(Debug, Deserialize)] struct ManifestFile { #[serde(default)] asset: Vec, } /// Why the manifest could not be loaded, or an asset could not be resolved. #[derive(Debug, thiserror::Error)] pub enum ManifestError { /// The override path was set but unreadable. #[error("cannot read media manifest at {path}: {source}")] Read { path: PathBuf, #[source] source: std::io::Error, }, /// The manifest is not valid TOML, or does not match the schema. #[error("media manifest is not valid: {0}")] Parse(#[from] toml::de::Error), /// Two entries claim the same id, so a reference is ambiguous. #[error("media manifest declares id {0:?} more than once")] DuplicateId(String), /// One or more curated assets failed to fetch or verify. Collected rather /// than returned one at a time: a curation pass wants the whole list. #[error("{} media asset(s) failed to resolve:\n{}", .0.len(), .0.join("\n"))] Unresolved(Vec), } /// The parsed manifest, indexed by asset id. #[derive(Debug, Default)] pub struct Manifest { assets: Vec, } impl Manifest { /// Load the manifest: the file named by [`MANIFEST_PATH_ENV`] if set, /// otherwise the copy embedded at build time. pub fn load() -> Result { match std::env::var(MANIFEST_PATH_ENV) { Ok(path) if !path.trim().is_empty() => { let path = PathBuf::from(path); let text = std::fs::read_to_string(&path).map_err(|source| ManifestError::Read { path: path.clone(), source, })?; tracing::info!(path = %path.display(), "example seed: using media manifest override"); Self::parse(&text) } _ => Self::parse(EMBEDDED_MANIFEST), } } /// Parse and validate manifest text. Separated from [`Self::load`] so the /// embedded manifest can be checked by a unit test with no filesystem. pub fn parse(text: &str) -> Result { let file: ManifestFile = toml::from_str(text)?; let mut seen = std::collections::HashSet::with_capacity(file.asset.len()); for asset in &file.asset { if !seen.insert(asset.id.as_str()) { return Err(ManifestError::DuplicateId(asset.id.clone())); } } Ok(Self { assets: file.asset }) } /// Every declared id, curated or not. Used by the roster-coverage test. pub fn ids(&self) -> impl Iterator { self.assets.iter().map(|a| a.id.as_str()) } /// Fetch every curated asset, verify its digest, and return the resolved set. /// /// Uncurated entries are skipped (their slots keep the placeholder). Any /// curated asset that fails is collected; the call returns /// [`ManifestError::Unresolved`] listing all of them rather than stopping at /// the first. pub async fn resolve(&self) -> Result { let curated: Vec<&Asset> = self.assets.iter().filter(|a| a.is_curated()).collect(); let total = self.assets.len(); if curated.is_empty() { tracing::warn!( declared = total, "example seed: no media curated yet; every slot keeps its generated placeholder" ); return Ok(ResolvedAssets::default()); } tracing::info!( curated = curated.len(), declared = total, "example seed: resolving curated media" ); let cache = cache_dir(); if let Err(e) = std::fs::create_dir_all(&cache) { tracing::warn!(dir = %cache.display(), error = ?e, "example seed: media cache unusable; fetching every asset fresh"); } crate::crypto::install_default_crypto_provider(); let client = reqwest::Client::builder() .timeout(FETCH_TIMEOUT) // Wikimedia's User-Agent policy refuses generic agents outright, and // some of the manifest is hosted there. Name the project and give a // contact, which is what the policy asks for. .user_agent("makenotwork-example-seed/1.0 (+https://makenot.work; info@makenot.work)") .build() .map_err(|e| ManifestError::Unresolved(vec![format!("http client: {e}")]))?; let mut resolved = HashMap::with_capacity(curated.len()); let mut failures = Vec::new(); for asset in curated { match fetch_asset(&client, &cache, asset).await { Ok(bytes) => { resolved.insert(asset.id.clone(), (asset.clone(), bytes)); } Err(reason) => failures.push(format!(" {}: {reason}", asset.id)), } } if !failures.is_empty() { return Err(ManifestError::Unresolved(failures)); } Ok(ResolvedAssets { assets: resolved }) } } /// Curated media, fetched and verified, ready to upload. #[derive(Debug, Default)] pub struct ResolvedAssets { assets: HashMap)>, } impl ResolvedAssets { /// The asset behind an id, if it was curated and resolved. pub fn get(&self, id: &str) -> Option<(&Asset, &[u8])> { self.assets.get(id).map(|(a, b)| (a, b.as_slice())) } /// The asset behind an optional reference, so call sites can pass /// `spec.cover` straight through. pub fn lookup(&self, id: Option<&str>) -> Option<(&Asset, &[u8])> { self.get(id?) } /// How many assets resolved. pub fn len(&self) -> usize { self.assets.len() } /// Whether nothing resolved (every slot is on its placeholder). pub fn is_empty(&self) -> bool { self.assets.is_empty() } /// A markdown credits list: one line per resolved asset, title linked to the /// source page, with author and licence. Sorted by title so a reseed with the /// same manifest produces the same page. /// /// Empty string when nothing resolved, so the caller can skip publishing. pub fn attribution_markdown(&self) -> String { if self.assets.is_empty() { return String::new(); } let mut lines: Vec = self .assets .values() .map(|(a, _)| { let author = a .author .as_deref() .map_or_else(String::new, |author| format!(" by {author}")); format!("- [{}]({}){} — {}", a.title, a.source, author, a.license) }) .collect(); lines.sort(); lines.dedup(); lines.join("\n") } } /// Where fetched assets are cached between runs. fn cache_dir() -> PathBuf { match std::env::var(CACHE_DIR_ENV) { Ok(dir) if !dir.trim().is_empty() => PathBuf::from(dir), _ => std::env::temp_dir().join("mnw-seed-media"), } } /// Fetch one asset, preferring the cache, and verify its digest. /// /// The cache is keyed by id, and a cached file is only trusted when the manifest /// pins a digest and the file matches it. An unpinned asset is re-fetched every /// run: without a digest there is nothing to tell a stale cache entry from a /// current one. async fn fetch_asset( client: &reqwest::Client, cache: &std::path::Path, asset: &Asset, ) -> Result, String> { let cached = cache.join(&asset.id); if let (Some(want), Ok(bytes)) = (asset.sha256.as_deref(), std::fs::read(&cached)) && digest_hex(&bytes).eq_ignore_ascii_case(want.trim()) { tracing::debug!(id = %asset.id, "example seed: media cache hit"); return Ok(bytes); } let url = asset.url.as_deref().unwrap_or_default().trim(); let response = client .get(url) .send() .await .map_err(|e| format!("fetching {url}: {e}"))?; if !response.status().is_success() { return Err(format!("fetching {url}: HTTP {}", response.status())); } // Refuse an oversized body before buffering it, when the server declares one. if let Some(len) = response.content_length() && len > MAX_ASSET_BYTES { return Err(format!( "fetching {url}: {len} bytes exceeds the {MAX_ASSET_BYTES}-byte asset ceiling" )); } let bytes = response .bytes() .await .map_err(|e| format!("reading {url}: {e}"))? .to_vec(); if bytes.len() as u64 > MAX_ASSET_BYTES { return Err(format!( "fetching {url}: {} bytes exceeds the {MAX_ASSET_BYTES}-byte asset ceiling", bytes.len() )); } if bytes.is_empty() { return Err(format!("fetching {url}: empty body")); } let got = digest_hex(&bytes); match asset.sha256.as_deref().map(str::trim) { Some(want) if !want.is_empty() => { if !got.eq_ignore_ascii_case(want) { return Err(format!( "digest mismatch for {url}: manifest pins {want}, fetched {got}. \ The asset changed at the source; re-check the licence before repinning." )); } } _ => tracing::warn!( id = %asset.id, sha256 = %got, "example seed: asset is unpinned; add sha256 = \"{got}\" to media-manifest.toml" ), } if let Err(e) = std::fs::write(&cached, &bytes) { tracing::debug!(id = %asset.id, error = ?e, "example seed: could not cache asset"); } tracing::info!(id = %asset.id, bytes = bytes.len(), "example seed: fetched media asset"); Ok(bytes) } /// Lowercase hex SHA-256. fn digest_hex(bytes: &[u8]) -> String { use sha2::{Digest, Sha256}; hex::encode(Sha256::digest(bytes)) } #[cfg(test)] mod tests { use super::*; #[test] fn embedded_manifest_parses() { Manifest::parse(EMBEDDED_MANIFEST).expect("the embedded manifest must always be valid"); } #[test] fn duplicate_ids_are_refused() { let text = r#" [[asset]] id = "a" media_type = "image/png" filename = "a.png" license = "CC0-1.0" title = "A" source = "https://example.test/a" [[asset]] id = "a" media_type = "image/png" filename = "b.png" license = "CC0-1.0" title = "B" source = "https://example.test/b" "#; assert!(matches!( Manifest::parse(text).unwrap_err(), ManifestError::DuplicateId(id) if id == "a" )); } #[test] fn an_asset_without_a_url_is_uncurated() { let text = r#" [[asset]] id = "a" url = " " media_type = "image/png" filename = "a.png" license = "CC0-1.0" title = "A" source = "https://example.test/a" "#; let manifest = Manifest::parse(text).unwrap(); assert!(!manifest.assets[0].is_curated()); } #[tokio::test] async fn an_uncurated_manifest_never_reaches_for_the_network() { // Synthetic rather than the embedded manifest, which is now fully // curated: this is a claim about the code path, and it should keep // holding whatever the shipped manifest looks like. let text = r#" [[asset]] id = "a" media_type = "image/png" filename = "a.png" license = "CC0-1.0" title = "A" source = "https://example.test/a" "#; let resolved = Manifest::parse(text).unwrap().resolve().await.unwrap(); assert!(resolved.is_empty()); } #[test] fn every_shipped_asset_is_curated_and_pinned() { // An unpinned asset still works, but it re-fetches every run and cannot // detect the file changing at the source. The shipped manifest was // curated in one pass with digests taken from the bytes that arrived, // so anything unpinned here is an entry someone added without running // it. let manifest = Manifest::load().expect("manifest loads"); for asset in &manifest.assets { assert!(asset.is_curated(), "{} has no url", asset.id); let digest = asset.sha256.as_deref().unwrap_or_default().trim(); assert_eq!(digest.len(), 64, "{} is not pinned", asset.id); assert!( digest.chars().all(|c| c.is_ascii_hexdigit()), "{} has a malformed digest", asset.id ); } } /// Fetch every shipped asset and check it against its pin. /// /// Ignored by default: it is ~90 MB over the network and depends on two /// museums staying up, neither of which belongs in a normal test run. Run it /// after editing the manifest, which is the moment it earns its cost: /// /// cargo test --lib seed::manifest -- --ignored --nocapture #[tokio::test] #[ignore = "network: fetches every asset in the manifest"] async fn every_shipped_asset_actually_resolves() { let manifest = Manifest::load().expect("manifest loads"); let declared = manifest.assets.len(); let resolved = manifest .resolve() .await .expect("every asset should resolve"); assert_eq!( resolved.len(), declared, "resolved {} of {declared} assets", resolved.len() ); assert!(!resolved.attribution_markdown().is_empty()); } #[test] fn every_shipped_asset_is_public_domain_and_verifiable() { // Rule 1 and rule 2 of the manifest header, enforced rather than // trusted. These files sit on a public box under a licence claim this // repo makes; a CC-BY asset slipping in is a licensing problem, and a // `source` that is not a reachable page makes the claim uncheckable. let manifest = Manifest::load().expect("manifest loads"); for asset in &manifest.assets { let licence = asset.license.to_ascii_lowercase(); assert!( licence.contains("cc0") || licence.contains("public domain"), "{} is licensed {:?}, which is not public domain or CC0", asset.id, asset.license ); assert!( asset.source.starts_with("https://"), "{} has no https source page", asset.id ); assert!( asset .url .as_deref() .is_some_and(|u| u.starts_with("https://")), "{} is not fetched over https", asset.id ); } } #[test] fn attribution_lists_resolved_assets_only() { let asset = Asset { id: "x".into(), url: Some("https://example.test/x.jpg".into()), sha256: None, media_type: "image/jpeg".into(), filename: "x.jpg".into(), license: "CC0-1.0".into(), title: "A Study".into(), author: Some("A. Person".into()), source: "https://example.test/x".into(), }; let mut assets = HashMap::new(); assets.insert("x".to_string(), (asset, vec![1, 2, 3])); let resolved = ResolvedAssets { assets }; assert_eq!( resolved.attribution_markdown(), "- [A Study](https://example.test/x) by A. Person — CC0-1.0" ); assert!(ResolvedAssets::default().attribution_markdown().is_empty()); } #[test] fn digest_is_lowercase_hex_sha256() { // Known vector: SHA-256 of the empty string. assert_eq!( digest_hex(b""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ); } }