//! Collecting a build's artifacts: finding them, proving they are the version //! that was asked for, hashing them, and pulling them back. use super::RecipeCtx; use crate::domain::Version; use crate::events::{self, Event}; use anyhow::{Context as _, Result}; use ops_exec::SyncOpts; use sha2::{Digest, Sha256}; use std::path::{Path, PathBuf}; use std::sync::Arc; /// Every `X.Y.Z`-shaped version embedded in an artifact file name. Each maximal /// run of digits-and-dots contributes its first three numeric fields: /// `GoingsOn_0.5.0_aarch64.dmg` and `demo-9.9.9.bin` both yield one version (the /// trailing `.bin`/`.dmg` dot is tolerated), while `latest.json` yields `[]` and /// the `64` in `x86_64` is not three fields. Only the `major.minor.patch` core is /// taken; a prerelease/build suffix is separated by `-`/`+` and not needed here. pub(super) fn versions_in_filename(name: &str) -> Vec { name.split(|c: char| !(c.is_ascii_digit() || c == '.')) .filter_map(|run| { let f: Vec<&str> = run.split('.').filter(|s| !s.is_empty()).collect(); if f.len() >= 3 && f[..3].iter().all(|s| s.chars().all(|c| c.is_ascii_digit())) { Version::parse(&format!("{}.{}.{}", f[0], f[1], f[2])).ok() } else { None } }) .collect() } /// Fail when a collected file's name embeds a version whose `major.minor.patch` /// is not the one being built. This is the guard against a stale checked-in /// artifact winning a glob: `ls -t ` once let /// `AudioFiles-0.4.0-x86_64.AppImage` ship against 0.5.0. A file whose name /// carries no version (an updater `latest.json`, a `.sig`) is not asserted — /// there is nothing to compare. Compared on the core so a prerelease build's /// plain `X.Y.Z` in the filename still matches. pub(super) fn assert_artifact_version(name: &str, expected: &Version) -> Result<()> { let versions = versions_in_filename(name); anyhow::ensure!( versions.is_empty() || versions.iter().any(|v| v.core() == expected.core()), "collected artifact `{name}` carries version {} but the build is {expected}; \ a stale artifact was left in the output dir — clean it so only {expected} remains", versions .iter() .map(ToString::to_string) .collect::>() .join("/"), ); Ok(()) } /// sha256 of a file, lowercase hex. Streams in 64 KiB chunks so a multi-GiB /// bundle never lands in memory whole. pub(super) fn sha256_file(path: &Path) -> Result { let mut file = std::fs::File::open(path).with_context(|| format!("hashing {}", path.display()))?; let mut hasher = Sha256::new(); std::io::copy(&mut file, &mut hasher) .with_context(|| format!("reading {} to hash", path.display()))?; Ok(hex_lower(&hasher.finalize())) } /// Every regular file under `root`, as `(path relative to root, absolute path)`, /// sorted by the relative path. /// /// **This walk has to match `bundle::digest_dir` in sando, file for file.** That /// function re-hashes an incoming bundle and refuses it when the bytes disagree /// with the manifest they arrived with, so a producer that walks differently /// produces a manifest the consumer will reject for an artifact nothing is wrong /// with. Three properties carry that agreement, and none is incidental: /// /// - **Recursive.** A bundle may carry a directory (migrations, resources), and /// a top-level-only listing would omit its contents from the manifest while /// the verifier hashed them. /// - **Symlinks are not followed, and not recorded.** Following one would let /// content from outside the bundle into its identity; recording the link /// itself would name a file the verifier does not hash. /// - **Relative paths, `/`-separated, sorted.** Readdir order is not guaranteed, /// so an unsorted manifest would differ run to run on one machine, never mind /// between two. pub(super) fn collected_files(root: &Path) -> std::io::Result> { fn walk(dir: &Path, root: &Path, out: &mut Vec<(String, PathBuf)>) -> std::io::Result<()> { for entry in std::fs::read_dir(dir)? { let entry = entry?; let ft = entry.file_type()?; let path = entry.path(); if ft.is_dir() { walk(&path, root, out)?; } else if ft.is_file() { let rel = path .strip_prefix(root) .unwrap_or(&path) .components() .map(|c| c.as_os_str().to_string_lossy()) .collect::>() .join("/"); out.push((rel, path)); } // Symlinks and other special files are intentionally ignored, // matching the verifier. } Ok(()) } let mut out = Vec::new(); walk(root, root, &mut out)?; out.sort_by(|a, b| a.0.cmp(&b.0)); Ok(out) } /// Lowercase-hex encode without pulling in a hex crate. pub(super) fn hex_lower(bytes: &[u8]) -> String { use std::fmt::Write as _; let mut s = String::with_capacity(bytes.len() * 2); for b in bytes { let _ = write!(s, "{b:02x}"); } s } /// Reject a glob that carries shell command metacharacters. Path and wildcard /// characters (`/ . * ? [ ] ~` etc.) are fine — the pattern reaches a login /// shell to be expanded — but a `;` or `$(...)` must not ride along and run. /// Not a privilege boundary (a recipe already runs arbitrary shell via `sh_ok`) /// but it keeps a malformed pattern from turning into a command. Shared by /// `collect` and `resolve_artifact`. pub(super) fn ensure_glob_safe(glob: &str) -> Result<()> { anyhow::ensure!( !glob.chars().any(|c| matches!( c, ';' | '&' | '|' | '$' | '`' | '\'' | '"' | '\\' | ' ' | '\n' | '(' | ')' | '<' | '>' )), "glob `{glob}` contains shell metacharacters" ); Ok(()) } /// Decide the single artifact a glob resolves to from a newline-separated /// listing of the paths that matched it. /// /// Demands exactly one match. Zero matches fail when `required` (return `""` /// when optional); more than one is always an error rather than an arbitrary /// newest-wins pick, because an ambiguous match means the build left stale /// artifacts behind and the wrong one could ship. pub(super) fn resolve_artifact_match(listing: &str, glob: &str, required: bool) -> Result { let matches: Vec<&str> = listing .lines() .map(str::trim) .filter(|l| !l.is_empty()) .collect(); match matches.as_slice() { [] if required => anyhow::bail!("no artifact matched glob `{glob}`"), [] => Ok(String::new()), [one] => Ok((*one).to_string()), many => anyhow::bail!( "glob `{glob}` is ambiguous: {} artifacts matched ({}). \ The build left more than one behind; clean stale artifacts so exactly one remains.", many.len(), many.join(", ") ), } } pub(super) fn dir_size(p: &Path) -> Option { let mut total = 0i64; for entry in std::fs::read_dir(p).ok()? { let entry = entry.ok()?; let md = entry.metadata().ok()?; if md.is_file() { total += md.len() as i64; } else if md.is_dir() { // Recurse so a bundle dir (a `.app`) reports its real size, not ~0. total += dir_size(&entry.path()).unwrap_or(0); } } Some(total) } impl RecipeCtx { /// Resolve `glob` on `host` to the single artifact it names. The `for` loop /// lists each existing match on its own line (and prints nothing — rather /// than a literal unexpanded pattern — when the glob matches no file), so /// the count is unambiguous. `required` controls whether zero matches is an /// error; more than one always is. See `resolve_artifact_match`. pub(super) fn resolve_artifact( self: &Arc, host: &str, glob: &str, required: bool, ) -> Result { ensure_glob_safe(glob)?; // `[ -e ]` guards against a non-matching glob surviving as its literal // self, and lists one path per line for the count. let cmd = format!("for __f in {glob}; do [ -e \"$__f\" ] && printf '%s\\n' \"$__f\"; done"); let (code, tail) = self.run(host, &cmd)?; anyhow::ensure!( code == 0, "resolving artifact glob `{glob}` on `{host}` exited {code}" ); resolve_artifact_match(&tail, glob, required) } /// Where this run's collected files land locally. /// /// Per target, not per version. Sharing one `dist_root///` /// across targets would have the hash loop below (which lists the directory) /// attribute a sibling's AppImage to the mac build's artifact record. It is /// also the layout the archive uses, and the two have to agree or the local /// copy and the deposited one are different shapes. pub(super) fn collect_dest(&self, app: &str, version: &str) -> PathBuf { self.cfg .dist_root .join(app) .join(version) .join(crate::archive::target_slug(self.target)) } pub(super) fn collect( self: &Arc, host: &str, glob: &str, app: &str, version: &str, ) -> Result<()> { let dest = self.collect_dest(app, version); let dest_s = dest.to_string_lossy().into_owned(); // The glob reaches a remote login shell intact (that's what expands it), // so command metacharacters stay barred. Path/wildcard chars are fine. ensure_glob_safe(glob)?; std::fs::create_dir_all(&dest) .with_context(|| format!("creating collect dest {dest_s}"))?; // The SYNC transport, not the host's exec executor: artifacts move over // ssh/rsync even from an agent host, whose `/pull` is confined to a // narrow `pull_root` that deliberately excludes the repo checkout these // artifacts are built in (see `state::build_sync`). The daemon still // runs the transfer itself, as it always has. let sync = self.host_sync(host)?; let opts = SyncOpts::precompressed(); // Bounded by the collect step's deadline (rsync of a multi-GiB artifact // can wedge on a stalled transport) and interruptible on supersession. let dest_pull = dest.clone(); self.run_bounded(&format!("collect {glob} from `{host}`"), async move { sync.pull_glob(glob, &dest_pull, &opts).await }) .with_context(|| format!("collect {glob} from `{host}`"))?; // Assert the version and hash every collected file. This is where a // stale artifact is caught: a file whose name embeds a different version // fails the collect (rather than silently winning a later glob), and the // sha256 recorded here is what `publish` writes into the release ledger // and what the artifact record's manifest is built from. // // Recursive, and keyed by path relative to the collect dir. That is not // a preference: Sando's intake re-hashes the bundle with its own walker, // which recurses and keys the same way, and refuses a bundle whose bytes // do not match the manifest it was handed. A top-level `read_dir` keyed // by file name agrees with that walker for a flat directory and diverges // the moment a bundle carries a subdirectory — the honest artifact would // be refused for a manifest that omitted everything nested. The two // walkers have to be the same walk. See `bundle::digest_dir` in sando. for (rel, path) in collected_files(&dest).with_context(|| format!("listing collect dest {dest_s}"))? { // The version check stays on the file NAME rather than the relative // path: it is looking for a stale `app_1.2.3.AppImage` beside the // one this release built, and a directory component is not that. let name = path .file_name() .map_or_else(|| rel.clone(), |n| n.to_string_lossy().into_owned()); assert_artifact_version(&name, &self.version)?; let digest = sha256_file(&path)?; self.record_artifact_hash(rel, digest); } // Deposit at the archive path, so this target's bytes have one address // whichever host produced them. A no-op when no archive is configured. // // Inside `collect`, not after the recipe: a failure here fails the // collect step, before sign and publish, rather than putting a red mark // on a release that has already shipped. And it is a failure, not a // warning — a deposit that is quietly skipped leaves the archive path // wrong for exactly the release nobody was watching, which is the thing // having one address is for. let (cfg, app_id, version, target) = ( self.cfg.clone(), self.app.clone(), self.version.clone(), self.target, ); let dest_archive = dest.clone(); self.run_bounded("deposit in the archive", async move { crate::archive::deposit(&cfg, &dest_archive, &app_id, &version, target).await })?; // Best-effort size accounting for the event. events::emit( &self.events, Event::ArtifactCollected { app: self.app.clone(), target: self.target, path: dest_s, bytes: dir_size(&dest).unwrap_or(0), }, ); Ok(()) } } #[cfg(test)] mod tests { use super::*; /// A parsed version, for the cases below. fn ver(s: &str) -> Version { Version::parse(s).unwrap() } /// Build the shared cross-crate bundle fixture under `root`. /// /// A binary at the top and two files in a subdirectory: the shape a service /// that ships its migrations has, and the case a flat walk gets wrong. pub(crate) fn write_bundle_fixture(root: &Path) { std::fs::create_dir_all(root.join("migrations")).unwrap(); std::fs::write(root.join("pom"), b"binary-bytes").unwrap(); std::fs::write(root.join("migrations/001_init.sql"), b"create table a;").unwrap(); std::fs::write(root.join("migrations/002_next.sql"), b"alter table a;").unwrap(); } /// The manifest text the fixture must produce, in BOTH crates. /// /// Sando's `bundle::digest_dir` has the identical constant and the identical /// fixture. That is the whole point: bento writes this text into the artifact /// record, sando recomputes it from the bytes that arrive, and an artifact is /// refused when they differ. Two walks, one answer, pinned from both ends — /// if either crate's walk drifts, its own test fails and names the drift /// rather than a release failing intake for a bundle nothing is wrong with. pub(crate) const BUNDLE_FIXTURE_MANIFEST: &str = concat!( "e4c908e219c533fa7ad7ea1634398f9bf51637ba20717769ada545bab26d7368 migrations/001_init.sql\n", "b026fd51bae096b34672cefdb781b6585b13efb53bc301d50c305f422552a380 migrations/002_next.sql\n", "71227a7f160afca3fb3c39f448735886dda7bd366252580c2222fb87d4bb4d85 pom\n", ); /// The producer half of the contract above: what `collect` hashes, turned /// into a manifest, is exactly the text the verifier will recompute. /// /// Nested files are included and addressed by relative path. Before this, /// `collect` listed only the top level, so `migrations/` contributed nothing /// to the manifest while sando's walker hashed both files in it — and the /// honest bundle was refused for a manifest that had omitted them. #[test] fn a_collected_bundle_manifests_exactly_as_the_verifier_will_read_it() { let dir = tempfile::tempdir().unwrap(); write_bundle_fixture(dir.path()); let files = collected_files(dir.path()).unwrap(); assert_eq!( files.iter().map(|(r, _)| r.as_str()).collect::>(), vec!["migrations/001_init.sql", "migrations/002_next.sql", "pom"], "recursive, relative, sorted" ); let hashes: Vec<(String, String)> = files .into_iter() .map(|(rel, path)| (rel, sha256_file(&path).unwrap())) .collect(); let manifest = ops_artifact::Manifest::new(hashes).unwrap(); assert_eq!(manifest.to_text(), BUNDLE_FIXTURE_MANIFEST); } /// A symlink is neither followed nor named. Following one would let bytes /// from outside the bundle into its identity; naming it would put a path in /// the manifest the verifier does not hash, which reads as a corrupt bundle. #[test] #[cfg(unix)] fn a_symlink_in_the_collect_dir_is_not_part_of_the_bundle() { let dir = tempfile::tempdir().unwrap(); write_bundle_fixture(dir.path()); let outside = dir.path().join("..").join("secret.env"); std::fs::write(&outside, b"TOKEN=1").ok(); std::os::unix::fs::symlink(&outside, dir.path().join("link.env")).unwrap(); let files = collected_files(dir.path()).unwrap(); assert!( !files.iter().any(|(rel, _)| rel.contains("link.env")), "{files:?}" ); } #[test] fn resolve_artifact_match_wants_exactly_one() { // Exactly one match: the path, trimmed of the listing's line noise. assert_eq!( resolve_artifact_match(" /d/App.AppImage \n", "*.AppImage", true).unwrap(), "/d/App.AppImage" ); } #[test] fn resolve_artifact_match_zero_depends_on_required() { // Required + zero matches is the case the old empty-string guard caught; // keep failing it. let err = resolve_artifact_match("", "*.dmg", true).unwrap_err(); assert!(err.to_string().contains("no artifact matched"), "{err}"); // Optional + zero matches resolves to empty (recipe skips the collect). assert_eq!( resolve_artifact_match("\n \n", "*.deb", false).unwrap(), "" ); } #[test] fn resolve_artifact_match_rejects_ambiguous() { // Two matches must throw rather than silently pick one — this is the // stale-newest-mtime hole the audit flagged. Applies even when optional. for required in [true, false] { let err = resolve_artifact_match("/d/old.deb\n/d/new.deb\n", "*.deb", required).unwrap_err(); let msg = err.to_string(); assert!(msg.contains("ambiguous"), "{msg}"); assert!(msg.contains("old.deb") && msg.contains("new.deb"), "{msg}"); } } #[test] fn ensure_glob_safe_allows_paths_bars_commands() { // Path and wildcard characters pass. assert!(ensure_glob_safe("~/Code/app/dist/*.AppImage").is_ok()); assert!(ensure_glob_safe("/t/App_1.2.3-x86_64.dmg").is_ok()); // A command substitution or separator does not. for bad in ["*.dmg; rm -rf /", "$(evil)", "a|b", "a b"] { assert!(ensure_glob_safe(bad).is_err(), "should reject {bad:?}"); } } #[test] fn versions_in_filename_extracts_only_real_semvers() { assert_eq!( versions_in_filename("GoingsOn_0.5.0_aarch64.dmg"), vec![ver("0.5.0")] ); assert_eq!( versions_in_filename("AudioFiles-0.4.0-x86_64.AppImage"), vec![ver("0.4.0")] ); // No three-part token ⇒ nothing (an updater manifest, a bare signature). assert!(versions_in_filename("latest.json").is_empty()); assert!(versions_in_filename("app.sig").is_empty()); } #[test] fn assert_artifact_version_rejects_a_stale_artifact() { // The 0.4.0 file sitting in the output dir against a 0.5.0 build. let err = assert_artifact_version("AudioFiles-0.4.0-x86_64.AppImage", &ver("0.5.0")).unwrap_err(); assert!(format!("{err:#}").contains("stale artifact"), "{err:#}"); // The matching version passes, and a versionless file is not asserted. assert_artifact_version("GoingsOn_0.5.0_aarch64.dmg", &ver("0.5.0")).unwrap(); assert_artifact_version("latest.json", &ver("0.5.0")).unwrap(); } #[test] fn sha256_file_is_lowercase_hex_of_contents() { let tmp = tempfile::tempdir().unwrap(); let f = tmp.path().join("a.bin"); std::fs::write(&f, b"abc").unwrap(); // Known SHA-256 of "abc". assert_eq!( sha256_file(&f).unwrap(), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" ); } }