Skip to main content

max / makenotwork

20.6 KB · 471 lines History Blame Raw
1 //! Collecting a build's artifacts: finding them, proving they are the version
2 //! that was asked for, hashing them, and pulling them back.
3
4 use super::RecipeCtx;
5 use crate::domain::Version;
6 use crate::events::{self, Event};
7 use anyhow::{Context as _, Result};
8 use ops_exec::SyncOpts;
9 use sha2::{Digest, Sha256};
10 use std::path::{Path, PathBuf};
11 use std::sync::Arc;
12
13 /// Every `X.Y.Z`-shaped version embedded in an artifact file name. Each maximal
14 /// run of digits-and-dots contributes its first three numeric fields:
15 /// `GoingsOn_0.5.0_aarch64.dmg` and `demo-9.9.9.bin` both yield one version (the
16 /// trailing `.bin`/`.dmg` dot is tolerated), while `latest.json` yields `[]` and
17 /// the `64` in `x86_64` is not three fields. Only the `major.minor.patch` core is
18 /// taken; a prerelease/build suffix is separated by `-`/`+` and not needed here.
19 pub(super) fn versions_in_filename(name: &str) -> Vec<Version> {
20 name.split(|c: char| !(c.is_ascii_digit() || c == '.'))
21 .filter_map(|run| {
22 let f: Vec<&str> = run.split('.').filter(|s| !s.is_empty()).collect();
23 if f.len() >= 3 && f[..3].iter().all(|s| s.chars().all(|c| c.is_ascii_digit())) {
24 Version::parse(&format!("{}.{}.{}", f[0], f[1], f[2])).ok()
25 } else {
26 None
27 }
28 })
29 .collect()
30 }
31
32 /// Fail when a collected file's name embeds a version whose `major.minor.patch`
33 /// is not the one being built. This is the guard against a stale checked-in
34 /// artifact winning a glob: `ls -t <glob>` once let
35 /// `AudioFiles-0.4.0-x86_64.AppImage` ship against 0.5.0. A file whose name
36 /// carries no version (an updater `latest.json`, a `.sig`) is not asserted —
37 /// there is nothing to compare. Compared on the core so a prerelease build's
38 /// plain `X.Y.Z` in the filename still matches.
39 pub(super) fn assert_artifact_version(name: &str, expected: &Version) -> Result<()> {
40 let versions = versions_in_filename(name);
41 anyhow::ensure!(
42 versions.is_empty() || versions.iter().any(|v| v.core() == expected.core()),
43 "collected artifact `{name}` carries version {} but the build is {expected}; \
44 a stale artifact was left in the output dir — clean it so only {expected} remains",
45 versions
46 .iter()
47 .map(ToString::to_string)
48 .collect::<Vec<_>>()
49 .join("/"),
50 );
51 Ok(())
52 }
53
54 /// sha256 of a file, lowercase hex. Streams in 64 KiB chunks so a multi-GiB
55 /// bundle never lands in memory whole.
56 pub(super) fn sha256_file(path: &Path) -> Result<String> {
57 let mut file =
58 std::fs::File::open(path).with_context(|| format!("hashing {}", path.display()))?;
59 let mut hasher = Sha256::new();
60 std::io::copy(&mut file, &mut hasher)
61 .with_context(|| format!("reading {} to hash", path.display()))?;
62 Ok(hex_lower(&hasher.finalize()))
63 }
64
65 /// Every regular file under `root`, as `(path relative to root, absolute path)`,
66 /// sorted by the relative path.
67 ///
68 /// **This walk has to match `bundle::digest_dir` in sando, file for file.** That
69 /// function re-hashes an incoming bundle and refuses it when the bytes disagree
70 /// with the manifest they arrived with, so a producer that walks differently
71 /// produces a manifest the consumer will reject for an artifact nothing is wrong
72 /// with. Three properties carry that agreement, and none is incidental:
73 ///
74 /// - **Recursive.** A bundle may carry a directory (migrations, resources), and
75 /// a top-level-only listing would omit its contents from the manifest while
76 /// the verifier hashed them.
77 /// - **Symlinks are not followed, and not recorded.** Following one would let
78 /// content from outside the bundle into its identity; recording the link
79 /// itself would name a file the verifier does not hash.
80 /// - **Relative paths, `/`-separated, sorted.** Readdir order is not guaranteed,
81 /// so an unsorted manifest would differ run to run on one machine, never mind
82 /// between two.
83 pub(super) fn collected_files(root: &Path) -> std::io::Result<Vec<(String, PathBuf)>> {
84 fn walk(dir: &Path, root: &Path, out: &mut Vec<(String, PathBuf)>) -> std::io::Result<()> {
85 for entry in std::fs::read_dir(dir)? {
86 let entry = entry?;
87 let ft = entry.file_type()?;
88 let path = entry.path();
89 if ft.is_dir() {
90 walk(&path, root, out)?;
91 } else if ft.is_file() {
92 let rel = path
93 .strip_prefix(root)
94 .unwrap_or(&path)
95 .components()
96 .map(|c| c.as_os_str().to_string_lossy())
97 .collect::<Vec<_>>()
98 .join("/");
99 out.push((rel, path));
100 }
101 // Symlinks and other special files are intentionally ignored,
102 // matching the verifier.
103 }
104 Ok(())
105 }
106 let mut out = Vec::new();
107 walk(root, root, &mut out)?;
108 out.sort_by(|a, b| a.0.cmp(&b.0));
109 Ok(out)
110 }
111
112 /// Lowercase-hex encode without pulling in a hex crate.
113 pub(super) fn hex_lower(bytes: &[u8]) -> String {
114 use std::fmt::Write as _;
115 let mut s = String::with_capacity(bytes.len() * 2);
116 for b in bytes {
117 let _ = write!(s, "{b:02x}");
118 }
119 s
120 }
121
122 /// Reject a glob that carries shell command metacharacters. Path and wildcard
123 /// characters (`/ . * ? [ ] ~` etc.) are fine — the pattern reaches a login
124 /// shell to be expanded — but a `;` or `$(...)` must not ride along and run.
125 /// Not a privilege boundary (a recipe already runs arbitrary shell via `sh_ok`)
126 /// but it keeps a malformed pattern from turning into a command. Shared by
127 /// `collect` and `resolve_artifact`.
128 pub(super) fn ensure_glob_safe(glob: &str) -> Result<()> {
129 anyhow::ensure!(
130 !glob.chars().any(|c| matches!(
131 c,
132 ';' | '&' | '|' | '$' | '`' | '\'' | '"' | '\\' | ' ' | '\n' | '(' | ')' | '<' | '>'
133 )),
134 "glob `{glob}` contains shell metacharacters"
135 );
136 Ok(())
137 }
138
139 /// Decide the single artifact a glob resolves to from a newline-separated
140 /// listing of the paths that matched it.
141 ///
142 /// Demands exactly one match. Zero matches fail when `required` (return `""`
143 /// when optional); more than one is always an error rather than an arbitrary
144 /// newest-wins pick, because an ambiguous match means the build left stale
145 /// artifacts behind and the wrong one could ship.
146 pub(super) fn resolve_artifact_match(listing: &str, glob: &str, required: bool) -> Result<String> {
147 let matches: Vec<&str> = listing
148 .lines()
149 .map(str::trim)
150 .filter(|l| !l.is_empty())
151 .collect();
152 match matches.as_slice() {
153 [] if required => anyhow::bail!("no artifact matched glob `{glob}`"),
154 [] => Ok(String::new()),
155 [one] => Ok((*one).to_string()),
156 many => anyhow::bail!(
157 "glob `{glob}` is ambiguous: {} artifacts matched ({}). \
158 The build left more than one behind; clean stale artifacts so exactly one remains.",
159 many.len(),
160 many.join(", ")
161 ),
162 }
163 }
164
165 pub(super) fn dir_size(p: &Path) -> Option<i64> {
166 let mut total = 0i64;
167 for entry in std::fs::read_dir(p).ok()? {
168 let entry = entry.ok()?;
169 let md = entry.metadata().ok()?;
170 if md.is_file() {
171 total += md.len() as i64;
172 } else if md.is_dir() {
173 // Recurse so a bundle dir (a `.app`) reports its real size, not ~0.
174 total += dir_size(&entry.path()).unwrap_or(0);
175 }
176 }
177 Some(total)
178 }
179
180 impl RecipeCtx {
181 /// Resolve `glob` on `host` to the single artifact it names. The `for` loop
182 /// lists each existing match on its own line (and prints nothing — rather
183 /// than a literal unexpanded pattern — when the glob matches no file), so
184 /// the count is unambiguous. `required` controls whether zero matches is an
185 /// error; more than one always is. See `resolve_artifact_match`.
186 pub(super) fn resolve_artifact(
187 self: &Arc<Self>,
188 host: &str,
189 glob: &str,
190 required: bool,
191 ) -> Result<String> {
192 ensure_glob_safe(glob)?;
193 // `[ -e ]` guards against a non-matching glob surviving as its literal
194 // self, and lists one path per line for the count.
195 let cmd = format!("for __f in {glob}; do [ -e \"$__f\" ] && printf '%s\\n' \"$__f\"; done");
196 let (code, tail) = self.run(host, &cmd)?;
197 anyhow::ensure!(
198 code == 0,
199 "resolving artifact glob `{glob}` on `{host}` exited {code}"
200 );
201 resolve_artifact_match(&tail, glob, required)
202 }
203
204 /// Where this run's collected files land locally.
205 ///
206 /// Per target, not per version. Sharing one `dist_root/<app>/<version>/`
207 /// across targets would have the hash loop below (which lists the directory)
208 /// attribute a sibling's AppImage to the mac build's artifact record. It is
209 /// also the layout the archive uses, and the two have to agree or the local
210 /// copy and the deposited one are different shapes.
211 pub(super) fn collect_dest(&self, app: &str, version: &str) -> PathBuf {
212 self.cfg
213 .dist_root
214 .join(app)
215 .join(version)
216 .join(crate::archive::target_slug(self.target))
217 }
218
219 pub(super) fn collect(
220 self: &Arc<Self>,
221 host: &str,
222 glob: &str,
223 app: &str,
224 version: &str,
225 ) -> Result<()> {
226 let dest = self.collect_dest(app, version);
227 let dest_s = dest.to_string_lossy().into_owned();
228 // The glob reaches a remote login shell intact (that's what expands it),
229 // so command metacharacters stay barred. Path/wildcard chars are fine.
230 ensure_glob_safe(glob)?;
231 std::fs::create_dir_all(&dest)
232 .with_context(|| format!("creating collect dest {dest_s}"))?;
233 // The SYNC transport, not the host's exec executor: artifacts move over
234 // ssh/rsync even from an agent host, whose `/pull` is confined to a
235 // narrow `pull_root` that deliberately excludes the repo checkout these
236 // artifacts are built in (see `state::build_sync`). The daemon still
237 // runs the transfer itself, as it always has.
238 let sync = self.host_sync(host)?;
239 let opts = SyncOpts::precompressed();
240 // Bounded by the collect step's deadline (rsync of a multi-GiB artifact
241 // can wedge on a stalled transport) and interruptible on supersession.
242 let dest_pull = dest.clone();
243 self.run_bounded(&format!("collect {glob} from `{host}`"), async move {
244 sync.pull_glob(glob, &dest_pull, &opts).await
245 })
246 .with_context(|| format!("collect {glob} from `{host}`"))?;
247 // Assert the version and hash every collected file. This is where a
248 // stale artifact is caught: a file whose name embeds a different version
249 // fails the collect (rather than silently winning a later glob), and the
250 // sha256 recorded here is what `publish` writes into the release ledger
251 // and what the artifact record's manifest is built from.
252 //
253 // Recursive, and keyed by path relative to the collect dir. That is not
254 // a preference: Sando's intake re-hashes the bundle with its own walker,
255 // which recurses and keys the same way, and refuses a bundle whose bytes
256 // do not match the manifest it was handed. A top-level `read_dir` keyed
257 // by file name agrees with that walker for a flat directory and diverges
258 // the moment a bundle carries a subdirectory — the honest artifact would
259 // be refused for a manifest that omitted everything nested. The two
260 // walkers have to be the same walk. See `bundle::digest_dir` in sando.
261 for (rel, path) in
262 collected_files(&dest).with_context(|| format!("listing collect dest {dest_s}"))?
263 {
264 // The version check stays on the file NAME rather than the relative
265 // path: it is looking for a stale `app_1.2.3.AppImage` beside the
266 // one this release built, and a directory component is not that.
267 let name = path
268 .file_name()
269 .map_or_else(|| rel.clone(), |n| n.to_string_lossy().into_owned());
270 assert_artifact_version(&name, &self.version)?;
271 let digest = sha256_file(&path)?;
272 self.record_artifact_hash(rel, digest);
273 }
274 // Deposit at the archive path, so this target's bytes have one address
275 // whichever host produced them. A no-op when no archive is configured.
276 //
277 // Inside `collect`, not after the recipe: a failure here fails the
278 // collect step, before sign and publish, rather than putting a red mark
279 // on a release that has already shipped. And it is a failure, not a
280 // warning — a deposit that is quietly skipped leaves the archive path
281 // wrong for exactly the release nobody was watching, which is the thing
282 // having one address is for.
283 let (cfg, app_id, version, target) = (
284 self.cfg.clone(),
285 self.app.clone(),
286 self.version.clone(),
287 self.target,
288 );
289 let dest_archive = dest.clone();
290 self.run_bounded("deposit in the archive", async move {
291 crate::archive::deposit(&cfg, &dest_archive, &app_id, &version, target).await
292 })?;
293 // Best-effort size accounting for the event.
294 events::emit(
295 &self.events,
296 Event::ArtifactCollected {
297 app: self.app.clone(),
298 target: self.target,
299 path: dest_s,
300 bytes: dir_size(&dest).unwrap_or(0),
301 },
302 );
303 Ok(())
304 }
305 }
306
307 #[cfg(test)]
308 mod tests {
309 use super::*;
310
311 /// A parsed version, for the cases below.
312 fn ver(s: &str) -> Version {
313 Version::parse(s).unwrap()
314 }
315
316 /// Build the shared cross-crate bundle fixture under `root`.
317 ///
318 /// A binary at the top and two files in a subdirectory: the shape a service
319 /// that ships its migrations has, and the case a flat walk gets wrong.
320 pub(crate) fn write_bundle_fixture(root: &Path) {
321 std::fs::create_dir_all(root.join("migrations")).unwrap();
322 std::fs::write(root.join("pom"), b"binary-bytes").unwrap();
323 std::fs::write(root.join("migrations/001_init.sql"), b"create table a;").unwrap();
324 std::fs::write(root.join("migrations/002_next.sql"), b"alter table a;").unwrap();
325 }
326
327 /// The manifest text the fixture must produce, in BOTH crates.
328 ///
329 /// Sando's `bundle::digest_dir` has the identical constant and the identical
330 /// fixture. That is the whole point: bento writes this text into the artifact
331 /// record, sando recomputes it from the bytes that arrive, and an artifact is
332 /// refused when they differ. Two walks, one answer, pinned from both ends —
333 /// if either crate's walk drifts, its own test fails and names the drift
334 /// rather than a release failing intake for a bundle nothing is wrong with.
335 pub(crate) const BUNDLE_FIXTURE_MANIFEST: &str = concat!(
336 "e4c908e219c533fa7ad7ea1634398f9bf51637ba20717769ada545bab26d7368 migrations/001_init.sql\n",
337 "b026fd51bae096b34672cefdb781b6585b13efb53bc301d50c305f422552a380 migrations/002_next.sql\n",
338 "71227a7f160afca3fb3c39f448735886dda7bd366252580c2222fb87d4bb4d85 pom\n",
339 );
340
341 /// The producer half of the contract above: what `collect` hashes, turned
342 /// into a manifest, is exactly the text the verifier will recompute.
343 ///
344 /// Nested files are included and addressed by relative path. Before this,
345 /// `collect` listed only the top level, so `migrations/` contributed nothing
346 /// to the manifest while sando's walker hashed both files in it — and the
347 /// honest bundle was refused for a manifest that had omitted them.
348 #[test]
349 fn a_collected_bundle_manifests_exactly_as_the_verifier_will_read_it() {
350 let dir = tempfile::tempdir().unwrap();
351 write_bundle_fixture(dir.path());
352
353 let files = collected_files(dir.path()).unwrap();
354 assert_eq!(
355 files.iter().map(|(r, _)| r.as_str()).collect::<Vec<_>>(),
356 vec!["migrations/001_init.sql", "migrations/002_next.sql", "pom"],
357 "recursive, relative, sorted"
358 );
359
360 let hashes: Vec<(String, String)> = files
361 .into_iter()
362 .map(|(rel, path)| (rel, sha256_file(&path).unwrap()))
363 .collect();
364 let manifest = ops_artifact::Manifest::new(hashes).unwrap();
365 assert_eq!(manifest.to_text(), BUNDLE_FIXTURE_MANIFEST);
366 }
367
368 /// A symlink is neither followed nor named. Following one would let bytes
369 /// from outside the bundle into its identity; naming it would put a path in
370 /// the manifest the verifier does not hash, which reads as a corrupt bundle.
371 #[test]
372 #[cfg(unix)]
373 fn a_symlink_in_the_collect_dir_is_not_part_of_the_bundle() {
374 let dir = tempfile::tempdir().unwrap();
375 write_bundle_fixture(dir.path());
376 let outside = dir.path().join("..").join("secret.env");
377 std::fs::write(&outside, b"TOKEN=1").ok();
378 std::os::unix::fs::symlink(&outside, dir.path().join("link.env")).unwrap();
379
380 let files = collected_files(dir.path()).unwrap();
381 assert!(
382 !files.iter().any(|(rel, _)| rel.contains("link.env")),
383 "{files:?}"
384 );
385 }
386
387 #[test]
388 fn resolve_artifact_match_wants_exactly_one() {
389 // Exactly one match: the path, trimmed of the listing's line noise.
390 assert_eq!(
391 resolve_artifact_match(" /d/App.AppImage \n", "*.AppImage", true).unwrap(),
392 "/d/App.AppImage"
393 );
394 }
395
396 #[test]
397 fn resolve_artifact_match_zero_depends_on_required() {
398 // Required + zero matches is the case the old empty-string guard caught;
399 // keep failing it.
400 let err = resolve_artifact_match("", "*.dmg", true).unwrap_err();
401 assert!(err.to_string().contains("no artifact matched"), "{err}");
402 // Optional + zero matches resolves to empty (recipe skips the collect).
403 assert_eq!(
404 resolve_artifact_match("\n \n", "*.deb", false).unwrap(),
405 ""
406 );
407 }
408
409 #[test]
410 fn resolve_artifact_match_rejects_ambiguous() {
411 // Two matches must throw rather than silently pick one — this is the
412 // stale-newest-mtime hole the audit flagged. Applies even when optional.
413 for required in [true, false] {
414 let err =
415 resolve_artifact_match("/d/old.deb\n/d/new.deb\n", "*.deb", required).unwrap_err();
416 let msg = err.to_string();
417 assert!(msg.contains("ambiguous"), "{msg}");
418 assert!(msg.contains("old.deb") && msg.contains("new.deb"), "{msg}");
419 }
420 }
421
422 #[test]
423 fn ensure_glob_safe_allows_paths_bars_commands() {
424 // Path and wildcard characters pass.
425 assert!(ensure_glob_safe("~/Code/app/dist/*.AppImage").is_ok());
426 assert!(ensure_glob_safe("/t/App_1.2.3-x86_64.dmg").is_ok());
427 // A command substitution or separator does not.
428 for bad in ["*.dmg; rm -rf /", "$(evil)", "a|b", "a b"] {
429 assert!(ensure_glob_safe(bad).is_err(), "should reject {bad:?}");
430 }
431 }
432
433 #[test]
434 fn versions_in_filename_extracts_only_real_semvers() {
435 assert_eq!(
436 versions_in_filename("GoingsOn_0.5.0_aarch64.dmg"),
437 vec![ver("0.5.0")]
438 );
439 assert_eq!(
440 versions_in_filename("AudioFiles-0.4.0-x86_64.AppImage"),
441 vec![ver("0.4.0")]
442 );
443 // No three-part token ⇒ nothing (an updater manifest, a bare signature).
444 assert!(versions_in_filename("latest.json").is_empty());
445 assert!(versions_in_filename("app.sig").is_empty());
446 }
447
448 #[test]
449 fn assert_artifact_version_rejects_a_stale_artifact() {
450 // The 0.4.0 file sitting in the output dir against a 0.5.0 build.
451 let err =
452 assert_artifact_version("AudioFiles-0.4.0-x86_64.AppImage", &ver("0.5.0")).unwrap_err();
453 assert!(format!("{err:#}").contains("stale artifact"), "{err:#}");
454 // The matching version passes, and a versionless file is not asserted.
455 assert_artifact_version("GoingsOn_0.5.0_aarch64.dmg", &ver("0.5.0")).unwrap();
456 assert_artifact_version("latest.json", &ver("0.5.0")).unwrap();
457 }
458
459 #[test]
460 fn sha256_file_is_lowercase_hex_of_contents() {
461 let tmp = tempfile::tempdir().unwrap();
462 let f = tmp.path().join("a.bin");
463 std::fs::write(&f, b"abc").unwrap();
464 // Known SHA-256 of "abc".
465 assert_eq!(
466 sha256_file(&f).unwrap(),
467 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
468 );
469 }
470 }
471