Skip to main content

max / makenotwork

8.4 KB · 225 lines History Blame Raw
1 //! Release-bundle content addressing. Design: wiki [[release-artifact-identity]].
2 //!
3 //! A release bundle is the whole staged release dir — the primary binary plus
4 //! `companions/<name>` plus everything from `cfg.release_contents` (static
5 //! assets, docs, error pages). Its identity is the **bundle digest**: the
6 //! sha256 of a `MANIFEST` listing one `<sha256> <relpath>` line per file,
7 //! sorted by relative path.
8 //!
9 //! Sorting by path makes the manifest order-independent and reproducible, and
10 //! buys two properties for free: node-side verification is per-file, so a
11 //! mismatch names the file that drifted rather than just reporting that
12 //! something did; and the manifest is plain text readable on a node with no
13 //! tooling.
14 //!
15 //! Hashing the *bundle* rather than the primary binary is deliberate: two
16 //! bundles with identical binaries but differing assets must not collide — that
17 //! is the exact drift that crash-looped prod on a missing `CDN_BASE_URL`
18 //! (postmortem 2026-07-09 #2).
19
20 use anyhow::{Context, Result};
21 use sha2::{Digest, Sha256};
22 use std::path::{Path, PathBuf};
23
24 /// The manifest file's name at the bundle root. Excluded from its own hash so
25 /// a bundle that carries its MANIFEST (for node-side verification) still
26 /// digests to the same value as the bundle before the file was written.
27 pub const MANIFEST_NAME: &str = "MANIFEST";
28
29 /// The full bundle digest recorded on `build_runs.bundle_digest` (64 hex), and
30 /// the 16-char prefix used as the content-addressed directory name.
31 #[derive(Debug, Clone, PartialEq, Eq)]
32 pub struct BundleDigest {
33 /// sha256 of the manifest text, lowercase hex, 64 chars.
34 pub full: String,
35 /// The manifest text itself, ready to write to `<bundle>/MANIFEST`.
36 pub manifest: String,
37 }
38
39 impl BundleDigest {
40 /// The 16-hex-char prefix used in `releases/<digest16>/`. Full digest is
41 /// kept in the DB; the path only needs enough to be collision-safe.
42 pub fn short(&self) -> &str {
43 &self.full[..16]
44 }
45 }
46
47 /// Compute the bundle digest for the staged release dir at `root`.
48 ///
49 /// Walks every regular file under `root` (recursively), hashes each, and builds
50 /// the sorted `MANIFEST`. The manifest file itself ([`MANIFEST_NAME`] at the
51 /// root) is skipped so the digest is stable whether or not it has been written
52 /// into the bundle yet.
53 ///
54 /// The walk + per-file hashing runs on a blocking thread: a release bundle is
55 /// hundreds of MB of binary, and hashing it must not stall the async runtime.
56 pub async fn digest_dir(root: &Path) -> Result<BundleDigest> {
57 let root = root.to_path_buf();
58 tokio::task::spawn_blocking(move || digest_dir_blocking(&root))
59 .await
60 .context("bundle digest task panicked")?
61 }
62
63 fn digest_dir_blocking(root: &Path) -> Result<BundleDigest> {
64 let mut files: Vec<PathBuf> = Vec::new();
65 collect_files(root, &mut files)
66 .with_context(|| format!("walking bundle dir {}", root.display()))?;
67
68 // Relative paths, sorted, so the manifest is order-independent across hosts
69 // and filesystems (readdir order is not guaranteed).
70 let mut rows: Vec<(String, PathBuf)> = Vec::with_capacity(files.len());
71 for abs in files {
72 let rel = abs
73 .strip_prefix(root)
74 .with_context(|| format!("{} not under bundle root", abs.display()))?;
75 // Skip the manifest file itself — it is not part of what it describes.
76 if rel.as_os_str() == MANIFEST_NAME {
77 continue;
78 }
79 let rel_str = rel_to_unix(rel);
80 rows.push((rel_str, abs));
81 }
82 rows.sort_by(|a, b| a.0.cmp(&b.0));
83
84 let mut manifest = String::new();
85 for (rel, abs) in &rows {
86 let hash = hash_file(abs).with_context(|| format!("hashing {}", abs.display()))?;
87 // Two spaces between hash and path, matching sha256sum's format so the
88 // manifest is checkable with standard tools on a node.
89 manifest.push_str(&hash);
90 manifest.push_str(" ");
91 manifest.push_str(rel);
92 manifest.push('\n');
93 }
94
95 let full = hex(&Sha256::digest(manifest.as_bytes()));
96 Ok(BundleDigest { full, manifest })
97 }
98
99 /// Recursively collect regular-file paths under `dir`. Symlinks are not
100 /// followed: a staged bundle is a plain tree of copied files, and following
101 /// links would let content outside the bundle leak into its identity.
102 fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) -> std::io::Result<()> {
103 for entry in std::fs::read_dir(dir)? {
104 let entry = entry?;
105 let ft = entry.file_type()?;
106 if ft.is_dir() {
107 collect_files(&entry.path(), out)?;
108 } else if ft.is_file() {
109 out.push(entry.path());
110 }
111 // symlinks / other special files are intentionally ignored.
112 }
113 Ok(())
114 }
115
116 fn hash_file(path: &Path) -> std::io::Result<String> {
117 use std::io::Read;
118 let mut file = std::fs::File::open(path)?;
119 let mut hasher = Sha256::new();
120 let mut buf = vec![0u8; 64 * 1024].into_boxed_slice();
121 loop {
122 let n = file.read(&mut buf)?;
123 if n == 0 {
124 break;
125 }
126 hasher.update(&buf[..n]);
127 }
128 Ok(hex(&hasher.finalize()))
129 }
130
131 /// Relative path as a forward-slash string, so a manifest built on one OS reads
132 /// the same on another. Sando builds on Linux, but keep the identity portable.
133 fn rel_to_unix(rel: &Path) -> String {
134 rel.components()
135 .map(|c| c.as_os_str().to_string_lossy())
136 .collect::<Vec<_>>()
137 .join("/")
138 }
139
140 fn hex(bytes: &[u8]) -> String {
141 use std::fmt::Write;
142 let mut s = String::with_capacity(bytes.len() * 2);
143 for b in bytes {
144 let _ = write!(s, "{b:02x}");
145 }
146 s
147 }
148
149 #[cfg(test)]
150 mod tests {
151 use super::*;
152
153 async fn write(root: &Path, rel: &str, bytes: &[u8]) {
154 let p = root.join(rel);
155 tokio::fs::create_dir_all(p.parent().unwrap())
156 .await
157 .unwrap();
158 tokio::fs::write(&p, bytes).await.unwrap();
159 }
160
161 #[tokio::test]
162 async fn digest_is_stable_and_lists_every_file_sorted() {
163 let dir = tempfile::tempdir().unwrap();
164 let root = dir.path();
165 write(root, "makenotwork", b"binary bytes").await;
166 write(root, "static/app.css", b"body{}").await;
167 write(root, "companions/mnw-cli", b"cli bytes").await;
168
169 let d = super::digest_dir(root).await.unwrap();
170 assert_eq!(d.full.len(), 64);
171 assert_eq!(d.short().len(), 16);
172 // Manifest lines are sorted by relative path.
173 let paths: Vec<&str> = d
174 .manifest
175 .lines()
176 .map(|l| l.split_once(" ").unwrap().1)
177 .collect();
178 assert_eq!(
179 paths,
180 ["companions/mnw-cli", "makenotwork", "static/app.css"]
181 );
182 }
183
184 #[tokio::test]
185 async fn digest_is_independent_of_creation_order() {
186 let a = tempfile::tempdir().unwrap();
187 write(a.path(), "z.txt", b"1").await;
188 write(a.path(), "a.txt", b"2").await;
189 let b = tempfile::tempdir().unwrap();
190 write(b.path(), "a.txt", b"2").await;
191 write(b.path(), "z.txt", b"1").await;
192 assert_eq!(
193 super::digest_dir(a.path()).await.unwrap().full,
194 super::digest_dir(b.path()).await.unwrap().full,
195 );
196 }
197
198 #[tokio::test]
199 async fn a_changed_asset_changes_the_digest_even_with_identical_binary() {
200 let a = tempfile::tempdir().unwrap();
201 write(a.path(), "makenotwork", b"same binary").await;
202 write(a.path(), "static/app.css", b"v1").await;
203 let b = tempfile::tempdir().unwrap();
204 write(b.path(), "makenotwork", b"same binary").await;
205 write(b.path(), "static/app.css", b"v2").await;
206 assert_ne!(
207 super::digest_dir(a.path()).await.unwrap().full,
208 super::digest_dir(b.path()).await.unwrap().full,
209 "asset drift with an identical binary must not collide (2026-07-09 #2)"
210 );
211 }
212
213 #[tokio::test]
214 async fn manifest_file_is_excluded_from_its_own_digest() {
215 let dir = tempfile::tempdir().unwrap();
216 write(dir.path(), "makenotwork", b"bytes").await;
217 let before = super::digest_dir(dir.path()).await.unwrap();
218 // Write the manifest into the bundle, as Stage B will for node-side
219 // verification; the digest must not change.
220 write(dir.path(), MANIFEST_NAME, before.manifest.as_bytes()).await;
221 let after = super::digest_dir(dir.path()).await.unwrap();
222 assert_eq!(before.full, after.full);
223 }
224 }
225