Skip to main content

max / makenotwork

10.2 KB · 259 lines History Blame Raw
1 //! Release-bundle content addressing. Design: wiki [[release-artifact-identity]].
2 //!
3 //! A release bundle is the whole staged release dir — the primary binary plus
4 //! `companions/<name>` plus everything from `cfg.release_contents` (static
5 //! assets, docs, error pages). Its identity is the **bundle digest**: the
6 //! sha256 of a `MANIFEST` listing one `<sha256> <relpath>` line per file,
7 //! sorted by relative path.
8 //!
9 //! Sorting by path makes the manifest order-independent and reproducible, and
10 //! buys two properties for free: node-side verification is per-file, so a
11 //! mismatch names the file that drifted rather than just reporting that
12 //! something did; and the manifest is plain text readable on a node with no
13 //! tooling.
14 //!
15 //! Hashing the *bundle* rather than the primary binary is deliberate: two
16 //! bundles with identical binaries but differing assets must not collide — that
17 //! is the exact drift that crash-looped prod on a missing `CDN_BASE_URL`
18 //! (postmortem 2026-07-09 #2).
19
20 use anyhow::{Context, Result};
21 use sha2::{Digest, Sha256};
22 use std::path::{Path, PathBuf};
23
24 /// The manifest file's name at the bundle root. Excluded from its own hash so
25 /// a bundle that carries its MANIFEST (for node-side verification) still
26 /// digests to the same value as the bundle before the file was written.
27 pub const MANIFEST_NAME: &str = "MANIFEST";
28
29 /// The full bundle digest recorded on `build_runs.bundle_digest` (64 hex), and
30 /// the 16-char prefix used as the content-addressed directory name.
31 #[derive(Debug, Clone, PartialEq, Eq)]
32 pub struct BundleDigest {
33 /// sha256 of the manifest text, lowercase hex, 64 chars.
34 pub full: String,
35 /// The manifest text itself, ready to write to `<bundle>/MANIFEST`.
36 pub manifest: String,
37 }
38
39 impl BundleDigest {
40 /// The 16-hex-char prefix used in `releases/<digest16>/`. Full digest is
41 /// kept in the DB; the path only needs enough to be collision-safe.
42 pub fn short(&self) -> &str {
43 &self.full[..16]
44 }
45 }
46
47 /// Compute the bundle digest for the staged release dir at `root`.
48 ///
49 /// Walks every regular file under `root` (recursively), hashes each, and builds
50 /// the sorted `MANIFEST`. The manifest file itself ([`MANIFEST_NAME`] at the
51 /// root) is skipped so the digest is stable whether or not it has been written
52 /// into the bundle yet.
53 ///
54 /// The walk + per-file hashing runs on a blocking thread: a release bundle is
55 /// hundreds of MB of binary, and hashing it must not stall the async runtime.
56 pub async fn digest_dir(root: &Path) -> Result<BundleDigest> {
57 let root = root.to_path_buf();
58 tokio::task::spawn_blocking(move || digest_dir_blocking(&root))
59 .await
60 .context("bundle digest task panicked")?
61 }
62
63 fn digest_dir_blocking(root: &Path) -> Result<BundleDigest> {
64 let mut files: Vec<PathBuf> = Vec::new();
65 collect_files(root, &mut files)
66 .with_context(|| format!("walking bundle dir {}", root.display()))?;
67
68 // Relative paths, sorted, so the manifest is order-independent across hosts
69 // and filesystems (readdir order is not guaranteed).
70 let mut rows: Vec<(String, PathBuf)> = Vec::with_capacity(files.len());
71 for abs in files {
72 let rel = abs
73 .strip_prefix(root)
74 .with_context(|| format!("{} not under bundle root", abs.display()))?;
75 // Skip the manifest file itself — it is not part of what it describes.
76 if rel.as_os_str() == MANIFEST_NAME {
77 continue;
78 }
79 let rel_str = rel_to_unix(rel);
80 rows.push((rel_str, abs));
81 }
82 rows.sort_by(|a, b| a.0.cmp(&b.0));
83
84 let mut manifest = String::new();
85 for (rel, abs) in &rows {
86 let hash = hash_file(abs).with_context(|| format!("hashing {}", abs.display()))?;
87 // Two spaces between hash and path, matching sha256sum's format so the
88 // manifest is checkable with standard tools on a node.
89 manifest.push_str(&hash);
90 manifest.push_str(" ");
91 manifest.push_str(rel);
92 manifest.push('\n');
93 }
94
95 let full = hex(&Sha256::digest(manifest.as_bytes()));
96 Ok(BundleDigest { full, manifest })
97 }
98
99 /// Recursively collect regular-file paths under `dir`. Symlinks are not
100 /// followed: a staged bundle is a plain tree of copied files, and following
101 /// links would let content outside the bundle leak into its identity.
102 fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) -> std::io::Result<()> {
103 for entry in std::fs::read_dir(dir)? {
104 let entry = entry?;
105 let ft = entry.file_type()?;
106 if ft.is_dir() {
107 collect_files(&entry.path(), out)?;
108 } else if ft.is_file() {
109 out.push(entry.path());
110 }
111 // symlinks / other special files are intentionally ignored.
112 }
113 Ok(())
114 }
115
116 fn hash_file(path: &Path) -> std::io::Result<String> {
117 use std::io::Read;
118 let mut file = std::fs::File::open(path)?;
119 let mut hasher = Sha256::new();
120 let mut buf = vec![0u8; 64 * 1024].into_boxed_slice();
121 loop {
122 let n = file.read(&mut buf)?;
123 if n == 0 {
124 break;
125 }
126 hasher.update(&buf[..n]);
127 }
128 Ok(hex(&hasher.finalize()))
129 }
130
131 /// Relative path as a forward-slash string, so a manifest built on one OS reads
132 /// the same on another. Sando builds on Linux, but keep the identity portable.
133 fn rel_to_unix(rel: &Path) -> String {
134 rel.components()
135 .map(|c| c.as_os_str().to_string_lossy())
136 .collect::<Vec<_>>()
137 .join("/")
138 }
139
140 fn hex(bytes: &[u8]) -> String {
141 use std::fmt::Write;
142 let mut s = String::with_capacity(bytes.len() * 2);
143 for b in bytes {
144 let _ = write!(s, "{b:02x}");
145 }
146 s
147 }
148
149 #[cfg(test)]
150 mod tests {
151 use super::*;
152
153 /// The manifest text the shared cross-crate fixture must produce, in BOTH
154 /// crates.
155 ///
156 /// Bento's `engine.rs` has the identical constant and the identical fixture.
157 /// Bento writes this text into the artifact record at `collect`; this walker
158 /// recomputes it from the bytes that arrive at intake, and an artifact whose
159 /// two answers differ is refused. Producer and consumer are different walks
160 /// in different repos, so the agreement is pinned from both ends — if either
161 /// drifts, its own test fails and names the drift, instead of a release
162 /// failing intake for a bundle nothing is wrong with.
163 ///
164 /// The nested directory is the case that matters: bento's collect used to
165 /// list only the top level, so `migrations/` contributed nothing to the
166 /// manifest while this walker hashed both files in it.
167 const BUNDLE_FIXTURE_MANIFEST: &str = concat!(
168 "e4c908e219c533fa7ad7ea1634398f9bf51637ba20717769ada545bab26d7368 migrations/001_init.sql\n",
169 "b026fd51bae096b34672cefdb781b6585b13efb53bc301d50c305f422552a380 migrations/002_next.sql\n",
170 "71227a7f160afca3fb3c39f448735886dda7bd366252580c2222fb87d4bb4d85 pom\n",
171 );
172
173 /// The consumer half of the contract: what arrives is read exactly the way
174 /// the producer described it.
175 #[tokio::test]
176 async fn the_shared_bundle_fixture_digests_as_the_producer_wrote_it() {
177 let tmp = tempfile::tempdir().unwrap();
178 let root = tmp.path();
179 write(root, "pom", b"binary-bytes").await;
180 write(root, "migrations/001_init.sql", b"create table a;").await;
181 write(root, "migrations/002_next.sql", b"alter table a;").await;
182
183 let d = digest_dir(root).await.unwrap();
184 assert_eq!(d.manifest, BUNDLE_FIXTURE_MANIFEST);
185 }
186
187 async fn write(root: &Path, rel: &str, bytes: &[u8]) {
188 let p = root.join(rel);
189 tokio::fs::create_dir_all(p.parent().unwrap())
190 .await
191 .unwrap();
192 tokio::fs::write(&p, bytes).await.unwrap();
193 }
194
195 #[tokio::test]
196 async fn digest_is_stable_and_lists_every_file_sorted() {
197 let dir = tempfile::tempdir().unwrap();
198 let root = dir.path();
199 write(root, "makenotwork", b"binary bytes").await;
200 write(root, "static/app.css", b"body{}").await;
201 write(root, "companions/mnw-cli", b"cli bytes").await;
202
203 let d = super::digest_dir(root).await.unwrap();
204 assert_eq!(d.full.len(), 64);
205 assert_eq!(d.short().len(), 16);
206 // Manifest lines are sorted by relative path.
207 let paths: Vec<&str> = d
208 .manifest
209 .lines()
210 .map(|l| l.split_once(" ").unwrap().1)
211 .collect();
212 assert_eq!(
213 paths,
214 ["companions/mnw-cli", "makenotwork", "static/app.css"]
215 );
216 }
217
218 #[tokio::test]
219 async fn digest_is_independent_of_creation_order() {
220 let a = tempfile::tempdir().unwrap();
221 write(a.path(), "z.txt", b"1").await;
222 write(a.path(), "a.txt", b"2").await;
223 let b = tempfile::tempdir().unwrap();
224 write(b.path(), "a.txt", b"2").await;
225 write(b.path(), "z.txt", b"1").await;
226 assert_eq!(
227 super::digest_dir(a.path()).await.unwrap().full,
228 super::digest_dir(b.path()).await.unwrap().full,
229 );
230 }
231
232 #[tokio::test]
233 async fn a_changed_asset_changes_the_digest_even_with_identical_binary() {
234 let a = tempfile::tempdir().unwrap();
235 write(a.path(), "makenotwork", b"same binary").await;
236 write(a.path(), "static/app.css", b"v1").await;
237 let b = tempfile::tempdir().unwrap();
238 write(b.path(), "makenotwork", b"same binary").await;
239 write(b.path(), "static/app.css", b"v2").await;
240 assert_ne!(
241 super::digest_dir(a.path()).await.unwrap().full,
242 super::digest_dir(b.path()).await.unwrap().full,
243 "asset drift with an identical binary must not collide (2026-07-09 #2)"
244 );
245 }
246
247 #[tokio::test]
248 async fn manifest_file_is_excluded_from_its_own_digest() {
249 let dir = tempfile::tempdir().unwrap();
250 write(dir.path(), "makenotwork", b"bytes").await;
251 let before = super::digest_dir(dir.path()).await.unwrap();
252 // Write the manifest into the bundle, as Stage B will for node-side
253 // verification; the digest must not change.
254 write(dir.path(), MANIFEST_NAME, before.manifest.as_bytes()).await;
255 let after = super::digest_dir(dir.path()).await.unwrap();
256 assert_eq!(before.full, after.full);
257 }
258 }
259