Skip to main content

max / makenotwork

sando: content-addressed release identity; key promote/burn-in on build id Make build_runs.id the artifact identity and demote version to a label (wiki release-artifact-identity). Closes the burn-in hole where `promote --version Y` inherited the source tier's clock/evidence, and the overwrite class where a dev rebuild reused releases/<version> in place. - migration 008: additive/nullable build_runs.bundle_digest/staged_path, gate_runs.build_id, deploys.build_id, tier_state.current/previous_build_id. Legacy NULL rows fall back to the version-string path. - bundle.rs: content-addressed MANIFEST digest (sha256sum format, self- excluded, no symlink follow). - staging: stage_local_bundle -> hash -> finalize_local_release renames staging/<build_id> to releases/<digest16> (atomic), writes MANIFEST. - promote resolves the source tier's current_build_id and refuses an explicit version that isn't it; build-evidence gates key on build_id. - node-side verification: sha256sum -c MANIFEST after rsync, before swap; legacy bundles without a MANIFEST skip rather than fail. - retention now counts builds (one digest dir per build); burn-in clock resets on build_id change via advance_tier. 246 tests green, clippy clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-23 23:31 UTC
Commit: 30b420e7a59d12efd1920d8dd5322beaa1e85fef
Parent: c2ae59e
11 files changed, +1089 insertions, -160 deletions
M sando/Cargo.lock +8
@@ -289,6 +289,12 @@ dependencies = [
289 289 ]
290 290
291 291 [[package]]
292 + name = "const-oid"
293 + version = "0.10.2"
294 + source = "registry+https://github.com/rust-lang/crates.io-index"
295 + checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
296 +
297 + [[package]]
292 298 name = "core-foundation"
293 299 version = "0.10.1"
294 300 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -403,6 +409,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
403 409 checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
404 410 dependencies = [
405 411 "block-buffer 0.12.1",
412 + "const-oid",
406 413 "crypto-common 0.2.2",
407 414 "ctutils",
408 415 ]
@@ -1618,6 +1625,7 @@ dependencies = [
1618 1625 "semver",
1619 1626 "serde",
1620 1627 "serde_json",
1628 + "sha2 0.11.0",
1621 1629 "sqlx",
1622 1630 "tempfile",
1623 1631 "thiserror",
M sando/daemon/Cargo.toml +1
@@ -25,6 +25,7 @@ anyhow = "1.0.102"
25 25 thiserror = "2.0.18"
26 26 chrono = { version = "0.4", features = ["serde"] }
27 27 semver = { version = "1.0", features = ["serde"] }
28 + sha2 = "0.11"
28 29
29 30 [dev-dependencies]
30 31 tempfile = "3.20"
A sando/daemon/migrations/008_build_identity.sql +36
@@ -0,0 +1,36 @@
1 + -- Release artifact identity: make build_runs.id the identity object and demote
2 + -- `version` to a label. Design: wiki [[release-artifact-identity]] (decided
3 + -- 2026-07-21). The defect it closes: `version` (a semver read from Cargo.toml)
4 + -- was simultaneously the artifact identity, the gate-evidence key, the
5 + -- deployed-state key, and the staging directory name. Versions bump at release,
6 + -- not per commit, so dev rebuilds overwrote releases/<version>/ in place while
7 + -- the gate rows and burn-in clock from an earlier build stayed green — promote
8 + -- shipped a binary that never passed the gates it was credited with.
9 + --
10 + -- build_runs (migration 007) is already one row per /rebuild and carries `sha`.
11 + -- It is the right identity object. Everything below is ADDITIVE and nullable:
12 + -- legacy rows keep NULL and are treated as "pre-identity". One fresh
13 + -- build-and-promote cycle re-anchors each tier. An honest NULL beats a
14 + -- fabricated backfill for live prod state.
15 +
16 + -- Identity lives on the build row.
17 + -- bundle_digest: sha256 of the bundle MANIFEST, full 64 hex. Answers "what
18 + -- bytes" — identity. Distinct from `sha` (provenance: "what source"), which
19 + -- already exists on build_runs. Cargo builds are not bit-reproducible, so a
20 + -- digest match across hosts is a strictly stronger claim than a sha match.
21 + -- staged_path: releases/<digest16> — the content-addressed dir the bundle was
22 + -- renamed into. Replaces versions.artifact_path as the source of truth.
23 + ALTER TABLE build_runs ADD COLUMN bundle_digest TEXT;
24 + ALTER TABLE build_runs ADD COLUMN staged_path TEXT;
25 +
26 + -- The evidence binds to the build it vouched for, not to a version string.
27 + ALTER TABLE gate_runs ADD COLUMN build_id INTEGER REFERENCES build_runs(id);
28 + ALTER TABLE deploys ADD COLUMN build_id INTEGER REFERENCES build_runs(id);
29 +
30 + -- Deployed state points at build rows. previous_build_id keeps exactly one step
31 + -- of rollback history, mirroring the existing (current|previous)_version pair.
32 + ALTER TABLE tier_state ADD COLUMN current_build_id INTEGER REFERENCES build_runs(id);
33 + ALTER TABLE tier_state ADD COLUMN previous_build_id INTEGER REFERENCES build_runs(id);
34 +
35 + CREATE INDEX gate_runs_by_build ON gate_runs(build_id, gate_kind);
36 + CREATE INDEX deploys_by_build ON deploys(build_id);
M sando/daemon/src/build.rs +86 -24
@@ -324,17 +324,20 @@ pub async fn stage_and_gate(
324 324 .await
325 325 .ok();
326 326
327 - // Stage the binary in the host's release_root so future gates and the
328 - // host self-deploy point at a stable path, not the worktree's target/.
327 + // Stage the bundle into `staging/<build_id>/` — a private scratch dir, not
328 + // yet a release. It is published content-addressed below, once its digest is
329 + // known. This is what makes overwrite unexpressible (wiki
330 + // [[release-artifact-identity]]): a build never touches another build's dir.
329 331 let host_release_root = &cfg.release_root;
330 - let staged = deploy::deploy_local(host_release_root, &art.version, &art.binary_paths).await?;
332 + let staging =
333 + deploy::stage_local_bundle(host_release_root, run_id.0, &art.binary_paths).await?;
331 334
332 - // Stage every entry from cfg.release_contents into the staged release dir.
333 - // This is how non-binary version-coupled content (static assets, docs,
334 - // error-pages, ...) makes it into the atomic deploy bundle. Projects opt
335 - // in via daemon config — the sando code carries no MNW-specific knowledge.
335 + // Stage every entry from cfg.release_contents into the staged bundle. This is
336 + // how non-binary version-coupled content (static assets, docs, error-pages,
337 + // ...) makes it into the atomic deploy bundle. Projects opt in via daemon
338 + // config — the sando code carries no MNW-specific knowledge.
336 339 for entry in &cfg.release_contents {
337 - stage_entry(&art.worktree, &staged, entry).await?;
340 + stage_entry(&art.worktree, &staging, entry).await?;
338 341 }
339 342
340 343 // Stage companion binaries as `companions/<name>` (the file itself) so they
@@ -343,7 +346,7 @@ pub async fn stage_and_gate(
343 346 // the topology. The nodes that opt in install them post-swap (see
344 347 // deploy::deploy_remote).
345 348 if !art.companion_paths.is_empty() {
346 - let dst_dir = staged.join("companions");
349 + let dst_dir = staging.join("companions");
347 350 tokio::fs::create_dir_all(&dst_dir)
348 351 .await
349 352 .with_context(|| format!("create staged companions dir {}", dst_dir.display()))?;
@@ -359,13 +362,39 @@ pub async fn stage_and_gate(
359 362 }
360 363 }
361 364
362 - let staged_bin = staged.join(cfg.primary_bin());
365 + // Content identity: hash the fully-staged bundle, write its MANIFEST into the
366 + // bundle (for node-side verification), then publish it at `releases/<digest16>`.
367 + // The digest is now load-bearing — a hashing failure fails the build rather
368 + // than shipping an unidentifiable artifact.
369 + let digest = crate::bundle::digest_dir(&staging)
370 + .await
371 + .context("hashing the staged bundle for content addressing")?;
372 + tokio::fs::write(
373 + staging.join(crate::bundle::MANIFEST_NAME),
374 + digest.manifest.as_bytes(),
375 + )
376 + .await
377 + .context("writing bundle MANIFEST")?;
378 + let released =
379 + deploy::finalize_local_release(host_release_root, &staging, digest.short()).await?;
380 +
381 + let staged_bin = released.join(cfg.primary_bin());
363 382 sqlx::query("UPDATE versions SET artifact_path = ? WHERE version = ?")
364 383 .bind(staged_bin.to_string_lossy().as_ref())
365 384 .bind(&art.version)
366 385 .execute(&pool)
367 386 .await?;
368 387
388 + // Record the identity on the build row: the full digest and the
389 + // content-addressed dir the bundle was published to. This is what promote
390 + // resolves the artifact through, and burn-in/retention key on.
391 + {
392 + let released_path = released.to_string_lossy();
393 + crate::runs::set_identity(&pool, run_id, &digest.full, &released_path)
394 + .await
395 + .ok();
396 + }
397 +
369 398 let host = topo
370 399 .tiers
371 400 .iter()
@@ -386,6 +415,9 @@ pub async fn stage_and_gate(
386 415 // boot_smoke) only — `node_health` never appears here, so there are no
387 416 // nodes to probe.
388 417 nodes: Vec::new(),
418 + // These gates vouch for this build; record its id so promote can resolve
419 + // the artifact through the evidence rather than a version string.
420 + build_id: Some(run_id.0),
389 421 };
390 422 let failed = gates::run_all(&ctx, &host.gates).await?;
391 423
@@ -396,7 +428,7 @@ pub async fn stage_and_gate(
396 428 // ultra-fuzz Run 2, S1). Held only for the atomic UPDATE, never the gates.
397 429 {
398 430 let _deploy_guard = deploy_lock.lock().await;
399 - crate::runs::advance_tier(&pool, "host", &art.version).await?;
431 + crate::runs::advance_tier(&pool, "host", &art.version, Some(run_id.0)).await?;
400 432 }
401 433 // Terminal verdict: unlike the phase pings above (best-effort), a dropped
402 434 // pass/fail write leaves the run wedged at `building`. Log it loudly if it
@@ -691,27 +723,57 @@ mod tests {
691 723 assert_eq!(result, "passed");
692 724 assert_eq!(summary, None);
693 725
694 - // versions.artifact_path now points at the staged primary binary, and
695 - // that file actually exists on disk under release_root.
726 + // Identity: the build row carries the bundle digest (64 hex) and the
727 + // content-addressed dir it was published to (releases/<digest16>).
728 + let (digest, staged_path): (Option<String>, Option<String>) =
729 + sqlx::query_as("SELECT bundle_digest, staged_path FROM build_runs WHERE id = ?")
730 + .bind(run_id.0)
731 + .fetch_one(&pool)
732 + .await
733 + .unwrap();
734 + let digest = digest.expect("bundle_digest recorded");
735 + let staged_path = staged_path.expect("staged_path recorded");
736 + assert_eq!(digest.len(), 64);
737 + let releases = tmp.path().join("release-root").join("releases");
738 + assert_eq!(
739 + std::path::Path::new(&staged_path),
740 + releases.join(&digest[..16]),
741 + "bundle is published content-addressed at releases/<digest16>"
742 + );
743 +
744 + // versions.artifact_path points at the primary binary inside that dir,
745 + // and it exists on disk.
696 746 let staged_bin: String =
697 747 sqlx::query_scalar("SELECT artifact_path FROM versions WHERE version = ?")
698 748 .bind(version.to_string())
699 749 .fetch_one(&pool)
700 750 .await
701 751 .unwrap();
702 - let expected = tmp
703 - .path()
704 - .join("release-root")
705 - .join("releases")
706 - .join(version.to_string())
707 - .join("makenotwork");
708 - assert_eq!(staged_bin, expected.to_string_lossy());
709 - assert!(expected.exists(), "staged binary missing at {expected:?}");
710 -
711 - // The `current` symlink flipped to the new release.
752 + let expected_bin = releases.join(&digest[..16]).join("makenotwork");
753 + assert_eq!(staged_bin, expected_bin.to_string_lossy());
754 + assert!(
755 + expected_bin.exists(),
756 + "staged binary missing at {expected_bin:?}"
757 + );
758 +
759 + // The bundle carries its MANIFEST (for node-side verification), and the
760 + // recorded digest recomputes over the published dir (MANIFEST excluded).
761 + assert!(
762 + releases.join(&digest[..16]).join("MANIFEST").exists(),
763 + "MANIFEST written into the bundle"
764 + );
765 + let recomputed = crate::bundle::digest_dir(std::path::Path::new(&staged_path))
766 + .await
767 + .unwrap();
768 + assert_eq!(
769 + digest, recomputed.full,
770 + "recorded digest matches the bundle"
771 + );
772 +
773 + // The `current` symlink flipped to the content-addressed release.
712 774 let link = tmp.path().join("release-root").join("current");
713 775 let target = std::fs::read_link(&link).expect("current is a symlink");
714 - assert_eq!(target, PathBuf::from(format!("releases/{version}")));
776 + assert_eq!(target, PathBuf::from(format!("releases/{}", &digest[..16])));
715 777 }
716 778
717 779 #[tokio::test]
A sando/daemon/src/bundle.rs +224
@@ -0,0 +1,224 @@
1 + //! Release-bundle content addressing. Design: wiki [[release-artifact-identity]].
2 + //!
3 + //! A release bundle is the whole staged release dir — the primary binary plus
4 + //! `companions/<name>` plus everything from `cfg.release_contents` (static
5 + //! assets, docs, error pages). Its identity is the **bundle digest**: the
6 + //! sha256 of a `MANIFEST` listing one `<sha256> <relpath>` line per file,
7 + //! sorted by relative path.
8 + //!
9 + //! Sorting by path makes the manifest order-independent and reproducible, and
10 + //! buys two properties for free: node-side verification is per-file, so a
11 + //! mismatch names the file that drifted rather than just reporting that
12 + //! something did; and the manifest is plain text readable on a node with no
13 + //! tooling.
14 + //!
15 + //! Hashing the *bundle* rather than the primary binary is deliberate: two
16 + //! bundles with identical binaries but differing assets must not collide — that
17 + //! is the exact drift that crash-looped prod on a missing `CDN_BASE_URL`
18 + //! (postmortem 2026-07-09 #2).
19 +
20 + use anyhow::{Context, Result};
21 + use sha2::{Digest, Sha256};
22 + use std::path::{Path, PathBuf};
23 +
24 + /// The manifest file's name at the bundle root. Excluded from its own hash so
25 + /// a bundle that carries its MANIFEST (for node-side verification) still
26 + /// digests to the same value as the bundle before the file was written.
27 + pub const MANIFEST_NAME: &str = "MANIFEST";
28 +
29 + /// The full bundle digest recorded on `build_runs.bundle_digest` (64 hex), and
30 + /// the 16-char prefix used as the content-addressed directory name.
31 + #[derive(Debug, Clone, PartialEq, Eq)]
32 + pub struct BundleDigest {
33 + /// sha256 of the manifest text, lowercase hex, 64 chars.
34 + pub full: String,
35 + /// The manifest text itself, ready to write to `<bundle>/MANIFEST`.
36 + pub manifest: String,
37 + }
38 +
39 + impl BundleDigest {
40 + /// The 16-hex-char prefix used in `releases/<digest16>/`. Full digest is
41 + /// kept in the DB; the path only needs enough to be collision-safe.
42 + pub fn short(&self) -> &str {
43 + &self.full[..16]
44 + }
45 + }
46 +
47 + /// Compute the bundle digest for the staged release dir at `root`.
48 + ///
49 + /// Walks every regular file under `root` (recursively), hashes each, and builds
50 + /// the sorted `MANIFEST`. The manifest file itself ([`MANIFEST_NAME`] at the
51 + /// root) is skipped so the digest is stable whether or not it has been written
52 + /// into the bundle yet.
53 + ///
54 + /// The walk + per-file hashing runs on a blocking thread: a release bundle is
55 + /// hundreds of MB of binary, and hashing it must not stall the async runtime.
56 + pub async fn digest_dir(root: &Path) -> Result<BundleDigest> {
57 + let root = root.to_path_buf();
58 + tokio::task::spawn_blocking(move || digest_dir_blocking(&root))
59 + .await
60 + .context("bundle digest task panicked")?
61 + }
62 +
63 + fn digest_dir_blocking(root: &Path) -> Result<BundleDigest> {
64 + let mut files: Vec<PathBuf> = Vec::new();
65 + collect_files(root, &mut files)
66 + .with_context(|| format!("walking bundle dir {}", root.display()))?;
67 +
68 + // Relative paths, sorted, so the manifest is order-independent across hosts
69 + // and filesystems (readdir order is not guaranteed).
70 + let mut rows: Vec<(String, PathBuf)> = Vec::with_capacity(files.len());
71 + for abs in files {
72 + let rel = abs
73 + .strip_prefix(root)
74 + .with_context(|| format!("{} not under bundle root", abs.display()))?;
75 + // Skip the manifest file itself — it is not part of what it describes.
76 + if rel.as_os_str() == MANIFEST_NAME {
77 + continue;
78 + }
79 + let rel_str = rel_to_unix(rel);
80 + rows.push((rel_str, abs));
81 + }
82 + rows.sort_by(|a, b| a.0.cmp(&b.0));
83 +
84 + let mut manifest = String::new();
85 + for (rel, abs) in &rows {
86 + let hash = hash_file(abs).with_context(|| format!("hashing {}", abs.display()))?;
87 + // Two spaces between hash and path, matching sha256sum's format so the
88 + // manifest is checkable with standard tools on a node.
89 + manifest.push_str(&hash);
90 + manifest.push_str(" ");
91 + manifest.push_str(rel);
92 + manifest.push('\n');
93 + }
94 +
95 + let full = hex(&Sha256::digest(manifest.as_bytes()));
96 + Ok(BundleDigest { full, manifest })
97 + }
98 +
99 + /// Recursively collect regular-file paths under `dir`. Symlinks are not
100 + /// followed: a staged bundle is a plain tree of copied files, and following
101 + /// links would let content outside the bundle leak into its identity.
102 + fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) -> std::io::Result<()> {
103 + for entry in std::fs::read_dir(dir)? {
104 + let entry = entry?;
105 + let ft = entry.file_type()?;
106 + if ft.is_dir() {
107 + collect_files(&entry.path(), out)?;
108 + } else if ft.is_file() {
109 + out.push(entry.path());
110 + }
111 + // symlinks / other special files are intentionally ignored.
112 + }
113 + Ok(())
114 + }
115 +
116 + fn hash_file(path: &Path) -> std::io::Result<String> {
117 + use std::io::Read;
118 + let mut file = std::fs::File::open(path)?;
119 + let mut hasher = Sha256::new();
120 + let mut buf = [0u8; 64 * 1024];
121 + loop {
122 + let n = file.read(&mut buf)?;
123 + if n == 0 {
124 + break;
125 + }
126 + hasher.update(&buf[..n]);
127 + }
128 + Ok(hex(&hasher.finalize()))
129 + }
130 +
131 + /// Relative path as a forward-slash string, so a manifest built on one OS reads
132 + /// the same on another. Sando builds on Linux, but keep the identity portable.
133 + fn rel_to_unix(rel: &Path) -> String {
134 + rel.components()
135 + .map(|c| c.as_os_str().to_string_lossy())
136 + .collect::<Vec<_>>()
137 + .join("/")
138 + }
139 +
140 + fn hex(bytes: &[u8]) -> String {
141 + use std::fmt::Write;
142 + let mut s = String::with_capacity(bytes.len() * 2);
143 + for b in bytes {
144 + let _ = write!(s, "{b:02x}");
145 + }
146 + s
147 + }
148 +
149 + #[cfg(test)]
150 + mod tests {
151 + use super::*;
152 +
153 + async fn write(root: &Path, rel: &str, bytes: &[u8]) {
154 + let p = root.join(rel);
155 + tokio::fs::create_dir_all(p.parent().unwrap())
156 + .await
157 + .unwrap();
158 + tokio::fs::write(&p, bytes).await.unwrap();
159 + }
160 +
161 + #[tokio::test]
162 + async fn digest_is_stable_and_lists_every_file_sorted() {
163 + let dir = tempfile::tempdir().unwrap();
164 + let root = dir.path();
165 + write(root, "makenotwork", b"binary bytes").await;
166 + write(root, "static/app.css", b"body{}").await;
167 + write(root, "companions/mnw-cli", b"cli bytes").await;
168 +
169 + let d = super::digest_dir(root).await.unwrap();
170 + assert_eq!(d.full.len(), 64);
171 + assert_eq!(d.short().len(), 16);
172 + // Manifest lines are sorted by relative path.
173 + let paths: Vec<&str> = d
174 + .manifest
175 + .lines()
176 + .map(|l| l.split_once(" ").unwrap().1)
177 + .collect();
178 + assert_eq!(
179 + paths,
180 + ["companions/mnw-cli", "makenotwork", "static/app.css"]
181 + );
182 + }
183 +
184 + #[tokio::test]
185 + async fn digest_is_independent_of_creation_order() {
186 + let a = tempfile::tempdir().unwrap();
187 + write(a.path(), "z.txt", b"1").await;
188 + write(a.path(), "a.txt", b"2").await;
189 + let b = tempfile::tempdir().unwrap();
190 + write(b.path(), "a.txt", b"2").await;
191 + write(b.path(), "z.txt", b"1").await;
192 + assert_eq!(
193 + super::digest_dir(a.path()).await.unwrap().full,
194 + super::digest_dir(b.path()).await.unwrap().full,
195 + );
196 + }
197 +
198 + #[tokio::test]
199 + async fn a_changed_asset_changes_the_digest_even_with_identical_binary() {
200 + let a = tempfile::tempdir().unwrap();
201 + write(a.path(), "makenotwork", b"same binary").await;
202 + write(a.path(), "static/app.css", b"v1").await;
203 + let b = tempfile::tempdir().unwrap();
204 + write(b.path(), "makenotwork", b"same binary").await;
205 + write(b.path(), "static/app.css", b"v2").await;
206 + assert_ne!(
207 + super::digest_dir(a.path()).await.unwrap().full,
208 + super::digest_dir(b.path()).await.unwrap().full,
209 + "asset drift with an identical binary must not collide (2026-07-09 #2)"
210 + );
211 + }
212 +
213 + #[tokio::test]
214 + async fn manifest_file_is_excluded_from_its_own_digest() {
215 + let dir = tempfile::tempdir().unwrap();
216 + write(dir.path(), "makenotwork", b"bytes").await;
217 + let before = super::digest_dir(dir.path()).await.unwrap();
218 + // Write the manifest into the bundle, as Stage B will for node-side
219 + // verification; the digest must not change.
220 + write(dir.path(), MANIFEST_NAME, before.manifest.as_bytes()).await;
221 + let after = super::digest_dir(dir.path()).await.unwrap();
222 + assert_eq!(before.full, after.full);
223 + }
224 + }
M sando/daemon/src/deploy.rs +270 -53
@@ -1,14 +1,18 @@
1 1 //! Atomic symlink-swap deploys.
2 2 //!
3 - //! Layout on every target (local host, A nodes, B nodes, ...):
3 + //! Layout on every target (local host, A nodes, B nodes, ...). Release dirs are
4 + //! named for their content digest, not the version (wiki note
5 + //! `release-artifact-identity`), so a rebuild can never overwrite an earlier
6 + //! build's dir in place:
4 7 //!
5 8 //! <release_root>/
6 9 //! releases/
7 - //! 0.8.1/
10 + //! a1b2c3d4e5f60718/ <- <digest16>
8 11 //! <bin_name>
9 - //! 0.8.2/
12 + //! MANIFEST <- <sha256> <relpath> per file, node-verified
13 + //! f0e1d2c3b4a59687/
10 14 //! <bin_name>
11 - //! current -> releases/0.8.2
15 + //! current -> releases/f0e1d2c3b4a59687
12 16 //!
13 17 //! `ln -sfn` swaps the symlink. systemd units point at
14 18 //! `<release_root>/current/<bin_name>` so reload-or-restart picks up the new
@@ -67,23 +71,67 @@ async fn run_checked(executor: &dyn Executor, script: &str, what: &str) -> Resul
67 71 Ok(out)
68 72 }
69 73
70 - pub async fn deploy_local(
74 + /// Stage built binaries into `staging/<build_id>/` on the Sando host — a
75 + /// private, mutable scratch dir that is not yet a release (no symlink, no gc).
76 + /// The caller adds `release_contents` + companions, hashes the result, writes
77 + /// the `MANIFEST`, then publishes it content-addressed via
78 + /// [`finalize_local_release`]. Splitting staging from publish is what lets the
79 + /// bundle be hashed before it is named (wiki [[release-artifact-identity]]).
80 + ///
81 + /// A stale `staging/<build_id>` from a killed prior run at the same id is
82 + /// removed first, so a retry stages clean.
83 + pub async fn stage_local_bundle(
71 84 release_root: &Path,
72 - version: &crate::domain::Version,
85 + build_id: i64,
73 86 binaries: &[PathBuf],
74 87 ) -> Result<PathBuf> {
75 - let release_dir = release_root.join("releases").join(version.to_string());
76 - tokio::fs::create_dir_all(&release_dir).await?;
88 + let staging = release_root.join("staging").join(build_id.to_string());
89 + if tokio::fs::try_exists(&staging).await.unwrap_or(false) {
90 + tokio::fs::remove_dir_all(&staging)
91 + .await
92 + .with_context(|| format!("clearing stale staging dir {}", staging.display()))?;
93 + }
94 + tokio::fs::create_dir_all(&staging).await?;
77 95 for binary in binaries {
78 96 let name = binary.file_name().context("binary path has no file name")?;
79 - let dest = release_dir.join(name);
97 + let dest = staging.join(name);
80 98 tokio::fs::copy(binary, &dest)
81 99 .await
82 100 .with_context(|| format!("copy {} -> {}", binary.display(), dest.display()))?;
83 101 }
102 + Ok(staging)
103 + }
104 +
105 + /// Publish a fully-staged bundle content-addressed: rename
106 + /// `staging/<build_id>` to `releases/<digest16>` (atomic same-filesystem
107 + /// rename), flip `current` to it, gc old releases. Returns the released dir.
108 + ///
109 + /// The rename is the load-bearing step: **a directory whose name derives from
110 + /// its contents cannot be rewritten, because rewriting it changes its name.**
111 + /// The overwrite class that let a dev rebuild inherit an earlier build's gate
112 + /// rows and burn-in clock stops being something to guard against and becomes
113 + /// something that cannot be expressed. If a release with this digest already
114 + /// exists (identical bytes rebuilt), the staging copy is redundant and dropped.
115 + pub async fn finalize_local_release(
116 + release_root: &Path,
117 + staging: &Path,
118 + digest16: &str,
119 + ) -> Result<PathBuf> {
120 + let releases = release_root.join("releases");
121 + tokio::fs::create_dir_all(&releases).await?;
122 + let released = releases.join(digest16);
123 +
124 + if tokio::fs::try_exists(&released).await.unwrap_or(false) {
125 + // Same digest already published — reuse it, discard the redundant stage.
126 + tokio::fs::remove_dir_all(staging).await.ok();
127 + } else {
128 + tokio::fs::rename(staging, &released)
129 + .await
130 + .with_context(|| format!("publish {} -> {}", staging.display(), released.display()))?;
131 + }
84 132
85 133 let current = release_root.join("current");
86 - let target = format!("releases/{version}");
134 + let target = format!("releases/{digest16}");
87 135 let out = Command::new("ln")
88 136 .args(["-sfn", &target])
89 137 .arg(&current)
@@ -98,7 +146,7 @@ pub async fn deploy_local(
98 146 if let Err(e) = gc_local_releases(release_root).await {
99 147 tracing::warn!(error = %e, "local release GC failed (non-fatal)");
100 148 }
101 - Ok(release_dir)
149 + Ok(released)
102 150 }
103 151
104 152 /// Deploy `staged_release_dir` (a directory built on the Sando host by
@@ -115,21 +163,43 @@ pub async fn deploy_node(
115 163 staged_release_dir: &Path,
116 164 primary_bin: &str,
117 165 ) -> Result<PathBuf> {
166 + // The release dir is named for its content digest (`releases/<digest16>`),
167 + // not the version. The node mirrors that name so host and node agree on the
168 + // artifact's identity; the version is only a log label here. Legacy staged
169 + // dirs (pre-identity, still `releases/<version>`) work unchanged — the name
170 + // is whatever the host staged under.
171 + let release_id = staged_release_dir
172 + .file_name()
173 + .and_then(|n| n.to_str())
174 + .with_context(|| {
175 + format!(
176 + "staged release dir {} has no usable name",
177 + staged_release_dir.display()
178 + )
179 + })?;
118 180 if node.ssh_target == "local" || node.ssh_target.is_empty() {
119 181 // Local deploy already happened when we staged on the Sando host.
120 182 // Just re-point `current` at the staged dir.
121 - return reset_local_current(executor, Path::new(&node.release_root), version).await;
183 + return reset_local_current(executor, Path::new(&node.release_root), release_id).await;
122 184 }
123 - deploy_remote(executor, node, version, staged_release_dir, primary_bin).await
185 + deploy_remote(
186 + executor,
187 + node,
188 + version,
189 + release_id,
190 + staged_release_dir,
191 + primary_bin,
192 + )
193 + .await
124 194 }
125 195
126 196 async fn reset_local_current(
127 197 executor: &dyn Executor,
128 198 release_root: &Path,
129 - version: &str,
199 + release_id: &str,
130 200 ) -> Result<PathBuf> {
131 201 let current = release_root.join("current");
132 - let target = format!("releases/{version}");
202 + let target = format!("releases/{release_id}");
133 203 run_checked(
134 204 executor,
135 205 &format!(
@@ -140,21 +210,22 @@ async fn reset_local_current(
140 210 "local symlink swap",
141 211 )
142 212 .await?;
143 - Ok(release_root.join("releases").join(version))
213 + Ok(release_root.join("releases").join(release_id))
144 214 }
145 215
146 216 async fn deploy_remote(
147 217 executor: &dyn Executor,
148 218 node: &Node,
149 219 version: &str,
220 + release_id: &str,
150 221 staged_release_dir: &Path,
151 222 primary_bin: &str,
152 223 ) -> Result<PathBuf> {
153 224 let release_root = &node.release_root;
154 225 let service = &node.service_name;
155 - let release_dir = format!("{release_root}/releases/{version}");
226 + let release_dir = format!("{release_root}/releases/{release_id}");
156 227
157 - tracing::info!(node = %node.name, version, "deploy: mkdir release dir");
228 + tracing::info!(node = %node.name, version, release_id, "deploy: mkdir release dir");
158 229 run_checked(
159 230 executor,
160 231 &format!("set -e; mkdir -p {q}", q = sh_quote(&release_dir)),
@@ -179,6 +250,22 @@ async fn deploy_remote(
179 250 .await
180 251 .context("rsync failed (current symlink left intact)")?;
181 252
253 + // Verify the bundle on the node against its own MANIFEST before the swap
254 + // (invariant 3, wiki [[release-artifact-identity]]). The MANIFEST shipped in
255 + // the bundle is exactly `sha256sum` check format (`<hash> <relpath>`), so
256 + // this re-hashes every file on the node and names any that drifted in
257 + // transit — the hash is load-bearing, not merely recorded. Bundles staged
258 + // by a pre-identity build carry no MANIFEST; those skip verification (logged)
259 + // rather than fail, so a mid-migration deploy of a legacy artifact still
260 + // ships. A mismatch fails the promote with the running service intact.
261 + run_checked(
262 + executor,
263 + &manifest_verify_script(&release_dir),
264 + "verifying bundle digest on node",
265 + )
266 + .await
267 + .context("node-side bundle verification failed (current symlink left intact)")?;
268 +
182 269 // Fail closed on a wrong-architecture binary before the symlink swap. The
183 270 // "never cross-compile" rule is enforced at build time (build_host check),
184 271 // but nothing verified the artifact's arch matched the *target* node — so
@@ -214,7 +301,7 @@ async fn deploy_remote(
214 301 "sudo /bin/systemctl reload-or-restart {}",
215 302 sh_quote(service)
216 303 );
217 - let swap_and_restart = swap_and_restart_script(release_root, version, &restart_cmd);
304 + let swap_and_restart = swap_and_restart_script(release_root, release_id, &restart_cmd);
218 305 run_checked(
219 306 executor,
220 307 &swap_and_restart,
@@ -229,7 +316,7 @@ async fn deploy_remote(
229 316 // promote: a companion is part of the deploy, not a best-effort side effect.
230 317 for c in &node.companions {
231 318 let src = format!(
232 - "{release_root}/releases/{version}/companions/{name}",
319 + "{release_root}/releases/{release_id}/companions/{name}",
233 320 name = c.name,
234 321 );
235 322 tracing::info!(node = %node.name, companion = %c.name, "deploy: install companion + restart");
@@ -248,7 +335,9 @@ async fn deploy_remote(
248 335 tracing::warn!(error = %e, "remote release GC failed (non-fatal)");
249 336 }
250 337
251 - Ok(PathBuf::from(release_root).join("releases").join(version))
338 + Ok(PathBuf::from(release_root)
339 + .join("releases")
340 + .join(release_id))
252 341 }
253 342
254 343 /// Absolute path of the node-side companion installer (shipped once per node;
@@ -339,12 +428,12 @@ fn config_check_script(env_file: &str, bin: &str) -> String {
339 428 ///
340 429 /// `restart_cmd` is injected (rather than hardcoded) so tests can drive the
341 430 /// failure and success paths with a `false`/`true` stand-in.
342 - fn swap_and_restart_script(release_root: &str, version: &str, restart_cmd: &str) -> String {
431 + fn swap_and_restart_script(release_root: &str, release_id: &str, restart_cmd: &str) -> String {
343 432 format!(
344 433 "set -e\n\
345 434 cd {root}\n\
346 435 prev=$(readlink current 2>/dev/null || true)\n\
347 - ln -sfn releases/{ver} current.new\n\
436 + ln -sfn releases/{rel} current.new\n\
348 437 mv -Tf current.new current\n\
349 438 if ! {restart}; then\n\
350 439 if [ -n \"$prev\" ]; then\n\
@@ -356,11 +445,34 @@ fn swap_and_restart_script(release_root: &str, version: &str, restart_cmd: &str)
356 445 exit 1\n\
357 446 fi\n",
358 447 root = sh_quote(release_root),
359 - ver = sh_quote(version),
448 + rel = sh_quote(release_id),
360 449 restart = restart_cmd,
361 450 )
362 451 }
363 452
453 + /// Shell that re-hashes the rsynced bundle on the node against its shipped
454 + /// `MANIFEST` and aborts (exit 1) if any file drifted (invariant 3, wiki note
455 + /// `release-artifact-identity`). The `MANIFEST` is `sha256sum` check format,
456 + /// so `sha256sum -c` verifies every listed file with node-native tooling and
457 + /// names the one that failed. `--strict` fails on a malformed manifest line;
458 + /// `--quiet` drops the per-file OK spam and keeps only failures.
459 + ///
460 + /// A bundle staged by a pre-identity build carries no `MANIFEST`; that is not an
461 + /// error — it logs a skip and exits 0, so a mid-migration deploy of a legacy
462 + /// artifact still ships. Once every tier has cycled once, every bundle has one.
463 + fn manifest_verify_script(release_dir: &str) -> String {
464 + format!(
465 + "set -e\n\
466 + cd {dir}\n\
467 + if [ ! -f MANIFEST ]; then\n\
468 + echo \"deploy: no MANIFEST in bundle; skipping digest verification (legacy artifact)\" >&2\n\
469 + exit 0\n\
470 + fi\n\
471 + sha256sum --quiet --strict -c MANIFEST\n",
472 + dir = sh_quote(release_dir),
473 + )
474 + }
475 +
364 476 /// Shell that aborts (exit 1) if `bin`'s ELF architecture doesn't match the
365 477 /// node it's running on. Reads the ELF `e_machine` field (2 bytes LE at offset
366 478 /// 18) and compares it to the value implied by `uname -m`. An arch we don't have
@@ -456,33 +568,46 @@ mod tests {
456 568 let release_root = root.join("releases-root");
457 569 tokio::fs::create_dir_all(&release_root).await.unwrap();
458 570
459 - let staged = deploy_local(
460 - &release_root,
461 - &"0.8.12".parse().unwrap(),
462 - &[primary.clone(), admin.clone()],
463 - )
464 - .await
465 - .expect("deploy_local should succeed");
571 + // Stage into staging/<build_id> (no publish yet).
572 + let staging = stage_local_bundle(&release_root, 42, &[primary.clone(), admin.clone()])
573 + .await
574 + .expect("stage_local_bundle should succeed");
575 + assert_eq!(staging, release_root.join("staging").join("42"));
576 + assert!(
577 + !release_root.join("current").exists(),
578 + "staging must not publish or flip current"
579 + );
466 580
467 - assert_eq!(staged, release_root.join("releases").join("0.8.12"));
581 + // Publish content-addressed at releases/<digest16>.
582 + let released = finalize_local_release(&release_root, &staging, "deadbeefcafe0000")
583 + .await
584 + .expect("finalize_local_release should succeed");
585 + assert_eq!(
586 + released,
587 + release_root.join("releases").join("deadbeefcafe0000")
588 + );
589 + assert!(
590 + !staging.exists(),
591 + "staging dir is consumed by the publish rename"
592 + );
468 593 assert_eq!(
469 - tokio::fs::read(staged.join("makenotwork")).await.unwrap(),
594 + tokio::fs::read(released.join("makenotwork")).await.unwrap(),
470 595 b"PRIMARY"
471 596 );
472 597 assert_eq!(
473 - tokio::fs::read(staged.join("mnw-admin")).await.unwrap(),
598 + tokio::fs::read(released.join("mnw-admin")).await.unwrap(),
474 599 b"ADMIN"
475 600 );
476 601
477 602 let current = release_root.join("current");
478 603 let target = tokio::fs::read_link(&current).await.unwrap();
479 - assert_eq!(target.to_string_lossy(), "releases/0.8.12");
604 + assert_eq!(target.to_string_lossy(), "releases/deadbeefcafe0000");
480 605 let via_current = tokio::fs::read(current.join("makenotwork")).await.unwrap();
481 606 assert_eq!(via_current, b"PRIMARY");
482 607 }
483 608
484 609 #[tokio::test]
485 - async fn deploy_local_second_release_swaps_symlink_and_keeps_old_dir() {
610 + async fn finalize_second_release_swaps_symlink_and_keeps_old_dir() {
486 611 let tmp = tempfile::tempdir().unwrap();
487 612 let root = tmp.path();
488 613 let src_dir = root.join("src");
@@ -493,28 +618,35 @@ mod tests {
493 618 let release_root = root.join("rr");
494 619 tokio::fs::create_dir_all(&release_root).await.unwrap();
495 620
496 - deploy_local(
497 - &release_root,
498 - &"0.1.0".parse().unwrap(),
499 - std::slice::from_ref(&bin),
500 - )
501 - .await
502 - .unwrap();
621 + // Two builds, distinct digests (distinct content) -> two release dirs.
622 + let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
623 + .await
624 + .unwrap();
625 + finalize_local_release(&release_root, &s1, "1111111111111111")
626 + .await
627 + .unwrap();
503 628 tokio::fs::write(&bin, b"V2").await.unwrap();
504 - deploy_local(
505 - &release_root,
506 - &"0.2.0".parse().unwrap(),
507 - std::slice::from_ref(&bin),
508 - )
509 - .await
510 - .unwrap();
629 + let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
630 + .await
631 + .unwrap();
632 + finalize_local_release(&release_root, &s2, "2222222222222222")
633 + .await
634 + .unwrap();
511 635
512 - assert!(release_root.join("releases/0.1.0/server").exists());
513 - assert!(release_root.join("releases/0.2.0/server").exists());
636 + assert!(
637 + release_root
638 + .join("releases/1111111111111111/server")
639 + .exists()
640 + );
641 + assert!(
642 + release_root
643 + .join("releases/2222222222222222/server")
644 + .exists()
645 + );
514 646 let target = tokio::fs::read_link(release_root.join("current"))
515 647 .await
516 648 .unwrap();
517 - assert_eq!(target.to_string_lossy(), "releases/0.2.0");
649 + assert_eq!(target.to_string_lossy(), "releases/2222222222222222");
518 650 let via_current = tokio::fs::read(release_root.join("current/server"))
519 651 .await
520 652 .unwrap();
@@ -522,6 +654,91 @@ mod tests {
522 654 }
523 655
524 656 #[tokio::test]
657 + async fn finalize_reuses_an_existing_release_of_the_same_digest() {
658 + let tmp = tempfile::tempdir().unwrap();
659 + let root = tmp.path();
660 + let bin = root.join("server");
661 + tokio::fs::write(&bin, b"BYTES").await.unwrap();
662 + let release_root = root.join("rr");
663 + tokio::fs::create_dir_all(&release_root).await.unwrap();
664 +
665 + let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
666 + .await
667 + .unwrap();
668 + finalize_local_release(&release_root, &s1, "abc123abc123abc1")
669 + .await
670 + .unwrap();
671 + // Same digest rebuilt (e.g. a re-run at the same content): finalize must
672 + // reuse the existing release and drop the redundant staging dir, not error.
673 + let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
674 + .await
675 + .unwrap();
676 + let released = finalize_local_release(&release_root, &s2, "abc123abc123abc1")
677 + .await
678 + .expect("finalize is idempotent on a repeated digest");
679 + assert_eq!(released, release_root.join("releases/abc123abc123abc1"));
680 + assert!(!s2.exists(), "redundant staging dropped");
681 + }
682 +
683 + #[tokio::test]
684 + async fn manifest_verify_script_passes_on_match_fails_on_drift_and_skips_when_absent() {
685 + // The node-side verification is a shell running `sha256sum -c MANIFEST`;
686 + // drive the real script through bash to prove it accepts a good bundle,
687 + // rejects a tampered one, and no-ops on a legacy (MANIFEST-less) bundle.
688 + let dir = tempfile::tempdir().unwrap();
689 + tokio::fs::write(dir.path().join("server"), b"BINARY")
690 + .await
691 + .unwrap();
692 + tokio::fs::create_dir(dir.path().join("static"))
693 + .await
694 + .unwrap();
695 + tokio::fs::write(dir.path().join("static/app.css"), b"body{}")
696 + .await
697 + .unwrap();
698 + let digest = crate::bundle::digest_dir(dir.path()).await.unwrap();
699 + tokio::fs::write(dir.path().join("MANIFEST"), digest.manifest.as_bytes())
700 + .await
701 + .unwrap();
702 +
703 + let run = |d: &std::path::Path| {
704 + let script = manifest_verify_script(d.to_str().unwrap());
705 + async move {
706 + Command::new("bash")
707 + .arg("-c")
708 + .arg(&script)
709 + .output()
710 + .await
711 + .unwrap()
712 + }
713 + };
714 +
715 + let ok = run(dir.path()).await;
716 + assert!(
717 + ok.status.success(),
718 + "matching bundle verifies: {}",
719 + String::from_utf8_lossy(&ok.stderr)
720 + );
721 +
722 + // Drift one file: sha256sum -c must fail (current symlink left intact).
723 + tokio::fs::write(dir.path().join("static/app.css"), b"TAMPERED")
724 + .await
725 + .unwrap();
726 + let bad = run(dir.path()).await;
727 + assert!(!bad.status.success(), "a drifted file fails verification");
728 +
729 + // Legacy bundle with no MANIFEST: skip, not fail.
730 + let legacy = tempfile::tempdir().unwrap();
731 + tokio::fs::write(legacy.path().join("server"), b"x")
732 + .await
733 + .unwrap();
734 + let skip = run(legacy.path()).await;
735 + assert!(
736 + skip.status.success(),
737 + "a bundle without a MANIFEST skips verification rather than failing"
738 + );
739 + }
740 +
741 + #[tokio::test]
525 742 async fn gc_local_releases_keeps_last_n_by_mtime() {
526 743 let tmp = tempfile::tempdir().unwrap();
527 744 let root = tmp.path();
M sando/daemon/src/gates.rs +16 -1
@@ -48,6 +48,12 @@ pub struct GateCtx {
48 48 /// gate runs on the host (where `node_health` never appears); filled at
49 49 /// promote time with each freshly-deployed node and its executor.
50 50 pub nodes: Vec<NodeProbe>,
51 + /// The `build_runs.id` this gate run vouches for — the artifact identity
52 + /// (wiki [[release-artifact-identity]]). Recorded on every `gate_runs` row so
53 + /// promote can resolve the artifact through the evidence for a specific build,
54 + /// not through a version string that a later rebuild can silently reuse.
55 + /// `None` for legacy/pre-identity runs and gate unit tests.
56 + pub build_id: Option<i64>,
51 57 }
52 58
53 59 /// One node the `node_health` gate verifies: its id, the systemd unit to
@@ -67,13 +73,14 @@ pub async fn run(ctx: &GateCtx, gate: &Gate) -> Result<GateOutcome> {
67 73 let started_at = Utc::now().to_rfc3339();
68 74
69 75 let id: i64 = sqlx::query_scalar(
70 - "INSERT INTO gate_runs (version, tier, gate_kind, started_at) VALUES (?, ?, ?, ?)
76 + "INSERT INTO gate_runs (version, tier, gate_kind, started_at, build_id) VALUES (?, ?, ?, ?, ?)
71 77 RETURNING id",
72 78 )
73 79 .bind(&ctx.version)
74 80 .bind(&ctx.tier)
75 81 .bind(kind)
76 82 .bind(&started_at)
83 + .bind(ctx.build_id)
77 84 .fetch_one(&ctx.pool)
78 85 .await?;
79 86 let run_id = GateRunId(id);
@@ -2132,6 +2139,7 @@ mod tests {
2132 2139 worktree: tmp.path().to_path_buf(),
2133 2140 events: events::channel(),
2134 2141 nodes: Vec::new(),
2142 + build_id: None,
2135 2143 };
2136 2144 let out = cargo_test(&ctx, GateRunId(1)).await.unwrap();
2137 2145 assert_eq!(
@@ -2166,6 +2174,7 @@ mod tests {
2166 2174 worktree: std::path::PathBuf::from("/tmp/wt"),
2167 2175 events: events::channel(),
2168 2176 nodes: Vec::new(),
2177 + build_id: None,
2169 2178 };
2170 2179 let dir = std::path::Path::new("/tmp/wt/x");
2171 2180
@@ -2198,6 +2207,7 @@ mod tests {
2198 2207 worktree: std::path::PathBuf::from("/tmp/wt"),
2199 2208 events: events::channel(),
2200 2209 nodes: Vec::new(),
2210 + build_id: None,
2201 2211 };
2202 2212 let mut t = target("shared/ops-exec");
2203 2213 t.all_features = true;
@@ -2285,6 +2295,7 @@ mod tests {
2285 2295 worktree: tmp.path().to_path_buf(),
2286 2296 events: events::channel(),
2287 2297 nodes: Vec::new(),
2298 + build_id: None,
2288 2299 };
2289 2300 let out = supply_chain(&ctx, GateRunId(1), GateKind::CargoAudit)
2290 2301 .await
@@ -2359,6 +2370,7 @@ mod tests {
2359 2370 worktree: std::path::PathBuf::from("/tmp/wt"),
2360 2371 events: events::channel(),
2361 2372 nodes: Vec::new(),
2373 + build_id: None,
2362 2374 };
2363 2375 let plain = crate::config::TestTarget {
2364 2376 dir: std::path::PathBuf::from("server"),
@@ -2490,6 +2502,7 @@ mod tests {
2490 2502 worktree: std::path::PathBuf::from("/tmp/unused"),
2491 2503 events: events::channel(),
2492 2504 nodes: Vec::new(),
2505 + build_id: None,
2493 2506 };
2494 2507 let out = run(&ctx, &Gate::BurnIn { hours: 24 }).await.unwrap();
2495 2508 assert_eq!(out.status_str(), "blocked");
@@ -2543,6 +2556,7 @@ mod tests {
2543 2556 worktree: std::path::PathBuf::new(),
2544 2557 events: events::channel(),
2545 2558 nodes: Vec::new(), // no nodes -> fail closed
2559 + build_id: None,
2546 2560 };
2547 2561 let out = run(&ctx, &Gate::NodeHealth).await.unwrap();
2548 2562 assert_eq!(out.status_str(), "blocked");
@@ -2863,6 +2877,7 @@ mod tests {
2863 2877 worktree: std::path::PathBuf::from("/tmp/unused"),
2864 2878 events: events::channel(),
2865 2879 nodes: Vec::new(),
2880 + build_id: None,
2866 2881 };
2867 2882 let out = run(&ctx, &Gate::CodeSmoke).await.unwrap();
2868 2883 assert_eq!(out.status_str(), "blocked");
M sando/daemon/src/lib.rs +1
@@ -15,6 +15,7 @@
15 15
16 16 pub mod backup;
17 17 pub mod build;
18 + pub mod bundle;
18 19 pub mod classify;
19 20 pub mod config;
20 21 pub mod db;
M sando/daemon/src/routes/mod.rs +280 -34
@@ -990,7 +990,7 @@ mod tests {
990 990 // A tier that configures no gates has nothing to satisfy.
991 991 let pool = fresh_pool().await;
992 992 seed(&pool, "host", "0.8.12").await;
993 - let pending = unsatisfied_gates(&pool, &tid("host"), &[], "0.8.12", false)
993 + let pending = unsatisfied_gates(&pool, &tid("host"), &[], "0.8.12", None, false)
994 994 .await
995 995 .unwrap();
996 996 assert_eq!(pending, Vec::<String>::new());
@@ -1004,9 +1004,10 @@ mod tests {
1004 1004 // straight through to prod.
1005 1005 let pool = fresh_pool().await;
1006 1006 seed(&pool, "a", "0.8.12").await;
1007 - let pending = unsatisfied_gates(&pool, &tid("a"), &[Gate::BootSmoke], "0.8.12", false)
1008 - .await
1009 - .unwrap();
1007 + let pending =
1008 + unsatisfied_gates(&pool, &tid("a"), &[Gate::BootSmoke], "0.8.12", None, false)
1009 + .await
1010 + .unwrap();
1010 1011 assert_eq!(pending, vec!["boot_smoke".to_string()]);
1011 1012 }
1012 1013
@@ -1021,6 +1022,7 @@ mod tests {
1021 1022 &tid("host"),
1022 1023 &[Gate::CargoTest, Gate::BootSmoke],
1023 1024 "0.8.12",
1025 + None,
1024 1026 false,
1025 1027 )
1026 1028 .await
@@ -1036,9 +1038,16 @@ mod tests {
1036 1038 seed(&pool, "host", "0.8.12").await;
1037 1039 insert_gate(&pool, "host", "0.8.12", "cargo_test", 0).await;
1038 1040 insert_gate(&pool, "host", "0.8.12", "cargo_test", 1).await;
1039 - let pending = unsatisfied_gates(&pool, &tid("host"), &[Gate::CargoTest], "0.8.12", false)
1040 - .await
1041 - .unwrap();
1041 + let pending = unsatisfied_gates(
1042 + &pool,
1043 + &tid("host"),
1044 + &[Gate::CargoTest],
1045 + "0.8.12",
1046 + None,
1047 + false,
1048 + )
1049 + .await
1050 + .unwrap();
1042 1051 assert!(pending.is_empty());
1043 1052 }
1044 1053
@@ -1077,9 +1086,16 @@ mod tests {
1077 1086
1078 1087 // Stale confirmation (recorded before this landing) -> unsatisfied.
1079 1088 insert_confirm(&pool, "a", "0.8.12", landed - chrono::Duration::hours(1)).await;
1080 - let pending = unsatisfied_gates(&pool, &tid("a"), &[Gate::ManualConfirm], "0.8.12", false)
1081 - .await
1082 - .unwrap();
1089 + let pending = unsatisfied_gates(
1090 + &pool,
1091 + &tid("a"),
1092 + &[Gate::ManualConfirm],
1093 + "0.8.12",
1094 + None,
1095 + false,
1096 + )
1097 + .await
1098 + .unwrap();
1083 1099 assert_eq!(
1084 1100 pending,
1085 1101 vec!["manual_confirm".to_string()],
@@ -1088,9 +1104,16 @@ mod tests {
1088 1104
1089 1105 // Fresh confirmation (after this landing) -> satisfied.
1090 1106 insert_confirm(&pool, "a", "0.8.12", landed + chrono::Duration::minutes(5)).await;
1091 - let pending = unsatisfied_gates(&pool, &tid("a"), &[Gate::ManualConfirm], "0.8.12", false)
1092 - .await
1093 - .unwrap();
1107 + let pending = unsatisfied_gates(
1108 + &pool,
1109 + &tid("a"),
1110 + &[Gate::ManualConfirm],
1111 + "0.8.12",
1112 + None,
1113 + false,
1114 + )
1115 + .await
1116 + .unwrap();
1094 1117 assert!(pending.is_empty(), "fresh confirm satisfies");
1095 1118 }
1096 1119
@@ -1101,9 +1124,16 @@ mod tests {
1101 1124 let pool = fresh_pool().await;
1102 1125 seed(&pool, "a", "0.8.12").await; // leaves burn_in_started_at NULL
1103 1126 insert_confirm(&pool, "a", "0.8.12", chrono::Utc::now()).await;
1104 - let pending = unsatisfied_gates(&pool, &tid("a"), &[Gate::ManualConfirm], "0.8.12", false)
1105 - .await
1106 - .unwrap();
1127 + let pending = unsatisfied_gates(
1128 + &pool,
1129 + &tid("a"),
1130 + &[Gate::ManualConfirm],
1131 + "0.8.12",
1132 + None,
1133 + false,
1134 + )
1135 + .await
1136 + .unwrap();
1107 1137 assert_eq!(
1108 1138 pending,
1109 1139 vec!["manual_confirm".to_string()],
@@ -1122,7 +1152,7 @@ mod tests {
1122 1152 insert_gate(&pool, "a", "0.8.12", "cargo_test", 0).await;
1123 1153 let gates = [Gate::BurnIn { hours: 48 }, Gate::CargoTest];
1124 1154
1125 - let normal = unsatisfied_gates(&pool, &tid("a"), &gates, "0.8.12", false)
1155 + let normal = unsatisfied_gates(&pool, &tid("a"), &gates, "0.8.12", None, false)
1126 1156 .await
1127 1157 .unwrap();
1128 1158 assert_eq!(
@@ -1130,7 +1160,7 @@ mod tests {
1130 1160 vec!["burn_in".to_string(), "cargo_test".to_string()]
1131 1161 );
1132 1162
1133 - let with_hotfix = unsatisfied_gates(&pool, &tid("a"), &gates, "0.8.12", true)
1163 + let with_hotfix = unsatisfied_gates(&pool, &tid("a"), &gates, "0.8.12", None, true)
1134 1164 .await
1135 1165 .unwrap();
1136 1166 assert_eq!(with_hotfix, vec!["cargo_test".to_string()]);
@@ -1152,6 +1182,7 @@ mod tests {
1152 1182 &tid("a"),
1153 1183 &[Gate::BurnIn { hours: 48 }],
1154 1184 "0.8.12",
1185 + None,
1155 1186 false,
1156 1187 )
1157 1188 .await
@@ -1171,9 +1202,16 @@ mod tests {
1171 1202 insert_gate(&pool, "a", "0.8.12", "cargo_test", 0).await;
1172 1203 insert_gate(&pool, "host", "0.8.11", "cargo_test", 0).await;
1173 1204
1174 - let pending = unsatisfied_gates(&pool, &tid("host"), &[Gate::CargoTest], "0.8.12", false)
1175 - .await
1176 - .unwrap();
1205 + let pending = unsatisfied_gates(
1206 + &pool,
1207 + &tid("host"),
1208 + &[Gate::CargoTest],
1209 + "0.8.12",
1210 + None,
1211 + false,
1212 + )
1213 + .await
1214 + .unwrap();
1177 1215 assert_eq!(pending, vec!["cargo_test".to_string()]);
1178 1216 }
1179 1217
@@ -1192,9 +1230,16 @@ mod tests {
1192 1230 .await
1193 1231 .unwrap();
1194 1232
1195 - let pending = unsatisfied_gates(&pool, &tid("host"), &[Gate::CargoTest], "0.8.12", false)
1196 - .await
1197 - .unwrap();
1233 + let pending = unsatisfied_gates(
1234 + &pool,
1235 + &tid("host"),
1236 + &[Gate::CargoTest],
1237 + "0.8.12",
1238 + None,
1239 + false,
1240 + )
1241 + .await
1242 + .unwrap();
1198 1243 assert_eq!(pending, vec!["cargo_test".to_string()]);
1199 1244 }
1200 1245
@@ -1551,7 +1596,9 @@ mod tests {
1551 1596 // Exercise the sealed forward-advance primitive itself — the same op
1552 1597 // /promote and the host build path both call (S1), not a copy of its SQL.
1553 1598 let v = crate::domain::Version::parse("2.0.0").unwrap();
1554 - crate::runs::advance_tier(&pool, "a", &v).await.unwrap();
1599 + crate::runs::advance_tier(&pool, "a", &v, None)
1600 + .await
1601 + .unwrap();
1555 1602
1556 1603 let (cur, prev): (Option<String>, Option<String>) = sqlx::query_as(
1557 1604 "SELECT current_version, previous_version FROM tier_state WHERE tier = 'a'",
@@ -1599,9 +1646,9 @@ mod tests {
1599 1646 }
1600 1647
1601 1648 let mut state = test_state().await;
1602 - // The rollback target needs a versions row (artifact parent = the staged
1603 - // dir, unused for local nodes but resolved by the helper).
1604 - sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('1.0.0','sha',datetime('now'),'/tmp/staged/makenotwork')")
1649 + // The rollback target needs a versions row. The release dir name comes
1650 + // from the artifact_path's parent (legacy layout: releases/<version>).
1651 + sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('1.0.0','sha',datetime('now'),'/tmp/staged/releases/1.0.0/makenotwork')")
1605 1652 .execute(&state.pool).await.unwrap();
1606 1653 let execs: crate::state::ExecutorMap = nodes
1607 1654 .iter()
@@ -1809,8 +1856,11 @@ mod tests {
1809 1856 .unwrap();
1810 1857 }
1811 1858 for v in ["1.0.0", "2.0.0", "3.0.0"] {
1812 - sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES (?, 'sha', datetime('now'), '/tmp/staged/makenotwork')")
1813 - .bind(v).execute(&state.pool).await.unwrap();
1859 + // Legacy (pre-identity) artifact_path is `releases/<version>/<bin>`,
1860 + // so the release dir the node mirrors is named for the version. The
1861 + // fixture reflects that real layout (parent basename == version).
1862 + sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES (?, 'sha', datetime('now'), ?)")
1863 + .bind(v).bind(format!("/tmp/staged/releases/{v}/makenotwork")).execute(&state.pool).await.unwrap();
1814 1864 }
1815 1865 sqlx::query("UPDATE tier_state SET current_version = '3.0.0' WHERE tier = 'host'")
1816 1866 .execute(&state.pool)
@@ -1979,8 +2029,8 @@ mod tests {
1979 2029 .bind(rr.to_string_lossy().into_owned())
1980 2030 .execute(&state.pool).await.unwrap();
1981 2031 for v in [prev, current] {
1982 - sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES (?, 'sha', datetime('now'), '/tmp/staged/makenotwork')")
1983 - .bind(v).execute(&state.pool).await.unwrap();
2032 + sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES (?, 'sha', datetime('now'), ?)")
2033 + .bind(v).bind(format!("/tmp/staged/releases/{v}/makenotwork")).execute(&state.pool).await.unwrap();
1984 2034 }
1985 2035 sqlx::query(
1986 2036 "UPDATE tier_state SET current_version = ?, previous_version = ? WHERE tier = 'a'",
@@ -2113,7 +2163,7 @@ mod tests {
2113 2163 )
2114 2164 .await
2115 2165 .unwrap();
2116 - sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('3.0.0','sha',datetime('now'),'/tmp/staged/makenotwork')")
2166 + sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('3.0.0','sha',datetime('now'),'/tmp/staged/releases/3.0.0/makenotwork')")
2117 2167 .execute(&pool).await.unwrap();
2118 2168 sqlx::query("UPDATE tier_state SET current_version = '3.0.0' WHERE tier = 'host'")
2119 2169 .execute(&pool)
@@ -2188,7 +2238,7 @@ mod tests {
2188 2238 state.executors = Arc::new(crate::state::ExecutorMap::new());
2189 2239 let pool = state.pool.clone();
2190 2240 // Promote 3.0.0 up from host, which configures no gates of its own.
2191 - sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('3.0.0','sha',datetime('now'),'/tmp/staged/makenotwork')")
2241 + sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('3.0.0','sha',datetime('now'),'/tmp/staged/releases/3.0.0/makenotwork')")
2192 2242 .execute(&pool).await.unwrap();
2193 2243 sqlx::query("UPDATE tier_state SET current_version = '3.0.0' WHERE tier = 'host'")
2194 2244 .execute(&pool)
@@ -2562,4 +2612,200 @@ mod tests {
2562 2612 .unwrap();
2563 2613 assert_eq!(resp.status(), StatusCode::NOT_FOUND);
2564 2614 }
2615 +
2616 + // ---- build-identity path (wiki release-artifact-identity) ----
2617 +
2618 + /// Insert a settled (passed) build_runs row with content identity and return
2619 + /// its id. `staged_path` is the content-addressed release dir the bundle was
2620 + /// published to (`releases/<digest16>`).
2621 + async fn seed_build(pool: &SqlitePool, sha: &str, version: &str, staged_path: &str) -> i64 {
2622 + sqlx::query_scalar(
2623 + "INSERT INTO build_runs (sha, version, phase, result, started_at, bundle_digest, staged_path)
2624 + VALUES (?, ?, 'done', 'passed', datetime('now'), ?, ?) RETURNING id",
2625 + )
2626 + .bind(sha)
2627 + .bind(version)
2628 + .bind(format!("{sha}-digest"))
2629 + .bind(staged_path)
2630 + .fetch_one(pool)
2631 + .await
2632 + .unwrap()
2633 + }
2634 +
2635 + async fn insert_gate_build(
2636 + pool: &SqlitePool,
2637 + tier: &str,
2638 + version: &str,
2639 + kind: &str,
2640 + passed: i64,
2641 + build_id: i64,
2642 + ) {
2643 + let status = if passed == 1 { "passed" } else { "failed" };
2644 + sqlx::query(
2645 + "INSERT INTO gate_runs (version, tier, gate_kind, started_at, finished_at, status, build_id) \
2646 + VALUES (?, ?, ?, datetime('now'), datetime('now'), ?, ?)",
2647 + )
2648 + .bind(version)
2649 + .bind(tier)
2650 + .bind(kind)
2651 + .bind(status)
2652 + .bind(build_id)
2653 + .execute(pool)
2654 + .await
2655 + .unwrap();
2656 + }
2657 +
2658 + #[tokio::test]
2659 + async fn unsatisfied_gates_keys_build_evidence_on_build_id() {
2660 + let pool = fresh_pool().await;
2661 + seed(&pool, "a", "3.0.0").await;
2662 + // Two builds of the SAME version string. Only b1's gate ran.
2663 + let b1 = seed_build(&pool, "sha1", "3.0.0", "/rel/1111111111111111").await;
2664 + let b2 = seed_build(&pool, "sha2", "3.0.0", "/rel/2222222222222222").await;
2665 + insert_gate_build(&pool, "a", "3.0.0", "cargo_test", 1, b1).await;
2666 +
2667 + // b1's own evidence satisfies.
2668 + let ok = unsatisfied_gates(
2669 + &pool,
2670 + &tid("a"),
2671 + &[Gate::CargoTest],
2672 + "3.0.0",
2673 + Some(b1),
2674 + false,
2675 + )
2676 + .await
2677 + .unwrap();
2678 + assert!(ok.is_empty(), "the build that passed its gate is satisfied");
2679 +
2680 + // b2 shares the version but has no gate row of its own: fail closed. This
2681 + // is the hole — a rebuild reusing the version must not ride b1's evidence.
2682 + let bad = unsatisfied_gates(
2683 + &pool,
2684 + &tid("a"),
2685 + &[Gate::CargoTest],
2686 + "3.0.0",
2687 + Some(b2),
2688 + false,
2689 + )
2690 + .await
2691 + .unwrap();
2692 + assert_eq!(
2693 + bad,
2694 + vec!["cargo_test".to_string()],
2695 + "a different build of the same version does not inherit the evidence"
2696 + );
2697 +
2698 + // Legacy (pre-identity) callers still resolve by version string.
2699 + let legacy = unsatisfied_gates(&pool, &tid("a"), &[Gate::CargoTest], "3.0.0", None, false)
2700 + .await
2701 + .unwrap();
2702 + assert!(legacy.is_empty(), "version-keyed legacy path unchanged");
2703 + }
2704 +
2705 + #[tokio::test]
2706 + async fn promote_rejects_an_explicit_version_that_is_not_the_source_build() {
2707 + // The burn-in hole: `promote --version Y` used to check the SOURCE tier's
2708 + // clock/evidence (which belong to whatever is current there), letting Y
2709 + // inherit another build's 48h. Now promote resolves the source's current
2710 + // build and refuses an explicit version that isn't it.
2711 + let state = test_state().await;
2712 + seed_version(&state.pool, "3.0.0").await;
2713 + let b = seed_build(&state.pool, "shaB", "3.0.0", "/rel/deadbeefdeadbeef").await;
2714 + sqlx::query(
2715 + "UPDATE tier_state SET current_version='3.0.0', current_build_id=? WHERE tier='host'",
2716 + )
2717 + .bind(b)
2718 + .execute(&state.pool)
2719 + .await
2720 + .unwrap();
2721 +
2722 + let err = promote_inner(
2723 + state,
2724 + "a".into(),
2725 + PromoteBody {
2726 + version: Some("2.0.0".into()),
2727 + ..Default::default()
2728 + },
2729 + )
2730 + .await
2731 + .expect_err("promoting a version other than the source build must be refused");
2732 + assert!(
2733 + matches!(&err, crate::error::Error::GateBlocked(m) if m.contains("vouched for")),
2734 + "the refusal must name the mismatch, got: {err:?}",
2735 + );
2736 + }
2737 +
2738 + #[tokio::test]
2739 + async fn promote_identity_path_deploys_the_source_build_and_advances_build_id() {
2740 + // Full promote through the identity path: source tier points at a build,
2741 + // promote resolves the artifact through it, deploys build_runs.staged_path
2742 + // (content-addressed), and advances the target's current_build_id.
2743 + let mut state = test_state().await;
2744 + let node_root = tempfile::tempdir().unwrap();
2745 + // Point the a-local node at a real tempdir so the symlink swap is checkable.
2746 + let mut topo = (*state.topo).clone();
2747 + topo.tiers[1].nodes[0].release_root = node_root.path().to_string_lossy().into_owned();
2748 + topo.tiers[1].gates = vec![]; // isolate the deploy fan-out
2749 + state.topo = Arc::new(topo);
2750 + state.executors = Arc::new(crate::state::build_executors(&state.topo));
2751 +
2752 + // deploys.node FKs into `nodes`, so the promote path needs the row.
2753 + sqlx::query("INSERT INTO nodes (name, tier, ssh_target, release_root) VALUES ('a-local', 'a', 'local', ?)")
2754 + .bind(node_root.path().to_string_lossy().into_owned())
2755 + .execute(&state.pool)
2756 + .await
2757 + .unwrap();
2758 +
2759 + seed_version(&state.pool, "3.0.0").await;
2760 + let staged = format!(
2761 + "{}/releases/abc123abc123abc1",
2762 + node_root.path().to_string_lossy()
2763 + );
2764 + let b = seed_build(&state.pool, "shaB", "3.0.0", &staged).await;
2765 + sqlx::query(
2766 + "UPDATE tier_state SET current_version='3.0.0', current_build_id=? WHERE tier='host'",
2767 + )
2768 + .bind(b)
2769 + .execute(&state.pool)
2770 + .await
2771 + .unwrap();
2772 +
2773 + let pool = state.pool.clone();
2774 + let Json(body) = promote_inner(state, "a".into(), PromoteBody::default())
2775 + .await
2776 + .expect("identity-path promote succeeds");
2777 + assert_eq!(body["version"], "3.0.0");
2778 +
2779 + // Target tier advanced with the build identity, not just the version.
2780 + let (cur_v, cur_b): (Option<String>, Option<i64>) = sqlx::query_as(
2781 + "SELECT current_version, current_build_id FROM tier_state WHERE tier = 'a'",
2782 + )
2783 + .fetch_one(&pool)
2784 + .await
2785 + .unwrap();
2786 + assert_eq!(cur_v.as_deref(), Some("3.0.0"));
2787 + assert_eq!(cur_b, Some(b), "target records the promoted build id");
2788 +
2789 + // The deploy row is attributed to the build.
2790 + let deploy_b: Option<i64> = sqlx::query_scalar(
2791 + "SELECT build_id FROM deploys WHERE tier = 'a' ORDER BY id DESC LIMIT 1",
2792 + )
2793 + .fetch_one(&pool)
2794 + .await
2795 + .unwrap();
2796 + assert_eq!(deploy_b, Some(b));
2797 +
2798 + // The node's `current` points at the content-addressed release dir, whose
2799 + // name is the staged_path's basename — not the version.
2800 + let link = tokio::fs::read_link(node_root.path().join("current"))
2801 + .await
2802 + .unwrap();
2803 + assert_eq!(link.to_string_lossy(), "releases/abc123abc123abc1");
2804 + }
2805 +
2806 + /// Insert a bare `versions` row (FK target for tier_state.current_version).
2807 + async fn seed_version(pool: &SqlitePool, version: &str) {
2808 + sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES (?, 'sha', datetime('now'), '/tmp/x') ON CONFLICT DO NOTHING")
2809 + .bind(version).execute(pool).await.unwrap();
2810 + }
2565 2811 }
M sando/daemon/src/routes/promotion.rs +126 -46
@@ -39,29 +39,97 @@ pub(super) async fn promote_inner(
39 39 )));
40 40 }
41 41
42 - // Resolve version: explicit if given, else the source tier's current.
43 - let version_str = match body.version {
44 - Some(v) => v,
45 - None => sqlx::query_scalar::<_, Option<String>>(
46 - "SELECT current_version FROM tier_state WHERE tier = ?",
47 - )
48 - .bind(&source.name)
49 - .fetch_optional(&s.pool)
50 - .await
51 - .map_err(crate::error::Error::Db)?
52 - .flatten()
53 - .ok_or_else(|| {
54 - crate::error::Error::GateBlocked(format!(
55 - "no version specified and tier {} has no current_version",
56 - source.name
57 - ))
58 - })?,
42 + // Resolve the artifact through the SOURCE tier's evidence, not a version
43 + // string (invariant 2, wiki [[release-artifact-identity]]). The build that is
44 + // *current on the source tier* is the one that burned in and passed its gates
45 + // there; promoting anything else lets a version string ride on another build's
46 + // evidence and clock — the "promote --version Y inherits X's 48h" hole. A
47 + // given `body.version` must name that same build.
48 + let source_build: Option<(Option<i64>, Option<String>, Option<String>)> = sqlx::query_as(
49 + "SELECT ts.current_build_id, br.version, br.staged_path
50 + FROM tier_state ts
51 + LEFT JOIN build_runs br ON br.id = ts.current_build_id
52 + WHERE ts.tier = ?",
53 + )
54 + .bind(&source.name)
55 + .fetch_optional(&s.pool)
56 + .await
57 + .map_err(crate::error::Error::Db)?;
58 +
59 + let (build_id, version_str, staged_dir) = match source_build {
60 + // Identity path: the source tier points at a recorded build.
61 + Some((Some(bid), ver, staged_path)) => {
62 + let version_str = ver.ok_or_else(|| {
63 + crate::error::Error::Other(anyhow::anyhow!(
64 + "source build {bid} on tier {} has no recorded version",
65 + source.name
66 + ))
67 + })?;
68 + let staged_path = staged_path.ok_or_else(|| {
69 + crate::error::Error::Other(anyhow::anyhow!(
70 + "source build {bid} ({version_str}) has no staged_path; cannot promote"
71 + ))
72 + })?;
73 + if let Some(req) = &body.version
74 + && req != &version_str
75 + {
76 + return Err(crate::error::Error::GateBlocked(format!(
77 + "tier {} is running build {bid} ({version_str}); refusing to promote an \
78 + explicit version {req} that is not the build the tier vouched for",
79 + source.name
80 + )));
81 + }
82 + (
83 + Some(bid),
84 + version_str,
85 + std::path::PathBuf::from(staged_path),
86 + )
87 + }
88 + // Legacy fallback: NULL current_build_id (a pre-identity tier). Resolve by
89 + // version string through the versions table, exactly as before. One clean
90 + // build-and-promote cycle re-anchors the tier onto the identity path.
91 + _ => {
92 + let version_str = match body.version.clone() {
93 + Some(v) => v,
94 + None => sqlx::query_scalar::<_, Option<String>>(
95 + "SELECT current_version FROM tier_state WHERE tier = ?",
96 + )
97 + .bind(&source.name)
98 + .fetch_optional(&s.pool)
99 + .await
100 + .map_err(crate::error::Error::Db)?
101 + .flatten()
102 + .ok_or_else(|| {
103 + crate::error::Error::GateBlocked(format!(
104 + "no version specified and tier {} has no current_version",
105 + source.name
106 + ))
107 + })?,
108 + };
109 + let bin: Option<(String,)> =
110 + sqlx::query_as("SELECT artifact_path FROM versions WHERE version = ?")
111 + .bind(&version_str)
112 + .fetch_optional(&s.pool)
113 + .await
114 + .map_err(crate::error::Error::Db)?;
115 + let Some((bin,)) = bin else {
116 + return Err(crate::error::Error::NotFound);
117 + };
118 + // `artifact_path` is the primary binary; the release dir is its parent.
119 + let staged_dir = std::path::PathBuf::from(&bin)
120 + .parent()
121 + .ok_or_else(|| {
122 + crate::error::Error::Other(anyhow::anyhow!("artifact_path has no parent"))
123 + })?
124 + .to_path_buf();
125 + (None, version_str, staged_dir)
126 + }
59 127 };
60 128 let version = crate::domain::Version::parse(&version_str)
61 129 .map_err(|e| crate::error::Error::Other(anyhow::anyhow!(e)))?;
62 130
63 131 // 1. Predecessor must have all of its configured gates satisfied for this
64 - // version (with optional hotfix override that skips burn_in). Evaluated
132 + // build (with optional hotfix override that skips burn_in). Evaluated
65 133 // against the topology gate list, so a gate that never ran blocks the
66 134 // promote instead of being treated as green.
67 135 let pending = unsatisfied_gates(
@@ -69,6 +137,7 @@ pub(super) async fn promote_inner(
69 137 &source.name,
70 138 &source.gates,
71 139 &version_str,
140 + build_id,
72 141 body.hotfix,
73 142 )
74 143 .await?;
@@ -81,23 +150,6 @@ pub(super) async fn promote_inner(
81 150 )));
82 151 }
83 152
84 - // 2. Look up the artifact for this version.
85 - let bin: Option<(String,)> =
86 - sqlx::query_as("SELECT artifact_path FROM versions WHERE version = ?")
87 - .bind(&version)
88 - .fetch_optional(&s.pool)
89 - .await
90 - .map_err(crate::error::Error::Db)?;
91 - let Some((bin,)) = bin else {
92 - return Err(crate::error::Error::NotFound);
93 - };
94 - let bin_path = std::path::PathBuf::from(bin);
95 - // `artifact_path` is the primary binary; the staged release dir is its parent.
96 - let staged_dir = bin_path
97 - .parent()
98 - .ok_or_else(|| crate::error::Error::Other(anyhow::anyhow!("artifact_path has no parent")))?
99 - .to_path_buf();
100 -
101 153 // The version this tier was running before this promote — the rollback
102 154 // target if a canary node fails partway through a multi-node rollout.
103 155 let prev_version: Option<String> =
@@ -148,13 +200,14 @@ pub(super) async fn promote_inner(
148 200 let outcome_json = serde_json::to_string(&outcome_obj)
149 201 .unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}"));
150 202 sqlx::query(
151 - "INSERT INTO deploys (version, tier, node, started_at, finished_at, outcome, outcome_json, hotfix, reset_burn_in)
152 - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
203 + "INSERT INTO deploys (version, tier, node, started_at, finished_at, outcome, outcome_json, hotfix, reset_burn_in, build_id)
204 + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
153 205 )
154 206 .bind(&version).bind(&target.name).bind(&node.name)
155 207 .bind(&started).bind(&finished).bind(outcome_obj.status_str())
156 208 .bind(&outcome_json)
157 209 .bind(body.hotfix as i64).bind(body.reset_burn_in as i64)
210 + .bind(build_id)
158 211 .execute(&s.pool).await.map_err(crate::error::Error::Db)?;
159 212 if let Some(e) = err_for_propagation {
160 213 let crate::outcome::DeployStatus::Failed { failure } = outcome_obj.status else {
@@ -301,6 +354,10 @@ pub(super) async fn promote_inner(
301 354 worktree: std::path::PathBuf::new(),
302 355 events: s.events.clone(),
303 356 nodes,
357 + // These post-deploy gate rows are the evidence the NEXT promote (this
358 + // tier -> the following one) resolves through, so they must carry the
359 + // build they vouch for.
360 + build_id,
304 361 };
305 362 post_deploy_failure = match crate::gates::run_all(&ctx, &post_deploy).await {
306 363 Ok(failed) if failed.is_empty() => None,
@@ -338,7 +395,7 @@ pub(super) async fn promote_inner(
338 395 // serialized against rollback and the host build path's advance.
339 396 // reset_burn_in on the *source* tier nulls its clock only when the operator
340 397 // explicitly asked.
341 - crate::runs::advance_tier(&s.pool, target.name.as_str(), &version)
398 + crate::runs::advance_tier(&s.pool, target.name.as_str(), &version, build_id)
342 399 .await
343 400 .map_err(crate::error::Error::Db)?;
344 401
@@ -497,11 +554,22 @@ pub(super) async fn clear_partial(s: &AppState, tier: &crate::domain::TierId) {
497 554 /// row would otherwise never flip to passed as time elapses);
498 555 /// - every other kind requires a `passed` row for (tier, version) — a missing
499 556 /// or non-passed latest row counts as unsatisfied.
557 + ///
558 + /// `build_id` is the identity of the artifact being promoted (wiki note
559 + /// `release-artifact-identity`). When `Some`, the deterministic build-evidence
560 + /// gates are checked against the row that vouched for *that build* rather than
561 + /// against any row that happens to carry the version string — this is what stops
562 + /// a `promote --version Y` from riding on gate rows a different build left under
563 + /// the same version. `None` is the legacy/pre-identity path: fall back to the
564 + /// version-keyed lookup so a mid-migration tier (NULL `current_build_id`) still
565 + /// promotes. `burn_in` is clock-based either way — the clock is reset on every
566 + /// advance, so it belongs to the build now current on the tier.
500 567 pub(super) async fn unsatisfied_gates(
501 568 pool: &sqlx::SqlitePool,
502 569 tier: &crate::domain::TierId,
503 570 gates: &[crate::topology::Gate],
504 571 version: &str,
572 + build_id: Option<i64>,
505 573 hotfix: bool,
506 574 ) -> std::result::Result<Vec<String>, crate::error::Error> {
507 575 use crate::topology::Gate;
@@ -580,14 +648,26 @@ pub(super) async fn unsatisfied_gates(
580 648 | Gate::NodeHealth => {
581 649 // Latest row for this configured gate kind; NULL/missing/any
582 650 // non-'passed' status all count as unsatisfied (fail closed).
583 - let status: Option<String> = sqlx::query_scalar(
584 - "SELECT status FROM gate_runs
585 - WHERE tier = ?1 AND version = ?2 AND gate_kind = ?3
586 - ORDER BY id DESC LIMIT 1",
587 - )
588 - .bind(tier.as_str())
589 - .bind(version)
590 - .bind(kind.as_str())
651 + // Keyed on build_id when the artifact has an identity (the
652 + // evidence must be for *this* build), else on the version string.
653 + let status: Option<String> = match build_id {
654 + Some(bid) => sqlx::query_scalar(
655 + "SELECT status FROM gate_runs
656 + WHERE tier = ?1 AND build_id = ?2 AND gate_kind = ?3
657 + ORDER BY id DESC LIMIT 1",
658 + )
659 + .bind(tier.as_str())
660 + .bind(bid)
661 + .bind(kind.as_str()),
662 + None => sqlx::query_scalar(
663 + "SELECT status FROM gate_runs
664 + WHERE tier = ?1 AND version = ?2 AND gate_kind = ?3
665 + ORDER BY id DESC LIMIT 1",
666 + )
667 + .bind(tier.as_str())
668 + .bind(version)
669 + .bind(kind.as_str()),
670 + }
591 671 .fetch_optional(pool)
592 672 .await
593 673 .map_err(crate::error::Error::Db)?
M sando/daemon/src/runs.rs +41 -2
@@ -79,19 +79,35 @@ pub async fn set_phase(pool: &SqlitePool, run_id: RunId, phase: Phase) -> Result
79 79 ///
80 80 /// Returns the raw `sqlx::Error` so the route layer can map it to its typed
81 81 /// `Error::Db`; anyhow callers (the build pipeline) get `?`-conversion for free.
82 + /// `build_id` is the `build_runs.id` of the build landing on the tier — the
83 + /// artifact identity (wiki [[release-artifact-identity]]). It advances in
84 + /// lockstep with the version label: `previous_build_id` takes the old
85 + /// `current_build_id` in the same self-referential UPDATE, so the build the
86 + /// tier just stepped off is the rollback target. `None` for legacy/pre-identity
87 + /// callers writes NULL — treated as "no recorded build" downstream, with the
88 + /// version-string path as the fallback.
89 + ///
90 + /// `burn_in_started_at = now` starts the tier's burn-in clock. Because the
91 + /// clock resets on every advance and the clock is what burn-in reads, the clock
92 + /// always belongs to the build now current on the tier — this is what stops a
93 + /// promote from crediting one build with another build's elapsed burn-in.
82 94 pub async fn advance_tier(
83 95 pool: &SqlitePool,
84 96 tier: &str,
85 97 version: &Version,
98 + build_id: Option<i64>,
86 99 ) -> Result<(), sqlx::Error> {
87 100 sqlx::query(
88 101 "UPDATE tier_state
89 - SET previous_version = current_version,
90 - current_version = ?,
102 + SET previous_version = current_version,
103 + current_version = ?,
104 + previous_build_id = current_build_id,
105 + current_build_id = ?,
91 106 burn_in_started_at = ?
92 107 WHERE tier = ?",
93 108 )
94 109 .bind(version)
110 + .bind(build_id)
95 111 .bind(Utc::now().to_rfc3339())
96 112 .bind(tier)
97 113 .execute(pool)
@@ -109,6 +125,29 @@ pub async fn set_version(pool: &SqlitePool, run_id: RunId, version: &Version) ->
109 125 Ok(())
110 126 }
111 127
128 + /// Record the build's content identity once its bundle has been staged and
129 + /// hashed: the full 64-hex `bundle_digest` and the `staged_path` it lives at.
130 + /// This is what promote/burn-in/retention key on once build id becomes the
131 + /// identity (wiki [[release-artifact-identity]]). Guarded on `building` so a
132 + /// settled run is never mutated.
133 + pub async fn set_identity(
134 + pool: &SqlitePool,
135 + run_id: RunId,
136 + bundle_digest: &str,
137 + staged_path: &str,
138 + ) -> Result<()> {
139 + sqlx::query(
140 + "UPDATE build_runs SET bundle_digest = ?, staged_path = ?
141 + WHERE id = ? AND result = 'building'",
142 + )
143 + .bind(bundle_digest)
144 + .bind(staged_path)
145 + .bind(run_id.0)
146 + .execute(pool)
147 + .await?;
148 + Ok(())
149 + }
150 +
112 151 /// Settle the run green. First terminal write wins (guarded on `building`).
113 152 pub async fn mark_passed(pool: &SqlitePool, run_id: RunId) -> Result<()> {
114 153 sqlx::query(