max / makenotwork

bento: close the seven release-orchestration audit findings Address the 2026-07-21 Bento audit (daemon 78 -> 99 tests, clippy clean): - Artifact identity: hash every artifact at collect, assert the version embedded in its filename matches the build, and record the sha256 in the release ledger (releases.artifact_hash, previously never written). - version_check preflight: cross-check tauri.conf.json vs Cargo.toml before any host pulls, so a version-source drift fails fast. - bentod --check-config, wired as ExecStartPre, so a config a fresh binary can't parse fails the start instead of crash-looping. - Per-host serialization lock so two targets on one host can't build concurrently in a shared checkout + keychain. - Per-step deadlines with cooperative cancellation, replacing the whole-build 2h guillotine that mismarked a legitimate fan-out failed while the blocking recipe bodies kept signing. - Per-(app, version) release view: build_view aggregates the latest run per target across all builds (a retry no longer collapses the matrix), plus a GET /release/{app}/{version} endpoint. - Opt-in all-targets-green publish gate (topology require_all_targets) to stop a partial release shipping while a sibling target is red or still building. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: Max Johnson <me@maxj.phd> · 2026-07-24 00:31 UTC

Commit: 5a0aede2c48b19952ce8a3aec65d0c26d97d9875

Parent: e33082c

12 files changed, +1063 insertions, -17 deletions

M bento/daemon/Cargo.lock +1

			@@ -193,6 +193,7 @@ dependencies = [
193	193		"semver",
194	194		"serde",
195	195		"serde_json",
	196	+	"sha2 0.10.9",
196	197		"sqlx",
197	198		"tempfile",
198	199		"thiserror",

M bento/daemon/Cargo.toml +1

			@@ -28,6 +28,7 @@ anyhow = "1.0.102"
28	28		thiserror = "2.0.18"
29	29		chrono = { version = "0.4", features = ["serde"] }
30	30		semver = { version = "1.0", features = ["serde"] }
	31	+	sha2 = "0.10"
31	32
32	33		[dev-dependencies]
33	34		async-trait = "0.1"

M bento/daemon/src/config.rs +7

			@@ -20,6 +20,12 @@ pub struct Config {
20	20		/// (`<logs_root>/<app>/<version>/<target>/<step>.log`).
21	21		#[serde(default = "default_logs_root")]
22	22		pub logs_root: PathBuf,
	23	+	/// Override the per-step wall-clock budget (in seconds) for EVERY step,
	24	+	/// replacing the per-kind defaults in `engine::step_budget`. Unset (the
	25	+	/// default) uses those. Mainly an escape hatch for a constrained host or a
	26	+	/// test that needs a short deadline.
	27	+	#[serde(default)]
	28	+	pub step_timeout_secs: Option<u64>,
23	29		}
24	30
25	31		fn default_logs_root() -> PathBuf {
			@@ -43,6 +49,7 @@ impl Config {
43	49		secrets_root: root.join("secrets"),
44	50		dist_root: root.join("dist"),
45	51		logs_root: root.join("logs"),
	52	+	step_timeout_secs: None,
46	53		}
47	54		}
48	55		}

M bento/daemon/src/domain.rs +7

			@@ -239,6 +239,13 @@ impl Version {
239	239		.map(Self)
240	240		.map_err(\|e\| format!("invalid semver `{s}`: {e}"))
241	241		}
	242	+
	243	+	/// The `(major, minor, patch)` core, ignoring any prerelease/build suffix.
	244	+	/// Used to match a build against the plain `X.Y.Z` embedded in an artifact
	245	+	/// file name, which never carries the suffix.
	246	+	pub fn core(&self) -> (u64, u64, u64) {
	247	+	(self.0.major, self.0.minor, self.0.patch)
	248	+	}
242	249		}
243	250
244	251		impl fmt::Display for Version {

M bento/daemon/src/engine.rs +390 -7

			@@ -21,7 +21,9 @@ use anyhow::{Context as _, Result};
21	21		use ops_core::live_log::LiveLog;
22	22		use ops_exec::{Action, Executor, ObserveKind, Step as OpStep, SyncOpts};
23	23		use rhai::{Engine, EvalAltResult, Map};
	24	+	use sha2::{Digest, Sha256};
24	25		use sqlx::SqlitePool;
	26	+	use std::collections::HashMap;
25	27		use std::path::{Path, PathBuf};
26	28		use std::sync::atomic::{AtomicBool, Ordering};
27	29		use std::sync::{Arc, Mutex};
			@@ -57,6 +59,39 @@ struct StepState {
57	59		/// recipe ignored the bool). Forces the step's recorded status to `Failed`
58	60		/// and bars `publish` (the step-success ledger).
59	61		failed: bool,
	62	+	/// Wall-clock deadline for this step. A command that runs past it fails the
	63	+	/// step (and unwinds the recipe) rather than wedging under the old
	64	+	/// whole-build guillotine, which a legitimate 5-target fan-out plus notary
	65	+	/// queueing could trip — mismarking every target failed while the blocking
	66	+	/// recipe bodies kept signing.
	67	+	deadline: std::time::Instant,
	68	+	}
	69	+
	70	+	/// Per-step wall-clock budget: a generous ceiling that catches a wedged command
	71	+	/// (a hung ssh, a stuck notary poll) without killing legitimately slow work.
	72	+	/// The old design bounded the whole build at 2h; this bounds each step so one
	73	+	/// slow step can't be blamed on another and a fan-out of slow-but-fine targets
	74	+	/// isn't guillotined. `Config::step_timeout_secs` overrides these per-kind
	75	+	/// defaults for every step; see [`RecipeCtx::step_budget`].
	76	+	fn default_step_budget(step: Step) -> std::time::Duration {
	77	+	use std::time::Duration;
	78	+	let mins = match step {
	79	+	Step::Checkout => 10,
	80	+	// clippy + full test suite, cold, on a workspace.
	81	+	Step::Prebuild => 45,
	82	+	// cargo tauri build, cold, universal bundles.
	83	+	Step::Build => 90,
	84	+	Step::Sign => 15,
	85	+	// Apple's notary queue + this step's bounded retries.
	86	+	Step::Notarize => 60,
	87	+	Step::Staple => 10,
	88	+	Step::Package => 30,
	89	+	Step::Verify => 10,
	90	+	// rsync of multi-GiB artifacts off the build host.
	91	+	Step::Collect => 30,
	92	+	Step::Publish => 20,
	93	+	};
	94	+	Duration::from_secs(mins * 60)
60	95		}
61	96
62	97		/// Everything a recipe's host functions need, shared (Arc) into each closure.
			@@ -104,6 +139,16 @@ pub struct RecipeCtx {
104	139		/// and before publish so a superseded recipe stops promptly rather than
105	140		/// running to completion (the blocking Rhai body can't be `abort()`ed).
106	141		cancel: Arc<AtomicBool>,
	142	+	/// The all-targets-green publish gate (topology `require_all_targets`).
	143	+	/// `Some(declared)` ⇒ `publish` refuses unless every one of `declared` has a
	144	+	/// successful latest run for this `(app, version)`. `None` ⇒ gate off, each
	145	+	/// target publishes independently.
	146	+	all_green_required: Option<Vec<Target>>,
	147	+	/// sha256 of each artifact hashed at `collect`, keyed by file name. `publish`
	148	+	/// reads it to record `releases.artifact_hash` for the bytes it ships, so the
	149	+	/// hash is the one computed when the artifact landed rather than a re-read
	150	+	/// that could see a different file. Absent ⇒ `publish` hashes on demand.
	151	+	artifact_hashes: Mutex<HashMap<String, String>>,
107	152		}
108	153
109	154		impl RecipeCtx {
			@@ -156,6 +201,7 @@ impl RecipeCtx {
156	201		ota: Arc<OtaRegistry>,
157	202		rt: Handle,
158	203		cancel: Arc<AtomicBool>,
	204	+	all_green_required: Option<Vec<Target>>,
159	205		) -> Self {
160	206		Self {
161	207		app,
			@@ -176,6 +222,8 @@ impl RecipeCtx {
176	222		gatekeeper_ok: Mutex::new(None),
177	223		failed_steps: Mutex::new(Vec::new()),
178	224		cancel,
	225	+	all_green_required,
	226	+	artifact_hashes: Mutex::new(HashMap::new()),
179	227		}
180	228		}
181	229
			@@ -270,10 +318,65 @@ impl RecipeCtx {
270	318		step,
271	319		log: Arc::new(AsyncMutex::new(log)),
272	320		failed: false,
	321	+	deadline: std::time::Instant::now() + self.step_budget(step),
273	322		});
274	323		Ok(())
275	324		}
276	325
	326	+	/// This build's budget for `step`: the `Config` override if set, else the
	327	+	/// per-kind default.
	328	+	fn step_budget(&self, step: Step) -> std::time::Duration {
	329	+	self.cfg
	330	+	.step_timeout_secs
	331	+	.map(std::time::Duration::from_secs)
	332	+	.unwrap_or_else(\|\| default_step_budget(step))
	333	+	}
	334	+
	335	+	/// The current step's wall-clock deadline (or a default if no step is open,
	336	+	/// which only happens before the first `step()` — commands then run under an
	337	+	/// implicit `Build` step opened by `ensure_step`).
	338	+	fn step_deadline(&self) -> std::time::Instant {
	339	+	self.current
	340	+	.lock()
	341	+	.unwrap()
	342	+	.as_ref()
	343	+	.map(\|s\| s.deadline)
	344	+	.unwrap_or_else(\|\| std::time::Instant::now() + self.step_budget(Step::Build))
	345	+	}
	346	+
	347	+	/// Drive `fut` on the runtime, but stop early on two conditions the recipe
	348	+	/// bodies otherwise cannot observe (they run synchronously on a blocking
	349	+	/// thread): the current step's deadline, and supersession by a newer build.
	350	+	/// Either turns into an error that fails the step and unwinds the recipe, so
	351	+	/// a wedged command no longer runs unbounded and a superseded build stops
	352	+	/// mid-step instead of only at the next step boundary.
	353	+	fn run_bounded<F, T>(&self, what: &str, fut: F) -> Result<T>
	354	+	where
	355	+	F: std::future::Future<Output = Result<T>>,
	356	+	{
	357	+	let deadline = self.step_deadline();
	358	+	let cancel = self.cancel.clone();
	359	+	self.rt.block_on(async move {
	360	+	tokio::pin!(fut);
	361	+	let watch = async {
	362	+	// Poll the cooperative cancel flag; the finalizer and a
	363	+	// superseding build both set it. Cheap next to a build step.
	364	+	while !cancel.load(Ordering::SeqCst) {
	365	+	tokio::time::sleep(std::time::Duration::from_millis(250)).await;
	366	+	}
	367	+	};
	368	+	tokio::select! {
	369	+	r = &mut fut => r,
	370	+	_ = tokio::time::sleep_until(deadline.into()) => {
	371	+	Err(anyhow::anyhow!("`{what}` exceeded its per-step deadline"))
	372	+	}
	373	+	_ = watch => {
	374	+	Err(anyhow::anyhow!("build superseded by a newer request; aborting `{what}`"))
	375	+	}
	376	+	}
	377	+	})
	378	+	}
	379	+
277	380		/// Flag the currently-open step as failed (no-op if none is open). Forces
278	381		/// its recorded status to `Failed` at `finish_step` and adds it to the
279	382		/// publish-barring ledger, even though the recipe kept running.
			@@ -374,8 +477,13 @@ impl RecipeCtx {
374	477		fn run(self: &Arc<Self>, host: &str, cmd: &str) -> Result<(i32, String)> {
375	478		let sink = self.ensure_step()?;
376	479		let exec = self.exec(host)?;
377		-	let step = OpStep::shell(action_for(self.current_step()), cmd.to_string());
378		-	let out = self.rt.block_on(async move {
	480	+	let cur = self.current_step();
	481	+	let step = OpStep::shell(action_for(cur), cmd.to_string());
	482	+	// Bounded by the step's deadline and interruptible on supersession, so a
	483	+	// hung command fails its step instead of running unbounded, and a
	484	+	// superseded build stops mid-step rather than only at the next boundary.
	485	+	let label = format!("{cur} command on `{host}`");
	486	+	let out = self.run_bounded(&label, async move {
379	487		let mut guard = sink.lock().await;
380	488		exec.run_streaming(&step, &mut *guard).await
381	489		})?;
			@@ -563,6 +671,139 @@ fn version_from_cargo_toml(raw: &str) -> Result<String> {
563	671		.context("no `[package].version` or `[workspace.package].version` in Cargo.toml")
564	672		}
565	673
	674	+	/// Cross-check every version source in a repo and confirm they all agree with
	675	+	/// the version being built, before a single host pulls or compiles.
	676	+	///
	677	+	/// `version_from_repo` reads exactly one file, so a `tauri.conf.json` at 0.5.0
	678	+	/// and a root `Cargo.toml` still at 0.4.0 build happily and file artifacts under
	679	+	/// whichever the runner happened to read. This reads every source present —
	680	+	/// `version_path` (when set), `src-tauri/tauri.conf.json`, and the root
	681	+	/// `Cargo.toml` — and fails loudly when any disagree, naming each file and its
	682	+	/// version. A source that is absent is skipped (a library crate with only a
	683	+	/// `Cargo.toml` has nothing to disagree with); the check never invents drift.
	684	+	///
	685	+	/// Scope: the JSON/TOML sources bentod itself reads. The iOS `gen/apple/project.yml`
	686	+	/// path (rewritten by a build-time `sed`) is out of scope here — it is asserted at
	687	+	/// its own build step — but the same drift class motivated this guard.
	688	+	pub fn check_version_consistency(
	689	+	repo: &str,
	690	+	version_path: Option<&str>,
	691	+	expected: &Version,
	692	+	) -> Result<()> {
	693	+	let root = expand_tilde(repo);
	694	+	// (human-readable source label, parsed version) for every source present.
	695	+	let mut found: Vec<(String, Version)> = Vec::new();
	696	+
	697	+	let mut consider = \|rel: &str, raw: &str, as_json: bool\| -> Result<()> {
	698	+	let ver = if as_json {
	699	+	version_from_tauri_json(raw)
	700	+	} else {
	701	+	version_from_cargo_toml(raw)
	702	+	}?;
	703	+	let parsed = Version::parse(&ver).map_err(\|e\| anyhow::anyhow!(e))?;
	704	+	found.push((rel.to_string(), parsed));
	705	+	Ok(())
	706	+	};
	707	+
	708	+	if let Some(vp) = version_path {
	709	+	let path = root.join(vp);
	710	+	let raw = std::fs::read_to_string(&path)
	711	+	.with_context(\|\| format!("reading version file {}", path.display()))?;
	712	+	consider(vp, &raw, vp.ends_with(".json"))?;
	713	+	}
	714	+	let tauri_conf = root.join("src-tauri").join("tauri.conf.json");
	715	+	if version_path != Some("src-tauri/tauri.conf.json") && tauri_conf.exists() {
	716	+	let raw = std::fs::read_to_string(&tauri_conf)
	717	+	.with_context(\|\| format!("reading {}", tauri_conf.display()))?;
	718	+	consider("src-tauri/tauri.conf.json", &raw, true)?;
	719	+	}
	720	+	let cargo_toml = root.join("Cargo.toml");
	721	+	if version_path != Some("Cargo.toml") && cargo_toml.exists() {
	722	+	// A Cargo.toml with neither `[package].version` nor
	723	+	// `[workspace.package].version` (a pure virtual workspace) carries no
	724	+	// version to check — skip it rather than fail.
	725	+	if let Ok(raw) = std::fs::read_to_string(&cargo_toml)
	726	+	&& version_from_cargo_toml(&raw).is_ok()
	727	+	{
	728	+	consider("Cargo.toml", &raw, false)?;
	729	+	}
	730	+	}
	731	+
	732	+	let disagree: Vec<&(String, Version)> = found.iter().filter(\|(_, v)\| v != expected).collect();
	733	+	anyhow::ensure!(
	734	+	disagree.is_empty(),
	735	+	"version drift in {repo}: building {expected} but {}",
	736	+	disagree
	737	+	.iter()
	738	+	.map(\|(src, v)\| format!("{src} says {v}"))
	739	+	.collect::<Vec<_>>()
	740	+	.join(", ")
	741	+	);
	742	+	Ok(())
	743	+	}
	744	+
	745	+	/// Every `X.Y.Z`-shaped version embedded in an artifact file name. Each maximal
	746	+	/// run of digits-and-dots contributes its first three numeric fields:
	747	+	/// `GoingsOn_0.5.0_aarch64.dmg` and `demo-9.9.9.bin` both yield one version (the
	748	+	/// trailing `.bin`/`.dmg` dot is tolerated), while `latest.json` yields `[]` and
	749	+	/// the `64` in `x86_64` is not three fields. Only the `major.minor.patch` core is
	750	+	/// taken; a prerelease/build suffix is separated by `-`/`+` and not needed here.
	751	+	fn versions_in_filename(name: &str) -> Vec<Version> {
	752	+	name.split(\|c: char\| !(c.is_ascii_digit() \|\| c == '.'))
	753	+	.filter_map(\|run\| {
	754	+	let f: Vec<&str> = run.split('.').filter(\|s\| !s.is_empty()).collect();
	755	+	if f.len() >= 3 && f[..3].iter().all(\|s\| s.chars().all(\|c\| c.is_ascii_digit())) {
	756	+	Version::parse(&format!("{}.{}.{}", f[0], f[1], f[2])).ok()
	757	+	} else {
	758	+	None
	759	+	}
	760	+	})
	761	+	.collect()
	762	+	}
	763	+
	764	+	/// Fail when a collected file's name embeds a version whose `major.minor.patch`
	765	+	/// is not the one being built. This is the guard against a stale checked-in
	766	+	/// artifact winning a glob: `ls -t <glob>` once let
	767	+	/// `AudioFiles-0.4.0-x86_64.AppImage` ship against 0.5.0. A file whose name
	768	+	/// carries no version (an updater `latest.json`, a `.sig`) is not asserted —
	769	+	/// there is nothing to compare. Compared on the core so a prerelease build's
	770	+	/// plain `X.Y.Z` in the filename still matches.
	771	+	fn assert_artifact_version(name: &str, expected: &Version) -> Result<()> {
	772	+	let versions = versions_in_filename(name);
	773	+	anyhow::ensure!(
	774	+	versions.is_empty() \|\| versions.iter().any(\|v\| v.core() == expected.core()),
	775	+	"collected artifact `{name}` carries version {} but the build is {expected}; \
	776	+	a stale artifact was left in the output dir — clean it so only {expected} remains",
	777	+	versions
	778	+	.iter()
	779	+	.map(ToString::to_string)
	780	+	.collect::<Vec<_>>()
	781	+	.join("/"),
	782	+	);
	783	+	Ok(())
	784	+	}
	785	+
	786	+	/// sha256 of a file, lowercase hex. Streams in 64 KiB chunks so a multi-GiB
	787	+	/// bundle never lands in memory whole.
	788	+	fn sha256_file(path: &Path) -> Result<String> {
	789	+	let mut file =
	790	+	std::fs::File::open(path).with_context(\|\| format!("hashing {}", path.display()))?;
	791	+	let mut hasher = Sha256::new();
	792	+	std::io::copy(&mut file, &mut hasher)
	793	+	.with_context(\|\| format!("reading {} to hash", path.display()))?;
	794	+	Ok(hex_lower(&hasher.finalize()))
	795	+	}
	796	+
	797	+	/// Lowercase-hex encode without pulling in a hex crate.
	798	+	fn hex_lower(bytes: &[u8]) -> String {
	799	+	use std::fmt::Write as _;
	800	+	let mut s = String::with_capacity(bytes.len() * 2);
	801	+	for b in bytes {
	802	+	let _ = write!(s, "{b:02x}");
	803	+	}
	804	+	s
	805	+	}
	806	+
566	807		/// Expand a leading `~/` to `$HOME`. Paths in the topology are written with `~`.
567	808		pub fn expand_tilde(p: &str) -> PathBuf {
568	809		if let Some(rest) = p.strip_prefix("~/")
			@@ -989,9 +1230,28 @@ impl RecipeCtx {
989	1230		// runs the transfer itself, as it always has.
990	1231		let sync = self.host_sync(host)?;
991	1232		let opts = SyncOpts::precompressed();
992		-	self.rt
993		-	.block_on(async { sync.pull_glob(glob, &dest, &opts).await })
994		-	.with_context(\|\| format!("collect {glob} from `{host}`"))?;
	1233	+	// Bounded by the collect step's deadline (rsync of a multi-GiB artifact
	1234	+	// can wedge on a stalled transport) and interruptible on supersession.
	1235	+	let dest_pull = dest.clone();
	1236	+	self.run_bounded(&format!("collect {glob} from `{host}`"), async move {
	1237	+	sync.pull_glob(glob, &dest_pull, &opts).await
	1238	+	})
	1239	+	.with_context(\|\| format!("collect {glob} from `{host}`"))?;
	1240	+	// Assert the version and hash every collected file. This is where a
	1241	+	// stale artifact is caught: a file whose name embeds a different version
	1242	+	// fails the collect (rather than silently winning a later glob), and the
	1243	+	// sha256 recorded here is what `publish` writes into the release ledger.
	1244	+	for entry in
	1245	+	std::fs::read_dir(&dest).with_context(\|\| format!("listing collect dest {dest_s}"))?
	1246	+	{
	1247	+	let entry = entry?;
	1248	+	let name = entry.file_name().to_string_lossy().into_owned();
	1249	+	assert_artifact_version(&name, &self.version)?;
	1250	+	if entry.file_type().map(\|t\| t.is_file()).unwrap_or(false) {
	1251	+	let digest = sha256_file(&entry.path())?;
	1252	+	self.artifact_hashes.lock().unwrap().insert(name, digest);
	1253	+	}
	1254	+	}
995	1255		// Best-effort size accounting for the event.
996	1256		events::emit(
997	1257		&self.events,
			@@ -1005,6 +1265,48 @@ impl RecipeCtx {
1005	1265		Ok(())
1006	1266		}
1007	1267
	1268	+	/// The all-targets-green gate: err unless every declared target OTHER than
	1269	+	/// the one publishing has a latest `target_runs` row of `ok` for this
	1270	+	/// `(app, version)`. A sibling with no run, a running run, or a failed
	1271	+	/// latest run all block the publish, naming what is not green.
	1272	+	fn assert_siblings_green(self: &Arc<Self>, declared: &[Target]) -> Result<()> {
	1273	+	let me = self.clone();
	1274	+	let (app_s, ver_s) = (self.app.to_string(), self.version.to_string());
	1275	+	let rows: Vec<(String, String)> = self.rt.block_on(async move {
	1276	+	sqlx::query_as(
	1277	+	"SELECT target, status FROM target_runs tr
	1278	+	WHERE app = ?1 AND version = ?2
	1279	+	AND id = (SELECT MAX(id) FROM target_runs
	1280	+	WHERE app = ?1 AND version = ?2 AND target = tr.target)",
	1281	+	)
	1282	+	.bind(app_s)
	1283	+	.bind(ver_s)
	1284	+	.fetch_all(&me.pool)
	1285	+	.await
	1286	+	.unwrap_or_default()
	1287	+	});
	1288	+	let status_of = \|t: &Target\| -> Option<String> {
	1289	+	let key = t.to_string();
	1290	+	rows.iter()
	1291	+	.find(\|(name, _)\| name == &key)
	1292	+	.map(\|(_, s)\| s.clone())
	1293	+	};
	1294	+	let not_green: Vec<String> = declared
	1295	+	.iter()
	1296	+	.filter(\|t\| **t != self.target) // the publishing target is the last mile
	1297	+	.filter(\|t\| status_of(t).as_deref() != Some("ok"))
	1298	+	.map(\|t\| format!("{t} ({})", status_of(t).unwrap_or_else(\|\| "no run".into())))
	1299	+	.collect();
	1300	+	anyhow::ensure!(
	1301	+	not_green.is_empty(),
	1302	+	"all-targets-green gate: refusing to publish {} {} — not green: {}",
	1303	+	self.app,
	1304	+	self.version,
	1305	+	not_green.join(", "),
	1306	+	);
	1307	+	Ok(())
	1308	+	}
	1309	+
1008	1310		fn publish(
1009	1311		self: &Arc<Self>,
1010	1312		channel: &str,
			@@ -1021,6 +1323,15 @@ impl RecipeCtx {
1021	1323		!self.is_cancelled(),
1022	1324		"build superseded by a newer request; refusing to publish"
1023	1325		);
	1326	+	// Opt-in all-targets-green gate: refuse a partial release. Every OTHER
	1327	+	// declared target of this (app, version) must have a successful latest
	1328	+	// run before this one ships, so macOS can't publish while windows is red
	1329	+	// or still building. The publishing target itself is the last mile (it
	1330	+	// reached publish, so its steps passed) and is not required to be green
	1331	+	// in the ledger yet.
	1332	+	if let Some(declared) = self.all_green_required.clone() {
	1333	+	self.assert_siblings_green(&declared)?;
	1334	+	}
1024	1335		let backend = self
1025	1336		.ota
1026	1337		.get(channel)
			@@ -1111,6 +1422,16 @@ impl RecipeCtx {
1111	1422		// publishers can't race the read-then-insert because the latest-wins slot
1112	1423		// (state::ActiveSlot) serializes them and a superseded run is cancelled
1113	1424		// before it reaches publish.
	1425	+	// The artifact's hash, recorded so the release ledger says exactly which
	1426	+	// bytes shipped. Prefer the digest computed at `collect`; fall back to
	1427	+	// hashing the file now (an absolute-path artifact never routed through
	1428	+	// `collect`). A hash failure must not fail an already-published release,
	1429	+	// so degrade to NULL rather than erroring.
	1430	+	let artifact_hash: Option<String> = artifact_path
	1431	+	.file_name()
	1432	+	.and_then(\|n\| n.to_str())
	1433	+	.and_then(\|n\| self.artifact_hashes.lock().unwrap().get(n).cloned())
	1434	+	.or_else(\|\| sha256_file(&artifact_path).ok());
1114	1435		let me = self.clone();
1115	1436		let (app_s, target_s, ver_s, chan_s) = (
1116	1437		app.to_string(),
			@@ -1121,13 +1442,14 @@ impl RecipeCtx {
1121	1442		self.rt
1122	1443		.block_on(async move {
1123	1444		sqlx::query(
1124		-	"INSERT OR IGNORE INTO releases (app, target, version, channel, published_at)
1125		-	VALUES (?, ?, ?, ?, ?)",
	1445	+	"INSERT OR IGNORE INTO releases (app, target, version, channel, artifact_hash, published_at)
	1446	+	VALUES (?, ?, ?, ?, ?, ?)",
1126	1447		)
1127	1448		.bind(app_s)
1128	1449		.bind(target_s)
1129	1450		.bind(ver_s)
1130	1451		.bind(chan_s)
	1452	+	.bind(artifact_hash)
1131	1453		.bind(Self::now())
1132	1454		.execute(&me.pool)
1133	1455		.await
			@@ -1370,6 +1692,7 @@ mod tests {
1370	1692		Arc::new(OtaRegistry::standard("https://makenot.work")),
1371	1693		tokio::runtime::Handle::current(),
1372	1694		cancel.clone(),
	1695	+	None,
1373	1696		));
1374	1697		// Cancelled: begin_step refuses before touching the DB (the ensure! is
1375	1698		// ahead of any block_on, so this is safe to call from the async test).
			@@ -1403,6 +1726,7 @@ mod tests {
1403	1726		Arc::new(OtaRegistry::standard("https://makenot.work")),
1404	1727		tokio::runtime::Handle::current(),
1405	1728		Arc::new(AtomicBool::new(false)),
	1729	+	None,
1406	1730		));
1407	1731		let engine = build_engine(ctx);
1408	1732		engine.eval::<String>("feature_flags()").unwrap()
			@@ -1650,6 +1974,145 @@ mod tests {
1650	1974		);
1651	1975		}
1652	1976
	1977	+	// ---- version-source cross-check (drift preflight) ----
	1978	+
	1979	+	fn ver(s: &str) -> Version {
	1980	+	Version::parse(s).unwrap()
	1981	+	}
	1982	+
	1983	+	#[test]
	1984	+	fn version_consistency_passes_when_all_sources_agree() {
	1985	+	let tmp = tempfile::tempdir().unwrap();
	1986	+	let repo = tmp.path();
	1987	+	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
	1988	+	std::fs::write(
	1989	+	repo.join("src-tauri/tauri.conf.json"),
	1990	+	r#"{"version":"0.5.0"}"#,
	1991	+	)
	1992	+	.unwrap();
	1993	+	std::fs::write(
	1994	+	repo.join("Cargo.toml"),
	1995	+	"[package]\nname = \"app\"\nversion = \"0.5.0\"\n",
	1996	+	)
	1997	+	.unwrap();
	1998	+	check_version_consistency(repo.to_str().unwrap(), None, &ver("0.5.0")).unwrap();
	1999	+	}
	2000	+
	2001	+	#[test]
	2002	+	fn version_consistency_flags_tauri_vs_cargo_drift() {
	2003	+	// The concrete finding: tauri.conf.json bumped to 0.5.0 but the root
	2004	+	// Cargo.toml left at 0.4.0. version_from_repo (one file) would miss it.
	2005	+	let tmp = tempfile::tempdir().unwrap();
	2006	+	let repo = tmp.path();
	2007	+	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
	2008	+	std::fs::write(
	2009	+	repo.join("src-tauri/tauri.conf.json"),
	2010	+	r#"{"version":"0.5.0"}"#,
	2011	+	)
	2012	+	.unwrap();
	2013	+	std::fs::write(
	2014	+	repo.join("Cargo.toml"),
	2015	+	"[package]\nname = \"app\"\nversion = \"0.4.0\"\n",
	2016	+	)
	2017	+	.unwrap();
	2018	+	let err =
	2019	+	check_version_consistency(repo.to_str().unwrap(), None, &ver("0.5.0")).unwrap_err();
	2020	+	let msg = format!("{err:#}");
	2021	+	assert!(msg.contains("Cargo.toml says 0.4.0"), "{msg}");
	2022	+	}
	2023	+
	2024	+	#[test]
	2025	+	fn version_consistency_flags_explicit_version_the_repo_does_not_reflect() {
	2026	+	let tmp = tempfile::tempdir().unwrap();
	2027	+	let repo = tmp.path();
	2028	+	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
	2029	+	std::fs::write(
	2030	+	repo.join("src-tauri/tauri.conf.json"),
	2031	+	r#"{"version":"0.5.0"}"#,
	2032	+	)
	2033	+	.unwrap();
	2034	+	let err =
	2035	+	check_version_consistency(repo.to_str().unwrap(), None, &ver("9.9.9")).unwrap_err();

Lines truncated

M bento/daemon/src/main.rs +20

			@@ -12,6 +12,24 @@ fn mnw_base_url() -> String {
12	12
13	13		#[tokio::main]
14	14		async fn main() -> Result<()> {
	15	+	// `--check-config`: load the daemon config + topology (which reads every
	16	+	// app's in-repo manifest) and exit 0, or fail with the parse error and a
	17	+	// non-zero code. Run before swapping in a freshly built bentod so a schema
	18	+	// change that can't parse the live config fails the install instead of
	19	+	// bricking the daemon — a per-app-config move once left it down for 20h.
	20	+	// Mirrors `sandod --check-config`.
	21	+	if std::env::args().skip(1).any(\|a\| a == "--check-config") {
	22	+	let cfg = config::Config::load()?;
	23	+	let topo = topology::Topology::load(&cfg.topology_path)?;
	24	+	println!(
	25	+	"bentod config OK: {} host(s), {} app(s) from {}",
	26	+	topo.hosts.len(),
	27	+	topo.app.len(),
	28	+	cfg.topology_path.display()
	29	+	);
	30	+	return Ok(());
	31	+	}
	32	+
15	33		tracing_subscriber::fmt()
16	34		.with_writer(std::io::stderr)
17	35		.with_env_filter(
			@@ -66,6 +84,7 @@ async fn main() -> Result<()> {
66	84
67	85		let executors = Arc::new(state::build_executors(&topo));
68	86		let syncs = Arc::new(state::build_syncs(&topo));
	87	+	let host_locks = state::build_host_locks(&topo);
69	88		let app_state = state::AppState {
70	89		pool,
71	90		topo,
			@@ -77,6 +96,7 @@ async fn main() -> Result<()> {
77	96		syncs,
78	97		active: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
79	98		api_token,
	99	+	host_locks,
80	100		};
81	101		// Disk retention: prune logs_root + dist_root to the newest N versions per
82	102		// app, at startup and every 6h, so neither root grows without bound. Skips

M bento/daemon/src/routes.rs +210 -3

			@@ -34,6 +34,7 @@ pub fn router(state: AppState) -> Router {
34	34		let open = Router::new()
35	35		.route("/state", get(get_state))
36	36		.route("/status.json", get(get_status_json))
	37	+	.route("/release/{app}/{version}", get(get_release))
37	38		.route("/logs/{app}/{version}/{target}/{step}", get(get_step_log))
38	39		.route("/events", get(events_ws));
39	40
			@@ -150,14 +151,28 @@ async fn get_state(State(s): State<AppState>) -> Result<Json<StateView>> {
150	151		///
151	152		/// Shared by `/state` and `/status.json` so the two cannot disagree about what
152	153		/// a build looks like.
	154	+	///
	155	+	/// The matrix is the per-`(app, version)` RELEASE view, not one `builds` row's
	156	+	/// targets: for each target, the latest `target_runs` row for this app+version
	157	+	/// across EVERY build of that version. A `/retry` inserts a fresh single-target
	158	+	/// build, so keying on `build_id` would collapse a 5-target release to the one
	159	+	/// retried row and lose the answer to "did every target of this version ship".
	160	+	/// Aggregating by `(app, version)` keeps the full matrix and folds the retry's
	161	+	/// newer result into just that target's cell.
153	162		async fn build_view(s: &AppState, b: &sqlx::sqlite::SqliteRow) -> Result<BuildView> {
154	163		let build_id: i64 = b.get("id");
	164	+	let app: String = b.get("app");
	165	+	let version: String = b.get("version");
155	166
156	167		let target_rows = sqlx::query(
157		-	"SELECT id, target, status, current_step, error FROM target_runs
158		-	WHERE build_id = ? ORDER BY target",
	168	+	"SELECT id, target, status, current_step, error FROM target_runs tr
	169	+	WHERE app = ?1 AND version = ?2
	170	+	AND id = (SELECT MAX(id) FROM target_runs
	171	+	WHERE app = ?1 AND version = ?2 AND target = tr.target)
	172	+	ORDER BY target",
159	173		)
160		-	.bind(build_id)
	174	+	.bind(&app)
	175	+	.bind(&version)
161	176		.fetch_all(&s.pool)
162	177		.await?;
163	178
			@@ -210,6 +225,81 @@ async fn get_status_json(State(s): State<AppState>) -> Result<Json<ops_status::P
210	225		Ok(Json(crate::status::payload(&view, chrono::Utc::now())))
211	226		}
212	227
	228	+	/// One release, answering "did every target of `{version}` ship?"
	229	+	#[derive(Serialize)]
	230	+	struct ReleaseView {
	231	+	app: String,
	232	+	version: String,
	233	+	/// Every target the app's manifest declares for this release.
	234	+	declared_targets: Vec<String>,
	235	+	/// The per-`(app, version)` matrix: latest run per target across every build
	236	+	/// (including retries) of this version.
	237	+	build: BuildView,
	238	+	/// Targets with a `releases` row at this version.
	239	+	published_targets: Vec<String>,
	240	+	/// Every declared target has a latest run of `ok` — the release built clean.
	241	+	all_targets_green: bool,
	242	+	/// `all_targets_green` AND every declared target is published.
	243	+	complete: bool,
	244	+	}
	245	+
	246	+	/// `GET /release/{app}/{version}` -- the release view for a SPECIFIC version, not
	247	+	/// just the latest. Unlike `/state`, a retry can't hide it and an unrelated
	248	+	/// newer build can't mask it; this is the durable "did 0.5.0 fully ship" query.
	249	+	async fn get_release(
	250	+	State(s): State<AppState>,
	251	+	Path((app, version)): Path<(String, String)>,
	252	+	) -> Result<Json<ReleaseView>> {
	253	+	let cfg = s
	254	+	.topo
	255	+	.app(&AppId::new(app.clone()))
	256	+	.ok_or_else(\|\| Error::BadRequest(format!("unknown app `{app}`")))?;
	257	+	let declared_targets: Vec<String> = cfg.targets.iter().map(ToString::to_string).collect();
	258	+
	259	+	// A representative build row for this exact version (newest wins for the
	260	+	// build-level fields; the target matrix is aggregated across all of them).
	261	+	let row = sqlx::query(
	262	+	"SELECT id, app, version, status, created_at FROM builds
	263	+	WHERE app = ? AND version = ? ORDER BY id DESC LIMIT 1",
	264	+	)
	265	+	.bind(&app)
	266	+	.bind(&version)
	267	+	.fetch_optional(&s.pool)
	268	+	.await?
	269	+	.ok_or(Error::NotFound)?;
	270	+	let build = build_view(&s, &row).await?;
	271	+
	272	+	let published_targets = sqlx::query_scalar::<_, String>(
	273	+	"SELECT DISTINCT target FROM releases WHERE app = ? AND version = ? ORDER BY target",
	274	+	)
	275	+	.bind(&app)
	276	+	.bind(&version)
	277	+	.fetch_all(&s.pool)
	278	+	.await?;
	279	+
	280	+	// Green = every declared target has a latest run that is `ok`.
	281	+	let all_targets_green = declared_targets.iter().all(\|d\| {
	282	+	build
	283	+	.targets
	284	+	.iter()
	285	+	.any(\|t\| &t.target == d && t.status == "ok")
	286	+	});
	287	+	let complete = all_targets_green
	288	+	&& declared_targets
	289	+	.iter()
	290	+	.all(\|d\| published_targets.contains(d));
	291	+
	292	+	Ok(Json(ReleaseView {
	293	+	app,
	294	+	version,
	295	+	declared_targets,
	296	+	build,
	297	+	published_targets,
	298	+	all_targets_green,
	299	+	complete,
	300	+	}))
	301	+	}
	302	+
213	303		/// One [`AppStatusView`] per app in the topology, name-ordered.
214	304		///
215	305		/// Every declared app appears whether or not it has ever been built: an app
			@@ -456,6 +546,7 @@ repo = "{}"
456	546		.unwrap();
457	547		let executors = Arc::new(crate::state::build_executors(&topo));
458	548		let syncs = Arc::new(crate::state::build_syncs(&topo));
	549	+	let host_locks = crate::state::build_host_locks(&topo);
459	550		AppState {
460	551		pool,
461	552		topo: Arc::new(topo),
			@@ -467,6 +558,7 @@ repo = "{}"
467	558		syncs,
468	559		active: Arc::new(Mutex::new(HashMap::new())),
469	560		api_token: None,
	561	+	host_locks,
470	562		}
471	563		}
472	564
			@@ -713,4 +805,119 @@ repo = "{}"
713	805		.unwrap();
714	806		assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
715	807		}
	808	+
	809	+	/// A two-target app + a two-host topology, for the release-view aggregation.
	810	+	async fn two_target_state(root: &std::path::Path) -> AppState {
	811	+	let cfg = Config::for_tests(root);
	812	+	let pool = crate::db::open(&cfg.db_path).await.unwrap();
	813	+	let repo = root.join("demo");
	814	+	std::fs::create_dir_all(&repo).unwrap();
	815	+	std::fs::write(
	816	+	repo.join("bento.toml"),
	817	+	"targets = [\"linux/x86_64\", \"macos/aarch64\"]\n",
	818	+	)
	819	+	.unwrap();
	820	+	let topo = Topology::from_str_for_tests(&format!(
	821	+	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
	822	+	[[host]]\nname = \"mbp\"\nssh = \"mbp\"\ntargets = [\"macos/aarch64\"]\n\
	823	+	[app.demo]\nrepo = \"{}\"\n",
	824	+	repo.display()
	825	+	))
	826	+	.unwrap();
	827	+	let executors = Arc::new(crate::state::build_executors(&topo));
	828	+	let syncs = Arc::new(crate::state::build_syncs(&topo));
	829	+	let host_locks = crate::state::build_host_locks(&topo);
	830	+	AppState {
	831	+	pool,
	832	+	topo: Arc::new(topo),
	833	+	cfg: Arc::new(cfg),
	834	+	prom: crate::metrics::test_handle(),
	835	+	events: crate::events::channel(),
	836	+	ota: Arc::new(OtaRegistry::standard("https://makenot.work")),
	837	+	executors,
	838	+	syncs,
	839	+	active: Arc::new(Mutex::new(HashMap::new())),
	840	+	api_token: None,
	841	+	host_locks,
	842	+	}
	843	+	}
	844	+
	845	+	async fn insert_target_run(pool: &sqlx::SqlitePool, ver: &str, target: &str, status: &str) {
	846	+	let bid: i64 = sqlx::query_scalar(
	847	+	"INSERT INTO builds (app, version, status, created_at) VALUES ('demo', ?, ?, '2026-07-23T00:00:00Z') RETURNING id",
	848	+	)
	849	+	.bind(ver)
	850	+	.bind(status)
	851	+	.fetch_one(pool)
	852	+	.await
	853	+	.unwrap();
	854	+	sqlx::query(
	855	+	"INSERT INTO target_runs (build_id, app, version, target, status, started_at)
	856	+	VALUES (?, 'demo', ?, ?, ?, '2026-07-23T00:00:00Z')",
	857	+	)
	858	+	.bind(bid)
	859	+	.bind(ver)
	860	+	.bind(target)
	861	+	.bind(status)
	862	+	.execute(pool)
	863	+	.await
	864	+	.unwrap();
	865	+	}
	866	+
	867	+	/// The audit's H2: `/retry` inserts a whole new single-target build, so a
	868	+	/// build-id-keyed view collapses a multi-target release to the retried row.
	869	+	/// The release view aggregates by (app, version), so a green retry of one
	870	+	/// target folds into that cell while the other target stays visible.
	871	+	#[tokio::test]
	872	+	async fn release_view_survives_a_single_target_retry() {
	873	+	let tmp = tempfile::tempdir().unwrap();
	874	+	let state = two_target_state(tmp.path()).await;
	875	+	let pool = state.pool.clone();
	876	+
	877	+	// Original release build: linux ok, macos failed.
	878	+	insert_target_run(&pool, "0.5.0", "linux/x86_64", "ok").await;
	879	+	insert_target_run(&pool, "0.5.0", "macos/aarch64", "failed").await;
	880	+	// Retry macos only — a fresh single-target build that succeeds.
	881	+	insert_target_run(&pool, "0.5.0", "macos/aarch64", "ok").await;
	882	+
	883	+	let app = router(state);
	884	+	let resp = app
	885	+	.oneshot(
	886	+	Request::builder()
	887	+	.uri("/release/demo/0.5.0")
	888	+	.body(Body::empty())
	889	+	.unwrap(),
	890	+	)
	891	+	.await
	892	+	.unwrap();
	893	+	assert_eq!(resp.status(), StatusCode::OK);
	894	+	let v: serde_json::Value = serde_json::from_str(&body_string(resp).await).unwrap();
	895	+
	896	+	// Both targets are present (the matrix did not collapse to the retry),
	897	+	// and macos shows the newer `ok`, not the earlier `failed`.
	898	+	let targets = v["build"]["targets"].as_array().unwrap();
	899	+	assert_eq!(targets.len(), 2, "full matrix preserved across the retry");
	900	+	let macos = targets
	901	+	.iter()
	902	+	.find(\|t\| t["target"] == "macos/aarch64")
	903	+	.unwrap();
	904	+	assert_eq!(macos["status"], "ok", "retry result wins the cell");
	905	+	assert_eq!(v["all_targets_green"], true);
	906	+	}
	907	+
	908	+	#[tokio::test]
	909	+	async fn release_view_404s_for_an_unbuilt_version() {
	910	+	let tmp = tempfile::tempdir().unwrap();
	911	+	let app = router(two_target_state(tmp.path()).await);
	912	+	let resp = app
	913	+	.oneshot(
	914	+	Request::builder()
	915	+	.uri("/release/demo/9.9.9")
	916	+	.body(Body::empty())
	917	+	.unwrap(),
	918	+	)
	919	+	.await
	920	+	.unwrap();
	921	+	assert_eq!(resp.status(), StatusCode::NOT_FOUND);
	922	+	}
716	923		}

M bento/daemon/src/runner.rs +346 -5

			@@ -71,6 +71,15 @@ pub async fn start_build(
71	71		version: Version,
72	72		targets: Vec<Target>,
73	73		) -> Result<i64> {
	74	+	// Preflight: every version source in the repo must agree with the version
	75	+	// being built, before any host pulls or compiles. `version_from_repo` reads
	76	+	// one file, so a tauri.conf.json/Cargo.toml (or explicit-version) mismatch
	77	+	// would otherwise sail through and file artifacts under the wrong version.
	78	+	if let Some(cfg) = state.topo.app(&app) {
	79	+	engine::check_version_consistency(&cfg.repo, cfg.version_path.as_deref(), &version)
	80	+	.context("version preflight")?;
	81	+	}
	82	+
74	83		let build_id: i64 = sqlx::query_scalar(
75	84		"INSERT INTO builds (app, version, status, created_at) VALUES (?, ?, 'running', ?) RETURNING id",
76	85		)
			@@ -254,6 +263,13 @@ async fn run_target(
254	263		.app(&app)
255	264		.map(\|a\| a.features.clone())
256	265		.unwrap_or_default();
	266	+	// The opt-in all-targets-green publish gate: pass the declared target set
	267	+	// (so `publish` can require every sibling green) only when the app turns it
	268	+	// on; otherwise None leaves independent per-target publishing unchanged.
	269	+	let all_green_required = state
	270	+	.topo
	271	+	.app(&app)
	272	+	.and_then(\|a\| a.require_all_targets.then(\|\| a.targets.clone()));
257	273
258	274		let ctx = Arc::new(RecipeCtx::new(
259	275		app.clone(),
			@@ -271,8 +287,21 @@ async fn run_target(
271	287		state.ota.clone(),
272	288		tokio::runtime::Handle::current(),
273	289		cancel,
	290	+	all_green_required,
274	291		));
275	292
	293	+	// Serialize per host: hold this host's lock for the whole recipe run so a
	294	+	// second target on the same box (goingson macos + ios both on mbp) can't
	295	+	// build concurrently in one checkout and corrupt the shared target/ +
	296	+	// keychain. Targets on different hosts hold different locks and still fan
	297	+	// out. Acquired at an await point, so a supersede-abort while queued drops
	298	+	// the task cleanly before it ever takes the lock. Not held during the
	299	+	// read-only preflight above.
	300	+	let _host_guard = match state.host_locks.get(&ctx.build_host).cloned() {
	301	+	Some(lock) => Some(lock.lock_owned().await),
	302	+	None => None,
	303	+	};
	304	+
276	305		// Rhai is synchronous; run the recipe (and its final step finalization) on
277	306		// a blocking thread so host functions can `block_on` without sitting on a
278	307		// runtime worker.
			@@ -398,11 +427,18 @@ async fn fail_target(
398	427		///
399	428		/// Event-driven: it joins the `JoinSet` rather than polling the DB. Each task's
400	429		/// outcome is supervised — a panicked task is logged (and its still-`running`
401		-	/// row is reconciled below), an aborted (superseded) task is expected. A
402		-	/// generous overall deadline bounds a wedged step (e.g. a hung ssh) so the build
403		-	/// can always finalize; on deadline the remaining tasks are aborted.
	430	+	/// row is reconciled below), an aborted (superseded) task is expected.
	431	+	///
	432	+	/// Per-step deadlines (see `engine::step_budget`) are the real bound on a wedged
	433	+	/// step now; this overall deadline is only a generous last-resort backstop for a
	434	+	/// hang outside a bounded command. When it trips it sets each still-running
	435	+	/// target's cooperative cancel FIRST — the blocking recipe bodies observe that
	436	+	/// at their next bounded command or step boundary and stop signing — then aborts
	437	+	/// the async wrappers. The old code aborted only the wrappers, which could not
	438	+	/// reach the blocking bodies, so a build was marked failed while codesign kept
	439	+	/// running on the mac.
404	440		async fn finalize_build(state: AppState, build_id: i64, mut set: tokio::task::JoinSet<()>) {
405		-	const MAX_WAIT: std::time::Duration = std::time::Duration::from_secs(2 * 60 * 60);
	441	+	const MAX_WAIT: std::time::Duration = std::time::Duration::from_secs(6 * 60 * 60);
406	442		let deadline = tokio::time::Instant::now() + MAX_WAIT;
407	443		loop {
408	444		match tokio::time::timeout_at(deadline, set.join_next()).await {
			@@ -418,8 +454,17 @@ async fn finalize_build(state: AppState, build_id: i64, mut set: tokio::task::Jo
418	454		Err(_elapsed) => {
419	455		tracing::error!(
420	456		build_id,
421		-	"finalize_build deadline hit; aborting remaining target tasks"
	457	+	"finalize_build backstop deadline hit; cancelling then aborting remaining targets"
422	458		);
	459	+	// Set the cooperative cancel on this build's still-running slots
	460	+	// BEFORE aborting, so the blocking recipe bodies actually stop
	461	+	// (abort() alone cannot reach a spawn_blocking body).
	462	+	{
	463	+	let active = state.active.lock().await;
	464	+	for slot in active.values().filter(\|s\| s.build_id == build_id) {
	465	+	slot.cancel.store(true, std::sync::atomic::Ordering::SeqCst);
	466	+	}
	467	+	}
423	468		set.abort_all();
424	469		while set.join_next().await.is_some() {}
425	470		break;
			@@ -602,6 +647,7 @@ mod tests {
602	647		fn test_state(pool: SqlitePool, topo: Topology, cfg: Config) -> AppState {
603	648		let executors = Arc::new(crate::state::build_executors(&topo));
604	649		let syncs = Arc::new(crate::state::build_syncs(&topo));
	650	+	let host_locks = crate::state::build_host_locks(&topo);
605	651		AppState {
606	652		pool,
607	653		topo: Arc::new(topo),
			@@ -613,6 +659,7 @@ mod tests {
613	659		syncs,
614	660		active: Arc::new(Mutex::new(HashMap::new())),
615	661		api_token: None,
	662	+	host_locks,
616	663		}
617	664		}
618	665
			@@ -1180,6 +1227,300 @@ repo = "{}"
1180	1227		count, 1,
1181	1228		"only the first (newer) publish should record a release"
1182	1229		);
	1230	+	// The recorded release carries the artifact's sha256 (64 hex chars),
	1231	+	// not a NULL — the ledger says which bytes shipped.
	1232	+	let hash: Option<String> =
	1233	+	sqlx::query_scalar("SELECT artifact_hash FROM releases WHERE version = '0.2.0'")
	1234	+	.fetch_one(&pool)
	1235	+	.await
	1236	+	.unwrap();
	1237	+	assert!(
	1238	+	hash.as_deref().is_some_and(\|h\| h.len() == 64),
	1239	+	"publish must record the artifact sha256, got {hash:?}"
	1240	+	);
	1241	+	}
	1242	+
	1243	+	/// The artifact-identity fix at collect: a stale, wrongly-versioned artifact
	1244	+	/// left in the output dir fails the collect instead of silently winning a
	1245	+	/// later glob. Here the build is 0.0.1 but the recipe produces a 9.9.9 file.
	1246	+	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
	1247	+	async fn collect_rejects_a_stale_versioned_artifact() {
	1248	+	let tmp = tempfile::tempdir().unwrap();
	1249	+	let root = tmp.path();
	1250	+	let repo = root.join("app");
	1251	+	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
	1252	+	std::fs::write(
	1253	+	repo.join("src-tauri/tauri.conf.json"),
	1254	+	r#"{"version":"0.0.1"}"#,
	1255	+	)
	1256	+	.unwrap();
	1257	+	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
	1258	+	std::fs::write(
	1259	+	repo.join("dist/recipes/linux.rhai"),
	1260	+	r#"
	1261	+	step("build");
	1262	+	let v = version_of("demo");
	1263	+	sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo-9.9.9.bin");
	1264	+	step("collect");
	1265	+	collect("fw13", "REPO/out/demo-9.9.9.bin", "demo", v);
	1266	+	"#
	1267	+	.replace("REPO", repo.to_str().unwrap()),
	1268	+	)
	1269	+	.unwrap();
	1270	+
	1271	+	let cfg = Config::for_tests(root);
	1272	+	let pool = crate::db::open(&cfg.db_path).await.unwrap();
	1273	+	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
	1274	+	let topo = Topology::from_str_for_tests(&format!(
	1275	+	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
	1276	+	pull_root = \"{repo}\"\n\n[app.demo]\nrepo = \"{repo}\"\n",
	1277	+	repo = repo.display()
	1278	+	))
	1279	+	.unwrap();
	1280	+	let state = test_state(pool.clone(), topo, cfg);
	1281	+	let build_id = start_build(
	1282	+	state.clone(),
	1283	+	AppId::new("demo"),
	1284	+	Version::parse("0.0.1").unwrap(),
	1285	+	vec!["linux/x86_64".parse().unwrap()],
	1286	+	)
	1287	+	.await
	1288	+	.unwrap();
	1289	+
	1290	+	let mut status = String::new();
	1291	+	let mut error = String::new();
	1292	+	for _ in 0..100 {
	1293	+	let row: Option<(String, Option<String>)> =
	1294	+	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
	1295	+	.bind(build_id)
	1296	+	.fetch_optional(&pool)
	1297	+	.await
	1298	+	.unwrap();
	1299	+	if let Some((s, e)) = row {
	1300	+	status = s;
	1301	+	error = e.unwrap_or_default();
	1302	+	if status != "running" {
	1303	+	break;
	1304	+	}
	1305	+	}
	1306	+	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
	1307	+	}
	1308	+	assert_eq!(
	1309	+	status, "failed",
	1310	+	"a mismatched-version artifact must fail collect"
	1311	+	);
	1312	+	assert!(
	1313	+	error.contains("stale artifact"),
	1314	+	"expected a stale-artifact error, got: {error}"
	1315	+	);
	1316	+	}
	1317	+
	1318	+	/// A command that runs past its step's deadline fails THAT step (and unwinds
	1319	+	/// the recipe) rather than wedging under the old whole-build guillotine. The
	1320	+	/// per-step budget is overridden to 1s here; the recipe then sleeps 30s, so
	1321	+	/// the step deadline — not the sleep — decides the outcome, quickly.
	1322	+	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
	1323	+	async fn a_step_that_exceeds_its_deadline_fails() {
	1324	+	let tmp = tempfile::tempdir().unwrap();
	1325	+	let root = tmp.path();
	1326	+	let repo = root.join("app");
	1327	+	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
	1328	+	std::fs::write(
	1329	+	repo.join("src-tauri/tauri.conf.json"),
	1330	+	r#"{"version":"0.0.1"}"#,
	1331	+	)
	1332	+	.unwrap();
	1333	+	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
	1334	+	std::fs::write(
	1335	+	repo.join("dist/recipes/linux.rhai"),
	1336	+	r#"
	1337	+	step("build");
	1338	+	sh_ok("fw13", "sleep 30");
	1339	+	"#,
	1340	+	)
	1341	+	.unwrap();
	1342	+
	1343	+	let mut cfg = Config::for_tests(root);
	1344	+	cfg.step_timeout_secs = Some(1); // every step's budget -> 1s
	1345	+	let pool = crate::db::open(&cfg.db_path).await.unwrap();
	1346	+	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
	1347	+	let topo = Topology::from_str_for_tests(&format!(
	1348	+	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
	1349	+	[app.demo]\nrepo = \"{}\"\n",
	1350	+	repo.display()
	1351	+	))
	1352	+	.unwrap();
	1353	+	let state = test_state(pool.clone(), topo, cfg);
	1354	+	let started = std::time::Instant::now();
	1355	+	let build_id = start_build(
	1356	+	state.clone(),
	1357	+	AppId::new("demo"),
	1358	+	Version::parse("0.0.1").unwrap(),
	1359	+	vec!["linux/x86_64".parse().unwrap()],
	1360	+	)
	1361	+	.await
	1362	+	.unwrap();
	1363	+
	1364	+	let mut status = String::new();
	1365	+	let mut error = String::new();
	1366	+	for _ in 0..100 {
	1367	+	let row: Option<(String, Option<String>)> =
	1368	+	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
	1369	+	.bind(build_id)
	1370	+	.fetch_optional(&pool)
	1371	+	.await
	1372	+	.unwrap();
	1373	+	if let Some((s, e)) = row {
	1374	+	status = s;
	1375	+	error = e.unwrap_or_default();
	1376	+	if status != "running" {
	1377	+	break;
	1378	+	}
	1379	+	}
	1380	+	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
	1381	+	}
	1382	+	assert_eq!(status, "failed", "a step past its deadline must fail");
	1383	+	assert!(
	1384	+	error.contains("per-step deadline"),
	1385	+	"expected a deadline error, got: {error}"
	1386	+	);
	1387	+	// The deadline (1s), not the 30s sleep, decided it — proof the command
	1388	+	// was actually interrupted rather than run to completion.
	1389	+	assert!(
	1390	+	started.elapsed() < std::time::Duration::from_secs(20),
	1391	+	"the step deadline must fire well before the sleep would finish"
	1392	+	);
	1393	+	}
	1394	+
	1395	+	/// A demo app that publishes linux and has the all-targets-green gate on; it
	1396	+	/// declares linux + macos, so publishing linux is gated on macos being green.
	1397	+	async fn gate_state(root: &std::path::Path) -> (AppState, sqlx::SqlitePool) {
	1398	+	let repo = root.join("app");
	1399	+	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
	1400	+	std::fs::write(
	1401	+	repo.join("src-tauri/tauri.conf.json"),
	1402	+	r#"{"version":"0.2.0"}"#,
	1403	+	)
	1404	+	.unwrap();
	1405	+	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
	1406	+	let artifact = repo.join("out/app.tar.gz");
	1407	+	std::fs::write(
	1408	+	repo.join("dist/recipes/linux.rhai"),
	1409	+	r#"
	1410	+	step("build");
	1411	+	sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
	1412	+	step("publish");
	1413	+	publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
	1414	+	"#
	1415	+	.replace("ARTIFACT", artifact.to_str().unwrap())
	1416	+	.replace("REPO", repo.to_str().unwrap()),
	1417	+	)
	1418	+	.unwrap();
	1419	+	std::fs::write(
	1420	+	repo.join("bento.toml"),
	1421	+	"targets = [\"linux/x86_64\", \"macos/aarch64\"]\nrequire_all_targets = true\n",
	1422	+	)
	1423	+	.unwrap();
	1424	+	let cfg = Config::for_tests(root);
	1425	+	let pool = crate::db::open(&cfg.db_path).await.unwrap();
	1426	+	let topo = Topology::from_str_for_tests(&format!(
	1427	+	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
	1428	+	[[host]]\nname = \"mbp\"\nssh = \"mbp\"\ntargets = [\"macos/aarch64\"]\n\
	1429	+	[app.demo]\nrepo = \"{}\"\n",
	1430	+	repo.display()
	1431	+	))
	1432	+	.unwrap();
	1433	+	(test_state(pool.clone(), topo, cfg), pool)
	1434	+	}
	1435	+
	1436	+	async fn await_target(pool: &sqlx::SqlitePool, build_id: i64) -> (String, String) {
	1437	+	for _ in 0..100 {
	1438	+	let row: Option<(String, Option<String>)> =
	1439	+	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
	1440	+	.bind(build_id)
	1441	+	.fetch_optional(pool)
	1442	+	.await
	1443	+	.unwrap();
	1444	+	if let Some((s, e)) = row
	1445	+	&& s != "running"
	1446	+	{
	1447	+	return (s, e.unwrap_or_default());
	1448	+	}
	1449	+	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
	1450	+	}
	1451	+	panic!("target never settled");
	1452	+	}
	1453	+
	1454	+	/// The all-targets-green gate blocks a partial release: linux tries to
	1455	+	/// publish while macos has no successful run, so publish is refused and the
	1456	+	/// target fails at its publish step.
	1457	+	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
	1458	+	async fn all_green_gate_blocks_publish_when_a_sibling_is_not_green() {
	1459	+	let tmp = tempfile::tempdir().unwrap();
	1460	+	let (state, pool) = gate_state(tmp.path()).await;
	1461	+	let build_id = start_build(
	1462	+	state,
	1463	+	AppId::new("demo"),
	1464	+	Version::parse("0.2.0").unwrap(),
	1465	+	vec!["linux/x86_64".parse().unwrap()],
	1466	+	)
	1467	+	.await
	1468	+	.unwrap();
	1469	+	let (status, error) = await_target(&pool, build_id).await;
	1470	+	assert_eq!(status, "failed", "a partial release must be blocked");
	1471	+	assert!(
	1472	+	error.contains("all-targets-green gate"),
	1473	+	"expected the gate to name itself, got: {error}"
	1474	+	);
	1475	+	// Nothing shipped.
	1476	+	let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
	1477	+	.fetch_one(&pool)
	1478	+	.await
	1479	+	.unwrap();
	1480	+	assert_eq!(count, 0, "a gated-off publish records no release");
	1481	+	}
	1482	+
	1483	+	/// With every sibling green, the gate lets the publish through. macos is
	1484	+	/// pre-recorded `ok` for this version, so linux's publish proceeds.
	1485	+	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
	1486	+	async fn all_green_gate_allows_publish_when_every_sibling_is_green() {
	1487	+	let tmp = tempfile::tempdir().unwrap();
	1488	+	let (state, pool) = gate_state(tmp.path()).await;
	1489	+	// Pre-record a green macos run at 0.2.0 (as if its target already built).
	1490	+	let bid: i64 = sqlx::query_scalar(
	1491	+	"INSERT INTO builds (app, version, status, created_at) VALUES ('demo','0.2.0','ok','2026-07-23T00:00:00Z') RETURNING id",
	1492	+	)
	1493	+	.fetch_one(&pool)
	1494	+	.await
	1495	+	.unwrap();
	1496	+	sqlx::query(
	1497	+	"INSERT INTO target_runs (build_id, app, version, target, status, started_at)
	1498	+	VALUES (?, 'demo', '0.2.0', 'macos/aarch64', 'ok', '2026-07-23T00:00:00Z')",
	1499	+	)
	1500	+	.bind(bid)
	1501	+	.execute(&pool)
	1502	+	.await
	1503	+	.unwrap();
	1504	+
	1505	+	let build_id = start_build(
	1506	+	state,
	1507	+	AppId::new("demo"),
	1508	+	Version::parse("0.2.0").unwrap(),
	1509	+	vec!["linux/x86_64".parse().unwrap()],
	1510	+	)
	1511	+	.await
	1512	+	.unwrap();
	1513	+	let (status, error) = await_target(&pool, build_id).await;
	1514	+	assert_eq!(
	1515	+	status, "ok",
	1516	+	"all siblings green -> publish proceeds ({error})"
	1517	+	);
	1518	+	let count: i64 =
	1519	+	sqlx::query_scalar("SELECT COUNT(*) FROM releases WHERE target = 'linux/x86_64'")
	1520	+	.fetch_one(&pool)
	1521	+	.await
	1522	+	.unwrap();
	1523	+	assert_eq!(count, 1, "linux published once the gate was satisfied");
1183	1524		}
1184	1525
1185	1526		/// The audit fix: a `build` step dispatched to a host whose executor lacks the

M bento/daemon/src/state.rs +55

			@@ -35,6 +35,26 @@ pub struct ActiveSlot {
35	35		/// handle); a finishing build reaps only the slots it still owns.
36	36		pub type ActiveBuilds = Arc<Mutex<HashMap<(AppId, Target), ActiveSlot>>>;
37	37
	38	+	/// One serialization lock per build host. A target holds its host's lock across
	39	+	/// the whole recipe run, so two targets that land on the same host never build
	40	+	/// concurrently in one checkout. Without it, goingson's `macos/aarch64` and
	41	+	/// `ios/universal` — both on mbp — run at once in `~/Code/Apps/goingson`:
	42	+	/// concurrent `git pull`, a shared `target/`, `release-ios.sh` rewriting
	43	+	/// `project.yml` mid-build, and (worst) a fixed-path build keychain that the
	44	+	/// second build deletes out from under the first's codesign.
	45	+	pub type HostLocks = Arc<HashMap<String, Arc<Mutex<()>>>>;
	46	+
	47	+	/// Build one serialization lock per host in the topology. Hosts are fixed at
	48	+	/// load, so the map is built once and never mutated.
	49	+	pub fn build_host_locks(topo: &Topology) -> HostLocks {
	50	+	Arc::new(
	51	+	topo.hosts
	52	+	.iter()
	53	+	.map(\|h\| (h.name.clone(), Arc::new(Mutex::new(()))))
	54	+	.collect(),
	55	+	)
	56	+	}
	57	+
38	58		#[derive(Clone)]
39	59		pub struct AppState {
40	60		pub pool: SqlitePool,
			@@ -60,6 +80,10 @@ pub struct AppState {
60	80		/// The value carries the owning `build_id` so a finished build reaps only
61	81		/// its own slots and never a superseding build's handle.
62	82		pub active: ActiveBuilds,
	83	+	/// One lock per host, held for a target's whole recipe run so two targets on
	84	+	/// the same host serialize instead of corrupting one shared checkout +
	85	+	/// keychain. See [`HostLocks`].
	86	+	pub host_locks: HostLocks,
63	87		}
64	88
65	89		/// Build one host's EXEC executor: `LocalExec` for `ssh = "local"`, `AgentRpc`
			@@ -266,6 +290,37 @@ mod tests {
266	290		);
267	291		}
268	292
	293	+	/// One serialization lock per host, and they are independent: holding mbp's
	294	+	/// lock (a macOS + iOS build serialize behind it) leaves fw13 free to build
	295	+	/// in parallel. This is the fan-out-across-hosts / serialize-within-a-host
	296	+	/// property the runner relies on.
	297	+	#[tokio::test]
	298	+	async fn host_locks_are_one_per_host_and_independent() {
	299	+	use crate::topology::Topology;
	300	+	let dir = tempfile::tempdir().unwrap();
	301	+	let repo = dir.path().join("goingson");
	302	+	std::fs::create_dir_all(&repo).unwrap();
	303	+	std::fs::write(repo.join("bento.toml"), "targets = [\"macos/aarch64\"]\n").unwrap();
	304	+	let topo = Topology::from_str_for_tests(&format!(
	305	+	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
	306	+	[[host]]\nname = \"mbp\"\nssh = \"mbp\"\ntargets = [\"macos/aarch64\"]\n\
	307	+	[app.goingson]\nrepo = \"{}\"\n",
	308	+	repo.display()
	309	+	))
	310	+	.unwrap();
	311	+	let locks = build_host_locks(&topo);
	312	+	assert_eq!(locks.len(), 2);
	313	+	let mbp = locks.get("mbp").expect("mbp lock").clone();
	314	+	let _held = mbp.clone().lock_owned().await;
	315	+	// Same host: a second acquire cannot proceed while the first is held.
	316	+	assert!(mbp.try_lock().is_err(), "same-host builds must serialize");
	317	+	// Different host: unaffected — it builds in parallel.
	318	+	assert!(
	319	+	locks.get("fw13").expect("fw13 lock").try_lock().is_ok(),
	320	+	"a different host must not be blocked"
	321	+	);
	322	+	}
	323	+
269	324		#[test]
270	325		fn agent_host_executor_permits_sign() {
271	326		// The mac host: agent transport, widened grant. Proves AgentRpc is

M bento/daemon/src/topology.rs +9

			@@ -86,6 +86,8 @@ struct AppManifest {
86	86		version_path: Option<String>,
87	87		#[serde(default)]
88	88		features: Vec<String>,
	89	+	#[serde(default)]
	90	+	require_all_targets: bool,
89	91		targets: Vec<Target>,
90	92		}
91	93
			@@ -182,6 +184,12 @@ pub struct AppConfig {
182	184		/// fails silently, producing a binary that builds and runs with a feature
183	185		/// quietly absent.
184	186		pub features: Vec<String>,
	187	+	/// Opt-in: `publish` refuses unless every declared target of this
	188	+	/// `(app, version)` has a successful latest run — the all-targets-green gate
	189	+	/// that stops a partial release (macOS published while windows is red or
	190	+	/// still building). Off by default so independent per-target publishing
	191	+	/// keeps working; a release that must ship as a set turns it on.
	192	+	pub require_all_targets: bool,
185	193		/// Targets this app ships.
186	194		pub targets: Vec<Target>,
187	195		}
			@@ -233,6 +241,7 @@ impl Topology {
233	241		recipe_dir: m.recipe_dir,
234	242		version_path: m.version_path,
235	243		features: m.features,
	244	+	require_all_targets: m.require_all_targets,
236	245		targets: m.targets,
237	246		},
238	247		);

M bento/deploy/bento-daemon.toml.example +6

			@@ -24,3 +24,9 @@ dist_root = "/home/max/Dist"
24	24
25	25		# Per-step run logs: <logs_root>/<app>/<version>/<target>/<step>.log
26	26		logs_root = "/home/max/.local/state/bento/logs"
	27	+
	28	+	# Optional: override the per-step wall-clock budget (seconds) for EVERY step,
	29	+	# replacing the per-kind defaults (build 90m, notarize 60m, sign 15m, ...). A
	30	+	# step that runs past its budget fails that step and unwinds the recipe, so a
	31	+	# hung command can't wedge a build. Leave unset to use the defaults.
	32	+	# step_timeout_secs = 5400

M bento/deploy/bentod.service +11 -2

			@@ -13,8 +13,12 @@
13	13		# systemctl --user enable --now bentod
14	14		#
15	15		# Watch: journalctl --user -u bentod -f
16		-	# Deploy a new bentod: build, copy to ~/.local/bin/bentod, `systemctl --user
17		-	# restart bentod` (no sudo — that's the point of a user service).
	16	+	# Deploy a new bentod: build, copy to ~/.local/bin/bentod, then validate the
	17	+	# fresh binary against the live config BEFORE restarting:
	18	+	# ~/.local/bin/bentod --check-config && systemctl --user restart bentod
	19	+	# (no sudo — that's the point of a user service). The unit also runs the same
	20	+	# --check-config as ExecStartPre, so a bad config fails the start rather than
	21	+	# crash-looping.
18	22		[Unit]
19	23		Description=Bento app build controller
20	24		After=network-online.target
			@@ -22,6 +26,11 @@ Wants=network-online.target
22	26
23	27		[Service]
24	28		Type=simple
	29	+	# Validate the config (daemon file + every app's in-repo manifest) before each
	30	+	# start. A binary or schema change that can't parse the live config fails the
	31	+	# unit here instead of crash-looping — the guard a per-app-config move needed
	32	+	# when it bricked bentod for 20h. Same check runs at install (see README).
	33	+	ExecStartPre=%h/.local/bin/bentod --check-config
25	34		ExecStart=%h/.local/bin/bentod
26	35		Restart=on-failure
27	36		RestartSec=5