max / makenotwork

139.2 KB · 3658 lines History Blame Raw

1	//! Build orchestration: fan a `(app, version)` out across its targets, each
2	//! running its recipe concurrently on the host that can build it.
3	//!
4	//! A build inserts one `builds` row, then spawns one task per target. Each
5	//! target task registers itself in the single-slot guard (a newer build for
6	//! the same `(app, target)` aborts the in-flight one — latest wins, but other
7	//! targets keep running, which is the fan-out), then runs the recipe.
8
9	use crate::domain::{AppId, Status, Step, Target, Version};
10	use crate::engine::{self, RecipeCtx};
11	use crate::events::{self, Event};
12	use crate::state::AppState;
13	use anyhow::{Context, Result};
14	use ops_exec::{Action, LogSink, Step as OpStep};
15	use std::path::PathBuf;
16	use std::sync::Arc;
17
18	/// A [`LogSink`] that drops what it's handed — the release preflight runs git on
19	/// each host for its exit code and (via a separate `rev-parse`) the sha in
20	/// `RunOutput`, not for a streamed log, so there is no step to stream into.
21	struct DiscardSink;
22
23	#[async_trait::async_trait]
24	impl LogSink for DiscardSink {
25	async fn write_chunk(&mut self, _bytes: &[u8]) {}
26	}
27
28	/// Preflight barrier: pin every host that will build a target for this release
29	/// to the tag `v<version>` and refuse the build unless they all report the SAME
30	/// commit. Recipes used to `git pull --ff-only` per host, so `mbp`/`astra`/`fw13`
31	/// each built whatever `main` was at pull time and the artifacts were filed
32	/// under the daemon host's version. This runs before any target task spawns, so
33	/// a mixed-source release is stopped before a single artifact is built.
34	///
35	/// "All" means all: a host that did not report is a refusal, not an abstention.
36	/// Comparing only the hosts that answered proves agreement among those, which is
37	/// not the claim the barrier is making.
38	///
39	/// Returns the branch each host was on before it was pinned, for
40	/// [`restore_branches`] to put back once the build settles, and the commit they
41	/// all agreed on. A host that is ALREADY detached has no branch to return to,
42	/// and is refused: that state means some earlier release never cleaned up, and
43	/// any commits made in the meantime are sitting on no branch at all.
44	///
45	/// The agreed sha is the release's provenance, and this is the only place it is
46	/// known to be one value rather than a per-host answer. Resolving it here and
47	/// carrying it forward is what lets an artifact record name the source it was
48	/// built from without asking a build host to be honest about it afterwards.
49	async fn pin_release(
50	state: &AppState,
51	app: &AppId,
52	version: &Version,
53	targets: &[Target],
54	) -> Result<Pinned> {
55	let cfg = state
56	.topo
57	.app(app)
58	.ok_or_else(\|\| anyhow::anyhow!("unknown app `{app}`"))?;
59	// Spelled per app: a repo holding one product tags `v0.4.1`, a repo holding
60	// several tags `pom-v0.4.1`. See `AppConfig::tag_format`.
61	let tag = cfg.tag_for(version);
62
63	// The distinct hosts across the requested targets (order-stable).
64	//
65	// A target whose host the topology cannot resolve is refused rather than
66	// skipped. Skipping it would leave that target out of the barrier while it
67	// still builds, which is the shape of hole this whole preflight exists to
68	// close: agreement proven among some hosts reads as agreement among all.
69	// `resolve_targets` rejects an unbuildable target before `/build`, so this
70	// is unreachable through the API and cheap to state anyway.
71	let mut hosts: Vec<String> = Vec::new();
72	for t in targets {
73	let h = state
74	.topo
75	.host_for(*t)
76	.ok_or_else(\|\| anyhow::anyhow!("release preflight: no host can build {t}"))?;
77	if !hosts.contains(&h.name) {
78	hosts.push(h.name.clone());
79	}
80	}
81	anyhow::ensure!(
82	!hosts.is_empty(),
83	"release preflight: no build hosts for v{version}; refusing to build a release \
84	nothing was pinned for"
85	);
86
87	let mut shas: Vec<(String, String)> = Vec::new();
88	let mut branches: Vec<(String, String)> = Vec::new();
89	for host in &hosts {
90	let exec = state
91	.executors
92	.get(host)
93	.ok_or_else(\|\| anyhow::anyhow!("no executor for host `{host}`"))?;
94	let mut sink = DiscardSink;
95	// Resolved per host, not once for the set: the checkouts need not be at
96	// the same path on every machine, and pinning windows-x86 with the unix
97	// path failed on the first git command of the release.
98	let repo = cfg.repo_for(host);
99
100	// Read the branch BEFORE pinning, while there is still one to read.
101	let branch_cmd = OpStep::shell(Action::Build, engine::git_current_branch_cmd(repo));
102	let out = exec
103	.run_streaming(&branch_cmd, &mut sink)
104	.await
105	.with_context(\|\| format!("release preflight: reading branch on `{host}`"))?;
106	let branch = String::from_utf8_lossy(&out.stdout).trim().to_string();
107	// `symbolic-ref -q` exits non-zero BOTH on a detached HEAD and when it
108	// could not read the repo at all (wrong path, no checkout, no git).
109	// Those need opposite responses, so a stderr that says anything is
110	// reported as itself rather than folded into the detached-HEAD advice.
111	let why = String::from_utf8_lossy(&out.stderr).trim().to_string();
112	anyhow::ensure!(
113	out.status.success() \|\| why.is_empty(),
114	"release preflight: reading the branch of `{repo}` on `{host}` failed: {why}"
115	);
116	anyhow::ensure!(
117	out.status.success() && !branch.is_empty(),
118	"release preflight: `{repo}` on `{host}` is on a detached HEAD, so there is \
119	no branch to restore it to after the release. An earlier release left it \
120	that way; any commits made since are on no branch and may exist nowhere \
121	else. Reattach it (`git checkout <branch>`, fast-forwarding if the commits \
122	should be kept) before releasing."
123	);
124	branches.push((host.clone(), branch));
125
126	// A dirty tree defeats the pin silently, which is worse than not pinning
127	// at all. `git checkout <tag>` does not fail on local modifications to
128	// files whose content is unchanged in the tag — it succeeds and KEEPS
129	// them. So a host with edits in the working tree builds those edits
130	// while reporting the tagged sha, and a second host with a clean tree
131	// builds something else. The rev-parse barrier below cannot see it:
132	// both hosts genuinely are on the same commit. Two architectures, two
133	// different binaries, one tag containing neither.
134	let dirty_cmd = OpStep::shell(Action::Build, engine::git_dirty_cmd(repo));
135	let out = exec
136	.run_streaming(&dirty_cmd, &mut sink)
137	.await
138	.with_context(\|\| format!("release preflight: reading tree state on `{host}`"))?;
139	let dirty = String::from_utf8_lossy(&out.stdout);
140	let dirty: Vec<&str> = dirty
141	.lines()
142	.map(str::trim)
143	.filter(\|l\| !l.is_empty())
144	.collect();
145	anyhow::ensure!(
146	dirty.is_empty(),
147	"release preflight: `{repo}` on `{host}` has uncommitted changes to tracked \
148	files, so the build would not be the tagged commit:\n {}\nCommit or stash \
149	them before releasing.",
150	dirty.join("\n "),
151	);
152
153	// Advisory: `fetch --all` is non-zero if any one of a repo's remotes is
154	// unreachable, and blaming the tag for a dead mirror is what made this
155	// preflight misreport. Only the checkout below is allowed to fail.
156	let fetch = OpStep::shell(Action::Build, engine::git_fetch_cmd(repo));
157	let _ = exec.run_streaming(&fetch, &mut sink).await;
158
159	let checkout = OpStep::shell(Action::Build, engine::git_checkout_tag_cmd(repo, &tag));
160	let out = exec
161	.run_streaming(&checkout, &mut sink)
162	.await
163	.with_context(\|\| format!("release preflight: checkout on `{host}`"))?;
164	if !out.status.success() {
165	// Ask git why before telling the operator. The two causes need
166	// opposite responses: tag-and-push, versus clean the working tree.
167	let probe = OpStep::shell(Action::Build, engine::git_tag_exists_cmd(repo, &tag));
168	let tag_exists = exec
169	.run_streaming(&probe, &mut sink)
170	.await
171	.is_ok_and(\|o\| o.status.success());
172	anyhow::bail!(
173	"release preflight: `git checkout {tag}` failed on `{host}`: {}",
174	engine::checkout_failure_reason(&tag, tag_exists)
175	);
176	}
177
178	let rev = OpStep::shell(Action::Build, engine::git_rev_parse_cmd(repo));
179	let out = exec
180	.run_streaming(&rev, &mut sink)
181	.await
182	.with_context(\|\| format!("release preflight: rev-parse on `{host}`"))?;
183	anyhow::ensure!(
184	out.status.success(),
185	"release preflight: `git rev-parse HEAD` failed on `{host}`"
186	);
187	let sha = String::from_utf8_lossy(&out.stdout).trim().to_string();
188	shas.push((host.clone(), sha));
189	}
190
191	// The barrier: every participating host must have reported, and every report
192	// must be the same commit. Reporting is checked first and separately from
193	// agreement, because a barrier that compares only what it received cannot
194	// tell unanimity from silence — three hosts agreeing while a fourth was
195	// never asked is exactly the mixed-source release this refuses.
196	let missing: Vec<&str> = hosts
197	.iter()
198	.filter(\|h\| !shas.iter().any(\|(sh, _)\| sh == *h))
199	.map(String::as_str)
200	.collect();
201	anyhow::ensure!(
202	missing.is_empty(),
203	"release preflight: {} did not report a commit for v{version}; refusing to build \
204	a release the barrier cannot vouch for",
205	missing.join(", "),
206	);
207	let empty_shas: Vec<&str> = shas
208	.iter()
209	.filter(\|(_, sha)\| sha.is_empty())
210	.map(\|(h, _)\| h.as_str())
211	.collect();
212	anyhow::ensure!(
213	empty_shas.is_empty(),
214	"release preflight: `git rev-parse HEAD` returned nothing on {} for v{version}",
215	empty_shas.join(", "),
216	);
217
218	let (first_host, first_sha) = &shas[0];
219	let mismatch: Vec<String> = shas
220	.iter()
221	.filter(\|(_, sha)\| sha != first_sha)
222	.map(\|(h, sha)\| format!("{h}={}", short(sha)))
223	.collect();
224	anyhow::ensure!(
225	mismatch.is_empty(),
226	"release preflight: build hosts are on different commits for v{version} \
227	({}={}, {}); refusing to build a release from mixed sources",
228	first_host,
229	short(first_sha),
230	mismatch.join(", "),
231	);
232	Ok(Pinned {
233	branches,
234	sha: first_sha.clone(),
235	})
236	}
237
238	/// What the preflight established: where each host's checkout was, and the one
239	/// commit every host is now on.
240	struct Pinned {
241	/// `(host, branch)` for [`restore_branches`].
242	branches: Vec<(String, String)>,
243	/// The commit all hosts agreed on. Empty when pinning is off (tests).
244	sha: String,
245	}
246
247	/// Put every host's checkout back on the branch [`pin_release`] found it on.
248	///
249	/// Best-effort and non-fatal: the release itself is already decided by the time
250	/// this runs, and failing a green build because a `git checkout` did not take
251	/// would be worse than the detached tree it is cleaning up. A failure is logged
252	/// loudly instead, because the state it leaves behind is the silent one.
253	async fn restore_branches(state: &AppState, app: &AppId, branches: &[(String, String)]) {
254	let Some(cfg) = state.topo.app(app) else {
255	return;
256	};
257	for (host, branch) in branches {
258	let Some(exec) = state.executors.get(host) else {
259	continue;
260	};
261	// Serialize against a target still holding this host for its recipe run,
262	// so the restore can't move the tree mid-build.
263	let _host_guard = match state.host_locks.get(host).cloned() {
264	Some(lock) => Some(lock.lock_owned().await),
265	None => None,
266	};
267	let mut sink = DiscardSink;
268	// The same per-host path `pin_release` detached; restoring the daemon's
269	// path on a host that keeps its checkout elsewhere would leave the real
270	// one detached and report success.
271	let repo = cfg.repo_for(host);
272	let step = OpStep::shell(Action::Build, engine::git_restore_branch_cmd(repo, branch));
273	match exec.run_streaming(&step, &mut sink).await {
274	Ok(out) if out.status.success() => {
275	tracing::debug!(%host, %branch, "restored checkout to its branch");
276	}
277	Ok(_) => tracing::error!(
278	%host, %branch, %repo,
279	"could not restore the checkout to its branch; it is left DETACHED at the \
280	release tag, and commits made there will belong to no branch"
281	),
282	Err(e) => tracing::error!(
283	%host, %branch, %repo, error = %e,
284	"could not restore the checkout to its branch; it is left DETACHED at the \
285	release tag, and commits made there will belong to no branch"
286	),
287	}
288	}
289	}
290
291	/// First 12 chars of a sha for a readable error.
292	fn short(sha: &str) -> &str {
293	sha.get(..12).unwrap_or(sha)
294	}
295
296	/// Resolve the version to build: explicit, or read from `tauri.conf.json`.
297	///
298	/// Returns typed errors so the route maps user mistakes (unknown app, bad
299	/// version string) to 400, and only genuine daemon-side failures (reading the
300	/// app's `tauri.conf.json`) to 500.
301	pub fn resolve_version(
302	state: &AppState,
303	app: &AppId,
304	explicit: Option<String>,
305	) -> crate::error::Result<Version> {
306	use crate::error::Error;
307	if let Some(v) = explicit {
308	return Version::parse(&v).map_err(Error::BadRequest);
309	}
310	let cfg = state
311	.topo
312	.app(app)
313	.ok_or_else(\|\| Error::BadRequest(format!("unknown app `{app}`")))?;
314	engine::version_from_repo(&cfg.repo, cfg.version_path.as_deref()).map_err(Error::Other)
315	}
316
317	/// Validate + default the target list against what the app ships. Unknown app,
318	/// an unshipped target, or a target no host can build are all client errors
319	/// (400), not server errors (500).
320	pub fn resolve_targets(
321	state: &AppState,
322	app: &AppId,
323	requested: Vec<Target>,
324	) -> crate::error::Result<Vec<Target>> {
325	use crate::error::Error;
326	let cfg = state
327	.topo
328	.app(app)
329	.ok_or_else(\|\| Error::BadRequest(format!("unknown app `{app}`")))?;
330	if requested.is_empty() {
331	return Ok(cfg.targets.clone());
332	}
333	for t in &requested {
334	if !cfg.targets.contains(t) {
335	return Err(Error::BadRequest(format!(
336	"app `{app}` does not ship target {t}"
337	)));
338	}
339	if state.topo.host_for(*t).is_none() {
340	return Err(Error::BadRequest(format!("no host can build {t}")));
341	}
342	}
343	Ok(requested)
344	}
345
346	/// Insert the build row and spawn per-target tasks. Returns the build id.
347	pub async fn start_build(
348	state: AppState,
349	app: AppId,
350	version: Version,
351	targets: Vec<Target>,
352	) -> Result<i64> {
353	// Preflight: every version source in the repo must agree with the version
354	// being built, before any host pulls or compiles. `version_from_repo` reads
355	// one file, so a tauri.conf.json/Cargo.toml (or explicit-version) mismatch
356	// would otherwise sail through and file artifacts under the wrong version.
357	if let Some(cfg) = state.topo.app(&app) {
358	engine::check_version_consistency(&cfg.repo, cfg.version_path.as_deref(), &version)
359	.context("version preflight")?;
360	}
361
362	// Pin every build host to the release tag and verify they agree, before any
363	// target task spawns. Off in tests (their repos aren't git checkouts).
364	let pinned = if state.cfg.pin_release_sha {
365	pin_release(&state, &app, &version, &targets)
366	.await
367	.context("release preflight")?
368	} else {
369	Pinned {
370	branches: Vec::new(),
371	sha: String::new(),
372	}
373	};
374
375	let build_id: i64 = sqlx::query_scalar(
376	"INSERT INTO builds (app, version, status, created_at) VALUES (?, ?, 'running', ?) RETURNING id",
377	)
378	.bind(app.as_str())
379	.bind(version.to_string())
380	.bind(chrono::Utc::now().to_rfc3339())
381	.fetch_one(&state.pool)
382	.await
383	.context("insert build")?;
384
385	events::emit(
386	&state.events,
387	Event::BuildRequested {
388	app: app.clone(),
389	version: version.clone(),
390	targets: targets.clone(),
391	},
392	);
393	crate::metrics::build_started();
394
395	// Spawn every target into a JoinSet so finalize_build can await completion
396	// event-driven (no DB polling) and observe each task's outcome (panic vs
397	// clean) for supervision.
398	let mut set: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
399	for target in targets {
400	let app = app.clone();
401	let version = version.clone();
402	let key = (app.clone(), target);
403	let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
404
405	// Latest-wins, done as ONE critical section: supersede the prior occupant
406	// (cooperatively cancel its recipe + abort its task) and install ours
407	// without ever releasing the lock, so two concurrent /build+/retry for the
408	// same (app, target) can't both pass "nothing to abort" and run together.
409	let mut active = state.active.lock().await;
410	if let Some(prev) = active.remove(&key)
411	&& !prev.abort.is_finished()
412	{
413	// The recipe runs on a blocking thread; abort() alone can't stop it,
414	// so set the cooperative flag the engine checks at step boundaries.
415	prev.cancel.store(true, std::sync::atomic::Ordering::SeqCst);
416	prev.abort.abort();
417	events::emit(
418	&state.events,
419	Event::TargetAborted {
420	app: app.clone(),
421	target,
422	},
423	);
424	}
425	let abort = set.spawn(run_target(
426	state.clone(),
427	build_id,
428	app,
429	version,
430	target,
431	cancel.clone(),
432	pinned.sha.clone(),
433	));
434	active.insert(
435	key,
436	crate::state::ActiveSlot {
437	build_id,
438	abort,
439	cancel,
440	},
441	);
442	crate::metrics::set_in_flight(active.len());
443	}
444
445	// Mark the build done once all target tasks settle, and put the pinned
446	// checkouts back on their branches. Spawned so /build returns immediately.
447	tokio::spawn(finalize_build(state, build_id, set, app, pinned.branches));
448	Ok(build_id)
449	}
450
451	/// Run one target's recipe end to end, updating its `target_runs` row. `cancel`
452	/// is set when a newer build supersedes this one; the engine checks it at step
453	/// boundaries and before publish so a superseded recipe can't advance or ship.
454	async fn run_target(
455	state: AppState,
456	build_id: i64,
457	app: AppId,
458	version: Version,
459	target: Target,
460	cancel: Arc<std::sync::atomic::AtomicBool>,
461	pinned_sha: String,
462	) {
463	let started = std::time::Instant::now();
464	let target_run_id: i64 = match sqlx::query_scalar(
465	"INSERT INTO target_runs (build_id, app, version, target, status, started_at)
466	VALUES (?, ?, ?, ?, 'running', ?) RETURNING id",
467	)
468	.bind(build_id)
469	.bind(app.as_str())
470	.bind(version.to_string())
471	.bind(target.to_string())
472	.bind(chrono::Utc::now().to_rfc3339())
473	.fetch_one(&state.pool)
474	.await
475	{
476	Ok(id) => id,
477	Err(e) => {
478	tracing::error!(%app, %target, error = %e, "could not create target_run");
479	return;
480	}
481	};
482
483	events::emit(
484	&state.events,
485	Event::TargetStart {
486	app: app.clone(),
487	version: version.clone(),
488	target,
489	},
490	);
491
492	let recipe_src = match read_recipe(&state, &app, target) {
493	Ok(s) => s,
494	Err(e) => {
495	fail_target(
496	&state,
497	target_run_id,
498	&app,
499	&version,
500	target,
501	Step::Checkout,
502	&format!("{e:#}"),
503	)
504	.await;
505	crate::metrics::target_finished(
506	&target.to_string(),
507	"failed",
508	started.elapsed().as_secs_f64(),
509	);
510	return;
511	}
512	};
513
514	// Pre-flight the build host before running the recipe. For local/ssh hosts
515	// this is a no-op; for the agent host (macOS in-session signing) it hits
516	// `/health` so a dead ops-agent fails here with a clear message — matching
517	// what the driver does — instead of erroring opaquely on the recipe's first
518	// dispatch to that host.
519	if let Some(host) = state.topo.host_for(target)
520	&& let Some(exec) = state.executors.get(&host.name)
521	&& let Err(e) = exec.preflight().await
522	{
523	fail_target(
524	&state,
525	target_run_id,
526	&app,
527	&version,
528	target,
529	Step::Checkout,
530	&format!("{e:#}"),
531	)
532	.await;
533	crate::metrics::target_finished(
534	&target.to_string(),
535	"failed",
536	started.elapsed().as_secs_f64(),
537	);
538	return;
539	}
540
541	// Host + checkout path for this target, resolved from the topology. Both are
542	// exposed to the recipe (`build_host()` / `repo()`); a missing host here is
543	// unreachable (resolve_targets rejected unbuildable targets before /build),
544	// but fall back rather than panic.
545	let build_host = state
546	.topo
547	.host_for(target)
548	.map(\|h\| h.name.clone())
549	.unwrap_or_default();
550	let build_host_ssh = state
551	.topo
552	.host_for(target)
553	.map(\|h\| h.ssh.clone())
554	.unwrap_or_default();
555	let repo = state
556	.topo
557	.app(&app)
558	.map(\|a\| a.repo.clone())
559	.unwrap_or_default();
560	let repo_by_host = state
561	.topo
562	.app(&app)
563	.map(\|a\| a.repo_by_host.clone())
564	.unwrap_or_default();
565	let features = state
566	.topo
567	.app(&app)
568	.map(\|a\| a.features.clone())
569	.unwrap_or_default();
570	let tag = state
571	.topo
572	.app(&app)
573	.map_or_else(\|\| format!("v{version}"), \|a\| a.tag_for(&version));
574	// The opt-in all-targets-green publish gate: pass the declared target set
575	// (so `publish` can require every sibling green) only when the app turns it
576	// on; otherwise None leaves independent per-target publishing unchanged.
577	let all_green_required = state
578	.topo
579	.app(&app)
580	.and_then(\|a\| a.require_all_targets.then(\|\| a.targets.clone()));
581
582	// A service's install destination for THIS target, plus an executor for it.
583	//
584	// The executor is added to a per-run copy of the map rather than to the
585	// daemon-wide one, so a deploy grant on a production host exists only for
586	// the duration of the run that needs it and only for the app that declared
587	// it. Nothing else can address that host: it is not in the topology, so no
588	// other app's recipe can name it.
589	let deploy = state
590	.topo
591	.app(&app)
592	.and_then(\|a\| a.deploy_for(target))
593	.cloned();
594	let execs = match &deploy {
595	Some(d) => {
596	let mut map = (*state.executors).clone();
597	map.insert(d.host.clone(), crate::state::build_deploy_executor(d));
598	Arc::new(map)
599	}
600	None => state.executors.clone(),
601	};
602
603	let ctx = Arc::new(
604	RecipeCtx::new(
605	app.clone(),
606	version.clone(),
607	target,
608	build_host,
609	build_host_ssh,
610	tag.clone(),
611	repo,
612	features,
613	// Gates the `verify` step's capability: a library's crate preflight is
614	// not an app's Gatekeeper check. See engine::action_for.
615	state.topo.app(&app).map(\|a\| a.kind).unwrap_or_default(),
616	target_run_id,
617	execs,
618	state.syncs.clone(),
619	deploy,
620	state.pool.clone(),
621	state.events.clone(),
622	state.cfg.clone(),
623	state.ota.clone(),
624	tokio::runtime::Handle::current(),
625	cancel,
626	all_green_required,
627	)
628	.with_repo_by_host(repo_by_host),
629	);
630
631	// Serialize per host: hold this host's lock for the whole recipe run so a
632	// second target on the same box (goingson macos + ios both on mbp) can't
633	// build concurrently in one checkout and corrupt the shared target/ +
634	// keychain. Targets on different hosts hold different locks and still fan
635	// out. Acquired at an await point, so a supersede-abort while queued drops
636	// the task cleanly before it ever takes the lock. Not held during the
637	// read-only preflight above.
638	let _host_guard = match state.host_locks.get(&ctx.build_host).cloned() {
639	Some(lock) => Some(lock.lock_owned().await),
640	None => None,
641	};
642
643	// Rhai is synchronous; run the recipe (and its final step finalization) on
644	// a blocking thread so host functions can `block_on` without sitting on a
645	// runtime worker.
646	let ctx_run = ctx.clone();
647	let outcome = tokio::task::spawn_blocking(move \|\| {
648	let engine = engine::build_engine(&ctx_run);
649	let res = engine.run(&recipe_src);
650	let last_step = ctx_run.current_step();
651	match &res {
652	Ok(()) => {
653	let _ = ctx_run.finish_step(Status::Ok);
654	}
655	Err(_) => {
656	let _ = ctx_run.finish_step(Status::Failed);
657	}
658	}
659	res.map_err(\|e\| (last_step, e.to_string()))
660	})
661	.await;
662
663	// Write the artifact record before the run is stamped terminal. It describes
664	// what was collected, so it runs whether the recipe succeeded or failed: a
665	// build that signed and collected an artifact and then failed at `publish`
666	// still produced bytes somebody may want the provenance of. Emit-only and
667	// non-fatal; nothing reads it yet.
668	let record_path = crate::artifact_record::emit(&state, &ctx, &pinned_sha).await;
669
670	match outcome {
671	Ok(Ok(())) => {
672	// Hand the artifact to Sando before the run is stamped ok, so a
673	// target reported green is one whose bytes reached the controller
674	// that decides whether they ship. Only on success: an artifact from
675	// a failed recipe is exactly what should not be offered for a
676	// deploy, whatever it managed to collect on the way down.
677	if let Err(e) =
678	handoff_for(&state, record_path.as_deref(), &app, &version, target).await
679	{
680	let msg = format!("{e:#}");
681	tracing::error!(%app, %target, error = %msg, "handing the artifact to sando failed");
682	fail_target(
683	&state,
684	target_run_id,
685	&app,
686	&version,
687	target,
688	Step::Handoff,
689	&msg,
690	)
691	.await;
692	crate::metrics::target_finished(
693	&target.to_string(),
694	"failed",
695	started.elapsed().as_secs_f64(),
696	);
697	return;
698	}
699
700	let artifacts = collected_artifacts(&state, &app, &version, target);
701	if let Err(e) = sqlx::query(
702	"UPDATE target_runs SET status = 'ok', current_step = NULL, finished_at = ? WHERE id = ?",
703	)
704	.bind(chrono::Utc::now().to_rfc3339())
705	.bind(target_run_id)
706	.execute(&state.pool)
707	.await
708	{
709	// A swallowed terminal write would leave the row `running`;
710	// finalize_build reconciles it, but log so the cause is visible.
711	tracing::error!(%app, %target, error = %e, "could not stamp target_run ok");
712	}
713	crate::metrics::target_finished(
714	&target.to_string(),
715	"ok",
716	started.elapsed().as_secs_f64(),
717	);
718	events::emit(
719	&state.events,
720	Event::TargetOk {
721	app,
722	version,
723	target,
724	artifacts,
725	},
726	);
727	}
728	Ok(Err((step, msg))) => {
729	fail_target(&state, target_run_id, &app, &version, target, step, &msg).await;
730	crate::metrics::target_finished(
731	&target.to_string(),
732	"failed",
733	started.elapsed().as_secs_f64(),
734	);
735	}
736	Err(join_err) => {
737	// Task was aborted (superseded) or panicked.
738	let status = if join_err.is_cancelled() {
739	"aborted"
740	} else {
741	"failed"
742	};
743	crate::metrics::target_finished(
744	&target.to_string(),
745	status,
746	started.elapsed().as_secs_f64(),
747	);
748	let msg = if join_err.is_cancelled() {
749	"aborted (superseded)".to_string()
750	} else {
751	format!("recipe task panicked: {join_err}")
752	};
753	fail_target(
754	&state,
755	target_run_id,
756	&app,
757	&version,
758	target,
759	Step::Build,
760	&msg,
761	)
762	.await;
763	}
764	}
765	}
766
767	async fn fail_target(
768	state: &AppState,
769	target_run_id: i64,
770	app: &AppId,
771	version: &Version,
772	target: Target,
773	step: Step,
774	error: &str,
775	) {
776	if let Err(e) = sqlx::query(
777	"UPDATE target_runs SET status = 'failed', current_step = NULL, error = ?, finished_at = ? WHERE id = ?",
778	)
779	.bind(error)
780	.bind(chrono::Utc::now().to_rfc3339())
781	.bind(target_run_id)
782	.execute(&state.pool)
783	.await
784	{
785	tracing::error!(%app, %target, error = %e, "could not stamp target_run failed");
786	}
787	events::emit(
788	&state.events,
789	Event::TargetFailed {
790	app: app.clone(),
791	version: version.clone(),
792	target,
793	step,
794	error: error.to_string(),
795	},
796	);
797	}
798
799	/// Await every target task of a build, then stamp the build's terminal status.
800	///
801	/// Event-driven: it joins the `JoinSet` rather than polling the DB. Each task's
802	/// outcome is supervised — a panicked task is logged (and its still-`running`
803	/// row is reconciled below), an aborted (superseded) task is expected.
804	///
805	/// Per-step deadlines (see `engine::step_budget`) are the real bound on a wedged
806	/// step now; this overall deadline is only a generous last-resort backstop for a
807	/// hang outside a bounded command. When it trips it sets each still-running
808	/// target's cooperative cancel FIRST — the blocking recipe bodies observe that
809	/// at their next bounded command or step boundary and stop signing — then aborts
810	/// the async wrappers. The old code aborted only the wrappers, which could not
811	/// reach the blocking bodies, so a build was marked failed while codesign kept
812	/// running on the mac.
813	async fn finalize_build(
814	state: AppState,
815	build_id: i64,
816	mut set: tokio::task::JoinSet<()>,
817	app: AppId,
818	pinned_branches: Vec<(String, String)>,
819	) {
820	const MAX_WAIT: std::time::Duration = std::time::Duration::from_hours(6);
821	let deadline = tokio::time::Instant::now() + MAX_WAIT;
822	loop {
823	match tokio::time::timeout_at(deadline, set.join_next()).await {
824	Ok(Some(Ok(()))) => {}
825	Ok(Some(Err(e))) => {
826	// A panic in run_target itself (outside its inner spawn_blocking,
827	// which is already caught). Cancellation = a superseding build.
828	if !e.is_cancelled() {
829	tracing::error!(build_id, error = %e, "target task panicked");
830	}
831	}
832	Ok(None) => break, // all targets settled
833	Err(_elapsed) => {
834	tracing::error!(
835	build_id,
836	"finalize_build backstop deadline hit; cancelling then aborting remaining targets"
837	);
838	// Set the cooperative cancel on this build's still-running slots
839	// BEFORE aborting, so the blocking recipe bodies actually stop
840	// (abort() alone cannot reach a spawn_blocking body).
841	{
842	let active = state.active.lock().await;
843	for slot in active.values().filter(\|s\| s.build_id == build_id) {
844	slot.cancel.store(true, std::sync::atomic::Ordering::SeqCst);
845	}
846	}
847	set.abort_all();
848	while set.join_next().await.is_some() {}
849	break;
850	}
851	}
852	}
853
854	// Reap this build's latest-wins slots (only our own — a superseding build
855	// owns a different build_id and is left intact).
856	let app_still_building = {
857	let mut active = state.active.lock().await;
858	active.retain(\|_, slot\| slot.build_id != build_id);
859	crate::metrics::set_in_flight(active.len());
860	active.keys().any(\|(a, _)\| a == &app)
861	};
862
863	// Undo the preflight's pin. Skipped if a superseding build for this same app
864	// still holds the checkout — it pinned the tree to ITS tag, and restoring the
865	// branch here would move that build off the commit it is releasing.
866	if !pinned_branches.is_empty() {
867	if app_still_building {
868	tracing::debug!(
869	build_id, %app,
870	"leaving the checkout pinned; a superseding build for this app is still running"
871	);
872	} else {
873	restore_branches(&state, &app, &pinned_branches).await;
874	}
875	}
876
877	let now = chrono::Utc::now().to_rfc3339();
878	// Any row still `running` is a panicked/aborted task that never stamped its
879	// terminal status — reconcile it so the build finalizes truthfully.
880	if let Err(e) = sqlx::query(
881	"UPDATE target_runs SET status = 'failed', \
882	error = COALESCE(error, 'target task ended before stamping its status'), \
883	finished_at = ? WHERE build_id = ? AND status = 'running'",
884	)
885	.bind(&now)
886	.bind(build_id)
887	.execute(&state.pool)
888	.await
889	{
890	tracing::error!(build_id, error = %e, "could not reconcile straggling target_runs");
891	}
892
893	let failed: i64 = sqlx::query_scalar(
894	"SELECT COUNT(*) FROM target_runs WHERE build_id = ? AND status = 'failed'",
895	)
896	.bind(build_id)
897	.fetch_one(&state.pool)
898	.await
899	.unwrap_or(0);
900	let status = if failed == 0 { "ok" } else { "failed" };
901	if let Err(e) = sqlx::query("UPDATE builds SET status = ?, finished_at = ? WHERE id = ?")
902	.bind(status)
903	.bind(&now)
904	.bind(build_id)
905	.execute(&state.pool)
906	.await
907	{
908	tracing::error!(build_id, error = %e, "could not stamp build terminal status");
909	}
910	}
911
912	/// Read the recipe text for `(app, target)` from the app's checkout on the
913	/// daemon host.
914	///
915	/// Apps use one recipe per platform (`<recipe_dir>/<platform>.rhai`), since
916	/// what they produce differs by platform. A library produces one crate whatever
917	/// host uploads it, so it uses a single `publish.rhai` — a `linux.rhai` naming
918	/// a registry upload would imply a per-platform artifact that does not exist.
919	fn read_recipe(state: &AppState, app: &AppId, target: Target) -> Result<String> {
920	let cfg = state
921	.topo
922	.app(app)
923	.ok_or_else(\|\| anyhow::anyhow!("unknown app `{app}`"))?;
924	let file = match cfg.kind {
925	crate::topology::Kind::Library => "publish.rhai".to_string(),
926	// A service is built per target like an app — the binary genuinely
927	// differs per platform, and so does where it lands — so it takes the
928	// same per-platform recipe naming rather than a single deploy.rhai.
929	crate::topology::Kind::App \| crate::topology::Kind::Service => {
930	format!("{}.rhai", target.platform.as_str())
931	}
932	};
933	let path: PathBuf = engine::expand_tilde(&cfg.repo)
934	.join(&cfg.recipe_dir)
935	.join(file);
936	std::fs::read_to_string(&path).with_context(\|\| format!("reading recipe {}", path.display()))
937	}
938
939	/// Send this target's finished bundle to the Sando configured for the app, if
940	/// one is. Nothing configured is a no-op and the ordinary case.
941	///
942	/// A configured handoff with no record is an error rather than a skip. It means
943	/// the recipe succeeded and collected nothing, or that the paperwork could not
944	/// be written — and for an app whose whole point is being deployed by Sando, a
945	/// build that produced nothing to hand over is not a green build. Everywhere
946	/// else a missing record stays the non-event it was.
947	async fn handoff_for(
948	state: &AppState,
949	record_path: Option<&std::path::Path>,
950	app: &AppId,
951	version: &Version,
952	target: Target,
953	) -> anyhow::Result<()> {
954	if !state.cfg.handoff.contains_key(app.as_str()) {
955	return Ok(());
956	}
957	let record_path = record_path.ok_or_else(\|\| {
958	anyhow::anyhow!(
959	"{app} hands off to sando, but this {target} run wrote no artifact record \
960	(nothing was collected, or the record could not be written)"
961	)
962	})?;
963	let dir = crate::archive::target_dir(&state.cfg.dist_root, app, version, target);
964	crate::handoff::send(&state.cfg, &dir, record_path, app, version, target).await
965	}
966
967	/// What this target run left in its collect directory.
968	///
969	/// Per target: the event reports what THIS run produced, and every target of a
970	/// version used to share one directory, so a mac build's `TargetOk` listed the
971	/// Linux AppImage a sibling had collected minutes earlier.
972	fn collected_artifacts(
973	state: &AppState,
974	app: &AppId,
975	version: &Version,
976	target: Target,
977	) -> Vec<String> {
978	let dir = crate::archive::target_dir(&state.cfg.dist_root, app, version, target);
979	let Ok(rd) = std::fs::read_dir(&dir) else {
980	return Vec::new();
981	};
982	rd.filter_map(std::result::Result::ok)
983	.map(\|e\| e.file_name().to_string_lossy().into_owned())
984	.collect()
985	}
986
987	#[cfg(test)]
988	mod tests {
989	use super::*;
990	use crate::config::Config;
991	use crate::ota::OtaRegistry;
992	use crate::topology::Topology;
993	use async_trait::async_trait;
994	use ops_exec::{CapabilitySet, Executor, LogSink, RunOutput, SyncOpts};
995	use sqlx::SqlitePool;
996	use std::collections::HashMap;
997	use std::os::unix::process::ExitStatusExt;
998	use std::sync::Arc;
999	use tokio::sync::Mutex;
1000
1001	/// A no-transport [`Executor`] for the paths that don't need a real command
1002	/// to run: its `preflight` is programmable (the agent host hits `/health`
1003	/// there, and a dead `ops-agent` must fail the target before the recipe
1004	/// dispatches), and every actual op is a success no-op.
1005	struct FakeExec {
1006	caps: CapabilitySet,
1007	preflight_err: Option<String>,
1008	}
1009
1010	impl FakeExec {
1011	fn preflight_fails(msg: &str) -> Arc<dyn Executor> {
1012	Arc::new(Self {
1013	caps: CapabilitySet::default(),
1014	preflight_err: Some(msg.to_string()),
1015	})
1016	}
1017	}
1018
1019	#[async_trait]
1020	impl Executor for FakeExec {
1021	async fn run_streaming(
1022	&self,
1023	_step: &ops_exec::Step,
1024	_sink: &mut dyn LogSink,
1025	) -> anyhow::Result<RunOutput> {
1026	Ok(RunOutput {
1027	status: std::process::ExitStatus::from_raw(0),
1028	stdout: Vec::new(),
1029	stderr: Vec::new(),
1030	})
1031	}
1032	async fn pull_file(
1033	&self,
1034	_r: &std::path::Path,
1035	_l: &std::path::Path,
1036	_o: &SyncOpts,
1037	) -> anyhow::Result<()> {
1038	Ok(())
1039	}
1040	async fn pull_dir(
1041	&self,
1042	_r: &std::path::Path,
1043	_l: &std::path::Path,
1044	_o: &SyncOpts,
1045	) -> anyhow::Result<()> {
1046	Ok(())
1047	}
1048	async fn pull_glob(
1049	&self,
1050	_g: &str,
1051	_l: &std::path::Path,
1052	_o: &SyncOpts,
1053	) -> anyhow::Result<()> {
1054	Ok(())
1055	}
1056	async fn push_dir(
1057	&self,
1058	_l: &std::path::Path,
1059	_r: &std::path::Path,
1060	_o: &SyncOpts,
1061	) -> anyhow::Result<()> {
1062	Ok(())
1063	}
1064	async fn preflight(&self) -> anyhow::Result<()> {
1065	match &self.preflight_err {
1066	Some(m) => anyhow::bail!("{m}"),
1067	None => Ok(()),
1068	}
1069	}
1070	fn capabilities(&self) -> &CapabilitySet {
1071	&self.caps
1072	}
1073	}
1074
1075	/// A recording, programmable [`Executor`] for the macOS sign chain. Every
1076	/// dispatched shell command is captured for assertion, and its exit code +
1077	/// stdout is chosen by the first rule whose needle the command contains — so
1078	/// a test can make `notarytool` report `Accepted`, make `codesign` fail, or
1079	/// make `spctl` emit the Gatekeeper sentinel without a real Mac or SSH. A
1080	/// rule may carry a sequence of responses (one per successive match) to
1081	/// drive the notarize retry loop; the last entry repeats once the sequence
1082	/// is exhausted. Unmatched commands succeed as empty no-ops (so a plain
1083	/// build `sh_ok` passes), and the sync/preflight ops are no-ops.
1084	struct ScriptedExec {
1085	caps: CapabilitySet,
1086	rules: Vec<ScriptRule>,
1087	log: Arc<std::sync::Mutex<Vec<String>>>,
1088	}
1089
1090	struct ScriptRule {
1091	needle: String,
1092	responses: Vec<(i32, String)>,
1093	calls: std::sync::atomic::AtomicUsize,
1094	}
1095
1096	impl ScriptedExec {
1097	fn new() -> Self {
1098	Self {
1099	// A mac host's real grant. Nothing in the dispatch path gates on
1100	// it (this fake never calls `gate`), but keep it coherent so
1101	// `capabilities()` is not a lie.
1102	caps: CapabilitySet::from_tokens(
1103	["build", "sign", "notarize", "staple"],
1104	["build-log", "artifact"],
1105	),
1106	rules: Vec::new(),
1107	log: Arc::new(std::sync::Mutex::new(Vec::new())),
1108	}
1109	}
1110
1111	/// Respond to every command containing `needle` with `(code, stdout)`.
1112	fn on(mut self, needle: &str, code: i32, stdout: &str) -> Self {
1113	self.rules.push(ScriptRule {
1114	needle: needle.to_string(),
1115	responses: vec![(code, stdout.to_string())],
1116	calls: std::sync::atomic::AtomicUsize::new(0),
1117	});
1118	self
1119	}
1120
1121	/// Respond to successive `needle` matches with successive responses; the
1122	/// last repeats once the list is exhausted. Drives the notarize retry.
1123	fn on_seq(mut self, needle: &str, responses: &[(i32, &str)]) -> Self {
1124	self.rules.push(ScriptRule {
1125	needle: needle.to_string(),
1126	responses: responses
1127	.iter()
1128	.map(\|(c, s)\| (c, (s).to_string()))
1129	.collect(),
1130	calls: std::sync::atomic::AtomicUsize::new(0),
1131	});
1132	self
1133	}
1134
1135	/// Every shell command this executor was asked to run, in order.
1136	fn commands(&self) -> Vec<String> {
1137	self.log.lock().unwrap().clone()
1138	}
1139	}
1140
1141	#[async_trait]
1142	impl Executor for ScriptedExec {
1143	async fn run_streaming(
1144	&self,
1145	step: &ops_exec::Step,
1146	_sink: &mut dyn LogSink,
1147	) -> anyhow::Result<RunOutput> {
1148	let cmd = step.argv.last().cloned().unwrap_or_default();
1149	self.log.lock().unwrap().push(cmd.clone());
1150	let (code, stdout) = self.rules.iter().find(\|r\| cmd.contains(&r.needle)).map_or(
1151	(0, String::new()),
1152	\|r\| {
1153	let i = r.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1154	r.responses[i.min(r.responses.len() - 1)].clone()
1155	},
1156	);
1157	Ok(RunOutput {
1158	// Shift into the wait-status word's exit-code byte so
1159	// `ExitStatus::code()` reports `code` exactly (a bare
1160	// `from_raw(1)` reads as a signal, yielding `None`).
1161	status: std::process::ExitStatus::from_raw(code << 8),
1162	stdout: stdout.into_bytes(),
1163	stderr: Vec::new(),
1164	})
1165	}
1166	async fn pull_file(
1167	&self,
1168	_r: &std::path::Path,
1169	_l: &std::path::Path,
1170	_o: &SyncOpts,
1171	) -> anyhow::Result<()> {
1172	Ok(())
1173	}
1174	async fn pull_dir(
1175	&self,
1176	_r: &std::path::Path,
1177	_l: &std::path::Path,
1178	_o: &SyncOpts,
1179	) -> anyhow::Result<()> {
1180	Ok(())
1181	}
1182	async fn pull_glob(
1183	&self,
1184	_g: &str,
1185	_l: &std::path::Path,
1186	_o: &SyncOpts,
1187	) -> anyhow::Result<()> {
1188	Ok(())
1189	}
1190	async fn push_dir(
1191	&self,
1192	_l: &std::path::Path,
1193	_r: &std::path::Path,
1194	_o: &SyncOpts,
1195	) -> anyhow::Result<()> {
1196	Ok(())
1197	}
1198	async fn preflight(&self) -> anyhow::Result<()> {
1199	Ok(())
1200	}
1201	fn capabilities(&self) -> &CapabilitySet {
1202	&self.caps
1203	}
1204	}
1205
1206	/// Assemble an [`AppState`] from the three per-test inputs, filling in the
1207	/// executors/syncs (built from `topo`) and the fixed test scaffolding
1208	/// (metrics handle, event bus, standard OTA registry, empty active map, no
1209	/// token). Every runner test builds the same struct around a different repo
1210	/// + recipe; this is that struct in one place.
1211	fn test_state(pool: SqlitePool, topo: Topology, cfg: Config) -> AppState {
1212	let executors = Arc::new(crate::state::build_executors(&topo));
1213	let syncs = Arc::new(crate::state::build_syncs(&topo));
1214	let host_locks = crate::state::build_host_locks(&topo);
1215	AppState {
1216	pool,
1217	topo: Arc::new(topo),
1218	cfg: Arc::new(cfg),
1219	prom: crate::metrics::test_handle(),
1220	events: crate::events::channel(),
1221	ota: Arc::new(OtaRegistry::standard("https://makenot.work")),
1222	executors,
1223	syncs,
1224	active: Arc::new(Mutex::new(HashMap::new())),
1225	api_token: None,
1226	host_locks,
1227	distribution: Arc::new(Mutex::new(HashMap::new())),
1228	http: crate::tls::builder().build().unwrap(),
1229	// Port 1 refuses instantly. A test must never probe production, and
1230	// a refusal is also faster than any timeout would be.
1231	mnw_base_url: "http://127.0.0.1:1".into(),
1232	}
1233	}
1234
1235	/// A `kind = "service"` release end to end: build, `glibc_check`, `deploy`,
1236	/// and a health assertion the recipe makes itself against the service host.
1237	///
1238	/// The whole point of the deploy step is that it dispatches to a host
1239	/// OUTSIDE the build topology, so this exercises the resolution
1240	/// (target -> `[[deploy]]` entry -> executor registered for the run), the
1241	/// staging, and the call into the privileged installer — with a fake
1242	/// installer standing in for the root script, which is the one part a test
1243	/// cannot run for real.
1244	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1245	async fn service_recipe_builds_then_deploys_and_verifies() {
1246	let tmp = tempfile::tempdir().unwrap();
1247	let root = tmp.path();
1248	let repo = root.join("svc");
1249	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1250	std::fs::write(repo.join("Cargo.toml"), "[package]\nversion = \"0.4.0\"\n").unwrap();
1251
1252	// Stand-in for the root installer: same three arguments, records what it
1253	// was asked to do instead of writing to /usr/local/bin and restarting a
1254	// unit. `install` + a marker file, so the test can assert the binary
1255	// that arrived is the binary that was built.
1256	let installer = root.join("install-service.sh");
1257	std::fs::write(
1258	&installer,
1259	"#!/bin/sh\nset -eu\ninstall -m 0755 \"$1\" \"$2\"\necho \"restarted $3\" >> \"$2.log\"\n",
1260	)
1261	.unwrap();
1262	std::fs::set_permissions(
1263	&installer,
1264	<std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o755),
1265	)
1266	.unwrap();
1267	let install_path = root.join("bin/svc");
1268	std::fs::create_dir_all(root.join("bin")).unwrap();
1269
1270	std::fs::write(
1271	repo.join("dist/recipes/linux.rhai"),
1272	r#"
1273	let v = version();
1274	step("build");
1275	sh_ok("fw13", "mkdir -p REPO/target/release && echo built-BIN > REPO/target/release/svc");
1276	step("verify");
1277	log(glibc_check("REPO/target/release/svc"));
1278	step("deploy");
1279	log(deploy("REPO/target/release/svc"));
1280	// The recipe owns what "healthy" means, and asserts it itself
1281	// against the host it just restarted.
1282	sh_ok(deploy_host(), "test -x " + install_path());
1283	"#
1284	.replace("REPO", repo.to_str().unwrap())
1285	.replace("BIN", "0.4.0"),
1286	)
1287	.unwrap();
1288
1289	std::fs::write(
1290	repo.join("bento.toml"),
1291	format!(
1292	r#"kind = "service"
1293	targets = ["linux/x86_64"]
1294	version_path = "Cargo.toml"
1295
1296	[[deploy]]
1297	target = "linux/x86_64"
1298	host = "local"
1299	install_path = "{}"
1300	service = "svc.service"
1301	health_url = "http://localhost:9100/api/health"
1302	"#,
1303	install_path.display()
1304	),
1305	)
1306	.unwrap();
1307
1308	let mut cfg = Config::for_tests(root);
1309	cfg.deploy_installer = installer.display().to_string();
1310	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1311	let topo = Topology::from_str_for_tests(&format!(
1312	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
1313	pull_root = \"{repo}\"\n\n[app.svc]\nrepo = \"{repo}\"\n",
1314	repo = repo.display()
1315	))
1316	.unwrap();
1317	let state = test_state(pool.clone(), topo, cfg);
1318
1319	let build_id = start_build(
1320	state.clone(),
1321	AppId::new("svc"),
1322	Version::parse("0.4.0").unwrap(),
1323	vec!["linux/x86_64".parse().unwrap()],
1324	)
1325	.await
1326	.unwrap();
1327
1328	let mut status = String::new();
1329	for _ in 0..100 {
1330	status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1331	.bind(build_id)
1332	.fetch_optional(&pool)
1333	.await
1334	.unwrap()
1335	.unwrap_or_else(\|\| "running".to_string());
1336	if status != "running" {
1337	break;
1338	}
1339	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1340	}
1341	assert_eq!(status, "ok", "service run should succeed");
1342
1343	let steps: Vec<(String, String)> = sqlx::query_as(
1344	"SELECT step, status FROM step_runs WHERE target_run_id IN \
1345	(SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id",
1346	)
1347	.bind(build_id)
1348	.fetch_all(&pool)
1349	.await
1350	.unwrap();
1351	assert_eq!(
1352	steps.iter().map(\|(s, _)\| s.as_str()).collect::<Vec<_>>(),
1353	vec!["build", "verify", "deploy"],
1354	"a service ends at deploy, not collect"
1355	);
1356	assert!(steps.iter().all(\|(_, st)\| st == "ok"), "{steps:?}");
1357
1358	// The bytes that were built are the bytes that landed, and the unit was
1359	// restarted only after the install succeeded.
1360	assert_eq!(
1361	std::fs::read_to_string(&install_path).unwrap().trim(),
1362	"built-0.4.0"
1363	);
1364	assert!(
1365	std::fs::read_to_string(install_path.with_extension("").with_file_name("svc.log"))
1366	.unwrap()
1367	.contains("restarted svc.service")
1368	);
1369	}
1370
1371	/// Stand up a tmp app repo + topology and run a real local recipe end to
1372	/// end: step transitions, streamed `sh_ok`, `version_of`, `log`, and a
1373	/// `collect` that pulls a built artifact into dist_root.
1374	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1375	async fn local_linux_recipe_runs_end_to_end() {
1376	let tmp = tempfile::tempdir().unwrap();
1377	let root = tmp.path();
1378
1379	// Fake app checkout: tauri.conf.json + a linux recipe.
1380	let repo = root.join("app");
1381	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1382	std::fs::write(
1383	repo.join("src-tauri/tauri.conf.json"),
1384	r#"{"version":"0.0.1"}"#,
1385	)
1386	.unwrap();
1387	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1388	// Build writes an artifact into the repo; collect pulls it to dist_root.
1389	std::fs::write(
1390	repo.join("dist/recipes/linux.rhai"),
1391	r#"
1392	step("build");
1393	let v = version_of("demo");
1394	log("building demo " + v);
1395	sh_ok("fw13", "echo compiling; mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
1396	step("collect");
1397	collect("fw13", "REPO/out/demo.bin", "demo", v);
1398	"#
1399	.replace("REPO", repo.to_str().unwrap()),
1400	)
1401	.unwrap();
1402
1403	let mut cfg = Config::for_tests(root);
1404	// Archive to a local directory, so the deposit a real release makes to
1405	// astra runs on the same code path here.
1406	cfg.archive = Some(crate::config::Archive {
1407	host: "local".into(),
1408	root: root.join("archive"),
1409	});
1410	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1411	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1412	let topo = Topology::from_str_for_tests(&format!(
1413	r#"
1414	[[host]]
1415	name = "fw13"
1416	ssh = "local"
1417	targets = ["linux/x86_64"]
1418	pull_root = "{repo}"
1419
1420	[app.demo]
1421	repo = "{repo}"
1422	"#,
1423	repo = repo.display()
1424	))
1425	.unwrap();
1426
1427	let state = test_state(pool.clone(), topo, cfg);
1428
1429	let app = AppId::new("demo");
1430	let version = Version::parse("0.0.1").unwrap();
1431	let build_id = start_build(
1432	state.clone(),
1433	app,
1434	version,
1435	vec!["linux/x86_64".parse().unwrap()],
1436	)
1437	.await
1438	.unwrap();
1439
1440	// Wait for the target run to settle. The row may not be inserted on the
1441	// first poll (run_target is spawned, not awaited), so treat a missing row
1442	// as still-pending rather than an error.
1443	let mut status = String::new();
1444	for _ in 0..100 {
1445	status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1446	.bind(build_id)
1447	.fetch_optional(&pool)
1448	.await
1449	.unwrap()
1450	.unwrap_or_else(\|\| "running".to_string());
1451	if status != "running" {
1452	break;
1453	}
1454	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1455	}
1456	assert_eq!(status, "ok", "target run should succeed");
1457
1458	// Both steps recorded and finished ok.
1459	let steps: Vec<(String, String)> =
1460	sqlx::query_as("SELECT step, status FROM step_runs WHERE target_run_id IN (SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id")
1461	.bind(build_id)
1462	.fetch_all(&pool)
1463	.await
1464	.unwrap();
1465	let names: Vec<&str> = steps.iter().map(\|(s, _)\| s.as_str()).collect();
1466	assert_eq!(names, vec!["build", "collect"]);
1467	assert!(steps.iter().all(\|(_, st)\| st == "ok"));
1468
1469	// Artifact landed in dist_root, under its own target, and a step log was
1470	// written. Both trees are keyed the same way, `<app>/<version>/<target>/`.
1471	let artifact = state.cfg.dist_root.join("demo/0.0.1/linux-x86_64/demo.bin");
1472	assert!(artifact.exists(), "collect should copy the artifact");
1473	// ...and the same bytes reached the archive, at the same path under its
1474	// own root. This is the answer to "where is demo 0.0.1 for linux".
1475	let archived = root.join("archive/demo/0.0.1/linux-x86_64/demo.bin");
1476	assert!(
1477	archived.exists(),
1478	"collect should deposit into the archive: {}",
1479	archived.display()
1480	);
1481	assert_eq!(
1482	std::fs::read(&archived).unwrap(),
1483	std::fs::read(&artifact).unwrap()
1484	);
1485	// Read the path off the ledger rather than rebuilding it: the log is
1486	// named for its step run id, and the point of that is that the row
1487	// resolves to exactly one file.
1488	let (run_id, log_ref): (i64, String) = sqlx::query_as(
1489	"SELECT id, log_ref FROM step_runs WHERE step = 'build' AND target_run_id IN \
1490	(SELECT id FROM target_runs WHERE build_id = ?)",
1491	)
1492	.bind(build_id)
1493	.fetch_one(&pool)
1494	.await
1495	.unwrap();
1496	let log = std::path::PathBuf::from(&log_ref);
1497	assert_eq!(
1498	log,
1499	state
1500	.cfg
1501	.logs_root
1502	.join(format!("demo/0.0.1/linux-x86_64/build.{run_id}.log")),
1503	"log path should be keyed on the step run id"
1504	);
1505	assert!(log.exists(), "build step log should exist");
1506	let body = std::fs::read_to_string(&log).unwrap();
1507	assert!(body.contains("compiling"));
1508	assert!(
1509	body.starts_with(&format!(
1510	"=== bento demo 0.0.1 linux/x86_64 step=build run_id={run_id} "
1511	)),
1512	"log should open with a run header naming its run: {body}"
1513	);
1514	}
1515
1516	/// Multi-target fan-out, which is what the daemon exists for: every other
1517	/// runner test drives exactly one target, so nothing covered `start_build`'s
1518	/// `JoinSet` fan-out or `finalize_build`'s rollup. One target fails and one
1519	/// succeeds — the failure must not abort its sibling, both must land their
1520	/// own terminal row, and the build must finalize `failed` because any failed
1521	/// target fails the build.
1522	#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1523	async fn multi_target_fan_out_rolls_up_partial_failure() {
1524	let tmp = tempfile::tempdir().unwrap();
1525	let root = tmp.path();
1526
1527	let repo = root.join("app");
1528	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1529	std::fs::write(
1530	repo.join("src-tauri/tauri.conf.json"),
1531	r#"{"version":"0.0.1"}"#,
1532	)
1533	.unwrap();
1534	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1535
1536	// Linux succeeds and collects a real artifact. The artifact is the proof
1537	// this target ran to completion rather than being torn down when its
1538	// sibling failed.
1539	std::fs::write(
1540	repo.join("dist/recipes/linux.rhai"),
1541	r#"
1542	step("build");
1543	let v = version_of("demo");
1544	sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
1545	step("collect");
1546	collect("fw13", "REPO/out/demo.bin", "demo", v);
1547	"#
1548	.replace("REPO", repo.to_str().unwrap()),
1549	)
1550	.unwrap();
1551	// Windows fails in its first step.
1552	std::fs::write(
1553	repo.join("dist/recipes/windows.rhai"),
1554	r#"
1555	step("build");
1556	sh_ok("winbox", "echo nope 1>&2; exit 1");
1557	"#,
1558	)
1559	.unwrap();
1560
1561	let cfg = Config::for_tests(root);
1562	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1563	std::fs::write(
1564	repo.join("bento.toml"),
1565	"targets = [\"linux/x86_64\", \"windows/x86_64\"]\n",
1566	)
1567	.unwrap();
1568	// Two hosts so each target resolves its own, mirroring the real
1569	// per-architecture topology.
1570	let topo = Topology::from_str_for_tests(&format!(
1571	r#"
1572	[[host]]
1573	name = "fw13"
1574	ssh = "local"
1575	targets = ["linux/x86_64"]
1576	pull_root = "{repo}"
1577
1578	[[host]]
1579	name = "winbox"
1580	ssh = "local"
1581	targets = ["windows/x86_64"]
1582
1583	[app.demo]
1584	repo = "{repo}"
1585	"#,
1586	repo = repo.display()
1587	))
1588	.unwrap();
1589
1590	let state = test_state(pool.clone(), topo, cfg);
1591
1592	let build_id = start_build(
1593	state.clone(),
1594	AppId::new("demo"),
1595	Version::parse("0.0.1").unwrap(),
1596	vec![
1597	"linux/x86_64".parse().unwrap(),
1598	"windows/x86_64".parse().unwrap(),
1599	],
1600	)
1601	.await
1602	.unwrap();
1603
1604	// Poll the build row, not the target rows: the build is terminal only
1605	// once finalize_build has joined every task and stamped it.
1606	let mut build_status = String::new();
1607	for _ in 0..200 {
1608	build_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
1609	.bind(build_id)
1610	.fetch_one(&pool)
1611	.await
1612	.unwrap();
1613	if build_status != "running" {
1614	break;
1615	}
1616	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1617	}
1618
1619	let runs: Vec<(String, String)> = sqlx::query_as(
1620	"SELECT target, status FROM target_runs WHERE build_id = ? ORDER BY target",
1621	)
1622	.bind(build_id)
1623	.fetch_all(&pool)
1624	.await
1625	.unwrap();
1626	assert_eq!(
1627	runs,
1628	vec![
1629	("linux/x86_64".to_string(), "ok".to_string()),
1630	("windows/x86_64".to_string(), "failed".to_string()),
1631	],
1632	"each target lands its own terminal row; one failing does not take the other down",
1633	);
1634
1635	// The surviving target finished its work, not merely its row.
1636	assert!(
1637	state
1638	.cfg
1639	.dist_root
1640	.join("demo/0.0.1/linux-x86_64/demo.bin")
1641	.exists(),
1642	"the succeeding target ran to completion and collected its artifact",
1643	);
1644
1645	// The failure is attributed to the target that failed, and only it.
1646	let err: Option<String> = sqlx::query_scalar(
1647	"SELECT error FROM target_runs WHERE build_id = ? AND target = 'windows/x86_64'",
1648	)
1649	.bind(build_id)
1650	.fetch_one(&pool)
1651	.await
1652	.unwrap();
1653	assert!(
1654	err.is_some_and(\|e\| !e.is_empty()),
1655	"a failed target records why",
1656	);
1657
1658	assert_eq!(
1659	build_status, "failed",
1660	"any failed target fails the build; a partial release must not read as ok",
1661	);
1662	let finished: Option<String> =
1663	sqlx::query_scalar("SELECT finished_at FROM builds WHERE id = ?")
1664	.bind(build_id)
1665	.fetch_one(&pool)
1666	.await
1667	.unwrap();
1668	assert!(finished.is_some(), "finalize_build stamps the finish time");
1669
1670	// finalize_build reaps its own latest-wins slots, so nothing is left
1671	// in flight to block a later build of the same targets.
1672	assert!(
1673	state.active.lock().await.is_empty(),
1674	"finalize_build reaps the slots it owned",
1675	);
1676	}
1677
1678	/// Latest-wins supersession, driven through `start_build` (not the leaf
1679	/// `begin_step` bail that's already covered). Two builds of the SAME
1680	/// (app, target) race: the second must cooperatively cancel + abort the
1681	/// first and take the single slot, the first must terminate non-`ok`, and
1682	/// the second must run to completion.
1683	#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1684	async fn a_newer_build_supersedes_the_in_flight_one_for_the_same_target() {
1685	let tmp = tempfile::tempdir().unwrap();
1686	let root = tmp.path();
1687
1688	let repo = root.join("app");
1689	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1690	std::fs::write(
1691	repo.join("src-tauri/tauri.conf.json"),
1692	r#"{"version":"0.0.1"}"#,
1693	)
1694	.unwrap();
1695	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1696	// A sleep long enough that the first build is still mid-recipe when the
1697	// second arrives; the engine checks the cancel flag at the step boundary
1698	// before "done", so the superseded run bails rather than finishing.
1699	std::fs::write(
1700	repo.join("dist/recipes/linux.rhai"),
1701	r#"
1702	step("build");
1703	sh_ok("fw13", "sleep 2");
1704	"#,
1705	)
1706	.unwrap();
1707
1708	let cfg = Config::for_tests(root);
1709	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1710	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1711	let topo = Topology::from_str_for_tests(&format!(
1712	r#"
1713	[[host]]
1714	name = "fw13"
1715	ssh = "local"
1716	targets = ["linux/x86_64"]
1717
1718	[app.demo]
1719	repo = "{}"
1720	"#,
1721	repo.display()
1722	))
1723	.unwrap();
1724	let state = test_state(pool.clone(), topo, cfg);
1725
1726	let target = "linux/x86_64".parse().unwrap();
1727	let first = start_build(
1728	state.clone(),
1729	AppId::new("demo"),
1730	Version::parse("0.0.1").unwrap(),
1731	vec![target],
1732	)
1733	.await
1734	.unwrap();
1735	// Let the first build register its slot and enter the recipe before the
1736	// second supersedes it (start_build inserts the slot synchronously, but
1737	// the recipe runs on a spawned task).
1738	tokio::time::sleep(std::time::Duration::from_millis(100)).await;
1739	let second = start_build(
1740	state.clone(),
1741	AppId::new("demo"),
1742	Version::parse("0.0.1").unwrap(),
1743	vec![target],
1744	)
1745	.await
1746	.unwrap();
1747	assert_ne!(first, second);
1748
1749	// Latest wins: exactly one slot for the key, owned by the second build.
1750	{
1751	let active = state.active.lock().await;
1752	assert_eq!(active.len(), 1, "supersession must not leave two slots");
1753	assert_eq!(
1754	active.values().next().unwrap().build_id,
1755	second,
1756	"the surviving slot belongs to the newer build",
1757	);
1758	}
1759
1760	// Wait for the second (surviving) build to finalize.
1761	let mut second_status = String::new();
1762	for _ in 0..200 {
1763	second_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
1764	.bind(second)
1765	.fetch_one(&pool)
1766	.await
1767	.unwrap();
1768	if second_status != "running" {
1769	break;
1770	}
1771	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1772	}
1773	assert_eq!(
1774	second_status, "ok",
1775	"the superseding build runs to completion"
1776	);
1777
1778	// The first build's target run terminated without succeeding — it was
1779	// cancelled/aborted, never stamped `ok`.
1780	let first_status: String =
1781	sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1782	.bind(first)
1783	.fetch_one(&pool)
1784	.await
1785	.unwrap();
1786	assert_ne!(
1787	first_status, "ok",
1788	"the superseded build must not complete successfully",
1789	);
1790
1791	// Both builds reaped their slots; nothing left in flight.
1792	assert!(
1793	state.active.lock().await.is_empty(),
1794	"every finalized build reaps its own slot",
1795	);
1796	}
1797
1798	/// A failing `preflight` (the agent host's `/health` probe when `ops-agent`
1799	/// is down) must fail the target BEFORE the recipe dispatches — the whole
1800	/// point of preflighting. Covered only via a fake here, since a real
1801	/// LocalExec/SshExec preflight is a no-op that always passes.
1802	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1803	async fn a_failing_preflight_fails_the_target_before_the_recipe_runs() {
1804	let tmp = tempfile::tempdir().unwrap();
1805	let root = tmp.path();
1806
1807	let repo = root.join("app");
1808	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1809	std::fs::write(
1810	repo.join("src-tauri/tauri.conf.json"),
1811	r#"{"version":"0.0.1"}"#,
1812	)
1813	.unwrap();
1814	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1815	// If the recipe ran it would drop this marker; preflight failing first
1816	// means it never does.
1817	let marker = root.join("recipe-ran");
1818	std::fs::write(
1819	repo.join("dist/recipes/linux.rhai"),
1820	r#"
1821	step("build");
1822	sh_ok("fw13", "touch MARKER");
1823	"#
1824	.replace("MARKER", marker.to_str().unwrap()),
1825	)
1826	.unwrap();
1827
1828	let cfg = Config::for_tests(root);
1829	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1830	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1831	let topo = Topology::from_str_for_tests(&format!(
1832	r#"
1833	[[host]]
1834	name = "fw13"
1835	ssh = "local"
1836	targets = ["linux/x86_64"]
1837
1838	[app.demo]
1839	repo = "{}"
1840	"#,
1841	repo.display()
1842	))
1843	.unwrap();
1844	let mut state = test_state(pool.clone(), topo, cfg);
1845	// Swap fw13's executor for one whose preflight fails.
1846	let mut execs = HashMap::new();
1847	execs.insert(
1848	"fw13".to_string(),
1849	FakeExec::preflight_fails("ops-agent not reachable at /health"),
1850	);
1851	state.executors = Arc::new(execs);
1852
1853	let build_id = start_build(
1854	state.clone(),
1855	AppId::new("demo"),
1856	Version::parse("0.0.1").unwrap(),
1857	vec!["linux/x86_64".parse().unwrap()],
1858	)
1859	.await
1860	.unwrap();
1861
1862	let mut status = String::new();
1863	let mut error = String::new();
1864	for _ in 0..100 {
1865	let row: Option<(String, Option<String>)> =
1866	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1867	.bind(build_id)
1868	.fetch_optional(&pool)
1869	.await
1870	.unwrap();
1871	if let Some((s, e)) = row {
1872	status = s;
1873	error = e.unwrap_or_default();
1874	if status != "running" {
1875	break;
1876	}
1877	}
1878	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1879	}
1880	assert_eq!(status, "failed", "a failed preflight must fail the target");
1881	assert!(
1882	error.contains("ops-agent not reachable"),
1883	"the failure must carry the preflight error, got: {error}"
1884	);
1885	assert!(
1886	!marker.exists(),
1887	"the recipe must NOT run when preflight fails"
1888	);
1889	}
1890
1891	/// A target whose recipe file is absent must fail at the `checkout` boundary
1892	/// with a readable "reading recipe" error, before any step runs — exercising
1893	/// `read_recipe`'s error branch and the `fail_target` path in `run_target`
1894	/// that neither the happy-path nor the failing-preflight test reaches. The
1895	/// app is configured to ship linux, but no `linux.rhai` is written.
1896	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1897	async fn a_target_with_no_recipe_file_fails_at_checkout() {
1898	let tmp = tempfile::tempdir().unwrap();
1899	let root = tmp.path();
1900
1901	let repo = root.join("app");
1902	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1903	std::fs::write(
1904	repo.join("src-tauri/tauri.conf.json"),
1905	r#"{"version":"0.0.1"}"#,
1906	)
1907	.unwrap();
1908	// The recipe dir exists but is empty — no linux.rhai.
1909	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1910
1911	let cfg = Config::for_tests(root);
1912	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1913	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1914	let topo = Topology::from_str_for_tests(&format!(
1915	r#"
1916	[[host]]
1917	name = "fw13"
1918	ssh = "local"
1919	targets = ["linux/x86_64"]
1920
1921	[app.demo]
1922	repo = "{}"
1923	"#,
1924	repo.display()
1925	))
1926	.unwrap();
1927	let state = test_state(pool.clone(), topo, cfg);
1928
1929	let build_id = start_build(
1930	state.clone(),
1931	AppId::new("demo"),
1932	Version::parse("0.0.1").unwrap(),
1933	vec!["linux/x86_64".parse().unwrap()],
1934	)
1935	.await
1936	.unwrap();
1937
1938	// Poll the build row: it is terminal only once finalize_build joins the
1939	// one (failing) target task.
1940	let mut build_status = String::new();
1941	for _ in 0..100 {
1942	build_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
1943	.bind(build_id)
1944	.fetch_one(&pool)
1945	.await
1946	.unwrap();
1947	if build_status != "running" {
1948	break;
1949	}
1950	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1951	}
1952
1953	// The target failed, and the error points at the unreadable recipe.
1954	let (status, error): (String, Option<String>) =
1955	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1956	.bind(build_id)
1957	.fetch_one(&pool)
1958	.await
1959	.unwrap();
1960	assert_eq!(status, "failed", "a missing recipe must fail the target");
1961	assert!(
1962	error.is_some_and(\|e\| e.contains("reading recipe")),
1963	"the failure must name the recipe it could not read",
1964	);
1965
1966	// The recipe never ran, so no step_runs row was ever created — the
1967	// failure is at the checkout boundary, ahead of any step.
1968	let step_count: i64 = sqlx::query_scalar(
1969	"SELECT COUNT(*) FROM step_runs WHERE target_run_id IN \
1970	(SELECT id FROM target_runs WHERE build_id = ?)",
1971	)
1972	.bind(build_id)
1973	.fetch_one(&pool)
1974	.await
1975	.unwrap();
1976	assert_eq!(
1977	step_count, 0,
1978	"no step should run when the recipe is absent"
1979	);
1980
1981	assert_eq!(
1982	build_status, "failed",
1983	"the build rolls up the target failure",
1984	);
1985	assert!(
1986	state.active.lock().await.is_empty(),
1987	"finalize_build reaps the slot even on the recipe-read failure path",
1988	);
1989	}
1990
1991	/// Run 2 S5: a publish whose version is not strictly newer than the latest
1992	/// already published for the same (app, target, channel) is refused — an
1993	/// older build cannot republish over a live newer release.
1994	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1995	async fn publish_rejects_a_non_monotonic_version() {
1996	let tmp = tempfile::tempdir().unwrap();
1997	let root = tmp.path();
1998	let repo = root.join("app");
1999	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2000	std::fs::write(
2001	repo.join("src-tauri/tauri.conf.json"),
2002	r#"{"version":"0.2.0"}"#,
2003	)
2004	.unwrap();
2005	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2006	let artifact = repo.join("out/app.tar.gz");
2007	// Build an artifact, publish 0.2.0 (records a release), then try to
2008	// publish the older 0.1.0 — the second publish must fail.
2009	std::fs::write(
2010	repo.join("dist/recipes/linux.rhai"),
2011	r#"
2012	step("build");
2013	sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
2014	step("publish");
2015	publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
2016	publish("tauri-mnw", "demo", "linux/x86_64", "0.1.0", "ARTIFACT", #{});
2017	"#
2018	.replace("ARTIFACT", artifact.to_str().unwrap())
2019	.replace("REPO", repo.to_str().unwrap()),
2020	)
2021	.unwrap();
2022
2023	let cfg = Config::for_tests(root);
2024	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2025	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2026	let topo = Topology::from_str_for_tests(&format!(
2027	r#"
2028	[[host]]
2029	name = "fw13"
2030	ssh = "local"
2031	targets = ["linux/x86_64"]
2032
2033	[app.demo]
2034	repo = "{}"
2035	"#,
2036	repo.display()
2037	))
2038	.unwrap();
2039	let state = test_state(pool.clone(), topo, cfg);
2040	let build_id = start_build(
2041	state.clone(),
2042	AppId::new("demo"),
2043	Version::parse("0.2.0").unwrap(),
2044	vec!["linux/x86_64".parse().unwrap()],
2045	)
2046	.await
2047	.unwrap();
2048
2049	let mut status = String::new();
2050	let mut error = String::new();
2051	for _ in 0..100 {
2052	let row: Option<(String, Option<String>)> =
2053	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2054	.bind(build_id)
2055	.fetch_optional(&pool)
2056	.await
2057	.unwrap();
2058	if let Some((s, e)) = row {
2059	status = s;
2060	error = e.unwrap_or_default();
2061	if status != "running" {
2062	break;
2063	}
2064	}
2065	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2066	}
2067	assert_eq!(status, "failed", "non-monotonic publish must fail the run");
2068	assert!(
2069	error.contains("not newer"),
2070	"expected monotonicity error, got: {error}"
2071	);
2072	// Exactly one release was recorded (0.2.0); 0.1.0 never landed.
2073	let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
2074	.fetch_one(&pool)
2075	.await
2076	.unwrap();
2077	assert_eq!(
2078	count, 1,
2079	"only the first (newer) publish should record a release"
2080	);
2081	// The recorded release carries the artifact's sha256 (64 hex chars),
2082	// not a NULL — the ledger says which bytes shipped.
2083	let hash: Option<String> =
2084	sqlx::query_scalar("SELECT artifact_hash FROM releases WHERE version = '0.2.0'")
2085	.fetch_one(&pool)
2086	.await
2087	.unwrap();
2088	assert!(
2089	hash.as_deref().is_some_and(\|h\| h.len() == 64),
2090	"publish must record the artifact sha256, got {hash:?}"
2091	);
2092	}
2093
2094	/// The artifact-identity fix at collect: a stale, wrongly-versioned artifact
2095	/// left in the output dir fails the collect instead of silently winning a
2096	/// later glob. Here the build is 0.0.1 but the recipe produces a 9.9.9 file.
2097	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2098	async fn collect_rejects_a_stale_versioned_artifact() {
2099	let tmp = tempfile::tempdir().unwrap();
2100	let root = tmp.path();
2101	let repo = root.join("app");
2102	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2103	std::fs::write(
2104	repo.join("src-tauri/tauri.conf.json"),
2105	r#"{"version":"0.0.1"}"#,
2106	)
2107	.unwrap();
2108	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2109	std::fs::write(
2110	repo.join("dist/recipes/linux.rhai"),
2111	r#"
2112	step("build");
2113	let v = version_of("demo");
2114	sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo-9.9.9.bin");
2115	step("collect");
2116	collect("fw13", "REPO/out/demo-9.9.9.bin", "demo", v);
2117	"#
2118	.replace("REPO", repo.to_str().unwrap()),
2119	)
2120	.unwrap();
2121
2122	let cfg = Config::for_tests(root);
2123	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2124	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2125	let topo = Topology::from_str_for_tests(&format!(
2126	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2127	pull_root = \"{repo}\"\n\n[app.demo]\nrepo = \"{repo}\"\n",
2128	repo = repo.display()
2129	))
2130	.unwrap();
2131	let state = test_state(pool.clone(), topo, cfg);
2132	let build_id = start_build(
2133	state.clone(),
2134	AppId::new("demo"),
2135	Version::parse("0.0.1").unwrap(),
2136	vec!["linux/x86_64".parse().unwrap()],
2137	)
2138	.await
2139	.unwrap();
2140
2141	let mut status = String::new();
2142	let mut error = String::new();
2143	for _ in 0..100 {
2144	let row: Option<(String, Option<String>)> =
2145	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2146	.bind(build_id)
2147	.fetch_optional(&pool)
2148	.await
2149	.unwrap();
2150	if let Some((s, e)) = row {
2151	status = s;
2152	error = e.unwrap_or_default();
2153	if status != "running" {
2154	break;
2155	}
2156	}
2157	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2158	}
2159	assert_eq!(
2160	status, "failed",
2161	"a mismatched-version artifact must fail collect"
2162	);
2163	assert!(
2164	error.contains("stale artifact"),
2165	"expected a stale-artifact error, got: {error}"
2166	);
2167	}
2168
2169	/// A command that runs past its step's deadline fails THAT step (and unwinds
2170	/// the recipe) rather than wedging under the old whole-build guillotine. The
2171	/// per-step budget is overridden to 1s here; the recipe then sleeps 30s, so
2172	/// the step deadline — not the sleep — decides the outcome, quickly.
2173	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2174	async fn a_step_that_exceeds_its_deadline_fails() {
2175	let tmp = tempfile::tempdir().unwrap();
2176	let root = tmp.path();
2177	let repo = root.join("app");
2178	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2179	std::fs::write(
2180	repo.join("src-tauri/tauri.conf.json"),
2181	r#"{"version":"0.0.1"}"#,
2182	)
2183	.unwrap();
2184	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2185	std::fs::write(
2186	repo.join("dist/recipes/linux.rhai"),
2187	r#"
2188	step("build");
2189	sh_ok("fw13", "sleep 30");
2190	"#,
2191	)
2192	.unwrap();
2193
2194	let mut cfg = Config::for_tests(root);
2195	cfg.step_timeout_secs = Some(1); // every step's budget -> 1s
2196	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2197	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2198	let topo = Topology::from_str_for_tests(&format!(
2199	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2200	[app.demo]\nrepo = \"{}\"\n",
2201	repo.display()
2202	))
2203	.unwrap();
2204	let state = test_state(pool.clone(), topo, cfg);
2205	let started = std::time::Instant::now();
2206	let build_id = start_build(
2207	state.clone(),
2208	AppId::new("demo"),
2209	Version::parse("0.0.1").unwrap(),
2210	vec!["linux/x86_64".parse().unwrap()],
2211	)
2212	.await
2213	.unwrap();
2214
2215	let mut status = String::new();
2216	let mut error = String::new();
2217	for _ in 0..100 {
2218	let row: Option<(String, Option<String>)> =
2219	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2220	.bind(build_id)
2221	.fetch_optional(&pool)
2222	.await
2223	.unwrap();
2224	if let Some((s, e)) = row {
2225	status = s;
2226	error = e.unwrap_or_default();
2227	if status != "running" {
2228	break;
2229	}
2230	}
2231	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2232	}
2233	assert_eq!(status, "failed", "a step past its deadline must fail");
2234	assert!(
2235	error.contains("per-step deadline"),
2236	"expected a deadline error, got: {error}"
2237	);
2238	// The deadline (1s), not the 30s sleep, decided it — proof the command
2239	// was actually interrupted rather than run to completion.
2240	assert!(
2241	started.elapsed() < std::time::Duration::from_secs(20),
2242	"the step deadline must fire well before the sleep would finish"
2243	);
2244	}
2245
2246	/// A demo app that publishes linux and has the all-targets-green gate on; it
2247	/// declares linux + macos, so publishing linux is gated on macos being green.
2248	async fn gate_state(root: &std::path::Path) -> (AppState, sqlx::SqlitePool) {
2249	let repo = root.join("app");
2250	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2251	std::fs::write(
2252	repo.join("src-tauri/tauri.conf.json"),
2253	r#"{"version":"0.2.0"}"#,
2254	)
2255	.unwrap();
2256	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2257	let artifact = repo.join("out/app.tar.gz");
2258	std::fs::write(
2259	repo.join("dist/recipes/linux.rhai"),
2260	r#"
2261	step("build");
2262	sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
2263	step("publish");
2264	publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
2265	"#
2266	.replace("ARTIFACT", artifact.to_str().unwrap())
2267	.replace("REPO", repo.to_str().unwrap()),
2268	)
2269	.unwrap();
2270	std::fs::write(
2271	repo.join("bento.toml"),
2272	"targets = [\"linux/x86_64\", \"macos/aarch64\"]\nrequire_all_targets = true\n",
2273	)
2274	.unwrap();
2275	let cfg = Config::for_tests(root);
2276	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2277	let topo = Topology::from_str_for_tests(&format!(
2278	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2279	[[host]]\nname = \"mbp\"\nssh = \"mbp\"\ntargets = [\"macos/aarch64\"]\n\
2280	[app.demo]\nrepo = \"{}\"\n",
2281	repo.display()
2282	))
2283	.unwrap();
2284	(test_state(pool.clone(), topo, cfg), pool)
2285	}
2286
2287	async fn await_target(pool: &sqlx::SqlitePool, build_id: i64) -> (String, String) {
2288	for _ in 0..100 {
2289	let row: Option<(String, Option<String>)> =
2290	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2291	.bind(build_id)
2292	.fetch_optional(pool)
2293	.await
2294	.unwrap();
2295	if let Some((s, e)) = row
2296	&& s != "running"
2297	{
2298	return (s, e.unwrap_or_default());
2299	}
2300	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2301	}
2302	panic!("target never settled");
2303	}
2304
2305	/// The all-targets-green gate blocks a partial release: linux tries to
2306	/// publish while macos has no successful run, so publish is refused and the
2307	/// target fails at its publish step.
2308	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2309	async fn all_green_gate_blocks_publish_when_a_sibling_is_not_green() {
2310	let tmp = tempfile::tempdir().unwrap();
2311	let (state, pool) = gate_state(tmp.path()).await;
2312	let build_id = start_build(
2313	state,
2314	AppId::new("demo"),
2315	Version::parse("0.2.0").unwrap(),
2316	vec!["linux/x86_64".parse().unwrap()],
2317	)
2318	.await
2319	.unwrap();
2320	let (status, error) = await_target(&pool, build_id).await;
2321	assert_eq!(status, "failed", "a partial release must be blocked");
2322	assert!(
2323	error.contains("all-targets-green gate"),
2324	"expected the gate to name itself, got: {error}"
2325	);
2326	// Nothing shipped.
2327	let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
2328	.fetch_one(&pool)
2329	.await
2330	.unwrap();
2331	assert_eq!(count, 0, "a gated-off publish records no release");
2332	}
2333
2334	/// With every sibling green, the gate lets the publish through. macos is
2335	/// pre-recorded `ok` for this version, so linux's publish proceeds.
2336	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2337	async fn all_green_gate_allows_publish_when_every_sibling_is_green() {
2338	let tmp = tempfile::tempdir().unwrap();
2339	let (state, pool) = gate_state(tmp.path()).await;
2340	// Pre-record a green macos run at 0.2.0 (as if its target already built).
2341	let bid: i64 = sqlx::query_scalar(
2342	"INSERT INTO builds (app, version, status, created_at) VALUES ('demo','0.2.0','ok','2026-07-23T00:00:00Z') RETURNING id",
2343	)
2344	.fetch_one(&pool)
2345	.await
2346	.unwrap();
2347	sqlx::query(
2348	"INSERT INTO target_runs (build_id, app, version, target, status, started_at)
2349	VALUES (?, 'demo', '0.2.0', 'macos/aarch64', 'ok', '2026-07-23T00:00:00Z')",
2350	)
2351	.bind(bid)
2352	.execute(&pool)
2353	.await
2354	.unwrap();
2355
2356	let build_id = start_build(
2357	state,
2358	AppId::new("demo"),
2359	Version::parse("0.2.0").unwrap(),
2360	vec!["linux/x86_64".parse().unwrap()],
2361	)
2362	.await
2363	.unwrap();
2364	let (status, error) = await_target(&pool, build_id).await;
2365	assert_eq!(
2366	status, "ok",
2367	"all siblings green -> publish proceeds ({error})"
2368	);
2369	let count: i64 =
2370	sqlx::query_scalar("SELECT COUNT(*) FROM releases WHERE target = 'linux/x86_64'")
2371	.fetch_one(&pool)
2372	.await
2373	.unwrap();
2374	assert_eq!(count, 1, "linux published once the gate was satisfied");
2375	}
2376
2377	/// A committed + tagged git repo whose linux recipe pins the host to the tag
2378	/// via `checkout_sha`. `tag` is created only when `Some`.
2379	fn init_git_app(repo: &std::path::Path, tauri_version: &str, tag: Option<&str>) {
2380	init_git_app_with_recipe(
2381	repo,
2382	tauri_version,
2383	tag,
2384	"step(\"checkout\");\nlet s = checkout_sha(build_host());\nlog(\"pinned \" + s);\n\
2385	step(\"build\");\nsh_ok(build_host(), \"true\");\n",
2386	);
2387	}
2388
2389	/// As [`init_git_app`], with the linux recipe spelled by the caller.
2390	fn init_git_app_with_recipe(
2391	repo: &std::path::Path,
2392	tauri_version: &str,
2393	tag: Option<&str>,
2394	recipe: &str,
2395	) {
2396	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2397	std::fs::write(
2398	repo.join("src-tauri/tauri.conf.json"),
2399	format!("{{\"version\":\"{tauri_version}\"}}"),
2400	)
2401	.unwrap();
2402	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2403	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2404	std::fs::write(repo.join("dist/recipes/linux.rhai"), recipe).unwrap();
2405	// Isolate from the dev's global git config (which forces signed tags).
2406	let run = \|args: &[&str]\| {
2407	let out = std::process::Command::new("git")
2408	.args(args)
2409	.current_dir(repo)
2410	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2411	.env("GIT_CONFIG_SYSTEM", "/dev/null")
2412	.output()
2413	.expect("git runs");
2414	assert!(
2415	out.status.success(),
2416	"git {args:?}: {}",
2417	String::from_utf8_lossy(&out.stderr)
2418	);
2419	};
2420	run(&["init", "-q"]);
2421	run(&["-c", "user.email=t@t", "-c", "user.name=t", "add", "-A"]);
2422	run(&[
2423	"-c",
2424	"user.email=t@t",
2425	"-c",
2426	"user.name=t",
2427	"commit",
2428	"-q",
2429	"-m",
2430	"init",
2431	]);
2432	if let Some(t) = tag {
2433	run(&["tag", t]);
2434	}
2435	}
2436
2437	fn one_host_topo(repo: &std::path::Path) -> Topology {
2438	Topology::from_str_for_tests(&format!(
2439	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2440	[app.demo]\nrepo = \"{}\"\n",
2441	repo.display()
2442	))
2443	.unwrap()
2444	}
2445
2446	/// The release preflight (pin on) pins the host to the tag and the build
2447	/// proceeds. Exercises both the barrier and the `checkout_sha` host function.
2448	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2449	async fn release_preflight_pins_and_builds_when_the_tag_is_present() {
2450	let tmp = tempfile::tempdir().unwrap();
2451	let repo = tmp.path().join("demo");
2452	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2453	let mut cfg = Config::for_tests(tmp.path());
2454	cfg.pin_release_sha = true;
2455	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2456	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2457	let build_id = start_build(
2458	state,
2459	AppId::new("demo"),
2460	Version::parse("0.0.1").unwrap(),
2461	vec!["linux/x86_64".parse().unwrap()],
2462	)
2463	.await
2464	.unwrap();
2465	let (status, error) = await_target(&pool, build_id).await;
2466	assert_eq!(status, "ok", "a pinned build should succeed ({error})");
2467	}
2468
2469	/// A pinned build writes an artifact record beside what it collected: the
2470	/// manifest of those bytes, the commit the preflight pinned, and the steps
2471	/// as artifact-scoped gates.
2472	///
2473	/// The three facts already existed and met nowhere — the per-file sha256 at
2474	/// `collect`, the sha at the preflight, the step outcomes in `step_runs` —
2475	/// which is how gates came to vouch for one thing while the deploy shipped
2476	/// another.
2477	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2478	async fn a_pinned_build_writes_an_artifact_record_for_what_it_collected() {
2479	let tmp = tempfile::tempdir().unwrap();
2480	let repo = tmp.path().join("demo");
2481	std::fs::create_dir_all(&repo).unwrap();
2482	let recipe = r#"
2483	step("checkout");
2484	let sha = checkout_sha(build_host());
2485	log("pinned " + sha);
2486	step("build");
2487	sh_ok(build_host(), "mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
2488	step("collect");
2489	collect(build_host(), "REPO/out/demo.bin", "demo", version());
2490	"#
2491	.replace("REPO", repo.to_str().unwrap());
2492	init_git_app_with_recipe(&repo, "0.0.1", Some("v0.0.1"), &recipe);
2493
2494	let mut cfg = Config::for_tests(tmp.path());
2495	cfg.pin_release_sha = true;
2496	let dist_root = cfg.dist_root.clone();
2497	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2498	let topo = Topology::from_str_for_tests(&format!(
2499	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2500	pull_root = \"{repo}\"\n[app.demo]\nrepo = \"{repo}\"\n",
2501	repo = repo.display()
2502	))
2503	.unwrap();
2504	let state = test_state(pool.clone(), topo, cfg);
2505	let build_id = start_build(
2506	state,
2507	AppId::new("demo"),
2508	Version::parse("0.0.1").unwrap(),
2509	vec!["linux/x86_64".parse().unwrap()],
2510	)
2511	.await
2512	.unwrap();
2513	let (status, error) = await_target(&pool, build_id).await;
2514	assert_eq!(status, "ok", "the build should succeed ({error})");
2515
2516	let path = crate::artifact_record::record_path(
2517	&dist_root,
2518	&AppId::new("demo"),
2519	&Version::parse("0.0.1").unwrap(),
2520	"linux/x86_64".parse().unwrap(),
2521	);
2522	let json = std::fs::read_to_string(&path)
2523	.unwrap_or_else(\|e\| panic!("record at {}: {e}", path.display()));
2524	// Parsing revalidates, so this also asserts the digest matches the
2525	// manifest and no environment-scoped gate slipped in.
2526	let record = ops_artifact::ArtifactRecord::parse(&json).unwrap();
2527
2528	assert_eq!(record.producer, "bento");
2529	assert_eq!(record.manifest.entries().len(), 1);
2530	assert_eq!(record.manifest.entries()[0].path, "demo.bin");
2531	assert_eq!(record.digest, record.manifest.digest());
2532
2533	// The provenance names the commit the preflight pinned, not a rebuild of
2534	// whatever the branch is now.
2535	let head = std::process::Command::new("git")
2536	.args(["rev-parse", "v0.0.1^{commit}"])
2537	.current_dir(&repo)
2538	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2539	.output()
2540	.unwrap();
2541	let head = String::from_utf8_lossy(&head.stdout).trim().to_string();
2542	assert_eq!(record.provenance.git_sha, head);
2543	assert_eq!(record.provenance.target, "linux/x86_64");
2544	assert_eq!(record.provenance.build_host, "fw13");
2545	assert!(!record.provenance.toolchain.is_empty());
2546
2547	let gates: Vec<&str> = record.gates.iter().map(\|g\| g.gate.as_str()).collect();
2548	assert_eq!(gates, ["checkout", "build", "collect"]);
2549	assert!(record.all_gates_passed());
2550	assert!(
2551	record
2552	.gates
2553	.iter()
2554	.all(\|g\| g.scope == ops_artifact::Scope::Artifact),
2555	"a build host cannot vouch for an environment"
2556	);
2557	}
2558
2559	/// The barrier refuses a target no host can build, instead of leaving it out
2560	/// of the pin and letting the remaining hosts vouch for the release.
2561	///
2562	/// The hole this closes is silence reading as agreement: the comparison used
2563	/// to run over the hosts that reported, so a target dropped for want of a
2564	/// host still built while the other hosts' unanimity looked like proof the
2565	/// whole release came from one commit.
2566	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2567	async fn release_preflight_refuses_a_target_no_host_can_build() {
2568	let tmp = tempfile::tempdir().unwrap();
2569	let repo = tmp.path().join("demo");
2570	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2571	let mut cfg = Config::for_tests(tmp.path());
2572	cfg.pin_release_sha = true;
2573	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2574	// The topology declares fw13 (linux/x86_64) only, so macOS has no host.
2575	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2576	let err = start_build(
2577	state,
2578	AppId::new("demo"),
2579	Version::parse("0.0.1").unwrap(),
2580	vec!["macos/aarch64".parse().unwrap()],
2581	)
2582	.await
2583	.unwrap_err();
2584	let msg = format!("{err:#}");
2585	assert!(
2586	msg.contains("no host can build macos/aarch64"),
2587	"the error must name the unbuildable target, got: {msg}"
2588	);
2589	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2590	.fetch_one(&pool)
2591	.await
2592	.unwrap();
2593	assert_eq!(builds, 0, "a refused preflight writes no build row");
2594	}
2595
2596	/// The preflight refuses a host whose tree has uncommitted changes to
2597	/// tracked files.
2598	///
2599	/// This is the hole the rev-parse barrier cannot see. `git checkout <tag>`
2600	/// does NOT fail on local modifications to files whose content is unchanged
2601	/// in the tag — it succeeds and keeps them. So a dirty host builds its edits
2602	/// while honestly reporting the tagged sha, and a clean host builds
2603	/// something else: two architectures, two different binaries, one tag
2604	/// containing neither combination. Caught on pom's first release, where
2605	/// fw13 had uncommitted changes in the serve path and astra did not.
2606	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2607	async fn release_preflight_refuses_a_dirty_working_tree() {
2608	let tmp = tempfile::tempdir().unwrap();
2609	let repo = tmp.path().join("demo");
2610	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2611
2612	// Modify a TRACKED file, exactly as an editor session would.
2613	let tracked = repo.join("src-tauri/tauri.conf.json");
2614	let body = std::fs::read_to_string(&tracked).unwrap();
2615	std::fs::write(&tracked, format!("{body}\n")).unwrap();
2616
2617	let mut cfg = Config::for_tests(tmp.path());
2618	cfg.pin_release_sha = true;
2619	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2620	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2621	let err = start_build(
2622	state,
2623	AppId::new("demo"),
2624	Version::parse("0.0.1").unwrap(),
2625	vec!["linux/x86_64".parse().unwrap()],
2626	)
2627	.await
2628	.unwrap_err();
2629	let msg = format!("{err:#}");
2630	assert!(
2631	msg.contains("uncommitted changes") && msg.contains("tauri.conf.json"),
2632	"the error must name the host and the files, got: {msg}"
2633	);
2634	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2635	.fetch_one(&pool)
2636	.await
2637	.unwrap();
2638	assert_eq!(builds, 0, "a refused preflight writes no build row");
2639	}
2640
2641	/// An UNTRACKED file does not fail a release. A build host accumulates
2642	/// editor scratch and stray logs, none of which reach the binary, so failing
2643	/// on them would be noise that trains an operator to bypass the gate.
2644	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2645	async fn release_preflight_tolerates_untracked_files() {
2646	let tmp = tempfile::tempdir().unwrap();
2647	let repo = tmp.path().join("demo");
2648	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2649	std::fs::write(repo.join("scratch.log"), "noise").unwrap();
2650
2651	let mut cfg = Config::for_tests(tmp.path());
2652	cfg.pin_release_sha = true;
2653	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2654	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2655	let build_id = start_build(
2656	state,
2657	AppId::new("demo"),
2658	Version::parse("0.0.1").unwrap(),
2659	vec!["linux/x86_64".parse().unwrap()],
2660	)
2661	.await
2662	.unwrap();
2663	let (status, error) = await_target(&pool, build_id).await;
2664	assert_eq!(
2665	status, "ok",
2666	"untracked files must not fail a release ({error})"
2667	);
2668	}
2669
2670	/// The preflight refuses the build (before any row is written) when the
2671	/// release tag does not exist on the host — a missing/unpushed tag can't
2672	/// silently fall back to whatever `main` is.
2673	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2674	async fn release_preflight_refuses_when_the_release_tag_is_missing() {
2675	let tmp = tempfile::tempdir().unwrap();
2676	let repo = tmp.path().join("demo");
2677	// App is 0.0.2 (so the version check passes) but only v0.0.1 is tagged.
2678	init_git_app(&repo, "0.0.2", Some("v0.0.1"));
2679	let mut cfg = Config::for_tests(tmp.path());
2680	cfg.pin_release_sha = true;
2681	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2682	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2683	let err = start_build(
2684	state,
2685	AppId::new("demo"),
2686	Version::parse("0.0.2").unwrap(),
2687	vec!["linux/x86_64".parse().unwrap()],
2688	)
2689	.await
2690	.unwrap_err();
2691	let msg = format!("{err:#}");
2692	assert!(
2693	msg.contains("release preflight") && msg.contains("v0.0.2"),
2694	"expected a preflight tag error, got: {msg}"
2695	);
2696	assert!(
2697	msg.contains("does not exist"),
2698	"the error should name the absent tag as the cause, got: {msg}"
2699	);
2700	// Refused before anything was recorded.
2701	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2702	.fetch_one(&pool)
2703	.await
2704	.unwrap();
2705	assert_eq!(builds, 0, "a refused preflight writes no build row");
2706	}
2707
2708	/// The branch a release was launched from is still checked out afterwards.
2709	///
2710	/// The preflight pins every host to `v<version>`, which detaches HEAD. That
2711	/// is correct during the build and wrong to leave behind: git does not warn,
2712	/// and later commits succeed while belonging to no branch. makeover shipped
2713	/// 2.3.0 from exactly that state on 2026-07-28 — the published commit existed
2714	/// only as a detached HEAD on one machine, on no branch and no remote.
2715	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2716	async fn a_release_leaves_the_checkout_on_its_branch() {
2717	let tmp = tempfile::tempdir().unwrap();
2718	let repo = tmp.path().join("demo");
2719	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2720	let branch_before = current_branch(&repo);
2721	assert!(!branch_before.is_empty(), "test repo starts on a branch");
2722
2723	let mut cfg = Config::for_tests(tmp.path());
2724	cfg.pin_release_sha = true;
2725	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2726	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2727	let build_id = start_build(
2728	state,
2729	AppId::new("demo"),
2730	Version::parse("0.0.1").unwrap(),
2731	vec!["linux/x86_64".parse().unwrap()],
2732	)
2733	.await
2734	.unwrap();
2735	let (status, error) = await_target(&pool, build_id).await;
2736	assert_eq!(status, "ok", "the build itself should pass ({error})");
2737
2738	// finalize_build restores after the targets settle, so give it a moment.
2739	for _ in 0..50 {
2740	if current_branch(&repo) == branch_before {
2741	break;
2742	}
2743	tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2744	}
2745	assert_eq!(
2746	current_branch(&repo),
2747	branch_before,
2748	"the release must put the checkout back on its branch, not leave it detached"
2749	);
2750	}
2751
2752	/// A checkout that is ALREADY detached is refused, rather than released from
2753	/// and left detached again. It means an earlier release never cleaned up, and
2754	/// anything committed since is on no branch — which is precisely the state
2755	/// that has to be looked at by a human before more releases pile onto it.
2756	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2757	async fn a_detached_checkout_is_refused_before_it_is_pinned_again() {
2758	let tmp = tempfile::tempdir().unwrap();
2759	let repo = tmp.path().join("demo");
2760	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2761	// Detach, standing in for a checkout an earlier release left on its tag.
2762	let out = std::process::Command::new("git")
2763	.args(["checkout", "--detach", "-q", "HEAD"])
2764	.current_dir(&repo)
2765	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2766	.env("GIT_CONFIG_SYSTEM", "/dev/null")
2767	.output()
2768	.expect("git runs");
2769	assert!(out.status.success());
2770	assert!(current_branch(&repo).is_empty(), "repo is detached");
2771
2772	let mut cfg = Config::for_tests(tmp.path());
2773	cfg.pin_release_sha = true;
2774	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2775	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2776	let err = start_build(
2777	state,
2778	AppId::new("demo"),
2779	Version::parse("0.0.1").unwrap(),
2780	vec!["linux/x86_64".parse().unwrap()],
2781	)
2782	.await
2783	.unwrap_err();
2784	let msg = format!("{err:#}");
2785	assert!(
2786	msg.contains("detached HEAD"),
2787	"the refusal should name the detached checkout, got: {msg}"
2788	);
2789	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2790	.fetch_one(&pool)
2791	.await
2792	.unwrap();
2793	assert_eq!(builds, 0, "a refused preflight writes no build row");
2794	}
2795
2796	/// The branch `repo` is on, empty on a detached HEAD.
2797	fn current_branch(repo: &std::path::Path) -> String {
2798	let out = std::process::Command::new("git")
2799	.args(["symbolic-ref", "-q", "--short", "HEAD"])
2800	.current_dir(repo)
2801	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2802	.env("GIT_CONFIG_SYSTEM", "/dev/null")
2803	.output()
2804	.expect("git runs");
2805	String::from_utf8_lossy(&out.stdout).trim().to_string()
2806	}
2807
2808	/// An unreachable remote does not fail a release whose tag is present.
2809	///
2810	/// The preflight used to run `fetch --all --tags --prune && checkout`, and
2811	/// `fetch --all` is non-zero if ANY remote fails. Every library repo carries
2812	/// three (`astra`, `mnw`, `srht`), so one dead mirror aborted the release and
2813	/// blamed it on a missing tag — which is how makeover v2.1.1 failed on
2814	/// 2026-07-26 four minutes after its tag was created. Fetch is advisory now;
2815	/// only the checkout decides.
2816	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2817	async fn a_dead_remote_does_not_fail_a_release_whose_tag_exists() {
2818	let tmp = tempfile::tempdir().unwrap();
2819	let repo = tmp.path().join("demo");
2820	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2821	// A remote that cannot possibly be fetched, standing in for an offline
2822	// astra or an srht mirror the repo was never pushed to.
2823	let out = std::process::Command::new("git")
2824	.args([
2825	"remote",
2826	"add",
2827	"srht",
2828	&tmp.path().join("nowhere.git").display().to_string(),
2829	])
2830	.current_dir(&repo)
2831	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2832	.env("GIT_CONFIG_SYSTEM", "/dev/null")
2833	.output()
2834	.expect("git runs");
2835	assert!(out.status.success());
2836
2837	let mut cfg = Config::for_tests(tmp.path());
2838	cfg.pin_release_sha = true;
2839	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2840	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2841	let build_id = start_build(
2842	state,
2843	AppId::new("demo"),
2844	Version::parse("0.0.1").unwrap(),
2845	vec!["linux/x86_64".parse().unwrap()],
2846	)
2847	.await
2848	.expect("a dead mirror must not refuse the release");
2849	let (status, error) = await_target(&pool, build_id).await;
2850	assert_eq!(status, "ok", "the tag is present, so this builds ({error})");
2851	}
2852
2853	/// The audit fix: a `build` step dispatched to a host whose executor lacks the
2854	/// `build` capability is denied at the transport BEFORE the command runs. This
2855	/// is the structural guarantee behind "never build on prod" — a recipe naming
2856	/// the wrong host can't compile there.
2857	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2858	async fn build_on_a_host_without_the_build_grant_is_denied() {
2859	let tmp = tempfile::tempdir().unwrap();
2860	let root = tmp.path();
2861
2862	let repo = root.join("app");
2863	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2864	std::fs::write(
2865	repo.join("src-tauri/tauri.conf.json"),
2866	r#"{"version":"0.0.1"}"#,
2867	)
2868	.unwrap();
2869	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2870	// The recipe (wrongly) tries to compile on `prod`, a host with no build
2871	// grant. A marker file would appear if the command actually ran.
2872	let marker = root.join("ran-on-prod");
2873	std::fs::write(
2874	repo.join("dist/recipes/linux.rhai"),
2875	r#"
2876	step("build");
2877	sh_ok("prod", "touch MARKER");
2878	"#
2879	.replace("MARKER", marker.to_str().unwrap()),
2880	)
2881	.unwrap();
2882
2883	let cfg = Config::for_tests(root);
2884	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2885	// fw13 builds linux; `prod` is local-but-restart-only (no build/package).
2886	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2887	let topo = Topology::from_str_for_tests(
2888	r#"
2889	[[host]]
2890	name = "fw13"
2891	ssh = "local"
2892	targets = ["linux/x86_64"]
2893
2894	[[host]]
2895	name = "prod"
2896	ssh = "local"
2897	actuate = ["restart"]
2898	observe = []
2899
2900	[app.demo]
2901	repo = "REPO"
2902	"#
2903	.replace("REPO", repo.to_str().unwrap())
2904	.as_str(),
2905	)
2906	.unwrap();
2907
2908	let state = test_state(pool.clone(), topo, cfg);
2909
2910	let build_id = start_build(
2911	state.clone(),
2912	AppId::new("demo"),
2913	Version::parse("0.0.1").unwrap(),
2914	vec!["linux/x86_64".parse().unwrap()],
2915	)
2916	.await
2917	.unwrap();
2918
2919	let mut status = String::new();
2920	let mut error = String::new();
2921	for _ in 0..100 {
2922	let row: Option<(String, Option<String>)> =
2923	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2924	.bind(build_id)
2925	.fetch_optional(&pool)
2926	.await
2927	.unwrap();
2928	if let Some((s, e)) = row {
2929	status = s;
2930	error = e.unwrap_or_default();
2931	if status != "running" {
2932	break;
2933	}
2934	}
2935	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2936	}
2937
2938	assert_eq!(status, "failed", "build on an ungranted host must fail");
2939	assert!(
2940	error.contains("capability denied") && error.contains("build"),
2941	"failure must be a capability denial, got: {error}"
2942	);
2943	assert!(!marker.exists(), "denied build step must NOT have executed");
2944	}
2945
2946	// ---- macOS sign / notarize / staple execution chain (via ScriptedExec) ----
2947
2948	/// Stand up a single-macOS-target app whose recipe is `recipe_body` (with the
2949	/// literal `ARTIFACT` replaced by a real, non-empty file on disk), dispatch
2950	/// every host command through `scripted`, run the build to a terminal state,
2951	/// and return the pool plus the final `(status, error)`. The returned
2952	/// [`tempfile::TempDir`] must be kept alive by the caller: it holds the
2953	/// sqlite DB the pool reads.
2954	async fn run_macos_recipe(
2955	scripted: Arc<ScriptedExec>,
2956	recipe_body: &str,
2957	backoff_secs: Option<u64>,
2958	) -> (tempfile::TempDir, SqlitePool, String, String) {
2959	let tmp = tempfile::tempdir().unwrap();
2960	let root = tmp.path();
2961	let repo = root.join("app");
2962	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2963	std::fs::write(
2964	repo.join("src-tauri/tauri.conf.json"),
2965	r#"{"version":"0.0.1"}"#,
2966	)
2967	.unwrap();
2968	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2969
2970	// A real, non-empty artifact so `publish`'s size floor is satisfied; the
2971	// build step is faked, so nothing else creates it.
2972	let artifact = repo.join("out/demo.dmg");
2973	std::fs::create_dir_all(artifact.parent().unwrap()).unwrap();
2974	std::fs::write(&artifact, b"dmg-bytes").unwrap();
2975
2976	std::fs::write(
2977	repo.join("dist/recipes/macos.rhai"),
2978	recipe_body.replace("ARTIFACT", artifact.to_str().unwrap()),
2979	)
2980	.unwrap();
2981
2982	let cfg = Config {
2983	notarize_backoff_secs: backoff_secs,
2984	..Config::for_tests(root)
2985	};
2986	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2987	std::fs::write(repo.join("bento.toml"), "targets = [\"macos/aarch64\"]\n").unwrap();
2988	let topo = Topology::from_str_for_tests(&format!(
2989	r#"
2990	[[host]]
2991	name = "mbp"
2992	ssh = "local"
2993	targets = ["macos/aarch64"]
2994
2995	[app.demo]
2996	repo = "{}"
2997	"#,
2998	repo.display()
2999	))
3000	.unwrap();
3001
3002	let mut state = test_state(pool.clone(), topo, cfg);
3003	// Route every host command through the scripted executor.
3004	let mut execs = HashMap::new();
3005	execs.insert("mbp".to_string(), scripted as Arc<dyn Executor>);
3006	state.executors = Arc::new(execs);
3007
3008	let build_id = start_build(
3009	state.clone(),
3010	AppId::new("demo"),
3011	Version::parse("0.0.1").unwrap(),
3012	vec!["macos/aarch64".parse().unwrap()],
3013	)
3014	.await
3015	.unwrap();
3016
3017	let mut status = String::new();
3018	let mut error = String::new();
3019	for _ in 0..100 {
3020	let row: Option<(String, Option<String>)> =
3021	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
3022	.bind(build_id)
3023	.fetch_optional(&pool)
3024	.await
3025	.unwrap();
3026	if let Some((s, e)) = row {
3027	status = s;
3028	error = e.unwrap_or_default();
3029	if status != "running" {
3030	break;
3031	}
3032	}
3033	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3034	}
3035	(tmp, pool, status, error)
3036	}
3037
3038	async fn release_count(pool: &SqlitePool) -> i64 {
3039	sqlx::query_scalar("SELECT COUNT(*) FROM releases")
3040	.fetch_one(pool)
3041	.await
3042	.unwrap()
3043	}
3044
3045	/// The whole macOS release chain end to end through a fake host: codesign,
3046	/// notarize (Accepted first try), staple, verify_gatekeeper, then publish.
3047	/// The recorded commands lock the exact incantations each host function
3048	/// dispatches, and a `releases` row proves the publish gate opened for a
3049	/// signed + notarized + Gatekeeper-accepted artifact.
3050	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3051	async fn a_macos_recipe_signs_notarizes_staples_verifies_and_publishes() {
3052	let scripted = Arc::new(
3053	ScriptedExec::new()
3054	.on("codesign", 0, "")
3055	.on("notarytool", 0, r#"{"status":"Accepted"}"#)
3056	.on("stapler staple", 0, "")
3057	.on(
3058	"spctl",
3059	0,
3060	"source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
3061	),
3062	);
3063	let recipe = r#"
3064	let h = build_host();
3065	step("build");
3066	sh_ok(h, "echo built");
3067	step("sign");
3068	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3069	notarize(h, "ARTIFACT");
3070	staple(h, "ARTIFACT");
3071	step("verify");
3072	if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
3073	step("publish");
3074	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3075	"#;
3076	let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, None).await;
3077
3078	assert_eq!(
3079	status, "ok",
3080	"signed+notarized macOS build should publish: {error}"
3081	);
3082	assert_eq!(
3083	release_count(&pool).await,
3084	1,
3085	"publish must record a release"
3086	);
3087
3088	// The exact shell incantations each host function dispatched.
3089	let cmds = scripted.commands();
3090	let has = \|needle: &str\| cmds.iter().any(\|c\| c.contains(needle));
3091	assert!(
3092	has("codesign --force --options runtime --timestamp --sign"),
3093	"codesign runtime+timestamp incantation, got: {cmds:?}"
3094	);
3095	assert!(
3096	has("xcrun notarytool submit"),
3097	"notarytool submit: {cmds:?}"
3098	);
3099	assert!(
3100	has("--wait --output-format json"),
3101	"notarytool --wait json: {cmds:?}"
3102	);
3103	assert!(has("xcrun stapler staple"), "stapler staple: {cmds:?}");
3104	assert!(has("spctl --assess"), "gatekeeper assess: {cmds:?}");
3105	}
3106
3107	/// A failing `codesign` fails the sign step and aborts the recipe before
3108	/// publish — an unsigned artifact never ships.
3109	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3110	async fn a_codesign_failure_fails_the_sign_step_and_blocks_publish() {
3111	let scripted = Arc::new(ScriptedExec::new().on("codesign", 1, ""));
3112	let recipe = r#"
3113	let h = build_host();
3114	step("build");
3115	sh_ok(h, "echo built");
3116	step("sign");
3117	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3118	step("publish");
3119	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3120	"#;
3121	let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
3122
3123	assert_eq!(status, "failed", "a failed codesign must fail the target");
3124	assert!(
3125	error.contains("codesign failed"),
3126	"error names the codesign failure, got: {error}"
3127	);
3128	assert_eq!(
3129	release_count(&pool).await,
3130	0,
3131	"nothing may publish after a codesign failure"
3132	);
3133	}
3134
3135	/// Even if the recipe ignores `verify_gatekeeper`'s returned `false`, the
3136	/// publish gate refuses the artifact: `verify_gatekeeper` both records the
3137	/// rejection and fails its step, and `publish` proves neither passed. This
3138	/// is the defense-in-depth the pure `PublishAuthority::prove` tests assert in
3139	/// isolation, here exercised through the real host-function path.
3140	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3141	async fn a_gatekeeper_rejection_bars_publish_even_if_the_recipe_ignores_it() {
3142	let scripted = Arc::new(
3143	ScriptedExec::new()
3144	.on("codesign", 0, "")
3145	.on("notarytool", 0, r#"{"status":"Accepted"}"#)
3146	.on("stapler staple", 0, "")
3147	// Gatekeeper says no: the sentinel is FAIL, not OK.
3148	.on("spctl", 0, "source=Unnotarized\nBENTO_GATEKEEPER_FAIL"),
3149	);
3150	let recipe = r#"
3151	let h = build_host();
3152	step("build");
3153	sh_ok(h, "echo built");
3154	step("sign");
3155	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3156	notarize(h, "ARTIFACT");
3157	staple(h, "ARTIFACT");
3158	step("verify");
3159	verify_gatekeeper(h, "ARTIFACT");
3160	step("publish");
3161	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3162	"#;
3163	let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
3164
3165	assert_eq!(
3166	status, "failed",
3167	"a Gatekeeper-rejected artifact must not publish"
3168	);
3169	assert!(
3170	!error.is_empty(),
3171	"the barred publish must surface an error"
3172	);
3173	assert_eq!(
3174	release_count(&pool).await,
3175	0,
3176	"no release for a rejected artifact"
3177	);
3178	}
3179
3180	/// The one flaky, network-bound step: `notarize` retries a non-`Accepted`
3181	/// result and succeeds on a later attempt. Two notarytool calls (reject then
3182	/// accept) then a recorded release prove the retry ran and the chain
3183	/// completed. Backoff is 0 so the retry sleep doesn't stall the test.
3184	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3185	async fn notarize_retries_a_non_accepted_result_then_succeeds() {
3186	let scripted = Arc::new(
3187	ScriptedExec::new()
3188	.on("codesign", 0, "")
3189	.on_seq(
3190	"notarytool",
3191	&[
3192	(0, r#"{"status":"In Progress"}"#),
3193	(0, r#"{"status":"Accepted"}"#),
3194	],
3195	)
3196	.on("stapler staple", 0, "")
3197	.on(
3198	"spctl",
3199	0,
3200	"source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
3201	),
3202	);
3203	let recipe = r#"
3204	let h = build_host();
3205	step("build");
3206	sh_ok(h, "echo built");
3207	step("sign");
3208	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3209	notarize(h, "ARTIFACT");
3210	staple(h, "ARTIFACT");
3211	step("verify");
3212	if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
3213	step("publish");
3214	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3215	"#;
3216	let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
3217
3218	assert_eq!(
3219	status, "ok",
3220	"notarize should succeed on the retry: {error}"
3221	);
3222	assert_eq!(
3223	release_count(&pool).await,
3224	1,
3225	"the retried build still publishes"
3226	);
3227	let notary_calls = scripted
3228	.commands()
3229	.iter()
3230	.filter(\|c\| c.contains("notarytool"))
3231	.count();
3232	assert_eq!(
3233	notary_calls, 2,
3234	"notarytool ran once, was rejected, then ran again"
3235	);
3236	}
3237
3238	/// `notarize` gives up after its bounded retries: three notarytool attempts,
3239	/// all non-`Accepted`, fail the target and bar publish.
3240	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3241	async fn notarize_fails_the_target_after_exhausting_its_retries() {
3242	let scripted = Arc::new(
3243	ScriptedExec::new()
3244	.on("codesign", 0, "")
3245	// Every attempt reports a still-pending status, never Accepted.
3246	.on("notarytool", 0, r#"{"status":"In Progress"}"#),
3247	);
3248	let recipe = r#"
3249	let h = build_host();
3250	step("build");
3251	sh_ok(h, "echo built");
3252	step("sign");
3253	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3254	notarize(h, "ARTIFACT");
3255	step("publish");
3256	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3257	"#;
3258	let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
3259
3260	assert_eq!(
3261	status, "failed",
3262	"exhausted notarization must fail the target"
3263	);
3264	assert!(
3265	error.contains("notarization failed after 3 attempts"),
3266	"error names the exhausted retry, got: {error}"
3267	);
3268	assert_eq!(
3269	release_count(&pool).await,
3270	0,
3271	"an unnotarized artifact never publishes"
3272	);
3273	let notary_calls = scripted
3274	.commands()
3275	.iter()
3276	.filter(\|c\| c.contains("notarytool"))
3277	.count();
3278	assert_eq!(notary_calls, 3, "the retry is bounded at three attempts");
3279	}
3280	// ---- item (5): SSH and Agent transports end to end, via a recording fake ----
3281	//
3282	// Every other test declares `ssh = "local"`, so the runner's two-plane
3283	// routing is only exercised at construction
3284	// (state::agent_host_syncs_over_ssh_never_the_agent): steps run over the
3285	// EXEC transport (state.executors -- SshExec, or the in-session AgentRpc for
3286	// a mac host) while artifacts move over the SYNC transport (state.syncs --
3287	// always ssh, NEVER the agent, whose confined `/pull` would 404 or force
3288	// `pull_root` wide enough to expose `~/.tauri/passwords.env`). These drive a
3289	// real recipe through `start_build` on a NON-local topology, replace the
3290	// real ssh/agent transports with a recording fake, and assert which plane
3291	// handled which operation -- the runtime form of that construction-time
3292	// invariant, on the ssh string every other test pins to "local".
3293
3294	/// Records shell commands and artifact pulls on separate logs, so a test can
3295	/// prove the exec transport built/signed and the sync transport collected --
3296	/// and that neither did the other's job -- without a live host or ssh.
3297	struct RecordingExec {
3298	caps: CapabilitySet,
3299	commands: Arc<std::sync::Mutex<Vec<String>>>,
3300	pulls: Arc<std::sync::Mutex<Vec<String>>>,
3301	}
3302
3303	impl RecordingExec {
3304	fn new() -> Arc<Self> {
3305	Arc::new(Self {
3306	// A mac build host's real grant. Nothing in this fake gates on it
3307	// (the real transports do), but keep it coherent so
3308	// `capabilities()` is not a lie.
3309	caps: CapabilitySet::from_tokens(
3310	["build", "sign", "notarize", "staple"],
3311	["build-log", "artifact"],
3312	),
3313	commands: Arc::new(std::sync::Mutex::new(Vec::new())),
3314	pulls: Arc::new(std::sync::Mutex::new(Vec::new())),
3315	})
3316	}
3317	fn commands(&self) -> Vec<String> {
3318	self.commands.lock().unwrap().clone()
3319	}
3320	fn pulls(&self) -> Vec<String> {
3321	self.pulls.lock().unwrap().clone()
3322	}
3323	}
3324
3325	#[async_trait]
3326	impl Executor for RecordingExec {
3327	async fn run_streaming(
3328	&self,
3329	step: &ops_exec::Step,
3330	_sink: &mut dyn LogSink,
3331	) -> anyhow::Result<RunOutput> {
3332	self.commands
3333	.lock()
3334	.unwrap()
3335	.push(step.argv.last().cloned().unwrap_or_default());
3336	Ok(RunOutput {
3337	status: std::process::ExitStatus::from_raw(0),
3338	stdout: Vec::new(),
3339	stderr: Vec::new(),
3340	})
3341	}
3342	async fn pull_file(
3343	&self,
3344	r: &std::path::Path,
3345	_l: &std::path::Path,
3346	_o: &SyncOpts,
3347	) -> anyhow::Result<()> {
3348	self.pulls
3349	.lock()
3350	.unwrap()
3351	.push(r.to_string_lossy().into_owned());
3352	Ok(())
3353	}
3354	async fn pull_dir(
3355	&self,
3356	r: &std::path::Path,
3357	_l: &std::path::Path,
3358	_o: &SyncOpts,
3359	) -> anyhow::Result<()> {
3360	self.pulls
3361	.lock()
3362	.unwrap()
3363	.push(r.to_string_lossy().into_owned());
3364	Ok(())
3365	}
3366	async fn pull_glob(
3367	&self,
3368	g: &str,
3369	_l: &std::path::Path,
3370	_o: &SyncOpts,
3371	) -> anyhow::Result<()> {
3372	self.pulls.lock().unwrap().push(g.to_string());
3373	Ok(())
3374	}
3375	async fn push_dir(
3376	&self,
3377	_l: &std::path::Path,
3378	_r: &std::path::Path,
3379	_o: &SyncOpts,
3380	) -> anyhow::Result<()> {
3381	Ok(())
3382	}
3383	async fn preflight(&self) -> anyhow::Result<()> {
3384	Ok(())
3385	}
3386	fn capabilities(&self) -> &CapabilitySet {
3387	&self.caps
3388	}
3389	}
3390
3391	/// Stand up a single-target app (host named `h1`) whose recipe is
3392	/// `recipe_body` (with `REPO` replaced by the checkout path), inject
3393	/// `exec_fake` as the host's EXEC transport and `sync_fake` as its SYNC
3394	/// transport, run the build to a terminal state, and return the tmpdir plus
3395	/// `(status, error)`. Unlike `test_state`'s `build_executors`, this replaces
3396	/// BOTH planes so no real ssh/agent transport is dialed. The returned
3397	/// [`tempfile::TempDir`] holds the sqlite DB and must outlive the caller's
3398	/// assertions.
3399	async fn run_two_plane(
3400	host_toml: &str,
3401	target: &str,
3402	recipe_file: &str,
3403	recipe_body: &str,
3404	exec_fake: Arc<dyn Executor>,
3405	sync_fake: Arc<dyn Executor>,
3406	) -> (tempfile::TempDir, String, String) {
3407	let tmp = tempfile::tempdir().unwrap();
3408	let root = tmp.path();
3409	let repo = root.join("app");
3410	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3411	std::fs::write(
3412	repo.join("src-tauri/tauri.conf.json"),
3413	r#"{"version":"0.0.1"}"#,
3414	)
3415	.unwrap();
3416	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
3417	std::fs::write(
3418	repo.join("dist/recipes").join(recipe_file),
3419	recipe_body.replace("REPO", repo.to_str().unwrap()),
3420	)
3421	.unwrap();
3422	std::fs::write(
3423	repo.join("bento.toml"),
3424	format!("targets = [\"{target}\"]\n"),
3425	)
3426	.unwrap();
3427
3428	let cfg = Config::for_tests(root);
3429	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3430	let topo = Topology::from_str_for_tests(&format!(
3431	"{}\n[app.demo]\nrepo = \"{}\"\n",
3432	host_toml.replace("REPO", repo.to_str().unwrap()),
3433	repo.display()
3434	))
3435	.unwrap();
3436
3437	let mut state = test_state(pool.clone(), topo, cfg);
3438	state.executors = Arc::new(HashMap::from([("h1".to_string(), exec_fake)]));
3439	state.syncs = Arc::new(HashMap::from([("h1".to_string(), sync_fake)]));
3440
3441	let build_id = start_build(
3442	state.clone(),
3443	AppId::new("demo"),
3444	Version::parse("0.0.1").unwrap(),
3445	vec![target.parse().unwrap()],
3446	)
3447	.await
3448	.unwrap();
3449
3450	let mut status = String::new();
3451	let mut error = String::new();
3452	for _ in 0..100 {
3453	let row: Option<(String, Option<String>)> =
3454	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
3455	.bind(build_id)
3456	.fetch_optional(&pool)
3457	.await
3458	.unwrap();
3459	if let Some((s, e)) = row {
3460	status = s;
3461	error = e.unwrap_or_default();
3462	if status != "running" {
3463	break;
3464	}
3465	}
3466	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3467	}
3468	(tmp, status, error)
3469	}
3470
3471	/// An agent (macOS) host signs over the AGENT transport but is collected
3472	/// from over SSH -- driven through the whole runner, not just `build_sync`.
3473	/// The sign chain's commands land on the exec plane and the artifact pull on
3474	/// the sync plane; crucially, the agent plane is asked to move NOTHING (a
3475	/// regression to one transport would route collect at `AgentRpc::pull_glob`,
3476	/// refused by design, or widen `pull_root` over the secret-bearing home dir).
3477	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3478	async fn an_agent_host_signs_over_the_agent_and_collects_over_ssh_end_to_end() {
3479	let agent = RecordingExec::new(); // exec plane (AgentRpc in prod)
3480	let ssh = RecordingExec::new(); // sync plane (SshExec in prod)
3481	let host = r#"
3482	[[host]]
3483	name = "h1"
3484	ssh = "mbp"
3485	targets = ["macos/aarch64"]
3486	transport = "agent"
3487	agent_url = "http://mbp:8765"
3488	actuate = ["build", "sign", "notarize", "staple"]
3489	observe = ["build-log", "gatekeeper", "artifact"]
3490	pull_root = "REPO"
3491	"#;
3492	let recipe = r#"
3493	let h = build_host();
3494	step("build");
3495	sh_ok(h, "echo built");
3496	step("sign");
3497	codesign(h, "Developer ID Application: Test", "REPO/out/demo.dmg");
3498	step("collect");
3499	collect(h, "REPO/out/*.dmg", "demo", "0.0.1");
3500	"#;
3501	let (_tmp, status, error) = run_two_plane(
3502	host,
3503	"macos/aarch64",
3504	"macos.rhai",
3505	recipe,
3506	agent.clone(),
3507	ssh.clone(),
3508	)
3509	.await;
3510	assert_eq!(status, "ok", "the recipe should complete: {error}");
3511
3512	// The build + sign commands ran on the AGENT (exec) transport.
3513	let agent_cmds = agent.commands();
3514	assert!(
3515	agent_cmds.iter().any(\|c\| c.contains("codesign")),
3516	"codesign rides the agent exec transport: {agent_cmds:?}"
3517	);
3518	assert!(
3519	agent_cmds.iter().any(\|c\| c.contains("echo built")),
3520	"the build step rides the agent exec transport: {agent_cmds:?}"
3521	);
3522	// ...and the agent moved NO artifacts. This is the load-bearing half:
3523	// AgentRpc::pull_glob is refused by design, so collect must not touch it.
3524	assert!(
3525	agent.pulls().is_empty(),
3526	"the agent transport must never collect artifacts: {:?}",
3527	agent.pulls()
3528	);
3529
3530	// The artifact was collected over the SSH (sync) transport...
3531	let ssh_pulls = ssh.pulls();
3532	assert!(
3533	ssh_pulls
3534	.iter()
3535	.any(\|p\| p.contains("demo.dmg") \|\| p.contains("*.dmg")),
3536	"collect rides the ssh sync transport: {ssh_pulls:?}"
3537	);
3538	// ...and the sync transport was never asked to run a build/sign command.
3539	assert!(
3540	ssh.commands().is_empty(),
3541	"the sync transport must never run host commands: {:?}",
3542	ssh.commands()
3543	);
3544	}
3545
3546	/// A plain (non-agent) host whose `ssh` is a remote alias, not "local" --
3547	/// the case every other test avoids. The recipe runs end to end through the
3548	/// fake, proving the runner drives a non-local host and still splits exec
3549	/// (build) from sync (collect) across the two transport maps.
3550	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3551	async fn a_non_local_ssh_host_runs_a_recipe_end_to_end() {
3552	let exec = RecordingExec::new();
3553	let sync = RecordingExec::new();
3554	let host = r#"
3555	[[host]]
3556	name = "h1"
3557	ssh = "astra"
3558	targets = ["linux/x86_64"]
3559	pull_root = "REPO"
3560	"#;
3561	let recipe = r#"
3562	let h = build_host();
3563	step("build");
3564	sh_ok(h, "echo compiling");
3565	step("collect");
3566	collect(h, "REPO/out/demo.bin", "demo", "0.0.1");
3567	"#;
3568	let (_tmp, status, error) = run_two_plane(
3569	host,
3570	"linux/x86_64",
3571	"linux.rhai",
3572	recipe,
3573	exec.clone(),
3574	sync.clone(),
3575	)
3576	.await;
3577	assert_eq!(status, "ok", "the recipe should complete: {error}");
3578
3579	assert!(
3580	exec.commands().iter().any(\|c\| c.contains("echo compiling")),
3581	"the build command rides the exec transport: {:?}",
3582	exec.commands()
3583	);
3584	assert!(
3585	exec.pulls().is_empty(),
3586	"the exec transport must not collect: {:?}",
3587	exec.pulls()
3588	);
3589	assert!(
3590	sync.pulls().iter().any(\|p\| p.contains("demo.bin")),
3591	"collect rides the sync transport: {:?}",
3592	sync.pulls()
3593	);
3594	assert!(
3595	sync.commands().is_empty(),
3596	"the sync transport must not run commands: {:?}",
3597	sync.commands()
3598	);
3599	}
3600	}
3601
3602	/// Every recipe of every configured app must parse.
3603	///
3604	/// A recipe is read and compiled at release time, on the build host, after the
3605	/// checkout has already run — so a typo in one is discovered at the worst
3606	/// possible moment. Compiling them here costs nothing and moves that discovery
3607	/// to `cargo test`. Skips when the live config is absent (CI, another host),
3608	/// like [`crate::topology`]'s live-config smoke test.
3609	#[cfg(test)]
3610	mod live_recipe_smoke {
3611	use crate::topology::Topology;
3612	use std::path::{Path, PathBuf};
3613
3614	#[test]
3615	fn live_recipes_compile_if_present() {
3616	let Some(home) = std::env::var_os("HOME") else {
3617	return;
3618	};
3619	let path = Path::new(&home).join(".config/bento/bento.toml");
3620	if !path.exists() {
3621	return;
3622	}
3623	let topo = Topology::load(&path).expect("live bento.toml must load");
3624	// Syntax only: the host functions are bound per run against a live
3625	// context, and Rhai resolves calls at eval time regardless.
3626	let engine = rhai::Engine::new();
3627	let mut checked = 0;
3628	for (name, cfg) in &topo.app {
3629	let dir = crate::engine::expand_tilde(&cfg.repo).join(&cfg.recipe_dir);
3630	// Every recipe in the directory, not just the ones the app's current
3631	// targets name. A recipe for a target that is temporarily not
3632	// shipped (windows, dropped from the manifests until its host is
3633	// real) still has to parse, and it is the one nothing else is
3634	// watching.
3635	let Ok(entries) = std::fs::read_dir(&dir) else {
3636	continue;
3637	};
3638	let mut files: Vec<PathBuf> = entries
3639	.filter_map(Result::ok)
3640	.map(\|e\| e.path())
3641	.filter(\|p\| p.extension().is_some_and(\|x\| x == "rhai"))
3642	.collect();
3643	files.sort();
3644	for p in files {
3645	let file = p.file_name().unwrap_or_default().to_string_lossy();
3646	let Ok(src) = std::fs::read_to_string(&p) else {
3647	continue;
3648	};
3649	engine
3650	.compile(&src)
3651	.unwrap_or_else(\|e\| panic!("recipe {name}/{file} does not parse: {e}"));
3652	checked += 1;
3653	}
3654	}
3655	assert!(checked > 0, "live config resolved no readable recipes");
3656	}
3657	}
3658