max / makenotwork

149.2 KB · 3882 lines History Blame Raw

1	//! Build orchestration: fan a `(app, version)` out across its targets, each
2	//! running its recipe concurrently on the host that can build it.
3	//!
4	//! A build inserts one `builds` row, then spawns one task per target. Each
5	//! target task registers itself in the single-slot guard (a newer build for
6	//! the same `(app, target)` aborts the in-flight one — latest wins, but other
7	//! targets keep running, which is the fan-out), then runs the recipe.
8
9	use crate::domain::{AppId, Status, Step, Target, Version};
10	use crate::engine::{self, RecipeCtx};
11	use crate::events::{self, Event};
12	use crate::state::AppState;
13	use anyhow::{Context, Result};
14	use ops_exec::{Action, Executor, LogSink, Step as OpStep};
15	use std::path::PathBuf;
16	use std::sync::Arc;
17
18	/// A [`LogSink`] that drops what it's handed — the release preflight runs git on
19	/// each host for its exit code and (via a separate `rev-parse`) the sha in
20	/// `RunOutput`, not for a streamed log, so there is no step to stream into.
21	struct DiscardSink;
22
23	#[async_trait::async_trait]
24	impl LogSink for DiscardSink {
25	async fn write_chunk(&mut self, _bytes: &[u8]) {}
26	}
27
28	/// Preflight barrier: pin every host that will build a target for this release
29	/// to the tag `v<version>` and refuse the build unless they all report the SAME
30	/// commit. Recipes used to `git pull --ff-only` per host, so `mbp`/`astra`/`fw13`
31	/// each built whatever `main` was at pull time and the artifacts were filed
32	/// under the daemon host's version. This runs before any target task spawns, so
33	/// a mixed-source release is stopped before a single artifact is built.
34	///
35	/// "All" means all: a host that did not report is a refusal, not an abstention.
36	/// Comparing only the hosts that answered proves agreement among those, which is
37	/// not the claim the barrier is making.
38	///
39	/// Returns the branch each host was on before it was pinned, for
40	/// [`restore_branches`] to put back once the build settles, and the commit they
41	/// all agreed on. A host that is ALREADY detached has no branch to return to,
42	/// and is refused: that state means some earlier release never cleaned up, and
43	/// any commits made in the meantime are sitting on no branch at all.
44	///
45	/// The agreed sha is the release's provenance, and this is the only place it is
46	/// known to be one value rather than a per-host answer. Resolving it here and
47	/// carrying it forward is what lets an artifact record name the source it was
48	/// built from without asking a build host to be honest about it afterwards.
49	///
50	/// A refusal unwinds: every host this already detached goes back to the branch
51	/// it was on. The hosts are pinned one after another, so a failure on the third
52	/// used to leave the first two sitting on the tag with nothing to restore them —
53	/// and the next attempt then refused for a detached HEAD, describing its own
54	/// last run while reading as the operator's mistake. The barrier either pins all
55	/// of them or leaves none of them moved.
56	async fn pin_release(
57	state: &AppState,
58	app: &AppId,
59	version: &Version,
60	targets: &[Target],
61	) -> Result<Pinned> {
62	let mut branches: Vec<(String, String)> = Vec::new();
63	match pin_hosts(state, app, version, targets, &mut branches).await {
64	Ok(sha) => Ok(Pinned { branches, sha }),
65	Err(e) => {
66	// Best-effort, and it cannot rescue the refusal: the release is
67	// already not happening, so a restore that fails must not replace
68	// the reason it was refused. `restore_branches` logs its own
69	// failures for exactly that reason.
70	restore_branches(state, app, &branches).await;
71	Err(e)
72	}
73	}
74	}
75
76	/// The body of [`pin_release`]: pin each host in turn and return the commit they
77	/// agreed on, recording into `branches` — before each checkout — the branch that
78	/// host was on. `branches` is an out-parameter rather than a return value
79	/// because the caller needs it on the failure path too, which is the whole point
80	/// of the split.
81	async fn pin_hosts(
82	state: &AppState,
83	app: &AppId,
84	version: &Version,
85	targets: &[Target],
86	branches: &mut Vec<(String, String)>,
87	) -> Result<String> {
88	let cfg = state
89	.topo
90	.app(app)
91	.ok_or_else(\|\| anyhow::anyhow!("unknown app `{app}`"))?;
92	// Spelled per app: a repo holding one product tags `v0.4.1`, a repo holding
93	// several tags `pom-v0.4.1`. See `AppConfig::tag_format`.
94	let tag = cfg.tag_for(version);
95
96	// The distinct hosts across the requested targets (order-stable).
97	//
98	// A target whose host the topology cannot resolve is refused rather than
99	// skipped. Skipping it would leave that target out of the barrier while it
100	// still builds, which is the shape of hole this whole preflight exists to
101	// close: agreement proven among some hosts reads as agreement among all.
102	// `resolve_targets` rejects an unbuildable target before `/build`, so this
103	// is unreachable through the API and cheap to state anyway.
104	let mut hosts: Vec<String> = Vec::new();
105	for t in targets {
106	let h = state
107	.topo
108	.host_for(*t)
109	.ok_or_else(\|\| anyhow::anyhow!("release preflight: no host can build {t}"))?;
110	if !hosts.contains(&h.name) {
111	hosts.push(h.name.clone());
112	}
113	}
114	anyhow::ensure!(
115	!hosts.is_empty(),
116	"release preflight: no build hosts for v{version}; refusing to build a release \
117	nothing was pinned for"
118	);
119
120	let mut shas: Vec<(String, String)> = Vec::new();
121	for host in &hosts {
122	let exec = state
123	.executors
124	.get(host)
125	.ok_or_else(\|\| anyhow::anyhow!("no executor for host `{host}`"))?;
126	let mut sink = DiscardSink;
127	// Resolved per host, not once for the set: the checkouts need not be at
128	// the same path on every machine, and pinning windows-x86 with the unix
129	// path failed on the first git command of the release.
130	let repo = cfg.repo_for(host);
131
132	// Read the branch BEFORE pinning, while there is still one to read.
133	let branch_cmd = OpStep::shell(Action::Build, engine::git_current_branch_cmd(repo));
134	let out = exec
135	.run_streaming(&branch_cmd, &mut sink)
136	.await
137	.with_context(\|\| format!("release preflight: reading branch on `{host}`"))?;
138	let branch = String::from_utf8_lossy(&out.stdout).trim().to_string();
139	// `symbolic-ref -q` exits non-zero BOTH on a detached HEAD and when it
140	// could not read the repo at all (wrong path, no checkout, no git).
141	// Those need opposite responses, so a stderr that says anything is
142	// reported as itself rather than folded into the detached-HEAD advice.
143	let why = String::from_utf8_lossy(&out.stderr).trim().to_string();
144	anyhow::ensure!(
145	out.status.success() \|\| why.is_empty(),
146	"release preflight: reading the branch of `{repo}` on `{host}` failed: {why}"
147	);
148	anyhow::ensure!(
149	out.status.success() && !branch.is_empty(),
150	"release preflight: `{repo}` on `{host}` is on a detached HEAD, so there is \
151	no branch to restore it to after the release. An earlier release left it \
152	that way; any commits made since are on no branch and may exist nowhere \
153	else. Reattach it (`git checkout <branch>`, fast-forwarding if the commits \
154	should be kept) before releasing."
155	);
156	branches.push((host.clone(), branch));
157
158	// A dirty tree defeats the pin silently, which is worse than not pinning
159	// at all. `git checkout <tag>` does not fail on local modifications to
160	// files whose content is unchanged in the tag — it succeeds and KEEPS
161	// them. So a host with edits in the working tree builds those edits
162	// while reporting the tagged sha, and a second host with a clean tree
163	// builds something else. The rev-parse barrier below cannot see it:
164	// both hosts genuinely are on the same commit. Two architectures, two
165	// different binaries, one tag containing neither.
166	let dirty_cmd = OpStep::shell(Action::Build, engine::git_dirty_cmd(repo));
167	let out = exec
168	.run_streaming(&dirty_cmd, &mut sink)
169	.await
170	.with_context(\|\| format!("release preflight: reading tree state on `{host}`"))?;
171	let dirty = String::from_utf8_lossy(&out.stdout);
172	let dirty: Vec<&str> = dirty
173	.lines()
174	.map(str::trim)
175	.filter(\|l\| !l.is_empty())
176	.collect();
177	anyhow::ensure!(
178	dirty.is_empty(),
179	"release preflight: `{repo}` on `{host}` has uncommitted changes to tracked \
180	files, so the build would not be the tagged commit:\n {}\nCommit or stash \
181	them before releasing.",
182	dirty.join("\n "),
183	);
184
185	// Advisory: `fetch --all` is non-zero if any one of a repo's remotes is
186	// unreachable, and blaming the tag for a dead mirror is what made this
187	// preflight misreport. Only the checkout below is allowed to fail.
188	let fetch = OpStep::shell(Action::Build, engine::git_fetch_cmd(repo));
189	let _ = exec.run_streaming(&fetch, &mut sink).await;
190
191	let checkout = OpStep::shell(Action::Build, engine::git_checkout_tag_cmd(repo, &tag));
192	let out = exec
193	.run_streaming(&checkout, &mut sink)
194	.await
195	.with_context(\|\| format!("release preflight: checkout on `{host}`"))?;
196	if !out.status.success() {
197	// Ask git why before telling the operator. The two causes need
198	// opposite responses: tag-and-push, versus clean the working tree.
199	let probe = OpStep::shell(Action::Build, engine::git_tag_exists_cmd(repo, &tag));
200	let tag_exists = exec
201	.run_streaming(&probe, &mut sink)
202	.await
203	.is_ok_and(\|o\| o.status.success());
204	// And which files, repo-wide. The gate above is app-scoped on
205	// purpose, but the checkout it precedes is not, so a clean app
206	// directory and a refused checkout are perfectly consistent — and
207	// between them they say nothing the operator can act on.
208	let blocking = repo_dirty_report(exec.as_ref(), repo).await;
209	anyhow::bail!(
210	"release preflight: `git checkout {tag}` failed on `{host}`: {}",
211	engine::checkout_failure_reason(&tag, tag_exists, blocking.as_deref())
212	);
213	}
214
215	let rev = OpStep::shell(Action::Build, engine::git_rev_parse_cmd(repo));
216	let out = exec
217	.run_streaming(&rev, &mut sink)
218	.await
219	.with_context(\|\| format!("release preflight: rev-parse on `{host}`"))?;
220	anyhow::ensure!(
221	out.status.success(),
222	"release preflight: `git rev-parse HEAD` failed on `{host}`"
223	);
224	let sha = String::from_utf8_lossy(&out.stdout).trim().to_string();
225	shas.push((host.clone(), sha));
226	}
227
228	// The barrier: every participating host must have reported, and every report
229	// must be the same commit. Reporting is checked first and separately from
230	// agreement, because a barrier that compares only what it received cannot
231	// tell unanimity from silence — three hosts agreeing while a fourth was
232	// never asked is exactly the mixed-source release this refuses.
233	let missing: Vec<&str> = hosts
234	.iter()
235	.filter(\|h\| !shas.iter().any(\|(sh, _)\| sh == *h))
236	.map(String::as_str)
237	.collect();
238	anyhow::ensure!(
239	missing.is_empty(),
240	"release preflight: {} did not report a commit for v{version}; refusing to build \
241	a release the barrier cannot vouch for",
242	missing.join(", "),
243	);
244	let empty_shas: Vec<&str> = shas
245	.iter()
246	.filter(\|(_, sha)\| sha.is_empty())
247	.map(\|(h, _)\| h.as_str())
248	.collect();
249	anyhow::ensure!(
250	empty_shas.is_empty(),
251	"release preflight: `git rev-parse HEAD` returned nothing on {} for v{version}",
252	empty_shas.join(", "),
253	);
254
255	let (first_host, first_sha) = &shas[0];
256	let mismatch: Vec<String> = shas
257	.iter()
258	.filter(\|(_, sha)\| sha != first_sha)
259	.map(\|(h, sha)\| format!("{h}={}", short(sha)))
260	.collect();
261	anyhow::ensure!(
262	mismatch.is_empty(),
263	"release preflight: build hosts are on different commits for v{version} \
264	({}={}, {}); refusing to build a release from mixed sources",
265	first_host,
266	short(first_sha),
267	mismatch.join(", "),
268	);
269	Ok(first_sha.clone())
270	}
271
272	/// What the preflight established: where each host's checkout was, and the one
273	/// commit every host is now on.
274	struct Pinned {
275	/// `(host, branch)` for [`restore_branches`].
276	branches: Vec<(String, String)>,
277	/// The commit all hosts agreed on. Empty when pinning is off (tests).
278	sha: String,
279	}
280
281	/// Which tracked files in `repo`'s whole repository have local changes, rendered
282	/// for an operator and marked where they fall outside the app's own directory.
283	///
284	/// Diagnosis, never a gate: both callers run it only after a `git checkout` has
285	/// already failed, and both treat a probe that itself fails as "nothing to add"
286	/// rather than as a second error. The reason the checkout failed is the message;
287	/// this only says which file to go and look at.
288	async fn repo_dirty_report(exec: &dyn Executor, repo: &str) -> Option<String> {
289	async fn read(exec: &dyn Executor, cmd: String) -> String {
290	let step = OpStep::shell(Action::Build, cmd);
291	let mut sink = DiscardSink;
292	match exec.run_streaming(&step, &mut sink).await {
293	Ok(out) if out.status.success() => String::from_utf8_lossy(&out.stdout).into_owned(),
294	_ => String::new(),
295	}
296	}
297	let status = read(exec, engine::git_repo_dirty_cmd(repo)).await;
298	let prefix = read(exec, engine::git_repo_prefix_cmd(repo)).await;
299	engine::dirty_paths_blocking_checkout(&status, &prefix)
300	}
301
302	/// Put every host's checkout back on the branch [`pin_release`] found it on.
303	///
304	/// Best-effort and non-fatal: the release itself is already decided by the time
305	/// this runs, and failing a green build because a `git checkout` did not take
306	/// would be worse than the detached tree it is cleaning up. A failure is logged
307	/// loudly instead, because the state it leaves behind is the silent one.
308	async fn restore_branches(state: &AppState, app: &AppId, branches: &[(String, String)]) {
309	let Some(cfg) = state.topo.app(app) else {
310	return;
311	};
312	for (host, branch) in branches {
313	let Some(exec) = state.executors.get(host) else {
314	continue;
315	};
316	// Serialize against a target still holding this host for its recipe run,
317	// so the restore can't move the tree mid-build.
318	let _host_guard = match state.host_locks.get(host).cloned() {
319	Some(lock) => Some(lock.lock_owned().await),
320	None => None,
321	};
322	let mut sink = DiscardSink;
323	// The same per-host path `pin_release` detached; restoring the daemon's
324	// path on a host that keeps its checkout elsewhere would leave the real
325	// one detached and report success.
326	let repo = cfg.repo_for(host);
327	let step = OpStep::shell(Action::Build, engine::git_restore_branch_cmd(repo, branch));
328	match exec.run_streaming(&step, &mut sink).await {
329	Ok(out) if out.status.success() => {
330	tracing::debug!(%host, %branch, "restored checkout to its branch");
331	}
332	// Name what held the restore up. The usual answer is a file the
333	// build itself wrote — a regenerated `Cargo.lock`, most often — and
334	// without it the next release refuses for uncommitted changes and
335	// the operator gets to guess whether the edit was theirs.
336	Ok(_) => {
337	let blocking = repo_dirty_report(exec.as_ref(), repo)
338	.await
339	.unwrap_or_else(\|\| "(none; the working tree is clean)".into());
340	tracing::error!(
341	%host, %branch, %repo, %blocking,
342	"could not restore the checkout to its branch; it is left DETACHED at the \
343	release tag, and commits made there will belong to no branch"
344	);
345	}
346	Err(e) => tracing::error!(
347	%host, %branch, %repo, error = %e,
348	"could not restore the checkout to its branch; it is left DETACHED at the \
349	release tag, and commits made there will belong to no branch"
350	),
351	}
352	}
353	}
354
355	/// First 12 chars of a sha for a readable error.
356	fn short(sha: &str) -> &str {
357	sha.get(..12).unwrap_or(sha)
358	}
359
360	/// Resolve the version to build: explicit, or read from `tauri.conf.json`.
361	///
362	/// Returns typed errors so the route maps user mistakes (unknown app, bad
363	/// version string) to 400, and only genuine daemon-side failures (reading the
364	/// app's `tauri.conf.json`) to 500.
365	pub fn resolve_version(
366	state: &AppState,
367	app: &AppId,
368	explicit: Option<String>,
369	) -> crate::error::Result<Version> {
370	use crate::error::Error;
371	if let Some(v) = explicit {
372	return Version::parse(&v).map_err(Error::BadRequest);
373	}
374	let cfg = state
375	.topo
376	.app(app)
377	.ok_or_else(\|\| Error::BadRequest(format!("unknown app `{app}`")))?;
378	engine::version_from_repo(&cfg.repo, cfg.version_path.as_deref()).map_err(Error::Other)
379	}
380
381	/// Validate + default the target list against what the app ships. Unknown app,
382	/// an unshipped target, or a target no host can build are all client errors
383	/// (400), not server errors (500).
384	pub fn resolve_targets(
385	state: &AppState,
386	app: &AppId,
387	requested: Vec<Target>,
388	) -> crate::error::Result<Vec<Target>> {
389	use crate::error::Error;
390	let cfg = state
391	.topo
392	.app(app)
393	.ok_or_else(\|\| Error::BadRequest(format!("unknown app `{app}`")))?;
394	if requested.is_empty() {
395	return Ok(cfg.targets.clone());
396	}
397	for t in &requested {
398	if !cfg.targets.contains(t) {
399	return Err(Error::BadRequest(format!(
400	"app `{app}` does not ship target {t}"
401	)));
402	}
403	if state.topo.host_for(*t).is_none() {
404	return Err(Error::BadRequest(format!("no host can build {t}")));
405	}
406	}
407	Ok(requested)
408	}
409
410	/// Insert the build row and spawn per-target tasks. Returns the build id.
411	pub async fn start_build(
412	state: AppState,
413	app: AppId,
414	version: Version,
415	targets: Vec<Target>,
416	) -> Result<i64> {
417	// Preflight: every version source in the repo must agree with the version
418	// being built, before any host pulls or compiles. `version_from_repo` reads
419	// one file, so a tauri.conf.json/Cargo.toml (or explicit-version) mismatch
420	// would otherwise sail through and file artifacts under the wrong version.
421	if let Some(cfg) = state.topo.app(&app) {
422	engine::check_version_consistency(&cfg.repo, cfg.version_path.as_deref(), &version)
423	.context("version preflight")?;
424	}
425
426	// Pin every build host to the release tag and verify they agree, before any
427	// target task spawns. Off in tests (their repos aren't git checkouts).
428	let pinned = if state.cfg.pin_release_sha {
429	pin_release(&state, &app, &version, &targets)
430	.await
431	.context("release preflight")?
432	} else {
433	Pinned {
434	branches: Vec::new(),
435	sha: String::new(),
436	}
437	};
438
439	let build_id: i64 = sqlx::query_scalar(
440	"INSERT INTO builds (app, version, status, created_at) VALUES (?, ?, 'running', ?) RETURNING id",
441	)
442	.bind(app.as_str())
443	.bind(version.to_string())
444	.bind(chrono::Utc::now().to_rfc3339())
445	.fetch_one(&state.pool)
446	.await
447	.context("insert build")?;
448
449	events::emit(
450	&state.events,
451	Event::BuildRequested {
452	app: app.clone(),
453	version: version.clone(),
454	targets: targets.clone(),
455	},
456	);
457	crate::metrics::build_started();
458
459	// Spawn every target into a JoinSet so finalize_build can await completion
460	// event-driven (no DB polling) and observe each task's outcome (panic vs
461	// clean) for supervision.
462	let mut set: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
463	for target in targets {
464	let app = app.clone();
465	let version = version.clone();
466	let key = (app.clone(), target);
467	let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
468
469	// Latest-wins, done as ONE critical section: supersede the prior occupant
470	// (cooperatively cancel its recipe + abort its task) and install ours
471	// without ever releasing the lock, so two concurrent /build+/retry for the
472	// same (app, target) can't both pass "nothing to abort" and run together.
473	let mut active = state.active.lock().await;
474	if let Some(prev) = active.remove(&key)
475	&& !prev.abort.is_finished()
476	{
477	// The recipe runs on a blocking thread; abort() alone can't stop it,
478	// so set the cooperative flag the engine checks at step boundaries.
479	prev.cancel.store(true, std::sync::atomic::Ordering::SeqCst);
480	prev.abort.abort();
481	events::emit(
482	&state.events,
483	Event::TargetAborted {
484	app: app.clone(),
485	target,
486	},
487	);
488	}
489	let abort = set.spawn(run_target(
490	state.clone(),
491	build_id,
492	app,
493	version,
494	target,
495	cancel.clone(),
496	pinned.sha.clone(),
497	));
498	active.insert(
499	key,
500	crate::state::ActiveSlot {
501	build_id,
502	abort,
503	cancel,
504	},
505	);
506	crate::metrics::set_in_flight(active.len());
507	}
508
509	// Mark the build done once all target tasks settle, and put the pinned
510	// checkouts back on their branches. Spawned so /build returns immediately.
511	tokio::spawn(finalize_build(state, build_id, set, app, pinned.branches));
512	Ok(build_id)
513	}
514
515	/// Run one target's recipe end to end, updating its `target_runs` row. `cancel`
516	/// is set when a newer build supersedes this one; the engine checks it at step
517	/// boundaries and before publish so a superseded recipe can't advance or ship.
518	async fn run_target(
519	state: AppState,
520	build_id: i64,
521	app: AppId,
522	version: Version,
523	target: Target,
524	cancel: Arc<std::sync::atomic::AtomicBool>,
525	pinned_sha: String,
526	) {
527	let started = std::time::Instant::now();
528	let target_run_id: i64 = match sqlx::query_scalar(
529	"INSERT INTO target_runs (build_id, app, version, target, status, started_at)
530	VALUES (?, ?, ?, ?, 'running', ?) RETURNING id",
531	)
532	.bind(build_id)
533	.bind(app.as_str())
534	.bind(version.to_string())
535	.bind(target.to_string())
536	.bind(chrono::Utc::now().to_rfc3339())
537	.fetch_one(&state.pool)
538	.await
539	{
540	Ok(id) => id,
541	Err(e) => {
542	tracing::error!(%app, %target, error = %e, "could not create target_run");
543	return;
544	}
545	};
546
547	events::emit(
548	&state.events,
549	Event::TargetStart {
550	app: app.clone(),
551	version: version.clone(),
552	target,
553	},
554	);
555
556	let recipe_src = match read_recipe(&state, &app, target) {
557	Ok(s) => s,
558	Err(e) => {
559	fail_target(
560	&state,
561	target_run_id,
562	&app,
563	&version,
564	target,
565	Step::Checkout,
566	&format!("{e:#}"),
567	)
568	.await;
569	crate::metrics::target_finished(
570	&target.to_string(),
571	"failed",
572	started.elapsed().as_secs_f64(),
573	);
574	return;
575	}
576	};
577
578	// Pre-flight the build host before running the recipe. For local/ssh hosts
579	// this is a no-op; for the agent host (macOS in-session signing) it hits
580	// `/health` so a dead ops-agent fails here with a clear message — matching
581	// what the driver does — instead of erroring opaquely on the recipe's first
582	// dispatch to that host.
583	if let Some(host) = state.topo.host_for(target)
584	&& let Some(exec) = state.executors.get(&host.name)
585	&& let Err(e) = exec.preflight().await
586	{
587	fail_target(
588	&state,
589	target_run_id,
590	&app,
591	&version,
592	target,
593	Step::Checkout,
594	&format!("{e:#}"),
595	)
596	.await;
597	crate::metrics::target_finished(
598	&target.to_string(),
599	"failed",
600	started.elapsed().as_secs_f64(),
601	);
602	return;
603	}
604
605	// Host + checkout path for this target, resolved from the topology. Both are
606	// exposed to the recipe (`build_host()` / `repo()`); a missing host here is
607	// unreachable (resolve_targets rejected unbuildable targets before /build),
608	// but fall back rather than panic.
609	let build_host = state
610	.topo
611	.host_for(target)
612	.map(\|h\| h.name.clone())
613	.unwrap_or_default();
614	let build_host_ssh = state
615	.topo
616	.host_for(target)
617	.map(\|h\| h.ssh.clone())
618	.unwrap_or_default();
619	let repo = state
620	.topo
621	.app(&app)
622	.map(\|a\| a.repo.clone())
623	.unwrap_or_default();
624	let repo_by_host = state
625	.topo
626	.app(&app)
627	.map(\|a\| a.repo_by_host.clone())
628	.unwrap_or_default();
629	let features = state
630	.topo
631	.app(&app)
632	.map(\|a\| a.features.clone())
633	.unwrap_or_default();
634	let tag = state
635	.topo
636	.app(&app)
637	.map_or_else(\|\| format!("v{version}"), \|a\| a.tag_for(&version));
638	// The opt-in all-targets-green publish gate: pass the declared target set
639	// (so `publish` can require every sibling green) only when the app turns it
640	// on; otherwise None leaves independent per-target publishing unchanged.
641	let all_green_required = state
642	.topo
643	.app(&app)
644	.and_then(\|a\| a.require_all_targets.then(\|\| a.targets.clone()));
645
646	// A service's install destination for THIS target, plus an executor for it.
647	//
648	// The executor is added to a per-run copy of the map rather than to the
649	// daemon-wide one, so a deploy grant on a production host exists only for
650	// the duration of the run that needs it and only for the app that declared
651	// it. Nothing else can address that host: it is not in the topology, so no
652	// other app's recipe can name it.
653	let deploy = state
654	.topo
655	.app(&app)
656	.and_then(\|a\| a.deploy_for(target))
657	.cloned();
658	let execs = match &deploy {
659	Some(d) => {
660	let mut map = (*state.executors).clone();
661	map.insert(d.host.clone(), crate::state::build_deploy_executor(d));
662	Arc::new(map)
663	}
664	None => state.executors.clone(),
665	};
666
667	let ctx = Arc::new(
668	RecipeCtx::new(
669	app.clone(),
670	version.clone(),
671	target,
672	build_host,
673	build_host_ssh,
674	tag.clone(),
675	repo,
676	features,
677	// Gates the `verify` step's capability: a library's crate preflight is
678	// not an app's Gatekeeper check. See engine::action_for.
679	state.topo.app(&app).map(\|a\| a.kind).unwrap_or_default(),
680	target_run_id,
681	execs,
682	state.syncs.clone(),
683	deploy,
684	state.pool.clone(),
685	state.events.clone(),
686	state.cfg.clone(),
687	state.ota.clone(),
688	tokio::runtime::Handle::current(),
689	cancel,
690	all_green_required,
691	)
692	.with_repo_by_host(repo_by_host),
693	);
694
695	// Serialize per host: hold this host's lock for the whole recipe run so a
696	// second target on the same box (goingson macos + ios both on mbp) can't
697	// build concurrently in one checkout and corrupt the shared target/ +
698	// keychain. Targets on different hosts hold different locks and still fan
699	// out. Acquired at an await point, so a supersede-abort while queued drops
700	// the task cleanly before it ever takes the lock. Not held during the
701	// read-only preflight above.
702	let _host_guard = match state.host_locks.get(&ctx.build_host).cloned() {
703	Some(lock) => Some(lock.lock_owned().await),
704	None => None,
705	};
706
707	// Rhai is synchronous; run the recipe (and its final step finalization) on
708	// a blocking thread so host functions can `block_on` without sitting on a
709	// runtime worker.
710	let ctx_run = ctx.clone();
711	let outcome = tokio::task::spawn_blocking(move \|\| {
712	let engine = engine::build_engine(&ctx_run);
713	let res = engine.run(&recipe_src);
714	let last_step = ctx_run.current_step();
715	match &res {
716	Ok(()) => {
717	let _ = ctx_run.finish_step(Status::Ok);
718	}
719	Err(_) => {
720	let _ = ctx_run.finish_step(Status::Failed);
721	}
722	}
723	res.map_err(\|e\| (last_step, e.to_string()))
724	})
725	.await;
726
727	// Write the artifact record before the run is stamped terminal. It describes
728	// what was collected, so it runs whether the recipe succeeded or failed: a
729	// build that signed and collected an artifact and then failed at `publish`
730	// still produced bytes somebody may want the provenance of. Emit-only and
731	// non-fatal; nothing reads it yet.
732	let record_path = crate::artifact_record::emit(&state, &ctx, &pinned_sha).await;
733
734	match outcome {
735	Ok(Ok(())) => {
736	// Hand the artifact to Sando before the run is stamped ok, so a
737	// target reported green is one whose bytes reached the controller
738	// that decides whether they ship. Only on success: an artifact from
739	// a failed recipe is exactly what should not be offered for a
740	// deploy, whatever it managed to collect on the way down.
741	if let Err(e) =
742	handoff_for(&state, record_path.as_deref(), &app, &version, target).await
743	{
744	let msg = format!("{e:#}");
745	tracing::error!(%app, %target, error = %msg, "handing the artifact to sando failed");
746	fail_target(
747	&state,
748	target_run_id,
749	&app,
750	&version,
751	target,
752	Step::Handoff,
753	&msg,
754	)
755	.await;
756	crate::metrics::target_finished(
757	&target.to_string(),
758	"failed",
759	started.elapsed().as_secs_f64(),
760	);
761	return;
762	}
763
764	let artifacts = collected_artifacts(&state, &app, &version, target);
765	if let Err(e) = sqlx::query(
766	"UPDATE target_runs SET status = 'ok', current_step = NULL, finished_at = ? WHERE id = ?",
767	)
768	.bind(chrono::Utc::now().to_rfc3339())
769	.bind(target_run_id)
770	.execute(&state.pool)
771	.await
772	{
773	// A swallowed terminal write would leave the row `running`;
774	// finalize_build reconciles it, but log so the cause is visible.
775	tracing::error!(%app, %target, error = %e, "could not stamp target_run ok");
776	}
777	crate::metrics::target_finished(
778	&target.to_string(),
779	"ok",
780	started.elapsed().as_secs_f64(),
781	);
782	events::emit(
783	&state.events,
784	Event::TargetOk {
785	app,
786	version,
787	target,
788	artifacts,
789	},
790	);
791	}
792	Ok(Err((step, msg))) => {
793	fail_target(&state, target_run_id, &app, &version, target, step, &msg).await;
794	crate::metrics::target_finished(
795	&target.to_string(),
796	"failed",
797	started.elapsed().as_secs_f64(),
798	);
799	}
800	Err(join_err) => {
801	// Task was aborted (superseded) or panicked.
802	let status = if join_err.is_cancelled() {
803	"aborted"
804	} else {
805	"failed"
806	};
807	crate::metrics::target_finished(
808	&target.to_string(),
809	status,
810	started.elapsed().as_secs_f64(),
811	);
812	let msg = if join_err.is_cancelled() {
813	"aborted (superseded)".to_string()
814	} else {
815	format!("recipe task panicked: {join_err}")
816	};
817	fail_target(
818	&state,
819	target_run_id,
820	&app,
821	&version,
822	target,
823	Step::Build,
824	&msg,
825	)
826	.await;
827	}
828	}
829	}
830
831	async fn fail_target(
832	state: &AppState,
833	target_run_id: i64,
834	app: &AppId,
835	version: &Version,
836	target: Target,
837	step: Step,
838	error: &str,
839	) {
840	if let Err(e) = sqlx::query(
841	"UPDATE target_runs SET status = 'failed', current_step = NULL, error = ?, finished_at = ? WHERE id = ?",
842	)
843	.bind(error)
844	.bind(chrono::Utc::now().to_rfc3339())
845	.bind(target_run_id)
846	.execute(&state.pool)
847	.await
848	{
849	tracing::error!(%app, %target, error = %e, "could not stamp target_run failed");
850	}
851	events::emit(
852	&state.events,
853	Event::TargetFailed {
854	app: app.clone(),
855	version: version.clone(),
856	target,
857	step,
858	error: error.to_string(),
859	},
860	);
861	}
862
863	/// Await every target task of a build, then stamp the build's terminal status.
864	///
865	/// Event-driven: it joins the `JoinSet` rather than polling the DB. Each task's
866	/// outcome is supervised — a panicked task is logged (and its still-`running`
867	/// row is reconciled below), an aborted (superseded) task is expected.
868	///
869	/// Per-step deadlines (see `engine::step_budget`) are the real bound on a wedged
870	/// step now; this overall deadline is only a generous last-resort backstop for a
871	/// hang outside a bounded command. When it trips it sets each still-running
872	/// target's cooperative cancel FIRST — the blocking recipe bodies observe that
873	/// at their next bounded command or step boundary and stop signing — then aborts
874	/// the async wrappers. The old code aborted only the wrappers, which could not
875	/// reach the blocking bodies, so a build was marked failed while codesign kept
876	/// running on the mac.
877	async fn finalize_build(
878	state: AppState,
879	build_id: i64,
880	mut set: tokio::task::JoinSet<()>,
881	app: AppId,
882	pinned_branches: Vec<(String, String)>,
883	) {
884	const MAX_WAIT: std::time::Duration = std::time::Duration::from_hours(6);
885	let deadline = tokio::time::Instant::now() + MAX_WAIT;
886	loop {
887	match tokio::time::timeout_at(deadline, set.join_next()).await {
888	Ok(Some(Ok(()))) => {}
889	Ok(Some(Err(e))) => {
890	// A panic in run_target itself (outside its inner spawn_blocking,
891	// which is already caught). Cancellation = a superseding build.
892	if !e.is_cancelled() {
893	tracing::error!(build_id, error = %e, "target task panicked");
894	}
895	}
896	Ok(None) => break, // all targets settled
897	Err(_elapsed) => {
898	tracing::error!(
899	build_id,
900	"finalize_build backstop deadline hit; cancelling then aborting remaining targets"
901	);
902	// Set the cooperative cancel on this build's still-running slots
903	// BEFORE aborting, so the blocking recipe bodies actually stop
904	// (abort() alone cannot reach a spawn_blocking body).
905	{
906	let active = state.active.lock().await;
907	for slot in active.values().filter(\|s\| s.build_id == build_id) {
908	slot.cancel.store(true, std::sync::atomic::Ordering::SeqCst);
909	}
910	}
911	set.abort_all();
912	while set.join_next().await.is_some() {}
913	break;
914	}
915	}
916	}
917
918	// Reap this build's latest-wins slots (only our own — a superseding build
919	// owns a different build_id and is left intact).
920	let app_still_building = {
921	let mut active = state.active.lock().await;
922	active.retain(\|_, slot\| slot.build_id != build_id);
923	crate::metrics::set_in_flight(active.len());
924	active.keys().any(\|(a, _)\| a == &app)
925	};
926
927	// Undo the preflight's pin. Skipped if a superseding build for this same app
928	// still holds the checkout — it pinned the tree to ITS tag, and restoring the
929	// branch here would move that build off the commit it is releasing.
930	if !pinned_branches.is_empty() {
931	if app_still_building {
932	tracing::debug!(
933	build_id, %app,
934	"leaving the checkout pinned; a superseding build for this app is still running"
935	);
936	} else {
937	restore_branches(&state, &app, &pinned_branches).await;
938	}
939	}
940
941	let now = chrono::Utc::now().to_rfc3339();
942	// Any row still `running` is a panicked/aborted task that never stamped its
943	// terminal status — reconcile it so the build finalizes truthfully.
944	if let Err(e) = sqlx::query(
945	"UPDATE target_runs SET status = 'failed', \
946	error = COALESCE(error, 'target task ended before stamping its status'), \
947	finished_at = ? WHERE build_id = ? AND status = 'running'",
948	)
949	.bind(&now)
950	.bind(build_id)
951	.execute(&state.pool)
952	.await
953	{
954	tracing::error!(build_id, error = %e, "could not reconcile straggling target_runs");
955	}
956
957	let failed: i64 = sqlx::query_scalar(
958	"SELECT COUNT(*) FROM target_runs WHERE build_id = ? AND status = 'failed'",
959	)
960	.bind(build_id)
961	.fetch_one(&state.pool)
962	.await
963	.unwrap_or(0);
964	let status = if failed == 0 { "ok" } else { "failed" };
965	if let Err(e) = sqlx::query("UPDATE builds SET status = ?, finished_at = ? WHERE id = ?")
966	.bind(status)
967	.bind(&now)
968	.bind(build_id)
969	.execute(&state.pool)
970	.await
971	{
972	tracing::error!(build_id, error = %e, "could not stamp build terminal status");
973	}
974	}
975
976	/// Read the recipe text for `(app, target)` from the app's checkout on the
977	/// daemon host.
978	///
979	/// Apps use one recipe per platform (`<recipe_dir>/<platform>.rhai`), since
980	/// what they produce differs by platform. A library produces one crate whatever
981	/// host uploads it, so it uses a single `publish.rhai` — a `linux.rhai` naming
982	/// a registry upload would imply a per-platform artifact that does not exist.
983	fn read_recipe(state: &AppState, app: &AppId, target: Target) -> Result<String> {
984	let cfg = state
985	.topo
986	.app(app)
987	.ok_or_else(\|\| anyhow::anyhow!("unknown app `{app}`"))?;
988	let file = match cfg.kind {
989	crate::topology::Kind::Library => "publish.rhai".to_string(),
990	// A service is built per target like an app — the binary genuinely
991	// differs per platform, and so does where it lands — so it takes the
992	// same per-platform recipe naming rather than a single deploy.rhai.
993	crate::topology::Kind::App \| crate::topology::Kind::Service => {
994	format!("{}.rhai", target.platform.as_str())
995	}
996	};
997	let path: PathBuf = engine::expand_tilde(&cfg.repo)
998	.join(&cfg.recipe_dir)
999	.join(file);
1000	std::fs::read_to_string(&path).with_context(\|\| format!("reading recipe {}", path.display()))
1001	}
1002
1003	/// Send this target's finished bundle to the Sando configured for the app, if
1004	/// one is. Nothing configured is a no-op and the ordinary case.
1005	///
1006	/// A configured handoff with no record is an error rather than a skip. It means
1007	/// the recipe succeeded and collected nothing, or that the paperwork could not
1008	/// be written — and for an app whose whole point is being deployed by Sando, a
1009	/// build that produced nothing to hand over is not a green build. Everywhere
1010	/// else a missing record stays the non-event it was.
1011	async fn handoff_for(
1012	state: &AppState,
1013	record_path: Option<&std::path::Path>,
1014	app: &AppId,
1015	version: &Version,
1016	target: Target,
1017	) -> anyhow::Result<()> {
1018	if !state.cfg.handoff.contains_key(app.as_str()) {
1019	return Ok(());
1020	}
1021	let record_path = record_path.ok_or_else(\|\| {
1022	anyhow::anyhow!(
1023	"{app} hands off to sando, but this {target} run wrote no artifact record \
1024	(nothing was collected, or the record could not be written)"
1025	)
1026	})?;
1027	let dir = crate::archive::target_dir(&state.cfg.dist_root, app, version, target);
1028	crate::handoff::send(&state.cfg, &dir, record_path, app, version, target).await
1029	}
1030
1031	/// What this target run left in its collect directory.
1032	///
1033	/// Per target: the event reports what THIS run produced, and every target of a
1034	/// version used to share one directory, so a mac build's `TargetOk` listed the
1035	/// Linux AppImage a sibling had collected minutes earlier.
1036	fn collected_artifacts(
1037	state: &AppState,
1038	app: &AppId,
1039	version: &Version,
1040	target: Target,
1041	) -> Vec<String> {
1042	let dir = crate::archive::target_dir(&state.cfg.dist_root, app, version, target);
1043	let Ok(rd) = std::fs::read_dir(&dir) else {
1044	return Vec::new();
1045	};
1046	rd.filter_map(std::result::Result::ok)
1047	.map(\|e\| e.file_name().to_string_lossy().into_owned())
1048	.collect()
1049	}
1050
1051	#[cfg(test)]
1052	mod tests {
1053	use super::*;
1054	use crate::config::Config;
1055	use crate::ota::OtaRegistry;
1056	use crate::topology::Topology;
1057	use async_trait::async_trait;
1058	use ops_exec::{CapabilitySet, Executor, LogSink, RunOutput, SyncOpts};
1059	use sqlx::SqlitePool;
1060	use std::collections::HashMap;
1061	use std::os::unix::process::ExitStatusExt;
1062	use std::sync::Arc;
1063	use tokio::sync::Mutex;
1064
1065	/// A no-transport [`Executor`] for the paths that don't need a real command
1066	/// to run: its `preflight` is programmable (the agent host hits `/health`
1067	/// there, and a dead `ops-agent` must fail the target before the recipe
1068	/// dispatches), and every actual op is a success no-op.
1069	struct FakeExec {
1070	caps: CapabilitySet,
1071	preflight_err: Option<String>,
1072	}
1073
1074	impl FakeExec {
1075	fn preflight_fails(msg: &str) -> Arc<dyn Executor> {
1076	Arc::new(Self {
1077	caps: CapabilitySet::default(),
1078	preflight_err: Some(msg.to_string()),
1079	})
1080	}
1081	}
1082
1083	#[async_trait]
1084	impl Executor for FakeExec {
1085	async fn run_streaming(
1086	&self,
1087	_step: &ops_exec::Step,
1088	_sink: &mut dyn LogSink,
1089	) -> anyhow::Result<RunOutput> {
1090	Ok(RunOutput {
1091	status: std::process::ExitStatus::from_raw(0),
1092	stdout: Vec::new(),
1093	stderr: Vec::new(),
1094	})
1095	}
1096	async fn pull_file(
1097	&self,
1098	_r: &std::path::Path,
1099	_l: &std::path::Path,
1100	_o: &SyncOpts,
1101	) -> anyhow::Result<()> {
1102	Ok(())
1103	}
1104	async fn pull_dir(
1105	&self,
1106	_r: &std::path::Path,
1107	_l: &std::path::Path,
1108	_o: &SyncOpts,
1109	) -> anyhow::Result<()> {
1110	Ok(())
1111	}
1112	async fn pull_glob(
1113	&self,
1114	_g: &str,
1115	_l: &std::path::Path,
1116	_o: &SyncOpts,
1117	) -> anyhow::Result<()> {
1118	Ok(())
1119	}
1120	async fn push_dir(
1121	&self,
1122	_l: &std::path::Path,
1123	_r: &std::path::Path,
1124	_o: &SyncOpts,
1125	) -> anyhow::Result<()> {
1126	Ok(())
1127	}
1128	async fn preflight(&self) -> anyhow::Result<()> {
1129	match &self.preflight_err {
1130	Some(m) => anyhow::bail!("{m}"),
1131	None => Ok(()),
1132	}
1133	}
1134	fn capabilities(&self) -> &CapabilitySet {
1135	&self.caps
1136	}
1137	}
1138
1139	/// A recording, programmable [`Executor`] for the macOS sign chain. Every
1140	/// dispatched shell command is captured for assertion, and its exit code +
1141	/// stdout is chosen by the first rule whose needle the command contains — so
1142	/// a test can make `notarytool` report `Accepted`, make `codesign` fail, or
1143	/// make `spctl` emit the Gatekeeper sentinel without a real Mac or SSH. A
1144	/// rule may carry a sequence of responses (one per successive match) to
1145	/// drive the notarize retry loop; the last entry repeats once the sequence
1146	/// is exhausted. Unmatched commands succeed as empty no-ops (so a plain
1147	/// build `sh_ok` passes), and the sync/preflight ops are no-ops.
1148	struct ScriptedExec {
1149	caps: CapabilitySet,
1150	rules: Vec<ScriptRule>,
1151	log: Arc<std::sync::Mutex<Vec<String>>>,
1152	}
1153
1154	struct ScriptRule {
1155	needle: String,
1156	responses: Vec<(i32, String)>,
1157	calls: std::sync::atomic::AtomicUsize,
1158	}
1159
1160	impl ScriptedExec {
1161	fn new() -> Self {
1162	Self {
1163	// A mac host's real grant. Nothing in the dispatch path gates on
1164	// it (this fake never calls `gate`), but keep it coherent so
1165	// `capabilities()` is not a lie.
1166	caps: CapabilitySet::from_tokens(
1167	["build", "sign", "notarize", "staple"],
1168	["build-log", "artifact"],
1169	),
1170	rules: Vec::new(),
1171	log: Arc::new(std::sync::Mutex::new(Vec::new())),
1172	}
1173	}
1174
1175	/// Respond to every command containing `needle` with `(code, stdout)`.
1176	fn on(mut self, needle: &str, code: i32, stdout: &str) -> Self {
1177	self.rules.push(ScriptRule {
1178	needle: needle.to_string(),
1179	responses: vec![(code, stdout.to_string())],
1180	calls: std::sync::atomic::AtomicUsize::new(0),
1181	});
1182	self
1183	}
1184
1185	/// Respond to successive `needle` matches with successive responses; the
1186	/// last repeats once the list is exhausted. Drives the notarize retry.
1187	fn on_seq(mut self, needle: &str, responses: &[(i32, &str)]) -> Self {
1188	self.rules.push(ScriptRule {
1189	needle: needle.to_string(),
1190	responses: responses
1191	.iter()
1192	.map(\|(c, s)\| (c, (s).to_string()))
1193	.collect(),
1194	calls: std::sync::atomic::AtomicUsize::new(0),
1195	});
1196	self
1197	}
1198
1199	/// Every shell command this executor was asked to run, in order.
1200	fn commands(&self) -> Vec<String> {
1201	self.log.lock().unwrap().clone()
1202	}
1203	}
1204
1205	#[async_trait]
1206	impl Executor for ScriptedExec {
1207	async fn run_streaming(
1208	&self,
1209	step: &ops_exec::Step,
1210	_sink: &mut dyn LogSink,
1211	) -> anyhow::Result<RunOutput> {
1212	let cmd = step.argv.last().cloned().unwrap_or_default();
1213	self.log.lock().unwrap().push(cmd.clone());
1214	let (code, stdout) = self.rules.iter().find(\|r\| cmd.contains(&r.needle)).map_or(
1215	(0, String::new()),
1216	\|r\| {
1217	let i = r.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1218	r.responses[i.min(r.responses.len() - 1)].clone()
1219	},
1220	);
1221	Ok(RunOutput {
1222	// Shift into the wait-status word's exit-code byte so
1223	// `ExitStatus::code()` reports `code` exactly (a bare
1224	// `from_raw(1)` reads as a signal, yielding `None`).
1225	status: std::process::ExitStatus::from_raw(code << 8),
1226	stdout: stdout.into_bytes(),
1227	stderr: Vec::new(),
1228	})
1229	}
1230	async fn pull_file(
1231	&self,
1232	_r: &std::path::Path,
1233	_l: &std::path::Path,
1234	_o: &SyncOpts,
1235	) -> anyhow::Result<()> {
1236	Ok(())
1237	}
1238	async fn pull_dir(
1239	&self,
1240	_r: &std::path::Path,
1241	_l: &std::path::Path,
1242	_o: &SyncOpts,
1243	) -> anyhow::Result<()> {
1244	Ok(())
1245	}
1246	async fn pull_glob(
1247	&self,
1248	_g: &str,
1249	_l: &std::path::Path,
1250	_o: &SyncOpts,
1251	) -> anyhow::Result<()> {
1252	Ok(())
1253	}
1254	async fn push_dir(
1255	&self,
1256	_l: &std::path::Path,
1257	_r: &std::path::Path,
1258	_o: &SyncOpts,
1259	) -> anyhow::Result<()> {
1260	Ok(())
1261	}
1262	async fn preflight(&self) -> anyhow::Result<()> {
1263	Ok(())
1264	}
1265	fn capabilities(&self) -> &CapabilitySet {
1266	&self.caps
1267	}
1268	}
1269
1270	/// Assemble an [`AppState`] from the three per-test inputs, filling in the
1271	/// executors/syncs (built from `topo`) and the fixed test scaffolding
1272	/// (metrics handle, event bus, standard OTA registry, empty active map, no
1273	/// token). Every runner test builds the same struct around a different repo
1274	/// + recipe; this is that struct in one place.
1275	fn test_state(pool: SqlitePool, topo: Topology, cfg: Config) -> AppState {
1276	let executors = Arc::new(crate::state::build_executors(&topo));
1277	let syncs = Arc::new(crate::state::build_syncs(&topo));
1278	let host_locks = crate::state::build_host_locks(&topo);
1279	AppState {
1280	pool,
1281	topo: Arc::new(topo),
1282	cfg: Arc::new(cfg),
1283	prom: crate::metrics::test_handle(),
1284	events: crate::events::channel(),
1285	ota: Arc::new(OtaRegistry::standard("https://makenot.work")),
1286	executors,
1287	syncs,
1288	active: Arc::new(Mutex::new(HashMap::new())),
1289	api_token: None,
1290	host_locks,
1291	distribution: Arc::new(Mutex::new(HashMap::new())),
1292	http: crate::tls::builder().build().unwrap(),
1293	// Port 1 refuses instantly. A test must never probe production, and
1294	// a refusal is also faster than any timeout would be.
1295	mnw_base_url: "http://127.0.0.1:1".into(),
1296	}
1297	}
1298
1299	/// A `kind = "service"` release end to end: build, `glibc_check`, `deploy`,
1300	/// and a health assertion the recipe makes itself against the service host.
1301	///
1302	/// The whole point of the deploy step is that it dispatches to a host
1303	/// OUTSIDE the build topology, so this exercises the resolution
1304	/// (target -> `[[deploy]]` entry -> executor registered for the run), the
1305	/// staging, and the call into the privileged installer — with a fake
1306	/// installer standing in for the root script, which is the one part a test
1307	/// cannot run for real.
1308	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1309	async fn service_recipe_builds_then_deploys_and_verifies() {
1310	let tmp = tempfile::tempdir().unwrap();
1311	let root = tmp.path();
1312	let repo = root.join("svc");
1313	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1314	std::fs::write(repo.join("Cargo.toml"), "[package]\nversion = \"0.4.0\"\n").unwrap();
1315
1316	// Stand-in for the root installer: same three arguments, records what it
1317	// was asked to do instead of writing to /usr/local/bin and restarting a
1318	// unit. `install` + a marker file, so the test can assert the binary
1319	// that arrived is the binary that was built.
1320	let installer = root.join("install-service.sh");
1321	std::fs::write(
1322	&installer,
1323	"#!/bin/sh\nset -eu\ninstall -m 0755 \"$1\" \"$2\"\necho \"restarted $3\" >> \"$2.log\"\n",
1324	)
1325	.unwrap();
1326	std::fs::set_permissions(
1327	&installer,
1328	<std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o755),
1329	)
1330	.unwrap();
1331	let install_path = root.join("bin/svc");
1332	std::fs::create_dir_all(root.join("bin")).unwrap();
1333
1334	std::fs::write(
1335	repo.join("dist/recipes/linux.rhai"),
1336	r#"
1337	let v = version();
1338	step("build");
1339	sh_ok("fw13", "mkdir -p REPO/target/release && echo built-BIN > REPO/target/release/svc");
1340	step("verify");
1341	log(glibc_check("REPO/target/release/svc"));
1342	step("deploy");
1343	log(deploy("REPO/target/release/svc"));
1344	// The recipe owns what "healthy" means, and asserts it itself
1345	// against the host it just restarted.
1346	sh_ok(deploy_host(), "test -x " + install_path());
1347	"#
1348	.replace("REPO", repo.to_str().unwrap())
1349	.replace("BIN", "0.4.0"),
1350	)
1351	.unwrap();
1352
1353	std::fs::write(
1354	repo.join("bento.toml"),
1355	format!(
1356	r#"kind = "service"
1357	targets = ["linux/x86_64"]
1358	version_path = "Cargo.toml"
1359
1360	[[deploy]]
1361	target = "linux/x86_64"
1362	host = "local"
1363	install_path = "{}"
1364	service = "svc.service"
1365	health_url = "http://localhost:9100/api/health"
1366	"#,
1367	install_path.display()
1368	),
1369	)
1370	.unwrap();
1371
1372	let mut cfg = Config::for_tests(root);
1373	cfg.deploy_installer = installer.display().to_string();
1374	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1375	let topo = Topology::from_str_for_tests(&format!(
1376	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
1377	pull_root = \"{repo}\"\n\n[app.svc]\nrepo = \"{repo}\"\n",
1378	repo = repo.display()
1379	))
1380	.unwrap();
1381	let state = test_state(pool.clone(), topo, cfg);
1382
1383	let build_id = start_build(
1384	state.clone(),
1385	AppId::new("svc"),
1386	Version::parse("0.4.0").unwrap(),
1387	vec!["linux/x86_64".parse().unwrap()],
1388	)
1389	.await
1390	.unwrap();
1391
1392	let mut status = String::new();
1393	for _ in 0..100 {
1394	status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1395	.bind(build_id)
1396	.fetch_optional(&pool)
1397	.await
1398	.unwrap()
1399	.unwrap_or_else(\|\| "running".to_string());
1400	if status != "running" {
1401	break;
1402	}
1403	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1404	}
1405	assert_eq!(status, "ok", "service run should succeed");
1406
1407	let steps: Vec<(String, String)> = sqlx::query_as(
1408	"SELECT step, status FROM step_runs WHERE target_run_id IN \
1409	(SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id",
1410	)
1411	.bind(build_id)
1412	.fetch_all(&pool)
1413	.await
1414	.unwrap();
1415	assert_eq!(
1416	steps.iter().map(\|(s, _)\| s.as_str()).collect::<Vec<_>>(),
1417	vec!["build", "verify", "deploy"],
1418	"a service ends at deploy, not collect"
1419	);
1420	assert!(steps.iter().all(\|(_, st)\| st == "ok"), "{steps:?}");
1421
1422	// The bytes that were built are the bytes that landed, and the unit was
1423	// restarted only after the install succeeded.
1424	assert_eq!(
1425	std::fs::read_to_string(&install_path).unwrap().trim(),
1426	"built-0.4.0"
1427	);
1428	assert!(
1429	std::fs::read_to_string(install_path.with_extension("").with_file_name("svc.log"))
1430	.unwrap()
1431	.contains("restarted svc.service")
1432	);
1433	}
1434
1435	/// Stand up a tmp app repo + topology and run a real local recipe end to
1436	/// end: step transitions, streamed `sh_ok`, `version_of`, `log`, and a
1437	/// `collect` that pulls a built artifact into dist_root.
1438	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1439	async fn local_linux_recipe_runs_end_to_end() {
1440	let tmp = tempfile::tempdir().unwrap();
1441	let root = tmp.path();
1442
1443	// Fake app checkout: tauri.conf.json + a linux recipe.
1444	let repo = root.join("app");
1445	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1446	std::fs::write(
1447	repo.join("src-tauri/tauri.conf.json"),
1448	r#"{"version":"0.0.1"}"#,
1449	)
1450	.unwrap();
1451	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1452	// Build writes an artifact into the repo; collect pulls it to dist_root.
1453	std::fs::write(
1454	repo.join("dist/recipes/linux.rhai"),
1455	r#"
1456	step("build");
1457	let v = version_of("demo");
1458	log("building demo " + v);
1459	sh_ok("fw13", "echo compiling; mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
1460	step("collect");
1461	collect("fw13", "REPO/out/demo.bin", "demo", v);
1462	"#
1463	.replace("REPO", repo.to_str().unwrap()),
1464	)
1465	.unwrap();
1466
1467	let mut cfg = Config::for_tests(root);
1468	// Archive to a local directory, so the deposit a real release makes to
1469	// astra runs on the same code path here.
1470	cfg.archive = Some(crate::config::Archive {
1471	host: "local".into(),
1472	root: root.join("archive"),
1473	});
1474	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1475	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1476	let topo = Topology::from_str_for_tests(&format!(
1477	r#"
1478	[[host]]
1479	name = "fw13"
1480	ssh = "local"
1481	targets = ["linux/x86_64"]
1482	pull_root = "{repo}"
1483
1484	[app.demo]
1485	repo = "{repo}"
1486	"#,
1487	repo = repo.display()
1488	))
1489	.unwrap();
1490
1491	let state = test_state(pool.clone(), topo, cfg);
1492
1493	let app = AppId::new("demo");
1494	let version = Version::parse("0.0.1").unwrap();
1495	let build_id = start_build(
1496	state.clone(),
1497	app,
1498	version,
1499	vec!["linux/x86_64".parse().unwrap()],
1500	)
1501	.await
1502	.unwrap();
1503
1504	// Wait for the target run to settle. The row may not be inserted on the
1505	// first poll (run_target is spawned, not awaited), so treat a missing row
1506	// as still-pending rather than an error.
1507	let mut status = String::new();
1508	for _ in 0..100 {
1509	status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1510	.bind(build_id)
1511	.fetch_optional(&pool)
1512	.await
1513	.unwrap()
1514	.unwrap_or_else(\|\| "running".to_string());
1515	if status != "running" {
1516	break;
1517	}
1518	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1519	}
1520	assert_eq!(status, "ok", "target run should succeed");
1521
1522	// Both steps recorded and finished ok.
1523	let steps: Vec<(String, String)> =
1524	sqlx::query_as("SELECT step, status FROM step_runs WHERE target_run_id IN (SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id")
1525	.bind(build_id)
1526	.fetch_all(&pool)
1527	.await
1528	.unwrap();
1529	let names: Vec<&str> = steps.iter().map(\|(s, _)\| s.as_str()).collect();
1530	assert_eq!(names, vec!["build", "collect"]);
1531	assert!(steps.iter().all(\|(_, st)\| st == "ok"));
1532
1533	// Artifact landed in dist_root, under its own target, and a step log was
1534	// written. Both trees are keyed the same way, `<app>/<version>/<target>/`.
1535	let artifact = state.cfg.dist_root.join("demo/0.0.1/linux-x86_64/demo.bin");
1536	assert!(artifact.exists(), "collect should copy the artifact");
1537	// ...and the same bytes reached the archive, at the same path under its
1538	// own root. This is the answer to "where is demo 0.0.1 for linux".
1539	let archived = root.join("archive/demo/0.0.1/linux-x86_64/demo.bin");
1540	assert!(
1541	archived.exists(),
1542	"collect should deposit into the archive: {}",
1543	archived.display()
1544	);
1545	assert_eq!(
1546	std::fs::read(&archived).unwrap(),
1547	std::fs::read(&artifact).unwrap()
1548	);
1549	// Read the path off the ledger rather than rebuilding it: the log is
1550	// named for its step run id, and the point of that is that the row
1551	// resolves to exactly one file.
1552	let (run_id, log_ref): (i64, String) = sqlx::query_as(
1553	"SELECT id, log_ref FROM step_runs WHERE step = 'build' AND target_run_id IN \
1554	(SELECT id FROM target_runs WHERE build_id = ?)",
1555	)
1556	.bind(build_id)
1557	.fetch_one(&pool)
1558	.await
1559	.unwrap();
1560	let log = std::path::PathBuf::from(&log_ref);
1561	assert_eq!(
1562	log,
1563	state
1564	.cfg
1565	.logs_root
1566	.join(format!("demo/0.0.1/linux-x86_64/build.{run_id}.log")),
1567	"log path should be keyed on the step run id"
1568	);
1569	assert!(log.exists(), "build step log should exist");
1570	let body = std::fs::read_to_string(&log).unwrap();
1571	assert!(body.contains("compiling"));
1572	assert!(
1573	body.starts_with(&format!(
1574	"=== bento demo 0.0.1 linux/x86_64 step=build run_id={run_id} "
1575	)),
1576	"log should open with a run header naming its run: {body}"
1577	);
1578	}
1579
1580	/// Multi-target fan-out, which is what the daemon exists for: every other
1581	/// runner test drives exactly one target, so nothing covered `start_build`'s
1582	/// `JoinSet` fan-out or `finalize_build`'s rollup. One target fails and one
1583	/// succeeds — the failure must not abort its sibling, both must land their
1584	/// own terminal row, and the build must finalize `failed` because any failed
1585	/// target fails the build.
1586	#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1587	async fn multi_target_fan_out_rolls_up_partial_failure() {
1588	let tmp = tempfile::tempdir().unwrap();
1589	let root = tmp.path();
1590
1591	let repo = root.join("app");
1592	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1593	std::fs::write(
1594	repo.join("src-tauri/tauri.conf.json"),
1595	r#"{"version":"0.0.1"}"#,
1596	)
1597	.unwrap();
1598	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1599
1600	// Linux succeeds and collects a real artifact. The artifact is the proof
1601	// this target ran to completion rather than being torn down when its
1602	// sibling failed.
1603	std::fs::write(
1604	repo.join("dist/recipes/linux.rhai"),
1605	r#"
1606	step("build");
1607	let v = version_of("demo");
1608	sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
1609	step("collect");
1610	collect("fw13", "REPO/out/demo.bin", "demo", v);
1611	"#
1612	.replace("REPO", repo.to_str().unwrap()),
1613	)
1614	.unwrap();
1615	// Windows fails in its first step.
1616	std::fs::write(
1617	repo.join("dist/recipes/windows.rhai"),
1618	r#"
1619	step("build");
1620	sh_ok("winbox", "echo nope 1>&2; exit 1");
1621	"#,
1622	)
1623	.unwrap();
1624
1625	let cfg = Config::for_tests(root);
1626	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1627	std::fs::write(
1628	repo.join("bento.toml"),
1629	"targets = [\"linux/x86_64\", \"windows/x86_64\"]\n",
1630	)
1631	.unwrap();
1632	// Two hosts so each target resolves its own, mirroring the real
1633	// per-architecture topology.
1634	let topo = Topology::from_str_for_tests(&format!(
1635	r#"
1636	[[host]]
1637	name = "fw13"
1638	ssh = "local"
1639	targets = ["linux/x86_64"]
1640	pull_root = "{repo}"
1641
1642	[[host]]
1643	name = "winbox"
1644	ssh = "local"
1645	targets = ["windows/x86_64"]
1646
1647	[app.demo]
1648	repo = "{repo}"
1649	"#,
1650	repo = repo.display()
1651	))
1652	.unwrap();
1653
1654	let state = test_state(pool.clone(), topo, cfg);
1655
1656	let build_id = start_build(
1657	state.clone(),
1658	AppId::new("demo"),
1659	Version::parse("0.0.1").unwrap(),
1660	vec![
1661	"linux/x86_64".parse().unwrap(),
1662	"windows/x86_64".parse().unwrap(),
1663	],
1664	)
1665	.await
1666	.unwrap();
1667
1668	// Poll the build row, not the target rows: the build is terminal only
1669	// once finalize_build has joined every task and stamped it.
1670	let mut build_status = String::new();
1671	for _ in 0..200 {
1672	build_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
1673	.bind(build_id)
1674	.fetch_one(&pool)
1675	.await
1676	.unwrap();
1677	if build_status != "running" {
1678	break;
1679	}
1680	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1681	}
1682
1683	let runs: Vec<(String, String)> = sqlx::query_as(
1684	"SELECT target, status FROM target_runs WHERE build_id = ? ORDER BY target",
1685	)
1686	.bind(build_id)
1687	.fetch_all(&pool)
1688	.await
1689	.unwrap();
1690	assert_eq!(
1691	runs,
1692	vec![
1693	("linux/x86_64".to_string(), "ok".to_string()),
1694	("windows/x86_64".to_string(), "failed".to_string()),
1695	],
1696	"each target lands its own terminal row; one failing does not take the other down",
1697	);
1698
1699	// The surviving target finished its work, not merely its row.
1700	assert!(
1701	state
1702	.cfg
1703	.dist_root
1704	.join("demo/0.0.1/linux-x86_64/demo.bin")
1705	.exists(),
1706	"the succeeding target ran to completion and collected its artifact",
1707	);
1708
1709	// The failure is attributed to the target that failed, and only it.
1710	let err: Option<String> = sqlx::query_scalar(
1711	"SELECT error FROM target_runs WHERE build_id = ? AND target = 'windows/x86_64'",
1712	)
1713	.bind(build_id)
1714	.fetch_one(&pool)
1715	.await
1716	.unwrap();
1717	assert!(
1718	err.is_some_and(\|e\| !e.is_empty()),
1719	"a failed target records why",
1720	);
1721
1722	assert_eq!(
1723	build_status, "failed",
1724	"any failed target fails the build; a partial release must not read as ok",
1725	);
1726	let finished: Option<String> =
1727	sqlx::query_scalar("SELECT finished_at FROM builds WHERE id = ?")
1728	.bind(build_id)
1729	.fetch_one(&pool)
1730	.await
1731	.unwrap();
1732	assert!(finished.is_some(), "finalize_build stamps the finish time");
1733
1734	// finalize_build reaps its own latest-wins slots, so nothing is left
1735	// in flight to block a later build of the same targets.
1736	assert!(
1737	state.active.lock().await.is_empty(),
1738	"finalize_build reaps the slots it owned",
1739	);
1740	}
1741
1742	/// Latest-wins supersession, driven through `start_build` (not the leaf
1743	/// `begin_step` bail that's already covered). Two builds of the SAME
1744	/// (app, target) race: the second must cooperatively cancel + abort the
1745	/// first and take the single slot, the first must terminate non-`ok`, and
1746	/// the second must run to completion.
1747	#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1748	async fn a_newer_build_supersedes_the_in_flight_one_for_the_same_target() {
1749	let tmp = tempfile::tempdir().unwrap();
1750	let root = tmp.path();
1751
1752	let repo = root.join("app");
1753	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1754	std::fs::write(
1755	repo.join("src-tauri/tauri.conf.json"),
1756	r#"{"version":"0.0.1"}"#,
1757	)
1758	.unwrap();
1759	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1760	// A sleep long enough that the first build is still mid-recipe when the
1761	// second arrives; the engine checks the cancel flag at the step boundary
1762	// before "done", so the superseded run bails rather than finishing.
1763	std::fs::write(
1764	repo.join("dist/recipes/linux.rhai"),
1765	r#"
1766	step("build");
1767	sh_ok("fw13", "sleep 2");
1768	"#,
1769	)
1770	.unwrap();
1771
1772	let cfg = Config::for_tests(root);
1773	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1774	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1775	let topo = Topology::from_str_for_tests(&format!(
1776	r#"
1777	[[host]]
1778	name = "fw13"
1779	ssh = "local"
1780	targets = ["linux/x86_64"]
1781
1782	[app.demo]
1783	repo = "{}"
1784	"#,
1785	repo.display()
1786	))
1787	.unwrap();
1788	let state = test_state(pool.clone(), topo, cfg);
1789
1790	let target = "linux/x86_64".parse().unwrap();
1791	let first = start_build(
1792	state.clone(),
1793	AppId::new("demo"),
1794	Version::parse("0.0.1").unwrap(),
1795	vec![target],
1796	)
1797	.await
1798	.unwrap();
1799	// Let the first build register its slot and enter the recipe before the
1800	// second supersedes it (start_build inserts the slot synchronously, but
1801	// the recipe runs on a spawned task).
1802	tokio::time::sleep(std::time::Duration::from_millis(100)).await;
1803	let second = start_build(
1804	state.clone(),
1805	AppId::new("demo"),
1806	Version::parse("0.0.1").unwrap(),
1807	vec![target],
1808	)
1809	.await
1810	.unwrap();
1811	assert_ne!(first, second);
1812
1813	// Latest wins: exactly one slot for the key, owned by the second build.
1814	{
1815	let active = state.active.lock().await;
1816	assert_eq!(active.len(), 1, "supersession must not leave two slots");
1817	assert_eq!(
1818	active.values().next().unwrap().build_id,
1819	second,
1820	"the surviving slot belongs to the newer build",
1821	);
1822	}
1823
1824	// Wait for the second (surviving) build to finalize.
1825	let mut second_status = String::new();
1826	for _ in 0..200 {
1827	second_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
1828	.bind(second)
1829	.fetch_one(&pool)
1830	.await
1831	.unwrap();
1832	if second_status != "running" {
1833	break;
1834	}
1835	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1836	}
1837	assert_eq!(
1838	second_status, "ok",
1839	"the superseding build runs to completion"
1840	);
1841
1842	// The first build's target run terminated without succeeding — it was
1843	// cancelled/aborted, never stamped `ok`.
1844	let first_status: String =
1845	sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1846	.bind(first)
1847	.fetch_one(&pool)
1848	.await
1849	.unwrap();
1850	assert_ne!(
1851	first_status, "ok",
1852	"the superseded build must not complete successfully",
1853	);
1854
1855	// Both builds reaped their slots; nothing left in flight.
1856	assert!(
1857	state.active.lock().await.is_empty(),
1858	"every finalized build reaps its own slot",
1859	);
1860	}
1861
1862	/// A failing `preflight` (the agent host's `/health` probe when `ops-agent`
1863	/// is down) must fail the target BEFORE the recipe dispatches — the whole
1864	/// point of preflighting. Covered only via a fake here, since a real
1865	/// LocalExec/SshExec preflight is a no-op that always passes.
1866	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1867	async fn a_failing_preflight_fails_the_target_before_the_recipe_runs() {
1868	let tmp = tempfile::tempdir().unwrap();
1869	let root = tmp.path();
1870
1871	let repo = root.join("app");
1872	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1873	std::fs::write(
1874	repo.join("src-tauri/tauri.conf.json"),
1875	r#"{"version":"0.0.1"}"#,
1876	)
1877	.unwrap();
1878	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1879	// If the recipe ran it would drop this marker; preflight failing first
1880	// means it never does.
1881	let marker = root.join("recipe-ran");
1882	std::fs::write(
1883	repo.join("dist/recipes/linux.rhai"),
1884	r#"
1885	step("build");
1886	sh_ok("fw13", "touch MARKER");
1887	"#
1888	.replace("MARKER", marker.to_str().unwrap()),
1889	)
1890	.unwrap();
1891
1892	let cfg = Config::for_tests(root);
1893	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1894	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1895	let topo = Topology::from_str_for_tests(&format!(
1896	r#"
1897	[[host]]
1898	name = "fw13"
1899	ssh = "local"
1900	targets = ["linux/x86_64"]
1901
1902	[app.demo]
1903	repo = "{}"
1904	"#,
1905	repo.display()
1906	))
1907	.unwrap();
1908	let mut state = test_state(pool.clone(), topo, cfg);
1909	// Swap fw13's executor for one whose preflight fails.
1910	let mut execs = HashMap::new();
1911	execs.insert(
1912	"fw13".to_string(),
1913	FakeExec::preflight_fails("ops-agent not reachable at /health"),
1914	);
1915	state.executors = Arc::new(execs);
1916
1917	let build_id = start_build(
1918	state.clone(),
1919	AppId::new("demo"),
1920	Version::parse("0.0.1").unwrap(),
1921	vec!["linux/x86_64".parse().unwrap()],
1922	)
1923	.await
1924	.unwrap();
1925
1926	let mut status = String::new();
1927	let mut error = String::new();
1928	for _ in 0..100 {
1929	let row: Option<(String, Option<String>)> =
1930	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1931	.bind(build_id)
1932	.fetch_optional(&pool)
1933	.await
1934	.unwrap();
1935	if let Some((s, e)) = row {
1936	status = s;
1937	error = e.unwrap_or_default();
1938	if status != "running" {
1939	break;
1940	}
1941	}
1942	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1943	}
1944	assert_eq!(status, "failed", "a failed preflight must fail the target");
1945	assert!(
1946	error.contains("ops-agent not reachable"),
1947	"the failure must carry the preflight error, got: {error}"
1948	);
1949	assert!(
1950	!marker.exists(),
1951	"the recipe must NOT run when preflight fails"
1952	);
1953	}
1954
1955	/// A target whose recipe file is absent must fail at the `checkout` boundary
1956	/// with a readable "reading recipe" error, before any step runs — exercising
1957	/// `read_recipe`'s error branch and the `fail_target` path in `run_target`
1958	/// that neither the happy-path nor the failing-preflight test reaches. The
1959	/// app is configured to ship linux, but no `linux.rhai` is written.
1960	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1961	async fn a_target_with_no_recipe_file_fails_at_checkout() {
1962	let tmp = tempfile::tempdir().unwrap();
1963	let root = tmp.path();
1964
1965	let repo = root.join("app");
1966	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1967	std::fs::write(
1968	repo.join("src-tauri/tauri.conf.json"),
1969	r#"{"version":"0.0.1"}"#,
1970	)
1971	.unwrap();
1972	// The recipe dir exists but is empty — no linux.rhai.
1973	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1974
1975	let cfg = Config::for_tests(root);
1976	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1977	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1978	let topo = Topology::from_str_for_tests(&format!(
1979	r#"
1980	[[host]]
1981	name = "fw13"
1982	ssh = "local"
1983	targets = ["linux/x86_64"]
1984
1985	[app.demo]
1986	repo = "{}"
1987	"#,
1988	repo.display()
1989	))
1990	.unwrap();
1991	let state = test_state(pool.clone(), topo, cfg);
1992
1993	let build_id = start_build(
1994	state.clone(),
1995	AppId::new("demo"),
1996	Version::parse("0.0.1").unwrap(),
1997	vec!["linux/x86_64".parse().unwrap()],
1998	)
1999	.await
2000	.unwrap();
2001
2002	// Poll the build row: it is terminal only once finalize_build joins the
2003	// one (failing) target task.
2004	let mut build_status = String::new();
2005	for _ in 0..100 {
2006	build_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
2007	.bind(build_id)
2008	.fetch_one(&pool)
2009	.await
2010	.unwrap();
2011	if build_status != "running" {
2012	break;
2013	}
2014	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2015	}
2016
2017	// The target failed, and the error points at the unreadable recipe.
2018	let (status, error): (String, Option<String>) =
2019	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2020	.bind(build_id)
2021	.fetch_one(&pool)
2022	.await
2023	.unwrap();
2024	assert_eq!(status, "failed", "a missing recipe must fail the target");
2025	assert!(
2026	error.is_some_and(\|e\| e.contains("reading recipe")),
2027	"the failure must name the recipe it could not read",
2028	);
2029
2030	// The recipe never ran, so no step_runs row was ever created — the
2031	// failure is at the checkout boundary, ahead of any step.
2032	let step_count: i64 = sqlx::query_scalar(
2033	"SELECT COUNT(*) FROM step_runs WHERE target_run_id IN \
2034	(SELECT id FROM target_runs WHERE build_id = ?)",
2035	)
2036	.bind(build_id)
2037	.fetch_one(&pool)
2038	.await
2039	.unwrap();
2040	assert_eq!(
2041	step_count, 0,
2042	"no step should run when the recipe is absent"
2043	);
2044
2045	assert_eq!(
2046	build_status, "failed",
2047	"the build rolls up the target failure",
2048	);
2049	assert!(
2050	state.active.lock().await.is_empty(),
2051	"finalize_build reaps the slot even on the recipe-read failure path",
2052	);
2053	}
2054
2055	/// Run 2 S5: a publish whose version is not strictly newer than the latest
2056	/// already published for the same (app, target, channel) is refused — an
2057	/// older build cannot republish over a live newer release.
2058	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2059	async fn publish_rejects_a_non_monotonic_version() {
2060	let tmp = tempfile::tempdir().unwrap();
2061	let root = tmp.path();
2062	let repo = root.join("app");
2063	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2064	std::fs::write(
2065	repo.join("src-tauri/tauri.conf.json"),
2066	r#"{"version":"0.2.0"}"#,
2067	)
2068	.unwrap();
2069	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2070	let artifact = repo.join("out/app.tar.gz");
2071	// Build an artifact, publish 0.2.0 (records a release), then try to
2072	// publish the older 0.1.0 — the second publish must fail.
2073	std::fs::write(
2074	repo.join("dist/recipes/linux.rhai"),
2075	r#"
2076	step("build");
2077	sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
2078	step("publish");
2079	publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
2080	publish("tauri-mnw", "demo", "linux/x86_64", "0.1.0", "ARTIFACT", #{});
2081	"#
2082	.replace("ARTIFACT", artifact.to_str().unwrap())
2083	.replace("REPO", repo.to_str().unwrap()),
2084	)
2085	.unwrap();
2086
2087	let cfg = Config::for_tests(root);
2088	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2089	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2090	let topo = Topology::from_str_for_tests(&format!(
2091	r#"
2092	[[host]]
2093	name = "fw13"
2094	ssh = "local"
2095	targets = ["linux/x86_64"]
2096
2097	[app.demo]
2098	repo = "{}"
2099	"#,
2100	repo.display()
2101	))
2102	.unwrap();
2103	let state = test_state(pool.clone(), topo, cfg);
2104	let build_id = start_build(
2105	state.clone(),
2106	AppId::new("demo"),
2107	Version::parse("0.2.0").unwrap(),
2108	vec!["linux/x86_64".parse().unwrap()],
2109	)
2110	.await
2111	.unwrap();
2112
2113	let mut status = String::new();
2114	let mut error = String::new();
2115	for _ in 0..100 {
2116	let row: Option<(String, Option<String>)> =
2117	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2118	.bind(build_id)
2119	.fetch_optional(&pool)
2120	.await
2121	.unwrap();
2122	if let Some((s, e)) = row {
2123	status = s;
2124	error = e.unwrap_or_default();
2125	if status != "running" {
2126	break;
2127	}
2128	}
2129	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2130	}
2131	assert_eq!(status, "failed", "non-monotonic publish must fail the run");
2132	assert!(
2133	error.contains("not newer"),
2134	"expected monotonicity error, got: {error}"
2135	);
2136	// Exactly one release was recorded (0.2.0); 0.1.0 never landed.
2137	let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
2138	.fetch_one(&pool)
2139	.await
2140	.unwrap();
2141	assert_eq!(
2142	count, 1,
2143	"only the first (newer) publish should record a release"
2144	);
2145	// The recorded release carries the artifact's sha256 (64 hex chars),
2146	// not a NULL — the ledger says which bytes shipped.
2147	let hash: Option<String> =
2148	sqlx::query_scalar("SELECT artifact_hash FROM releases WHERE version = '0.2.0'")
2149	.fetch_one(&pool)
2150	.await
2151	.unwrap();
2152	assert!(
2153	hash.as_deref().is_some_and(\|h\| h.len() == 64),
2154	"publish must record the artifact sha256, got {hash:?}"
2155	);
2156	}
2157
2158	/// The artifact-identity fix at collect: a stale, wrongly-versioned artifact
2159	/// left in the output dir fails the collect instead of silently winning a
2160	/// later glob. Here the build is 0.0.1 but the recipe produces a 9.9.9 file.
2161	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2162	async fn collect_rejects_a_stale_versioned_artifact() {
2163	let tmp = tempfile::tempdir().unwrap();
2164	let root = tmp.path();
2165	let repo = root.join("app");
2166	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2167	std::fs::write(
2168	repo.join("src-tauri/tauri.conf.json"),
2169	r#"{"version":"0.0.1"}"#,
2170	)
2171	.unwrap();
2172	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2173	std::fs::write(
2174	repo.join("dist/recipes/linux.rhai"),
2175	r#"
2176	step("build");
2177	let v = version_of("demo");
2178	sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo-9.9.9.bin");
2179	step("collect");
2180	collect("fw13", "REPO/out/demo-9.9.9.bin", "demo", v);
2181	"#
2182	.replace("REPO", repo.to_str().unwrap()),
2183	)
2184	.unwrap();
2185
2186	let cfg = Config::for_tests(root);
2187	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2188	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2189	let topo = Topology::from_str_for_tests(&format!(
2190	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2191	pull_root = \"{repo}\"\n\n[app.demo]\nrepo = \"{repo}\"\n",
2192	repo = repo.display()
2193	))
2194	.unwrap();
2195	let state = test_state(pool.clone(), topo, cfg);
2196	let build_id = start_build(
2197	state.clone(),
2198	AppId::new("demo"),
2199	Version::parse("0.0.1").unwrap(),
2200	vec!["linux/x86_64".parse().unwrap()],
2201	)
2202	.await
2203	.unwrap();
2204
2205	let mut status = String::new();
2206	let mut error = String::new();
2207	for _ in 0..100 {
2208	let row: Option<(String, Option<String>)> =
2209	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2210	.bind(build_id)
2211	.fetch_optional(&pool)
2212	.await
2213	.unwrap();
2214	if let Some((s, e)) = row {
2215	status = s;
2216	error = e.unwrap_or_default();
2217	if status != "running" {
2218	break;
2219	}
2220	}
2221	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2222	}
2223	assert_eq!(
2224	status, "failed",
2225	"a mismatched-version artifact must fail collect"
2226	);
2227	assert!(
2228	error.contains("stale artifact"),
2229	"expected a stale-artifact error, got: {error}"
2230	);
2231	}
2232
2233	/// A command that runs past its step's deadline fails THAT step (and unwinds
2234	/// the recipe) rather than wedging under the old whole-build guillotine. The
2235	/// per-step budget is overridden to 1s here; the recipe then sleeps 30s, so
2236	/// the step deadline — not the sleep — decides the outcome, quickly.
2237	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2238	async fn a_step_that_exceeds_its_deadline_fails() {
2239	let tmp = tempfile::tempdir().unwrap();
2240	let root = tmp.path();
2241	let repo = root.join("app");
2242	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2243	std::fs::write(
2244	repo.join("src-tauri/tauri.conf.json"),
2245	r#"{"version":"0.0.1"}"#,
2246	)
2247	.unwrap();
2248	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2249	std::fs::write(
2250	repo.join("dist/recipes/linux.rhai"),
2251	r#"
2252	step("build");
2253	sh_ok("fw13", "sleep 30");
2254	"#,
2255	)
2256	.unwrap();
2257
2258	let mut cfg = Config::for_tests(root);
2259	cfg.step_timeout_secs = Some(1); // every step's budget -> 1s
2260	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2261	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2262	let topo = Topology::from_str_for_tests(&format!(
2263	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2264	[app.demo]\nrepo = \"{}\"\n",
2265	repo.display()
2266	))
2267	.unwrap();
2268	let state = test_state(pool.clone(), topo, cfg);
2269	let started = std::time::Instant::now();
2270	let build_id = start_build(
2271	state.clone(),
2272	AppId::new("demo"),
2273	Version::parse("0.0.1").unwrap(),
2274	vec!["linux/x86_64".parse().unwrap()],
2275	)
2276	.await
2277	.unwrap();
2278
2279	let mut status = String::new();
2280	let mut error = String::new();
2281	for _ in 0..100 {
2282	let row: Option<(String, Option<String>)> =
2283	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2284	.bind(build_id)
2285	.fetch_optional(&pool)
2286	.await
2287	.unwrap();
2288	if let Some((s, e)) = row {
2289	status = s;
2290	error = e.unwrap_or_default();
2291	if status != "running" {
2292	break;
2293	}
2294	}
2295	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2296	}
2297	assert_eq!(status, "failed", "a step past its deadline must fail");
2298	assert!(
2299	error.contains("per-step deadline"),
2300	"expected a deadline error, got: {error}"
2301	);
2302	// The deadline (1s), not the 30s sleep, decided it — proof the command
2303	// was actually interrupted rather than run to completion.
2304	assert!(
2305	started.elapsed() < std::time::Duration::from_secs(20),
2306	"the step deadline must fire well before the sleep would finish"
2307	);
2308	}
2309
2310	/// A demo app that publishes linux and has the all-targets-green gate on; it
2311	/// declares linux + macos, so publishing linux is gated on macos being green.
2312	async fn gate_state(root: &std::path::Path) -> (AppState, sqlx::SqlitePool) {
2313	let repo = root.join("app");
2314	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2315	std::fs::write(
2316	repo.join("src-tauri/tauri.conf.json"),
2317	r#"{"version":"0.2.0"}"#,
2318	)
2319	.unwrap();
2320	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2321	let artifact = repo.join("out/app.tar.gz");
2322	std::fs::write(
2323	repo.join("dist/recipes/linux.rhai"),
2324	r#"
2325	step("build");
2326	sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
2327	step("publish");
2328	publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
2329	"#
2330	.replace("ARTIFACT", artifact.to_str().unwrap())
2331	.replace("REPO", repo.to_str().unwrap()),
2332	)
2333	.unwrap();
2334	std::fs::write(
2335	repo.join("bento.toml"),
2336	"targets = [\"linux/x86_64\", \"macos/aarch64\"]\nrequire_all_targets = true\n",
2337	)
2338	.unwrap();
2339	let cfg = Config::for_tests(root);
2340	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2341	let topo = Topology::from_str_for_tests(&format!(
2342	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2343	[[host]]\nname = \"mbp\"\nssh = \"mbp\"\ntargets = [\"macos/aarch64\"]\n\
2344	[app.demo]\nrepo = \"{}\"\n",
2345	repo.display()
2346	))
2347	.unwrap();
2348	(test_state(pool.clone(), topo, cfg), pool)
2349	}
2350
2351	async fn await_target(pool: &sqlx::SqlitePool, build_id: i64) -> (String, String) {
2352	for _ in 0..100 {
2353	let row: Option<(String, Option<String>)> =
2354	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2355	.bind(build_id)
2356	.fetch_optional(pool)
2357	.await
2358	.unwrap();
2359	if let Some((s, e)) = row
2360	&& s != "running"
2361	{
2362	return (s, e.unwrap_or_default());
2363	}
2364	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2365	}
2366	panic!("target never settled");
2367	}
2368
2369	/// The all-targets-green gate blocks a partial release: linux tries to
2370	/// publish while macos has no successful run, so publish is refused and the
2371	/// target fails at its publish step.
2372	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2373	async fn all_green_gate_blocks_publish_when_a_sibling_is_not_green() {
2374	let tmp = tempfile::tempdir().unwrap();
2375	let (state, pool) = gate_state(tmp.path()).await;
2376	let build_id = start_build(
2377	state,
2378	AppId::new("demo"),
2379	Version::parse("0.2.0").unwrap(),
2380	vec!["linux/x86_64".parse().unwrap()],
2381	)
2382	.await
2383	.unwrap();
2384	let (status, error) = await_target(&pool, build_id).await;
2385	assert_eq!(status, "failed", "a partial release must be blocked");
2386	assert!(
2387	error.contains("all-targets-green gate"),
2388	"expected the gate to name itself, got: {error}"
2389	);
2390	// Nothing shipped.
2391	let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
2392	.fetch_one(&pool)
2393	.await
2394	.unwrap();
2395	assert_eq!(count, 0, "a gated-off publish records no release");
2396	}
2397
2398	/// With every sibling green, the gate lets the publish through. macos is
2399	/// pre-recorded `ok` for this version, so linux's publish proceeds.
2400	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2401	async fn all_green_gate_allows_publish_when_every_sibling_is_green() {
2402	let tmp = tempfile::tempdir().unwrap();
2403	let (state, pool) = gate_state(tmp.path()).await;
2404	// Pre-record a green macos run at 0.2.0 (as if its target already built).
2405	let bid: i64 = sqlx::query_scalar(
2406	"INSERT INTO builds (app, version, status, created_at) VALUES ('demo','0.2.0','ok','2026-07-23T00:00:00Z') RETURNING id",
2407	)
2408	.fetch_one(&pool)
2409	.await
2410	.unwrap();
2411	sqlx::query(
2412	"INSERT INTO target_runs (build_id, app, version, target, status, started_at)
2413	VALUES (?, 'demo', '0.2.0', 'macos/aarch64', 'ok', '2026-07-23T00:00:00Z')",
2414	)
2415	.bind(bid)
2416	.execute(&pool)
2417	.await
2418	.unwrap();
2419
2420	let build_id = start_build(
2421	state,
2422	AppId::new("demo"),
2423	Version::parse("0.2.0").unwrap(),
2424	vec!["linux/x86_64".parse().unwrap()],
2425	)
2426	.await
2427	.unwrap();
2428	let (status, error) = await_target(&pool, build_id).await;
2429	assert_eq!(
2430	status, "ok",
2431	"all siblings green -> publish proceeds ({error})"
2432	);
2433	let count: i64 =
2434	sqlx::query_scalar("SELECT COUNT(*) FROM releases WHERE target = 'linux/x86_64'")
2435	.fetch_one(&pool)
2436	.await
2437	.unwrap();
2438	assert_eq!(count, 1, "linux published once the gate was satisfied");
2439	}
2440
2441	/// A committed + tagged git repo whose linux recipe pins the host to the tag
2442	/// via `checkout_sha`. `tag` is created only when `Some`.
2443	fn init_git_app(repo: &std::path::Path, tauri_version: &str, tag: Option<&str>) {
2444	init_git_app_with_recipe(
2445	repo,
2446	tauri_version,
2447	tag,
2448	"step(\"checkout\");\nlet s = checkout_sha(build_host());\nlog(\"pinned \" + s);\n\
2449	step(\"build\");\nsh_ok(build_host(), \"true\");\n",
2450	);
2451	}
2452
2453	/// As [`init_git_app`], with the linux recipe spelled by the caller.
2454	fn init_git_app_with_recipe(
2455	repo: &std::path::Path,
2456	tauri_version: &str,
2457	tag: Option<&str>,
2458	recipe: &str,
2459	) {
2460	init_git_app_shipping(repo, tauri_version, tag, recipe, "[\"linux/x86_64\"]");
2461	}
2462
2463	/// As [`init_git_app_with_recipe`], with the manifest's target list spelled
2464	/// by the caller — a second target is what puts a second HOST in the
2465	/// preflight, which is the only way to reach its unwind path.
2466	fn init_git_app_shipping(
2467	repo: &std::path::Path,
2468	tauri_version: &str,
2469	tag: Option<&str>,
2470	recipe: &str,
2471	targets: &str,
2472	) {
2473	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2474	std::fs::write(
2475	repo.join("src-tauri/tauri.conf.json"),
2476	format!("{{\"version\":\"{tauri_version}\"}}"),
2477	)
2478	.unwrap();
2479	std::fs::write(repo.join("bento.toml"), format!("targets = {targets}\n")).unwrap();
2480	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2481	std::fs::write(repo.join("dist/recipes/linux.rhai"), recipe).unwrap();
2482	// Isolate from the dev's global git config (which forces signed tags).
2483	let run = \|args: &[&str]\| {
2484	let out = std::process::Command::new("git")
2485	.args(args)
2486	.current_dir(repo)
2487	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2488	.env("GIT_CONFIG_SYSTEM", "/dev/null")
2489	.output()
2490	.expect("git runs");
2491	assert!(
2492	out.status.success(),
2493	"git {args:?}: {}",
2494	String::from_utf8_lossy(&out.stderr)
2495	);
2496	};
2497	run(&["init", "-q"]);
2498	run(&["-c", "user.email=t@t", "-c", "user.name=t", "add", "-A"]);
2499	run(&[
2500	"-c",
2501	"user.email=t@t",
2502	"-c",
2503	"user.name=t",
2504	"commit",
2505	"-q",
2506	"-m",
2507	"init",
2508	]);
2509	if let Some(t) = tag {
2510	run(&["tag", t]);
2511	}
2512	}
2513
2514	fn one_host_topo(repo: &std::path::Path) -> Topology {
2515	Topology::from_str_for_tests(&format!(
2516	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2517	[app.demo]\nrepo = \"{}\"\n",
2518	repo.display()
2519	))
2520	.unwrap()
2521	}
2522
2523	/// The release preflight (pin on) pins the host to the tag and the build
2524	/// proceeds. Exercises both the barrier and the `checkout_sha` host function.
2525	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2526	async fn release_preflight_pins_and_builds_when_the_tag_is_present() {
2527	let tmp = tempfile::tempdir().unwrap();
2528	let repo = tmp.path().join("demo");
2529	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2530	let mut cfg = Config::for_tests(tmp.path());
2531	cfg.pin_release_sha = true;
2532	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2533	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2534	let build_id = start_build(
2535	state,
2536	AppId::new("demo"),
2537	Version::parse("0.0.1").unwrap(),
2538	vec!["linux/x86_64".parse().unwrap()],
2539	)
2540	.await
2541	.unwrap();
2542	let (status, error) = await_target(&pool, build_id).await;
2543	assert_eq!(status, "ok", "a pinned build should succeed ({error})");
2544	}
2545
2546	/// A pinned build writes an artifact record beside what it collected: the
2547	/// manifest of those bytes, the commit the preflight pinned, and the steps
2548	/// as artifact-scoped gates.
2549	///
2550	/// The three facts already existed and met nowhere — the per-file sha256 at
2551	/// `collect`, the sha at the preflight, the step outcomes in `step_runs` —
2552	/// which is how gates came to vouch for one thing while the deploy shipped
2553	/// another.
2554	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2555	async fn a_pinned_build_writes_an_artifact_record_for_what_it_collected() {
2556	let tmp = tempfile::tempdir().unwrap();
2557	let repo = tmp.path().join("demo");
2558	std::fs::create_dir_all(&repo).unwrap();
2559	let recipe = r#"
2560	step("checkout");
2561	let sha = checkout_sha(build_host());
2562	log("pinned " + sha);
2563	step("build");
2564	sh_ok(build_host(), "mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
2565	step("collect");
2566	collect(build_host(), "REPO/out/demo.bin", "demo", version());
2567	"#
2568	.replace("REPO", repo.to_str().unwrap());
2569	init_git_app_with_recipe(&repo, "0.0.1", Some("v0.0.1"), &recipe);
2570
2571	let mut cfg = Config::for_tests(tmp.path());
2572	cfg.pin_release_sha = true;
2573	let dist_root = cfg.dist_root.clone();
2574	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2575	let topo = Topology::from_str_for_tests(&format!(
2576	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2577	pull_root = \"{repo}\"\n[app.demo]\nrepo = \"{repo}\"\n",
2578	repo = repo.display()
2579	))
2580	.unwrap();
2581	let state = test_state(pool.clone(), topo, cfg);
2582	let build_id = start_build(
2583	state,
2584	AppId::new("demo"),
2585	Version::parse("0.0.1").unwrap(),
2586	vec!["linux/x86_64".parse().unwrap()],
2587	)
2588	.await
2589	.unwrap();
2590	let (status, error) = await_target(&pool, build_id).await;
2591	assert_eq!(status, "ok", "the build should succeed ({error})");
2592
2593	let path = crate::artifact_record::record_path(
2594	&dist_root,
2595	&AppId::new("demo"),
2596	&Version::parse("0.0.1").unwrap(),
2597	"linux/x86_64".parse().unwrap(),
2598	);
2599	let json = std::fs::read_to_string(&path)
2600	.unwrap_or_else(\|e\| panic!("record at {}: {e}", path.display()));
2601	// Parsing revalidates, so this also asserts the digest matches the
2602	// manifest and no environment-scoped gate slipped in.
2603	let record = ops_artifact::ArtifactRecord::parse(&json).unwrap();
2604
2605	assert_eq!(record.producer, "bento");
2606	assert_eq!(record.manifest.entries().len(), 1);
2607	assert_eq!(record.manifest.entries()[0].path, "demo.bin");
2608	assert_eq!(record.digest, record.manifest.digest());
2609
2610	// The provenance names the commit the preflight pinned, not a rebuild of
2611	// whatever the branch is now.
2612	let head = std::process::Command::new("git")
2613	.args(["rev-parse", "v0.0.1^{commit}"])
2614	.current_dir(&repo)
2615	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2616	.output()
2617	.unwrap();
2618	let head = String::from_utf8_lossy(&head.stdout).trim().to_string();
2619	assert_eq!(record.provenance.git_sha, head);
2620	assert_eq!(record.provenance.target, "linux/x86_64");
2621	assert_eq!(record.provenance.build_host, "fw13");
2622	assert!(!record.provenance.toolchain.is_empty());
2623
2624	let gates: Vec<&str> = record.gates.iter().map(\|g\| g.gate.as_str()).collect();
2625	assert_eq!(gates, ["checkout", "build", "collect"]);
2626	assert!(record.all_gates_passed());
2627	assert!(
2628	record
2629	.gates
2630	.iter()
2631	.all(\|g\| g.scope == ops_artifact::Scope::Artifact),
2632	"a build host cannot vouch for an environment"
2633	);
2634	}
2635
2636	/// The barrier refuses a target no host can build, instead of leaving it out
2637	/// of the pin and letting the remaining hosts vouch for the release.
2638	///
2639	/// The hole this closes is silence reading as agreement: the comparison used
2640	/// to run over the hosts that reported, so a target dropped for want of a
2641	/// host still built while the other hosts' unanimity looked like proof the
2642	/// whole release came from one commit.
2643	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2644	async fn release_preflight_refuses_a_target_no_host_can_build() {
2645	let tmp = tempfile::tempdir().unwrap();
2646	let repo = tmp.path().join("demo");
2647	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2648	let mut cfg = Config::for_tests(tmp.path());
2649	cfg.pin_release_sha = true;
2650	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2651	// The topology declares fw13 (linux/x86_64) only, so macOS has no host.
2652	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2653	let err = start_build(
2654	state,
2655	AppId::new("demo"),
2656	Version::parse("0.0.1").unwrap(),
2657	vec!["macos/aarch64".parse().unwrap()],
2658	)
2659	.await
2660	.unwrap_err();
2661	let msg = format!("{err:#}");
2662	assert!(
2663	msg.contains("no host can build macos/aarch64"),
2664	"the error must name the unbuildable target, got: {msg}"
2665	);
2666	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2667	.fetch_one(&pool)
2668	.await
2669	.unwrap();
2670	assert_eq!(builds, 0, "a refused preflight writes no build row");
2671	}
2672
2673	/// The preflight refuses a host whose tree has uncommitted changes to
2674	/// tracked files.
2675	///
2676	/// This is the hole the rev-parse barrier cannot see. `git checkout <tag>`
2677	/// does NOT fail on local modifications to files whose content is unchanged
2678	/// in the tag — it succeeds and keeps them. So a dirty host builds its edits
2679	/// while honestly reporting the tagged sha, and a clean host builds
2680	/// something else: two architectures, two different binaries, one tag
2681	/// containing neither combination. Caught on pom's first release, where
2682	/// fw13 had uncommitted changes in the serve path and astra did not.
2683	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2684	async fn release_preflight_refuses_a_dirty_working_tree() {
2685	let tmp = tempfile::tempdir().unwrap();
2686	let repo = tmp.path().join("demo");
2687	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2688
2689	// Modify a TRACKED file, exactly as an editor session would.
2690	let tracked = repo.join("src-tauri/tauri.conf.json");
2691	let body = std::fs::read_to_string(&tracked).unwrap();
2692	std::fs::write(&tracked, format!("{body}\n")).unwrap();
2693
2694	let mut cfg = Config::for_tests(tmp.path());
2695	cfg.pin_release_sha = true;
2696	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2697	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2698	let err = start_build(
2699	state,
2700	AppId::new("demo"),
2701	Version::parse("0.0.1").unwrap(),
2702	vec!["linux/x86_64".parse().unwrap()],
2703	)
2704	.await
2705	.unwrap_err();
2706	let msg = format!("{err:#}");
2707	assert!(
2708	msg.contains("uncommitted changes") && msg.contains("tauri.conf.json"),
2709	"the error must name the host and the files, got: {msg}"
2710	);
2711	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2712	.fetch_one(&pool)
2713	.await
2714	.unwrap();
2715	assert_eq!(builds, 0, "a refused preflight writes no build row");
2716	}
2717
2718	/// An UNTRACKED file does not fail a release. A build host accumulates
2719	/// editor scratch and stray logs, none of which reach the binary, so failing
2720	/// on them would be noise that trains an operator to bypass the gate.
2721	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2722	async fn release_preflight_tolerates_untracked_files() {
2723	let tmp = tempfile::tempdir().unwrap();
2724	let repo = tmp.path().join("demo");
2725	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2726	std::fs::write(repo.join("scratch.log"), "noise").unwrap();
2727
2728	let mut cfg = Config::for_tests(tmp.path());
2729	cfg.pin_release_sha = true;
2730	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2731	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2732	let build_id = start_build(
2733	state,
2734	AppId::new("demo"),
2735	Version::parse("0.0.1").unwrap(),
2736	vec!["linux/x86_64".parse().unwrap()],
2737	)
2738	.await
2739	.unwrap();
2740	let (status, error) = await_target(&pool, build_id).await;
2741	assert_eq!(
2742	status, "ok",
2743	"untracked files must not fail a release ({error})"
2744	);
2745	}
2746
2747	/// The preflight refuses the build (before any row is written) when the
2748	/// release tag does not exist on the host — a missing/unpushed tag can't
2749	/// silently fall back to whatever `main` is.
2750	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2751	async fn release_preflight_refuses_when_the_release_tag_is_missing() {
2752	let tmp = tempfile::tempdir().unwrap();
2753	let repo = tmp.path().join("demo");
2754	// App is 0.0.2 (so the version check passes) but only v0.0.1 is tagged.
2755	init_git_app(&repo, "0.0.2", Some("v0.0.1"));
2756	let mut cfg = Config::for_tests(tmp.path());
2757	cfg.pin_release_sha = true;
2758	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2759	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2760	let err = start_build(
2761	state,
2762	AppId::new("demo"),
2763	Version::parse("0.0.2").unwrap(),
2764	vec!["linux/x86_64".parse().unwrap()],
2765	)
2766	.await
2767	.unwrap_err();
2768	let msg = format!("{err:#}");
2769	assert!(
2770	msg.contains("release preflight") && msg.contains("v0.0.2"),
2771	"expected a preflight tag error, got: {msg}"
2772	);
2773	assert!(
2774	msg.contains("does not exist"),
2775	"the error should name the absent tag as the cause, got: {msg}"
2776	);
2777	// Refused before anything was recorded.
2778	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2779	.fetch_one(&pool)
2780	.await
2781	.unwrap();
2782	assert_eq!(builds, 0, "a refused preflight writes no build row");
2783	}
2784
2785	/// The branch a release was launched from is still checked out afterwards.
2786	///
2787	/// The preflight pins every host to `v<version>`, which detaches HEAD. That
2788	/// is correct during the build and wrong to leave behind: git does not warn,
2789	/// and later commits succeed while belonging to no branch. makeover shipped
2790	/// 2.3.0 from exactly that state on 2026-07-28 — the published commit existed
2791	/// only as a detached HEAD on one machine, on no branch and no remote.
2792	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2793	async fn a_release_leaves_the_checkout_on_its_branch() {
2794	let tmp = tempfile::tempdir().unwrap();
2795	let repo = tmp.path().join("demo");
2796	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2797	let branch_before = current_branch(&repo);
2798	assert!(!branch_before.is_empty(), "test repo starts on a branch");
2799
2800	let mut cfg = Config::for_tests(tmp.path());
2801	cfg.pin_release_sha = true;
2802	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2803	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2804	let build_id = start_build(
2805	state,
2806	AppId::new("demo"),
2807	Version::parse("0.0.1").unwrap(),
2808	vec!["linux/x86_64".parse().unwrap()],
2809	)
2810	.await
2811	.unwrap();
2812	let (status, error) = await_target(&pool, build_id).await;
2813	assert_eq!(status, "ok", "the build itself should pass ({error})");
2814
2815	// finalize_build restores after the targets settle, so give it a moment.
2816	for _ in 0..50 {
2817	if current_branch(&repo) == branch_before {
2818	break;
2819	}
2820	tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2821	}
2822	assert_eq!(
2823	current_branch(&repo),
2824	branch_before,
2825	"the release must put the checkout back on its branch, not leave it detached"
2826	);
2827	}
2828
2829	/// A checkout that is ALREADY detached is refused, rather than released from
2830	/// and left detached again. It means an earlier release never cleaned up, and
2831	/// anything committed since is on no branch — which is precisely the state
2832	/// that has to be looked at by a human before more releases pile onto it.
2833	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2834	async fn a_detached_checkout_is_refused_before_it_is_pinned_again() {
2835	let tmp = tempfile::tempdir().unwrap();
2836	let repo = tmp.path().join("demo");
2837	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2838	// Detach, standing in for a checkout an earlier release left on its tag.
2839	let out = std::process::Command::new("git")
2840	.args(["checkout", "--detach", "-q", "HEAD"])
2841	.current_dir(&repo)
2842	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2843	.env("GIT_CONFIG_SYSTEM", "/dev/null")
2844	.output()
2845	.expect("git runs");
2846	assert!(out.status.success());
2847	assert!(current_branch(&repo).is_empty(), "repo is detached");
2848
2849	let mut cfg = Config::for_tests(tmp.path());
2850	cfg.pin_release_sha = true;
2851	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2852	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2853	let err = start_build(
2854	state,
2855	AppId::new("demo"),
2856	Version::parse("0.0.1").unwrap(),
2857	vec!["linux/x86_64".parse().unwrap()],
2858	)
2859	.await
2860	.unwrap_err();
2861	let msg = format!("{err:#}");
2862	assert!(
2863	msg.contains("detached HEAD"),
2864	"the refusal should name the detached checkout, got: {msg}"
2865	);
2866	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2867	.fetch_one(&pool)
2868	.await
2869	.unwrap();
2870	assert_eq!(builds, 0, "a refused preflight writes no build row");
2871	}
2872
2873	/// A preflight that refuses on a LATER host puts the earlier ones back.
2874	///
2875	/// The hosts are pinned one at a time, so by the time the second is refused
2876	/// the first is already detached at the tag. Nothing used to restore it: the
2877	/// branches were carried on the success value, which a refusal throws away.
2878	/// The next attempt then refused that host for a detached HEAD — true, and
2879	/// describing its own last run, which reads to the operator as their own
2880	/// mistake. Hit three times releasing pom on 2026-08-09.
2881	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2882	async fn a_refused_preflight_reattaches_the_hosts_it_already_pinned() {
2883	let tmp = tempfile::tempdir().unwrap();
2884	// fw13's checkout is tagged and clean, so it pins. mbp's has no v0.0.1,
2885	// so it is refused after fw13 has already been moved.
2886	let repo = tmp.path().join("demo");
2887	init_git_app_shipping(
2888	&repo,
2889	"0.0.1",
2890	Some("v0.0.1"),
2891	"step(\"build\");\nsh_ok(build_host(), \"true\");\n",
2892	"[\"linux/x86_64\", \"macos/aarch64\"]",
2893	);
2894	let other = tmp.path().join("demo-mbp");
2895	init_git_app_shipping(
2896	&other,
2897	"0.0.1",
2898	None,
2899	"step(\"build\");\nsh_ok(build_host(), \"true\");\n",
2900	"[\"linux/x86_64\", \"macos/aarch64\"]",
2901	);
2902	let branch_before = current_branch(&repo);
2903	assert!(
2904	!branch_before.is_empty(),
2905	"fw13's checkout starts on a branch"
2906	);
2907
2908	let mut cfg = Config::for_tests(tmp.path());
2909	cfg.pin_release_sha = true;
2910	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2911	let topo = Topology::from_str_for_tests(&format!(
2912	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2913	[[host]]\nname = \"mbp\"\nssh = \"local\"\ntargets = [\"macos/aarch64\"]\n\
2914	[app.demo]\nrepo = \"{}\"\n[app.demo.repo_by_host]\nmbp = \"{}\"\n",
2915	repo.display(),
2916	other.display(),
2917	))
2918	.unwrap();
2919	let state = test_state(pool.clone(), topo, cfg);
2920	let err = start_build(
2921	state,
2922	AppId::new("demo"),
2923	Version::parse("0.0.1").unwrap(),
2924	vec![
2925	"linux/x86_64".parse().unwrap(),
2926	"macos/aarch64".parse().unwrap(),
2927	],
2928	)
2929	.await
2930	.unwrap_err();
2931	let msg = format!("{err:#}");
2932	assert!(
2933	msg.contains("does not exist"),
2934	"the refusal should still be mbp's missing tag, got: {msg}"
2935	);
2936	assert_eq!(
2937	current_branch(&repo),
2938	branch_before,
2939	"a refused preflight must reattach the host it already pinned"
2940	);
2941	}
2942
2943	/// A checkout refused by a dirty file OUTSIDE the app names that file, and
2944	/// says it is outside.
2945	///
2946	/// The dirty gate is scoped to the app's directory on purpose — an edit in
2947	/// `server/` is not something pom's build compiles, and refusing pom's
2948	/// release for it is how a gate gets bypassed. But `git checkout <tag>` acts
2949	/// on the whole repository, and MNW is one `.git` over the server, sando,
2950	/// multithreaded and pom. So the gate passes, the checkout fails, and
2951	/// between them they used to say only "uncommitted changes in the checkout?"
2952	/// about a directory that was demonstrably clean.
2953	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2954	async fn a_dirty_file_outside_the_app_is_named_as_such_when_it_blocks_the_checkout() {
2955	let tmp = tempfile::tempdir().unwrap();
2956	let root = tmp.path().join("monorepo");
2957	let app = root.join("pom");
2958	init_git_app(&app, "0.0.1", None);
2959	let git = \|args: &[&str]\| {
2960	let out = std::process::Command::new("git")
2961	.args(args)
2962	.current_dir(&root)
2963	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2964	.env("GIT_CONFIG_SYSTEM", "/dev/null")
2965	.env("GIT_AUTHOR_NAME", "t")
2966	.env("GIT_AUTHOR_EMAIL", "t@t")
2967	.env("GIT_COMMITTER_NAME", "t")
2968	.env("GIT_COMMITTER_EMAIL", "t@t")
2969	.output()
2970	.expect("git runs");
2971	assert!(
2972	out.status.success(),
2973	"git {args:?}: {}",
2974	String::from_utf8_lossy(&out.stderr)
2975	);
2976	};
2977	// `init_git_app` made `pom/` its own repo; the monorepo is the parent, so
2978	// drop that and re-init one `.git` over both products.
2979	std::fs::remove_dir_all(app.join(".git")).unwrap();
2980	std::fs::create_dir_all(root.join("server")).unwrap();
2981	std::fs::write(root.join("server/Cargo.lock"), "version = 4\n").unwrap();
2982	git(&["init", "-q"]);
2983	git(&["add", "-A"]);
2984	git(&["commit", "-q", "-m", "init"]);
2985	git(&["tag", "v0.0.1"]);
2986	// The tag and the branch differ in `server/`, and the working copy of
2987	// that file is modified — so the checkout cannot carry the edit across.
2988	std::fs::write(root.join("server/Cargo.lock"), "version = 4\n# moved on\n").unwrap();
2989	git(&["add", "-A"]);
2990	git(&["commit", "-q", "-m", "server moves"]);
2991	std::fs::write(
2992	root.join("server/Cargo.lock"),
2993	"version = 4\n# local edit\n",
2994	)
2995	.unwrap();
2996
2997	let mut cfg = Config::for_tests(tmp.path());
2998	cfg.pin_release_sha = true;
2999	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3000	let state = test_state(pool.clone(), one_host_topo(&app), cfg);
3001	let err = start_build(
3002	state,
3003	AppId::new("demo"),
3004	Version::parse("0.0.1").unwrap(),
3005	vec!["linux/x86_64".parse().unwrap()],
3006	)
3007	.await
3008	.unwrap_err();
3009	let msg = format!("{err:#}");
3010	assert!(
3011	msg.contains("server/Cargo.lock"),
3012	"the refusal must name the file that blocked the checkout, got: {msg}"
3013	);
3014	assert!(
3015	msg.contains("outside pom/"),
3016	"and must say the file is not this app's, got: {msg}"
3017	);
3018	}
3019
3020	/// The branch `repo` is on, empty on a detached HEAD.
3021	fn current_branch(repo: &std::path::Path) -> String {
3022	let out = std::process::Command::new("git")
3023	.args(["symbolic-ref", "-q", "--short", "HEAD"])
3024	.current_dir(repo)
3025	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3026	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3027	.output()
3028	.expect("git runs");
3029	String::from_utf8_lossy(&out.stdout).trim().to_string()
3030	}
3031
3032	/// An unreachable remote does not fail a release whose tag is present.
3033	///
3034	/// The preflight used to run `fetch --all --tags --prune && checkout`, and
3035	/// `fetch --all` is non-zero if ANY remote fails. Every library repo carries
3036	/// three (`astra`, `mnw`, `srht`), so one dead mirror aborted the release and
3037	/// blamed it on a missing tag — which is how makeover v2.1.1 failed on
3038	/// 2026-07-26 four minutes after its tag was created. Fetch is advisory now;
3039	/// only the checkout decides.
3040	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3041	async fn a_dead_remote_does_not_fail_a_release_whose_tag_exists() {
3042	let tmp = tempfile::tempdir().unwrap();
3043	let repo = tmp.path().join("demo");
3044	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
3045	// A remote that cannot possibly be fetched, standing in for an offline
3046	// astra or an srht mirror the repo was never pushed to.
3047	let out = std::process::Command::new("git")
3048	.args([
3049	"remote",
3050	"add",
3051	"srht",
3052	&tmp.path().join("nowhere.git").display().to_string(),
3053	])
3054	.current_dir(&repo)
3055	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3056	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3057	.output()
3058	.expect("git runs");
3059	assert!(out.status.success());
3060
3061	let mut cfg = Config::for_tests(tmp.path());
3062	cfg.pin_release_sha = true;
3063	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3064	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
3065	let build_id = start_build(
3066	state,
3067	AppId::new("demo"),
3068	Version::parse("0.0.1").unwrap(),
3069	vec!["linux/x86_64".parse().unwrap()],
3070	)
3071	.await
3072	.expect("a dead mirror must not refuse the release");
3073	let (status, error) = await_target(&pool, build_id).await;
3074	assert_eq!(status, "ok", "the tag is present, so this builds ({error})");
3075	}
3076
3077	/// The audit fix: a `build` step dispatched to a host whose executor lacks the
3078	/// `build` capability is denied at the transport BEFORE the command runs. This
3079	/// is the structural guarantee behind "never build on prod" — a recipe naming
3080	/// the wrong host can't compile there.
3081	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3082	async fn build_on_a_host_without_the_build_grant_is_denied() {
3083	let tmp = tempfile::tempdir().unwrap();
3084	let root = tmp.path();
3085
3086	let repo = root.join("app");
3087	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3088	std::fs::write(
3089	repo.join("src-tauri/tauri.conf.json"),
3090	r#"{"version":"0.0.1"}"#,
3091	)
3092	.unwrap();
3093	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
3094	// The recipe (wrongly) tries to compile on `prod`, a host with no build
3095	// grant. A marker file would appear if the command actually ran.
3096	let marker = root.join("ran-on-prod");
3097	std::fs::write(
3098	repo.join("dist/recipes/linux.rhai"),
3099	r#"
3100	step("build");
3101	sh_ok("prod", "touch MARKER");
3102	"#
3103	.replace("MARKER", marker.to_str().unwrap()),
3104	)
3105	.unwrap();
3106
3107	let cfg = Config::for_tests(root);
3108	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3109	// fw13 builds linux; `prod` is local-but-restart-only (no build/package).
3110	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
3111	let topo = Topology::from_str_for_tests(
3112	r#"
3113	[[host]]
3114	name = "fw13"
3115	ssh = "local"
3116	targets = ["linux/x86_64"]
3117
3118	[[host]]
3119	name = "prod"
3120	ssh = "local"
3121	actuate = ["restart"]
3122	observe = []
3123
3124	[app.demo]
3125	repo = "REPO"
3126	"#
3127	.replace("REPO", repo.to_str().unwrap())
3128	.as_str(),
3129	)
3130	.unwrap();
3131
3132	let state = test_state(pool.clone(), topo, cfg);
3133
3134	let build_id = start_build(
3135	state.clone(),
3136	AppId::new("demo"),
3137	Version::parse("0.0.1").unwrap(),
3138	vec!["linux/x86_64".parse().unwrap()],
3139	)
3140	.await
3141	.unwrap();
3142
3143	let mut status = String::new();
3144	let mut error = String::new();
3145	for _ in 0..100 {
3146	let row: Option<(String, Option<String>)> =
3147	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
3148	.bind(build_id)
3149	.fetch_optional(&pool)
3150	.await
3151	.unwrap();
3152	if let Some((s, e)) = row {
3153	status = s;
3154	error = e.unwrap_or_default();
3155	if status != "running" {
3156	break;
3157	}
3158	}
3159	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3160	}
3161
3162	assert_eq!(status, "failed", "build on an ungranted host must fail");
3163	assert!(
3164	error.contains("capability denied") && error.contains("build"),
3165	"failure must be a capability denial, got: {error}"
3166	);
3167	assert!(!marker.exists(), "denied build step must NOT have executed");
3168	}
3169
3170	// ---- macOS sign / notarize / staple execution chain (via ScriptedExec) ----
3171
3172	/// Stand up a single-macOS-target app whose recipe is `recipe_body` (with the
3173	/// literal `ARTIFACT` replaced by a real, non-empty file on disk), dispatch
3174	/// every host command through `scripted`, run the build to a terminal state,
3175	/// and return the pool plus the final `(status, error)`. The returned
3176	/// [`tempfile::TempDir`] must be kept alive by the caller: it holds the
3177	/// sqlite DB the pool reads.
3178	async fn run_macos_recipe(
3179	scripted: Arc<ScriptedExec>,
3180	recipe_body: &str,
3181	backoff_secs: Option<u64>,
3182	) -> (tempfile::TempDir, SqlitePool, String, String) {
3183	let tmp = tempfile::tempdir().unwrap();
3184	let root = tmp.path();
3185	let repo = root.join("app");
3186	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3187	std::fs::write(
3188	repo.join("src-tauri/tauri.conf.json"),
3189	r#"{"version":"0.0.1"}"#,
3190	)
3191	.unwrap();
3192	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
3193
3194	// A real, non-empty artifact so `publish`'s size floor is satisfied; the
3195	// build step is faked, so nothing else creates it.
3196	let artifact = repo.join("out/demo.dmg");
3197	std::fs::create_dir_all(artifact.parent().unwrap()).unwrap();
3198	std::fs::write(&artifact, b"dmg-bytes").unwrap();
3199
3200	std::fs::write(
3201	repo.join("dist/recipes/macos.rhai"),
3202	recipe_body.replace("ARTIFACT", artifact.to_str().unwrap()),
3203	)
3204	.unwrap();
3205
3206	let cfg = Config {
3207	notarize_backoff_secs: backoff_secs,
3208	..Config::for_tests(root)
3209	};
3210	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3211	std::fs::write(repo.join("bento.toml"), "targets = [\"macos/aarch64\"]\n").unwrap();
3212	let topo = Topology::from_str_for_tests(&format!(
3213	r#"
3214	[[host]]
3215	name = "mbp"
3216	ssh = "local"
3217	targets = ["macos/aarch64"]
3218
3219	[app.demo]
3220	repo = "{}"
3221	"#,
3222	repo.display()
3223	))
3224	.unwrap();
3225
3226	let mut state = test_state(pool.clone(), topo, cfg);
3227	// Route every host command through the scripted executor.
3228	let mut execs = HashMap::new();
3229	execs.insert("mbp".to_string(), scripted as Arc<dyn Executor>);
3230	state.executors = Arc::new(execs);
3231
3232	let build_id = start_build(
3233	state.clone(),
3234	AppId::new("demo"),
3235	Version::parse("0.0.1").unwrap(),
3236	vec!["macos/aarch64".parse().unwrap()],
3237	)
3238	.await
3239	.unwrap();
3240
3241	let mut status = String::new();
3242	let mut error = String::new();
3243	for _ in 0..100 {
3244	let row: Option<(String, Option<String>)> =
3245	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
3246	.bind(build_id)
3247	.fetch_optional(&pool)
3248	.await
3249	.unwrap();
3250	if let Some((s, e)) = row {
3251	status = s;
3252	error = e.unwrap_or_default();
3253	if status != "running" {
3254	break;
3255	}
3256	}
3257	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3258	}
3259	(tmp, pool, status, error)
3260	}
3261
3262	async fn release_count(pool: &SqlitePool) -> i64 {
3263	sqlx::query_scalar("SELECT COUNT(*) FROM releases")
3264	.fetch_one(pool)
3265	.await
3266	.unwrap()
3267	}
3268
3269	/// The whole macOS release chain end to end through a fake host: codesign,
3270	/// notarize (Accepted first try), staple, verify_gatekeeper, then publish.
3271	/// The recorded commands lock the exact incantations each host function
3272	/// dispatches, and a `releases` row proves the publish gate opened for a
3273	/// signed + notarized + Gatekeeper-accepted artifact.
3274	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3275	async fn a_macos_recipe_signs_notarizes_staples_verifies_and_publishes() {
3276	let scripted = Arc::new(
3277	ScriptedExec::new()
3278	.on("codesign", 0, "")
3279	.on("notarytool", 0, r#"{"status":"Accepted"}"#)
3280	.on("stapler staple", 0, "")
3281	.on(
3282	"spctl",
3283	0,
3284	"source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
3285	),
3286	);
3287	let recipe = r#"
3288	let h = build_host();
3289	step("build");
3290	sh_ok(h, "echo built");
3291	step("sign");
3292	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3293	notarize(h, "ARTIFACT");
3294	staple(h, "ARTIFACT");
3295	step("verify");
3296	if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
3297	step("publish");
3298	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3299	"#;
3300	let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, None).await;
3301
3302	assert_eq!(
3303	status, "ok",
3304	"signed+notarized macOS build should publish: {error}"
3305	);
3306	assert_eq!(
3307	release_count(&pool).await,
3308	1,
3309	"publish must record a release"
3310	);
3311
3312	// The exact shell incantations each host function dispatched.
3313	let cmds = scripted.commands();
3314	let has = \|needle: &str\| cmds.iter().any(\|c\| c.contains(needle));
3315	assert!(
3316	has("codesign --force --options runtime --timestamp --sign"),
3317	"codesign runtime+timestamp incantation, got: {cmds:?}"
3318	);
3319	assert!(
3320	has("xcrun notarytool submit"),
3321	"notarytool submit: {cmds:?}"
3322	);
3323	assert!(
3324	has("--wait --output-format json"),
3325	"notarytool --wait json: {cmds:?}"
3326	);
3327	assert!(has("xcrun stapler staple"), "stapler staple: {cmds:?}");
3328	assert!(has("spctl --assess"), "gatekeeper assess: {cmds:?}");
3329	}
3330
3331	/// A failing `codesign` fails the sign step and aborts the recipe before
3332	/// publish — an unsigned artifact never ships.
3333	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3334	async fn a_codesign_failure_fails_the_sign_step_and_blocks_publish() {
3335	let scripted = Arc::new(ScriptedExec::new().on("codesign", 1, ""));
3336	let recipe = r#"
3337	let h = build_host();
3338	step("build");
3339	sh_ok(h, "echo built");
3340	step("sign");
3341	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3342	step("publish");
3343	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3344	"#;
3345	let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
3346
3347	assert_eq!(status, "failed", "a failed codesign must fail the target");
3348	assert!(
3349	error.contains("codesign failed"),
3350	"error names the codesign failure, got: {error}"
3351	);
3352	assert_eq!(
3353	release_count(&pool).await,
3354	0,
3355	"nothing may publish after a codesign failure"
3356	);
3357	}
3358
3359	/// Even if the recipe ignores `verify_gatekeeper`'s returned `false`, the
3360	/// publish gate refuses the artifact: `verify_gatekeeper` both records the
3361	/// rejection and fails its step, and `publish` proves neither passed. This
3362	/// is the defense-in-depth the pure `PublishAuthority::prove` tests assert in
3363	/// isolation, here exercised through the real host-function path.
3364	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3365	async fn a_gatekeeper_rejection_bars_publish_even_if_the_recipe_ignores_it() {
3366	let scripted = Arc::new(
3367	ScriptedExec::new()
3368	.on("codesign", 0, "")
3369	.on("notarytool", 0, r#"{"status":"Accepted"}"#)
3370	.on("stapler staple", 0, "")
3371	// Gatekeeper says no: the sentinel is FAIL, not OK.
3372	.on("spctl", 0, "source=Unnotarized\nBENTO_GATEKEEPER_FAIL"),
3373	);
3374	let recipe = r#"
3375	let h = build_host();
3376	step("build");
3377	sh_ok(h, "echo built");
3378	step("sign");
3379	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3380	notarize(h, "ARTIFACT");
3381	staple(h, "ARTIFACT");
3382	step("verify");
3383	verify_gatekeeper(h, "ARTIFACT");
3384	step("publish");
3385	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3386	"#;
3387	let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
3388
3389	assert_eq!(
3390	status, "failed",
3391	"a Gatekeeper-rejected artifact must not publish"
3392	);
3393	assert!(
3394	!error.is_empty(),
3395	"the barred publish must surface an error"
3396	);
3397	assert_eq!(
3398	release_count(&pool).await,
3399	0,
3400	"no release for a rejected artifact"
3401	);
3402	}
3403
3404	/// The one flaky, network-bound step: `notarize` retries a non-`Accepted`
3405	/// result and succeeds on a later attempt. Two notarytool calls (reject then
3406	/// accept) then a recorded release prove the retry ran and the chain
3407	/// completed. Backoff is 0 so the retry sleep doesn't stall the test.
3408	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3409	async fn notarize_retries_a_non_accepted_result_then_succeeds() {
3410	let scripted = Arc::new(
3411	ScriptedExec::new()
3412	.on("codesign", 0, "")
3413	.on_seq(
3414	"notarytool",
3415	&[
3416	(0, r#"{"status":"In Progress"}"#),
3417	(0, r#"{"status":"Accepted"}"#),
3418	],
3419	)
3420	.on("stapler staple", 0, "")
3421	.on(
3422	"spctl",
3423	0,
3424	"source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
3425	),
3426	);
3427	let recipe = r#"
3428	let h = build_host();
3429	step("build");
3430	sh_ok(h, "echo built");
3431	step("sign");
3432	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3433	notarize(h, "ARTIFACT");
3434	staple(h, "ARTIFACT");
3435	step("verify");
3436	if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
3437	step("publish");
3438	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3439	"#;
3440	let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
3441
3442	assert_eq!(
3443	status, "ok",
3444	"notarize should succeed on the retry: {error}"
3445	);
3446	assert_eq!(
3447	release_count(&pool).await,
3448	1,
3449	"the retried build still publishes"
3450	);
3451	let notary_calls = scripted
3452	.commands()
3453	.iter()
3454	.filter(\|c\| c.contains("notarytool"))
3455	.count();
3456	assert_eq!(
3457	notary_calls, 2,
3458	"notarytool ran once, was rejected, then ran again"
3459	);
3460	}
3461
3462	/// `notarize` gives up after its bounded retries: three notarytool attempts,
3463	/// all non-`Accepted`, fail the target and bar publish.
3464	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3465	async fn notarize_fails_the_target_after_exhausting_its_retries() {
3466	let scripted = Arc::new(
3467	ScriptedExec::new()
3468	.on("codesign", 0, "")
3469	// Every attempt reports a still-pending status, never Accepted.
3470	.on("notarytool", 0, r#"{"status":"In Progress"}"#),
3471	);
3472	let recipe = r#"
3473	let h = build_host();
3474	step("build");
3475	sh_ok(h, "echo built");
3476	step("sign");
3477	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3478	notarize(h, "ARTIFACT");
3479	step("publish");
3480	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3481	"#;
3482	let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
3483
3484	assert_eq!(
3485	status, "failed",
3486	"exhausted notarization must fail the target"
3487	);
3488	assert!(
3489	error.contains("notarization failed after 3 attempts"),
3490	"error names the exhausted retry, got: {error}"
3491	);
3492	assert_eq!(
3493	release_count(&pool).await,
3494	0,
3495	"an unnotarized artifact never publishes"
3496	);
3497	let notary_calls = scripted
3498	.commands()
3499	.iter()
3500	.filter(\|c\| c.contains("notarytool"))
3501	.count();
3502	assert_eq!(notary_calls, 3, "the retry is bounded at three attempts");
3503	}
3504	// ---- item (5): SSH and Agent transports end to end, via a recording fake ----
3505	//
3506	// Every other test declares `ssh = "local"`, so the runner's two-plane
3507	// routing is only exercised at construction
3508	// (state::agent_host_syncs_over_ssh_never_the_agent): steps run over the
3509	// EXEC transport (state.executors -- SshExec, or the in-session AgentRpc for
3510	// a mac host) while artifacts move over the SYNC transport (state.syncs --
3511	// always ssh, NEVER the agent, whose confined `/pull` would 404 or force
3512	// `pull_root` wide enough to expose `~/.tauri/passwords.env`). These drive a
3513	// real recipe through `start_build` on a NON-local topology, replace the
3514	// real ssh/agent transports with a recording fake, and assert which plane
3515	// handled which operation -- the runtime form of that construction-time
3516	// invariant, on the ssh string every other test pins to "local".
3517
3518	/// Records shell commands and artifact pulls on separate logs, so a test can
3519	/// prove the exec transport built/signed and the sync transport collected --
3520	/// and that neither did the other's job -- without a live host or ssh.
3521	struct RecordingExec {
3522	caps: CapabilitySet,
3523	commands: Arc<std::sync::Mutex<Vec<String>>>,
3524	pulls: Arc<std::sync::Mutex<Vec<String>>>,
3525	}
3526
3527	impl RecordingExec {
3528	fn new() -> Arc<Self> {
3529	Arc::new(Self {
3530	// A mac build host's real grant. Nothing in this fake gates on it
3531	// (the real transports do), but keep it coherent so
3532	// `capabilities()` is not a lie.
3533	caps: CapabilitySet::from_tokens(
3534	["build", "sign", "notarize", "staple"],
3535	["build-log", "artifact"],
3536	),
3537	commands: Arc::new(std::sync::Mutex::new(Vec::new())),
3538	pulls: Arc::new(std::sync::Mutex::new(Vec::new())),
3539	})
3540	}
3541	fn commands(&self) -> Vec<String> {
3542	self.commands.lock().unwrap().clone()
3543	}
3544	fn pulls(&self) -> Vec<String> {
3545	self.pulls.lock().unwrap().clone()
3546	}
3547	}
3548
3549	#[async_trait]
3550	impl Executor for RecordingExec {
3551	async fn run_streaming(
3552	&self,
3553	step: &ops_exec::Step,
3554	_sink: &mut dyn LogSink,
3555	) -> anyhow::Result<RunOutput> {
3556	self.commands
3557	.lock()
3558	.unwrap()
3559	.push(step.argv.last().cloned().unwrap_or_default());
3560	Ok(RunOutput {
3561	status: std::process::ExitStatus::from_raw(0),
3562	stdout: Vec::new(),
3563	stderr: Vec::new(),
3564	})
3565	}
3566	async fn pull_file(
3567	&self,
3568	r: &std::path::Path,
3569	_l: &std::path::Path,
3570	_o: &SyncOpts,
3571	) -> anyhow::Result<()> {
3572	self.pulls
3573	.lock()
3574	.unwrap()
3575	.push(r.to_string_lossy().into_owned());
3576	Ok(())
3577	}
3578	async fn pull_dir(
3579	&self,
3580	r: &std::path::Path,
3581	_l: &std::path::Path,
3582	_o: &SyncOpts,
3583	) -> anyhow::Result<()> {
3584	self.pulls
3585	.lock()
3586	.unwrap()
3587	.push(r.to_string_lossy().into_owned());
3588	Ok(())
3589	}
3590	async fn pull_glob(
3591	&self,
3592	g: &str,
3593	_l: &std::path::Path,
3594	_o: &SyncOpts,
3595	) -> anyhow::Result<()> {
3596	self.pulls.lock().unwrap().push(g.to_string());
3597	Ok(())
3598	}
3599	async fn push_dir(
3600	&self,
3601	_l: &std::path::Path,
3602	_r: &std::path::Path,
3603	_o: &SyncOpts,
3604	) -> anyhow::Result<()> {
3605	Ok(())
3606	}
3607	async fn preflight(&self) -> anyhow::Result<()> {
3608	Ok(())
3609	}
3610	fn capabilities(&self) -> &CapabilitySet {
3611	&self.caps
3612	}
3613	}
3614
3615	/// Stand up a single-target app (host named `h1`) whose recipe is
3616	/// `recipe_body` (with `REPO` replaced by the checkout path), inject
3617	/// `exec_fake` as the host's EXEC transport and `sync_fake` as its SYNC
3618	/// transport, run the build to a terminal state, and return the tmpdir plus
3619	/// `(status, error)`. Unlike `test_state`'s `build_executors`, this replaces
3620	/// BOTH planes so no real ssh/agent transport is dialed. The returned
3621	/// [`tempfile::TempDir`] holds the sqlite DB and must outlive the caller's
3622	/// assertions.
3623	async fn run_two_plane(
3624	host_toml: &str,
3625	target: &str,
3626	recipe_file: &str,
3627	recipe_body: &str,
3628	exec_fake: Arc<dyn Executor>,
3629	sync_fake: Arc<dyn Executor>,
3630	) -> (tempfile::TempDir, String, String) {
3631	let tmp = tempfile::tempdir().unwrap();
3632	let root = tmp.path();
3633	let repo = root.join("app");
3634	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3635	std::fs::write(
3636	repo.join("src-tauri/tauri.conf.json"),
3637	r#"{"version":"0.0.1"}"#,
3638	)
3639	.unwrap();
3640	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
3641	std::fs::write(
3642	repo.join("dist/recipes").join(recipe_file),
3643	recipe_body.replace("REPO", repo.to_str().unwrap()),
3644	)
3645	.unwrap();
3646	std::fs::write(
3647	repo.join("bento.toml"),
3648	format!("targets = [\"{target}\"]\n"),
3649	)
3650	.unwrap();
3651
3652	let cfg = Config::for_tests(root);
3653	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3654	let topo = Topology::from_str_for_tests(&format!(
3655	"{}\n[app.demo]\nrepo = \"{}\"\n",
3656	host_toml.replace("REPO", repo.to_str().unwrap()),
3657	repo.display()
3658	))
3659	.unwrap();
3660
3661	let mut state = test_state(pool.clone(), topo, cfg);
3662	state.executors = Arc::new(HashMap::from([("h1".to_string(), exec_fake)]));
3663	state.syncs = Arc::new(HashMap::from([("h1".to_string(), sync_fake)]));
3664
3665	let build_id = start_build(
3666	state.clone(),
3667	AppId::new("demo"),
3668	Version::parse("0.0.1").unwrap(),
3669	vec![target.parse().unwrap()],
3670	)
3671	.await
3672	.unwrap();
3673
3674	let mut status = String::new();
3675	let mut error = String::new();
3676	for _ in 0..100 {
3677	let row: Option<(String, Option<String>)> =
3678	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
3679	.bind(build_id)
3680	.fetch_optional(&pool)
3681	.await
3682	.unwrap();
3683	if let Some((s, e)) = row {
3684	status = s;
3685	error = e.unwrap_or_default();
3686	if status != "running" {
3687	break;
3688	}
3689	}
3690	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3691	}
3692	(tmp, status, error)
3693	}
3694
3695	/// An agent (macOS) host signs over the AGENT transport but is collected
3696	/// from over SSH -- driven through the whole runner, not just `build_sync`.
3697	/// The sign chain's commands land on the exec plane and the artifact pull on
3698	/// the sync plane; crucially, the agent plane is asked to move NOTHING (a
3699	/// regression to one transport would route collect at `AgentRpc::pull_glob`,
3700	/// refused by design, or widen `pull_root` over the secret-bearing home dir).
3701	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3702	async fn an_agent_host_signs_over_the_agent_and_collects_over_ssh_end_to_end() {
3703	let agent = RecordingExec::new(); // exec plane (AgentRpc in prod)
3704	let ssh = RecordingExec::new(); // sync plane (SshExec in prod)
3705	let host = r#"
3706	[[host]]
3707	name = "h1"
3708	ssh = "mbp"
3709	targets = ["macos/aarch64"]
3710	transport = "agent"
3711	agent_url = "http://mbp:8765"
3712	actuate = ["build", "sign", "notarize", "staple"]
3713	observe = ["build-log", "gatekeeper", "artifact"]
3714	pull_root = "REPO"
3715	"#;
3716	let recipe = r#"
3717	let h = build_host();
3718	step("build");
3719	sh_ok(h, "echo built");
3720	step("sign");
3721	codesign(h, "Developer ID Application: Test", "REPO/out/demo.dmg");
3722	step("collect");
3723	collect(h, "REPO/out/*.dmg", "demo", "0.0.1");
3724	"#;
3725	let (_tmp, status, error) = run_two_plane(
3726	host,
3727	"macos/aarch64",
3728	"macos.rhai",
3729	recipe,
3730	agent.clone(),
3731	ssh.clone(),
3732	)
3733	.await;
3734	assert_eq!(status, "ok", "the recipe should complete: {error}");
3735
3736	// The build + sign commands ran on the AGENT (exec) transport.
3737	let agent_cmds = agent.commands();
3738	assert!(
3739	agent_cmds.iter().any(\|c\| c.contains("codesign")),
3740	"codesign rides the agent exec transport: {agent_cmds:?}"
3741	);
3742	assert!(
3743	agent_cmds.iter().any(\|c\| c.contains("echo built")),
3744	"the build step rides the agent exec transport: {agent_cmds:?}"
3745	);
3746	// ...and the agent moved NO artifacts. This is the load-bearing half:
3747	// AgentRpc::pull_glob is refused by design, so collect must not touch it.
3748	assert!(
3749	agent.pulls().is_empty(),
3750	"the agent transport must never collect artifacts: {:?}",
3751	agent.pulls()
3752	);
3753
3754	// The artifact was collected over the SSH (sync) transport...
3755	let ssh_pulls = ssh.pulls();
3756	assert!(
3757	ssh_pulls
3758	.iter()
3759	.any(\|p\| p.contains("demo.dmg") \|\| p.contains("*.dmg")),
3760	"collect rides the ssh sync transport: {ssh_pulls:?}"
3761	);
3762	// ...and the sync transport was never asked to run a build/sign command.
3763	assert!(
3764	ssh.commands().is_empty(),
3765	"the sync transport must never run host commands: {:?}",
3766	ssh.commands()
3767	);
3768	}
3769
3770	/// A plain (non-agent) host whose `ssh` is a remote alias, not "local" --
3771	/// the case every other test avoids. The recipe runs end to end through the
3772	/// fake, proving the runner drives a non-local host and still splits exec
3773	/// (build) from sync (collect) across the two transport maps.
3774	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3775	async fn a_non_local_ssh_host_runs_a_recipe_end_to_end() {
3776	let exec = RecordingExec::new();
3777	let sync = RecordingExec::new();
3778	let host = r#"
3779	[[host]]
3780	name = "h1"
3781	ssh = "astra"
3782	targets = ["linux/x86_64"]
3783	pull_root = "REPO"
3784	"#;
3785	let recipe = r#"
3786	let h = build_host();
3787	step("build");
3788	sh_ok(h, "echo compiling");
3789	step("collect");
3790	collect(h, "REPO/out/demo.bin", "demo", "0.0.1");
3791	"#;
3792	let (_tmp, status, error) = run_two_plane(
3793	host,
3794	"linux/x86_64",
3795	"linux.rhai",
3796	recipe,
3797	exec.clone(),
3798	sync.clone(),
3799	)
3800	.await;
3801	assert_eq!(status, "ok", "the recipe should complete: {error}");
3802
3803	assert!(
3804	exec.commands().iter().any(\|c\| c.contains("echo compiling")),
3805	"the build command rides the exec transport: {:?}",
3806	exec.commands()
3807	);
3808	assert!(
3809	exec.pulls().is_empty(),
3810	"the exec transport must not collect: {:?}",
3811	exec.pulls()
3812	);
3813	assert!(
3814	sync.pulls().iter().any(\|p\| p.contains("demo.bin")),
3815	"collect rides the sync transport: {:?}",
3816	sync.pulls()
3817	);
3818	assert!(
3819	sync.commands().is_empty(),
3820	"the sync transport must not run commands: {:?}",
3821	sync.commands()
3822	);
3823	}
3824	}
3825
3826	/// Every recipe of every configured app must parse.
3827	///
3828	/// A recipe is read and compiled at release time, on the build host, after the
3829	/// checkout has already run — so a typo in one is discovered at the worst
3830	/// possible moment. Compiling them here costs nothing and moves that discovery
3831	/// to `cargo test`. Skips when the live config is absent (CI, another host),
3832	/// like [`crate::topology`]'s live-config smoke test.
3833	#[cfg(test)]
3834	mod live_recipe_smoke {
3835	use crate::topology::Topology;
3836	use std::path::{Path, PathBuf};
3837
3838	#[test]
3839	fn live_recipes_compile_if_present() {
3840	let Some(home) = std::env::var_os("HOME") else {
3841	return;
3842	};
3843	let path = Path::new(&home).join(".config/bento/bento.toml");
3844	if !path.exists() {
3845	return;
3846	}
3847	let topo = Topology::load(&path).expect("live bento.toml must load");
3848	// Syntax only: the host functions are bound per run against a live
3849	// context, and Rhai resolves calls at eval time regardless.
3850	let engine = rhai::Engine::new();
3851	let mut checked = 0;
3852	for (name, cfg) in &topo.app {
3853	let dir = crate::engine::expand_tilde(&cfg.repo).join(&cfg.recipe_dir);
3854	// Every recipe in the directory, not just the ones the app's current
3855	// targets name. A recipe for a target that is temporarily not
3856	// shipped (windows, dropped from the manifests until its host is
3857	// real) still has to parse, and it is the one nothing else is
3858	// watching.
3859	let Ok(entries) = std::fs::read_dir(&dir) else {
3860	continue;
3861	};
3862	let mut files: Vec<PathBuf> = entries
3863	.filter_map(Result::ok)
3864	.map(\|e\| e.path())
3865	.filter(\|p\| p.extension().is_some_and(\|x\| x == "rhai"))
3866	.collect();
3867	files.sort();
3868	for p in files {
3869	let file = p.file_name().unwrap_or_default().to_string_lossy();
3870	let Ok(src) = std::fs::read_to_string(&p) else {
3871	continue;
3872	};
3873	engine
3874	.compile(&src)
3875	.unwrap_or_else(\|e\| panic!("recipe {name}/{file} does not parse: {e}"));
3876	checked += 1;
3877	}
3878	}
3879	assert!(checked > 0, "live config resolved no readable recipes");
3880	}
3881	}
3882