max / makenotwork

161.3 KB · 4195 lines History Blame Raw

1	//! Build orchestration: fan a `(app, version)` out across its targets, each
2	//! running its recipe concurrently on the host that can build it.
3	//!
4	//! A build inserts one `builds` row, then spawns one task per target. Each
5	//! target task registers itself in the single-slot guard (a newer build for
6	//! the same `(app, target)` aborts the in-flight one — latest wins, but other
7	//! targets keep running, which is the fan-out), then runs the recipe.
8
9	use crate::domain::{AppId, Status, Step, Target, Version};
10	use crate::engine::{self, RecipeCtx};
11	use crate::events::{self, Event};
12	use crate::state::AppState;
13	use anyhow::{Context, Result};
14	use ops_exec::{Action, LogSink, Step as OpStep};
15	use std::path::PathBuf;
16	use std::sync::Arc;
17
18	/// A [`LogSink`] that drops what it's handed — the release preflight runs git on
19	/// each host for its exit code and (via a separate `rev-parse`) the sha in
20	/// `RunOutput`, not for a streamed log, so there is no step to stream into.
21	struct DiscardSink;
22
23	#[async_trait::async_trait]
24	impl LogSink for DiscardSink {
25	async fn write_chunk(&mut self, _bytes: &[u8]) {}
26	}
27
28	/// Preflight barrier: put every host that will build a target for this release
29	/// into a worktree detached at the tag `v<version>`, and refuse the build unless
30	/// they all report the SAME commit. Recipes used to `git pull --ff-only` per
31	/// host, so `mbp`/`astra`/`fw13` each built whatever `main` was at pull time and
32	/// the artifacts were filed under the daemon host's version. This runs before
33	/// any target task spawns, so a mixed-source release is stopped before a single
34	/// artifact is built.
35	///
36	/// "All" means all: a host that did not report is a refusal, not an abstention.
37	/// Comparing only the hosts that answered proves agreement among those, which is
38	/// not the claim the barrier is making.
39	///
40	/// The agreed sha is the release's provenance, and this is the only place it is
41	/// known to be one value rather than a per-host answer. Resolving it here and
42	/// carrying it forward is what lets an artifact record name the source it was
43	/// built from without asking a build host to be honest about it afterwards.
44	///
45	/// ## The tree a release builds in is Bento's, not the operator's
46	///
47	/// This used to check the tag out in the ordinary checkout and put that tree
48	/// back on its branch afterwards. Three things came out of that, and all three
49	/// are gone with it (ruled 2026-08-24, task `6b808f26`):
50	///
51	/// - A release pinned a tree somebody works in. Editing in Helix while one ran
52	/// meant editing the tag's content for the length of the build.
53	/// - The build dirtied that tree (`cargo` rewrites `Cargo.lock` under the
54	/// `[patch]` block), so the restore's `git checkout <branch>` failed and left
55	/// the tree detached at the tag — and the next release was refused for a
56	/// detached HEAD, or for an uncommitted change nobody made.
57	/// - The preflight therefore had to refuse a dirty or already-detached checkout,
58	/// which is a release blocked by the state of a tree it no longer reads.
59	///
60	/// A worktree costs the working files only — it shares the repository's object
61	/// store — and is re-pinned with `--force`, which is safe precisely because
62	/// nothing but Bento writes there. Where it lives is [`Host::worktree_root`],
63	/// and that doc carries why the path has to sit under `~/Code`.
64	///
65	/// Returns where each host will build, for the recipes to be pointed at, and the
66	/// commit they all agreed on. Nothing to unwind on a refusal: a host prepared
67	/// before the refusal is left holding a worktree at a tag, which is what the
68	/// next release would put it at anyway.
69	async fn pin_release(
70	state: &AppState,
71	app: &AppId,
72	version: &Version,
73	targets: &[Target],
74	) -> Result<Pinned> {
75	let cfg = state
76	.topo
77	.app(app)
78	.ok_or_else(\|\| anyhow::anyhow!("unknown app `{app}`"))?;
79	// Spelled per app: a repo holding one product tags `v0.4.1`, a repo holding
80	// several tags `pom-v0.4.1`. See `AppConfig::tag_format`.
81	let tag = cfg.tag_for(version);
82
83	// The distinct hosts across the requested targets (order-stable).
84	//
85	// A target whose host the topology cannot resolve is refused rather than
86	// skipped. Skipping it would leave that target out of the barrier while it
87	// still builds, which is the shape of hole this whole preflight exists to
88	// close: agreement proven among some hosts reads as agreement among all.
89	// `resolve_targets` rejects an unbuildable target before `/build`, so this
90	// is unreachable through the API and cheap to state anyway.
91	let mut hosts: Vec<&crate::topology::Host> = Vec::new();
92	for t in targets {
93	let h = state
94	.topo
95	.host_for(*t)
96	.ok_or_else(\|\| anyhow::anyhow!("no host can build {t}"))?;
97	if !hosts.iter().any(\|seen\| seen.name == h.name) {
98	hosts.push(h);
99	}
100	}
101	anyhow::ensure!(
102	!hosts.is_empty(),
103	"no build hosts for v{version}; refusing to build a release nothing was pinned for"
104	);
105
106	// What each host IS, before what each host is ON. A build host that was
107	// rebuilt into something else is refused here rather than producing an
108	// artifact whose libc floor nobody expected. Ordered before the worktree
109	// work because it is one cheap round-trip and because a machine that is not
110	// what the topology says it is should not be asked to check anything out.
111	for host in &hosts {
112	check_host_identity(state, host).await?;
113	}
114
115	let mut prepared: Vec<PinnedHost> = Vec::new();
116	for host in &hosts {
117	prepared.push(prepare_worktree(state, app, cfg, host, &tag).await?);
118	}
119
120	// The barrier: every participating host must have reported, and every report
121	// must be the same commit. Reporting is checked first and separately from
122	// agreement, because a barrier that compares only what it received cannot
123	// tell unanimity from silence — three hosts agreeing while a fourth was
124	// never asked is exactly the mixed-source release this refuses.
125	let missing: Vec<&str> = hosts
126	.iter()
127	.filter(\|h\| !prepared.iter().any(\|p\| p.host == h.name))
128	.map(\|h\| h.name.as_str())
129	.collect();
130	anyhow::ensure!(
131	missing.is_empty(),
132	"{} did not report a commit for v{version}; refusing to build a release the \
133	barrier cannot vouch for",
134	missing.join(", "),
135	);
136	let empty_shas: Vec<&str> = prepared
137	.iter()
138	.filter(\|p\| p.sha.is_empty())
139	.map(\|p\| p.host.as_str())
140	.collect();
141	anyhow::ensure!(
142	empty_shas.is_empty(),
143	"`git rev-parse HEAD` returned nothing on {} for v{version}",
144	empty_shas.join(", "),
145	);
146
147	let first = &prepared[0];
148	let mismatch: Vec<String> = prepared
149	.iter()
150	.filter(\|p\| p.sha != first.sha)
151	.map(\|p\| format!("{}={}", p.host, short(&p.sha)))
152	.collect();
153	anyhow::ensure!(
154	mismatch.is_empty(),
155	"build hosts are on different commits for v{version} \
156	({}={}, {}); refusing to build a release from mixed sources",
157	first.host,
158	short(&first.sha),
159	mismatch.join(", "),
160	);
161
162	let sha = first.sha.clone();
163	Ok(Pinned {
164	build_dirs: prepared
165	.into_iter()
166	.map(\|p\| (p.host, p.build_dir))
167	.collect(),
168	sha,
169	})
170	}
171
172	/// Confirm a build host is what the topology declares it to be.
173	///
174	/// The sha barrier below proves every host is on the same source. This proves
175	/// every host is the machine the topology thinks it is, which the sha barrier
176	/// cannot see: three hosts can agree on a commit while one of them was rebuilt
177	/// last week onto a base whose libc is two versions ahead, and the only symptom
178	/// is a binary that will not start on the box it ships to.
179	///
180	/// A host that declares neither field is skipped and logged as skipped. The
181	/// macOS and Windows hosts are legitimately in that state, so a refusal would
182	/// take those pipelines down to buy nothing.
183	async fn check_host_identity(state: &AppState, host: &crate::topology::Host) -> Result<()> {
184	if host.base_image.is_none() && host.libc.is_none() {
185	tracing::info!(host = %host.name, "preflight: host declares no base image; not checked");
186	return Ok(());
187	}
188	let exec = state
189	.executors
190	.get(&host.name)
191	.ok_or_else(\|\| anyhow::anyhow!("no executor for host `{}`", host.name))?;
192	let step = OpStep::shell(Action::Build, ops_core::base_image::probe_cmd());
193	let mut sink = DiscardSink;
194	let out = exec
195	.run_streaming(&step, &mut sink)
196	.await
197	.with_context(\|\| format!("asking `{}` what it is", host.name))?;
198	anyhow::ensure!(
199	out.status.success(),
200	"`{}` could not report its base image: {}",
201	host.name,
202	String::from_utf8_lossy(&out.stderr).trim(),
203	);
204	let reported = ops_core::base_image::parse_probe(&String::from_utf8_lossy(&out.stdout));
205	let checked = ops_core::base_image::check(
206	&host.name,
207	host.base_image.as_ref(),
208	host.libc.as_deref(),
209	&reported,
210	)
211	.context("a build host is not what the topology declares it to be")?;
212	if let Some(what) = checked {
213	tracing::info!(host = %host.name, "preflight: identity checked, {what}");
214	}
215	Ok(())
216	}
217
218	/// Put one host's worktree at the release tag and report the commit it landed
219	/// on, creating the worktree the first time this app is released there.
220	///
221	/// The worktree's own path is derived on the host rather than assumed: a repo
222	/// holding several products is one `.git` over all of them, so `repo` for pom is
223	/// `~/Code/MNW/pom` and the tree to make a worktree of is `~/Code/MNW`. One
224	/// `rev-parse` answers both halves — which repository, and where the app sits
225	/// inside it.
226	async fn prepare_worktree(
227	state: &AppState,
228	app: &AppId,
229	cfg: &crate::topology::AppConfig,
230	host: &crate::topology::Host,
231	tag: &str,
232	) -> Result<PinnedHost> {
233	let exec = state
234	.executors
235	.get(&host.name)
236	.ok_or_else(\|\| anyhow::anyhow!("no executor for host `{}`", host.name))?;
237	let name = &host.name;
238	// Resolved per host, not once for the set: the checkouts need not be at the
239	// same path on every machine, and pinning windows-x86 with the unix path
240	// failed on the first git command of the release.
241	let repo = cfg.repo_for(name);
242
243	let read = \|cmd: String\| async {
244	let step = OpStep::shell(Action::Build, cmd);
245	let mut sink = DiscardSink;
246	exec.run_streaming(&step, &mut sink).await
247	};
248
249	let out = read(engine::git_toplevel_and_prefix_cmd(repo))
250	.await
251	.with_context(\|\| format!("reading `{repo}` on `{name}`"))?;
252	let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();
253	anyhow::ensure!(
254	out.status.success(),
255	"`{repo}` on `{name}` is not a git checkout: {stderr}"
256	);
257	let (toplevel, prefix) = engine::parse_toplevel_and_prefix(&String::from_utf8_lossy(
258	&out.stdout,
259	))
260	.ok_or_else(\|\| anyhow::anyhow!("could not read where `{repo}` is checked out on `{name}`"))?;
261
262	let worktree = host
263	.worktree_for(engine::repo_dir_name(&toplevel), app)
264	.ok_or_else(\|\| {
265	anyhow::anyhow!(
266	"host `{name}` sets no worktree_root, so there is nowhere to build \
267	tagged source"
268	)
269	})?;
270
271	// Advisory, and deliberately run against the ordinary checkout: a worktree
272	// shares its repository's object store, so a tag fetched here is a tag the
273	// worktree can be put at. `fetch --all` exits non-zero if ANY remote fails
274	// and the library repos carry three, so a dead mirror must not decide a
275	// release. Only the checkout below is allowed to fail.
276	let _ = read(engine::git_fetch_cmd(repo)).await;
277
278	let exists = read(engine::git_worktree_probe_cmd(&worktree))
279	.await
280	.is_ok_and(\|o\| o.status.success());
281	let pin = if exists {
282	engine::git_worktree_pin_cmd(&worktree, tag)
283	} else {
284	// Prune first: a worktree directory somebody deleted by hand is still
285	// registered in the repository, and `worktree add` then refuses the path
286	// as already in use rather than rebuilding it.
287	let _ = read(engine::git_worktree_prune_cmd(&toplevel)).await;
288	engine::git_worktree_add_cmd(&toplevel, &worktree, tag)
289	};
290	let out = read(pin)
291	.await
292	.with_context(\|\| format!("pinning the worktree on `{name}`"))?;
293	if !out.status.success() {
294	// Ask git why before telling the operator. The usual answer is a tag
295	// that was never created or never pushed, which is a different job from
296	// anything about the worktree itself.
297	let tag_exists = read(engine::git_tag_exists_cmd(repo, tag))
298	.await
299	.is_ok_and(\|o\| o.status.success());
300	anyhow::bail!(
301	"could not put `{worktree}` on `{name}` at {tag}: {}",
302	engine::worktree_failure_reason(tag, tag_exists, &String::from_utf8_lossy(&out.stderr))
303	);
304	}
305
306	let out = read(engine::git_rev_parse_cmd(&worktree))
307	.await
308	.with_context(\|\| format!("rev-parse on `{name}`"))?;
309	anyhow::ensure!(
310	out.status.success(),
311	"`git rev-parse HEAD` failed in `{worktree}` on `{name}`"
312	);
313
314	Ok(PinnedHost {
315	host: name.clone(),
316	build_dir: engine::app_dir_in_worktree(&worktree, &prefix),
317	sha: String::from_utf8_lossy(&out.stdout).trim().to_string(),
318	})
319	}
320
321	/// What the preflight established: where each host builds this release, and the
322	/// one commit every one of those trees is on.
323	struct Pinned {
324	/// `(host, directory the recipe builds in)`. Becomes the run's
325	/// `repo_by_host`, so `repo()` in a recipe is the worktree and no recipe
326	/// knows this happened.
327	build_dirs: Vec<(String, String)>,
328	/// The commit all hosts agreed on. Empty when pinning is off (tests).
329	sha: String,
330	}
331
332	/// One host's prepared worktree, before the barrier has compared them.
333	struct PinnedHost {
334	host: String,
335	/// The app's own directory inside the worktree — the worktree root for a
336	/// repo holding one product, `<worktree>/pom` for one holding several.
337	build_dir: String,
338	sha: String,
339	}
340
341	/// Confirm the tag states the version being built, in every file that states a
342	/// version.
343	///
344	/// Read out of the tag with `git show` on a build host, so it needs no worktree
345	/// path of its own and no assumption that the daemon has a copy of the repo:
346	/// one host is enough, since the barrier has already proven they are all on the
347	/// same commit.
348	///
349	/// Skipped when nothing was pinned, which is `pin_release_sha = false` — tests,
350	/// whose repos are plain directories with no tag to read.
351	async fn check_version_at_tag(
352	state: &AppState,
353	app: &AppId,
354	version: &Version,
355	pinned: &Pinned,
356	) -> Result<()> {
357	let Some((host, dir)) = pinned.build_dirs.first() else {
358	return Ok(());
359	};
360	let Some(cfg) = state.topo.app(app) else {
361	return Ok(());
362	};
363	let Some(exec) = state.executors.get(host) else {
364	return Ok(());
365	};
366	let tag = cfg.tag_for(version);
367	let version_path = cfg.version_path.as_deref();
368
369	let mut sources: Vec<(String, String)> = Vec::new();
370	for rel in engine::version_sources(version_path) {
371	let step = OpStep::shell(Action::Build, engine::git_show_file_cmd(dir, &tag, &rel));
372	let mut sink = DiscardSink;
373	let out = exec
374	.run_streaming(&step, &mut sink)
375	.await
376	.with_context(\|\| format!("reading {rel} from {tag} on `{host}`"))?;
377	if out.status.success() {
378	sources.push((rel, String::from_utf8_lossy(&out.stdout).into_owned()));
379	} else if version_path == Some(rel.as_str()) {
380	// A file the app NAMES as its version source has to be there, or
381	// the check would pass by failing to look.
382	anyhow::bail!("`{rel}` is not in {tag} on `{host}`, but the app declares it");
383	}
384	}
385	engine::versions_agree(
386	&format!("{tag} (read on `{host}`)"),
387	&sources,
388	version_path,
389	version,
390	)
391	}
392
393	/// First 12 chars of a sha for a readable error.
394	fn short(sha: &str) -> &str {
395	sha.get(..12).unwrap_or(sha)
396	}
397
398	/// Resolve the version to build: explicit, or read from `tauri.conf.json`.
399	///
400	/// Returns typed errors so the route maps user mistakes (unknown app, bad
401	/// version string) to 400, and only genuine daemon-side failures (reading the
402	/// app's `tauri.conf.json`) to 500.
403	pub fn resolve_version(
404	state: &AppState,
405	app: &AppId,
406	explicit: Option<String>,
407	) -> crate::error::Result<Version> {
408	use crate::error::Error;
409	if let Some(v) = explicit {
410	return Version::parse(&v).map_err(Error::BadRequest);
411	}
412	let cfg = state
413	.topo
414	.app(app)
415	.ok_or_else(\|\| Error::BadRequest(format!("unknown app `{app}`")))?;
416	engine::version_from_repo(&cfg.repo, cfg.version_path.as_deref()).map_err(Error::Other)
417	}
418
419	/// Validate + default the target list against what the app ships. Unknown app,
420	/// an unshipped target, or a target no host can build are all client errors
421	/// (400), not server errors (500).
422	pub fn resolve_targets(
423	state: &AppState,
424	app: &AppId,
425	requested: Vec<Target>,
426	) -> crate::error::Result<Vec<Target>> {
427	use crate::error::Error;
428	let cfg = state
429	.topo
430	.app(app)
431	.ok_or_else(\|\| Error::BadRequest(format!("unknown app `{app}`")))?;
432	if requested.is_empty() {
433	return Ok(cfg.targets.clone());
434	}
435	for t in &requested {
436	if !cfg.targets.contains(t) {
437	return Err(Error::BadRequest(format!(
438	"app `{app}` does not ship target {t}"
439	)));
440	}
441	if state.topo.host_for(*t).is_none() {
442	return Err(Error::BadRequest(format!("no host can build {t}")));
443	}
444	}
445	Ok(requested)
446	}
447
448	/// Insert the build row and spawn per-target tasks. Returns the build id.
449	pub async fn start_build(
450	state: AppState,
451	app: AppId,
452	version: Version,
453	targets: Vec<Target>,
454	) -> Result<i64> {
455	// Pin every build host to the release tag and verify they agree, before any
456	// target task spawns. Off in tests (their repos aren't git checkouts).
457	let pinned = Arc::new(if state.cfg.pin_release_sha {
458	pin_release(&state, &app, &version, &targets)
459	.await
460	.context("release preflight")?
461	} else {
462	Pinned {
463	build_dirs: Vec::new(),
464	sha: String::new(),
465	}
466	});
467
468	// Then: every version source in the TAG must agree with the version being
469	// built, before a single host compiles. `version_from_repo` reads one file,
470	// so a `tauri.conf.json`/`Cargo.toml` (or explicit-version) mismatch would
471	// otherwise sail through and file artifacts under the wrong version.
472	//
473	// After the pin rather than before it, and read from the tag rather than
474	// from the checkout, because those are two different trees now. Asking the
475	// checkout answered a question about a tree the release does not build: a
476	// working copy one commit ahead failed a release of the tag behind it, and
477	// a `Cargo.toml` that disagreed with the tag it was tagged in passed.
478	check_version_at_tag(&state, &app, &version, &pinned)
479	.await
480	.context("version preflight")?;
481
482	let build_id: i64 = sqlx::query_scalar(
483	"INSERT INTO builds (app, version, status, created_at) VALUES (?, ?, 'running', ?) RETURNING id",
484	)
485	.bind(app.as_str())
486	.bind(version.to_string())
487	.bind(chrono::Utc::now().to_rfc3339())
488	.fetch_one(&state.pool)
489	.await
490	.context("insert build")?;
491
492	events::emit(
493	&state.events,
494	Event::BuildRequested {
495	app: app.clone(),
496	version: version.clone(),
497	targets: targets.clone(),
498	},
499	);
500	crate::metrics::build_started();
501
502	// Spawn every target into a JoinSet so finalize_build can await completion
503	// event-driven (no DB polling) and observe each task's outcome (panic vs
504	// clean) for supervision.
505	let mut set: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
506	for target in targets {
507	let app = app.clone();
508	let version = version.clone();
509	let key = (app.clone(), target);
510	let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
511
512	// Latest-wins, done as ONE critical section: supersede the prior occupant
513	// (cooperatively cancel its recipe + abort its task) and install ours
514	// without ever releasing the lock, so two concurrent /build+/retry for the
515	// same (app, target) can't both pass "nothing to abort" and run together.
516	let mut active = state.active.lock().await;
517	if let Some(prev) = active.remove(&key)
518	&& !prev.abort.is_finished()
519	{
520	// The recipe runs on a blocking thread; abort() alone can't stop it,
521	// so set the cooperative flag the engine checks at step boundaries.
522	prev.cancel.store(true, std::sync::atomic::Ordering::SeqCst);
523	prev.abort.abort();
524	events::emit(
525	&state.events,
526	Event::TargetAborted {
527	app: app.clone(),
528	target,
529	},
530	);
531	}
532	let abort = set.spawn(run_target(
533	state.clone(),
534	build_id,
535	app,
536	version,
537	target,
538	cancel.clone(),
539	pinned.clone(),
540	));
541	active.insert(
542	key,
543	crate::state::ActiveSlot {
544	build_id,
545	abort,
546	cancel,
547	},
548	);
549	crate::metrics::set_in_flight(active.len());
550	}
551
552	// Mark the build done once all target tasks settle. Spawned so /build
553	// returns immediately. Nothing to put back: the release built in Bento's own
554	// worktrees, and the checkouts were never moved.
555	tokio::spawn(finalize_build(state, build_id, set));
556	Ok(build_id)
557	}
558
559	/// Run one target's recipe end to end, updating its `target_runs` row. `cancel`
560	/// is set when a newer build supersedes this one; the engine checks it at step
561	/// boundaries and before publish so a superseded recipe can't advance or ship.
562	async fn run_target(
563	state: AppState,
564	build_id: i64,
565	app: AppId,
566	version: Version,
567	target: Target,
568	cancel: Arc<std::sync::atomic::AtomicBool>,
569	pinned: Arc<Pinned>,
570	) {
571	let started = std::time::Instant::now();
572	let target_run_id: i64 = match sqlx::query_scalar(
573	"INSERT INTO target_runs (build_id, app, version, target, status, started_at)
574	VALUES (?, ?, ?, ?, 'running', ?) RETURNING id",
575	)
576	.bind(build_id)
577	.bind(app.as_str())
578	.bind(version.to_string())
579	.bind(target.to_string())
580	.bind(chrono::Utc::now().to_rfc3339())
581	.fetch_one(&state.pool)
582	.await
583	{
584	Ok(id) => id,
585	Err(e) => {
586	tracing::error!(%app, %target, error = %e, "could not create target_run");
587	return;
588	}
589	};
590
591	events::emit(
592	&state.events,
593	Event::TargetStart {
594	app: app.clone(),
595	version: version.clone(),
596	target,
597	},
598	);
599
600	let recipe_src = match read_recipe(&state, &app, target) {
601	Ok(s) => s,
602	Err(e) => {
603	fail_target(
604	&state,
605	target_run_id,
606	&app,
607	&version,
608	target,
609	Step::Checkout,
610	&format!("{e:#}"),
611	)
612	.await;
613	crate::metrics::target_finished(
614	&target.to_string(),
615	"failed",
616	started.elapsed().as_secs_f64(),
617	);
618	return;
619	}
620	};
621
622	// Pre-flight the build host before running the recipe. For local/ssh hosts
623	// this is a no-op; for the agent host (macOS in-session signing) it hits
624	// `/health` so a dead ops-agent fails here with a clear message — matching
625	// what the driver does — instead of erroring opaquely on the recipe's first
626	// dispatch to that host.
627	if let Some(host) = state.topo.host_for(target)
628	&& let Some(exec) = state.executors.get(&host.name)
629	&& let Err(e) = exec.preflight().await
630	{
631	fail_target(
632	&state,
633	target_run_id,
634	&app,
635	&version,
636	target,
637	Step::Checkout,
638	&format!("{e:#}"),
639	)
640	.await;
641	crate::metrics::target_finished(
642	&target.to_string(),
643	"failed",
644	started.elapsed().as_secs_f64(),
645	);
646	return;
647	}
648
649	// Host + checkout path for this target, resolved from the topology. Both are
650	// exposed to the recipe (`build_host()` / `repo()`); a missing host here is
651	// unreachable (resolve_targets rejected unbuildable targets before /build),
652	// but fall back rather than panic.
653	let build_host = state
654	.topo
655	.host_for(target)
656	.map(\|h\| h.name.clone())
657	.unwrap_or_default();
658	let build_host_ssh = state
659	.topo
660	.host_for(target)
661	.map(\|h\| h.ssh.clone())
662	.unwrap_or_default();
663	// Where the recipe builds. The preflight prepared a worktree per host and
664	// these are those paths, so `repo()` in a recipe is Bento's own tree and no
665	// recipe knows the difference — which is the whole reason the checkout could
666	// stop being what a release builds in. The topology's paths are the fallback
667	// for `pin_release_sha = false`, which is tests: their repos are plain
668	// directories with no git in them to make a worktree of.
669	let repo_by_host: std::collections::HashMap<String, String> = if pinned.build_dirs.is_empty() {
670	state
671	.topo
672	.app(&app)
673	.map(\|a\| a.repo_by_host.clone())
674	.unwrap_or_default()
675	} else {
676	pinned.build_dirs.iter().cloned().collect()
677	};
678	// The daemon-local path, which is a different question: `crate_preflight`
679	// runs `cargo metadata` on the daemon's own box rather than on a build host.
680	// Point it at the local host's worktree when there is one, so it reads the
681	// tagged manifest the release is publishing and not whatever the working
682	// copy happens to say.
683	let repo = state
684	.topo
685	.hosts
686	.iter()
687	.find(\|h\| h.ssh == "local")
688	.and_then(\|h\| repo_by_host.get(&h.name).cloned())
689	.or_else(\|\| state.topo.app(&app).map(\|a\| a.repo.clone()))
690	.unwrap_or_default();
691	let features = state
692	.topo
693	.app(&app)
694	.map(\|a\| a.features.clone())
695	.unwrap_or_default();
696	let tag = state
697	.topo
698	.app(&app)
699	.map_or_else(\|\| format!("v{version}"), \|a\| a.tag_for(&version));
700	// The opt-in all-targets-green publish gate: pass the declared target set
701	// (so `publish` can require every sibling green) only when the app turns it
702	// on; otherwise None leaves independent per-target publishing unchanged.
703	let all_green_required = state
704	.topo
705	.app(&app)
706	.and_then(\|a\| a.require_all_targets.then(\|\| a.targets.clone()));
707
708	// A service's install destination for THIS target, plus an executor for it.
709	//
710	// The executor is added to a per-run copy of the map rather than to the
711	// daemon-wide one, so a deploy grant on a production host exists only for
712	// the duration of the run that needs it and only for the app that declared
713	// it. Nothing else can address that host: it is not in the topology, so no
714	// other app's recipe can name it.
715	let deploy = state
716	.topo
717	.app(&app)
718	.and_then(\|a\| a.deploy_for(target))
719	.cloned();
720	let execs = match &deploy {
721	Some(d) => {
722	let mut map = (*state.executors).clone();
723	map.insert(d.host.clone(), crate::state::build_deploy_executor(d));
724	Arc::new(map)
725	}
726	None => state.executors.clone(),
727	};
728
729	let ctx = Arc::new(
730	RecipeCtx::new(
731	app.clone(),
732	version.clone(),
733	target,
734	build_host,
735	build_host_ssh,
736	tag.clone(),
737	repo,
738	features,
739	// Gates the `verify` step's capability: a library's crate preflight is
740	// not an app's Gatekeeper check. See engine::action_for.
741	state.topo.app(&app).map(\|a\| a.kind).unwrap_or_default(),
742	target_run_id,
743	execs,
744	state.syncs.clone(),
745	deploy,
746	state.pool.clone(),
747	state.events.clone(),
748	state.cfg.clone(),
749	state.ota.clone(),
750	tokio::runtime::Handle::current(),
751	cancel,
752	all_green_required,
753	)
754	.with_repo_by_host(repo_by_host),
755	);
756
757	// Serialize per host: hold this host's lock for the whole recipe run so a
758	// second target on the same box (goingson macos + ios both on mbp) can't
759	// build concurrently in one checkout and corrupt the shared target/ +
760	// keychain. Targets on different hosts hold different locks and still fan
761	// out. Acquired at an await point, so a supersede-abort while queued drops
762	// the task cleanly before it ever takes the lock. Not held during the
763	// read-only preflight above.
764	let _host_guard = match state.host_locks.get(&ctx.build_host).cloned() {
765	Some(lock) => Some(lock.lock_owned().await),
766	None => None,
767	};
768
769	// Rhai is synchronous; run the recipe (and its final step finalization) on
770	// a blocking thread so host functions can `block_on` without sitting on a
771	// runtime worker.
772	let ctx_run = ctx.clone();
773	let outcome = tokio::task::spawn_blocking(move \|\| {
774	let engine = engine::build_engine(&ctx_run);
775	let res = engine.run(&recipe_src);
776	let last_step = ctx_run.current_step();
777	match &res {
778	Ok(()) => {
779	let _ = ctx_run.finish_step(Status::Ok);
780	}
781	Err(_) => {
782	let _ = ctx_run.finish_step(Status::Failed);
783	}
784	}
785	res.map_err(\|e\| (last_step, e.to_string()))
786	})
787	.await;
788
789	// Write the artifact record before the run is stamped terminal. It describes
790	// what was collected, so it runs whether the recipe succeeded or failed: a
791	// build that signed and collected an artifact and then failed at `publish`
792	// still produced bytes somebody may want the provenance of. Emit-only and
793	// non-fatal; nothing reads it yet.
794	let record_path = crate::artifact_record::emit(&state, &ctx, &pinned.sha).await;
795
796	match outcome {
797	Ok(Ok(())) => {
798	// Hand the artifact to Sando before the run is stamped ok, so a
799	// target reported green is one whose bytes reached the controller
800	// that decides whether they ship. Only on success: an artifact from
801	// a failed recipe is exactly what should not be offered for a
802	// deploy, whatever it managed to collect on the way down.
803	if let Err(e) =
804	handoff_for(&state, record_path.as_deref(), &app, &version, target).await
805	{
806	let msg = format!("{e:#}");
807	tracing::error!(%app, %target, error = %msg, "handing the artifact to sando failed");
808	fail_target(
809	&state,
810	target_run_id,
811	&app,
812	&version,
813	target,
814	Step::Handoff,
815	&msg,
816	)
817	.await;
818	crate::metrics::target_finished(
819	&target.to_string(),
820	"failed",
821	started.elapsed().as_secs_f64(),
822	);
823	return;
824	}
825
826	let artifacts = collected_artifacts(&state, &app, &version, target);
827	if let Err(e) = sqlx::query(
828	"UPDATE target_runs SET status = 'ok', current_step = NULL, finished_at = ? WHERE id = ?",
829	)
830	.bind(chrono::Utc::now().to_rfc3339())
831	.bind(target_run_id)
832	.execute(&state.pool)
833	.await
834	{
835	// A swallowed terminal write would leave the row `running`;
836	// finalize_build reconciles it, but log so the cause is visible.
837	tracing::error!(%app, %target, error = %e, "could not stamp target_run ok");
838	}
839	crate::metrics::target_finished(
840	&target.to_string(),
841	"ok",
842	started.elapsed().as_secs_f64(),
843	);
844	events::emit(
845	&state.events,
846	Event::TargetOk {
847	app,
848	version,
849	target,
850	artifacts,
851	},
852	);
853	}
854	Ok(Err((step, msg))) => {
855	fail_target(&state, target_run_id, &app, &version, target, step, &msg).await;
856	crate::metrics::target_finished(
857	&target.to_string(),
858	"failed",
859	started.elapsed().as_secs_f64(),
860	);
861	}
862	Err(join_err) => {
863	// Task was aborted (superseded) or panicked.
864	let status = if join_err.is_cancelled() {
865	"aborted"
866	} else {
867	"failed"
868	};
869	crate::metrics::target_finished(
870	&target.to_string(),
871	status,
872	started.elapsed().as_secs_f64(),
873	);
874	let msg = if join_err.is_cancelled() {
875	"aborted (superseded)".to_string()
876	} else {
877	format!("recipe task panicked: {join_err}")
878	};
879	fail_target(
880	&state,
881	target_run_id,
882	&app,
883	&version,
884	target,
885	Step::Build,
886	&msg,
887	)
888	.await;
889	}
890	}
891	}
892
893	async fn fail_target(
894	state: &AppState,
895	target_run_id: i64,
896	app: &AppId,
897	version: &Version,
898	target: Target,
899	step: Step,
900	error: &str,
901	) {
902	if let Err(e) = sqlx::query(
903	"UPDATE target_runs SET status = 'failed', current_step = NULL, error = ?, finished_at = ? WHERE id = ?",
904	)
905	.bind(error)
906	.bind(chrono::Utc::now().to_rfc3339())
907	.bind(target_run_id)
908	.execute(&state.pool)
909	.await
910	{
911	tracing::error!(%app, %target, error = %e, "could not stamp target_run failed");
912	}
913	events::emit(
914	&state.events,
915	Event::TargetFailed {
916	app: app.clone(),
917	version: version.clone(),
918	target,
919	step,
920	error: error.to_string(),
921	},
922	);
923	}
924
925	/// Await every target task of a build, then stamp the build's terminal status.
926	///
927	/// Event-driven: it joins the `JoinSet` rather than polling the DB. Each task's
928	/// outcome is supervised — a panicked task is logged (and its still-`running`
929	/// row is reconciled below), an aborted (superseded) task is expected.
930	///
931	/// Per-step deadlines (see `engine::step_budget`) are the real bound on a wedged
932	/// step now; this overall deadline is only a generous last-resort backstop for a
933	/// hang outside a bounded command. When it trips it sets each still-running
934	/// target's cooperative cancel FIRST — the blocking recipe bodies observe that
935	/// at their next bounded command or step boundary and stop signing — then aborts
936	/// the async wrappers. The old code aborted only the wrappers, which could not
937	/// reach the blocking bodies, so a build was marked failed while codesign kept
938	/// running on the mac.
939	async fn finalize_build(state: AppState, build_id: i64, mut set: tokio::task::JoinSet<()>) {
940	const MAX_WAIT: std::time::Duration = std::time::Duration::from_hours(6);
941	let deadline = tokio::time::Instant::now() + MAX_WAIT;
942	loop {
943	match tokio::time::timeout_at(deadline, set.join_next()).await {
944	Ok(Some(Ok(()))) => {}
945	Ok(Some(Err(e))) => {
946	// A panic in run_target itself (outside its inner spawn_blocking,
947	// which is already caught). Cancellation = a superseding build.
948	if !e.is_cancelled() {
949	tracing::error!(build_id, error = %e, "target task panicked");
950	}
951	}
952	Ok(None) => break, // all targets settled
953	Err(_elapsed) => {
954	tracing::error!(
955	build_id,
956	"finalize_build backstop deadline hit; cancelling then aborting remaining targets"
957	);
958	// Set the cooperative cancel on this build's still-running slots
959	// BEFORE aborting, so the blocking recipe bodies actually stop
960	// (abort() alone cannot reach a spawn_blocking body).
961	{
962	let active = state.active.lock().await;
963	for slot in active.values().filter(\|s\| s.build_id == build_id) {
964	slot.cancel.store(true, std::sync::atomic::Ordering::SeqCst);
965	}
966	}
967	set.abort_all();
968	while set.join_next().await.is_some() {}
969	break;
970	}
971	}
972	}
973
974	// Reap this build's latest-wins slots (only our own — a superseding build
975	// owns a different build_id and is left intact).
976	{
977	let mut active = state.active.lock().await;
978	active.retain(\|_, slot\| slot.build_id != build_id);
979	crate::metrics::set_in_flight(active.len());
980	}
981
982	let now = chrono::Utc::now().to_rfc3339();
983	// Any row still `running` is a panicked/aborted task that never stamped its
984	// terminal status — reconcile it so the build finalizes truthfully.
985	if let Err(e) = sqlx::query(
986	"UPDATE target_runs SET status = 'failed', \
987	error = COALESCE(error, 'target task ended before stamping its status'), \
988	finished_at = ? WHERE build_id = ? AND status = 'running'",
989	)
990	.bind(&now)
991	.bind(build_id)
992	.execute(&state.pool)
993	.await
994	{
995	tracing::error!(build_id, error = %e, "could not reconcile straggling target_runs");
996	}
997
998	let failed: i64 = sqlx::query_scalar(
999	"SELECT COUNT(*) FROM target_runs WHERE build_id = ? AND status = 'failed'",
1000	)
1001	.bind(build_id)
1002	.fetch_one(&state.pool)
1003	.await
1004	.unwrap_or(0);
1005	let status = if failed == 0 { "ok" } else { "failed" };
1006	if let Err(e) = sqlx::query("UPDATE builds SET status = ?, finished_at = ? WHERE id = ?")
1007	.bind(status)
1008	.bind(&now)
1009	.bind(build_id)
1010	.execute(&state.pool)
1011	.await
1012	{
1013	tracing::error!(build_id, error = %e, "could not stamp build terminal status");
1014	}
1015	}
1016
1017	/// Read the recipe text for `(app, target)` from the app's checkout on the
1018	/// daemon host.
1019	///
1020	/// Apps use one recipe per platform (`<recipe_dir>/<platform>.rhai`), since
1021	/// what they produce differs by platform. A library produces one crate whatever
1022	/// host uploads it, so it uses a single `publish.rhai` — a `linux.rhai` naming
1023	/// a registry upload would imply a per-platform artifact that does not exist.
1024	fn read_recipe(state: &AppState, app: &AppId, target: Target) -> Result<String> {
1025	let cfg = state
1026	.topo
1027	.app(app)
1028	.ok_or_else(\|\| anyhow::anyhow!("unknown app `{app}`"))?;
1029	let file = match cfg.kind {
1030	crate::topology::Kind::Library => "publish.rhai".to_string(),
1031	// A service is built per target like an app — the binary genuinely
1032	// differs per platform, and so does where it lands — so it takes the
1033	// same per-platform recipe naming rather than a single deploy.rhai.
1034	crate::topology::Kind::App \| crate::topology::Kind::Service => {
1035	format!("{}.rhai", target.platform.as_str())
1036	}
1037	};
1038	let path: PathBuf = engine::expand_tilde(&cfg.repo)
1039	.join(&cfg.recipe_dir)
1040	.join(file);
1041	std::fs::read_to_string(&path).with_context(\|\| format!("reading recipe {}", path.display()))
1042	}
1043
1044	/// Send this target's finished bundle to the Sando configured for the app, if
1045	/// one is. Nothing configured is a no-op and the ordinary case.
1046	///
1047	/// A configured handoff with no record is an error rather than a skip. It means
1048	/// the recipe succeeded and collected nothing, or that the paperwork could not
1049	/// be written — and for an app whose whole point is being deployed by Sando, a
1050	/// build that produced nothing to hand over is not a green build. Everywhere
1051	/// else a missing record stays the non-event it was.
1052	async fn handoff_for(
1053	state: &AppState,
1054	record_path: Option<&std::path::Path>,
1055	app: &AppId,
1056	version: &Version,
1057	target: Target,
1058	) -> anyhow::Result<()> {
1059	if !state.cfg.handoff.contains_key(app.as_str()) {
1060	return Ok(());
1061	}
1062	let record_path = record_path.ok_or_else(\|\| {
1063	anyhow::anyhow!(
1064	"{app} hands off to sando, but this {target} run wrote no artifact record \
1065	(nothing was collected, or the record could not be written)"
1066	)
1067	})?;
1068	let dir = crate::archive::target_dir(&state.cfg.dist_root, app, version, target);
1069	crate::handoff::send(&state.cfg, &dir, record_path, app, version, target).await
1070	}
1071
1072	/// What this target run left in its collect directory.
1073	///
1074	/// Per target: the event reports what THIS run produced, and every target of a
1075	/// version used to share one directory, so a mac build's `TargetOk` listed the
1076	/// Linux AppImage a sibling had collected minutes earlier.
1077	fn collected_artifacts(
1078	state: &AppState,
1079	app: &AppId,
1080	version: &Version,
1081	target: Target,
1082	) -> Vec<String> {
1083	let dir = crate::archive::target_dir(&state.cfg.dist_root, app, version, target);
1084	let Ok(rd) = std::fs::read_dir(&dir) else {
1085	return Vec::new();
1086	};
1087	rd.filter_map(std::result::Result::ok)
1088	.map(\|e\| e.file_name().to_string_lossy().into_owned())
1089	.collect()
1090	}
1091
1092	#[cfg(test)]
1093	mod tests {
1094	use super::*;
1095	use crate::config::Config;
1096	use crate::ota::OtaRegistry;
1097	use crate::topology::Topology;
1098	use async_trait::async_trait;
1099	use ops_exec::{CapabilitySet, Executor, LogSink, RunOutput, SyncOpts};
1100	use sqlx::SqlitePool;
1101	use std::collections::HashMap;
1102	use std::os::unix::process::ExitStatusExt;
1103	use std::sync::Arc;
1104	use tokio::sync::Mutex;
1105
1106	/// A no-transport [`Executor`] for the paths that don't need a real command
1107	/// to run: its `preflight` is programmable (the agent host hits `/health`
1108	/// there, and a dead `ops-agent` must fail the target before the recipe
1109	/// dispatches), and every actual op is a success no-op.
1110	struct FakeExec {
1111	caps: CapabilitySet,
1112	preflight_err: Option<String>,
1113	}
1114
1115	impl FakeExec {
1116	fn preflight_fails(msg: &str) -> Arc<dyn Executor> {
1117	Arc::new(Self {
1118	caps: CapabilitySet::default(),
1119	preflight_err: Some(msg.to_string()),
1120	})
1121	}
1122	}
1123
1124	#[async_trait]
1125	impl Executor for FakeExec {
1126	async fn run_streaming(
1127	&self,
1128	_step: &ops_exec::Step,
1129	_sink: &mut dyn LogSink,
1130	) -> anyhow::Result<RunOutput> {
1131	Ok(RunOutput {
1132	status: std::process::ExitStatus::from_raw(0),
1133	stdout: Vec::new(),
1134	stderr: Vec::new(),
1135	})
1136	}
1137	async fn pull_file(
1138	&self,
1139	_r: &std::path::Path,
1140	_l: &std::path::Path,
1141	_o: &SyncOpts,
1142	) -> anyhow::Result<()> {
1143	Ok(())
1144	}
1145	async fn pull_dir(
1146	&self,
1147	_r: &std::path::Path,
1148	_l: &std::path::Path,
1149	_o: &SyncOpts,
1150	) -> anyhow::Result<()> {
1151	Ok(())
1152	}
1153	async fn pull_glob(
1154	&self,
1155	_g: &str,
1156	_l: &std::path::Path,
1157	_o: &SyncOpts,
1158	) -> anyhow::Result<()> {
1159	Ok(())
1160	}
1161	async fn push_dir(
1162	&self,
1163	_l: &std::path::Path,
1164	_r: &std::path::Path,
1165	_o: &SyncOpts,
1166	) -> anyhow::Result<()> {
1167	Ok(())
1168	}
1169	async fn preflight(&self) -> anyhow::Result<()> {
1170	match &self.preflight_err {
1171	Some(m) => anyhow::bail!("{m}"),
1172	None => Ok(()),
1173	}
1174	}
1175	fn capabilities(&self) -> &CapabilitySet {
1176	&self.caps
1177	}
1178	}
1179
1180	/// A recording, programmable [`Executor`] for the macOS sign chain. Every
1181	/// dispatched shell command is captured for assertion, and its exit code +
1182	/// stdout is chosen by the first rule whose needle the command contains — so
1183	/// a test can make `notarytool` report `Accepted`, make `codesign` fail, or
1184	/// make `spctl` emit the Gatekeeper sentinel without a real Mac or SSH. A
1185	/// rule may carry a sequence of responses (one per successive match) to
1186	/// drive the notarize retry loop; the last entry repeats once the sequence
1187	/// is exhausted. Unmatched commands succeed as empty no-ops (so a plain
1188	/// build `sh_ok` passes), and the sync/preflight ops are no-ops.
1189	struct ScriptedExec {
1190	caps: CapabilitySet,
1191	rules: Vec<ScriptRule>,
1192	log: Arc<std::sync::Mutex<Vec<String>>>,
1193	}
1194
1195	struct ScriptRule {
1196	needle: String,
1197	responses: Vec<(i32, String)>,
1198	calls: std::sync::atomic::AtomicUsize,
1199	}
1200
1201	impl ScriptedExec {
1202	fn new() -> Self {
1203	Self {
1204	// A mac host's real grant. Nothing in the dispatch path gates on
1205	// it (this fake never calls `gate`), but keep it coherent so
1206	// `capabilities()` is not a lie.
1207	caps: CapabilitySet::from_tokens(
1208	["build", "sign", "notarize", "staple"],
1209	["build-log", "artifact"],
1210	),
1211	rules: Vec::new(),
1212	log: Arc::new(std::sync::Mutex::new(Vec::new())),
1213	}
1214	}
1215
1216	/// Respond to every command containing `needle` with `(code, stdout)`.
1217	fn on(mut self, needle: &str, code: i32, stdout: &str) -> Self {
1218	self.rules.push(ScriptRule {
1219	needle: needle.to_string(),
1220	responses: vec![(code, stdout.to_string())],
1221	calls: std::sync::atomic::AtomicUsize::new(0),
1222	});
1223	self
1224	}
1225
1226	/// Respond to successive `needle` matches with successive responses; the
1227	/// last repeats once the list is exhausted. Drives the notarize retry.
1228	fn on_seq(mut self, needle: &str, responses: &[(i32, &str)]) -> Self {
1229	self.rules.push(ScriptRule {
1230	needle: needle.to_string(),
1231	responses: responses
1232	.iter()
1233	.map(\|(c, s)\| (c, (s).to_string()))
1234	.collect(),
1235	calls: std::sync::atomic::AtomicUsize::new(0),
1236	});
1237	self
1238	}
1239
1240	/// Every shell command this executor was asked to run, in order.
1241	fn commands(&self) -> Vec<String> {
1242	self.log.lock().unwrap().clone()
1243	}
1244	}
1245
1246	#[async_trait]
1247	impl Executor for ScriptedExec {
1248	async fn run_streaming(
1249	&self,
1250	step: &ops_exec::Step,
1251	_sink: &mut dyn LogSink,
1252	) -> anyhow::Result<RunOutput> {
1253	let cmd = step.argv.last().cloned().unwrap_or_default();
1254	self.log.lock().unwrap().push(cmd.clone());
1255	let (code, stdout) = self.rules.iter().find(\|r\| cmd.contains(&r.needle)).map_or(
1256	(0, String::new()),
1257	\|r\| {
1258	let i = r.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1259	r.responses[i.min(r.responses.len() - 1)].clone()
1260	},
1261	);
1262	Ok(RunOutput {
1263	// Shift into the wait-status word's exit-code byte so
1264	// `ExitStatus::code()` reports `code` exactly (a bare
1265	// `from_raw(1)` reads as a signal, yielding `None`).
1266	status: std::process::ExitStatus::from_raw(code << 8),
1267	stdout: stdout.into_bytes(),
1268	stderr: Vec::new(),
1269	})
1270	}
1271	async fn pull_file(
1272	&self,
1273	_r: &std::path::Path,
1274	_l: &std::path::Path,
1275	_o: &SyncOpts,
1276	) -> anyhow::Result<()> {
1277	Ok(())
1278	}
1279	async fn pull_dir(
1280	&self,
1281	_r: &std::path::Path,
1282	_l: &std::path::Path,
1283	_o: &SyncOpts,
1284	) -> anyhow::Result<()> {
1285	Ok(())
1286	}
1287	async fn pull_glob(
1288	&self,
1289	_g: &str,
1290	_l: &std::path::Path,
1291	_o: &SyncOpts,
1292	) -> anyhow::Result<()> {
1293	Ok(())
1294	}
1295	async fn push_dir(
1296	&self,
1297	_l: &std::path::Path,
1298	_r: &std::path::Path,
1299	_o: &SyncOpts,
1300	) -> anyhow::Result<()> {
1301	Ok(())
1302	}
1303	async fn preflight(&self) -> anyhow::Result<()> {
1304	Ok(())
1305	}
1306	fn capabilities(&self) -> &CapabilitySet {
1307	&self.caps
1308	}
1309	}
1310
1311	/// Assemble an [`AppState`] from the three per-test inputs, filling in the
1312	/// executors/syncs (built from `topo`) and the fixed test scaffolding
1313	/// (metrics handle, event bus, standard OTA registry, empty active map, no
1314	/// token). Every runner test builds the same struct around a different repo
1315	/// + recipe; this is that struct in one place.
1316	fn test_state(pool: SqlitePool, topo: Topology, cfg: Config) -> AppState {
1317	let executors = Arc::new(crate::state::build_executors(&topo));
1318	let syncs = Arc::new(crate::state::build_syncs(&topo));
1319	let host_locks = crate::state::build_host_locks(&topo);
1320	AppState {
1321	pool,
1322	topo: Arc::new(topo),
1323	cfg: Arc::new(cfg),
1324	prom: crate::metrics::test_handle(),
1325	events: crate::events::channel(),
1326	ota: Arc::new(OtaRegistry::standard("https://makenot.work")),
1327	executors,
1328	syncs,
1329	active: Arc::new(Mutex::new(HashMap::new())),
1330	api_token: None,
1331	host_locks,
1332	distribution: Arc::new(Mutex::new(HashMap::new())),
1333	http: crate::tls::builder().build().unwrap(),
1334	// Port 1 refuses instantly. A test must never probe production, and
1335	// a refusal is also faster than any timeout would be.
1336	mnw_base_url: "http://127.0.0.1:1".into(),
1337	}
1338	}
1339
1340	/// A `kind = "service"` release end to end: build, `glibc_check`, `deploy`,
1341	/// and a health assertion the recipe makes itself against the service host.
1342	///
1343	/// The whole point of the deploy step is that it dispatches to a host
1344	/// OUTSIDE the build topology, so this exercises the resolution
1345	/// (target -> `[[deploy]]` entry -> executor registered for the run), the
1346	/// staging, and the call into the privileged installer — with a fake
1347	/// installer standing in for the root script, which is the one part a test
1348	/// cannot run for real.
1349	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1350	async fn service_recipe_builds_then_deploys_and_verifies() {
1351	let tmp = tempfile::tempdir().unwrap();
1352	let root = tmp.path();
1353	let repo = root.join("svc");
1354	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1355	std::fs::write(repo.join("Cargo.toml"), "[package]\nversion = \"0.4.0\"\n").unwrap();
1356
1357	// Stand-in for the root installer: same three arguments, records what it
1358	// was asked to do instead of writing to /usr/local/bin and restarting a
1359	// unit. `install` + a marker file, so the test can assert the binary
1360	// that arrived is the binary that was built.
1361	let installer = root.join("install-service.sh");
1362	std::fs::write(
1363	&installer,
1364	"#!/bin/sh\nset -eu\ninstall -m 0755 \"$1\" \"$2\"\necho \"restarted $3\" >> \"$2.log\"\n",
1365	)
1366	.unwrap();
1367	std::fs::set_permissions(
1368	&installer,
1369	<std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o755),
1370	)
1371	.unwrap();
1372	let install_path = root.join("bin/svc");
1373	std::fs::create_dir_all(root.join("bin")).unwrap();
1374
1375	std::fs::write(
1376	repo.join("dist/recipes/linux.rhai"),
1377	r#"
1378	let v = version();
1379	step("build");
1380	sh_ok("fw13", "mkdir -p REPO/target/release && echo built-BIN > REPO/target/release/svc");
1381	step("verify");
1382	log(glibc_check("REPO/target/release/svc"));
1383	step("deploy");
1384	log(deploy("REPO/target/release/svc"));
1385	// The recipe owns what "healthy" means, and asserts it itself
1386	// against the host it just restarted.
1387	sh_ok(deploy_host(), "test -x " + install_path());
1388	"#
1389	.replace("REPO", repo.to_str().unwrap())
1390	.replace("BIN", "0.4.0"),
1391	)
1392	.unwrap();
1393
1394	std::fs::write(
1395	repo.join("bento.toml"),
1396	format!(
1397	r#"kind = "service"
1398	targets = ["linux/x86_64"]
1399	version_path = "Cargo.toml"
1400
1401	[[deploy]]
1402	target = "linux/x86_64"
1403	host = "local"
1404	install_path = "{}"
1405	service = "svc.service"
1406	health_url = "http://localhost:9100/api/health"
1407	"#,
1408	install_path.display()
1409	),
1410	)
1411	.unwrap();
1412
1413	let mut cfg = Config::for_tests(root);
1414	cfg.deploy_installer = installer.display().to_string();
1415	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1416	let topo = Topology::from_str_for_tests(&format!(
1417	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
1418	pull_root = \"{repo}\"\n\n[app.svc]\nrepo = \"{repo}\"\n",
1419	repo = repo.display()
1420	))
1421	.unwrap();
1422	let state = test_state(pool.clone(), topo, cfg);
1423
1424	let build_id = start_build(
1425	state.clone(),
1426	AppId::new("svc"),
1427	Version::parse("0.4.0").unwrap(),
1428	vec!["linux/x86_64".parse().unwrap()],
1429	)
1430	.await
1431	.unwrap();
1432
1433	let mut status = String::new();
1434	for _ in 0..100 {
1435	status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1436	.bind(build_id)
1437	.fetch_optional(&pool)
1438	.await
1439	.unwrap()
1440	.unwrap_or_else(\|\| "running".to_string());
1441	if status != "running" {
1442	break;
1443	}
1444	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1445	}
1446	assert_eq!(status, "ok", "service run should succeed");
1447
1448	let steps: Vec<(String, String)> = sqlx::query_as(
1449	"SELECT step, status FROM step_runs WHERE target_run_id IN \
1450	(SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id",
1451	)
1452	.bind(build_id)
1453	.fetch_all(&pool)
1454	.await
1455	.unwrap();
1456	assert_eq!(
1457	steps.iter().map(\|(s, _)\| s.as_str()).collect::<Vec<_>>(),
1458	vec!["build", "verify", "deploy"],
1459	"a service ends at deploy, not collect"
1460	);
1461	assert!(steps.iter().all(\|(_, st)\| st == "ok"), "{steps:?}");
1462
1463	// The bytes that were built are the bytes that landed, and the unit was
1464	// restarted only after the install succeeded.
1465	assert_eq!(
1466	std::fs::read_to_string(&install_path).unwrap().trim(),
1467	"built-0.4.0"
1468	);
1469	assert!(
1470	std::fs::read_to_string(install_path.with_extension("").with_file_name("svc.log"))
1471	.unwrap()
1472	.contains("restarted svc.service")
1473	);
1474	}
1475
1476	/// Stand up a tmp app repo + topology and run a real local recipe end to
1477	/// end: step transitions, streamed `sh_ok`, `version_of`, `log`, and a
1478	/// `collect` that pulls a built artifact into dist_root.
1479	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1480	async fn local_linux_recipe_runs_end_to_end() {
1481	let tmp = tempfile::tempdir().unwrap();
1482	let root = tmp.path();
1483
1484	// Fake app checkout: tauri.conf.json + a linux recipe.
1485	let repo = root.join("app");
1486	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1487	std::fs::write(
1488	repo.join("src-tauri/tauri.conf.json"),
1489	r#"{"version":"0.0.1"}"#,
1490	)
1491	.unwrap();
1492	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1493	// Build writes an artifact into the repo; collect pulls it to dist_root.
1494	std::fs::write(
1495	repo.join("dist/recipes/linux.rhai"),
1496	r#"
1497	step("build");
1498	let v = version_of("demo");
1499	log("building demo " + v);
1500	sh_ok("fw13", "echo compiling; mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
1501	step("collect");
1502	collect("fw13", "REPO/out/demo.bin", "demo", v);
1503	"#
1504	.replace("REPO", repo.to_str().unwrap()),
1505	)
1506	.unwrap();
1507
1508	let mut cfg = Config::for_tests(root);
1509	// Archive to a local directory, so the deposit a real release makes to
1510	// astra runs on the same code path here.
1511	cfg.archive = Some(crate::config::Archive {
1512	host: "local".into(),
1513	root: root.join("archive"),
1514	});
1515	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1516	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1517	let topo = Topology::from_str_for_tests(&format!(
1518	r#"
1519	[[host]]
1520	name = "fw13"
1521	ssh = "local"
1522	targets = ["linux/x86_64"]
1523	pull_root = "{repo}"
1524
1525	[app.demo]
1526	repo = "{repo}"
1527	"#,
1528	repo = repo.display()
1529	))
1530	.unwrap();
1531
1532	let state = test_state(pool.clone(), topo, cfg);
1533
1534	let app = AppId::new("demo");
1535	let version = Version::parse("0.0.1").unwrap();
1536	let build_id = start_build(
1537	state.clone(),
1538	app,
1539	version,
1540	vec!["linux/x86_64".parse().unwrap()],
1541	)
1542	.await
1543	.unwrap();
1544
1545	// Wait for the target run to settle. The row may not be inserted on the
1546	// first poll (run_target is spawned, not awaited), so treat a missing row
1547	// as still-pending rather than an error.
1548	let mut status = String::new();
1549	for _ in 0..100 {
1550	status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1551	.bind(build_id)
1552	.fetch_optional(&pool)
1553	.await
1554	.unwrap()
1555	.unwrap_or_else(\|\| "running".to_string());
1556	if status != "running" {
1557	break;
1558	}
1559	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1560	}
1561	assert_eq!(status, "ok", "target run should succeed");
1562
1563	// Both steps recorded and finished ok.
1564	let steps: Vec<(String, String)> =
1565	sqlx::query_as("SELECT step, status FROM step_runs WHERE target_run_id IN (SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id")
1566	.bind(build_id)
1567	.fetch_all(&pool)
1568	.await
1569	.unwrap();
1570	let names: Vec<&str> = steps.iter().map(\|(s, _)\| s.as_str()).collect();
1571	assert_eq!(names, vec!["build", "collect"]);
1572	assert!(steps.iter().all(\|(_, st)\| st == "ok"));
1573
1574	// Artifact landed in dist_root, under its own target, and a step log was
1575	// written. Both trees are keyed the same way, `<app>/<version>/<target>/`.
1576	let artifact = state.cfg.dist_root.join("demo/0.0.1/linux-x86_64/demo.bin");
1577	assert!(artifact.exists(), "collect should copy the artifact");
1578	// ...and the same bytes reached the archive, at the same path under its
1579	// own root. This is the answer to "where is demo 0.0.1 for linux".
1580	let archived = root.join("archive/demo/0.0.1/linux-x86_64/demo.bin");
1581	assert!(
1582	archived.exists(),
1583	"collect should deposit into the archive: {}",
1584	archived.display()
1585	);
1586	assert_eq!(
1587	std::fs::read(&archived).unwrap(),
1588	std::fs::read(&artifact).unwrap()
1589	);
1590	// Read the path off the ledger rather than rebuilding it: the log is
1591	// named for its step run id, and the point of that is that the row
1592	// resolves to exactly one file.
1593	let (run_id, log_ref): (i64, String) = sqlx::query_as(
1594	"SELECT id, log_ref FROM step_runs WHERE step = 'build' AND target_run_id IN \
1595	(SELECT id FROM target_runs WHERE build_id = ?)",
1596	)
1597	.bind(build_id)
1598	.fetch_one(&pool)
1599	.await
1600	.unwrap();
1601	let log = std::path::PathBuf::from(&log_ref);
1602	assert_eq!(
1603	log,
1604	state
1605	.cfg
1606	.logs_root
1607	.join(format!("demo/0.0.1/linux-x86_64/build.{run_id}.log")),
1608	"log path should be keyed on the step run id"
1609	);
1610	assert!(log.exists(), "build step log should exist");
1611	let body = std::fs::read_to_string(&log).unwrap();
1612	assert!(body.contains("compiling"));
1613	assert!(
1614	body.starts_with(&format!(
1615	"=== bento demo 0.0.1 linux/x86_64 step=build run_id={run_id} "
1616	)),
1617	"log should open with a run header naming its run: {body}"
1618	);
1619	}
1620
1621	/// Multi-target fan-out, which is what the daemon exists for: every other
1622	/// runner test drives exactly one target, so nothing covered `start_build`'s
1623	/// `JoinSet` fan-out or `finalize_build`'s rollup. One target fails and one
1624	/// succeeds — the failure must not abort its sibling, both must land their
1625	/// own terminal row, and the build must finalize `failed` because any failed
1626	/// target fails the build.
1627	#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1628	async fn multi_target_fan_out_rolls_up_partial_failure() {
1629	let tmp = tempfile::tempdir().unwrap();
1630	let root = tmp.path();
1631
1632	let repo = root.join("app");
1633	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1634	std::fs::write(
1635	repo.join("src-tauri/tauri.conf.json"),
1636	r#"{"version":"0.0.1"}"#,
1637	)
1638	.unwrap();
1639	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1640
1641	// Linux succeeds and collects a real artifact. The artifact is the proof
1642	// this target ran to completion rather than being torn down when its
1643	// sibling failed.
1644	std::fs::write(
1645	repo.join("dist/recipes/linux.rhai"),
1646	r#"
1647	step("build");
1648	let v = version_of("demo");
1649	sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
1650	step("collect");
1651	collect("fw13", "REPO/out/demo.bin", "demo", v);
1652	"#
1653	.replace("REPO", repo.to_str().unwrap()),
1654	)
1655	.unwrap();
1656	// Windows fails in its first step.
1657	std::fs::write(
1658	repo.join("dist/recipes/windows.rhai"),
1659	r#"
1660	step("build");
1661	sh_ok("winbox", "echo nope 1>&2; exit 1");
1662	"#,
1663	)
1664	.unwrap();
1665
1666	let cfg = Config::for_tests(root);
1667	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1668	std::fs::write(
1669	repo.join("bento.toml"),
1670	"targets = [\"linux/x86_64\", \"windows/x86_64\"]\n",
1671	)
1672	.unwrap();
1673	// Two hosts so each target resolves its own, mirroring the real
1674	// per-architecture topology.
1675	let topo = Topology::from_str_for_tests(&format!(
1676	r#"
1677	[[host]]
1678	name = "fw13"
1679	ssh = "local"
1680	targets = ["linux/x86_64"]
1681	pull_root = "{repo}"
1682
1683	[[host]]
1684	name = "winbox"
1685	ssh = "local"
1686	targets = ["windows/x86_64"]
1687
1688	[app.demo]
1689	repo = "{repo}"
1690	"#,
1691	repo = repo.display()
1692	))
1693	.unwrap();
1694
1695	let state = test_state(pool.clone(), topo, cfg);
1696
1697	let build_id = start_build(
1698	state.clone(),
1699	AppId::new("demo"),
1700	Version::parse("0.0.1").unwrap(),
1701	vec![
1702	"linux/x86_64".parse().unwrap(),
1703	"windows/x86_64".parse().unwrap(),
1704	],
1705	)
1706	.await
1707	.unwrap();
1708
1709	// Poll the build row, not the target rows: the build is terminal only
1710	// once finalize_build has joined every task and stamped it.
1711	let mut build_status = String::new();
1712	for _ in 0..200 {
1713	build_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
1714	.bind(build_id)
1715	.fetch_one(&pool)
1716	.await
1717	.unwrap();
1718	if build_status != "running" {
1719	break;
1720	}
1721	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1722	}
1723
1724	let runs: Vec<(String, String)> = sqlx::query_as(
1725	"SELECT target, status FROM target_runs WHERE build_id = ? ORDER BY target",
1726	)
1727	.bind(build_id)
1728	.fetch_all(&pool)
1729	.await
1730	.unwrap();
1731	assert_eq!(
1732	runs,
1733	vec![
1734	("linux/x86_64".to_string(), "ok".to_string()),
1735	("windows/x86_64".to_string(), "failed".to_string()),
1736	],
1737	"each target lands its own terminal row; one failing does not take the other down",
1738	);
1739
1740	// The surviving target finished its work, not merely its row.
1741	assert!(
1742	state
1743	.cfg
1744	.dist_root
1745	.join("demo/0.0.1/linux-x86_64/demo.bin")
1746	.exists(),
1747	"the succeeding target ran to completion and collected its artifact",
1748	);
1749
1750	// The failure is attributed to the target that failed, and only it.
1751	let err: Option<String> = sqlx::query_scalar(
1752	"SELECT error FROM target_runs WHERE build_id = ? AND target = 'windows/x86_64'",
1753	)
1754	.bind(build_id)
1755	.fetch_one(&pool)
1756	.await
1757	.unwrap();
1758	assert!(
1759	err.is_some_and(\|e\| !e.is_empty()),
1760	"a failed target records why",
1761	);
1762
1763	assert_eq!(
1764	build_status, "failed",
1765	"any failed target fails the build; a partial release must not read as ok",
1766	);
1767	let finished: Option<String> =
1768	sqlx::query_scalar("SELECT finished_at FROM builds WHERE id = ?")
1769	.bind(build_id)
1770	.fetch_one(&pool)
1771	.await
1772	.unwrap();
1773	assert!(finished.is_some(), "finalize_build stamps the finish time");
1774
1775	// finalize_build reaps its own latest-wins slots, so nothing is left
1776	// in flight to block a later build of the same targets.
1777	assert!(
1778	state.active.lock().await.is_empty(),
1779	"finalize_build reaps the slots it owned",
1780	);
1781	}
1782
1783	/// Latest-wins supersession, driven through `start_build` (not the leaf
1784	/// `begin_step` bail that's already covered). Two builds of the SAME
1785	/// (app, target) race: the second must cooperatively cancel + abort the
1786	/// first and take the single slot, the first must terminate non-`ok`, and
1787	/// the second must run to completion.
1788	#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1789	async fn a_newer_build_supersedes_the_in_flight_one_for_the_same_target() {
1790	let tmp = tempfile::tempdir().unwrap();
1791	let root = tmp.path();
1792
1793	let repo = root.join("app");
1794	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1795	std::fs::write(
1796	repo.join("src-tauri/tauri.conf.json"),
1797	r#"{"version":"0.0.1"}"#,
1798	)
1799	.unwrap();
1800	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1801	// A sleep long enough that the first build is still mid-recipe when the
1802	// second arrives; the engine checks the cancel flag at the step boundary
1803	// before "done", so the superseded run bails rather than finishing.
1804	std::fs::write(
1805	repo.join("dist/recipes/linux.rhai"),
1806	r#"
1807	step("build");
1808	sh_ok("fw13", "sleep 2");
1809	"#,
1810	)
1811	.unwrap();
1812
1813	let cfg = Config::for_tests(root);
1814	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1815	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1816	let topo = Topology::from_str_for_tests(&format!(
1817	r#"
1818	[[host]]
1819	name = "fw13"
1820	ssh = "local"
1821	targets = ["linux/x86_64"]
1822
1823	[app.demo]
1824	repo = "{}"
1825	"#,
1826	repo.display()
1827	))
1828	.unwrap();
1829	let state = test_state(pool.clone(), topo, cfg);
1830
1831	let target = "linux/x86_64".parse().unwrap();
1832	let first = start_build(
1833	state.clone(),
1834	AppId::new("demo"),
1835	Version::parse("0.0.1").unwrap(),
1836	vec![target],
1837	)
1838	.await
1839	.unwrap();
1840	// Let the first build register its slot and enter the recipe before the
1841	// second supersedes it (start_build inserts the slot synchronously, but
1842	// the recipe runs on a spawned task).
1843	tokio::time::sleep(std::time::Duration::from_millis(100)).await;
1844	let second = start_build(
1845	state.clone(),
1846	AppId::new("demo"),
1847	Version::parse("0.0.1").unwrap(),
1848	vec![target],
1849	)
1850	.await
1851	.unwrap();
1852	assert_ne!(first, second);
1853
1854	// Latest wins: exactly one slot for the key, owned by the second build.
1855	{
1856	let active = state.active.lock().await;
1857	assert_eq!(active.len(), 1, "supersession must not leave two slots");
1858	assert_eq!(
1859	active.values().next().unwrap().build_id,
1860	second,
1861	"the surviving slot belongs to the newer build",
1862	);
1863	}
1864
1865	// Wait for the second (surviving) build to finalize.
1866	let mut second_status = String::new();
1867	for _ in 0..200 {
1868	second_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
1869	.bind(second)
1870	.fetch_one(&pool)
1871	.await
1872	.unwrap();
1873	if second_status != "running" {
1874	break;
1875	}
1876	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1877	}
1878	assert_eq!(
1879	second_status, "ok",
1880	"the superseding build runs to completion"
1881	);
1882
1883	// The first build's target run terminated without succeeding — it was
1884	// cancelled/aborted, never stamped `ok`.
1885	let first_status: String =
1886	sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1887	.bind(first)
1888	.fetch_one(&pool)
1889	.await
1890	.unwrap();
1891	assert_ne!(
1892	first_status, "ok",
1893	"the superseded build must not complete successfully",
1894	);
1895
1896	// Both builds reaped their slots; nothing left in flight.
1897	assert!(
1898	state.active.lock().await.is_empty(),
1899	"every finalized build reaps its own slot",
1900	);
1901	}
1902
1903	/// A failing `preflight` (the agent host's `/health` probe when `ops-agent`
1904	/// is down) must fail the target BEFORE the recipe dispatches — the whole
1905	/// point of preflighting. Covered only via a fake here, since a real
1906	/// LocalExec/SshExec preflight is a no-op that always passes.
1907	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1908	async fn a_failing_preflight_fails_the_target_before_the_recipe_runs() {
1909	let tmp = tempfile::tempdir().unwrap();
1910	let root = tmp.path();
1911
1912	let repo = root.join("app");
1913	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1914	std::fs::write(
1915	repo.join("src-tauri/tauri.conf.json"),
1916	r#"{"version":"0.0.1"}"#,
1917	)
1918	.unwrap();
1919	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1920	// If the recipe ran it would drop this marker; preflight failing first
1921	// means it never does.
1922	let marker = root.join("recipe-ran");
1923	std::fs::write(
1924	repo.join("dist/recipes/linux.rhai"),
1925	r#"
1926	step("build");
1927	sh_ok("fw13", "touch MARKER");
1928	"#
1929	.replace("MARKER", marker.to_str().unwrap()),
1930	)
1931	.unwrap();
1932
1933	let cfg = Config::for_tests(root);
1934	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1935	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1936	let topo = Topology::from_str_for_tests(&format!(
1937	r#"
1938	[[host]]
1939	name = "fw13"
1940	ssh = "local"
1941	targets = ["linux/x86_64"]
1942
1943	[app.demo]
1944	repo = "{}"
1945	"#,
1946	repo.display()
1947	))
1948	.unwrap();
1949	let mut state = test_state(pool.clone(), topo, cfg);
1950	// Swap fw13's executor for one whose preflight fails.
1951	let mut execs = HashMap::new();
1952	execs.insert(
1953	"fw13".to_string(),
1954	FakeExec::preflight_fails("ops-agent not reachable at /health"),
1955	);
1956	state.executors = Arc::new(execs);
1957
1958	let build_id = start_build(
1959	state.clone(),
1960	AppId::new("demo"),
1961	Version::parse("0.0.1").unwrap(),
1962	vec!["linux/x86_64".parse().unwrap()],
1963	)
1964	.await
1965	.unwrap();
1966
1967	let mut status = String::new();
1968	let mut error = String::new();
1969	for _ in 0..100 {
1970	let row: Option<(String, Option<String>)> =
1971	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1972	.bind(build_id)
1973	.fetch_optional(&pool)
1974	.await
1975	.unwrap();
1976	if let Some((s, e)) = row {
1977	status = s;
1978	error = e.unwrap_or_default();
1979	if status != "running" {
1980	break;
1981	}
1982	}
1983	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1984	}
1985	assert_eq!(status, "failed", "a failed preflight must fail the target");
1986	assert!(
1987	error.contains("ops-agent not reachable"),
1988	"the failure must carry the preflight error, got: {error}"
1989	);
1990	assert!(
1991	!marker.exists(),
1992	"the recipe must NOT run when preflight fails"
1993	);
1994	}
1995
1996	/// A target whose recipe file is absent must fail at the `checkout` boundary
1997	/// with a readable "reading recipe" error, before any step runs — exercising
1998	/// `read_recipe`'s error branch and the `fail_target` path in `run_target`
1999	/// that neither the happy-path nor the failing-preflight test reaches. The
2000	/// app is configured to ship linux, but no `linux.rhai` is written.
2001	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2002	async fn a_target_with_no_recipe_file_fails_at_checkout() {
2003	let tmp = tempfile::tempdir().unwrap();
2004	let root = tmp.path();
2005
2006	let repo = root.join("app");
2007	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2008	std::fs::write(
2009	repo.join("src-tauri/tauri.conf.json"),
2010	r#"{"version":"0.0.1"}"#,
2011	)
2012	.unwrap();
2013	// The recipe dir exists but is empty — no linux.rhai.
2014	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2015
2016	let cfg = Config::for_tests(root);
2017	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2018	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2019	let topo = Topology::from_str_for_tests(&format!(
2020	r#"
2021	[[host]]
2022	name = "fw13"
2023	ssh = "local"
2024	targets = ["linux/x86_64"]
2025
2026	[app.demo]
2027	repo = "{}"
2028	"#,
2029	repo.display()
2030	))
2031	.unwrap();
2032	let state = test_state(pool.clone(), topo, cfg);
2033
2034	let build_id = start_build(
2035	state.clone(),
2036	AppId::new("demo"),
2037	Version::parse("0.0.1").unwrap(),
2038	vec!["linux/x86_64".parse().unwrap()],
2039	)
2040	.await
2041	.unwrap();
2042
2043	// Poll the build row: it is terminal only once finalize_build joins the
2044	// one (failing) target task.
2045	let mut build_status = String::new();
2046	for _ in 0..100 {
2047	build_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
2048	.bind(build_id)
2049	.fetch_one(&pool)
2050	.await
2051	.unwrap();
2052	if build_status != "running" {
2053	break;
2054	}
2055	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2056	}
2057
2058	// The target failed, and the error points at the unreadable recipe.
2059	let (status, error): (String, Option<String>) =
2060	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2061	.bind(build_id)
2062	.fetch_one(&pool)
2063	.await
2064	.unwrap();
2065	assert_eq!(status, "failed", "a missing recipe must fail the target");
2066	assert!(
2067	error.is_some_and(\|e\| e.contains("reading recipe")),
2068	"the failure must name the recipe it could not read",
2069	);
2070
2071	// The recipe never ran, so no step_runs row was ever created — the
2072	// failure is at the checkout boundary, ahead of any step.
2073	let step_count: i64 = sqlx::query_scalar(
2074	"SELECT COUNT(*) FROM step_runs WHERE target_run_id IN \
2075	(SELECT id FROM target_runs WHERE build_id = ?)",
2076	)
2077	.bind(build_id)
2078	.fetch_one(&pool)
2079	.await
2080	.unwrap();
2081	assert_eq!(
2082	step_count, 0,
2083	"no step should run when the recipe is absent"
2084	);
2085
2086	assert_eq!(
2087	build_status, "failed",
2088	"the build rolls up the target failure",
2089	);
2090	assert!(
2091	state.active.lock().await.is_empty(),
2092	"finalize_build reaps the slot even on the recipe-read failure path",
2093	);
2094	}
2095
2096	/// Run 2 S5: a publish whose version is not strictly newer than the latest
2097	/// already published for the same (app, target, channel) is refused — an
2098	/// older build cannot republish over a live newer release.
2099	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2100	async fn publish_rejects_a_non_monotonic_version() {
2101	let tmp = tempfile::tempdir().unwrap();
2102	let root = tmp.path();
2103	let repo = root.join("app");
2104	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2105	std::fs::write(
2106	repo.join("src-tauri/tauri.conf.json"),
2107	r#"{"version":"0.2.0"}"#,
2108	)
2109	.unwrap();
2110	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2111	let artifact = repo.join("out/app.tar.gz");
2112	// Build an artifact, publish 0.2.0 (records a release), then try to
2113	// publish the older 0.1.0 — the second publish must fail.
2114	std::fs::write(
2115	repo.join("dist/recipes/linux.rhai"),
2116	r#"
2117	step("build");
2118	sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
2119	step("publish");
2120	publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
2121	publish("tauri-mnw", "demo", "linux/x86_64", "0.1.0", "ARTIFACT", #{});
2122	"#
2123	.replace("ARTIFACT", artifact.to_str().unwrap())
2124	.replace("REPO", repo.to_str().unwrap()),
2125	)
2126	.unwrap();
2127
2128	let cfg = Config::for_tests(root);
2129	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2130	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2131	let topo = Topology::from_str_for_tests(&format!(
2132	r#"
2133	[[host]]
2134	name = "fw13"
2135	ssh = "local"
2136	targets = ["linux/x86_64"]
2137
2138	[app.demo]
2139	repo = "{}"
2140	"#,
2141	repo.display()
2142	))
2143	.unwrap();
2144	let state = test_state(pool.clone(), topo, cfg);
2145	let build_id = start_build(
2146	state.clone(),
2147	AppId::new("demo"),
2148	Version::parse("0.2.0").unwrap(),
2149	vec!["linux/x86_64".parse().unwrap()],
2150	)
2151	.await
2152	.unwrap();
2153
2154	let mut status = String::new();
2155	let mut error = String::new();
2156	for _ in 0..100 {
2157	let row: Option<(String, Option<String>)> =
2158	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2159	.bind(build_id)
2160	.fetch_optional(&pool)
2161	.await
2162	.unwrap();
2163	if let Some((s, e)) = row {
2164	status = s;
2165	error = e.unwrap_or_default();
2166	if status != "running" {
2167	break;
2168	}
2169	}
2170	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2171	}
2172	assert_eq!(status, "failed", "non-monotonic publish must fail the run");
2173	assert!(
2174	error.contains("not newer"),
2175	"expected monotonicity error, got: {error}"
2176	);
2177	// Exactly one release was recorded (0.2.0); 0.1.0 never landed.
2178	let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
2179	.fetch_one(&pool)
2180	.await
2181	.unwrap();
2182	assert_eq!(
2183	count, 1,
2184	"only the first (newer) publish should record a release"
2185	);
2186	// The recorded release carries the artifact's sha256 (64 hex chars),
2187	// not a NULL — the ledger says which bytes shipped.
2188	let hash: Option<String> =
2189	sqlx::query_scalar("SELECT artifact_hash FROM releases WHERE version = '0.2.0'")
2190	.fetch_one(&pool)
2191	.await
2192	.unwrap();
2193	assert!(
2194	hash.as_deref().is_some_and(\|h\| h.len() == 64),
2195	"publish must record the artifact sha256, got {hash:?}"
2196	);
2197	}
2198
2199	/// The artifact-identity fix at collect: a stale, wrongly-versioned artifact
2200	/// left in the output dir fails the collect instead of silently winning a
2201	/// later glob. Here the build is 0.0.1 but the recipe produces a 9.9.9 file.
2202	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2203	async fn collect_rejects_a_stale_versioned_artifact() {
2204	let tmp = tempfile::tempdir().unwrap();
2205	let root = tmp.path();
2206	let repo = root.join("app");
2207	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2208	std::fs::write(
2209	repo.join("src-tauri/tauri.conf.json"),
2210	r#"{"version":"0.0.1"}"#,
2211	)
2212	.unwrap();
2213	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2214	std::fs::write(
2215	repo.join("dist/recipes/linux.rhai"),
2216	r#"
2217	step("build");
2218	let v = version_of("demo");
2219	sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo-9.9.9.bin");
2220	step("collect");
2221	collect("fw13", "REPO/out/demo-9.9.9.bin", "demo", v);
2222	"#
2223	.replace("REPO", repo.to_str().unwrap()),
2224	)
2225	.unwrap();
2226
2227	let cfg = Config::for_tests(root);
2228	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2229	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2230	let topo = Topology::from_str_for_tests(&format!(
2231	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2232	pull_root = \"{repo}\"\n\n[app.demo]\nrepo = \"{repo}\"\n",
2233	repo = repo.display()
2234	))
2235	.unwrap();
2236	let state = test_state(pool.clone(), topo, cfg);
2237	let build_id = start_build(
2238	state.clone(),
2239	AppId::new("demo"),
2240	Version::parse("0.0.1").unwrap(),
2241	vec!["linux/x86_64".parse().unwrap()],
2242	)
2243	.await
2244	.unwrap();
2245
2246	let mut status = String::new();
2247	let mut error = String::new();
2248	for _ in 0..100 {
2249	let row: Option<(String, Option<String>)> =
2250	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2251	.bind(build_id)
2252	.fetch_optional(&pool)
2253	.await
2254	.unwrap();
2255	if let Some((s, e)) = row {
2256	status = s;
2257	error = e.unwrap_or_default();
2258	if status != "running" {
2259	break;
2260	}
2261	}
2262	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2263	}
2264	assert_eq!(
2265	status, "failed",
2266	"a mismatched-version artifact must fail collect"
2267	);
2268	assert!(
2269	error.contains("stale artifact"),
2270	"expected a stale-artifact error, got: {error}"
2271	);
2272	}
2273
2274	/// A command that runs past its step's deadline fails THAT step (and unwinds
2275	/// the recipe) rather than wedging under the old whole-build guillotine. The
2276	/// per-step budget is overridden to 1s here; the recipe then sleeps 30s, so
2277	/// the step deadline — not the sleep — decides the outcome, quickly.
2278	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2279	async fn a_step_that_exceeds_its_deadline_fails() {
2280	let tmp = tempfile::tempdir().unwrap();
2281	let root = tmp.path();
2282	let repo = root.join("app");
2283	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2284	std::fs::write(
2285	repo.join("src-tauri/tauri.conf.json"),
2286	r#"{"version":"0.0.1"}"#,
2287	)
2288	.unwrap();
2289	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2290	std::fs::write(
2291	repo.join("dist/recipes/linux.rhai"),
2292	r#"
2293	step("build");
2294	sh_ok("fw13", "sleep 30");
2295	"#,
2296	)
2297	.unwrap();
2298
2299	let mut cfg = Config::for_tests(root);
2300	cfg.step_timeout_secs = Some(1); // every step's budget -> 1s
2301	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2302	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2303	let topo = Topology::from_str_for_tests(&format!(
2304	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2305	[app.demo]\nrepo = \"{}\"\n",
2306	repo.display()
2307	))
2308	.unwrap();
2309	let state = test_state(pool.clone(), topo, cfg);
2310	let started = std::time::Instant::now();
2311	let build_id = start_build(
2312	state.clone(),
2313	AppId::new("demo"),
2314	Version::parse("0.0.1").unwrap(),
2315	vec!["linux/x86_64".parse().unwrap()],
2316	)
2317	.await
2318	.unwrap();
2319
2320	let mut status = String::new();
2321	let mut error = String::new();
2322	for _ in 0..100 {
2323	let row: Option<(String, Option<String>)> =
2324	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2325	.bind(build_id)
2326	.fetch_optional(&pool)
2327	.await
2328	.unwrap();
2329	if let Some((s, e)) = row {
2330	status = s;
2331	error = e.unwrap_or_default();
2332	if status != "running" {
2333	break;
2334	}
2335	}
2336	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2337	}
2338	assert_eq!(status, "failed", "a step past its deadline must fail");
2339	assert!(
2340	error.contains("per-step deadline"),
2341	"expected a deadline error, got: {error}"
2342	);
2343	// The deadline (1s), not the 30s sleep, decided it — proof the command
2344	// was actually interrupted rather than run to completion.
2345	assert!(
2346	started.elapsed() < std::time::Duration::from_secs(20),
2347	"the step deadline must fire well before the sleep would finish"
2348	);
2349	}
2350
2351	/// A demo app that publishes linux and has the all-targets-green gate on; it
2352	/// declares linux + macos, so publishing linux is gated on macos being green.
2353	async fn gate_state(root: &std::path::Path) -> (AppState, sqlx::SqlitePool) {
2354	let repo = root.join("app");
2355	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2356	std::fs::write(
2357	repo.join("src-tauri/tauri.conf.json"),
2358	r#"{"version":"0.2.0"}"#,
2359	)
2360	.unwrap();
2361	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2362	let artifact = repo.join("out/app.tar.gz");
2363	std::fs::write(
2364	repo.join("dist/recipes/linux.rhai"),
2365	r#"
2366	step("build");
2367	sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
2368	step("publish");
2369	publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
2370	"#
2371	.replace("ARTIFACT", artifact.to_str().unwrap())
2372	.replace("REPO", repo.to_str().unwrap()),
2373	)
2374	.unwrap();
2375	std::fs::write(
2376	repo.join("bento.toml"),
2377	"targets = [\"linux/x86_64\", \"macos/aarch64\"]\nrequire_all_targets = true\n",
2378	)
2379	.unwrap();
2380	let cfg = Config::for_tests(root);
2381	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2382	let topo = Topology::from_str_for_tests(&format!(
2383	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2384	[[host]]\nname = \"mbp\"\nssh = \"mbp\"\ntargets = [\"macos/aarch64\"]\n\
2385	[app.demo]\nrepo = \"{}\"\n",
2386	repo.display()
2387	))
2388	.unwrap();
2389	(test_state(pool.clone(), topo, cfg), pool)
2390	}
2391
2392	async fn await_target(pool: &sqlx::SqlitePool, build_id: i64) -> (String, String) {
2393	for _ in 0..100 {
2394	let row: Option<(String, Option<String>)> =
2395	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2396	.bind(build_id)
2397	.fetch_optional(pool)
2398	.await
2399	.unwrap();
2400	if let Some((s, e)) = row
2401	&& s != "running"
2402	{
2403	return (s, e.unwrap_or_default());
2404	}
2405	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2406	}
2407	panic!("target never settled");
2408	}
2409
2410	/// The all-targets-green gate blocks a partial release: linux tries to
2411	/// publish while macos has no successful run, so publish is refused and the
2412	/// target fails at its publish step.
2413	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2414	async fn all_green_gate_blocks_publish_when_a_sibling_is_not_green() {
2415	let tmp = tempfile::tempdir().unwrap();
2416	let (state, pool) = gate_state(tmp.path()).await;
2417	let build_id = start_build(
2418	state,
2419	AppId::new("demo"),
2420	Version::parse("0.2.0").unwrap(),
2421	vec!["linux/x86_64".parse().unwrap()],
2422	)
2423	.await
2424	.unwrap();
2425	let (status, error) = await_target(&pool, build_id).await;
2426	assert_eq!(status, "failed", "a partial release must be blocked");
2427	assert!(
2428	error.contains("all-targets-green gate"),
2429	"expected the gate to name itself, got: {error}"
2430	);
2431	// Nothing shipped.
2432	let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
2433	.fetch_one(&pool)
2434	.await
2435	.unwrap();
2436	assert_eq!(count, 0, "a gated-off publish records no release");
2437	}
2438
2439	/// With every sibling green, the gate lets the publish through. macos is
2440	/// pre-recorded `ok` for this version, so linux's publish proceeds.
2441	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2442	async fn all_green_gate_allows_publish_when_every_sibling_is_green() {
2443	let tmp = tempfile::tempdir().unwrap();
2444	let (state, pool) = gate_state(tmp.path()).await;
2445	// Pre-record a green macos run at 0.2.0 (as if its target already built).
2446	let bid: i64 = sqlx::query_scalar(
2447	"INSERT INTO builds (app, version, status, created_at) VALUES ('demo','0.2.0','ok','2026-07-23T00:00:00Z') RETURNING id",
2448	)
2449	.fetch_one(&pool)
2450	.await
2451	.unwrap();
2452	sqlx::query(
2453	"INSERT INTO target_runs (build_id, app, version, target, status, started_at)
2454	VALUES (?, 'demo', '0.2.0', 'macos/aarch64', 'ok', '2026-07-23T00:00:00Z')",
2455	)
2456	.bind(bid)
2457	.execute(&pool)
2458	.await
2459	.unwrap();
2460
2461	let build_id = start_build(
2462	state,
2463	AppId::new("demo"),
2464	Version::parse("0.2.0").unwrap(),
2465	vec!["linux/x86_64".parse().unwrap()],
2466	)
2467	.await
2468	.unwrap();
2469	let (status, error) = await_target(&pool, build_id).await;
2470	assert_eq!(
2471	status, "ok",
2472	"all siblings green -> publish proceeds ({error})"
2473	);
2474	let count: i64 =
2475	sqlx::query_scalar("SELECT COUNT(*) FROM releases WHERE target = 'linux/x86_64'")
2476	.fetch_one(&pool)
2477	.await
2478	.unwrap();
2479	assert_eq!(count, 1, "linux published once the gate was satisfied");
2480	}
2481
2482	/// A committed + tagged git repo whose linux recipe pins the host to the tag
2483	/// via `checkout_sha`. `tag` is created only when `Some`.
2484	fn init_git_app(repo: &std::path::Path, tauri_version: &str, tag: Option<&str>) {
2485	init_git_app_with_recipe(
2486	repo,
2487	tauri_version,
2488	tag,
2489	"step(\"checkout\");\nlet s = checkout_sha(build_host());\nlog(\"pinned \" + s);\n\
2490	step(\"build\");\nsh_ok(build_host(), \"true\");\n",
2491	);
2492	}
2493
2494	/// As [`init_git_app`], with the linux recipe spelled by the caller.
2495	fn init_git_app_with_recipe(
2496	repo: &std::path::Path,
2497	tauri_version: &str,
2498	tag: Option<&str>,
2499	recipe: &str,
2500	) {
2501	init_git_app_shipping(repo, tauri_version, tag, recipe, "[\"linux/x86_64\"]");
2502	}
2503
2504	/// As [`init_git_app_with_recipe`], with the manifest's target list spelled
2505	/// by the caller — a second target is what puts a second HOST in the
2506	/// preflight, which is the only way to reach its unwind path.
2507	fn init_git_app_shipping(
2508	repo: &std::path::Path,
2509	tauri_version: &str,
2510	tag: Option<&str>,
2511	recipe: &str,
2512	targets: &str,
2513	) {
2514	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2515	std::fs::write(
2516	repo.join("src-tauri/tauri.conf.json"),
2517	format!("{{\"version\":\"{tauri_version}\"}}"),
2518	)
2519	.unwrap();
2520	std::fs::write(repo.join("bento.toml"), format!("targets = {targets}\n")).unwrap();
2521	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2522	std::fs::write(repo.join("dist/recipes/linux.rhai"), recipe).unwrap();
2523	// Isolate from the dev's global git config (which forces signed tags).
2524	let run = \|args: &[&str]\| {
2525	let out = std::process::Command::new("git")
2526	.args(args)
2527	.current_dir(repo)
2528	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2529	.env("GIT_CONFIG_SYSTEM", "/dev/null")
2530	.output()
2531	.expect("git runs");
2532	assert!(
2533	out.status.success(),
2534	"git {args:?}: {}",
2535	String::from_utf8_lossy(&out.stderr)
2536	);
2537	};
2538	run(&["init", "-q"]);
2539	run(&["-c", "user.email=t@t", "-c", "user.name=t", "add", "-A"]);
2540	run(&[
2541	"-c",
2542	"user.email=t@t",
2543	"-c",
2544	"user.name=t",
2545	"commit",
2546	"-q",
2547	"-m",
2548	"init",
2549	]);
2550	if let Some(t) = tag {
2551	run(&["tag", t]);
2552	}
2553	}
2554
2555	/// A worktree root beside the checkout, the shape production uses: a hidden
2556	/// directory in the tree the repos live in, not inside any of them.
2557	fn worktree_root(repo: &std::path::Path) -> std::path::PathBuf {
2558	repo.parent().expect("repo has a parent").join(".bento")
2559	}
2560
2561	/// Where a release of `demo` out of `repo` builds: the worktree, plus the
2562	/// app's own prefix inside it for a repo holding several products.
2563	fn build_dir(repo: &std::path::Path, prefix: &str) -> std::path::PathBuf {
2564	let repo_dir = repo.file_name().expect("repo has a name");
2565	worktree_root(repo).join(repo_dir).join("demo").join(prefix)
2566	}
2567
2568	fn one_host_topo(repo: &std::path::Path) -> Topology {
2569	Topology::from_str_for_tests(&format!(
2570	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2571	worktree_root = \"{}\"\n\
2572	[app.demo]\nrepo = \"{}\"\n",
2573	worktree_root(repo).display(),
2574	repo.display()
2575	))
2576	.unwrap()
2577	}
2578
2579	/// The release preflight (pin on) pins the host to the tag and the build
2580	/// proceeds. Exercises both the barrier and the `checkout_sha` host function.
2581	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2582	async fn release_preflight_pins_and_builds_when_the_tag_is_present() {
2583	let tmp = tempfile::tempdir().unwrap();
2584	let repo = tmp.path().join("demo");
2585	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2586	let mut cfg = Config::for_tests(tmp.path());
2587	cfg.pin_release_sha = true;
2588	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2589	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2590	let build_id = start_build(
2591	state,
2592	AppId::new("demo"),
2593	Version::parse("0.0.1").unwrap(),
2594	vec!["linux/x86_64".parse().unwrap()],
2595	)
2596	.await
2597	.unwrap();
2598	let (status, error) = await_target(&pool, build_id).await;
2599	assert_eq!(status, "ok", "a pinned build should succeed ({error})");
2600	assert!(
2601	build_dir(&repo, "")
2602	.join("src-tauri/tauri.conf.json")
2603	.exists(),
2604	"the release must have built in the worktree, not the checkout"
2605	);
2606	assert_eq!(
2607	head_sha(&build_dir(&repo, "")),
2608	tag_sha(&repo, "v0.0.1"),
2609	"and that worktree must be at the release tag"
2610	);
2611	}
2612
2613	/// A pinned build writes an artifact record beside what it collected: the
2614	/// manifest of those bytes, the commit the preflight pinned, and the steps
2615	/// as artifact-scoped gates.
2616	///
2617	/// The three facts already existed and met nowhere — the per-file sha256 at
2618	/// `collect`, the sha at the preflight, the step outcomes in `step_runs` —
2619	/// which is how gates came to vouch for one thing while the deploy shipped
2620	/// another.
2621	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2622	async fn a_pinned_build_writes_an_artifact_record_for_what_it_collected() {
2623	let tmp = tempfile::tempdir().unwrap();
2624	let repo = tmp.path().join("demo");
2625	std::fs::create_dir_all(&repo).unwrap();
2626	// The recipe builds and collects out of `repo()`, which is the worktree.
2627	// That also proves the collect: an artifact built in a tree the sync gate
2628	// does not trust is refused, and the worktree root is an artifact root
2629	// for exactly this reason (`Host::artifact_roots`).
2630	let recipe = r#"
2631	step("checkout");
2632	let sha = checkout_sha(build_host());
2633	log("pinned " + sha);
2634	step("build");
2635	sh_ok(build_host(), "mkdir -p " + repo() + "/out && echo bin > " + repo() + "/out/demo.bin");
2636	step("collect");
2637	collect(build_host(), repo() + "/out/demo.bin", "demo", version());
2638	"#;
2639	init_git_app_with_recipe(&repo, "0.0.1", Some("v0.0.1"), recipe);
2640
2641	let mut cfg = Config::for_tests(tmp.path());
2642	cfg.pin_release_sha = true;
2643	let dist_root = cfg.dist_root.clone();
2644	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2645	// No `pull_root`: the worktree root is the artifact root here, which is
2646	// the arrangement production uses.
2647	let topo = Topology::from_str_for_tests(&format!(
2648	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2649	worktree_root = \"{wt}\"\n[app.demo]\nrepo = \"{repo}\"\n",
2650	wt = worktree_root(&repo).display(),
2651	repo = repo.display()
2652	))
2653	.unwrap();
2654	let state = test_state(pool.clone(), topo, cfg);
2655	let build_id = start_build(
2656	state,
2657	AppId::new("demo"),
2658	Version::parse("0.0.1").unwrap(),
2659	vec!["linux/x86_64".parse().unwrap()],
2660	)
2661	.await
2662	.unwrap();
2663	let (status, error) = await_target(&pool, build_id).await;
2664	assert_eq!(status, "ok", "the build should succeed ({error})");
2665
2666	let path = crate::artifact_record::record_path(
2667	&dist_root,
2668	&AppId::new("demo"),
2669	&Version::parse("0.0.1").unwrap(),
2670	"linux/x86_64".parse().unwrap(),
2671	);
2672	let json = std::fs::read_to_string(&path)
2673	.unwrap_or_else(\|e\| panic!("record at {}: {e}", path.display()));
2674	// Parsing revalidates, so this also asserts the digest matches the
2675	// manifest and no environment-scoped gate slipped in.
2676	let record = ops_artifact::ArtifactRecord::parse(&json).unwrap();
2677
2678	assert_eq!(record.producer, "bento");
2679	assert_eq!(record.manifest.entries().len(), 1);
2680	assert_eq!(record.manifest.entries()[0].path, "demo.bin");
2681	assert_eq!(record.digest, record.manifest.digest());
2682
2683	// The provenance names the commit the preflight pinned, not a rebuild of
2684	// whatever the branch is now.
2685	let head = std::process::Command::new("git")
2686	.args(["rev-parse", "v0.0.1^{commit}"])
2687	.current_dir(&repo)
2688	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2689	.output()
2690	.unwrap();
2691	let head = String::from_utf8_lossy(&head.stdout).trim().to_string();
2692	assert_eq!(record.provenance.git_sha, head);
2693	assert_eq!(record.provenance.target, "linux/x86_64");
2694	assert_eq!(record.provenance.build_host, "fw13");
2695	assert!(!record.provenance.toolchain.is_empty());
2696
2697	let gates: Vec<&str> = record.gates.iter().map(\|g\| g.gate.as_str()).collect();
2698	assert_eq!(gates, ["checkout", "build", "collect"]);
2699	assert!(record.all_gates_passed());
2700	assert!(
2701	record
2702	.gates
2703	.iter()
2704	.all(\|g\| g.scope == ops_artifact::Scope::Artifact),
2705	"a build host cannot vouch for an environment"
2706	);
2707	}
2708
2709	/// The barrier refuses a target no host can build, instead of leaving it out
2710	/// of the pin and letting the remaining hosts vouch for the release.
2711	///
2712	/// The hole this closes is silence reading as agreement: the comparison used
2713	/// to run over the hosts that reported, so a target dropped for want of a
2714	/// host still built while the other hosts' unanimity looked like proof the
2715	/// whole release came from one commit.
2716	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2717	async fn release_preflight_refuses_a_target_no_host_can_build() {
2718	let tmp = tempfile::tempdir().unwrap();
2719	let repo = tmp.path().join("demo");
2720	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2721	let mut cfg = Config::for_tests(tmp.path());
2722	cfg.pin_release_sha = true;
2723	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2724	// The topology declares fw13 (linux/x86_64) only, so macOS has no host.
2725	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2726	let err = start_build(
2727	state,
2728	AppId::new("demo"),
2729	Version::parse("0.0.1").unwrap(),
2730	vec!["macos/aarch64".parse().unwrap()],
2731	)
2732	.await
2733	.unwrap_err();
2734	let msg = format!("{err:#}");
2735	assert!(
2736	msg.contains("no host can build macos/aarch64"),
2737	"the error must name the unbuildable target, got: {msg}"
2738	);
2739	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2740	.fetch_one(&pool)
2741	.await
2742	.unwrap();
2743	assert_eq!(builds, 0, "a refused preflight writes no build row");
2744	}
2745
2746	/// A dirty checkout neither refuses a release nor reaches it.
2747	///
2748	/// The preflight used to refuse one, and had to: `git checkout <tag>` does
2749	/// NOT fail on local modifications to files whose content is unchanged in
2750	/// the tag — it succeeds and keeps them, so a dirty host built its edits
2751	/// while honestly reporting the tagged sha. Caught on pom's first release,
2752	/// where fw13 had uncommitted changes in the serve path and astra did not.
2753	///
2754	/// The worktree answers it instead of gating on it: the release is built from
2755	/// a tree the edit is not in. So the edit cannot reach the binary, and
2756	/// refusing a release because somebody has unsaved work is no longer a thing
2757	/// this does.
2758	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2759	async fn a_dirty_checkout_neither_refuses_a_release_nor_reaches_it() {
2760	let tmp = tempfile::tempdir().unwrap();
2761	let repo = tmp.path().join("demo");
2762	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2763
2764	// Modify a TRACKED file, exactly as an editor session would.
2765	let tracked = repo.join("src-tauri/tauri.conf.json");
2766	// Still valid JSON and still version 0.0.1: the version preflight reads
2767	// this file on the daemon's box before anything is pinned, so an edit
2768	// that broke it would fail the release for a reason this test is not
2769	// about.
2770	let edited = "{\"version\":\"0.0.1\",\"unsaved\":true}".to_string();
2771	std::fs::write(&tracked, &edited).unwrap();
2772
2773	let mut cfg = Config::for_tests(tmp.path());
2774	cfg.pin_release_sha = true;
2775	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2776	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2777	let build_id = start_build(
2778	state,
2779	AppId::new("demo"),
2780	Version::parse("0.0.1").unwrap(),
2781	vec!["linux/x86_64".parse().unwrap()],
2782	)
2783	.await
2784	.unwrap();
2785	let (status, error) = await_target(&pool, build_id).await;
2786	assert_eq!(
2787	status, "ok",
2788	"an edit elsewhere must not stop a release ({error})"
2789	);
2790	assert_eq!(
2791	std::fs::read_to_string(&tracked).unwrap(),
2792	edited,
2793	"and the edit must still be there afterwards"
2794	);
2795	assert_ne!(
2796	std::fs::read_to_string(build_dir(&repo, "").join("src-tauri/tauri.conf.json"))
2797	.unwrap(),
2798	edited,
2799	"what was built is the tag's content, not the edit"
2800	);
2801	}
2802
2803	/// An UNTRACKED file does not fail a release. A build host accumulates
2804	/// editor scratch and stray logs, none of which reach the binary, so failing
2805	/// on them would be noise that trains an operator to bypass the gate.
2806	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2807	async fn release_preflight_tolerates_untracked_files() {
2808	let tmp = tempfile::tempdir().unwrap();
2809	let repo = tmp.path().join("demo");
2810	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2811	std::fs::write(repo.join("scratch.log"), "noise").unwrap();
2812
2813	let mut cfg = Config::for_tests(tmp.path());
2814	cfg.pin_release_sha = true;
2815	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2816	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2817	let build_id = start_build(
2818	state,
2819	AppId::new("demo"),
2820	Version::parse("0.0.1").unwrap(),
2821	vec!["linux/x86_64".parse().unwrap()],
2822	)
2823	.await
2824	.unwrap();
2825	let (status, error) = await_target(&pool, build_id).await;
2826	assert_eq!(
2827	status, "ok",
2828	"untracked files must not fail a release ({error})"
2829	);
2830	}
2831
2832	/// The preflight refuses the build (before any row is written) when the
2833	/// release tag does not exist on the host — a missing/unpushed tag can't
2834	/// silently fall back to whatever `main` is.
2835	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2836	async fn release_preflight_refuses_when_the_release_tag_is_missing() {
2837	let tmp = tempfile::tempdir().unwrap();
2838	let repo = tmp.path().join("demo");
2839	// App is 0.0.2 (so the version check passes) but only v0.0.1 is tagged.
2840	init_git_app(&repo, "0.0.2", Some("v0.0.1"));
2841	let mut cfg = Config::for_tests(tmp.path());
2842	cfg.pin_release_sha = true;
2843	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2844	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2845	let err = start_build(
2846	state,
2847	AppId::new("demo"),
2848	Version::parse("0.0.2").unwrap(),
2849	vec!["linux/x86_64".parse().unwrap()],
2850	)
2851	.await
2852	.unwrap_err();
2853	let msg = format!("{err:#}");
2854	assert!(
2855	msg.contains("release preflight") && msg.contains("v0.0.2"),
2856	"expected a preflight tag error, got: {msg}"
2857	);
2858	assert!(
2859	msg.contains("does not exist"),
2860	"the error should name the absent tag as the cause, got: {msg}"
2861	);
2862	// Refused before anything was recorded.
2863	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2864	.fetch_one(&pool)
2865	.await
2866	.unwrap();
2867	assert_eq!(builds, 0, "a refused preflight writes no build row");
2868	}
2869
2870	/// A release does not touch the checkout at all: same branch, same commit,
2871	/// same working tree.
2872	///
2873	/// It used to check the tag out there and put the branch back afterwards,
2874	/// which is a tree somebody works in being moved under them for the length of
2875	/// a build — and when the restore failed, left detached. makeover shipped
2876	/// 2.3.0 from exactly that state on 2026-07-28, the published commit on no
2877	/// branch and no remote. The worktree makes the whole question moot, so this
2878	/// asserts the strong form rather than the recovery.
2879	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2880	async fn a_release_never_moves_the_checkout() {
2881	let tmp = tempfile::tempdir().unwrap();
2882	let repo = tmp.path().join("demo");
2883	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2884	let branch_before = current_branch(&repo);
2885	let head_before = head_sha(&repo);
2886	assert!(!branch_before.is_empty(), "test repo starts on a branch");
2887
2888	let mut cfg = Config::for_tests(tmp.path());
2889	cfg.pin_release_sha = true;
2890	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2891	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2892	let build_id = start_build(
2893	state,
2894	AppId::new("demo"),
2895	Version::parse("0.0.1").unwrap(),
2896	vec!["linux/x86_64".parse().unwrap()],
2897	)
2898	.await
2899	.unwrap();
2900	let (status, error) = await_target(&pool, build_id).await;
2901	assert_eq!(status, "ok", "the build itself should pass ({error})");
2902
2903	assert_eq!(
2904	current_branch(&repo),
2905	branch_before,
2906	"the checkout must still be on its branch"
2907	);
2908	assert_eq!(head_sha(&repo), head_before, "and at the same commit");
2909	assert_eq!(repo_status(&repo), "", "and with the same working tree");
2910	}
2911
2912	/// A detached checkout no longer refuses a release.
2913	///
2914	/// It used to, and had to: a detached tree meant an earlier release never
2915	/// cleaned up, and the release about to start had no branch to put it back
2916	/// on. Releases do not move that tree any more, so its HEAD is not their
2917	/// business — and the state that produced the refusal is one releases can no
2918	/// longer create.
2919	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2920	async fn a_detached_checkout_no_longer_refuses_a_release() {
2921	let tmp = tempfile::tempdir().unwrap();
2922	let repo = tmp.path().join("demo");
2923	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2924	// Detach, standing in for a checkout an earlier release left on its tag.
2925	let out = std::process::Command::new("git")
2926	.args(["checkout", "--detach", "-q", "HEAD"])
2927	.current_dir(&repo)
2928	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2929	.env("GIT_CONFIG_SYSTEM", "/dev/null")
2930	.output()
2931	.expect("git runs");
2932	assert!(out.status.success());
2933	assert!(current_branch(&repo).is_empty(), "repo is detached");
2934
2935	let head_before = head_sha(&repo);
2936
2937	let mut cfg = Config::for_tests(tmp.path());
2938	cfg.pin_release_sha = true;
2939	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2940	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2941	let build_id = start_build(
2942	state,
2943	AppId::new("demo"),
2944	Version::parse("0.0.1").unwrap(),
2945	vec!["linux/x86_64".parse().unwrap()],
2946	)
2947	.await
2948	.unwrap();
2949	let (status, error) = await_target(&pool, build_id).await;
2950	assert_eq!(
2951	status, "ok",
2952	"a detached checkout is not a reason to refuse ({error})"
2953	);
2954	assert!(
2955	current_branch(&repo).is_empty() && head_sha(&repo) == head_before,
2956	"and the release left it exactly as detached as it found it"
2957	);
2958	}
2959
2960	/// A build that rewrites a tracked file does not stop the next release.
2961	///
2962	/// `cargo build` rewrites `Cargo.lock` under the `[patch]` block, so a build
2963	/// leaves the tree it ran in dirty. That used to be the operator's problem:
2964	/// the tree was the ordinary checkout, the branch could not be checked back
2965	/// out over the edit, and the next release was refused for a dirty tree or a
2966	/// detached HEAD. Three releases on astra on 2026-08-23, three manual
2967	/// cleanups, none of the diffs meaning anything.
2968	///
2969	/// Now the dirt lands in Bento's own worktree and the next release forces
2970	/// past it, which is safe precisely because nothing else writes there. So
2971	/// this releases the same app twice with a build that dirties the tree, and
2972	/// the second one is the assertion.
2973	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2974	async fn a_build_that_dirties_its_tree_does_not_refuse_the_next_release() {
2975	let tmp = tempfile::tempdir().unwrap();
2976	let repo = tmp.path().join("demo");
2977	std::fs::create_dir_all(&repo).unwrap();
2978	std::fs::write(repo.join("Cargo.lock"), "version = 4\n").unwrap();
2979	init_git_app_with_recipe(
2980	&repo,
2981	"0.0.1",
2982	Some("v0.0.1"),
2983	"step(\"build\");\nsh_ok(build_host(), \"echo churn >> \" + repo() + \"/Cargo.lock\");\n",
2984	);
2985	// The tag and the branch disagree about the lockfile, which is what made
2986	// the old restore fail rather than silently carry the edit across.
2987	let git = git_in(&repo);
2988	std::fs::write(repo.join("Cargo.lock"), "version = 4\n# moved on\n").unwrap();
2989	git(&["add", "-A"]);
2990	git(&["commit", "-q", "-m", "lock moves on"]);
2991
2992	let mut cfg = Config::for_tests(tmp.path());
2993	cfg.pin_release_sha = true;
2994	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2995	let topo = one_host_topo(&repo);
2996
2997	for attempt in 1..=2 {
2998	let state = test_state(pool.clone(), topo.clone(), cfg.clone());
2999	let build_id = start_build(
3000	state,
3001	AppId::new("demo"),
3002	Version::parse("0.0.1").unwrap(),
3003	vec!["linux/x86_64".parse().unwrap()],
3004	)
3005	.await
3006	.unwrap_or_else(\|e\| panic!("release {attempt} refused: {e:#}"));
3007	let (status, error) = await_target(&pool, build_id).await;
3008	assert_eq!(status, "ok", "release {attempt} should build ({error})");
3009	}
3010
3011	assert_eq!(
3012	std::fs::read_to_string(build_dir(&repo, "").join("Cargo.lock")).unwrap(),
3013	"version = 4\nchurn\n",
3014	"the second release started from the tag's lockfile, not the first's leavings"
3015	);
3016	assert_eq!(repo_status(&repo), "", "and the checkout was never in it");
3017	}
3018
3019	/// A working copy ahead of the tag does not fail the version check.
3020	///
3021	/// The check used to read the checkout, so releasing v0.0.1 while `main` had
3022	/// already moved to 0.0.2 was refused for "version drift" — about a tree the
3023	/// release does not build. It reads the tag now, and the tag says 0.0.1.
3024	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3025	async fn a_working_copy_ahead_of_the_tag_does_not_fail_the_version_check() {
3026	let tmp = tempfile::tempdir().unwrap();
3027	let repo = tmp.path().join("demo");
3028	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
3029	// main moves on, untagged, exactly as it does the day after a release.
3030	let git = git_in(&repo);
3031	std::fs::write(
3032	repo.join("src-tauri/tauri.conf.json"),
3033	"{\"version\":\"0.0.2\"}",
3034	)
3035	.unwrap();
3036	git(&["add", "-A"]);
3037	git(&["commit", "-q", "-m", "0.0.2"]);
3038
3039	let mut cfg = Config::for_tests(tmp.path());
3040	cfg.pin_release_sha = true;
3041	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3042	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
3043	let build_id = start_build(
3044	state,
3045	AppId::new("demo"),
3046	Version::parse("0.0.1").unwrap(),
3047	vec!["linux/x86_64".parse().unwrap()],
3048	)
3049	.await
3050	.expect("releasing the tag behind main must not be version drift");
3051	let (status, error) = await_target(&pool, build_id).await;
3052	assert_eq!(status, "ok", "({error})");
3053	}
3054
3055	/// A tag that disagrees with itself is refused, naming the file.
3056	///
3057	/// This is the drift worth catching and the one the old check could not see:
3058	/// `tauri.conf.json` at 0.0.1 and `Cargo.toml` still at 0.0.2, committed and
3059	/// tagged that way. Reading the checkout would have compared against
3060	/// whatever the working copy happened to say instead.
3061	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3062	async fn a_tag_whose_manifest_disagrees_with_it_is_refused() {
3063	let tmp = tempfile::tempdir().unwrap();
3064	let repo = tmp.path().join("demo");
3065	std::fs::create_dir_all(&repo).unwrap();
3066	std::fs::write(
3067	repo.join("Cargo.toml"),
3068	"[package]\nname = \"demo\"\nversion = \"0.0.2\"\n",
3069	)
3070	.unwrap();
3071	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
3072
3073	let mut cfg = Config::for_tests(tmp.path());
3074	cfg.pin_release_sha = true;
3075	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3076	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
3077	let err = start_build(
3078	state,
3079	AppId::new("demo"),
3080	Version::parse("0.0.1").unwrap(),
3081	vec!["linux/x86_64".parse().unwrap()],
3082	)
3083	.await
3084	.unwrap_err();
3085	let msg = format!("{err:#}");
3086	assert!(
3087	msg.contains("version drift") && msg.contains("Cargo.toml says 0.0.2"),
3088	"the refusal must name the file and what it says, got: {msg}"
3089	);
3090	assert!(
3091	msg.contains("v0.0.1"),
3092	"and say which tag it read, got: {msg}"
3093	);
3094	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
3095	.fetch_one(&pool)
3096	.await
3097	.unwrap();
3098	assert_eq!(builds, 0, "a refused version preflight writes no build row");
3099	}
3100
3101	/// A preflight that refuses on a LATER host leaves every checkout alone.
3102	///
3103	/// The hosts are prepared one at a time, so when the second is refused the
3104	/// first has already been through the whole preflight. That used to leave it
3105	/// detached at the tag with nothing to restore it, and the next attempt
3106	/// refused it for a detached HEAD — true, describing its own last run, and
3107	/// reading to the operator as their own mistake. Hit three times releasing
3108	/// pom on 2026-08-09. There is no unwind to get right now: the checkouts were
3109	/// never moved, and the worktrees a refusal leaves behind are where the next
3110	/// release would have put them.
3111	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3112	async fn a_refused_preflight_leaves_every_checkout_untouched() {
3113	let tmp = tempfile::tempdir().unwrap();
3114	// fw13's checkout is tagged and clean, so it pins. mbp's has no v0.0.1,
3115	// so it is refused after fw13 has already been moved.
3116	let repo = tmp.path().join("demo");
3117	init_git_app_shipping(
3118	&repo,
3119	"0.0.1",
3120	Some("v0.0.1"),
3121	"step(\"build\");\nsh_ok(build_host(), \"true\");\n",
3122	"[\"linux/x86_64\", \"macos/aarch64\"]",
3123	);
3124	let other = tmp.path().join("demo-mbp");
3125	init_git_app_shipping(
3126	&other,
3127	"0.0.1",
3128	None,
3129	"step(\"build\");\nsh_ok(build_host(), \"true\");\n",
3130	"[\"linux/x86_64\", \"macos/aarch64\"]",
3131	);
3132	let branch_before = current_branch(&repo);
3133	assert!(
3134	!branch_before.is_empty(),
3135	"fw13's checkout starts on a branch"
3136	);
3137
3138	let mut cfg = Config::for_tests(tmp.path());
3139	cfg.pin_release_sha = true;
3140	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3141	let topo = Topology::from_str_for_tests(&format!(
3142	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
3143	worktree_root = \"{wt}\"\n\
3144	[[host]]\nname = \"mbp\"\nssh = \"local\"\ntargets = [\"macos/aarch64\"]\n\
3145	worktree_root = \"{wt}\"\n\
3146	[app.demo]\nrepo = \"{}\"\n[app.demo.repo_by_host]\nmbp = \"{}\"\n",
3147	repo.display(),
3148	other.display(),
3149	wt = worktree_root(&repo).display(),
3150	))
3151	.unwrap();
3152	let state = test_state(pool.clone(), topo, cfg);
3153	let err = start_build(
3154	state,
3155	AppId::new("demo"),
3156	Version::parse("0.0.1").unwrap(),
3157	vec![
3158	"linux/x86_64".parse().unwrap(),
3159	"macos/aarch64".parse().unwrap(),
3160	],
3161	)
3162	.await
3163	.unwrap_err();
3164	let msg = format!("{err:#}");
3165	assert!(
3166	msg.contains("does not exist"),
3167	"the refusal should still be mbp's missing tag, got: {msg}"
3168	);
3169	assert_eq!(
3170	current_branch(&repo),
3171	branch_before,
3172	"a refused preflight must not have moved the host it prepared first"
3173	);
3174	assert_eq!(repo_status(&repo), "", "nor left anything in its tree");
3175	}
3176
3177	/// An edit elsewhere in a repo holding several products does not touch a
3178	/// release of one of them.
3179	///
3180	/// This is the case the old design could not get right, in either direction.
3181	/// The dirty gate was scoped to the app's own directory on purpose — an edit
3182	/// in `server/` is not something pom's build compiles — but `git checkout
3183	/// <tag>` acts on the whole repository, and MNW is one `.git` over the
3184	/// server, sando, multithreaded and pom. So the gate passed and the checkout
3185	/// then failed, on a file the operator had been told was none of this
3186	/// release's business.
3187	///
3188	/// A worktree is made of the repository too, so the release still gets the
3189	/// whole tagged tree — it just gets its own copy of it, and the edit stays
3190	/// where its author left it.
3191	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3192	async fn an_edit_elsewhere_in_the_repo_survives_a_release_and_stays_out_of_it() {
3193	let tmp = tempfile::tempdir().unwrap();
3194	let root = tmp.path().join("monorepo");
3195	let app = root.join("pom");
3196	init_git_app(&app, "0.0.1", None);
3197	let git = \|args: &[&str]\| {
3198	let out = std::process::Command::new("git")
3199	.args(args)
3200	.current_dir(&root)
3201	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3202	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3203	.env("GIT_AUTHOR_NAME", "t")
3204	.env("GIT_AUTHOR_EMAIL", "t@t")
3205	.env("GIT_COMMITTER_NAME", "t")
3206	.env("GIT_COMMITTER_EMAIL", "t@t")
3207	.output()
3208	.expect("git runs");
3209	assert!(
3210	out.status.success(),
3211	"git {args:?}: {}",
3212	String::from_utf8_lossy(&out.stderr)
3213	);
3214	};
3215	// `init_git_app` made `pom/` its own repo; the monorepo is the parent, so
3216	// drop that and re-init one `.git` over both products.
3217	std::fs::remove_dir_all(app.join(".git")).unwrap();
3218	std::fs::create_dir_all(root.join("server")).unwrap();
3219	std::fs::write(root.join("server/Cargo.lock"), "version = 4\n").unwrap();
3220	git(&["init", "-q"]);
3221	git(&["add", "-A"]);
3222	git(&["commit", "-q", "-m", "init"]);
3223	git(&["tag", "v0.0.1"]);
3224	// The tag and the branch differ in `server/`, and the working copy of
3225	// that file is modified — so the checkout cannot carry the edit across.
3226	std::fs::write(root.join("server/Cargo.lock"), "version = 4\n# moved on\n").unwrap();
3227	git(&["add", "-A"]);
3228	git(&["commit", "-q", "-m", "server moves"]);
3229	std::fs::write(
3230	root.join("server/Cargo.lock"),
3231	"version = 4\n# local edit\n",
3232	)
3233	.unwrap();
3234
3235	let mut cfg = Config::for_tests(tmp.path());
3236	cfg.pin_release_sha = true;
3237	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3238	// The worktree root sits beside the monorepo, not inside it.
3239	let topo = Topology::from_str_for_tests(&format!(
3240	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
3241	worktree_root = \"{wt}\"\n[app.demo]\nrepo = \"{repo}\"\n",
3242	wt = worktree_root(&root).display(),
3243	repo = app.display()
3244	))
3245	.unwrap();
3246	let state = test_state(pool.clone(), topo, cfg);
3247	let build_id = start_build(
3248	state,
3249	AppId::new("demo"),
3250	Version::parse("0.0.1").unwrap(),
3251	vec!["linux/x86_64".parse().unwrap()],
3252	)
3253	.await
3254	.unwrap();
3255	let (status, error) = await_target(&pool, build_id).await;
3256	assert_eq!(
3257	status, "ok",
3258	"an edit in server/ must not stop pom's release ({error})"
3259	);
3260	assert_eq!(
3261	std::fs::read_to_string(root.join("server/Cargo.lock")).unwrap(),
3262	"version = 4\n# local edit\n",
3263	"the edit must be exactly where its author left it"
3264	);
3265	// The worktree is of the whole repository, at the tag: `server/` is
3266	// there, and it is the tagged content rather than either the branch tip
3267	// or the edit.
3268	let worktree = worktree_root(&root).join("monorepo").join("demo");
3269	assert_eq!(
3270	std::fs::read_to_string(worktree.join("server/Cargo.lock")).unwrap(),
3271	"version = 4\n",
3272	"the release built the tag's server/, not the working copy's"
3273	);
3274	}
3275
3276	/// The branch `repo` is on, empty on a detached HEAD.
3277	/// Tracked files with local changes, as `git status --porcelain` writes
3278	/// them, joined by newlines and empty for a clean tree.
3279	fn repo_status(repo: &std::path::Path) -> String {
3280	let out = std::process::Command::new("git")
3281	.args(["status", "--porcelain", "--untracked-files=no"])
3282	.current_dir(repo)
3283	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3284	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3285	.output()
3286	.expect("git runs");
3287	String::from_utf8_lossy(&out.stdout).trim_end().to_string()
3288	}
3289
3290	/// Run git in `dir`, isolated from the dev's global config and with an
3291	/// identity, panicking with git's own stderr on failure.
3292	fn git_in(dir: &std::path::Path) -> impl Fn(&[&str]) + '_ {
3293	move \|args: &[&str]\| {
3294	let out = std::process::Command::new("git")
3295	.args(args)
3296	.current_dir(dir)
3297	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3298	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3299	.env("GIT_AUTHOR_NAME", "t")
3300	.env("GIT_AUTHOR_EMAIL", "t@t")
3301	.env("GIT_COMMITTER_NAME", "t")
3302	.env("GIT_COMMITTER_EMAIL", "t@t")
3303	.output()
3304	.expect("git runs");
3305	assert!(
3306	out.status.success(),
3307	"git {args:?}: {}",
3308	String::from_utf8_lossy(&out.stderr)
3309	);
3310	}
3311	}
3312
3313	/// The commit a checkout (or worktree) is on.
3314	fn head_sha(repo: &std::path::Path) -> String {
3315	git_read(repo, &["rev-parse", "HEAD"])
3316	}
3317
3318	/// The commit a tag names.
3319	fn tag_sha(repo: &std::path::Path, tag: &str) -> String {
3320	git_read(repo, &["rev-parse", &format!("{tag}^{{commit}}")])
3321	}
3322
3323	fn git_read(dir: &std::path::Path, args: &[&str]) -> String {
3324	let out = std::process::Command::new("git")
3325	.args(args)
3326	.current_dir(dir)
3327	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3328	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3329	.output()
3330	.expect("git runs");
3331	String::from_utf8_lossy(&out.stdout).trim().to_string()
3332	}
3333
3334	fn current_branch(repo: &std::path::Path) -> String {
3335	let out = std::process::Command::new("git")
3336	.args(["symbolic-ref", "-q", "--short", "HEAD"])
3337	.current_dir(repo)
3338	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3339	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3340	.output()
3341	.expect("git runs");
3342	String::from_utf8_lossy(&out.stdout).trim().to_string()
3343	}
3344
3345	/// An unreachable remote does not fail a release whose tag is present.
3346	///
3347	/// The preflight used to run `fetch --all --tags --prune && checkout`, and
3348	/// `fetch --all` is non-zero if ANY remote fails. Every library repo carries
3349	/// three (`astra`, `mnw`, `srht`), so one dead mirror aborted the release and
3350	/// blamed it on a missing tag — which is how makeover v2.1.1 failed on
3351	/// 2026-07-26 four minutes after its tag was created. Fetch is advisory now;
3352	/// only the checkout decides.
3353	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3354	async fn a_dead_remote_does_not_fail_a_release_whose_tag_exists() {
3355	let tmp = tempfile::tempdir().unwrap();
3356	let repo = tmp.path().join("demo");
3357	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
3358	// A remote that cannot possibly be fetched, standing in for an offline
3359	// astra or an srht mirror the repo was never pushed to.
3360	let out = std::process::Command::new("git")
3361	.args([
3362	"remote",
3363	"add",
3364	"srht",
3365	&tmp.path().join("nowhere.git").display().to_string(),
3366	])
3367	.current_dir(&repo)
3368	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3369	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3370	.output()
3371	.expect("git runs");
3372	assert!(out.status.success());
3373
3374	let mut cfg = Config::for_tests(tmp.path());
3375	cfg.pin_release_sha = true;
3376	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3377	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
3378	let build_id = start_build(
3379	state,
3380	AppId::new("demo"),
3381	Version::parse("0.0.1").unwrap(),
3382	vec!["linux/x86_64".parse().unwrap()],
3383	)
3384	.await
3385	.expect("a dead mirror must not refuse the release");
3386	let (status, error) = await_target(&pool, build_id).await;
3387	assert_eq!(status, "ok", "the tag is present, so this builds ({error})");
3388	}
3389
3390	/// The audit fix: a `build` step dispatched to a host whose executor lacks the
3391	/// `build` capability is denied at the transport BEFORE the command runs. This
3392	/// is the structural guarantee behind "never build on prod" — a recipe naming
3393	/// the wrong host can't compile there.
3394	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3395	async fn build_on_a_host_without_the_build_grant_is_denied() {
3396	let tmp = tempfile::tempdir().unwrap();
3397	let root = tmp.path();
3398
3399	let repo = root.join("app");
3400	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3401	std::fs::write(
3402	repo.join("src-tauri/tauri.conf.json"),
3403	r#"{"version":"0.0.1"}"#,
3404	)
3405	.unwrap();
3406	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
3407	// The recipe (wrongly) tries to compile on `prod`, a host with no build
3408	// grant. A marker file would appear if the command actually ran.
3409	let marker = root.join("ran-on-prod");
3410	std::fs::write(
3411	repo.join("dist/recipes/linux.rhai"),
3412	r#"
3413	step("build");
3414	sh_ok("prod", "touch MARKER");
3415	"#
3416	.replace("MARKER", marker.to_str().unwrap()),
3417	)
3418	.unwrap();
3419
3420	let cfg = Config::for_tests(root);
3421	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3422	// fw13 builds linux; `prod` is local-but-restart-only (no build/package).
3423	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
3424	let topo = Topology::from_str_for_tests(
3425	r#"
3426	[[host]]
3427	name = "fw13"
3428	ssh = "local"
3429	targets = ["linux/x86_64"]
3430
3431	[[host]]
3432	name = "prod"
3433	ssh = "local"
3434	actuate = ["restart"]
3435	observe = []
3436
3437	[app.demo]
3438	repo = "REPO"
3439	"#
3440	.replace("REPO", repo.to_str().unwrap())
3441	.as_str(),
3442	)
3443	.unwrap();
3444
3445	let state = test_state(pool.clone(), topo, cfg);
3446
3447	let build_id = start_build(
3448	state.clone(),
3449	AppId::new("demo"),
3450	Version::parse("0.0.1").unwrap(),
3451	vec!["linux/x86_64".parse().unwrap()],
3452	)
3453	.await
3454	.unwrap();
3455
3456	let mut status = String::new();
3457	let mut error = String::new();
3458	for _ in 0..100 {
3459	let row: Option<(String, Option<String>)> =
3460	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
3461	.bind(build_id)
3462	.fetch_optional(&pool)
3463	.await
3464	.unwrap();
3465	if let Some((s, e)) = row {
3466	status = s;
3467	error = e.unwrap_or_default();
3468	if status != "running" {
3469	break;
3470	}
3471	}
3472	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3473	}
3474
3475	assert_eq!(status, "failed", "build on an ungranted host must fail");
3476	assert!(
3477	error.contains("capability denied") && error.contains("build"),
3478	"failure must be a capability denial, got: {error}"
3479	);
3480	assert!(!marker.exists(), "denied build step must NOT have executed");
3481	}
3482
3483	// ---- macOS sign / notarize / staple execution chain (via ScriptedExec) ----
3484
3485	/// Stand up a single-macOS-target app whose recipe is `recipe_body` (with the
3486	/// literal `ARTIFACT` replaced by a real, non-empty file on disk), dispatch
3487	/// every host command through `scripted`, run the build to a terminal state,
3488	/// and return the pool plus the final `(status, error)`. The returned
3489	/// [`tempfile::TempDir`] must be kept alive by the caller: it holds the
3490	/// sqlite DB the pool reads.
3491	async fn run_macos_recipe(
3492	scripted: Arc<ScriptedExec>,
3493	recipe_body: &str,
3494	backoff_secs: Option<u64>,
3495	) -> (tempfile::TempDir, SqlitePool, String, String) {
3496	let tmp = tempfile::tempdir().unwrap();
3497	let root = tmp.path();
3498	let repo = root.join("app");
3499	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3500	std::fs::write(
3501	repo.join("src-tauri/tauri.conf.json"),
3502	r#"{"version":"0.0.1"}"#,
3503	)
3504	.unwrap();
3505	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
3506
3507	// A real, non-empty artifact so `publish`'s size floor is satisfied; the
3508	// build step is faked, so nothing else creates it.
3509	let artifact = repo.join("out/demo.dmg");
3510	std::fs::create_dir_all(artifact.parent().unwrap()).unwrap();
3511	std::fs::write(&artifact, b"dmg-bytes").unwrap();
3512
3513	std::fs::write(
3514	repo.join("dist/recipes/macos.rhai"),
3515	recipe_body.replace("ARTIFACT", artifact.to_str().unwrap()),
3516	)
3517	.unwrap();
3518
3519	let cfg = Config {
3520	notarize_backoff_secs: backoff_secs,
3521	..Config::for_tests(root)
3522	};
3523	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3524	std::fs::write(repo.join("bento.toml"), "targets = [\"macos/aarch64\"]\n").unwrap();
3525	let topo = Topology::from_str_for_tests(&format!(
3526	r#"
3527	[[host]]
3528	name = "mbp"
3529	ssh = "local"
3530	targets = ["macos/aarch64"]
3531
3532	[app.demo]
3533	repo = "{}"
3534	"#,
3535	repo.display()
3536	))
3537	.unwrap();
3538
3539	let mut state = test_state(pool.clone(), topo, cfg);
3540	// Route every host command through the scripted executor.
3541	let mut execs = HashMap::new();
3542	execs.insert("mbp".to_string(), scripted as Arc<dyn Executor>);
3543	state.executors = Arc::new(execs);
3544
3545	let build_id = start_build(
3546	state.clone(),
3547	AppId::new("demo"),
3548	Version::parse("0.0.1").unwrap(),
3549	vec!["macos/aarch64".parse().unwrap()],
3550	)
3551	.await
3552	.unwrap();
3553
3554	let mut status = String::new();
3555	let mut error = String::new();
3556	for _ in 0..100 {
3557	let row: Option<(String, Option<String>)> =
3558	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
3559	.bind(build_id)
3560	.fetch_optional(&pool)
3561	.await
3562	.unwrap();
3563	if let Some((s, e)) = row {
3564	status = s;
3565	error = e.unwrap_or_default();
3566	if status != "running" {
3567	break;
3568	}
3569	}
3570	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3571	}
3572	(tmp, pool, status, error)
3573	}
3574
3575	async fn release_count(pool: &SqlitePool) -> i64 {
3576	sqlx::query_scalar("SELECT COUNT(*) FROM releases")
3577	.fetch_one(pool)
3578	.await
3579	.unwrap()
3580	}
3581
3582	/// The whole macOS release chain end to end through a fake host: codesign,
3583	/// notarize (Accepted first try), staple, verify_gatekeeper, then publish.
3584	/// The recorded commands lock the exact incantations each host function
3585	/// dispatches, and a `releases` row proves the publish gate opened for a
3586	/// signed + notarized + Gatekeeper-accepted artifact.
3587	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3588	async fn a_macos_recipe_signs_notarizes_staples_verifies_and_publishes() {
3589	let scripted = Arc::new(
3590	ScriptedExec::new()
3591	.on("codesign", 0, "")
3592	.on("notarytool", 0, r#"{"status":"Accepted"}"#)
3593	.on("stapler staple", 0, "")
3594	.on(
3595	"spctl",
3596	0,
3597	"source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
3598	),
3599	);
3600	let recipe = r#"
3601	let h = build_host();
3602	step("build");
3603	sh_ok(h, "echo built");
3604	step("sign");
3605	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3606	notarize(h, "ARTIFACT");
3607	staple(h, "ARTIFACT");
3608	step("verify");
3609	if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
3610	step("publish");
3611	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3612	"#;
3613	let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, None).await;
3614
3615	assert_eq!(
3616	status, "ok",
3617	"signed+notarized macOS build should publish: {error}"
3618	);
3619	assert_eq!(
3620	release_count(&pool).await,
3621	1,
3622	"publish must record a release"
3623	);
3624
3625	// The exact shell incantations each host function dispatched.
3626	let cmds = scripted.commands();
3627	let has = \|needle: &str\| cmds.iter().any(\|c\| c.contains(needle));
3628	assert!(
3629	has("codesign --force --options runtime --timestamp --sign"),
3630	"codesign runtime+timestamp incantation, got: {cmds:?}"
3631	);
3632	assert!(
3633	has("xcrun notarytool submit"),
3634	"notarytool submit: {cmds:?}"
3635	);
3636	assert!(
3637	has("--wait --output-format json"),
3638	"notarytool --wait json: {cmds:?}"
3639	);
3640	assert!(has("xcrun stapler staple"), "stapler staple: {cmds:?}");
3641	assert!(has("spctl --assess"), "gatekeeper assess: {cmds:?}");
3642	}
3643
3644	/// A failing `codesign` fails the sign step and aborts the recipe before
3645	/// publish — an unsigned artifact never ships.
3646	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3647	async fn a_codesign_failure_fails_the_sign_step_and_blocks_publish() {
3648	let scripted = Arc::new(ScriptedExec::new().on("codesign", 1, ""));
3649	let recipe = r#"
3650	let h = build_host();
3651	step("build");
3652	sh_ok(h, "echo built");
3653	step("sign");
3654	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3655	step("publish");
3656	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3657	"#;
3658	let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
3659
3660	assert_eq!(status, "failed", "a failed codesign must fail the target");
3661	assert!(
3662	error.contains("codesign failed"),
3663	"error names the codesign failure, got: {error}"
3664	);
3665	assert_eq!(
3666	release_count(&pool).await,
3667	0,
3668	"nothing may publish after a codesign failure"
3669	);
3670	}
3671
3672	/// Even if the recipe ignores `verify_gatekeeper`'s returned `false`, the
3673	/// publish gate refuses the artifact: `verify_gatekeeper` both records the
3674	/// rejection and fails its step, and `publish` proves neither passed. This
3675	/// is the defense-in-depth the pure `PublishAuthority::prove` tests assert in
3676	/// isolation, here exercised through the real host-function path.
3677	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3678	async fn a_gatekeeper_rejection_bars_publish_even_if_the_recipe_ignores_it() {
3679	let scripted = Arc::new(
3680	ScriptedExec::new()
3681	.on("codesign", 0, "")
3682	.on("notarytool", 0, r#"{"status":"Accepted"}"#)
3683	.on("stapler staple", 0, "")
3684	// Gatekeeper says no: the sentinel is FAIL, not OK.
3685	.on("spctl", 0, "source=Unnotarized\nBENTO_GATEKEEPER_FAIL"),
3686	);
3687	let recipe = r#"
3688	let h = build_host();
3689	step("build");
3690	sh_ok(h, "echo built");
3691	step("sign");
3692	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3693	notarize(h, "ARTIFACT");
3694	staple(h, "ARTIFACT");
3695	step("verify");
3696	verify_gatekeeper(h, "ARTIFACT");
3697	step("publish");
3698	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3699	"#;
3700	let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
3701
3702	assert_eq!(
3703	status, "failed",
3704	"a Gatekeeper-rejected artifact must not publish"
3705	);
3706	assert!(
3707	!error.is_empty(),
3708	"the barred publish must surface an error"
3709	);
3710	assert_eq!(
3711	release_count(&pool).await,
3712	0,
3713	"no release for a rejected artifact"
3714	);
3715	}
3716
3717	/// The one flaky, network-bound step: `notarize` retries a non-`Accepted`
3718	/// result and succeeds on a later attempt. Two notarytool calls (reject then
3719	/// accept) then a recorded release prove the retry ran and the chain
3720	/// completed. Backoff is 0 so the retry sleep doesn't stall the test.
3721	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3722	async fn notarize_retries_a_non_accepted_result_then_succeeds() {
3723	let scripted = Arc::new(
3724	ScriptedExec::new()
3725	.on("codesign", 0, "")
3726	.on_seq(
3727	"notarytool",
3728	&[
3729	(0, r#"{"status":"In Progress"}"#),
3730	(0, r#"{"status":"Accepted"}"#),
3731	],
3732	)
3733	.on("stapler staple", 0, "")
3734	.on(
3735	"spctl",
3736	0,
3737	"source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
3738	),
3739	);
3740	let recipe = r#"
3741	let h = build_host();
3742	step("build");
3743	sh_ok(h, "echo built");
3744	step("sign");
3745	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3746	notarize(h, "ARTIFACT");
3747	staple(h, "ARTIFACT");
3748	step("verify");
3749	if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
3750	step("publish");
3751	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3752	"#;
3753	let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
3754
3755	assert_eq!(
3756	status, "ok",
3757	"notarize should succeed on the retry: {error}"
3758	);
3759	assert_eq!(
3760	release_count(&pool).await,
3761	1,
3762	"the retried build still publishes"
3763	);
3764	let notary_calls = scripted
3765	.commands()
3766	.iter()
3767	.filter(\|c\| c.contains("notarytool"))
3768	.count();
3769	assert_eq!(
3770	notary_calls, 2,
3771	"notarytool ran once, was rejected, then ran again"
3772	);
3773	}
3774
3775	/// `notarize` gives up after its bounded retries: three notarytool attempts,
3776	/// all non-`Accepted`, fail the target and bar publish.
3777	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3778	async fn notarize_fails_the_target_after_exhausting_its_retries() {
3779	let scripted = Arc::new(
3780	ScriptedExec::new()
3781	.on("codesign", 0, "")
3782	// Every attempt reports a still-pending status, never Accepted.
3783	.on("notarytool", 0, r#"{"status":"In Progress"}"#),
3784	);
3785	let recipe = r#"
3786	let h = build_host();
3787	step("build");
3788	sh_ok(h, "echo built");
3789	step("sign");
3790	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3791	notarize(h, "ARTIFACT");
3792	step("publish");
3793	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3794	"#;
3795	let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
3796
3797	assert_eq!(
3798	status, "failed",
3799	"exhausted notarization must fail the target"
3800	);
3801	assert!(
3802	error.contains("notarization failed after 3 attempts"),
3803	"error names the exhausted retry, got: {error}"
3804	);
3805	assert_eq!(
3806	release_count(&pool).await,
3807	0,
3808	"an unnotarized artifact never publishes"
3809	);
3810	let notary_calls = scripted
3811	.commands()
3812	.iter()
3813	.filter(\|c\| c.contains("notarytool"))
3814	.count();
3815	assert_eq!(notary_calls, 3, "the retry is bounded at three attempts");
3816	}
3817	// ---- item (5): SSH and Agent transports end to end, via a recording fake ----
3818	//
3819	// Every other test declares `ssh = "local"`, so the runner's two-plane
3820	// routing is only exercised at construction
3821	// (state::agent_host_syncs_over_ssh_never_the_agent): steps run over the
3822	// EXEC transport (state.executors -- SshExec, or the in-session AgentRpc for
3823	// a mac host) while artifacts move over the SYNC transport (state.syncs --
3824	// always ssh, NEVER the agent, whose confined `/pull` would 404 or force
3825	// `pull_root` wide enough to expose `~/.tauri/passwords.env`). These drive a
3826	// real recipe through `start_build` on a NON-local topology, replace the
3827	// real ssh/agent transports with a recording fake, and assert which plane
3828	// handled which operation -- the runtime form of that construction-time
3829	// invariant, on the ssh string every other test pins to "local".
3830
3831	/// Records shell commands and artifact pulls on separate logs, so a test can
3832	/// prove the exec transport built/signed and the sync transport collected --
3833	/// and that neither did the other's job -- without a live host or ssh.
3834	struct RecordingExec {
3835	caps: CapabilitySet,
3836	commands: Arc<std::sync::Mutex<Vec<String>>>,
3837	pulls: Arc<std::sync::Mutex<Vec<String>>>,
3838	}
3839
3840	impl RecordingExec {
3841	fn new() -> Arc<Self> {
3842	Arc::new(Self {
3843	// A mac build host's real grant. Nothing in this fake gates on it
3844	// (the real transports do), but keep it coherent so
3845	// `capabilities()` is not a lie.
3846	caps: CapabilitySet::from_tokens(
3847	["build", "sign", "notarize", "staple"],
3848	["build-log", "artifact"],
3849	),
3850	commands: Arc::new(std::sync::Mutex::new(Vec::new())),
3851	pulls: Arc::new(std::sync::Mutex::new(Vec::new())),
3852	})
3853	}
3854	fn commands(&self) -> Vec<String> {
3855	self.commands.lock().unwrap().clone()
3856	}
3857	fn pulls(&self) -> Vec<String> {
3858	self.pulls.lock().unwrap().clone()
3859	}
3860	}
3861
3862	#[async_trait]
3863	impl Executor for RecordingExec {
3864	async fn run_streaming(
3865	&self,
3866	step: &ops_exec::Step,
3867	_sink: &mut dyn LogSink,
3868	) -> anyhow::Result<RunOutput> {
3869	self.commands
3870	.lock()
3871	.unwrap()
3872	.push(step.argv.last().cloned().unwrap_or_default());
3873	Ok(RunOutput {
3874	status: std::process::ExitStatus::from_raw(0),
3875	stdout: Vec::new(),
3876	stderr: Vec::new(),
3877	})
3878	}
3879	async fn pull_file(
3880	&self,
3881	r: &std::path::Path,
3882	_l: &std::path::Path,
3883	_o: &SyncOpts,
3884	) -> anyhow::Result<()> {
3885	self.pulls
3886	.lock()
3887	.unwrap()
3888	.push(r.to_string_lossy().into_owned());
3889	Ok(())
3890	}
3891	async fn pull_dir(
3892	&self,
3893	r: &std::path::Path,
3894	_l: &std::path::Path,
3895	_o: &SyncOpts,
3896	) -> anyhow::Result<()> {
3897	self.pulls
3898	.lock()
3899	.unwrap()
3900	.push(r.to_string_lossy().into_owned());
3901	Ok(())
3902	}
3903	async fn pull_glob(
3904	&self,
3905	g: &str,
3906	_l: &std::path::Path,
3907	_o: &SyncOpts,
3908	) -> anyhow::Result<()> {
3909	self.pulls.lock().unwrap().push(g.to_string());
3910	Ok(())
3911	}
3912	async fn push_dir(
3913	&self,
3914	_l: &std::path::Path,
3915	_r: &std::path::Path,
3916	_o: &SyncOpts,
3917	) -> anyhow::Result<()> {
3918	Ok(())
3919	}
3920	async fn preflight(&self) -> anyhow::Result<()> {
3921	Ok(())
3922	}
3923	fn capabilities(&self) -> &CapabilitySet {
3924	&self.caps
3925	}
3926	}
3927
3928	/// Stand up a single-target app (host named `h1`) whose recipe is
3929	/// `recipe_body` (with `REPO` replaced by the checkout path), inject
3930	/// `exec_fake` as the host's EXEC transport and `sync_fake` as its SYNC
3931	/// transport, run the build to a terminal state, and return the tmpdir plus
3932	/// `(status, error)`. Unlike `test_state`'s `build_executors`, this replaces
3933	/// BOTH planes so no real ssh/agent transport is dialed. The returned
3934	/// [`tempfile::TempDir`] holds the sqlite DB and must outlive the caller's
3935	/// assertions.
3936	async fn run_two_plane(
3937	host_toml: &str,
3938	target: &str,
3939	recipe_file: &str,
3940	recipe_body: &str,
3941	exec_fake: Arc<dyn Executor>,
3942	sync_fake: Arc<dyn Executor>,
3943	) -> (tempfile::TempDir, String, String) {
3944	let tmp = tempfile::tempdir().unwrap();
3945	let root = tmp.path();
3946	let repo = root.join("app");
3947	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3948	std::fs::write(
3949	repo.join("src-tauri/tauri.conf.json"),
3950	r#"{"version":"0.0.1"}"#,
3951	)
3952	.unwrap();
3953	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
3954	std::fs::write(
3955	repo.join("dist/recipes").join(recipe_file),
3956	recipe_body.replace("REPO", repo.to_str().unwrap()),
3957	)
3958	.unwrap();
3959	std::fs::write(
3960	repo.join("bento.toml"),
3961	format!("targets = [\"{target}\"]\n"),
3962	)
3963	.unwrap();
3964
3965	let cfg = Config::for_tests(root);
3966	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3967	let topo = Topology::from_str_for_tests(&format!(
3968	"{}\n[app.demo]\nrepo = \"{}\"\n",
3969	host_toml.replace("REPO", repo.to_str().unwrap()),
3970	repo.display()
3971	))
3972	.unwrap();
3973
3974	let mut state = test_state(pool.clone(), topo, cfg);
3975	state.executors = Arc::new(HashMap::from([("h1".to_string(), exec_fake)]));
3976	state.syncs = Arc::new(HashMap::from([("h1".to_string(), sync_fake)]));
3977
3978	let build_id = start_build(
3979	state.clone(),
3980	AppId::new("demo"),
3981	Version::parse("0.0.1").unwrap(),
3982	vec![target.parse().unwrap()],
3983	)
3984	.await
3985	.unwrap();
3986
3987	let mut status = String::new();
3988	let mut error = String::new();
3989	for _ in 0..100 {
3990	let row: Option<(String, Option<String>)> =
3991	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
3992	.bind(build_id)
3993	.fetch_optional(&pool)
3994	.await
3995	.unwrap();
3996	if let Some((s, e)) = row {
3997	status = s;
3998	error = e.unwrap_or_default();
3999	if status != "running" {
4000	break;
4001	}
4002	}
4003	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
4004	}
4005	(tmp, status, error)
4006	}
4007
4008	/// An agent (macOS) host signs over the AGENT transport but is collected
4009	/// from over SSH -- driven through the whole runner, not just `build_sync`.
4010	/// The sign chain's commands land on the exec plane and the artifact pull on
4011	/// the sync plane; crucially, the agent plane is asked to move NOTHING (a
4012	/// regression to one transport would route collect at `AgentRpc::pull_glob`,
4013	/// refused by design, or widen `pull_root` over the secret-bearing home dir).
4014	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4015	async fn an_agent_host_signs_over_the_agent_and_collects_over_ssh_end_to_end() {
4016	let agent = RecordingExec::new(); // exec plane (AgentRpc in prod)
4017	let ssh = RecordingExec::new(); // sync plane (SshExec in prod)
4018	let host = r#"
4019	[[host]]
4020	name = "h1"
4021	ssh = "mbp"
4022	targets = ["macos/aarch64"]
4023	transport = "agent"
4024	agent_url = "http://mbp:8765"
4025	actuate = ["build", "sign", "notarize", "staple"]
4026	observe = ["build-log", "gatekeeper", "artifact"]
4027	pull_root = "REPO"
4028	"#;
4029	let recipe = r#"
4030	let h = build_host();
4031	step("build");
4032	sh_ok(h, "echo built");
4033	step("sign");
4034	codesign(h, "Developer ID Application: Test", "REPO/out/demo.dmg");
4035	step("collect");
4036	collect(h, "REPO/out/*.dmg", "demo", "0.0.1");
4037	"#;
4038	let (_tmp, status, error) = run_two_plane(
4039	host,
4040	"macos/aarch64",
4041	"macos.rhai",
4042	recipe,
4043	agent.clone(),
4044	ssh.clone(),
4045	)
4046	.await;
4047	assert_eq!(status, "ok", "the recipe should complete: {error}");
4048
4049	// The build + sign commands ran on the AGENT (exec) transport.
4050	let agent_cmds = agent.commands();
4051	assert!(
4052	agent_cmds.iter().any(\|c\| c.contains("codesign")),
4053	"codesign rides the agent exec transport: {agent_cmds:?}"
4054	);
4055	assert!(
4056	agent_cmds.iter().any(\|c\| c.contains("echo built")),
4057	"the build step rides the agent exec transport: {agent_cmds:?}"
4058	);
4059	// ...and the agent moved NO artifacts. This is the load-bearing half:
4060	// AgentRpc::pull_glob is refused by design, so collect must not touch it.
4061	assert!(
4062	agent.pulls().is_empty(),
4063	"the agent transport must never collect artifacts: {:?}",
4064	agent.pulls()
4065	);
4066
4067	// The artifact was collected over the SSH (sync) transport...
4068	let ssh_pulls = ssh.pulls();
4069	assert!(
4070	ssh_pulls
4071	.iter()
4072	.any(\|p\| p.contains("demo.dmg") \|\| p.contains("*.dmg")),
4073	"collect rides the ssh sync transport: {ssh_pulls:?}"
4074	);
4075	// ...and the sync transport was never asked to run a build/sign command.
4076	assert!(
4077	ssh.commands().is_empty(),
4078	"the sync transport must never run host commands: {:?}",
4079	ssh.commands()
4080	);
4081	}
4082
4083	/// A plain (non-agent) host whose `ssh` is a remote alias, not "local" --
4084	/// the case every other test avoids. The recipe runs end to end through the
4085	/// fake, proving the runner drives a non-local host and still splits exec
4086	/// (build) from sync (collect) across the two transport maps.
4087	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4088	async fn a_non_local_ssh_host_runs_a_recipe_end_to_end() {
4089	let exec = RecordingExec::new();
4090	let sync = RecordingExec::new();
4091	let host = r#"
4092	[[host]]
4093	name = "h1"
4094	ssh = "astra"
4095	targets = ["linux/x86_64"]
4096	pull_root = "REPO"
4097	"#;
4098	let recipe = r#"
4099	let h = build_host();
4100	step("build");
4101	sh_ok(h, "echo compiling");
4102	step("collect");
4103	collect(h, "REPO/out/demo.bin", "demo", "0.0.1");
4104	"#;
4105	let (_tmp, status, error) = run_two_plane(
4106	host,
4107	"linux/x86_64",
4108	"linux.rhai",
4109	recipe,
4110	exec.clone(),
4111	sync.clone(),
4112	)
4113	.await;
4114	assert_eq!(status, "ok", "the recipe should complete: {error}");
4115
4116	assert!(
4117	exec.commands().iter().any(\|c\| c.contains("echo compiling")),
4118	"the build command rides the exec transport: {:?}",
4119	exec.commands()
4120	);
4121	assert!(
4122	exec.pulls().is_empty(),
4123	"the exec transport must not collect: {:?}",
4124	exec.pulls()
4125	);
4126	assert!(
4127	sync.pulls().iter().any(\|p\| p.contains("demo.bin")),
4128	"collect rides the sync transport: {:?}",
4129	sync.pulls()
4130	);
4131	assert!(
4132	sync.commands().is_empty(),
4133	"the sync transport must not run commands: {:?}",
4134	sync.commands()
4135	);
4136	}
4137	}
4138
4139	/// Every recipe of every configured app must parse.
4140	///
4141	/// A recipe is read and compiled at release time, on the build host, after the
4142	/// checkout has already run — so a typo in one is discovered at the worst
4143	/// possible moment. Compiling them here costs nothing and moves that discovery
4144	/// to `cargo test`. Skips when the live config is absent (CI, another host),
4145	/// like [`crate::topology`]'s live-config smoke test.
4146	#[cfg(test)]
4147	mod live_recipe_smoke {
4148	use crate::topology::Topology;
4149	use std::path::{Path, PathBuf};
4150
4151	#[test]
4152	fn live_recipes_compile_if_present() {
4153	let Some(home) = std::env::var_os("HOME") else {
4154	return;
4155	};
4156	let path = Path::new(&home).join(".config/bento/bento.toml");
4157	if !path.exists() {
4158	return;
4159	}
4160	let topo = Topology::load(&path).expect("live bento.toml must load");
4161	// Syntax only: the host functions are bound per run against a live
4162	// context, and Rhai resolves calls at eval time regardless.
4163	let engine = rhai::Engine::new();
4164	let mut checked = 0;
4165	for (name, cfg) in &topo.app {
4166	let dir = crate::engine::expand_tilde(&cfg.repo).join(&cfg.recipe_dir);
4167	// Every recipe in the directory, not just the ones the app's current
4168	// targets name. A recipe for a target that is temporarily not
4169	// shipped (windows, dropped from the manifests until its host is
4170	// real) still has to parse, and it is the one nothing else is
4171	// watching.
4172	let Ok(entries) = std::fs::read_dir(&dir) else {
4173	continue;
4174	};
4175	let mut files: Vec<PathBuf> = entries
4176	.filter_map(Result::ok)
4177	.map(\|e\| e.path())
4178	.filter(\|p\| p.extension().is_some_and(\|x\| x == "rhai"))
4179	.collect();
4180	files.sort();
4181	for p in files {
4182	let file = p.file_name().unwrap_or_default().to_string_lossy();
4183	let Ok(src) = std::fs::read_to_string(&p) else {
4184	continue;
4185	};
4186	engine
4187	.compile(&src)
4188	.unwrap_or_else(\|e\| panic!("recipe {name}/{file} does not parse: {e}"));
4189	checked += 1;
4190	}
4191	}
4192	assert!(checked > 0, "live config resolved no readable recipes");
4193	}
4194	}
4195