max / makenotwork

158.7 KB · 4157 lines History Blame Raw

1	//! Build orchestration: fan a `(app, version)` out across its targets, each
2	//! running its recipe concurrently on the host that can build it.
3	//!
4	//! A build inserts one `builds` row, then spawns one task per target. Each
5	//! target task registers itself in the single-slot guard (a newer build for
6	//! the same `(app, target)` aborts the in-flight one — latest wins, but other
7	//! targets keep running, which is the fan-out), then runs the recipe.
8
9	use crate::domain::{AppId, Status, Step, Target, Version};
10	use crate::engine::{self, RecipeCtx};
11	use crate::events::{self, Event};
12	use crate::state::AppState;
13	use anyhow::{Context, Result};
14	use ops_exec::{Action, LogSink, Step as OpStep};
15	use std::path::PathBuf;
16	use std::sync::Arc;
17
18	/// A [`LogSink`] that drops what it's handed — the release preflight runs git on
19	/// each host for its exit code and (via a separate `rev-parse`) the sha in
20	/// `RunOutput`, not for a streamed log, so there is no step to stream into.
21	struct DiscardSink;
22
23	#[async_trait::async_trait]
24	impl LogSink for DiscardSink {
25	async fn write_chunk(&mut self, _bytes: &[u8]) {}
26	}
27
28	/// Preflight barrier: put every host that will build a target for this release
29	/// into a worktree detached at the tag `v<version>`, and refuse the build unless
30	/// they all report the SAME commit. Without it each host builds whatever its own
31	/// checkout holds and the artifacts are filed under the daemon host's version.
32	/// This runs before any target task spawns, so a mixed-source release is stopped
33	/// before a single artifact is built.
34	///
35	/// "All" means all: a host that did not report is a refusal, not an abstention.
36	/// Comparing only the hosts that answered proves agreement among those, which is
37	/// not the claim the barrier is making.
38	///
39	/// The agreed sha is the release's provenance, and this is the only place it is
40	/// known to be one value rather than a per-host answer. Resolving it here and
41	/// carrying it forward is what lets an artifact record name the source it was
42	/// built from without asking a build host to be honest about it afterwards.
43	///
44	/// ## The tree a release builds in is Bento's, not the operator's
45	///
46	/// A release never touches the operator's checkout. Checking the tag out there
47	/// would pin a tree somebody works in for the length of the build, dirty it
48	/// (`cargo` rewrites `Cargo.lock` under the `[patch]` block) so the restore
49	/// fails and leaves it detached at the tag, and force the preflight to refuse a
50	/// dirty or already-detached checkout it does not read.
51	///
52	/// A worktree costs the working files only, since it shares the repository's
53	/// object store, and is re-pinned with `--force`, which is safe precisely because
54	/// nothing but Bento writes there. Where it lives is [`Host::worktree_root`],
55	/// and that doc carries why the path has to sit under `~/Code`.
56	///
57	/// Returns where each host will build, for the recipes to be pointed at, and the
58	/// commit they all agreed on. Nothing to unwind on a refusal: a host prepared
59	/// before the refusal is left holding a worktree at a tag, which is what the
60	/// next release would put it at anyway.
61	async fn pin_release(
62	state: &AppState,
63	app: &AppId,
64	version: &Version,
65	targets: &[Target],
66	) -> Result<Pinned> {
67	let cfg = state
68	.topo
69	.app(app)
70	.ok_or_else(\|\| anyhow::anyhow!("unknown app `{app}`"))?;
71	// Spelled per app: a repo holding one product tags `v0.4.1`, a repo holding
72	// several tags `pom-v0.4.1`. See `AppConfig::tag_format`.
73	let tag = cfg.tag_for(version);
74
75	// The distinct hosts across the requested targets (order-stable).
76	//
77	// A target whose host the topology cannot resolve is refused rather than
78	// skipped. Skipping it would leave that target out of the barrier while it
79	// still builds, which is the shape of hole this whole preflight exists to
80	// close: agreement proven among some hosts reads as agreement among all.
81	// `resolve_targets` rejects an unbuildable target before `/build`, so this
82	// is unreachable through the API and cheap to state anyway.
83	let mut hosts: Vec<&crate::topology::Host> = Vec::new();
84	for t in targets {
85	let h = state
86	.topo
87	.host_for(*t)
88	.ok_or_else(\|\| anyhow::anyhow!("no host can build {t}"))?;
89	if !hosts.iter().any(\|seen\| seen.name == h.name) {
90	hosts.push(h);
91	}
92	}
93	anyhow::ensure!(
94	!hosts.is_empty(),
95	"no build hosts for v{version}; refusing to build a release nothing was pinned for"
96	);
97
98	// What each host IS, before what each host is ON. A build host that was
99	// rebuilt into something else is refused here rather than producing an
100	// artifact whose libc floor nobody expected. Ordered before the worktree
101	// work because it is one cheap round-trip and because a machine that is not
102	// what the topology says it is should not be asked to check anything out.
103	for host in &hosts {
104	check_host_identity(state, host).await?;
105	}
106
107	let mut prepared: Vec<PinnedHost> = Vec::new();
108	for host in &hosts {
109	prepared.push(prepare_worktree(state, app, cfg, host, &tag).await?);
110	}
111
112	// The barrier: every participating host must have reported, and every report
113	// must be the same commit. Reporting is checked first and separately from
114	// agreement, because a barrier that compares only what it received cannot
115	// tell unanimity from silence — three hosts agreeing while a fourth was
116	// never asked is exactly the mixed-source release this refuses.
117	let missing: Vec<&str> = hosts
118	.iter()
119	.filter(\|h\| !prepared.iter().any(\|p\| p.host == h.name))
120	.map(\|h\| h.name.as_str())
121	.collect();
122	anyhow::ensure!(
123	missing.is_empty(),
124	"{} did not report a commit for v{version}; refusing to build a release the \
125	barrier cannot vouch for",
126	missing.join(", "),
127	);
128	let empty_shas: Vec<&str> = prepared
129	.iter()
130	.filter(\|p\| p.sha.is_empty())
131	.map(\|p\| p.host.as_str())
132	.collect();
133	anyhow::ensure!(
134	empty_shas.is_empty(),
135	"`git rev-parse HEAD` returned nothing on {} for v{version}",
136	empty_shas.join(", "),
137	);
138
139	let first = &prepared[0];
140	let mismatch: Vec<String> = prepared
141	.iter()
142	.filter(\|p\| p.sha != first.sha)
143	.map(\|p\| format!("{}={}", p.host, short(&p.sha)))
144	.collect();
145	anyhow::ensure!(
146	mismatch.is_empty(),
147	"build hosts are on different commits for v{version} \
148	({}={}, {}); refusing to build a release from mixed sources",
149	first.host,
150	short(&first.sha),
151	mismatch.join(", "),
152	);
153
154	let sha = first.sha.clone();
155	Ok(Pinned {
156	build_dirs: prepared
157	.into_iter()
158	.map(\|p\| (p.host, p.build_dir))
159	.collect(),
160	sha,
161	})
162	}
163
164	/// Confirm a build host is what the topology declares it to be.
165	///
166	/// The sha barrier below proves every host is on the same source. This proves
167	/// every host is the machine the topology thinks it is, which the sha barrier
168	/// cannot see: three hosts can agree on a commit while one of them was rebuilt
169	/// last week onto a base whose libc is two versions ahead, and the only symptom
170	/// is a binary that will not start on the box it ships to.
171	///
172	/// A host that declares neither field is skipped and logged as skipped. The
173	/// macOS and Windows hosts are legitimately in that state, so a refusal would
174	/// take those pipelines down to buy nothing.
175	async fn check_host_identity(state: &AppState, host: &crate::topology::Host) -> Result<()> {
176	if host.base_image.is_none() && host.libc.is_none() {
177	tracing::info!(host = %host.name, "preflight: host declares no base image; not checked");
178	return Ok(());
179	}
180	let exec = state
181	.executors
182	.get(&host.name)
183	.ok_or_else(\|\| anyhow::anyhow!("no executor for host `{}`", host.name))?;
184	let step = OpStep::shell(Action::Build, ops_core::base_image::probe_cmd());
185	let mut sink = DiscardSink;
186	let out = exec
187	.run_streaming(&step, &mut sink)
188	.await
189	.with_context(\|\| format!("asking `{}` what it is", host.name))?;
190	anyhow::ensure!(
191	out.status.success(),
192	"`{}` could not report its base image: {}",
193	host.name,
194	String::from_utf8_lossy(&out.stderr).trim(),
195	);
196	let reported = ops_core::base_image::parse_probe(&String::from_utf8_lossy(&out.stdout));
197	let checked = ops_core::base_image::check(
198	&host.name,
199	host.base_image.as_ref(),
200	host.libc.as_deref(),
201	&reported,
202	)
203	.context("a build host is not what the topology declares it to be")?;
204	if let Some(what) = checked {
205	tracing::info!(host = %host.name, "preflight: identity checked, {what}");
206	}
207	Ok(())
208	}
209
210	/// Put one host's worktree at the release tag and report the commit it landed
211	/// on, creating the worktree the first time this app is released there.
212	///
213	/// The worktree's own path is derived on the host rather than assumed: a repo
214	/// holding several products is one `.git` over all of them, so `repo` for pom is
215	/// `~/Code/MNW/pom` and the tree to make a worktree of is `~/Code/MNW`. One
216	/// `rev-parse` answers both halves — which repository, and where the app sits
217	/// inside it.
218	async fn prepare_worktree(
219	state: &AppState,
220	app: &AppId,
221	cfg: &crate::topology::AppConfig,
222	host: &crate::topology::Host,
223	tag: &str,
224	) -> Result<PinnedHost> {
225	let exec = state
226	.executors
227	.get(&host.name)
228	.ok_or_else(\|\| anyhow::anyhow!("no executor for host `{}`", host.name))?;
229	let name = &host.name;
230	// Resolved per host, not once for the set: the checkouts need not be at the
231	// same path on every machine, and pinning windows-x86 with the unix path
232	// failed on the first git command of the release.
233	let repo = cfg.repo_for(name);
234
235	let read = \|cmd: String\| async {
236	let step = OpStep::shell(Action::Build, cmd);
237	let mut sink = DiscardSink;
238	exec.run_streaming(&step, &mut sink).await
239	};
240
241	let out = read(engine::git_toplevel_and_prefix_cmd(repo))
242	.await
243	.with_context(\|\| format!("reading `{repo}` on `{name}`"))?;
244	let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();
245	anyhow::ensure!(
246	out.status.success(),
247	"`{repo}` on `{name}` is not a git checkout: {stderr}"
248	);
249	let (toplevel, prefix) = engine::parse_toplevel_and_prefix(&String::from_utf8_lossy(
250	&out.stdout,
251	))
252	.ok_or_else(\|\| anyhow::anyhow!("could not read where `{repo}` is checked out on `{name}`"))?;
253
254	let worktree = host
255	.worktree_for(engine::repo_dir_name(&toplevel), app)
256	.ok_or_else(\|\| {
257	anyhow::anyhow!(
258	"host `{name}` sets no worktree_root, so there is nowhere to build \
259	tagged source"
260	)
261	})?;
262
263	// Advisory, and deliberately run against the ordinary checkout: a worktree
264	// shares its repository's object store, so a tag fetched here is a tag the
265	// worktree can be put at. `fetch --all` exits non-zero if ANY remote fails
266	// and the library repos carry three, so a dead mirror must not decide a
267	// release. Only the checkout below is allowed to fail.
268	let _ = read(engine::git_fetch_cmd(repo)).await;
269
270	let exists = read(engine::git_worktree_probe_cmd(&worktree))
271	.await
272	.is_ok_and(\|o\| o.status.success());
273	let pin = if exists {
274	engine::git_worktree_pin_cmd(&worktree, tag)
275	} else {
276	// Prune first: a worktree directory somebody deleted by hand is still
277	// registered in the repository, and `worktree add` then refuses the path
278	// as already in use rather than rebuilding it.
279	let _ = read(engine::git_worktree_prune_cmd(&toplevel)).await;
280	engine::git_worktree_add_cmd(&toplevel, &worktree, tag)
281	};
282	let out = read(pin)
283	.await
284	.with_context(\|\| format!("pinning the worktree on `{name}`"))?;
285	if !out.status.success() {
286	// Ask git why before telling the operator. The usual answer is a tag
287	// that was never created or never pushed, which is a different job from
288	// anything about the worktree itself.
289	let tag_exists = read(engine::git_tag_exists_cmd(repo, tag))
290	.await
291	.is_ok_and(\|o\| o.status.success());
292	anyhow::bail!(
293	"could not put `{worktree}` on `{name}` at {tag}: {}",
294	engine::worktree_failure_reason(tag, tag_exists, &String::from_utf8_lossy(&out.stderr))
295	);
296	}
297
298	let out = read(engine::git_rev_parse_cmd(&worktree))
299	.await
300	.with_context(\|\| format!("rev-parse on `{name}`"))?;
301	anyhow::ensure!(
302	out.status.success(),
303	"`git rev-parse HEAD` failed in `{worktree}` on `{name}`"
304	);
305
306	Ok(PinnedHost {
307	host: name.clone(),
308	build_dir: engine::app_dir_in_worktree(&worktree, &prefix),
309	sha: String::from_utf8_lossy(&out.stdout).trim().to_string(),
310	})
311	}
312
313	/// What the preflight established: where each host builds this release, and the
314	/// one commit every one of those trees is on.
315	struct Pinned {
316	/// `(host, directory the recipe builds in)`. Becomes the run's
317	/// `repo_by_host`, so `repo()` in a recipe is the worktree and no recipe
318	/// knows this happened.
319	build_dirs: Vec<(String, String)>,
320	/// The commit all hosts agreed on. Empty when pinning is off (tests).
321	sha: String,
322	}
323
324	/// One host's prepared worktree, before the barrier has compared them.
325	struct PinnedHost {
326	host: String,
327	/// The app's own directory inside the worktree — the worktree root for a
328	/// repo holding one product, `<worktree>/pom` for one holding several.
329	build_dir: String,
330	sha: String,
331	}
332
333	/// Confirm the tag states the version being built, in every file that states a
334	/// version.
335	///
336	/// Read out of the tag with `git show` on a build host, so it needs no worktree
337	/// path of its own and no assumption that the daemon has a copy of the repo:
338	/// one host is enough, since the barrier has already proven they are all on the
339	/// same commit.
340	///
341	/// Skipped when nothing was pinned, which is `pin_release_sha = false` — tests,
342	/// whose repos are plain directories with no tag to read.
343	async fn check_version_at_tag(
344	state: &AppState,
345	app: &AppId,
346	version: &Version,
347	pinned: &Pinned,
348	) -> Result<()> {
349	let Some((host, dir)) = pinned.build_dirs.first() else {
350	return Ok(());
351	};
352	let Some(cfg) = state.topo.app(app) else {
353	return Ok(());
354	};
355	let Some(exec) = state.executors.get(host) else {
356	return Ok(());
357	};
358	let tag = cfg.tag_for(version);
359	let version_path = cfg.version_path.as_deref();
360
361	let mut sources: Vec<(String, String)> = Vec::new();
362	for rel in engine::version_sources(version_path) {
363	let step = OpStep::shell(Action::Build, engine::git_show_file_cmd(dir, &tag, &rel));
364	let mut sink = DiscardSink;
365	let out = exec
366	.run_streaming(&step, &mut sink)
367	.await
368	.with_context(\|\| format!("reading {rel} from {tag} on `{host}`"))?;
369	if out.status.success() {
370	sources.push((rel, String::from_utf8_lossy(&out.stdout).into_owned()));
371	} else if version_path == Some(rel.as_str()) {
372	// A file the app NAMES as its version source has to be there, or
373	// the check would pass by failing to look.
374	anyhow::bail!("`{rel}` is not in {tag} on `{host}`, but the app declares it");
375	}
376	}
377	engine::versions_agree(
378	&format!("{tag} (read on `{host}`)"),
379	&sources,
380	version_path,
381	version,
382	)
383	}
384
385	/// First 12 chars of a sha for a readable error.
386	fn short(sha: &str) -> &str {
387	sha.get(..12).unwrap_or(sha)
388	}
389
390	/// Resolve the version to build: explicit, or read from `tauri.conf.json`.
391	///
392	/// Returns typed errors so the route maps user mistakes (unknown app, bad
393	/// version string) to 400, and only genuine daemon-side failures (reading the
394	/// app's `tauri.conf.json`) to 500.
395	pub fn resolve_version(
396	state: &AppState,
397	app: &AppId,
398	explicit: Option<String>,
399	) -> crate::error::Result<Version> {
400	use crate::error::Error;
401	if let Some(v) = explicit {
402	return Version::parse(&v).map_err(Error::BadRequest);
403	}
404	let cfg = state
405	.topo
406	.app(app)
407	.ok_or_else(\|\| Error::BadRequest(format!("unknown app `{app}`")))?;
408	engine::version_from_repo(&cfg.repo, cfg.version_path.as_deref()).map_err(Error::Other)
409	}
410
411	/// Validate + default the target list against what the app ships. Unknown app,
412	/// an unshipped target, or a target no host can build are all client errors
413	/// (400), not server errors (500).
414	pub fn resolve_targets(
415	state: &AppState,
416	app: &AppId,
417	requested: Vec<Target>,
418	) -> crate::error::Result<Vec<Target>> {
419	use crate::error::Error;
420	let cfg = state
421	.topo
422	.app(app)
423	.ok_or_else(\|\| Error::BadRequest(format!("unknown app `{app}`")))?;
424	if requested.is_empty() {
425	return Ok(cfg.targets.clone());
426	}
427	for t in &requested {
428	if !cfg.targets.contains(t) {
429	return Err(Error::BadRequest(format!(
430	"app `{app}` does not ship target {t}"
431	)));
432	}
433	if state.topo.host_for(*t).is_none() {
434	return Err(Error::BadRequest(format!("no host can build {t}")));
435	}
436	}
437	Ok(requested)
438	}
439
440	/// Insert the build row and spawn per-target tasks. Returns the build id.
441	pub async fn start_build(
442	state: AppState,
443	app: AppId,
444	version: Version,
445	targets: Vec<Target>,
446	) -> Result<i64> {
447	// Pin every build host to the release tag and verify they agree, before any
448	// target task spawns. Off in tests (their repos aren't git checkouts).
449	let pinned = Arc::new(if state.cfg.pin_release_sha {
450	pin_release(&state, &app, &version, &targets)
451	.await
452	.context("release preflight")?
453	} else {
454	Pinned {
455	build_dirs: Vec::new(),
456	sha: String::new(),
457	}
458	});
459
460	// Then: every version source in the TAG must agree with the version being
461	// built, before a single host compiles. `version_from_repo` reads one file,
462	// so a `tauri.conf.json`/`Cargo.toml` (or explicit-version) mismatch would
463	// otherwise sail through and file artifacts under the wrong version.
464	//
465	// After the pin rather than before it, and read from the tag rather than
466	// from the checkout, because those are two different trees now. Asking the
467	// checkout answered a question about a tree the release does not build: a
468	// working copy one commit ahead failed a release of the tag behind it, and
469	// a `Cargo.toml` that disagreed with the tag it was tagged in passed.
470	check_version_at_tag(&state, &app, &version, &pinned)
471	.await
472	.context("version preflight")?;
473
474	let build_id: i64 = sqlx::query_scalar(
475	"INSERT INTO builds (app, version, status, created_at) VALUES (?, ?, 'running', ?) RETURNING id",
476	)
477	.bind(app.as_str())
478	.bind(version.to_string())
479	.bind(chrono::Utc::now().to_rfc3339())
480	.fetch_one(&state.pool)
481	.await
482	.context("insert build")?;
483
484	events::emit(
485	&state.events,
486	Event::BuildRequested {
487	app: app.clone(),
488	version: version.clone(),
489	targets: targets.clone(),
490	},
491	);
492	crate::metrics::build_started();
493
494	// Spawn every target into a JoinSet so finalize_build can await completion
495	// event-driven (no DB polling) and observe each task's outcome (panic vs
496	// clean) for supervision.
497	let mut set: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
498	for target in targets {
499	let app = app.clone();
500	let version = version.clone();
501	let key = (app.clone(), target);
502	let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
503
504	// Latest-wins, done as ONE critical section: supersede the prior occupant
505	// (cooperatively cancel its recipe + abort its task) and install ours
506	// without ever releasing the lock, so two concurrent /build+/retry for the
507	// same (app, target) can't both pass "nothing to abort" and run together.
508	let mut active = state.active.lock().await;
509	if let Some(prev) = active.remove(&key)
510	&& !prev.abort.is_finished()
511	{
512	// The recipe runs on a blocking thread; abort() alone can't stop it,
513	// so set the cooperative flag the engine checks at step boundaries.
514	prev.cancel.store(true, std::sync::atomic::Ordering::SeqCst);
515	prev.abort.abort();
516	events::emit(
517	&state.events,
518	Event::TargetAborted {
519	app: app.clone(),
520	target,
521	},
522	);
523	}
524	let abort = set.spawn(run_target(
525	state.clone(),
526	build_id,
527	app,
528	version,
529	target,
530	cancel.clone(),
531	pinned.clone(),
532	));
533	active.insert(
534	key,
535	crate::state::ActiveSlot {
536	build_id,
537	abort,
538	cancel,
539	},
540	);
541	crate::metrics::set_in_flight(active.len());
542	}
543
544	// Mark the build done once all target tasks settle. Spawned so /build
545	// returns immediately. Nothing to put back: the release built in Bento's own
546	// worktrees, and the checkouts were never moved.
547	tokio::spawn(finalize_build(state, build_id, set));
548	Ok(build_id)
549	}
550
551	/// Run one target's recipe end to end, updating its `target_runs` row. `cancel`
552	/// is set when a newer build supersedes this one; the engine checks it at step
553	/// boundaries and before publish so a superseded recipe can't advance or ship.
554	async fn run_target(
555	state: AppState,
556	build_id: i64,
557	app: AppId,
558	version: Version,
559	target: Target,
560	cancel: Arc<std::sync::atomic::AtomicBool>,
561	pinned: Arc<Pinned>,
562	) {
563	let started = std::time::Instant::now();
564	let target_run_id: i64 = match sqlx::query_scalar(
565	"INSERT INTO target_runs (build_id, app, version, target, status, started_at)
566	VALUES (?, ?, ?, ?, 'running', ?) RETURNING id",
567	)
568	.bind(build_id)
569	.bind(app.as_str())
570	.bind(version.to_string())
571	.bind(target.to_string())
572	.bind(chrono::Utc::now().to_rfc3339())
573	.fetch_one(&state.pool)
574	.await
575	{
576	Ok(id) => id,
577	Err(e) => {
578	tracing::error!(%app, %target, error = %e, "could not create target_run");
579	return;
580	}
581	};
582
583	events::emit(
584	&state.events,
585	Event::TargetStart {
586	app: app.clone(),
587	version: version.clone(),
588	target,
589	},
590	);
591
592	let recipe_src = match read_recipe(&state, &app, target) {
593	Ok(s) => s,
594	Err(e) => {
595	fail_target(
596	&state,
597	target_run_id,
598	&app,
599	&version,
600	target,
601	Step::Checkout,
602	&format!("{e:#}"),
603	)
604	.await;
605	crate::metrics::target_finished(
606	&target.to_string(),
607	"failed",
608	started.elapsed().as_secs_f64(),
609	);
610	return;
611	}
612	};
613
614	// Pre-flight the build host before running the recipe. For local/ssh hosts
615	// this is a no-op; for the agent host (macOS in-session signing) it hits
616	// `/health` so a dead ops-agent fails here with a clear message — matching
617	// what the driver does — instead of erroring opaquely on the recipe's first
618	// dispatch to that host.
619	if let Some(host) = state.topo.host_for(target)
620	&& let Some(exec) = state.executors.get(&host.name)
621	&& let Err(e) = exec.preflight().await
622	{
623	fail_target(
624	&state,
625	target_run_id,
626	&app,
627	&version,
628	target,
629	Step::Checkout,
630	&format!("{e:#}"),
631	)
632	.await;
633	crate::metrics::target_finished(
634	&target.to_string(),
635	"failed",
636	started.elapsed().as_secs_f64(),
637	);
638	return;
639	}
640
641	// Host + checkout path for this target, resolved from the topology. Both are
642	// exposed to the recipe (`build_host()` / `repo()`); a missing host here is
643	// unreachable (resolve_targets rejected unbuildable targets before /build),
644	// but fall back rather than panic.
645	let build_host = state
646	.topo
647	.host_for(target)
648	.map(\|h\| h.name.clone())
649	.unwrap_or_default();
650	let build_host_ssh = state
651	.topo
652	.host_for(target)
653	.map(\|h\| h.ssh.clone())
654	.unwrap_or_default();
655	// Where the recipe builds. The preflight prepared a worktree per host and
656	// these are those paths, so `repo()` in a recipe is Bento's own tree and no
657	// recipe knows the difference — which is the whole reason the checkout could
658	// stop being what a release builds in. The topology's paths are the fallback
659	// for `pin_release_sha = false`, which is tests: their repos are plain
660	// directories with no git in them to make a worktree of.
661	let repo_by_host: std::collections::HashMap<String, String> = if pinned.build_dirs.is_empty() {
662	state
663	.topo
664	.app(&app)
665	.map(\|a\| a.repo_by_host.clone())
666	.unwrap_or_default()
667	} else {
668	pinned.build_dirs.iter().cloned().collect()
669	};
670	// The daemon-local path, which is a different question: `crate_preflight`
671	// runs `cargo metadata` on the daemon's own box rather than on a build host.
672	// Point it at the local host's worktree when there is one, so it reads the
673	// tagged manifest the release is publishing and not whatever the working
674	// copy happens to say.
675	let repo = state
676	.topo
677	.hosts
678	.iter()
679	.find(\|h\| h.ssh == "local")
680	.and_then(\|h\| repo_by_host.get(&h.name).cloned())
681	.or_else(\|\| state.topo.app(&app).map(\|a\| a.repo.clone()))
682	.unwrap_or_default();
683	let features = state
684	.topo
685	.app(&app)
686	.map(\|a\| a.features.clone())
687	.unwrap_or_default();
688	let tag = state
689	.topo
690	.app(&app)
691	.map_or_else(\|\| format!("v{version}"), \|a\| a.tag_for(&version));
692	// The opt-in all-targets-green publish gate: pass the declared target set
693	// (so `publish` can require every sibling green) only when the app turns it
694	// on; otherwise None leaves independent per-target publishing unchanged.
695	let all_green_required = state
696	.topo
697	.app(&app)
698	.and_then(\|a\| a.require_all_targets.then(\|\| a.targets.clone()));
699
700	// A service's install destination for THIS target, plus an executor for it.
701	//
702	// The executor is added to a per-run copy of the map rather than to the
703	// daemon-wide one, so a deploy grant on a production host exists only for
704	// the duration of the run that needs it and only for the app that declared
705	// it. Nothing else can address that host: it is not in the topology, so no
706	// other app's recipe can name it.
707	let deploy = state
708	.topo
709	.app(&app)
710	.and_then(\|a\| a.deploy_for(target))
711	.cloned();
712	let execs = match &deploy {
713	Some(d) => {
714	let mut map = (*state.executors).clone();
715	map.insert(d.host.clone(), crate::state::build_deploy_executor(d));
716	Arc::new(map)
717	}
718	None => state.executors.clone(),
719	};
720
721	let ctx = Arc::new(
722	RecipeCtx::new(
723	app.clone(),
724	version.clone(),
725	target,
726	build_host,
727	build_host_ssh,
728	tag.clone(),
729	repo,
730	features,
731	// Gates the `verify` step's capability: a library's crate preflight is
732	// not an app's Gatekeeper check. See engine::action_for.
733	state.topo.app(&app).map(\|a\| a.kind).unwrap_or_default(),
734	target_run_id,
735	execs,
736	state.syncs.clone(),
737	deploy,
738	state.pool.clone(),
739	state.events.clone(),
740	state.cfg.clone(),
741	state.ota.clone(),
742	tokio::runtime::Handle::current(),
743	cancel,
744	all_green_required,
745	)
746	.with_repo_by_host(repo_by_host),
747	);
748
749	// Serialize per host: hold this host's lock for the whole recipe run so a
750	// second target on the same box (goingson macos + ios both on mbp) can't
751	// build concurrently in one checkout and corrupt the shared target/ +
752	// keychain. Targets on different hosts hold different locks and still fan
753	// out. Acquired at an await point, so a supersede-abort while queued drops
754	// the task cleanly before it ever takes the lock. Not held during the
755	// read-only preflight above.
756	let _host_guard = match state.host_locks.get(&ctx.build_host).cloned() {
757	Some(lock) => Some(lock.lock_owned().await),
758	None => None,
759	};
760
761	// Rhai is synchronous; run the recipe (and its final step finalization) on
762	// a blocking thread so host functions can `block_on` without sitting on a
763	// runtime worker.
764	let ctx_run = ctx.clone();
765	let outcome = tokio::task::spawn_blocking(move \|\| {
766	let engine = engine::build_engine(&ctx_run);
767	let res = engine.run(&recipe_src);
768	let last_step = ctx_run.current_step();
769	match &res {
770	Ok(()) => {
771	let _ = ctx_run.finish_step(Status::Ok);
772	}
773	Err(_) => {
774	let _ = ctx_run.finish_step(Status::Failed);
775	}
776	}
777	res.map_err(\|e\| (last_step, e.to_string()))
778	})
779	.await;
780
781	// Write the artifact record before the run is stamped terminal. It describes
782	// what was collected, so it runs whether the recipe succeeded or failed: a
783	// build that signed and collected an artifact and then failed at `publish`
784	// still produced bytes somebody may want the provenance of. Emit-only and
785	// non-fatal; nothing reads it yet.
786	let record_path = crate::artifact_record::emit(&state, &ctx, &pinned.sha).await;
787
788	match outcome {
789	Ok(Ok(())) => {
790	// Hand the artifact to Sando before the run is stamped ok, so a
791	// target reported green is one whose bytes reached the controller
792	// that decides whether they ship. Only on success: an artifact from
793	// a failed recipe is exactly what should not be offered for a
794	// deploy, whatever it managed to collect on the way down.
795	if let Err(e) =
796	handoff_for(&state, record_path.as_deref(), &app, &version, target).await
797	{
798	let msg = format!("{e:#}");
799	tracing::error!(%app, %target, error = %msg, "handing the artifact to sando failed");
800	fail_target(
801	&state,
802	target_run_id,
803	&app,
804	&version,
805	target,
806	Step::Handoff,
807	&msg,
808	)
809	.await;
810	crate::metrics::target_finished(
811	&target.to_string(),
812	"failed",
813	started.elapsed().as_secs_f64(),
814	);
815	return;
816	}
817
818	let artifacts = collected_artifacts(&state, &app, &version, target);
819	if let Err(e) = sqlx::query(
820	"UPDATE target_runs SET status = 'ok', current_step = NULL, finished_at = ? WHERE id = ?",
821	)
822	.bind(chrono::Utc::now().to_rfc3339())
823	.bind(target_run_id)
824	.execute(&state.pool)
825	.await
826	{
827	// A swallowed terminal write would leave the row `running`;
828	// finalize_build reconciles it, but log so the cause is visible.
829	tracing::error!(%app, %target, error = %e, "could not stamp target_run ok");
830	}
831	crate::metrics::target_finished(
832	&target.to_string(),
833	"ok",
834	started.elapsed().as_secs_f64(),
835	);
836	events::emit(
837	&state.events,
838	Event::TargetOk {
839	app,
840	version,
841	target,
842	artifacts,
843	},
844	);
845	}
846	Ok(Err((step, msg))) => {
847	fail_target(&state, target_run_id, &app, &version, target, step, &msg).await;
848	crate::metrics::target_finished(
849	&target.to_string(),
850	"failed",
851	started.elapsed().as_secs_f64(),
852	);
853	}
854	Err(join_err) => {
855	// Task was aborted (superseded) or panicked.
856	let status = if join_err.is_cancelled() {
857	"aborted"
858	} else {
859	"failed"
860	};
861	crate::metrics::target_finished(
862	&target.to_string(),
863	status,
864	started.elapsed().as_secs_f64(),
865	);
866	let msg = if join_err.is_cancelled() {
867	"aborted (superseded)".to_string()
868	} else {
869	format!("recipe task panicked: {join_err}")
870	};
871	fail_target(
872	&state,
873	target_run_id,
874	&app,
875	&version,
876	target,
877	Step::Build,
878	&msg,
879	)
880	.await;
881	}
882	}
883	}
884
885	async fn fail_target(
886	state: &AppState,
887	target_run_id: i64,
888	app: &AppId,
889	version: &Version,
890	target: Target,
891	step: Step,
892	error: &str,
893	) {
894	if let Err(e) = sqlx::query(
895	"UPDATE target_runs SET status = 'failed', current_step = NULL, error = ?, finished_at = ? WHERE id = ?",
896	)
897	.bind(error)
898	.bind(chrono::Utc::now().to_rfc3339())
899	.bind(target_run_id)
900	.execute(&state.pool)
901	.await
902	{
903	tracing::error!(%app, %target, error = %e, "could not stamp target_run failed");
904	}
905	events::emit(
906	&state.events,
907	Event::TargetFailed {
908	app: app.clone(),
909	version: version.clone(),
910	target,
911	step,
912	error: error.to_string(),
913	},
914	);
915	}
916
917	/// Await every target task of a build, then stamp the build's terminal status.
918	///
919	/// Event-driven: it joins the `JoinSet` rather than polling the DB. Each task's
920	/// outcome is supervised — a panicked task is logged (and its still-`running`
921	/// row is reconciled below), an aborted (superseded) task is expected.
922	///
923	/// Per-step deadlines (see `engine::step_budget`) are the real bound on a wedged
924	/// step now; this overall deadline is only a generous last-resort backstop for a
925	/// hang outside a bounded command. When it trips it sets each still-running
926	/// target's cooperative cancel FIRST (the blocking recipe bodies observe that at
927	/// their next bounded command or step boundary and stop signing) and then aborts
928	/// the async wrappers. Aborting the wrappers alone cannot reach the blocking
929	/// bodies, which marks a build failed while codesign keeps running on the mac.
930	async fn finalize_build(state: AppState, build_id: i64, mut set: tokio::task::JoinSet<()>) {
931	const MAX_WAIT: std::time::Duration = std::time::Duration::from_hours(6);
932	let deadline = tokio::time::Instant::now() + MAX_WAIT;
933	loop {
934	match tokio::time::timeout_at(deadline, set.join_next()).await {
935	Ok(Some(Ok(()))) => {}
936	Ok(Some(Err(e))) => {
937	// A panic in run_target itself (outside its inner spawn_blocking,
938	// which is already caught). Cancellation = a superseding build.
939	if !e.is_cancelled() {
940	tracing::error!(build_id, error = %e, "target task panicked");
941	}
942	}
943	Ok(None) => break, // all targets settled
944	Err(_elapsed) => {
945	tracing::error!(
946	build_id,
947	"finalize_build backstop deadline hit; cancelling then aborting remaining targets"
948	);
949	// Set the cooperative cancel on this build's still-running slots
950	// BEFORE aborting, so the blocking recipe bodies actually stop
951	// (abort() alone cannot reach a spawn_blocking body).
952	{
953	let active = state.active.lock().await;
954	for slot in active.values().filter(\|s\| s.build_id == build_id) {
955	slot.cancel.store(true, std::sync::atomic::Ordering::SeqCst);
956	}
957	}
958	set.abort_all();
959	while set.join_next().await.is_some() {}
960	break;
961	}
962	}
963	}
964
965	// Reap this build's latest-wins slots (only our own — a superseding build
966	// owns a different build_id and is left intact).
967	{
968	let mut active = state.active.lock().await;
969	active.retain(\|_, slot\| slot.build_id != build_id);
970	crate::metrics::set_in_flight(active.len());
971	}
972
973	let now = chrono::Utc::now().to_rfc3339();
974	// Any row still `running` is a panicked/aborted task that never stamped its
975	// terminal status — reconcile it so the build finalizes truthfully.
976	if let Err(e) = sqlx::query(
977	"UPDATE target_runs SET status = 'failed', \
978	error = COALESCE(error, 'target task ended before stamping its status'), \
979	finished_at = ? WHERE build_id = ? AND status = 'running'",
980	)
981	.bind(&now)
982	.bind(build_id)
983	.execute(&state.pool)
984	.await
985	{
986	tracing::error!(build_id, error = %e, "could not reconcile straggling target_runs");
987	}
988
989	let failed: i64 = sqlx::query_scalar(
990	"SELECT COUNT(*) FROM target_runs WHERE build_id = ? AND status = 'failed'",
991	)
992	.bind(build_id)
993	.fetch_one(&state.pool)
994	.await
995	.unwrap_or(0);
996	let status = if failed == 0 { "ok" } else { "failed" };
997	if let Err(e) = sqlx::query("UPDATE builds SET status = ?, finished_at = ? WHERE id = ?")
998	.bind(status)
999	.bind(&now)
1000	.bind(build_id)
1001	.execute(&state.pool)
1002	.await
1003	{
1004	tracing::error!(build_id, error = %e, "could not stamp build terminal status");
1005	}
1006	}
1007
1008	/// Read the recipe text for `(app, target)` from the app's checkout on the
1009	/// daemon host.
1010	///
1011	/// Apps use one recipe per platform (`<recipe_dir>/<platform>.rhai`), since
1012	/// what they produce differs by platform. A library produces one crate whatever
1013	/// host uploads it, so it uses a single `publish.rhai` — a `linux.rhai` naming
1014	/// a registry upload would imply a per-platform artifact that does not exist.
1015	fn read_recipe(state: &AppState, app: &AppId, target: Target) -> Result<String> {
1016	let cfg = state
1017	.topo
1018	.app(app)
1019	.ok_or_else(\|\| anyhow::anyhow!("unknown app `{app}`"))?;
1020	let file = match cfg.kind {
1021	crate::topology::Kind::Library => "publish.rhai".to_string(),
1022	// A service is built per target like an app — the binary genuinely
1023	// differs per platform, and so does where it lands — so it takes the
1024	// same per-platform recipe naming rather than a single deploy.rhai.
1025	crate::topology::Kind::App \| crate::topology::Kind::Service => {
1026	format!("{}.rhai", target.platform.as_str())
1027	}
1028	};
1029	let path: PathBuf = engine::expand_tilde(&cfg.repo)
1030	.join(&cfg.recipe_dir)
1031	.join(file);
1032	std::fs::read_to_string(&path).with_context(\|\| format!("reading recipe {}", path.display()))
1033	}
1034
1035	/// Send this target's finished bundle to the Sando configured for the app, if
1036	/// one is. Nothing configured is a no-op and the ordinary case.
1037	///
1038	/// A configured handoff with no record is an error rather than a skip. It means
1039	/// the recipe succeeded and collected nothing, or that the paperwork could not
1040	/// be written — and for an app whose whole point is being deployed by Sando, a
1041	/// build that produced nothing to hand over is not a green build. Everywhere
1042	/// else a missing record stays the non-event it was.
1043	async fn handoff_for(
1044	state: &AppState,
1045	record_path: Option<&std::path::Path>,
1046	app: &AppId,
1047	version: &Version,
1048	target: Target,
1049	) -> anyhow::Result<()> {
1050	if !state.cfg.handoff.contains_key(app.as_str()) {
1051	return Ok(());
1052	}
1053	let record_path = record_path.ok_or_else(\|\| {
1054	anyhow::anyhow!(
1055	"{app} hands off to sando, but this {target} run wrote no artifact record \
1056	(nothing was collected, or the record could not be written)"
1057	)
1058	})?;
1059	let dir = crate::archive::target_dir(&state.cfg.dist_root, app, version, target);
1060	crate::handoff::send(&state.cfg, &dir, record_path, app, version, target).await
1061	}
1062
1063	/// What this target run left in its collect directory.
1064	///
1065	/// Per target: the event reports what THIS run produced. Sharing one directory
1066	/// across a version's targets would have a mac build's `TargetOk` list the Linux
1067	/// AppImage a sibling collected minutes earlier.
1068	fn collected_artifacts(
1069	state: &AppState,
1070	app: &AppId,
1071	version: &Version,
1072	target: Target,
1073	) -> Vec<String> {
1074	let dir = crate::archive::target_dir(&state.cfg.dist_root, app, version, target);
1075	let Ok(rd) = std::fs::read_dir(&dir) else {
1076	return Vec::new();
1077	};
1078	rd.filter_map(std::result::Result::ok)
1079	.map(\|e\| e.file_name().to_string_lossy().into_owned())
1080	.collect()
1081	}
1082
1083	#[cfg(test)]
1084	mod tests {
1085	use super::*;
1086	use crate::config::Config;
1087	use crate::ota::OtaRegistry;
1088	use crate::topology::Topology;
1089	use async_trait::async_trait;
1090	use ops_exec::{CapabilitySet, Executor, LogSink, RunOutput, SyncOpts};
1091	use sqlx::SqlitePool;
1092	use std::collections::HashMap;
1093	use std::os::unix::process::ExitStatusExt;
1094	use std::sync::Arc;
1095	use tokio::sync::Mutex;
1096
1097	/// A no-transport [`Executor`] for the paths that don't need a real command
1098	/// to run: its `preflight` is programmable (the agent host hits `/health`
1099	/// there, and a dead `ops-agent` must fail the target before the recipe
1100	/// dispatches), and every actual op is a success no-op.
1101	struct FakeExec {
1102	caps: CapabilitySet,
1103	preflight_err: Option<String>,
1104	}
1105
1106	impl FakeExec {
1107	fn preflight_fails(msg: &str) -> Arc<dyn Executor> {
1108	Arc::new(Self {
1109	caps: CapabilitySet::default(),
1110	preflight_err: Some(msg.to_string()),
1111	})
1112	}
1113	}
1114
1115	#[async_trait]
1116	impl Executor for FakeExec {
1117	async fn run_streaming(
1118	&self,
1119	_step: &ops_exec::Step,
1120	_sink: &mut dyn LogSink,
1121	) -> anyhow::Result<RunOutput> {
1122	Ok(RunOutput {
1123	status: std::process::ExitStatus::from_raw(0),
1124	stdout: Vec::new(),
1125	stderr: Vec::new(),
1126	})
1127	}
1128	async fn pull_file(
1129	&self,
1130	_r: &std::path::Path,
1131	_l: &std::path::Path,
1132	_o: &SyncOpts,
1133	) -> anyhow::Result<()> {
1134	Ok(())
1135	}
1136	async fn pull_dir(
1137	&self,
1138	_r: &std::path::Path,
1139	_l: &std::path::Path,
1140	_o: &SyncOpts,
1141	) -> anyhow::Result<()> {
1142	Ok(())
1143	}
1144	async fn pull_glob(
1145	&self,
1146	_g: &str,
1147	_l: &std::path::Path,
1148	_o: &SyncOpts,
1149	) -> anyhow::Result<()> {
1150	Ok(())
1151	}
1152	async fn push_dir(
1153	&self,
1154	_l: &std::path::Path,
1155	_r: &std::path::Path,
1156	_o: &SyncOpts,
1157	) -> anyhow::Result<()> {
1158	Ok(())
1159	}
1160	async fn preflight(&self) -> anyhow::Result<()> {
1161	match &self.preflight_err {
1162	Some(m) => anyhow::bail!("{m}"),
1163	None => Ok(()),
1164	}
1165	}
1166	fn capabilities(&self) -> &CapabilitySet {
1167	&self.caps
1168	}
1169	}
1170
1171	/// A recording, programmable [`Executor`] for the macOS sign chain. Every
1172	/// dispatched shell command is captured for assertion, and its exit code +
1173	/// stdout is chosen by the first rule whose needle the command contains — so
1174	/// a test can make `notarytool` report `Accepted`, make `codesign` fail, or
1175	/// make `spctl` emit the Gatekeeper sentinel without a real Mac or SSH. A
1176	/// rule may carry a sequence of responses (one per successive match) to
1177	/// drive the notarize retry loop; the last entry repeats once the sequence
1178	/// is exhausted. Unmatched commands succeed as empty no-ops (so a plain
1179	/// build `sh_ok` passes), and the sync/preflight ops are no-ops.
1180	struct ScriptedExec {
1181	caps: CapabilitySet,
1182	rules: Vec<ScriptRule>,
1183	log: Arc<std::sync::Mutex<Vec<String>>>,
1184	}
1185
1186	struct ScriptRule {
1187	needle: String,
1188	responses: Vec<(i32, String)>,
1189	calls: std::sync::atomic::AtomicUsize,
1190	}
1191
1192	impl ScriptedExec {
1193	fn new() -> Self {
1194	Self {
1195	// A mac host's real grant. Nothing in the dispatch path gates on
1196	// it (this fake never calls `gate`), but keep it coherent so
1197	// `capabilities()` is not a lie.
1198	caps: CapabilitySet::from_tokens(
1199	["build", "sign", "notarize", "staple"],
1200	["build-log", "artifact"],
1201	),
1202	rules: Vec::new(),
1203	log: Arc::new(std::sync::Mutex::new(Vec::new())),
1204	}
1205	}
1206
1207	/// Respond to every command containing `needle` with `(code, stdout)`.
1208	fn on(mut self, needle: &str, code: i32, stdout: &str) -> Self {
1209	self.rules.push(ScriptRule {
1210	needle: needle.to_string(),
1211	responses: vec![(code, stdout.to_string())],
1212	calls: std::sync::atomic::AtomicUsize::new(0),
1213	});
1214	self
1215	}
1216
1217	/// Respond to successive `needle` matches with successive responses; the
1218	/// last repeats once the list is exhausted. Drives the notarize retry.
1219	fn on_seq(mut self, needle: &str, responses: &[(i32, &str)]) -> Self {
1220	self.rules.push(ScriptRule {
1221	needle: needle.to_string(),
1222	responses: responses
1223	.iter()
1224	.map(\|(c, s)\| (c, (s).to_string()))
1225	.collect(),
1226	calls: std::sync::atomic::AtomicUsize::new(0),
1227	});
1228	self
1229	}
1230
1231	/// Every shell command this executor was asked to run, in order.
1232	fn commands(&self) -> Vec<String> {
1233	self.log.lock().unwrap().clone()
1234	}
1235	}
1236
1237	#[async_trait]
1238	impl Executor for ScriptedExec {
1239	async fn run_streaming(
1240	&self,
1241	step: &ops_exec::Step,
1242	_sink: &mut dyn LogSink,
1243	) -> anyhow::Result<RunOutput> {
1244	let cmd = step.argv.last().cloned().unwrap_or_default();
1245	self.log.lock().unwrap().push(cmd.clone());
1246	let (code, stdout) = self.rules.iter().find(\|r\| cmd.contains(&r.needle)).map_or(
1247	(0, String::new()),
1248	\|r\| {
1249	let i = r.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1250	r.responses[i.min(r.responses.len() - 1)].clone()
1251	},
1252	);
1253	Ok(RunOutput {
1254	// Shift into the wait-status word's exit-code byte so
1255	// `ExitStatus::code()` reports `code` exactly (a bare
1256	// `from_raw(1)` reads as a signal, yielding `None`).
1257	status: std::process::ExitStatus::from_raw(code << 8),
1258	stdout: stdout.into_bytes(),
1259	stderr: Vec::new(),
1260	})
1261	}
1262	async fn pull_file(
1263	&self,
1264	_r: &std::path::Path,
1265	_l: &std::path::Path,
1266	_o: &SyncOpts,
1267	) -> anyhow::Result<()> {
1268	Ok(())
1269	}
1270	async fn pull_dir(
1271	&self,
1272	_r: &std::path::Path,
1273	_l: &std::path::Path,
1274	_o: &SyncOpts,
1275	) -> anyhow::Result<()> {
1276	Ok(())
1277	}
1278	async fn pull_glob(
1279	&self,
1280	_g: &str,
1281	_l: &std::path::Path,
1282	_o: &SyncOpts,
1283	) -> anyhow::Result<()> {
1284	Ok(())
1285	}
1286	async fn push_dir(
1287	&self,
1288	_l: &std::path::Path,
1289	_r: &std::path::Path,
1290	_o: &SyncOpts,
1291	) -> anyhow::Result<()> {
1292	Ok(())
1293	}
1294	async fn preflight(&self) -> anyhow::Result<()> {
1295	Ok(())
1296	}
1297	fn capabilities(&self) -> &CapabilitySet {
1298	&self.caps
1299	}
1300	}
1301
1302	/// Assemble an [`AppState`] from the three per-test inputs, filling in the
1303	/// executors/syncs (built from `topo`) and the fixed test scaffolding
1304	/// (metrics handle, event bus, standard OTA registry, empty active map, no
1305	/// token). Every runner test builds the same struct around a different repo
1306	/// + recipe; this is that struct in one place.
1307	fn test_state(pool: SqlitePool, topo: Topology, cfg: Config) -> AppState {
1308	let executors = Arc::new(crate::state::build_executors(&topo));
1309	let syncs = Arc::new(crate::state::build_syncs(&topo));
1310	let host_locks = crate::state::build_host_locks(&topo);
1311	AppState {
1312	pool,
1313	topo: Arc::new(topo),
1314	cfg: Arc::new(cfg),
1315	prom: crate::metrics::test_handle(),
1316	events: crate::events::channel(),
1317	ota: Arc::new(OtaRegistry::standard("https://makenot.work")),
1318	executors,
1319	syncs,
1320	active: Arc::new(Mutex::new(HashMap::new())),
1321	api_token: None,
1322	host_locks,
1323	distribution: Arc::new(Mutex::new(HashMap::new())),
1324	http: crate::tls::builder().build().unwrap(),
1325	// Port 1 refuses instantly. A test must never probe production, and
1326	// a refusal is also faster than any timeout would be.
1327	mnw_base_url: "http://127.0.0.1:1".into(),
1328	}
1329	}
1330
1331	/// A `kind = "service"` release end to end: build, `glibc_check`, `deploy`,
1332	/// and a health assertion the recipe makes itself against the service host.
1333	///
1334	/// The whole point of the deploy step is that it dispatches to a host
1335	/// OUTSIDE the build topology, so this exercises the resolution
1336	/// (target -> `[[deploy]]` entry -> executor registered for the run), the
1337	/// staging, and the call into the privileged installer — with a fake
1338	/// installer standing in for the root script, which is the one part a test
1339	/// cannot run for real.
1340	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1341	async fn service_recipe_builds_then_deploys_and_verifies() {
1342	let tmp = tempfile::tempdir().unwrap();
1343	let root = tmp.path();
1344	let repo = root.join("svc");
1345	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1346	std::fs::write(repo.join("Cargo.toml"), "[package]\nversion = \"0.4.0\"\n").unwrap();
1347
1348	// Stand-in for the root installer: same three arguments, records what it
1349	// was asked to do instead of writing to /usr/local/bin and restarting a
1350	// unit. `install` + a marker file, so the test can assert the binary
1351	// that arrived is the binary that was built.
1352	let installer = root.join("install-service.sh");
1353	std::fs::write(
1354	&installer,
1355	"#!/bin/sh\nset -eu\ninstall -m 0755 \"$1\" \"$2\"\necho \"restarted $3\" >> \"$2.log\"\n",
1356	)
1357	.unwrap();
1358	std::fs::set_permissions(
1359	&installer,
1360	<std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o755),
1361	)
1362	.unwrap();
1363	let install_path = root.join("bin/svc");
1364	std::fs::create_dir_all(root.join("bin")).unwrap();
1365
1366	std::fs::write(
1367	repo.join("dist/recipes/linux.rhai"),
1368	r#"
1369	let v = version();
1370	step("build");
1371	sh_ok("fw13", "mkdir -p REPO/target/release && echo built-BIN > REPO/target/release/svc");
1372	step("verify");
1373	log(glibc_check("REPO/target/release/svc"));
1374	step("deploy");
1375	log(deploy("REPO/target/release/svc"));
1376	// The recipe owns what "healthy" means, and asserts it itself
1377	// against the host it just restarted.
1378	sh_ok(deploy_host(), "test -x " + install_path());
1379	"#
1380	.replace("REPO", repo.to_str().unwrap())
1381	.replace("BIN", "0.4.0"),
1382	)
1383	.unwrap();
1384
1385	std::fs::write(
1386	repo.join("bento.toml"),
1387	format!(
1388	r#"kind = "service"
1389	targets = ["linux/x86_64"]
1390	version_path = "Cargo.toml"
1391
1392	[[deploy]]
1393	target = "linux/x86_64"
1394	host = "local"
1395	install_path = "{}"
1396	service = "svc.service"
1397	health_url = "http://localhost:9100/api/health"
1398	"#,
1399	install_path.display()
1400	),
1401	)
1402	.unwrap();
1403
1404	let mut cfg = Config::for_tests(root);
1405	cfg.deploy_installer = installer.display().to_string();
1406	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1407	let topo = Topology::from_str_for_tests(&format!(
1408	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
1409	pull_root = \"{repo}\"\n\n[app.svc]\nrepo = \"{repo}\"\n",
1410	repo = repo.display()
1411	))
1412	.unwrap();
1413	let state = test_state(pool.clone(), topo, cfg);
1414
1415	let build_id = start_build(
1416	state.clone(),
1417	AppId::new("svc"),
1418	Version::parse("0.4.0").unwrap(),
1419	vec!["linux/x86_64".parse().unwrap()],
1420	)
1421	.await
1422	.unwrap();
1423
1424	let mut status = String::new();
1425	for _ in 0..100 {
1426	status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1427	.bind(build_id)
1428	.fetch_optional(&pool)
1429	.await
1430	.unwrap()
1431	.unwrap_or_else(\|\| "running".to_string());
1432	if status != "running" {
1433	break;
1434	}
1435	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1436	}
1437	assert_eq!(status, "ok", "service run should succeed");
1438
1439	let steps: Vec<(String, String)> = sqlx::query_as(
1440	"SELECT step, status FROM step_runs WHERE target_run_id IN \
1441	(SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id",
1442	)
1443	.bind(build_id)
1444	.fetch_all(&pool)
1445	.await
1446	.unwrap();
1447	assert_eq!(
1448	steps.iter().map(\|(s, _)\| s.as_str()).collect::<Vec<_>>(),
1449	vec!["build", "verify", "deploy"],
1450	"a service ends at deploy, not collect"
1451	);
1452	assert!(steps.iter().all(\|(_, st)\| st == "ok"), "{steps:?}");
1453
1454	// The bytes that were built are the bytes that landed, and the unit was
1455	// restarted only after the install succeeded.
1456	assert_eq!(
1457	std::fs::read_to_string(&install_path).unwrap().trim(),
1458	"built-0.4.0"
1459	);
1460	assert!(
1461	std::fs::read_to_string(install_path.with_extension("").with_file_name("svc.log"))
1462	.unwrap()
1463	.contains("restarted svc.service")
1464	);
1465	}
1466
1467	/// Stand up a tmp app repo + topology and run a real local recipe end to
1468	/// end: step transitions, streamed `sh_ok`, `version_of`, `log`, and a
1469	/// `collect` that pulls a built artifact into dist_root.
1470	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1471	async fn local_linux_recipe_runs_end_to_end() {
1472	let tmp = tempfile::tempdir().unwrap();
1473	let root = tmp.path();
1474
1475	// Fake app checkout: tauri.conf.json + a linux recipe.
1476	let repo = root.join("app");
1477	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1478	std::fs::write(
1479	repo.join("src-tauri/tauri.conf.json"),
1480	r#"{"version":"0.0.1"}"#,
1481	)
1482	.unwrap();
1483	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1484	// Build writes an artifact into the repo; collect pulls it to dist_root.
1485	std::fs::write(
1486	repo.join("dist/recipes/linux.rhai"),
1487	r#"
1488	step("build");
1489	let v = version_of("demo");
1490	log("building demo " + v);
1491	sh_ok("fw13", "echo compiling; mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
1492	step("collect");
1493	collect("fw13", "REPO/out/demo.bin", "demo", v);
1494	"#
1495	.replace("REPO", repo.to_str().unwrap()),
1496	)
1497	.unwrap();
1498
1499	let mut cfg = Config::for_tests(root);
1500	// Archive to a local directory, so the deposit a real release makes to
1501	// astra runs on the same code path here.
1502	cfg.archive = Some(crate::config::Archive {
1503	host: "local".into(),
1504	root: root.join("archive"),
1505	});
1506	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1507	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1508	let topo = Topology::from_str_for_tests(&format!(
1509	r#"
1510	[[host]]
1511	name = "fw13"
1512	ssh = "local"
1513	targets = ["linux/x86_64"]
1514	pull_root = "{repo}"
1515
1516	[app.demo]
1517	repo = "{repo}"
1518	"#,
1519	repo = repo.display()
1520	))
1521	.unwrap();
1522
1523	let state = test_state(pool.clone(), topo, cfg);
1524
1525	let app = AppId::new("demo");
1526	let version = Version::parse("0.0.1").unwrap();
1527	let build_id = start_build(
1528	state.clone(),
1529	app,
1530	version,
1531	vec!["linux/x86_64".parse().unwrap()],
1532	)
1533	.await
1534	.unwrap();
1535
1536	// Wait for the target run to settle. The row may not be inserted on the
1537	// first poll (run_target is spawned, not awaited), so treat a missing row
1538	// as still-pending rather than an error.
1539	let mut status = String::new();
1540	for _ in 0..100 {
1541	status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1542	.bind(build_id)
1543	.fetch_optional(&pool)
1544	.await
1545	.unwrap()
1546	.unwrap_or_else(\|\| "running".to_string());
1547	if status != "running" {
1548	break;
1549	}
1550	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1551	}
1552	assert_eq!(status, "ok", "target run should succeed");
1553
1554	// Both steps recorded and finished ok.
1555	let steps: Vec<(String, String)> =
1556	sqlx::query_as("SELECT step, status FROM step_runs WHERE target_run_id IN (SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id")
1557	.bind(build_id)
1558	.fetch_all(&pool)
1559	.await
1560	.unwrap();
1561	let names: Vec<&str> = steps.iter().map(\|(s, _)\| s.as_str()).collect();
1562	assert_eq!(names, vec!["build", "collect"]);
1563	assert!(steps.iter().all(\|(_, st)\| st == "ok"));
1564
1565	// Artifact landed in dist_root, under its own target, and a step log was
1566	// written. Both trees are keyed the same way, `<app>/<version>/<target>/`.
1567	let artifact = state.cfg.dist_root.join("demo/0.0.1/linux-x86_64/demo.bin");
1568	assert!(artifact.exists(), "collect should copy the artifact");
1569	// ...and the same bytes reached the archive, at the same path under its
1570	// own root. This is the answer to "where is demo 0.0.1 for linux".
1571	let archived = root.join("archive/demo/0.0.1/linux-x86_64/demo.bin");
1572	assert!(
1573	archived.exists(),
1574	"collect should deposit into the archive: {}",
1575	archived.display()
1576	);
1577	assert_eq!(
1578	std::fs::read(&archived).unwrap(),
1579	std::fs::read(&artifact).unwrap()
1580	);
1581	// Read the path off the ledger rather than rebuilding it: the log is
1582	// named for its step run id, and the point of that is that the row
1583	// resolves to exactly one file.
1584	let (run_id, log_ref): (i64, String) = sqlx::query_as(
1585	"SELECT id, log_ref FROM step_runs WHERE step = 'build' AND target_run_id IN \
1586	(SELECT id FROM target_runs WHERE build_id = ?)",
1587	)
1588	.bind(build_id)
1589	.fetch_one(&pool)
1590	.await
1591	.unwrap();
1592	let log = std::path::PathBuf::from(&log_ref);
1593	assert_eq!(
1594	log,
1595	state
1596	.cfg
1597	.logs_root
1598	.join(format!("demo/0.0.1/linux-x86_64/build.{run_id}.log")),
1599	"log path should be keyed on the step run id"
1600	);
1601	assert!(log.exists(), "build step log should exist");
1602	let body = std::fs::read_to_string(&log).unwrap();
1603	assert!(body.contains("compiling"));
1604	assert!(
1605	body.starts_with(&format!(
1606	"=== bento demo 0.0.1 linux/x86_64 step=build run_id={run_id} "
1607	)),
1608	"log should open with a run header naming its run: {body}"
1609	);
1610	}
1611
1612	/// Multi-target fan-out, which is what the daemon exists for: every other
1613	/// runner test drives exactly one target, so nothing covered `start_build`'s
1614	/// `JoinSet` fan-out or `finalize_build`'s rollup. One target fails and one
1615	/// succeeds — the failure must not abort its sibling, both must land their
1616	/// own terminal row, and the build must finalize `failed` because any failed
1617	/// target fails the build.
1618	#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1619	async fn multi_target_fan_out_rolls_up_partial_failure() {
1620	let tmp = tempfile::tempdir().unwrap();
1621	let root = tmp.path();
1622
1623	let repo = root.join("app");
1624	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1625	std::fs::write(
1626	repo.join("src-tauri/tauri.conf.json"),
1627	r#"{"version":"0.0.1"}"#,
1628	)
1629	.unwrap();
1630	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1631
1632	// Linux succeeds and collects a real artifact. The artifact is the proof
1633	// this target ran to completion rather than being torn down when its
1634	// sibling failed.
1635	std::fs::write(
1636	repo.join("dist/recipes/linux.rhai"),
1637	r#"
1638	step("build");
1639	let v = version_of("demo");
1640	sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
1641	step("collect");
1642	collect("fw13", "REPO/out/demo.bin", "demo", v);
1643	"#
1644	.replace("REPO", repo.to_str().unwrap()),
1645	)
1646	.unwrap();
1647	// Windows fails in its first step.
1648	std::fs::write(
1649	repo.join("dist/recipes/windows.rhai"),
1650	r#"
1651	step("build");
1652	sh_ok("winbox", "echo nope 1>&2; exit 1");
1653	"#,
1654	)
1655	.unwrap();
1656
1657	let cfg = Config::for_tests(root);
1658	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1659	std::fs::write(
1660	repo.join("bento.toml"),
1661	"targets = [\"linux/x86_64\", \"windows/x86_64\"]\n",
1662	)
1663	.unwrap();
1664	// Two hosts so each target resolves its own, mirroring the real
1665	// per-architecture topology.
1666	let topo = Topology::from_str_for_tests(&format!(
1667	r#"
1668	[[host]]
1669	name = "fw13"
1670	ssh = "local"
1671	targets = ["linux/x86_64"]
1672	pull_root = "{repo}"
1673
1674	[[host]]
1675	name = "winbox"
1676	ssh = "local"
1677	targets = ["windows/x86_64"]
1678
1679	[app.demo]
1680	repo = "{repo}"
1681	"#,
1682	repo = repo.display()
1683	))
1684	.unwrap();
1685
1686	let state = test_state(pool.clone(), topo, cfg);
1687
1688	let build_id = start_build(
1689	state.clone(),
1690	AppId::new("demo"),
1691	Version::parse("0.0.1").unwrap(),
1692	vec![
1693	"linux/x86_64".parse().unwrap(),
1694	"windows/x86_64".parse().unwrap(),
1695	],
1696	)
1697	.await
1698	.unwrap();
1699
1700	// Poll the build row, not the target rows: the build is terminal only
1701	// once finalize_build has joined every task and stamped it.
1702	let mut build_status = String::new();
1703	for _ in 0..200 {
1704	build_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
1705	.bind(build_id)
1706	.fetch_one(&pool)
1707	.await
1708	.unwrap();
1709	if build_status != "running" {
1710	break;
1711	}
1712	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1713	}
1714
1715	let runs: Vec<(String, String)> = sqlx::query_as(
1716	"SELECT target, status FROM target_runs WHERE build_id = ? ORDER BY target",
1717	)
1718	.bind(build_id)
1719	.fetch_all(&pool)
1720	.await
1721	.unwrap();
1722	assert_eq!(
1723	runs,
1724	vec![
1725	("linux/x86_64".to_string(), "ok".to_string()),
1726	("windows/x86_64".to_string(), "failed".to_string()),
1727	],
1728	"each target lands its own terminal row; one failing does not take the other down",
1729	);
1730
1731	// The surviving target finished its work, not merely its row.
1732	assert!(
1733	state
1734	.cfg
1735	.dist_root
1736	.join("demo/0.0.1/linux-x86_64/demo.bin")
1737	.exists(),
1738	"the succeeding target ran to completion and collected its artifact",
1739	);
1740
1741	// The failure is attributed to the target that failed, and only it.
1742	let err: Option<String> = sqlx::query_scalar(
1743	"SELECT error FROM target_runs WHERE build_id = ? AND target = 'windows/x86_64'",
1744	)
1745	.bind(build_id)
1746	.fetch_one(&pool)
1747	.await
1748	.unwrap();
1749	assert!(
1750	err.is_some_and(\|e\| !e.is_empty()),
1751	"a failed target records why",
1752	);
1753
1754	assert_eq!(
1755	build_status, "failed",
1756	"any failed target fails the build; a partial release must not read as ok",
1757	);
1758	let finished: Option<String> =
1759	sqlx::query_scalar("SELECT finished_at FROM builds WHERE id = ?")
1760	.bind(build_id)
1761	.fetch_one(&pool)
1762	.await
1763	.unwrap();
1764	assert!(finished.is_some(), "finalize_build stamps the finish time");
1765
1766	// finalize_build reaps its own latest-wins slots, so nothing is left
1767	// in flight to block a later build of the same targets.
1768	assert!(
1769	state.active.lock().await.is_empty(),
1770	"finalize_build reaps the slots it owned",
1771	);
1772	}
1773
1774	/// Latest-wins supersession, driven through `start_build` (not the leaf
1775	/// `begin_step` bail that's already covered). Two builds of the SAME
1776	/// (app, target) race: the second must cooperatively cancel + abort the
1777	/// first and take the single slot, the first must terminate non-`ok`, and
1778	/// the second must run to completion.
1779	#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1780	async fn a_newer_build_supersedes_the_in_flight_one_for_the_same_target() {
1781	let tmp = tempfile::tempdir().unwrap();
1782	let root = tmp.path();
1783
1784	let repo = root.join("app");
1785	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1786	std::fs::write(
1787	repo.join("src-tauri/tauri.conf.json"),
1788	r#"{"version":"0.0.1"}"#,
1789	)
1790	.unwrap();
1791	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1792	// A sleep long enough that the first build is still mid-recipe when the
1793	// second arrives; the engine checks the cancel flag at the step boundary
1794	// before "done", so the superseded run bails rather than finishing.
1795	std::fs::write(
1796	repo.join("dist/recipes/linux.rhai"),
1797	r#"
1798	step("build");
1799	sh_ok("fw13", "sleep 2");
1800	"#,
1801	)
1802	.unwrap();
1803
1804	let cfg = Config::for_tests(root);
1805	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1806	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1807	let topo = Topology::from_str_for_tests(&format!(
1808	r#"
1809	[[host]]
1810	name = "fw13"
1811	ssh = "local"
1812	targets = ["linux/x86_64"]
1813
1814	[app.demo]
1815	repo = "{}"
1816	"#,
1817	repo.display()
1818	))
1819	.unwrap();
1820	let state = test_state(pool.clone(), topo, cfg);
1821
1822	let target = "linux/x86_64".parse().unwrap();
1823	let first = start_build(
1824	state.clone(),
1825	AppId::new("demo"),
1826	Version::parse("0.0.1").unwrap(),
1827	vec![target],
1828	)
1829	.await
1830	.unwrap();
1831	// Let the first build register its slot and enter the recipe before the
1832	// second supersedes it (start_build inserts the slot synchronously, but
1833	// the recipe runs on a spawned task).
1834	tokio::time::sleep(std::time::Duration::from_millis(100)).await;
1835	let second = start_build(
1836	state.clone(),
1837	AppId::new("demo"),
1838	Version::parse("0.0.1").unwrap(),
1839	vec![target],
1840	)
1841	.await
1842	.unwrap();
1843	assert_ne!(first, second);
1844
1845	// Latest wins: exactly one slot for the key, owned by the second build.
1846	{
1847	let active = state.active.lock().await;
1848	assert_eq!(active.len(), 1, "supersession must not leave two slots");
1849	assert_eq!(
1850	active.values().next().unwrap().build_id,
1851	second,
1852	"the surviving slot belongs to the newer build",
1853	);
1854	}
1855
1856	// Wait for the second (surviving) build to finalize.
1857	let mut second_status = String::new();
1858	for _ in 0..200 {
1859	second_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
1860	.bind(second)
1861	.fetch_one(&pool)
1862	.await
1863	.unwrap();
1864	if second_status != "running" {
1865	break;
1866	}
1867	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1868	}
1869	assert_eq!(
1870	second_status, "ok",
1871	"the superseding build runs to completion"
1872	);
1873
1874	// The first build's target run terminated without succeeding — it was
1875	// cancelled/aborted, never stamped `ok`.
1876	let first_status: String =
1877	sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1878	.bind(first)
1879	.fetch_one(&pool)
1880	.await
1881	.unwrap();
1882	assert_ne!(
1883	first_status, "ok",
1884	"the superseded build must not complete successfully",
1885	);
1886
1887	// Both builds reaped their slots; nothing left in flight.
1888	assert!(
1889	state.active.lock().await.is_empty(),
1890	"every finalized build reaps its own slot",
1891	);
1892	}
1893
1894	/// A failing `preflight` (the agent host's `/health` probe when `ops-agent`
1895	/// is down) must fail the target BEFORE the recipe dispatches — the whole
1896	/// point of preflighting. Covered only via a fake here, since a real
1897	/// LocalExec/SshExec preflight is a no-op that always passes.
1898	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1899	async fn a_failing_preflight_fails_the_target_before_the_recipe_runs() {
1900	let tmp = tempfile::tempdir().unwrap();
1901	let root = tmp.path();
1902
1903	let repo = root.join("app");
1904	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1905	std::fs::write(
1906	repo.join("src-tauri/tauri.conf.json"),
1907	r#"{"version":"0.0.1"}"#,
1908	)
1909	.unwrap();
1910	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1911	// If the recipe ran it would drop this marker; preflight failing first
1912	// means it never does.
1913	let marker = root.join("recipe-ran");
1914	std::fs::write(
1915	repo.join("dist/recipes/linux.rhai"),
1916	r#"
1917	step("build");
1918	sh_ok("fw13", "touch MARKER");
1919	"#
1920	.replace("MARKER", marker.to_str().unwrap()),
1921	)
1922	.unwrap();
1923
1924	let cfg = Config::for_tests(root);
1925	let pool = crate::db::open(&cfg.db_path).await.unwrap();
1926	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1927	let topo = Topology::from_str_for_tests(&format!(
1928	r#"
1929	[[host]]
1930	name = "fw13"
1931	ssh = "local"
1932	targets = ["linux/x86_64"]
1933
1934	[app.demo]
1935	repo = "{}"
1936	"#,
1937	repo.display()
1938	))
1939	.unwrap();
1940	let mut state = test_state(pool.clone(), topo, cfg);
1941	// Swap fw13's executor for one whose preflight fails.
1942	let mut execs = HashMap::new();
1943	execs.insert(
1944	"fw13".to_string(),
1945	FakeExec::preflight_fails("ops-agent not reachable at /health"),
1946	);
1947	state.executors = Arc::new(execs);
1948
1949	let build_id = start_build(
1950	state.clone(),
1951	AppId::new("demo"),
1952	Version::parse("0.0.1").unwrap(),
1953	vec!["linux/x86_64".parse().unwrap()],
1954	)
1955	.await
1956	.unwrap();
1957
1958	let mut status = String::new();
1959	let mut error = String::new();
1960	for _ in 0..100 {
1961	let row: Option<(String, Option<String>)> =
1962	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1963	.bind(build_id)
1964	.fetch_optional(&pool)
1965	.await
1966	.unwrap();
1967	if let Some((s, e)) = row {
1968	status = s;
1969	error = e.unwrap_or_default();
1970	if status != "running" {
1971	break;
1972	}
1973	}
1974	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1975	}
1976	assert_eq!(status, "failed", "a failed preflight must fail the target");
1977	assert!(
1978	error.contains("ops-agent not reachable"),
1979	"the failure must carry the preflight error, got: {error}"
1980	);
1981	assert!(
1982	!marker.exists(),
1983	"the recipe must NOT run when preflight fails"
1984	);
1985	}
1986
1987	/// A target whose recipe file is absent must fail at the `checkout` boundary
1988	/// with a readable "reading recipe" error, before any step runs — exercising
1989	/// `read_recipe`'s error branch and the `fail_target` path in `run_target`
1990	/// that neither the happy-path nor the failing-preflight test reaches. The
1991	/// app is configured to ship linux, but no `linux.rhai` is written.
1992	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1993	async fn a_target_with_no_recipe_file_fails_at_checkout() {
1994	let tmp = tempfile::tempdir().unwrap();
1995	let root = tmp.path();
1996
1997	let repo = root.join("app");
1998	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1999	std::fs::write(
2000	repo.join("src-tauri/tauri.conf.json"),
2001	r#"{"version":"0.0.1"}"#,
2002	)
2003	.unwrap();
2004	// The recipe dir exists but is empty — no linux.rhai.
2005	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2006
2007	let cfg = Config::for_tests(root);
2008	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2009	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2010	let topo = Topology::from_str_for_tests(&format!(
2011	r#"
2012	[[host]]
2013	name = "fw13"
2014	ssh = "local"
2015	targets = ["linux/x86_64"]
2016
2017	[app.demo]
2018	repo = "{}"
2019	"#,
2020	repo.display()
2021	))
2022	.unwrap();
2023	let state = test_state(pool.clone(), topo, cfg);
2024
2025	let build_id = start_build(
2026	state.clone(),
2027	AppId::new("demo"),
2028	Version::parse("0.0.1").unwrap(),
2029	vec!["linux/x86_64".parse().unwrap()],
2030	)
2031	.await
2032	.unwrap();
2033
2034	// Poll the build row: it is terminal only once finalize_build joins the
2035	// one (failing) target task.
2036	let mut build_status = String::new();
2037	for _ in 0..100 {
2038	build_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
2039	.bind(build_id)
2040	.fetch_one(&pool)
2041	.await
2042	.unwrap();
2043	if build_status != "running" {
2044	break;
2045	}
2046	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2047	}
2048
2049	// The target failed, and the error points at the unreadable recipe.
2050	let (status, error): (String, Option<String>) =
2051	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2052	.bind(build_id)
2053	.fetch_one(&pool)
2054	.await
2055	.unwrap();
2056	assert_eq!(status, "failed", "a missing recipe must fail the target");
2057	assert!(
2058	error.is_some_and(\|e\| e.contains("reading recipe")),
2059	"the failure must name the recipe it could not read",
2060	);
2061
2062	// The recipe never ran, so no step_runs row was ever created — the
2063	// failure is at the checkout boundary, ahead of any step.
2064	let step_count: i64 = sqlx::query_scalar(
2065	"SELECT COUNT(*) FROM step_runs WHERE target_run_id IN \
2066	(SELECT id FROM target_runs WHERE build_id = ?)",
2067	)
2068	.bind(build_id)
2069	.fetch_one(&pool)
2070	.await
2071	.unwrap();
2072	assert_eq!(
2073	step_count, 0,
2074	"no step should run when the recipe is absent"
2075	);
2076
2077	assert_eq!(
2078	build_status, "failed",
2079	"the build rolls up the target failure",
2080	);
2081	assert!(
2082	state.active.lock().await.is_empty(),
2083	"finalize_build reaps the slot even on the recipe-read failure path",
2084	);
2085	}
2086
2087	/// Run 2 S5: a publish whose version is not strictly newer than the latest
2088	/// already published for the same (app, target, channel) is refused — an
2089	/// older build cannot republish over a live newer release.
2090	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2091	async fn publish_rejects_a_non_monotonic_version() {
2092	let tmp = tempfile::tempdir().unwrap();
2093	let root = tmp.path();
2094	let repo = root.join("app");
2095	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2096	std::fs::write(
2097	repo.join("src-tauri/tauri.conf.json"),
2098	r#"{"version":"0.2.0"}"#,
2099	)
2100	.unwrap();
2101	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2102	let artifact = repo.join("out/app.tar.gz");
2103	// Build an artifact, publish 0.2.0 (records a release), then try to
2104	// publish the older 0.1.0 — the second publish must fail.
2105	std::fs::write(
2106	repo.join("dist/recipes/linux.rhai"),
2107	r#"
2108	step("build");
2109	sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
2110	step("publish");
2111	publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
2112	publish("tauri-mnw", "demo", "linux/x86_64", "0.1.0", "ARTIFACT", #{});
2113	"#
2114	.replace("ARTIFACT", artifact.to_str().unwrap())
2115	.replace("REPO", repo.to_str().unwrap()),
2116	)
2117	.unwrap();
2118
2119	let cfg = Config::for_tests(root);
2120	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2121	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2122	let topo = Topology::from_str_for_tests(&format!(
2123	r#"
2124	[[host]]
2125	name = "fw13"
2126	ssh = "local"
2127	targets = ["linux/x86_64"]
2128
2129	[app.demo]
2130	repo = "{}"
2131	"#,
2132	repo.display()
2133	))
2134	.unwrap();
2135	let state = test_state(pool.clone(), topo, cfg);
2136	let build_id = start_build(
2137	state.clone(),
2138	AppId::new("demo"),
2139	Version::parse("0.2.0").unwrap(),
2140	vec!["linux/x86_64".parse().unwrap()],
2141	)
2142	.await
2143	.unwrap();
2144
2145	let mut status = String::new();
2146	let mut error = String::new();
2147	for _ in 0..100 {
2148	let row: Option<(String, Option<String>)> =
2149	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2150	.bind(build_id)
2151	.fetch_optional(&pool)
2152	.await
2153	.unwrap();
2154	if let Some((s, e)) = row {
2155	status = s;
2156	error = e.unwrap_or_default();
2157	if status != "running" {
2158	break;
2159	}
2160	}
2161	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2162	}
2163	assert_eq!(status, "failed", "non-monotonic publish must fail the run");
2164	assert!(
2165	error.contains("not newer"),
2166	"expected monotonicity error, got: {error}"
2167	);
2168	// Exactly one release was recorded (0.2.0); 0.1.0 never landed.
2169	let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
2170	.fetch_one(&pool)
2171	.await
2172	.unwrap();
2173	assert_eq!(
2174	count, 1,
2175	"only the first (newer) publish should record a release"
2176	);
2177	// The recorded release carries the artifact's sha256 (64 hex chars),
2178	// not a NULL — the ledger says which bytes shipped.
2179	let hash: Option<String> =
2180	sqlx::query_scalar("SELECT artifact_hash FROM releases WHERE version = '0.2.0'")
2181	.fetch_one(&pool)
2182	.await
2183	.unwrap();
2184	assert!(
2185	hash.as_deref().is_some_and(\|h\| h.len() == 64),
2186	"publish must record the artifact sha256, got {hash:?}"
2187	);
2188	}
2189
2190	/// The artifact-identity fix at collect: a stale, wrongly-versioned artifact
2191	/// left in the output dir fails the collect instead of silently winning a
2192	/// later glob. Here the build is 0.0.1 but the recipe produces a 9.9.9 file.
2193	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2194	async fn collect_rejects_a_stale_versioned_artifact() {
2195	let tmp = tempfile::tempdir().unwrap();
2196	let root = tmp.path();
2197	let repo = root.join("app");
2198	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2199	std::fs::write(
2200	repo.join("src-tauri/tauri.conf.json"),
2201	r#"{"version":"0.0.1"}"#,
2202	)
2203	.unwrap();
2204	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2205	std::fs::write(
2206	repo.join("dist/recipes/linux.rhai"),
2207	r#"
2208	step("build");
2209	let v = version_of("demo");
2210	sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo-9.9.9.bin");
2211	step("collect");
2212	collect("fw13", "REPO/out/demo-9.9.9.bin", "demo", v);
2213	"#
2214	.replace("REPO", repo.to_str().unwrap()),
2215	)
2216	.unwrap();
2217
2218	let cfg = Config::for_tests(root);
2219	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2220	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2221	let topo = Topology::from_str_for_tests(&format!(
2222	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2223	pull_root = \"{repo}\"\n\n[app.demo]\nrepo = \"{repo}\"\n",
2224	repo = repo.display()
2225	))
2226	.unwrap();
2227	let state = test_state(pool.clone(), topo, cfg);
2228	let build_id = start_build(
2229	state.clone(),
2230	AppId::new("demo"),
2231	Version::parse("0.0.1").unwrap(),
2232	vec!["linux/x86_64".parse().unwrap()],
2233	)
2234	.await
2235	.unwrap();
2236
2237	let mut status = String::new();
2238	let mut error = String::new();
2239	for _ in 0..100 {
2240	let row: Option<(String, Option<String>)> =
2241	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2242	.bind(build_id)
2243	.fetch_optional(&pool)
2244	.await
2245	.unwrap();
2246	if let Some((s, e)) = row {
2247	status = s;
2248	error = e.unwrap_or_default();
2249	if status != "running" {
2250	break;
2251	}
2252	}
2253	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2254	}
2255	assert_eq!(
2256	status, "failed",
2257	"a mismatched-version artifact must fail collect"
2258	);
2259	assert!(
2260	error.contains("stale artifact"),
2261	"expected a stale-artifact error, got: {error}"
2262	);
2263	}
2264
2265	/// A command that runs past its step's deadline fails THAT step and unwinds
2266	/// the recipe. The per-step budget is overridden to 1s here; the recipe then
2267	/// sleeps 30s, so the step deadline, not the sleep, decides the outcome.
2268	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2269	async fn a_step_that_exceeds_its_deadline_fails() {
2270	let tmp = tempfile::tempdir().unwrap();
2271	let root = tmp.path();
2272	let repo = root.join("app");
2273	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2274	std::fs::write(
2275	repo.join("src-tauri/tauri.conf.json"),
2276	r#"{"version":"0.0.1"}"#,
2277	)
2278	.unwrap();
2279	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2280	std::fs::write(
2281	repo.join("dist/recipes/linux.rhai"),
2282	r#"
2283	step("build");
2284	sh_ok("fw13", "sleep 30");
2285	"#,
2286	)
2287	.unwrap();
2288
2289	let mut cfg = Config::for_tests(root);
2290	cfg.step_timeout_secs = Some(1); // every step's budget -> 1s
2291	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2292	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2293	let topo = Topology::from_str_for_tests(&format!(
2294	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2295	[app.demo]\nrepo = \"{}\"\n",
2296	repo.display()
2297	))
2298	.unwrap();
2299	let state = test_state(pool.clone(), topo, cfg);
2300	let started = std::time::Instant::now();
2301	let build_id = start_build(
2302	state.clone(),
2303	AppId::new("demo"),
2304	Version::parse("0.0.1").unwrap(),
2305	vec!["linux/x86_64".parse().unwrap()],
2306	)
2307	.await
2308	.unwrap();
2309
2310	let mut status = String::new();
2311	let mut error = String::new();
2312	for _ in 0..100 {
2313	let row: Option<(String, Option<String>)> =
2314	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2315	.bind(build_id)
2316	.fetch_optional(&pool)
2317	.await
2318	.unwrap();
2319	if let Some((s, e)) = row {
2320	status = s;
2321	error = e.unwrap_or_default();
2322	if status != "running" {
2323	break;
2324	}
2325	}
2326	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2327	}
2328	assert_eq!(status, "failed", "a step past its deadline must fail");
2329	assert!(
2330	error.contains("per-step deadline"),
2331	"expected a deadline error, got: {error}"
2332	);
2333	// The deadline (1s), not the 30s sleep, decided it — proof the command
2334	// was actually interrupted rather than run to completion.
2335	assert!(
2336	started.elapsed() < std::time::Duration::from_secs(20),
2337	"the step deadline must fire well before the sleep would finish"
2338	);
2339	}
2340
2341	/// A demo app that publishes linux and has the all-targets-green gate on; it
2342	/// declares linux + macos, so publishing linux is gated on macos being green.
2343	async fn gate_state(root: &std::path::Path) -> (AppState, sqlx::SqlitePool) {
2344	let repo = root.join("app");
2345	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2346	std::fs::write(
2347	repo.join("src-tauri/tauri.conf.json"),
2348	r#"{"version":"0.2.0"}"#,
2349	)
2350	.unwrap();
2351	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2352	let artifact = repo.join("out/app.tar.gz");
2353	std::fs::write(
2354	repo.join("dist/recipes/linux.rhai"),
2355	r#"
2356	step("build");
2357	sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
2358	step("publish");
2359	publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
2360	"#
2361	.replace("ARTIFACT", artifact.to_str().unwrap())
2362	.replace("REPO", repo.to_str().unwrap()),
2363	)
2364	.unwrap();
2365	std::fs::write(
2366	repo.join("bento.toml"),
2367	"targets = [\"linux/x86_64\", \"macos/aarch64\"]\nrequire_all_targets = true\n",
2368	)
2369	.unwrap();
2370	let cfg = Config::for_tests(root);
2371	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2372	let topo = Topology::from_str_for_tests(&format!(
2373	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2374	[[host]]\nname = \"mbp\"\nssh = \"mbp\"\ntargets = [\"macos/aarch64\"]\n\
2375	[app.demo]\nrepo = \"{}\"\n",
2376	repo.display()
2377	))
2378	.unwrap();
2379	(test_state(pool.clone(), topo, cfg), pool)
2380	}
2381
2382	async fn await_target(pool: &sqlx::SqlitePool, build_id: i64) -> (String, String) {
2383	for _ in 0..100 {
2384	let row: Option<(String, Option<String>)> =
2385	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2386	.bind(build_id)
2387	.fetch_optional(pool)
2388	.await
2389	.unwrap();
2390	if let Some((s, e)) = row
2391	&& s != "running"
2392	{
2393	return (s, e.unwrap_or_default());
2394	}
2395	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2396	}
2397	panic!("target never settled");
2398	}
2399
2400	/// The all-targets-green gate blocks a partial release: linux tries to
2401	/// publish while macos has no successful run, so publish is refused and the
2402	/// target fails at its publish step.
2403	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2404	async fn all_green_gate_blocks_publish_when_a_sibling_is_not_green() {
2405	let tmp = tempfile::tempdir().unwrap();
2406	let (state, pool) = gate_state(tmp.path()).await;
2407	let build_id = start_build(
2408	state,
2409	AppId::new("demo"),
2410	Version::parse("0.2.0").unwrap(),
2411	vec!["linux/x86_64".parse().unwrap()],
2412	)
2413	.await
2414	.unwrap();
2415	let (status, error) = await_target(&pool, build_id).await;
2416	assert_eq!(status, "failed", "a partial release must be blocked");
2417	assert!(
2418	error.contains("all-targets-green gate"),
2419	"expected the gate to name itself, got: {error}"
2420	);
2421	// Nothing shipped.
2422	let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
2423	.fetch_one(&pool)
2424	.await
2425	.unwrap();
2426	assert_eq!(count, 0, "a gated-off publish records no release");
2427	}
2428
2429	/// With every sibling green, the gate lets the publish through. macos is
2430	/// pre-recorded `ok` for this version, so linux's publish proceeds.
2431	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2432	async fn all_green_gate_allows_publish_when_every_sibling_is_green() {
2433	let tmp = tempfile::tempdir().unwrap();
2434	let (state, pool) = gate_state(tmp.path()).await;
2435	// Pre-record a green macos run at 0.2.0 (as if its target already built).
2436	let bid: i64 = sqlx::query_scalar(
2437	"INSERT INTO builds (app, version, status, created_at) VALUES ('demo','0.2.0','ok','2026-07-23T00:00:00Z') RETURNING id",
2438	)
2439	.fetch_one(&pool)
2440	.await
2441	.unwrap();
2442	sqlx::query(
2443	"INSERT INTO target_runs (build_id, app, version, target, status, started_at)
2444	VALUES (?, 'demo', '0.2.0', 'macos/aarch64', 'ok', '2026-07-23T00:00:00Z')",
2445	)
2446	.bind(bid)
2447	.execute(&pool)
2448	.await
2449	.unwrap();
2450
2451	let build_id = start_build(
2452	state,
2453	AppId::new("demo"),
2454	Version::parse("0.2.0").unwrap(),
2455	vec!["linux/x86_64".parse().unwrap()],
2456	)
2457	.await
2458	.unwrap();
2459	let (status, error) = await_target(&pool, build_id).await;
2460	assert_eq!(
2461	status, "ok",
2462	"all siblings green -> publish proceeds ({error})"
2463	);
2464	let count: i64 =
2465	sqlx::query_scalar("SELECT COUNT(*) FROM releases WHERE target = 'linux/x86_64'")
2466	.fetch_one(&pool)
2467	.await
2468	.unwrap();
2469	assert_eq!(count, 1, "linux published once the gate was satisfied");
2470	}
2471
2472	/// A committed + tagged git repo whose linux recipe pins the host to the tag
2473	/// via `checkout_sha`. `tag` is created only when `Some`.
2474	fn init_git_app(repo: &std::path::Path, tauri_version: &str, tag: Option<&str>) {
2475	init_git_app_with_recipe(
2476	repo,
2477	tauri_version,
2478	tag,
2479	"step(\"checkout\");\nlet s = checkout_sha(build_host());\nlog(\"pinned \" + s);\n\
2480	step(\"build\");\nsh_ok(build_host(), \"true\");\n",
2481	);
2482	}
2483
2484	/// As [`init_git_app`], with the linux recipe spelled by the caller.
2485	fn init_git_app_with_recipe(
2486	repo: &std::path::Path,
2487	tauri_version: &str,
2488	tag: Option<&str>,
2489	recipe: &str,
2490	) {
2491	init_git_app_shipping(repo, tauri_version, tag, recipe, "[\"linux/x86_64\"]");
2492	}
2493
2494	/// As [`init_git_app_with_recipe`], with the manifest's target list spelled
2495	/// by the caller — a second target is what puts a second HOST in the
2496	/// preflight, which is the only way to reach its unwind path.
2497	fn init_git_app_shipping(
2498	repo: &std::path::Path,
2499	tauri_version: &str,
2500	tag: Option<&str>,
2501	recipe: &str,
2502	targets: &str,
2503	) {
2504	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2505	std::fs::write(
2506	repo.join("src-tauri/tauri.conf.json"),
2507	format!("{{\"version\":\"{tauri_version}\"}}"),
2508	)
2509	.unwrap();
2510	std::fs::write(repo.join("bento.toml"), format!("targets = {targets}\n")).unwrap();
2511	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2512	std::fs::write(repo.join("dist/recipes/linux.rhai"), recipe).unwrap();
2513	// Isolate from the dev's global git config (which forces signed tags).
2514	let run = \|args: &[&str]\| {
2515	let out = std::process::Command::new("git")
2516	.args(args)
2517	.current_dir(repo)
2518	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2519	.env("GIT_CONFIG_SYSTEM", "/dev/null")
2520	.output()
2521	.expect("git runs");
2522	assert!(
2523	out.status.success(),
2524	"git {args:?}: {}",
2525	String::from_utf8_lossy(&out.stderr)
2526	);
2527	};
2528	run(&["init", "-q"]);
2529	run(&["-c", "user.email=t@t", "-c", "user.name=t", "add", "-A"]);
2530	run(&[
2531	"-c",
2532	"user.email=t@t",
2533	"-c",
2534	"user.name=t",
2535	"commit",
2536	"-q",
2537	"-m",
2538	"init",
2539	]);
2540	if let Some(t) = tag {
2541	run(&["tag", t]);
2542	}
2543	}
2544
2545	/// A worktree root beside the checkout, the shape production uses: a hidden
2546	/// directory in the tree the repos live in, not inside any of them.
2547	fn worktree_root(repo: &std::path::Path) -> std::path::PathBuf {
2548	repo.parent().expect("repo has a parent").join(".bento")
2549	}
2550
2551	/// Where a release of `demo` out of `repo` builds: the worktree, plus the
2552	/// app's own prefix inside it for a repo holding several products.
2553	fn build_dir(repo: &std::path::Path, prefix: &str) -> std::path::PathBuf {
2554	let repo_dir = repo.file_name().expect("repo has a name");
2555	worktree_root(repo).join(repo_dir).join("demo").join(prefix)
2556	}
2557
2558	fn one_host_topo(repo: &std::path::Path) -> Topology {
2559	Topology::from_str_for_tests(&format!(
2560	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2561	worktree_root = \"{}\"\n\
2562	[app.demo]\nrepo = \"{}\"\n",
2563	worktree_root(repo).display(),
2564	repo.display()
2565	))
2566	.unwrap()
2567	}
2568
2569	/// The release preflight (pin on) pins the host to the tag and the build
2570	/// proceeds. Exercises both the barrier and the `checkout_sha` host function.
2571	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2572	async fn release_preflight_pins_and_builds_when_the_tag_is_present() {
2573	let tmp = tempfile::tempdir().unwrap();
2574	let repo = tmp.path().join("demo");
2575	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2576	let mut cfg = Config::for_tests(tmp.path());
2577	cfg.pin_release_sha = true;
2578	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2579	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2580	let build_id = start_build(
2581	state,
2582	AppId::new("demo"),
2583	Version::parse("0.0.1").unwrap(),
2584	vec!["linux/x86_64".parse().unwrap()],
2585	)
2586	.await
2587	.unwrap();
2588	let (status, error) = await_target(&pool, build_id).await;
2589	assert_eq!(status, "ok", "a pinned build should succeed ({error})");
2590	assert!(
2591	build_dir(&repo, "")
2592	.join("src-tauri/tauri.conf.json")
2593	.exists(),
2594	"the release must have built in the worktree, not the checkout"
2595	);
2596	assert_eq!(
2597	head_sha(&build_dir(&repo, "")),
2598	tag_sha(&repo, "v0.0.1"),
2599	"and that worktree must be at the release tag"
2600	);
2601	}
2602
2603	/// A pinned build writes an artifact record beside what it collected: the
2604	/// manifest of those bytes, the commit the preflight pinned, and the steps
2605	/// as artifact-scoped gates.
2606	///
2607	/// The three facts already existed and met nowhere — the per-file sha256 at
2608	/// `collect`, the sha at the preflight, the step outcomes in `step_runs` —
2609	/// which is how gates came to vouch for one thing while the deploy shipped
2610	/// another.
2611	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2612	async fn a_pinned_build_writes_an_artifact_record_for_what_it_collected() {
2613	let tmp = tempfile::tempdir().unwrap();
2614	let repo = tmp.path().join("demo");
2615	std::fs::create_dir_all(&repo).unwrap();
2616	// The recipe builds and collects out of `repo()`, which is the worktree.
2617	// That also proves the collect: an artifact built in a tree the sync gate
2618	// does not trust is refused, and the worktree root is an artifact root
2619	// for exactly this reason (`Host::artifact_roots`).
2620	let recipe = r#"
2621	step("checkout");
2622	let sha = checkout_sha(build_host());
2623	log("pinned " + sha);
2624	step("build");
2625	sh_ok(build_host(), "mkdir -p " + repo() + "/out && echo bin > " + repo() + "/out/demo.bin");
2626	step("collect");
2627	collect(build_host(), repo() + "/out/demo.bin", "demo", version());
2628	"#;
2629	init_git_app_with_recipe(&repo, "0.0.1", Some("v0.0.1"), recipe);
2630
2631	let mut cfg = Config::for_tests(tmp.path());
2632	cfg.pin_release_sha = true;
2633	let dist_root = cfg.dist_root.clone();
2634	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2635	// No `pull_root`: the worktree root is the artifact root here, which is
2636	// the arrangement production uses.
2637	let topo = Topology::from_str_for_tests(&format!(
2638	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2639	worktree_root = \"{wt}\"\n[app.demo]\nrepo = \"{repo}\"\n",
2640	wt = worktree_root(&repo).display(),
2641	repo = repo.display()
2642	))
2643	.unwrap();
2644	let state = test_state(pool.clone(), topo, cfg);
2645	let build_id = start_build(
2646	state,
2647	AppId::new("demo"),
2648	Version::parse("0.0.1").unwrap(),
2649	vec!["linux/x86_64".parse().unwrap()],
2650	)
2651	.await
2652	.unwrap();
2653	let (status, error) = await_target(&pool, build_id).await;
2654	assert_eq!(status, "ok", "the build should succeed ({error})");
2655
2656	let path = crate::artifact_record::record_path(
2657	&dist_root,
2658	&AppId::new("demo"),
2659	&Version::parse("0.0.1").unwrap(),
2660	"linux/x86_64".parse().unwrap(),
2661	);
2662	let json = std::fs::read_to_string(&path)
2663	.unwrap_or_else(\|e\| panic!("record at {}: {e}", path.display()));
2664	// Parsing revalidates, so this also asserts the digest matches the
2665	// manifest and no environment-scoped gate slipped in.
2666	let record = ops_artifact::ArtifactRecord::parse(&json).unwrap();
2667
2668	assert_eq!(record.producer, "bento");
2669	assert_eq!(record.manifest.entries().len(), 1);
2670	assert_eq!(record.manifest.entries()[0].path, "demo.bin");
2671	assert_eq!(record.digest, record.manifest.digest());
2672
2673	// The provenance names the commit the preflight pinned, not a rebuild of
2674	// whatever the branch is now.
2675	let head = std::process::Command::new("git")
2676	.args(["rev-parse", "v0.0.1^{commit}"])
2677	.current_dir(&repo)
2678	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2679	.output()
2680	.unwrap();
2681	let head = String::from_utf8_lossy(&head.stdout).trim().to_string();
2682	assert_eq!(record.provenance.git_sha, head);
2683	assert_eq!(record.provenance.target, "linux/x86_64");
2684	assert_eq!(record.provenance.build_host, "fw13");
2685	assert!(!record.provenance.toolchain.is_empty());
2686
2687	let gates: Vec<&str> = record.gates.iter().map(\|g\| g.gate.as_str()).collect();
2688	assert_eq!(gates, ["checkout", "build", "collect"]);
2689	assert!(record.all_gates_passed());
2690	assert!(
2691	record
2692	.gates
2693	.iter()
2694	.all(\|g\| g.scope == ops_artifact::Scope::Artifact),
2695	"a build host cannot vouch for an environment"
2696	);
2697	}
2698
2699	/// The barrier refuses a target no host can build, instead of leaving it out
2700	/// of the pin and letting the remaining hosts vouch for the release.
2701	///
2702	/// The hole this closes is silence reading as agreement: the comparison used
2703	/// to run over the hosts that reported, so a target dropped for want of a
2704	/// host still built while the other hosts' unanimity looked like proof the
2705	/// whole release came from one commit.
2706	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2707	async fn release_preflight_refuses_a_target_no_host_can_build() {
2708	let tmp = tempfile::tempdir().unwrap();
2709	let repo = tmp.path().join("demo");
2710	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2711	let mut cfg = Config::for_tests(tmp.path());
2712	cfg.pin_release_sha = true;
2713	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2714	// The topology declares fw13 (linux/x86_64) only, so macOS has no host.
2715	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2716	let err = start_build(
2717	state,
2718	AppId::new("demo"),
2719	Version::parse("0.0.1").unwrap(),
2720	vec!["macos/aarch64".parse().unwrap()],
2721	)
2722	.await
2723	.unwrap_err();
2724	let msg = format!("{err:#}");
2725	assert!(
2726	msg.contains("no host can build macos/aarch64"),
2727	"the error must name the unbuildable target, got: {msg}"
2728	);
2729	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2730	.fetch_one(&pool)
2731	.await
2732	.unwrap();
2733	assert_eq!(builds, 0, "a refused preflight writes no build row");
2734	}
2735
2736	/// A dirty checkout neither refuses a release nor reaches it.
2737	///
2738	/// `git checkout <tag>` does NOT fail on local modifications to files whose
2739	/// content is unchanged in the tag: it succeeds and keeps them, so a dirty
2740	/// host would build its edits while honestly reporting the tagged sha.
2741	///
2742	/// The worktree answers that instead of gating on it. The release is built
2743	/// from a tree the edit is not in, so the edit cannot reach the binary and a
2744	/// release is never refused because somebody has unsaved work.
2745	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2746	async fn a_dirty_checkout_neither_refuses_a_release_nor_reaches_it() {
2747	let tmp = tempfile::tempdir().unwrap();
2748	let repo = tmp.path().join("demo");
2749	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2750
2751	// Modify a TRACKED file, exactly as an editor session would.
2752	let tracked = repo.join("src-tauri/tauri.conf.json");
2753	// Still valid JSON and still version 0.0.1: the version preflight reads
2754	// this file on the daemon's box before anything is pinned, so an edit
2755	// that broke it would fail the release for a reason this test is not
2756	// about.
2757	let edited = "{\"version\":\"0.0.1\",\"unsaved\":true}".to_string();
2758	std::fs::write(&tracked, &edited).unwrap();
2759
2760	let mut cfg = Config::for_tests(tmp.path());
2761	cfg.pin_release_sha = true;
2762	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2763	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2764	let build_id = start_build(
2765	state,
2766	AppId::new("demo"),
2767	Version::parse("0.0.1").unwrap(),
2768	vec!["linux/x86_64".parse().unwrap()],
2769	)
2770	.await
2771	.unwrap();
2772	let (status, error) = await_target(&pool, build_id).await;
2773	assert_eq!(
2774	status, "ok",
2775	"an edit elsewhere must not stop a release ({error})"
2776	);
2777	assert_eq!(
2778	std::fs::read_to_string(&tracked).unwrap(),
2779	edited,
2780	"and the edit must still be there afterwards"
2781	);
2782	assert_ne!(
2783	std::fs::read_to_string(build_dir(&repo, "").join("src-tauri/tauri.conf.json"))
2784	.unwrap(),
2785	edited,
2786	"what was built is the tag's content, not the edit"
2787	);
2788	}
2789
2790	/// An UNTRACKED file does not fail a release. A build host accumulates
2791	/// editor scratch and stray logs, none of which reach the binary, so failing
2792	/// on them would be noise that trains an operator to bypass the gate.
2793	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2794	async fn release_preflight_tolerates_untracked_files() {
2795	let tmp = tempfile::tempdir().unwrap();
2796	let repo = tmp.path().join("demo");
2797	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2798	std::fs::write(repo.join("scratch.log"), "noise").unwrap();
2799
2800	let mut cfg = Config::for_tests(tmp.path());
2801	cfg.pin_release_sha = true;
2802	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2803	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2804	let build_id = start_build(
2805	state,
2806	AppId::new("demo"),
2807	Version::parse("0.0.1").unwrap(),
2808	vec!["linux/x86_64".parse().unwrap()],
2809	)
2810	.await
2811	.unwrap();
2812	let (status, error) = await_target(&pool, build_id).await;
2813	assert_eq!(
2814	status, "ok",
2815	"untracked files must not fail a release ({error})"
2816	);
2817	}
2818
2819	/// The preflight refuses the build (before any row is written) when the
2820	/// release tag does not exist on the host — a missing/unpushed tag can't
2821	/// silently fall back to whatever `main` is.
2822	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2823	async fn release_preflight_refuses_when_the_release_tag_is_missing() {
2824	let tmp = tempfile::tempdir().unwrap();
2825	let repo = tmp.path().join("demo");
2826	// App is 0.0.2 (so the version check passes) but only v0.0.1 is tagged.
2827	init_git_app(&repo, "0.0.2", Some("v0.0.1"));
2828	let mut cfg = Config::for_tests(tmp.path());
2829	cfg.pin_release_sha = true;
2830	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2831	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2832	let err = start_build(
2833	state,
2834	AppId::new("demo"),
2835	Version::parse("0.0.2").unwrap(),
2836	vec!["linux/x86_64".parse().unwrap()],
2837	)
2838	.await
2839	.unwrap_err();
2840	let msg = format!("{err:#}");
2841	assert!(
2842	msg.contains("release preflight") && msg.contains("v0.0.2"),
2843	"expected a preflight tag error, got: {msg}"
2844	);
2845	assert!(
2846	msg.contains("does not exist"),
2847	"the error should name the absent tag as the cause, got: {msg}"
2848	);
2849	// Refused before anything was recorded.
2850	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2851	.fetch_one(&pool)
2852	.await
2853	.unwrap();
2854	assert_eq!(builds, 0, "a refused preflight writes no build row");
2855	}
2856
2857	/// A release does not touch the checkout at all: same branch, same commit,
2858	/// same working tree.
2859	///
2860	/// Checking the tag out there would move a tree somebody works in under them
2861	/// for the length of a build, and leave it detached whenever the restore
2862	/// failed. The worktree makes the question moot, so this asserts the strong
2863	/// form rather than a recovery.
2864	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2865	async fn a_release_never_moves_the_checkout() {
2866	let tmp = tempfile::tempdir().unwrap();
2867	let repo = tmp.path().join("demo");
2868	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2869	let branch_before = current_branch(&repo);
2870	let head_before = head_sha(&repo);
2871	assert!(!branch_before.is_empty(), "test repo starts on a branch");
2872
2873	let mut cfg = Config::for_tests(tmp.path());
2874	cfg.pin_release_sha = true;
2875	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2876	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2877	let build_id = start_build(
2878	state,
2879	AppId::new("demo"),
2880	Version::parse("0.0.1").unwrap(),
2881	vec!["linux/x86_64".parse().unwrap()],
2882	)
2883	.await
2884	.unwrap();
2885	let (status, error) = await_target(&pool, build_id).await;
2886	assert_eq!(status, "ok", "the build itself should pass ({error})");
2887
2888	assert_eq!(
2889	current_branch(&repo),
2890	branch_before,
2891	"the checkout must still be on its branch"
2892	);
2893	assert_eq!(head_sha(&repo), head_before, "and at the same commit");
2894	assert_eq!(repo_status(&repo), "", "and with the same working tree");
2895	}
2896
2897	/// A detached checkout does not refuse a release.
2898	///
2899	/// Releases do not move that tree, so its HEAD is not their business.
2900	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2901	async fn a_detached_checkout_no_longer_refuses_a_release() {
2902	let tmp = tempfile::tempdir().unwrap();
2903	let repo = tmp.path().join("demo");
2904	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2905	// Detach, standing in for a checkout an earlier release left on its tag.
2906	let out = std::process::Command::new("git")
2907	.args(["checkout", "--detach", "-q", "HEAD"])
2908	.current_dir(&repo)
2909	.env("GIT_CONFIG_GLOBAL", "/dev/null")
2910	.env("GIT_CONFIG_SYSTEM", "/dev/null")
2911	.output()
2912	.expect("git runs");
2913	assert!(out.status.success());
2914	assert!(current_branch(&repo).is_empty(), "repo is detached");
2915
2916	let head_before = head_sha(&repo);
2917
2918	let mut cfg = Config::for_tests(tmp.path());
2919	cfg.pin_release_sha = true;
2920	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2921	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2922	let build_id = start_build(
2923	state,
2924	AppId::new("demo"),
2925	Version::parse("0.0.1").unwrap(),
2926	vec!["linux/x86_64".parse().unwrap()],
2927	)
2928	.await
2929	.unwrap();
2930	let (status, error) = await_target(&pool, build_id).await;
2931	assert_eq!(
2932	status, "ok",
2933	"a detached checkout is not a reason to refuse ({error})"
2934	);
2935	assert!(
2936	current_branch(&repo).is_empty() && head_sha(&repo) == head_before,
2937	"and the release left it exactly as detached as it found it"
2938	);
2939	}
2940
2941	/// A build that rewrites a tracked file does not stop the next release.
2942	///
2943	/// `cargo build` rewrites `Cargo.lock` under the `[patch]` block, so a build
2944	/// leaves the tree it ran in dirty. The dirt lands in Bento's own worktree
2945	/// and the next release forces past it, which is safe precisely because
2946	/// nothing else writes there. So this releases the same app twice with a
2947	/// build that dirties the tree, and the second one is the assertion.
2948	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2949	async fn a_build_that_dirties_its_tree_does_not_refuse_the_next_release() {
2950	let tmp = tempfile::tempdir().unwrap();
2951	let repo = tmp.path().join("demo");
2952	std::fs::create_dir_all(&repo).unwrap();
2953	std::fs::write(repo.join("Cargo.lock"), "version = 4\n").unwrap();
2954	init_git_app_with_recipe(
2955	&repo,
2956	"0.0.1",
2957	Some("v0.0.1"),
2958	"step(\"build\");\nsh_ok(build_host(), \"echo churn >> \" + repo() + \"/Cargo.lock\");\n",
2959	);
2960	// The tag and the branch disagree about the lockfile.
2961	let git = git_in(&repo);
2962	std::fs::write(repo.join("Cargo.lock"), "version = 4\n# moved on\n").unwrap();
2963	git(&["add", "-A"]);
2964	git(&["commit", "-q", "-m", "lock moves on"]);
2965
2966	let mut cfg = Config::for_tests(tmp.path());
2967	cfg.pin_release_sha = true;
2968	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2969	let topo = one_host_topo(&repo);
2970
2971	for attempt in 1..=2 {
2972	let state = test_state(pool.clone(), topo.clone(), cfg.clone());
2973	let build_id = start_build(
2974	state,
2975	AppId::new("demo"),
2976	Version::parse("0.0.1").unwrap(),
2977	vec!["linux/x86_64".parse().unwrap()],
2978	)
2979	.await
2980	.unwrap_or_else(\|e\| panic!("release {attempt} refused: {e:#}"));
2981	let (status, error) = await_target(&pool, build_id).await;
2982	assert_eq!(status, "ok", "release {attempt} should build ({error})");
2983	}
2984
2985	assert_eq!(
2986	std::fs::read_to_string(build_dir(&repo, "").join("Cargo.lock")).unwrap(),
2987	"version = 4\nchurn\n",
2988	"the second release started from the tag's lockfile, not the first's leavings"
2989	);
2990	assert_eq!(repo_status(&repo), "", "and the checkout was never in it");
2991	}
2992
2993	/// A working copy ahead of the tag does not fail the version check.
2994	///
2995	/// The check reads the tag, not the checkout, so releasing v0.0.1 while
2996	/// `main` has moved to 0.0.2 is not version drift: the tag says 0.0.1.
2997	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2998	async fn a_working_copy_ahead_of_the_tag_does_not_fail_the_version_check() {
2999	let tmp = tempfile::tempdir().unwrap();
3000	let repo = tmp.path().join("demo");
3001	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
3002	// main moves on, untagged, exactly as it does the day after a release.
3003	let git = git_in(&repo);
3004	std::fs::write(
3005	repo.join("src-tauri/tauri.conf.json"),
3006	"{\"version\":\"0.0.2\"}",
3007	)
3008	.unwrap();
3009	git(&["add", "-A"]);
3010	git(&["commit", "-q", "-m", "0.0.2"]);
3011
3012	let mut cfg = Config::for_tests(tmp.path());
3013	cfg.pin_release_sha = true;
3014	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3015	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
3016	let build_id = start_build(
3017	state,
3018	AppId::new("demo"),
3019	Version::parse("0.0.1").unwrap(),
3020	vec!["linux/x86_64".parse().unwrap()],
3021	)
3022	.await
3023	.expect("releasing the tag behind main must not be version drift");
3024	let (status, error) = await_target(&pool, build_id).await;
3025	assert_eq!(status, "ok", "({error})");
3026	}
3027
3028	/// A tag that disagrees with itself is refused, naming the file.
3029	///
3030	/// The drift worth catching: `tauri.conf.json` at 0.0.1 and `Cargo.toml`
3031	/// still at 0.0.2, committed and tagged that way. Reading the checkout would
3032	/// compare against whatever the working copy happens to say instead.
3033	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3034	async fn a_tag_whose_manifest_disagrees_with_it_is_refused() {
3035	let tmp = tempfile::tempdir().unwrap();
3036	let repo = tmp.path().join("demo");
3037	std::fs::create_dir_all(&repo).unwrap();
3038	std::fs::write(
3039	repo.join("Cargo.toml"),
3040	"[package]\nname = \"demo\"\nversion = \"0.0.2\"\n",
3041	)
3042	.unwrap();
3043	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
3044
3045	let mut cfg = Config::for_tests(tmp.path());
3046	cfg.pin_release_sha = true;
3047	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3048	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
3049	let err = start_build(
3050	state,
3051	AppId::new("demo"),
3052	Version::parse("0.0.1").unwrap(),
3053	vec!["linux/x86_64".parse().unwrap()],
3054	)
3055	.await
3056	.unwrap_err();
3057	let msg = format!("{err:#}");
3058	assert!(
3059	msg.contains("version drift") && msg.contains("Cargo.toml says 0.0.2"),
3060	"the refusal must name the file and what it says, got: {msg}"
3061	);
3062	assert!(
3063	msg.contains("v0.0.1"),
3064	"and say which tag it read, got: {msg}"
3065	);
3066	let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
3067	.fetch_one(&pool)
3068	.await
3069	.unwrap();
3070	assert_eq!(builds, 0, "a refused version preflight writes no build row");
3071	}
3072
3073	/// A preflight that refuses on a LATER host leaves every checkout alone.
3074	///
3075	/// The hosts are prepared one at a time, so when the second is refused the
3076	/// first has already been through the whole preflight. There is no unwind to
3077	/// get right: the checkouts were never moved, and the worktrees a refusal
3078	/// leaves behind are where the next release would have put them.
3079	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3080	async fn a_refused_preflight_leaves_every_checkout_untouched() {
3081	let tmp = tempfile::tempdir().unwrap();
3082	// fw13's checkout is tagged and clean, so it pins. mbp's has no v0.0.1,
3083	// so it is refused after fw13 has already been moved.
3084	let repo = tmp.path().join("demo");
3085	init_git_app_shipping(
3086	&repo,
3087	"0.0.1",
3088	Some("v0.0.1"),
3089	"step(\"build\");\nsh_ok(build_host(), \"true\");\n",
3090	"[\"linux/x86_64\", \"macos/aarch64\"]",
3091	);
3092	let other = tmp.path().join("demo-mbp");
3093	init_git_app_shipping(
3094	&other,
3095	"0.0.1",
3096	None,
3097	"step(\"build\");\nsh_ok(build_host(), \"true\");\n",
3098	"[\"linux/x86_64\", \"macos/aarch64\"]",
3099	);
3100	let branch_before = current_branch(&repo);
3101	assert!(
3102	!branch_before.is_empty(),
3103	"fw13's checkout starts on a branch"
3104	);
3105
3106	let mut cfg = Config::for_tests(tmp.path());
3107	cfg.pin_release_sha = true;
3108	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3109	let topo = Topology::from_str_for_tests(&format!(
3110	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
3111	worktree_root = \"{wt}\"\n\
3112	[[host]]\nname = \"mbp\"\nssh = \"local\"\ntargets = [\"macos/aarch64\"]\n\
3113	worktree_root = \"{wt}\"\n\
3114	[app.demo]\nrepo = \"{}\"\n[app.demo.repo_by_host]\nmbp = \"{}\"\n",
3115	repo.display(),
3116	other.display(),
3117	wt = worktree_root(&repo).display(),
3118	))
3119	.unwrap();
3120	let state = test_state(pool.clone(), topo, cfg);
3121	let err = start_build(
3122	state,
3123	AppId::new("demo"),
3124	Version::parse("0.0.1").unwrap(),
3125	vec![
3126	"linux/x86_64".parse().unwrap(),
3127	"macos/aarch64".parse().unwrap(),
3128	],
3129	)
3130	.await
3131	.unwrap_err();
3132	let msg = format!("{err:#}");
3133	assert!(
3134	msg.contains("does not exist"),
3135	"the refusal should still be mbp's missing tag, got: {msg}"
3136	);
3137	assert_eq!(
3138	current_branch(&repo),
3139	branch_before,
3140	"a refused preflight must not have moved the host it prepared first"
3141	);
3142	assert_eq!(repo_status(&repo), "", "nor left anything in its tree");
3143	}
3144
3145	/// An edit elsewhere in a repo holding several products does not touch a
3146	/// release of one of them.
3147	///
3148	/// An edit in `server/` is not something pom's build compiles, but `git
3149	/// checkout <tag>` acts on the whole repository, and MNW is one `.git` over
3150	/// the server, sando, multithreaded and pom.
3151	///
3152	/// A worktree is made of the repository too, so the release still gets the
3153	/// whole tagged tree, in its own copy, and the edit stays where its author
3154	/// left it.
3155	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3156	async fn an_edit_elsewhere_in_the_repo_survives_a_release_and_stays_out_of_it() {
3157	let tmp = tempfile::tempdir().unwrap();
3158	let root = tmp.path().join("monorepo");
3159	let app = root.join("pom");
3160	init_git_app(&app, "0.0.1", None);
3161	let git = \|args: &[&str]\| {
3162	let out = std::process::Command::new("git")
3163	.args(args)
3164	.current_dir(&root)
3165	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3166	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3167	.env("GIT_AUTHOR_NAME", "t")
3168	.env("GIT_AUTHOR_EMAIL", "t@t")
3169	.env("GIT_COMMITTER_NAME", "t")
3170	.env("GIT_COMMITTER_EMAIL", "t@t")
3171	.output()
3172	.expect("git runs");
3173	assert!(
3174	out.status.success(),
3175	"git {args:?}: {}",
3176	String::from_utf8_lossy(&out.stderr)
3177	);
3178	};
3179	// `init_git_app` made `pom/` its own repo; the monorepo is the parent, so
3180	// drop that and re-init one `.git` over both products.
3181	std::fs::remove_dir_all(app.join(".git")).unwrap();
3182	std::fs::create_dir_all(root.join("server")).unwrap();
3183	std::fs::write(root.join("server/Cargo.lock"), "version = 4\n").unwrap();
3184	git(&["init", "-q"]);
3185	git(&["add", "-A"]);
3186	git(&["commit", "-q", "-m", "init"]);
3187	git(&["tag", "v0.0.1"]);
3188	// The tag and the branch differ in `server/`, and the working copy of
3189	// that file is modified — so the checkout cannot carry the edit across.
3190	std::fs::write(root.join("server/Cargo.lock"), "version = 4\n# moved on\n").unwrap();
3191	git(&["add", "-A"]);
3192	git(&["commit", "-q", "-m", "server moves"]);
3193	std::fs::write(
3194	root.join("server/Cargo.lock"),
3195	"version = 4\n# local edit\n",
3196	)
3197	.unwrap();
3198
3199	let mut cfg = Config::for_tests(tmp.path());
3200	cfg.pin_release_sha = true;
3201	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3202	// The worktree root sits beside the monorepo, not inside it.
3203	let topo = Topology::from_str_for_tests(&format!(
3204	"[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
3205	worktree_root = \"{wt}\"\n[app.demo]\nrepo = \"{repo}\"\n",
3206	wt = worktree_root(&root).display(),
3207	repo = app.display()
3208	))
3209	.unwrap();
3210	let state = test_state(pool.clone(), topo, cfg);
3211	let build_id = start_build(
3212	state,
3213	AppId::new("demo"),
3214	Version::parse("0.0.1").unwrap(),
3215	vec!["linux/x86_64".parse().unwrap()],
3216	)
3217	.await
3218	.unwrap();
3219	let (status, error) = await_target(&pool, build_id).await;
3220	assert_eq!(
3221	status, "ok",
3222	"an edit in server/ must not stop pom's release ({error})"
3223	);
3224	assert_eq!(
3225	std::fs::read_to_string(root.join("server/Cargo.lock")).unwrap(),
3226	"version = 4\n# local edit\n",
3227	"the edit must be exactly where its author left it"
3228	);
3229	// The worktree is of the whole repository, at the tag: `server/` is
3230	// there, and it is the tagged content rather than either the branch tip
3231	// or the edit.
3232	let worktree = worktree_root(&root).join("monorepo").join("demo");
3233	assert_eq!(
3234	std::fs::read_to_string(worktree.join("server/Cargo.lock")).unwrap(),
3235	"version = 4\n",
3236	"the release built the tag's server/, not the working copy's"
3237	);
3238	}
3239
3240	/// The branch `repo` is on, empty on a detached HEAD.
3241	/// Tracked files with local changes, as `git status --porcelain` writes
3242	/// them, joined by newlines and empty for a clean tree.
3243	fn repo_status(repo: &std::path::Path) -> String {
3244	let out = std::process::Command::new("git")
3245	.args(["status", "--porcelain", "--untracked-files=no"])
3246	.current_dir(repo)
3247	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3248	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3249	.output()
3250	.expect("git runs");
3251	String::from_utf8_lossy(&out.stdout).trim_end().to_string()
3252	}
3253
3254	/// Run git in `dir`, isolated from the dev's global config and with an
3255	/// identity, panicking with git's own stderr on failure.
3256	fn git_in(dir: &std::path::Path) -> impl Fn(&[&str]) + '_ {
3257	move \|args: &[&str]\| {
3258	let out = std::process::Command::new("git")
3259	.args(args)
3260	.current_dir(dir)
3261	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3262	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3263	.env("GIT_AUTHOR_NAME", "t")
3264	.env("GIT_AUTHOR_EMAIL", "t@t")
3265	.env("GIT_COMMITTER_NAME", "t")
3266	.env("GIT_COMMITTER_EMAIL", "t@t")
3267	.output()
3268	.expect("git runs");
3269	assert!(
3270	out.status.success(),
3271	"git {args:?}: {}",
3272	String::from_utf8_lossy(&out.stderr)
3273	);
3274	}
3275	}
3276
3277	/// The commit a checkout (or worktree) is on.
3278	fn head_sha(repo: &std::path::Path) -> String {
3279	git_read(repo, &["rev-parse", "HEAD"])
3280	}
3281
3282	/// The commit a tag names.
3283	fn tag_sha(repo: &std::path::Path, tag: &str) -> String {
3284	git_read(repo, &["rev-parse", &format!("{tag}^{{commit}}")])
3285	}
3286
3287	fn git_read(dir: &std::path::Path, args: &[&str]) -> String {
3288	let out = std::process::Command::new("git")
3289	.args(args)
3290	.current_dir(dir)
3291	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3292	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3293	.output()
3294	.expect("git runs");
3295	String::from_utf8_lossy(&out.stdout).trim().to_string()
3296	}
3297
3298	fn current_branch(repo: &std::path::Path) -> String {
3299	let out = std::process::Command::new("git")
3300	.args(["symbolic-ref", "-q", "--short", "HEAD"])
3301	.current_dir(repo)
3302	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3303	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3304	.output()
3305	.expect("git runs");
3306	String::from_utf8_lossy(&out.stdout).trim().to_string()
3307	}
3308
3309	/// An unreachable remote does not fail a release whose tag is present.
3310	///
3311	/// `fetch --all` is non-zero if ANY remote fails, and every library repo
3312	/// carries three (`astra`, `mnw`, `srht`), so a dead mirror would abort the
3313	/// release and blame it on a missing tag. Fetch is advisory; only the
3314	/// checkout decides.
3315	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3316	async fn a_dead_remote_does_not_fail_a_release_whose_tag_exists() {
3317	let tmp = tempfile::tempdir().unwrap();
3318	let repo = tmp.path().join("demo");
3319	init_git_app(&repo, "0.0.1", Some("v0.0.1"));
3320	// A remote that cannot possibly be fetched, standing in for an offline
3321	// astra or an srht mirror the repo was never pushed to.
3322	let out = std::process::Command::new("git")
3323	.args([
3324	"remote",
3325	"add",
3326	"srht",
3327	&tmp.path().join("nowhere.git").display().to_string(),
3328	])
3329	.current_dir(&repo)
3330	.env("GIT_CONFIG_GLOBAL", "/dev/null")
3331	.env("GIT_CONFIG_SYSTEM", "/dev/null")
3332	.output()
3333	.expect("git runs");
3334	assert!(out.status.success());
3335
3336	let mut cfg = Config::for_tests(tmp.path());
3337	cfg.pin_release_sha = true;
3338	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3339	let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
3340	let build_id = start_build(
3341	state,
3342	AppId::new("demo"),
3343	Version::parse("0.0.1").unwrap(),
3344	vec!["linux/x86_64".parse().unwrap()],
3345	)
3346	.await
3347	.expect("a dead mirror must not refuse the release");
3348	let (status, error) = await_target(&pool, build_id).await;
3349	assert_eq!(status, "ok", "the tag is present, so this builds ({error})");
3350	}
3351
3352	/// The audit fix: a `build` step dispatched to a host whose executor lacks the
3353	/// `build` capability is denied at the transport BEFORE the command runs. This
3354	/// is the structural guarantee behind "never build on prod" — a recipe naming
3355	/// the wrong host can't compile there.
3356	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3357	async fn build_on_a_host_without_the_build_grant_is_denied() {
3358	let tmp = tempfile::tempdir().unwrap();
3359	let root = tmp.path();
3360
3361	let repo = root.join("app");
3362	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3363	std::fs::write(
3364	repo.join("src-tauri/tauri.conf.json"),
3365	r#"{"version":"0.0.1"}"#,
3366	)
3367	.unwrap();
3368	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
3369	// The recipe (wrongly) tries to compile on `prod`, a host with no build
3370	// grant. A marker file would appear if the command actually ran.
3371	let marker = root.join("ran-on-prod");
3372	std::fs::write(
3373	repo.join("dist/recipes/linux.rhai"),
3374	r#"
3375	step("build");
3376	sh_ok("prod", "touch MARKER");
3377	"#
3378	.replace("MARKER", marker.to_str().unwrap()),
3379	)
3380	.unwrap();
3381
3382	let cfg = Config::for_tests(root);
3383	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3384	// fw13 builds linux; `prod` is local-but-restart-only (no build/package).
3385	std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
3386	let topo = Topology::from_str_for_tests(
3387	r#"
3388	[[host]]
3389	name = "fw13"
3390	ssh = "local"
3391	targets = ["linux/x86_64"]
3392
3393	[[host]]
3394	name = "prod"
3395	ssh = "local"
3396	actuate = ["restart"]
3397	observe = []
3398
3399	[app.demo]
3400	repo = "REPO"
3401	"#
3402	.replace("REPO", repo.to_str().unwrap())
3403	.as_str(),
3404	)
3405	.unwrap();
3406
3407	let state = test_state(pool.clone(), topo, cfg);
3408
3409	let build_id = start_build(
3410	state.clone(),
3411	AppId::new("demo"),
3412	Version::parse("0.0.1").unwrap(),
3413	vec!["linux/x86_64".parse().unwrap()],
3414	)
3415	.await
3416	.unwrap();
3417
3418	let mut status = String::new();
3419	let mut error = String::new();
3420	for _ in 0..100 {
3421	let row: Option<(String, Option<String>)> =
3422	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
3423	.bind(build_id)
3424	.fetch_optional(&pool)
3425	.await
3426	.unwrap();
3427	if let Some((s, e)) = row {
3428	status = s;
3429	error = e.unwrap_or_default();
3430	if status != "running" {
3431	break;
3432	}
3433	}
3434	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3435	}
3436
3437	assert_eq!(status, "failed", "build on an ungranted host must fail");
3438	assert!(
3439	error.contains("capability denied") && error.contains("build"),
3440	"failure must be a capability denial, got: {error}"
3441	);
3442	assert!(!marker.exists(), "denied build step must NOT have executed");
3443	}
3444
3445	// ---- macOS sign / notarize / staple execution chain (via ScriptedExec) ----
3446
3447	/// Stand up a single-macOS-target app whose recipe is `recipe_body` (with the
3448	/// literal `ARTIFACT` replaced by a real, non-empty file on disk), dispatch
3449	/// every host command through `scripted`, run the build to a terminal state,
3450	/// and return the pool plus the final `(status, error)`. The returned
3451	/// [`tempfile::TempDir`] must be kept alive by the caller: it holds the
3452	/// sqlite DB the pool reads.
3453	async fn run_macos_recipe(
3454	scripted: Arc<ScriptedExec>,
3455	recipe_body: &str,
3456	backoff_secs: Option<u64>,
3457	) -> (tempfile::TempDir, SqlitePool, String, String) {
3458	let tmp = tempfile::tempdir().unwrap();
3459	let root = tmp.path();
3460	let repo = root.join("app");
3461	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3462	std::fs::write(
3463	repo.join("src-tauri/tauri.conf.json"),
3464	r#"{"version":"0.0.1"}"#,
3465	)
3466	.unwrap();
3467	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
3468
3469	// A real, non-empty artifact so `publish`'s size floor is satisfied; the
3470	// build step is faked, so nothing else creates it.
3471	let artifact = repo.join("out/demo.dmg");
3472	std::fs::create_dir_all(artifact.parent().unwrap()).unwrap();
3473	std::fs::write(&artifact, b"dmg-bytes").unwrap();
3474
3475	std::fs::write(
3476	repo.join("dist/recipes/macos.rhai"),
3477	recipe_body.replace("ARTIFACT", artifact.to_str().unwrap()),
3478	)
3479	.unwrap();
3480
3481	let cfg = Config {
3482	notarize_backoff_secs: backoff_secs,
3483	..Config::for_tests(root)
3484	};
3485	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3486	std::fs::write(repo.join("bento.toml"), "targets = [\"macos/aarch64\"]\n").unwrap();
3487	let topo = Topology::from_str_for_tests(&format!(
3488	r#"
3489	[[host]]
3490	name = "mbp"
3491	ssh = "local"
3492	targets = ["macos/aarch64"]
3493
3494	[app.demo]
3495	repo = "{}"
3496	"#,
3497	repo.display()
3498	))
3499	.unwrap();
3500
3501	let mut state = test_state(pool.clone(), topo, cfg);
3502	// Route every host command through the scripted executor.
3503	let mut execs = HashMap::new();
3504	execs.insert("mbp".to_string(), scripted as Arc<dyn Executor>);
3505	state.executors = Arc::new(execs);
3506
3507	let build_id = start_build(
3508	state.clone(),
3509	AppId::new("demo"),
3510	Version::parse("0.0.1").unwrap(),
3511	vec!["macos/aarch64".parse().unwrap()],
3512	)
3513	.await
3514	.unwrap();
3515
3516	let mut status = String::new();
3517	let mut error = String::new();
3518	for _ in 0..100 {
3519	let row: Option<(String, Option<String>)> =
3520	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
3521	.bind(build_id)
3522	.fetch_optional(&pool)
3523	.await
3524	.unwrap();
3525	if let Some((s, e)) = row {
3526	status = s;
3527	error = e.unwrap_or_default();
3528	if status != "running" {
3529	break;
3530	}
3531	}
3532	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3533	}
3534	(tmp, pool, status, error)
3535	}
3536
3537	async fn release_count(pool: &SqlitePool) -> i64 {
3538	sqlx::query_scalar("SELECT COUNT(*) FROM releases")
3539	.fetch_one(pool)
3540	.await
3541	.unwrap()
3542	}
3543
3544	/// The whole macOS release chain end to end through a fake host: codesign,
3545	/// notarize (Accepted first try), staple, verify_gatekeeper, then publish.
3546	/// The recorded commands lock the exact incantations each host function
3547	/// dispatches, and a `releases` row proves the publish gate opened for a
3548	/// signed + notarized + Gatekeeper-accepted artifact.
3549	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3550	async fn a_macos_recipe_signs_notarizes_staples_verifies_and_publishes() {
3551	let scripted = Arc::new(
3552	ScriptedExec::new()
3553	.on("codesign", 0, "")
3554	.on("notarytool", 0, r#"{"status":"Accepted"}"#)
3555	.on("stapler staple", 0, "")
3556	.on(
3557	"spctl",
3558	0,
3559	"source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
3560	),
3561	);
3562	let recipe = r#"
3563	let h = build_host();
3564	step("build");
3565	sh_ok(h, "echo built");
3566	step("sign");
3567	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3568	notarize(h, "ARTIFACT");
3569	staple(h, "ARTIFACT");
3570	step("verify");
3571	if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
3572	step("publish");
3573	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3574	"#;
3575	let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, None).await;
3576
3577	assert_eq!(
3578	status, "ok",
3579	"signed+notarized macOS build should publish: {error}"
3580	);
3581	assert_eq!(
3582	release_count(&pool).await,
3583	1,
3584	"publish must record a release"
3585	);
3586
3587	// The exact shell incantations each host function dispatched.
3588	let cmds = scripted.commands();
3589	let has = \|needle: &str\| cmds.iter().any(\|c\| c.contains(needle));
3590	assert!(
3591	has("codesign --force --options runtime --timestamp --sign"),
3592	"codesign runtime+timestamp incantation, got: {cmds:?}"
3593	);
3594	assert!(
3595	has("xcrun notarytool submit"),
3596	"notarytool submit: {cmds:?}"
3597	);
3598	assert!(
3599	has("--wait --output-format json"),
3600	"notarytool --wait json: {cmds:?}"
3601	);
3602	assert!(has("xcrun stapler staple"), "stapler staple: {cmds:?}");
3603	assert!(has("spctl --assess"), "gatekeeper assess: {cmds:?}");
3604	}
3605
3606	/// A failing `codesign` fails the sign step and aborts the recipe before
3607	/// publish — an unsigned artifact never ships.
3608	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3609	async fn a_codesign_failure_fails_the_sign_step_and_blocks_publish() {
3610	let scripted = Arc::new(ScriptedExec::new().on("codesign", 1, ""));
3611	let recipe = r#"
3612	let h = build_host();
3613	step("build");
3614	sh_ok(h, "echo built");
3615	step("sign");
3616	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3617	step("publish");
3618	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3619	"#;
3620	let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
3621
3622	assert_eq!(status, "failed", "a failed codesign must fail the target");
3623	assert!(
3624	error.contains("codesign failed"),
3625	"error names the codesign failure, got: {error}"
3626	);
3627	assert_eq!(
3628	release_count(&pool).await,
3629	0,
3630	"nothing may publish after a codesign failure"
3631	);
3632	}
3633
3634	/// Even if the recipe ignores `verify_gatekeeper`'s returned `false`, the
3635	/// publish gate refuses the artifact: `verify_gatekeeper` both records the
3636	/// rejection and fails its step, and `publish` proves neither passed. This
3637	/// is the defense-in-depth the pure `PublishAuthority::prove` tests assert in
3638	/// isolation, here exercised through the real host-function path.
3639	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3640	async fn a_gatekeeper_rejection_bars_publish_even_if_the_recipe_ignores_it() {
3641	let scripted = Arc::new(
3642	ScriptedExec::new()
3643	.on("codesign", 0, "")
3644	.on("notarytool", 0, r#"{"status":"Accepted"}"#)
3645	.on("stapler staple", 0, "")
3646	// Gatekeeper says no: the sentinel is FAIL, not OK.
3647	.on("spctl", 0, "source=Unnotarized\nBENTO_GATEKEEPER_FAIL"),
3648	);
3649	let recipe = r#"
3650	let h = build_host();
3651	step("build");
3652	sh_ok(h, "echo built");
3653	step("sign");
3654	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3655	notarize(h, "ARTIFACT");
3656	staple(h, "ARTIFACT");
3657	step("verify");
3658	verify_gatekeeper(h, "ARTIFACT");
3659	step("publish");
3660	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3661	"#;
3662	let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
3663
3664	assert_eq!(
3665	status, "failed",
3666	"a Gatekeeper-rejected artifact must not publish"
3667	);
3668	assert!(
3669	!error.is_empty(),
3670	"the barred publish must surface an error"
3671	);
3672	assert_eq!(
3673	release_count(&pool).await,
3674	0,
3675	"no release for a rejected artifact"
3676	);
3677	}
3678
3679	/// The one flaky, network-bound step: `notarize` retries a non-`Accepted`
3680	/// result and succeeds on a later attempt. Two notarytool calls (reject then
3681	/// accept) then a recorded release prove the retry ran and the chain
3682	/// completed. Backoff is 0 so the retry sleep doesn't stall the test.
3683	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3684	async fn notarize_retries_a_non_accepted_result_then_succeeds() {
3685	let scripted = Arc::new(
3686	ScriptedExec::new()
3687	.on("codesign", 0, "")
3688	.on_seq(
3689	"notarytool",
3690	&[
3691	(0, r#"{"status":"In Progress"}"#),
3692	(0, r#"{"status":"Accepted"}"#),
3693	],
3694	)
3695	.on("stapler staple", 0, "")
3696	.on(
3697	"spctl",
3698	0,
3699	"source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
3700	),
3701	);
3702	let recipe = r#"
3703	let h = build_host();
3704	step("build");
3705	sh_ok(h, "echo built");
3706	step("sign");
3707	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3708	notarize(h, "ARTIFACT");
3709	staple(h, "ARTIFACT");
3710	step("verify");
3711	if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
3712	step("publish");
3713	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3714	"#;
3715	let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
3716
3717	assert_eq!(
3718	status, "ok",
3719	"notarize should succeed on the retry: {error}"
3720	);
3721	assert_eq!(
3722	release_count(&pool).await,
3723	1,
3724	"the retried build still publishes"
3725	);
3726	let notary_calls = scripted
3727	.commands()
3728	.iter()
3729	.filter(\|c\| c.contains("notarytool"))
3730	.count();
3731	assert_eq!(
3732	notary_calls, 2,
3733	"notarytool ran once, was rejected, then ran again"
3734	);
3735	}
3736
3737	/// `notarize` gives up after its bounded retries: three notarytool attempts,
3738	/// all non-`Accepted`, fail the target and bar publish.
3739	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3740	async fn notarize_fails_the_target_after_exhausting_its_retries() {
3741	let scripted = Arc::new(
3742	ScriptedExec::new()
3743	.on("codesign", 0, "")
3744	// Every attempt reports a still-pending status, never Accepted.
3745	.on("notarytool", 0, r#"{"status":"In Progress"}"#),
3746	);
3747	let recipe = r#"
3748	let h = build_host();
3749	step("build");
3750	sh_ok(h, "echo built");
3751	step("sign");
3752	codesign(h, "Developer ID Application: Test", "ARTIFACT");
3753	notarize(h, "ARTIFACT");
3754	step("publish");
3755	publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
3756	"#;
3757	let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
3758
3759	assert_eq!(
3760	status, "failed",
3761	"exhausted notarization must fail the target"
3762	);
3763	assert!(
3764	error.contains("notarization failed after 3 attempts"),
3765	"error names the exhausted retry, got: {error}"
3766	);
3767	assert_eq!(
3768	release_count(&pool).await,
3769	0,
3770	"an unnotarized artifact never publishes"
3771	);
3772	let notary_calls = scripted
3773	.commands()
3774	.iter()
3775	.filter(\|c\| c.contains("notarytool"))
3776	.count();
3777	assert_eq!(notary_calls, 3, "the retry is bounded at three attempts");
3778	}
3779	// ---- item (5): SSH and Agent transports end to end, via a recording fake ----
3780	//
3781	// Every other test declares `ssh = "local"`, so the runner's two-plane
3782	// routing is only exercised at construction
3783	// (state::agent_host_syncs_over_ssh_never_the_agent): steps run over the
3784	// EXEC transport (state.executors -- SshExec, or the in-session AgentRpc for
3785	// a mac host) while artifacts move over the SYNC transport (state.syncs --
3786	// always ssh, NEVER the agent, whose confined `/pull` would 404 or force
3787	// `pull_root` wide enough to expose `~/.tauri/passwords.env`). These drive a
3788	// real recipe through `start_build` on a NON-local topology, replace the
3789	// real ssh/agent transports with a recording fake, and assert which plane
3790	// handled which operation -- the runtime form of that construction-time
3791	// invariant, on the ssh string every other test pins to "local".
3792
3793	/// Records shell commands and artifact pulls on separate logs, so a test can
3794	/// prove the exec transport built/signed and the sync transport collected --
3795	/// and that neither did the other's job -- without a live host or ssh.
3796	struct RecordingExec {
3797	caps: CapabilitySet,
3798	commands: Arc<std::sync::Mutex<Vec<String>>>,
3799	pulls: Arc<std::sync::Mutex<Vec<String>>>,
3800	}
3801
3802	impl RecordingExec {
3803	fn new() -> Arc<Self> {
3804	Arc::new(Self {
3805	// A mac build host's real grant. Nothing in this fake gates on it
3806	// (the real transports do), but keep it coherent so
3807	// `capabilities()` is not a lie.
3808	caps: CapabilitySet::from_tokens(
3809	["build", "sign", "notarize", "staple"],
3810	["build-log", "artifact"],
3811	),
3812	commands: Arc::new(std::sync::Mutex::new(Vec::new())),
3813	pulls: Arc::new(std::sync::Mutex::new(Vec::new())),
3814	})
3815	}
3816	fn commands(&self) -> Vec<String> {
3817	self.commands.lock().unwrap().clone()
3818	}
3819	fn pulls(&self) -> Vec<String> {
3820	self.pulls.lock().unwrap().clone()
3821	}
3822	}
3823
3824	#[async_trait]
3825	impl Executor for RecordingExec {
3826	async fn run_streaming(
3827	&self,
3828	step: &ops_exec::Step,
3829	_sink: &mut dyn LogSink,
3830	) -> anyhow::Result<RunOutput> {
3831	self.commands
3832	.lock()
3833	.unwrap()
3834	.push(step.argv.last().cloned().unwrap_or_default());
3835	Ok(RunOutput {
3836	status: std::process::ExitStatus::from_raw(0),
3837	stdout: Vec::new(),
3838	stderr: Vec::new(),
3839	})
3840	}
3841	async fn pull_file(
3842	&self,
3843	r: &std::path::Path,
3844	_l: &std::path::Path,
3845	_o: &SyncOpts,
3846	) -> anyhow::Result<()> {
3847	self.pulls
3848	.lock()
3849	.unwrap()
3850	.push(r.to_string_lossy().into_owned());
3851	Ok(())
3852	}
3853	async fn pull_dir(
3854	&self,
3855	r: &std::path::Path,
3856	_l: &std::path::Path,
3857	_o: &SyncOpts,
3858	) -> anyhow::Result<()> {
3859	self.pulls
3860	.lock()
3861	.unwrap()
3862	.push(r.to_string_lossy().into_owned());
3863	Ok(())
3864	}
3865	async fn pull_glob(
3866	&self,
3867	g: &str,
3868	_l: &std::path::Path,
3869	_o: &SyncOpts,
3870	) -> anyhow::Result<()> {
3871	self.pulls.lock().unwrap().push(g.to_string());
3872	Ok(())
3873	}
3874	async fn push_dir(
3875	&self,
3876	_l: &std::path::Path,
3877	_r: &std::path::Path,
3878	_o: &SyncOpts,
3879	) -> anyhow::Result<()> {
3880	Ok(())
3881	}
3882	async fn preflight(&self) -> anyhow::Result<()> {
3883	Ok(())
3884	}
3885	fn capabilities(&self) -> &CapabilitySet {
3886	&self.caps
3887	}
3888	}
3889
3890	/// Stand up a single-target app (host named `h1`) whose recipe is
3891	/// `recipe_body` (with `REPO` replaced by the checkout path), inject
3892	/// `exec_fake` as the host's EXEC transport and `sync_fake` as its SYNC
3893	/// transport, run the build to a terminal state, and return the tmpdir plus
3894	/// `(status, error)`. Unlike `test_state`'s `build_executors`, this replaces
3895	/// BOTH planes so no real ssh/agent transport is dialed. The returned
3896	/// [`tempfile::TempDir`] holds the sqlite DB and must outlive the caller's
3897	/// assertions.
3898	async fn run_two_plane(
3899	host_toml: &str,
3900	target: &str,
3901	recipe_file: &str,
3902	recipe_body: &str,
3903	exec_fake: Arc<dyn Executor>,
3904	sync_fake: Arc<dyn Executor>,
3905	) -> (tempfile::TempDir, String, String) {
3906	let tmp = tempfile::tempdir().unwrap();
3907	let root = tmp.path();
3908	let repo = root.join("app");
3909	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3910	std::fs::write(
3911	repo.join("src-tauri/tauri.conf.json"),
3912	r#"{"version":"0.0.1"}"#,
3913	)
3914	.unwrap();
3915	std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
3916	std::fs::write(
3917	repo.join("dist/recipes").join(recipe_file),
3918	recipe_body.replace("REPO", repo.to_str().unwrap()),
3919	)
3920	.unwrap();
3921	std::fs::write(
3922	repo.join("bento.toml"),
3923	format!("targets = [\"{target}\"]\n"),
3924	)
3925	.unwrap();
3926
3927	let cfg = Config::for_tests(root);
3928	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3929	let topo = Topology::from_str_for_tests(&format!(
3930	"{}\n[app.demo]\nrepo = \"{}\"\n",
3931	host_toml.replace("REPO", repo.to_str().unwrap()),
3932	repo.display()
3933	))
3934	.unwrap();
3935
3936	let mut state = test_state(pool.clone(), topo, cfg);
3937	state.executors = Arc::new(HashMap::from([("h1".to_string(), exec_fake)]));
3938	state.syncs = Arc::new(HashMap::from([("h1".to_string(), sync_fake)]));
3939
3940	let build_id = start_build(
3941	state.clone(),
3942	AppId::new("demo"),
3943	Version::parse("0.0.1").unwrap(),
3944	vec![target.parse().unwrap()],
3945	)
3946	.await
3947	.unwrap();
3948
3949	let mut status = String::new();
3950	let mut error = String::new();
3951	for _ in 0..100 {
3952	let row: Option<(String, Option<String>)> =
3953	sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
3954	.bind(build_id)
3955	.fetch_optional(&pool)
3956	.await
3957	.unwrap();
3958	if let Some((s, e)) = row {
3959	status = s;
3960	error = e.unwrap_or_default();
3961	if status != "running" {
3962	break;
3963	}
3964	}
3965	tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3966	}
3967	(tmp, status, error)
3968	}
3969
3970	/// An agent (macOS) host signs over the AGENT transport but is collected
3971	/// from over SSH -- driven through the whole runner, not just `build_sync`.
3972	/// The sign chain's commands land on the exec plane and the artifact pull on
3973	/// the sync plane; crucially, the agent plane is asked to move NOTHING (a
3974	/// regression to one transport would route collect at `AgentRpc::pull_glob`,
3975	/// refused by design, or widen `pull_root` over the secret-bearing home dir).
3976	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3977	async fn an_agent_host_signs_over_the_agent_and_collects_over_ssh_end_to_end() {
3978	let agent = RecordingExec::new(); // exec plane (AgentRpc in prod)
3979	let ssh = RecordingExec::new(); // sync plane (SshExec in prod)
3980	let host = r#"
3981	[[host]]
3982	name = "h1"
3983	ssh = "mbp"
3984	targets = ["macos/aarch64"]
3985	transport = "agent"
3986	agent_url = "http://mbp:8765"
3987	actuate = ["build", "sign", "notarize", "staple"]
3988	observe = ["build-log", "gatekeeper", "artifact"]
3989	pull_root = "REPO"
3990	"#;
3991	let recipe = r#"
3992	let h = build_host();
3993	step("build");
3994	sh_ok(h, "echo built");
3995	step("sign");
3996	codesign(h, "Developer ID Application: Test", "REPO/out/demo.dmg");
3997	step("collect");
3998	collect(h, "REPO/out/*.dmg", "demo", "0.0.1");
3999	"#;
4000	let (_tmp, status, error) = run_two_plane(
4001	host,
4002	"macos/aarch64",
4003	"macos.rhai",
4004	recipe,
4005	agent.clone(),
4006	ssh.clone(),
4007	)
4008	.await;
4009	assert_eq!(status, "ok", "the recipe should complete: {error}");
4010
4011	// The build + sign commands ran on the AGENT (exec) transport.
4012	let agent_cmds = agent.commands();
4013	assert!(
4014	agent_cmds.iter().any(\|c\| c.contains("codesign")),
4015	"codesign rides the agent exec transport: {agent_cmds:?}"
4016	);
4017	assert!(
4018	agent_cmds.iter().any(\|c\| c.contains("echo built")),
4019	"the build step rides the agent exec transport: {agent_cmds:?}"
4020	);
4021	// ...and the agent moved NO artifacts. This is the load-bearing half:
4022	// AgentRpc::pull_glob is refused by design, so collect must not touch it.
4023	assert!(
4024	agent.pulls().is_empty(),
4025	"the agent transport must never collect artifacts: {:?}",
4026	agent.pulls()
4027	);
4028
4029	// The artifact was collected over the SSH (sync) transport...
4030	let ssh_pulls = ssh.pulls();
4031	assert!(
4032	ssh_pulls
4033	.iter()
4034	.any(\|p\| p.contains("demo.dmg") \|\| p.contains("*.dmg")),
4035	"collect rides the ssh sync transport: {ssh_pulls:?}"
4036	);
4037	// ...and the sync transport was never asked to run a build/sign command.
4038	assert!(
4039	ssh.commands().is_empty(),
4040	"the sync transport must never run host commands: {:?}",
4041	ssh.commands()
4042	);
4043	}
4044
4045	/// A plain (non-agent) host whose `ssh` is a remote alias, not "local" --
4046	/// the case every other test avoids. The recipe runs end to end through the
4047	/// fake, proving the runner drives a non-local host and still splits exec
4048	/// (build) from sync (collect) across the two transport maps.
4049	#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4050	async fn a_non_local_ssh_host_runs_a_recipe_end_to_end() {
4051	let exec = RecordingExec::new();
4052	let sync = RecordingExec::new();
4053	let host = r#"
4054	[[host]]
4055	name = "h1"
4056	ssh = "astra"
4057	targets = ["linux/x86_64"]
4058	pull_root = "REPO"
4059	"#;
4060	let recipe = r#"
4061	let h = build_host();
4062	step("build");
4063	sh_ok(h, "echo compiling");
4064	step("collect");
4065	collect(h, "REPO/out/demo.bin", "demo", "0.0.1");
4066	"#;
4067	let (_tmp, status, error) = run_two_plane(
4068	host,
4069	"linux/x86_64",
4070	"linux.rhai",
4071	recipe,
4072	exec.clone(),
4073	sync.clone(),
4074	)
4075	.await;
4076	assert_eq!(status, "ok", "the recipe should complete: {error}");
4077
4078	assert!(
4079	exec.commands().iter().any(\|c\| c.contains("echo compiling")),
4080	"the build command rides the exec transport: {:?}",
4081	exec.commands()
4082	);
4083	assert!(
4084	exec.pulls().is_empty(),
4085	"the exec transport must not collect: {:?}",
4086	exec.pulls()
4087	);
4088	assert!(
4089	sync.pulls().iter().any(\|p\| p.contains("demo.bin")),
4090	"collect rides the sync transport: {:?}",
4091	sync.pulls()
4092	);
4093	assert!(
4094	sync.commands().is_empty(),
4095	"the sync transport must not run commands: {:?}",
4096	sync.commands()
4097	);
4098	}
4099	}
4100
4101	/// Every recipe of every configured app must parse.
4102	///
4103	/// A recipe is read and compiled at release time, on the build host, after the
4104	/// checkout has already run — so a typo in one is discovered at the worst
4105	/// possible moment. Compiling them here costs nothing and moves that discovery
4106	/// to `cargo test`. Skips when the live config is absent (CI, another host),
4107	/// like [`crate::topology`]'s live-config smoke test.
4108	#[cfg(test)]
4109	mod live_recipe_smoke {
4110	use crate::topology::Topology;
4111	use std::path::{Path, PathBuf};
4112
4113	#[test]
4114	fn live_recipes_compile_if_present() {
4115	let Some(home) = std::env::var_os("HOME") else {
4116	return;
4117	};
4118	let path = Path::new(&home).join(".config/bento/bento.toml");
4119	if !path.exists() {
4120	return;
4121	}
4122	let topo = Topology::load(&path).expect("live bento.toml must load");
4123	// Syntax only: the host functions are bound per run against a live
4124	// context, and Rhai resolves calls at eval time regardless.
4125	let engine = rhai::Engine::new();
4126	let mut checked = 0;
4127	for (name, cfg) in &topo.app {
4128	let dir = crate::engine::expand_tilde(&cfg.repo).join(&cfg.recipe_dir);
4129	// Every recipe in the directory, not just the ones the app's current
4130	// targets name. A recipe for a target that is temporarily not
4131	// shipped (windows, dropped from the manifests until its host is
4132	// real) still has to parse, and it is the one nothing else is
4133	// watching.
4134	let Ok(entries) = std::fs::read_dir(&dir) else {
4135	continue;
4136	};
4137	let mut files: Vec<PathBuf> = entries
4138	.filter_map(Result::ok)
4139	.map(\|e\| e.path())
4140	.filter(\|p\| p.extension().is_some_and(\|x\| x == "rhai"))
4141	.collect();
4142	files.sort();
4143	for p in files {
4144	let file = p.file_name().unwrap_or_default().to_string_lossy();
4145	let Ok(src) = std::fs::read_to_string(&p) else {
4146	continue;
4147	};
4148	engine
4149	.compile(&src)
4150	.unwrap_or_else(\|e\| panic!("recipe {name}/{file} does not parse: {e}"));
4151	checked += 1;
4152	}
4153	}
4154	assert!(checked > 0, "live config resolved no readable recipes");
4155	}
4156	}
4157