max / makenotwork

157.1 KB · 3746 lines History Blame Raw

1	//! Rhai recipe engine + host-function API.
2	//!
3	//! A `(app, target)` resolves to a `.rhai` recipe composed from a shared step
4	//! vocabulary. The daemon embeds Rhai and registers the host functions recipes
5	//! call; the recipe is the orchestration, the host functions are the
6	//! privileged primitives (run a command, read a secret, collect artifacts,
7	//! publish). Recipes are otherwise sandboxed — no arbitrary FS/network except
8	//! through these functions — matching the Balanced Breakfast plugin model.
9	//!
10	//! Rhai is synchronous; the engine runs each recipe on a blocking thread
11	//! (`spawn_blocking`, see [`crate::runner`]) and host functions bridge to async
12	//! work via `Handle::block_on`. That is sound only off a runtime worker thread,
13	//! which `spawn_blocking` guarantees.
14
15	use crate::config::Config;
16	use crate::domain::{AppId, Status, Step, StepRunId, Target, Version};
17	use crate::events::{self, Event, EventTx};
18	use crate::ota::{OtaRegistry, PublishAuthority, Release};
19	use crate::state::ExecutorMap;
20	use crate::topology::{DeployTarget, Kind};
21	use anyhow::{Context as _, Result};
22	use ops_core::live_log::LiveLog;
23	use ops_exec::{Action, Executor, ObserveKind, Step as OpStep, SyncOpts};
24	use rhai::{Engine, EvalAltResult, Map};
25	use sha2::{Digest, Sha256};
26	use sqlx::SqlitePool;
27	use std::collections::HashMap;
28	use std::path::{Path, PathBuf};
29	use std::sync::atomic::{AtomicBool, Ordering};
30	use std::sync::{Arc, Mutex};
31	use tokio::runtime::Handle;
32	use tokio::sync::Mutex as AsyncMutex;
33
34	/// The capability label for a command, derived from the open recipe step. A
35	/// recipe's `sh("mbp", …)` under `step("sign")` becomes an `Action::Sign`, gated
36	/// by the mac host's `sign` grant — so recipes stay unchanged while every command
37	/// is capability-checked at its transport. `Verify` is read-only (an observe).
38	/// The capability a step's commands are gated on.
39	///
40	/// `Verify` depends on what is being released, which is the one place this is
41	/// not a property of the step alone. An app's verify is a Gatekeeper check on a
42	/// signed bundle, and the `gatekeeper` observe is granted implicitly to hosts
43	/// that can `sign` (`CapabilitySet::from_tokens`) precisely so that pairing
44	/// holds. A library's verify is a crate preflight: it runs `cargo` on the build
45	/// host and asks the registry a question. Gating that on Gatekeeper asks a Linux
46	/// host for a macOS code-signing capability it can never honestly hold, and the
47	/// only way to satisfy it would be to declare the capability falsely.
48	fn action_for(step: Step, kind: Kind) -> Action {
49	match step {
50	Step::Checkout \| Step::Prebuild \| Step::Build => Action::Build,
51	Step::Sign => Action::Sign,
52	Step::Notarize => Action::Notarize,
53	Step::Staple => Action::Staple,
54	Step::Package => Action::Package,
55	Step::Verify => match kind {
56	Kind::App => Action::Observe(ObserveKind::Custom("gatekeeper".into())),
57	// Running the build toolchain to inspect a crate or a service
58	// binary, which is what `build` means on a host. Neither has a
59	// bundle for Gatekeeper to have an opinion about.
60	Kind::Library \| Kind::Service => Action::Build,
61	},
62	// Publish/Collect/Handoff run on the daemon, not through a host
63	// executor; this label only applies if a recipe runs a bare `sh` while
64	// one is open. `handoff` is the daemon's own post-recipe motion and no
65	// recipe should open it at all — naming it here costs nothing and beats
66	// a wildcard that would silently absorb the next step somebody adds.
67	Step::Publish \| Step::Collect \| Step::Handoff => Action::Package,
68	// The one step that dispatches to a host OUTSIDE the build topology.
69	// Every command a recipe runs while `deploy` is open — the install, the
70	// restart, the health assertion — carries this action, so it reaches the
71	// service host only through the deploy grant and reaches a build host
72	// not at all (no build host is granted `deploy`).
73	Step::Deploy => Action::Deploy,
74	}
75	}
76
77	/// The currently-open step within a recipe run: its DB row id, which step it
78	/// is, and the live-log sink that `sh`/`log` stream into.
79	struct StepState {
80	run_id: StepRunId,
81	step: Step,
82	log: Arc<AsyncMutex<LiveLog>>,
83	/// Set when something in the step recorded a hard failure the recipe did
84	/// not abort on (e.g. `verify_gatekeeper` rejected the artifact but the
85	/// recipe ignored the bool). Forces the step's recorded status to `Failed`
86	/// and bars `publish` (the step-success ledger).
87	failed: bool,
88	/// Wall-clock deadline for this step. A command that runs past it fails the
89	/// step (and unwinds the recipe) rather than wedging under the old
90	/// whole-build guillotine, which a legitimate 5-target fan-out plus notary
91	/// queueing could trip — mismarking every target failed while the blocking
92	/// recipe bodies kept signing.
93	deadline: std::time::Instant,
94	}
95
96	/// Per-step wall-clock budget: a generous ceiling that catches a wedged command
97	/// (a hung ssh, a stuck notary poll) without killing legitimately slow work.
98	/// The old design bounded the whole build at 2h; this bounds each step so one
99	/// slow step can't be blamed on another and a fan-out of slow-but-fine targets
100	/// isn't guillotined. `Config::step_timeout_secs` overrides these per-kind
101	/// defaults for every step; see [`RecipeCtx::step_budget`].
102	fn default_step_budget(step: Step) -> std::time::Duration {
103	use std::time::Duration;
104	let mins = match step {
105	Step::Checkout => 10,
106	// clippy + full test suite, cold, on a workspace.
107	Step::Prebuild => 45,
108	// cargo tauri build, cold, universal bundles.
109	Step::Build => 90,
110	Step::Sign => 15,
111	// Apple's notary queue + this step's bounded retries.
112	Step::Notarize => 60,
113	Step::Staple => 10,
114	Step::Package => 30,
115	Step::Verify => 10,
116	// rsync of multi-GiB artifacts off the build host.
117	Step::Collect => 30,
118	Step::Publish => 20,
119	// A binary push, an install, a unit restart, and a health poll. Minutes
120	// of work; the ceiling is for a wedged transport, not slow work.
121	Step::Deploy => 15,
122	// Unused by any recipe — the daemon runs the handoff itself, outside a
123	// step's clock — and matched to `collect`, since it moves the same
124	// bytes the same way and would wedge for the same reasons.
125	Step::Handoff => 30,
126	};
127	Duration::from_secs(mins * 60)
128	}
129
130	/// Where a service's binary is staged on the host that will run it, before the
131	/// privileged installer moves it into place.
132	///
133	/// A fixed, unguessable-by-accident path rather than a recipe-chosen one,
134	/// because the installer refuses any source outside it. That refusal is the
135	/// only thing standing between the NOPASSWD sudo grant and `install`-as-root to
136	/// an arbitrary path, so both ends have to name the same constant. `/var/tmp`
137	/// rather than `/tmp` so a staged binary survives a `PrivateTmp` unit and a
138	/// systemd tmpfiles sweep between staging and install.
139	pub const DEPLOY_STAGING_ROOT: &str = "/var/tmp/bento-deploy";
140
141	/// Everything a recipe's host functions need, shared (Arc) into each closure.
142	pub struct RecipeCtx {
143	pub app: AppId,
144	pub version: Version,
145	pub target: Target,
146	/// Name of the host this target builds on (resolved from the topology by the
147	/// runner). Recipes read it via `build_host()` so one per-platform recipe can
148	/// dispatch to the right host across arches (linux x86_64 -> fw13, aarch64 ->
149	/// astra) without hard-coding a host name.
150	pub build_host: String,
151	/// The build host's SSH destination (topology `ssh`), as opposed to its
152	/// name. Deploy compares it against the service host's to tell "build and
153	/// run on the same box" from "two hosts that need a transfer" — a question
154	/// the host NAMES cannot answer, since a build host and a deploy
155	/// destination are declared in different files and need not agree on one.
156	pub build_host_ssh: String,
157	/// This release's git tag, rendered from the app's `tag_format`. Held here
158	/// rather than derived from the version because a repo holding several
159	/// products spells it per product (`pom-v0.4.1`), and the recipe, the
160	/// preflight barrier and the failure message all have to agree on it.
161	pub tag: String,
162	/// The app's default checkout path (topology `repo`, `~`-prefixed). Read it
163	/// through [`RecipeCtx::repo_for`] for anything that runs on a build host;
164	/// this field alone is the daemon-local answer.
165	pub repo: String,
166	/// Per-host checkout overrides (topology `repo_by_host`). Empty for every app
167	/// that has not declared one, which is all of them but the Windows-shipping
168	/// ones.
169	pub repo_by_host: HashMap<String, String>,
170	/// Cargo features this app's release builds enable (topology `features`).
171	/// Recipes read it via `feature_flags()`.
172	pub features: Vec<String>,
173	/// App or library. Decides which capability a `verify` step is gated on;
174	/// see [`action_for`].
175	pub kind: Kind,
176	pub target_run_id: i64,
177	/// Capability-scoped executor per build host. Recipe commands dispatch
178	/// through these — the transport (local / ssh / in-session agent) and the
179	/// capability gate are the executor's, not the engine's.
180	pub execs: Arc<ExecutorMap>,
181	/// Sync transport per build host, used only by `collect` to pull artifacts
182	/// back. Never the agent, even for an agent host — see `state::build_sync`.
183	/// Not an execution path.
184	pub syncs: Arc<ExecutorMap>,
185	/// Where this target installs, for a `kind = "service"` app. `None` for an
186	/// app or a library, which makes every deploy host function fail with that
187	/// as the reason rather than with a missing-host error.
188	///
189	/// The runner resolves it from the app manifest's `[[deploy]]` table and
190	/// registers its executor into `execs` under the destination's host string,
191	/// so `sh_ok(deploy_host(), ...)` reaches the service host through the same
192	/// capability gate as everything else.
193	pub deploy: Option<DeployTarget>,
194	pub pool: SqlitePool,
195	pub events: EventTx,
196	pub cfg: Arc<Config>,
197	pub ota: Arc<OtaRegistry>,
198	pub rt: Handle,
199	current: Mutex<Option<StepState>>,
200	/// Gatekeeper verdict recorded by `verify_gatekeeper`: `None` = never run,
201	/// `Some(false)` = ran and rejected, `Some(true)` = accepted. `publish`
202	/// requires `Some(true)` for a macOS/iOS artifact (the proof it is signed +
203	/// notarized).
204	gatekeeper_ok: Mutex<Option<bool>>,
205	/// Steps finalized as `Failed` during this run. A non-empty ledger bars
206	/// `publish` — an artifact is never shipped after a step failed, even if the
207	/// recipe ignored the failure and ran on.
208	failed_steps: Mutex<Vec<Step>>,
209	/// Set when a newer build supersedes this run. Checked at step boundaries
210	/// and before publish so a superseded recipe stops promptly rather than
211	/// running to completion (the blocking Rhai body can't be `abort()`ed).
212	cancel: Arc<AtomicBool>,
213	/// The all-targets-green publish gate (topology `require_all_targets`).
214	/// `Some(declared)` ⇒ `publish` refuses unless every one of `declared` has a
215	/// successful latest run for this `(app, version)`. `None` ⇒ gate off, each
216	/// target publishes independently.
217	all_green_required: Option<Vec<Target>>,
218	/// sha256 of each artifact hashed at `collect`, keyed by file name. `publish`
219	/// reads it to record `releases.artifact_hash` for the bytes it ships, so the
220	/// hash is the one computed when the artifact landed rather than a re-read
221	/// that could see a different file. Absent ⇒ `publish` hashes on demand.
222	artifact_hashes: Mutex<HashMap<String, String>>,
223	}
224
225	impl RecipeCtx {
226	/// Versions of `name` already on crates.io. A network failure yields an
227	/// empty list: preflight then cannot claim a version is a duplicate, and
228	/// `cargo publish` still refuses one, so the check degrades to advisory
229	/// rather than blocking a release on registry availability.
230	fn published_versions(name: &str) -> Vec<String> {
231	let url = format!("https://crates.io/api/v1/crates/{name}");
232	let Ok(out) = std::process::Command::new("curl")
233	.args([
234	"-sS",
235	"--max-time",
236	"15",
237	"-H",
238	"User-Agent: bento-preflight",
239	&url,
240	])
241	.output()
242	else {
243	return Vec::new();
244	};
245	let Ok(v) = serde_json::from_slice::<serde_json::Value>(&out.stdout) else {
246	return Vec::new();
247	};
248	v.get("versions")
249	.and_then(\|x\| x.as_array())
250	.map(\|a\| {
251	a.iter()
252	.filter_map(\|x\| x.get("num").and_then(\|n\| n.as_str()).map(str::to_string))
253	.collect()
254	})
255	.unwrap_or_default()
256	}
257
258	#[allow(clippy::too_many_arguments)]
259	pub fn new(
260	app: AppId,
261	version: Version,
262	target: Target,
263	build_host: String,
264	build_host_ssh: String,
265	tag: String,
266	repo: String,
267	features: Vec<String>,
268	kind: Kind,
269	target_run_id: i64,
270	execs: Arc<ExecutorMap>,
271	syncs: Arc<ExecutorMap>,
272	deploy: Option<DeployTarget>,
273	pool: SqlitePool,
274	events: EventTx,
275	cfg: Arc<Config>,
276	ota: Arc<OtaRegistry>,
277	rt: Handle,
278	cancel: Arc<AtomicBool>,
279	all_green_required: Option<Vec<Target>>,
280	) -> Self {
281	Self {
282	app,
283	version,
284	target,
285	build_host,
286	build_host_ssh,
287	tag,
288	repo,
289	repo_by_host: HashMap::new(),
290	features,
291	kind,
292	target_run_id,
293	execs,
294	syncs,
295	deploy,
296	pool,
297	events,
298	cfg,
299	ota,
300	rt,
301	current: Mutex::new(None),
302	gatekeeper_ok: Mutex::new(None),
303	failed_steps: Mutex::new(Vec::new()),
304	cancel,
305	all_green_required,
306	artifact_hashes: Mutex::new(HashMap::new()),
307	}
308	}
309
310	/// Declare the app's per-host checkout overrides (topology `repo_by_host`).
311	///
312	/// Separate from `new` because it is empty for every app that has not opted
313	/// in, and `new` already carries twenty arguments no test wants a
314	/// twenty-first of.
315	#[must_use]
316	pub fn with_repo_by_host(mut self, repo_by_host: HashMap<String, String>) -> Self {
317	self.repo_by_host = repo_by_host;
318	self
319	}
320
321	/// Where this app is checked out on `host` (topology `AppConfig::repo_for`).
322	///
323	/// Every git command a recipe or the engine runs on a build host goes
324	/// through this. The bare `repo` field is the daemon-local path.
325	pub fn repo_for(&self, host: &str) -> &str {
326	self.repo_by_host
327	.get(host)
328	.map_or(self.repo.as_str(), String::as_str)
329	}
330
331	/// Whether a newer build has superseded this run.
332	fn is_cancelled(&self) -> bool {
333	self.cancel.load(Ordering::SeqCst)
334	}
335
336	fn now() -> String {
337	chrono::Utc::now().to_rfc3339()
338	}
339
340	/// `<logs_root>/<app>/<version>/<target-with-slash-as-dash>/<step>.<run_id>.log`.
341	///
342	/// The run id is in the filename because the rest of the key is not unique:
343	/// re-running an app at a version it already built (a retry, or a rebuild of
344	/// an already-published release) reopens the same path, and the old file
345	/// appended to shows two runs' output with nothing marking the boundary. The
346	/// step ledger records a run id per step, so keying the file on it makes a
347	/// ledger row resolve to exactly one file and keeps the earlier run readable.
348	fn log_path(&self, step: Step, run_id: StepRunId) -> PathBuf {
349	self.log_dir()
350	.join(format!("{}.{}.log", step.as_str(), run_id.0))
351	}
352
353	/// `<logs_root>/<app>/<version>/<target-with-slash-as-dash>/`.
354	fn log_dir(&self) -> PathBuf {
355	let target_dir = self.target.to_string().replace('/', "-");
356	self.cfg
357	.logs_root
358	.join(self.app.as_str())
359	.join(self.version.to_string())
360	.join(target_dir)
361	}
362
363	/// Close the previous step (as `Ok`), open a new one: insert its DB row,
364	/// open a live log whose chunks broadcast `StepLogChunk`, emit `StepStart`.
365	fn begin_step(self: &Arc<Self>, step: Step) -> Result<()> {
366	anyhow::ensure!(
367	!self.is_cancelled(),
368	"build superseded by a newer request; aborting before `{}`",
369	step.as_str()
370	);
371	self.finish_step(Status::Ok)?;
372	let me = self.clone();
373	let started = Self::now();
374	let started_for_header = started.clone();
375	let run_id = self.rt.block_on(async move {
376	// The step row and the target's current_step pointer are one logical
377	// state — write them atomically so a failure can't leave a `running`
378	// step row while current_step still names the previous step.
379	let mut tx = me.pool.begin().await.context("begin step tx")?;
380	// log_ref names the run id, which only exists once the row does, so
381	// the path is written back inside the same transaction rather than
382	// guessed beforehand.
383	let id: i64 = sqlx::query_scalar(
384	"INSERT INTO step_runs (target_run_id, step, status, started_at)
385	VALUES (?, ?, 'running', ?) RETURNING id",
386	)
387	.bind(me.target_run_id)
388	.bind(step.as_str())
389	.bind(&started)
390	.fetch_one(&mut *tx)
391	.await
392	.context("insert step_run")?;
393	let log_ref = me
394	.log_path(step, StepRunId(id))
395	.to_string_lossy()
396	.into_owned();
397	sqlx::query("UPDATE step_runs SET log_ref = ? WHERE id = ?")
398	.bind(&log_ref)
399	.bind(id)
400	.execute(&mut *tx)
401	.await
402	.context("set step_run log_ref")?;
403	sqlx::query("UPDATE target_runs SET current_step = ? WHERE id = ?")
404	.bind(step.as_str())
405	.bind(me.target_run_id)
406	.execute(&mut *tx)
407	.await
408	.context("update current_step")?;
409	tx.commit().await.context("commit step tx")?;
410	anyhow::Ok(StepRunId(id))
411	})?;
412
413	// Live log: each chunk fans out as a StepLogChunk event keyed by run_id.
414	let events = self.events.clone();
415	let cb_run_id = run_id;
416	let mut log = self.rt.block_on(LiveLog::open(
417	self.log_path(step, run_id),
418	Box::new(move \|seq, text\| {
419	events::emit(
420	&events,
421	Event::StepLogChunk {
422	run_id: cb_run_id,
423	seq,
424	text: text.to_string(),
425	},
426	);
427	}),
428	));
429
430	events::emit(
431	&self.events,
432	Event::StepStart {
433	run_id,
434	app: self.app.clone(),
435	version: self.version.clone(),
436	target: self.target,
437	step,
438	},
439	);
440
441	// A log that names its own run: reading one tells you which ledger row
442	// it belongs to without going back to the DB, and a file that somehow
443	// does get appended to still shows where the second run began. Written
444	// after `StepStart` so its chunk event cannot precede the step it
445	// belongs to.
446	let header = format!(
447	"=== bento {app} {version} {target} step={step} run_id={run_id} started={started_for_header} ===\n",
448	app = self.app.as_str(),
449	version = self.version,
450	target = self.target,
451	step = step.as_str(),
452	);
453	self.rt.block_on(async {
454	use ops_core::remote::LogSink as _;
455	log.write_chunk(header.as_bytes()).await;
456	});
457
458	*self.current.lock().unwrap() = Some(StepState {
459	run_id,
460	step,
461	log: Arc::new(AsyncMutex::new(log)),
462	failed: false,
463	deadline: std::time::Instant::now() + self.step_budget(step),
464	});
465	Ok(())
466	}
467
468	/// This build's budget for `step`: the `Config` override if set, else the
469	/// per-kind default.
470	fn step_budget(&self, step: Step) -> std::time::Duration {
471	self.cfg
472	.step_timeout_secs
473	.map_or_else(\|\| default_step_budget(step), std::time::Duration::from_secs)
474	}
475
476	/// The current step's wall-clock deadline (or a default if no step is open,
477	/// which only happens before the first `step()` — commands then run under an
478	/// implicit `Build` step opened by `ensure_step`).
479	fn step_deadline(&self) -> std::time::Instant {
480	self.current.lock().unwrap().as_ref().map_or_else(
481	\|\| std::time::Instant::now() + self.step_budget(Step::Build),
482	\|s\| s.deadline,
483	)
484	}
485
486	/// Drive `fut` on the runtime, but stop early on two conditions the recipe
487	/// bodies otherwise cannot observe (they run synchronously on a blocking
488	/// thread): the current step's deadline, and supersession by a newer build.
489	/// Either turns into an error that fails the step and unwinds the recipe, so
490	/// a wedged command no longer runs unbounded and a superseded build stops
491	/// mid-step instead of only at the next step boundary.
492	fn run_bounded<F, T>(&self, what: &str, fut: F) -> Result<T>
493	where
494	F: std::future::Future<Output = Result<T>>,
495	{
496	let deadline = self.step_deadline();
497	let cancel = self.cancel.clone();
498	self.rt.block_on(async move {
499	tokio::pin!(fut);
500	let watch = async {
501	// Poll the cooperative cancel flag; the finalizer and a
502	// superseding build both set it. Cheap next to a build step.
503	while !cancel.load(Ordering::SeqCst) {
504	tokio::time::sleep(std::time::Duration::from_millis(250)).await;
505	}
506	};
507	tokio::select! {
508	r = &mut fut => r,
509	() = tokio::time::sleep_until(deadline.into()) => {
510	Err(anyhow::anyhow!("`{what}` exceeded its per-step deadline"))
511	}
512	() = watch => {
513	Err(anyhow::anyhow!("build superseded by a newer request; aborting `{what}`"))
514	}
515	}
516	})
517	}
518
519	/// Flag the currently-open step as failed (no-op if none is open). Forces
520	/// its recorded status to `Failed` at `finish_step` and adds it to the
521	/// publish-barring ledger, even though the recipe kept running.
522	fn fail_current_step(&self) {
523	if let Some(st) = self.current.lock().unwrap().as_mut() {
524	st.failed = true;
525	}
526	}
527
528	/// Finalize the open step (if any): close its log, stamp the DB row, emit
529	/// `StepDone`. Idempotent when no step is open. A step flagged via
530	/// [`fail_current_step`] is recorded `Failed` regardless of the requested
531	/// status, and added to the ledger `publish` consults.
532	pub fn finish_step(self: &Arc<Self>, status: Status) -> Result<()> {
533	let st = self.current.lock().unwrap().take();
534	let Some(st) = st else { return Ok(()) };
535	let status = if st.failed { Status::Failed } else { status };
536	if status == Status::Failed {
537	self.failed_steps.lock().unwrap().push(st.step);
538	}
539	let me = self.clone();
540	self.rt.block_on(async move {
541	// Drop all log refs so the sink can be owned + flushed.
542	if let Ok(m) = Arc::try_unwrap(st.log) {
543	m.into_inner().close().await;
544	}
545	if let Err(e) = sqlx::query(
546	"UPDATE step_runs SET status = ?, finished_at = ? WHERE id = ?",
547	)
548	.bind(status.as_str())
549	.bind(Self::now())
550	.bind(st.run_id.0)
551	.execute(&me.pool)
552	.await
553	{
554	tracing::error!(step = st.step.as_str(), error = %e, "could not stamp step_run status");
555	}
556	});
557	events::emit(
558	&self.events,
559	Event::StepDone {
560	run_id: st.run_id,
561	app: self.app.clone(),
562	target: self.target,
563	step: st.step,
564	status,
565	},
566	);
567	Ok(())
568	}
569
570	/// Ensure a step is open; default to `Build` if a recipe runs a command
571	/// before declaring one.
572	fn ensure_step(self: &Arc<Self>) -> Result<Arc<AsyncMutex<LiveLog>>> {
573	if self.current.lock().unwrap().is_none() {
574	self.begin_step(Step::Build)?;
575	}
576	Ok(self.current.lock().unwrap().as_ref().unwrap().log.clone())
577	}
578
579	/// The step currently open, or `Build` as a default for failure
580	/// attribution before any step was declared.
581	pub fn current_step(&self) -> Step {
582	self.current
583	.lock()
584	.unwrap()
585	.as_ref()
586	.map_or(Step::Build, \|s\| s.step)
587	}
588
589	/// The capability-scoped executor for `name`, or an error if the host isn't
590	/// in the topology.
591	fn exec(&self, name: &str) -> Result<Arc<dyn ops_exec::Executor>> {
592	self.execs
593	.get(name)
594	.cloned()
595	.ok_or_else(\|\| anyhow::anyhow!("unknown build host `{name}` (not in topology)"))
596	}
597
598	/// The ssh string for `name` (for `collect`'s remote scp source).
599	/// The transport that moves artifacts off `name`. Distinct from
600	/// [`RecipeCtx::exec_for`]'s executor: an agent host signs over `AgentRpc`
601	/// but is collected from over ssh (`state::build_sync`).
602	fn host_sync(&self, name: &str) -> Result<Arc<dyn Executor>> {
603	self.syncs
604	.get(name)
605	.cloned()
606	.ok_or_else(\|\| anyhow::anyhow!("unknown build host `{name}` (not in topology)"))
607	}
608
609	/// Run `cmd` on `host` through its capability-scoped executor, streaming into
610	/// the current step's log. The command's [`Action`] is derived from the open
611	/// step (see [`action_for`]) and gated at the transport before dispatch — so a
612	/// `build` step on a host without the `build` grant is denied, and the macOS
613	/// sign steps ride the in-session `AgentRpc` transport automatically. Returns
614	/// exit code + a tail of stdout for the recipe to branch on.
615	fn run(self: &Arc<Self>, host: &str, cmd: &str) -> Result<(i32, String)> {
616	// The service host is addressed as a service host whatever step is open.
617	// Deriving the action from the step is right for a build host, where the
618	// step IS the work; on a service host it would ask for `build` during a
619	// `verify` and be denied for a reason unrelated to what was attempted.
620	let action = match &self.deploy {
621	Some(d) if d.host == host => Action::Deploy,
622	_ => action_for(self.current_step(), self.kind),
623	};
624	self.run_as(host, cmd, action)
625	}
626
627	/// `run`, with the [`Action`] stated rather than resolved. Used where the
628	/// caller already knows which plane it is on.
629	fn run_as(self: &Arc<Self>, host: &str, cmd: &str, action: Action) -> Result<(i32, String)> {
630	let sink = self.ensure_step()?;
631	let exec = self.exec(host)?;
632	let cur = self.current_step();
633	let step = OpStep::shell(action, cmd.to_string());
634	// Echo the command before running it. Without this a log says what
635	// happened but not what was asked, and a gate that prints nothing when
636	// it passes (`cargo fmt --all --check`) is indistinguishable from a gate
637	// that never ran.
638	let echo = format!("$ [{host}] {cmd}\n");
639	// Bounded by the step's deadline and interruptible on supersession, so a
640	// hung command fails its step instead of running unbounded, and a
641	// superseded build stops mid-step rather than only at the next boundary.
642	let label = format!("{cur} command on `{host}`");
643	let out = self.run_bounded(&label, async move {
644	use ops_core::remote::LogSink as _;
645	let mut guard = sink.lock().await;
646	guard.write_chunk(echo.as_bytes()).await;
647	exec.run_streaming(&step, &mut *guard).await
648	})?;
649	let code = out.status.code().unwrap_or(-1);
650	let stdout = String::from_utf8_lossy(&out.stdout);
651	let tail: String = stdout
652	.chars()
653	.rev()
654	.take(2000)
655	.collect::<Vec<_>>()
656	.into_iter()
657	.rev()
658	.collect();
659	Ok((code, tail))
660	}
661
662	/// Assert `host`'s build tree is at the release tag and return the commit,
663	/// which is what a recipe's `checkout` step logs. Fetch + checkout stream
664	/// into the current step's log; the sha comes from a separate `rev-parse` so
665	/// its stdout is only the sha.
666	///
667	/// The preflight has already put this tree at the tag before any recipe ran —
668	/// this is the same operation again, on purpose, so that a recipe's own
669	/// `checkout` step is a real step with a real log rather than a claim about
670	/// something that happened elsewhere. Re-running it is cheap and, because the
671	/// tree is Bento's own worktree, forcing: a build that has already dirtied it
672	/// must not be able to fail its own retry.
673	fn checkout_sha(self: &Arc<Self>, host: &str) -> Result<String> {
674	// Every command below runs ON `host`, so the path is that host's, not the
675	// daemon's. Windows is why: its worktree is under `C:/Users/me/Code/...`.
676	let repo = self.repo_for(host).to_string();
677	// A failing mirror is not a failing release: fetch is advisory, and only
678	// the checkout decides. Its output still streams into the step log, so an
679	// unreachable remote stays visible without being fatal.
680	let _ = self.run(host, &git_fetch_cmd(&repo))?;
681	let (code, err) = self.run(host, &git_worktree_pin_cmd(&repo, &self.tag))?;
682	if code != 0 {
683	let (probe, _) = self.run(host, &git_tag_exists_cmd(&repo, &self.tag))?;
684	anyhow::bail!(
685	"checkout of {} failed on `{host}`: {}",
686	self.tag,
687	worktree_failure_reason(&self.tag, probe == 0, &err)
688	);
689	}
690	let (code, tail) = self.run(host, &git_rev_parse_cmd(&repo))?;
691	anyhow::ensure!(code == 0, "rev-parse failed on `{host}`");
692	Ok(tail.trim().to_string())
693	}
694
695	/// Every artifact this run collected, `file name -> sha256`.
696	///
697	/// Already computed at `collect`, which is the only moment the bytes are
698	/// known to be the ones that landed.
699	pub fn artifact_hashes(&self) -> HashMap<String, String> {
700	self.artifact_hashes.lock().unwrap().clone()
701	}
702
703	/// Resolve `glob` on `host` to the single artifact it names. The `for` loop
704	/// lists each existing match on its own line (and prints nothing — rather
705	/// than a literal unexpanded pattern — when the glob matches no file), so
706	/// the count is unambiguous. `required` controls whether zero matches is an
707	/// error; more than one always is. See `resolve_artifact_match`.
708	fn resolve_artifact(
709	self: &Arc<Self>,
710	host: &str,
711	glob: &str,
712	required: bool,
713	) -> Result<String> {
714	ensure_glob_safe(glob)?;
715	// `[ -e ]` guards against a non-matching glob surviving as its literal
716	// self, and lists one path per line for the count.
717	let cmd = format!("for __f in {glob}; do [ -e \"$__f\" ] && printf '%s\\n' \"$__f\"; done");
718	let (code, tail) = self.run(host, &cmd)?;
719	anyhow::ensure!(
720	code == 0,
721	"resolving artifact glob `{glob}` on `{host}` exited {code}"
722	);
723	resolve_artifact_match(&tail, glob, required)
724	}
725	}
726
727	// ----- error bridging: anyhow -> Rhai runtime error -----
728
729	// Rhai host functions return `Result<_, Box<EvalAltResult>>` by convention, so
730	// this bridge must yield the boxed form to be usable with `.map_err(rhai_err)`.
731	#[allow(
732	clippy::unnecessary_box_returns,
733	reason = "rhai's error type is used boxed throughout its host-function API"
734	)]
735	fn rhai_err(e: impl std::fmt::Display) -> Box<EvalAltResult> {
736	Box::new(EvalAltResult::ErrorRuntime(
737	e.to_string().into(),
738	rhai::Position::NONE,
739	))
740	}
741
742	/// A crate's publish-relevant metadata, read from `cargo metadata`.
743	#[derive(Debug, Clone)]
744	pub struct CrateMeta {
745	pub name: String,
746	pub version: String,
747	pub repository: Option<String>,
748	pub description: Option<String>,
749	pub licensed: bool,
750	}
751
752	/// Parse the fields that matter for publishing out of `cargo metadata` JSON.
753	pub fn crate_meta_from_json(raw: &str) -> Result<CrateMeta> {
754	let v: serde_json::Value = serde_json::from_str(raw).context("parsing cargo metadata")?;
755	let p = v
756	.get("packages")
757	.and_then(\|p\| p.as_array())
758	.and_then(\|a\| a.first())
759	.context("cargo metadata reported no package")?;
760	let str_field = \|k: &str\| {
761	p.get(k)
762	.and_then(\|x\| x.as_str())
763	.filter(\|s\| !s.is_empty())
764	.map(str::to_string)
765	};
766	Ok(CrateMeta {
767	name: str_field("name").context("package has no name")?,
768	version: str_field("version").context("package has no version")?,
769	repository: str_field("repository"),
770	description: str_field("description"),
771	licensed: str_field("license").is_some() \|\| str_field("license_file").is_some(),
772	})
773	}
774
775	/// Everything wrong with a crate's metadata, as messages. Empty means publishable.
776	///
777	/// Checks only what crates.io records permanently. A published version cannot
778	/// be edited, only yanked, and yanking does not correct a wrong URL — so these
779	/// are the last moment any of it can be fixed.
780	pub fn crate_publish_problems(
781	meta: &CrateMeta,
782	repo_clonable: bool,
783	published: &[String],
784	credentials_present: bool,
785	) -> Vec<String> {
786	let mut out = Vec::new();
787	if !credentials_present {
788	out.push(
789	"no crates.io credentials on the publishing host: `cargo login` there first. \
790	Checked now rather than at the upload, so this fails in seconds instead of \
791	after a full build and verify."
792	.to_string(),
793	);
794	}
795	match &meta.repository {
796	None => out.push(
797	"no `repository` field: the crates.io page will show no source link, permanently"
798	.to_string(),
799	),
800	Some(url) if !repo_clonable => out.push(format!(
801	"`repository` is not publicly clonable: {url} \
802	(wrong URL, or the repo is private)"
803	)),
804	Some(_) => {}
805	}
806	if meta.description.is_none() {
807	out.push("no `description`: crates.io requires one".to_string());
808	}
809	if !meta.licensed {
810	out.push("no `license` or `license-file`".to_string());
811	}
812	if published.iter().any(\|v\| v == &meta.version) {
813	out.push(format!(
814	"version {} is already published; bump it",
815	meta.version
816	));
817	}
818	out
819	}
820
821	/// Read the app's version from its checkout on the daemon host. With
822	/// `version_path` set (topology `version_path`), read exactly that file — a
823	/// `.json` as a Tauri config, anything else as a `Cargo.toml`. Unset (the Tauri
824	/// default), try `src-tauri/tauri.conf.json` then the root `Cargo.toml`. Used by
825	/// the runner's default-version path.
826	pub fn version_from_repo(repo: &str, version_path: Option<&str>) -> Result<Version> {
827	let root = expand_tilde(repo);
828	if let Some(vp) = version_path {
829	let path = root.join(vp);
830	let raw = std::fs::read_to_string(&path)
831	.with_context(\|\| format!("reading version file {}", path.display()))?;
832	let ver = if std::path::Path::new(vp)
833	.extension()
834	.is_some_and(\|e\| e.eq_ignore_ascii_case("json"))
835	{
836	version_from_tauri_json(&raw)?
837	} else {
838	version_from_cargo_toml(&raw)?
839	};
840	return Version::parse(&ver).map_err(\|e\| anyhow::anyhow!(e));
841	}
842	let tauri_conf = root.join("src-tauri").join("tauri.conf.json");
843	if tauri_conf.exists() {
844	let raw = std::fs::read_to_string(&tauri_conf)
845	.with_context(\|\| format!("reading {}", tauri_conf.display()))?;
846	return Version::parse(&version_from_tauri_json(&raw)?).map_err(\|e\| anyhow::anyhow!(e));
847	}
848	let cargo_toml = root.join("Cargo.toml");
849	let raw = std::fs::read_to_string(&cargo_toml).with_context(\|\| {
850	format!(
851	"reading {} (no tauri.conf.json either)",
852	cargo_toml.display()
853	)
854	})?;
855	Version::parse(&version_from_cargo_toml(&raw)?).map_err(\|e\| anyhow::anyhow!(e))
856	}
857
858	/// Extract `version` from raw `tauri.conf.json` text.
859	fn version_from_tauri_json(raw: &str) -> Result<String> {
860	let v: serde_json::Value = serde_json::from_str(raw).context("parsing tauri.conf.json")?;
861	v.get("version")
862	.and_then(\|x\| x.as_str())
863	.map(str::to_owned)
864	.context("no `version` in tauri.conf.json")
865	}
866
867	/// Extract the version from raw `Cargo.toml` text — `[package].version` (a leaf
868	/// crate) or `[workspace.package].version` (a workspace that sets it).
869	fn version_from_cargo_toml(raw: &str) -> Result<String> {
870	let doc: toml::Value = toml::from_str(raw).context("parsing Cargo.toml")?;
871	doc.get("package")
872	.and_then(\|p\| p.get("version"))
873	.or_else(\|\| {
874	doc.get("workspace")
875	.and_then(\|w\| w.get("package"))
876	.and_then(\|p\| p.get("version"))
877	})
878	.and_then(\|v\| v.as_str())
879	.map(str::to_owned)
880	.context("no `[package].version` or `[workspace.package].version` in Cargo.toml")
881	}
882
883	/// Cross-check every version source in a repo and confirm they all agree with
884	/// the version being built, before a single host pulls or compiles.
885	///
886	/// `version_from_repo` reads exactly one file, so a `tauri.conf.json` at 0.5.0
887	/// and a root `Cargo.toml` still at 0.4.0 build happily and file artifacts under
888	/// whichever the runner happened to read. This reads every source present —
889	/// `version_path` (when set), `src-tauri/tauri.conf.json`, and the root
890	/// `Cargo.toml` — and fails loudly when any disagree, naming each file and its
891	/// version. A source that is absent is skipped (a library crate with only a
892	/// `Cargo.toml` has nothing to disagree with); the check never invents drift.
893	///
894	/// Scope: the JSON/TOML sources bentod itself reads. The iOS `gen/apple/project.yml`
895	/// path (rewritten by a build-time `sed`) is out of scope here — it is asserted at
896	/// its own build step — but the same drift class motivated this guard.
897	pub fn check_version_consistency(
898	repo: &str,
899	version_path: Option<&str>,
900	expected: &Version,
901	) -> Result<()> {
902	let root = expand_tilde(repo);
903	let mut sources: Vec<(String, String)> = Vec::new();
904	for rel in version_sources(version_path) {
905	let path = root.join(&rel);
906	// A source that is absent is skipped — a library crate with only a
907	// `Cargo.toml` has nothing to disagree with — but one the app NAMES
908	// must be readable, or the check would pass by failing to look.
909	match std::fs::read_to_string(&path) {
910	Ok(raw) => sources.push((rel, raw)),
911	Err(e) if version_path == Some(rel.as_str()) => {
912	return Err(e).with_context(\|\| format!("reading version file {}", path.display()));
913	}
914	Err(_) => {}
915	}
916	}
917	versions_agree(repo, &sources, version_path, expected)
918	}
919
920	/// The files a repo can state its version in, in the order they are read:
921	/// whatever the app names, then the two conventional ones.
922	///
923	/// The app's own `version_path` is never read twice, which is why this is a
924	/// function rather than a constant.
925	pub fn version_sources(version_path: Option<&str>) -> Vec<String> {
926	let mut rels: Vec<String> = version_path.into_iter().map(str::to_string).collect();
927	for conventional in ["src-tauri/tauri.conf.json", "Cargo.toml"] {
928	if version_path != Some(conventional) {
929	rels.push(conventional.to_string());
930	}
931	}
932	rels
933	}
934
935	/// The judgement half of [`check_version_consistency`], over sources somebody
936	/// else read.
937	///
938	/// Split out so the same rule can be applied to files read out of the release
939	/// TAG on a build host, which is where the question actually belongs: the tree
940	/// a release compiles is the tag's, so a `Cargo.toml` that disagrees with the
941	/// tag it is tagged in is the drift worth refusing. Reading the working copy
942	/// instead answered a question about a tree the release does not build.
943	///
944	/// `where_` is only for the error message — a path, or a tag and a host.
945	pub fn versions_agree(
946	where_: &str,
947	sources: &[(String, String)],
948	version_path: Option<&str>,
949	expected: &Version,
950	) -> Result<()> {
951	let mut found: Vec<(String, Version)> = Vec::new();
952	for (rel, raw) in sources {
953	// The app's own `version_path` can be either shape, so it is decided by
954	// extension; the two conventional sources are what they are.
955	let as_json = std::path::Path::new(rel)
956	.extension()
957	.is_some_and(\|e\| e.eq_ignore_ascii_case("json"));
958	let ver = if as_json {
959	version_from_tauri_json(raw)
960	} else {
961	// A Cargo.toml with neither `[package].version` nor
962	// `[workspace.package].version` (a pure virtual workspace) carries
963	// no version to check — skip it rather than fail. An app that NAMED
964	// this file is held to it.
965	match version_from_cargo_toml(raw) {
966	Ok(v) => Ok(v),
967	Err(e) if version_path == Some(rel.as_str()) => Err(e),
968	Err(_) => continue,
969	}
970	}?;
971	found.push((
972	rel.clone(),
973	Version::parse(&ver).map_err(\|e\| anyhow::anyhow!(e))?,
974	));
975	}
976
977	let disagree: Vec<&(String, Version)> = found.iter().filter(\|(_, v)\| v != expected).collect();
978	anyhow::ensure!(
979	disagree.is_empty(),
980	"version drift in {where_}: building {expected} but {}",
981	disagree
982	.iter()
983	.map(\|(src, v)\| format!("{src} says {v}"))
984	.collect::<Vec<_>>()
985	.join(", ")
986	);
987	Ok(())
988	}
989
990	/// Read one file as it exists in `tag`, without checking anything out.
991	///
992	/// `<rev>:./<path>` resolves the path relative to `-C`, so this is asked from
993	/// the app's own directory and needs no knowledge of where that sits inside the
994	/// repository. A non-zero exit means the file is not in the tag, which is the
995	/// same "absent, so nothing to disagree with" the local read treats it as.
996	pub fn git_show_file_cmd(dir: &str, tag: &str, rel: &str) -> String {
997	format!("git -C \"{dir}\" show \"{tag}:./{rel}\"")
998	}
999
1000	/// Every `X.Y.Z`-shaped version embedded in an artifact file name. Each maximal
1001	/// run of digits-and-dots contributes its first three numeric fields:
1002	/// `GoingsOn_0.5.0_aarch64.dmg` and `demo-9.9.9.bin` both yield one version (the
1003	/// trailing `.bin`/`.dmg` dot is tolerated), while `latest.json` yields `[]` and
1004	/// the `64` in `x86_64` is not three fields. Only the `major.minor.patch` core is
1005	/// taken; a prerelease/build suffix is separated by `-`/`+` and not needed here.
1006	fn versions_in_filename(name: &str) -> Vec<Version> {
1007	name.split(\|c: char\| !(c.is_ascii_digit() \|\| c == '.'))
1008	.filter_map(\|run\| {
1009	let f: Vec<&str> = run.split('.').filter(\|s\| !s.is_empty()).collect();
1010	if f.len() >= 3 && f[..3].iter().all(\|s\| s.chars().all(\|c\| c.is_ascii_digit())) {
1011	Version::parse(&format!("{}.{}.{}", f[0], f[1], f[2])).ok()
1012	} else {
1013	None
1014	}
1015	})
1016	.collect()
1017	}
1018
1019	/// Fail when a collected file's name embeds a version whose `major.minor.patch`
1020	/// is not the one being built. This is the guard against a stale checked-in
1021	/// artifact winning a glob: `ls -t <glob>` once let
1022	/// `AudioFiles-0.4.0-x86_64.AppImage` ship against 0.5.0. A file whose name
1023	/// carries no version (an updater `latest.json`, a `.sig`) is not asserted —
1024	/// there is nothing to compare. Compared on the core so a prerelease build's
1025	/// plain `X.Y.Z` in the filename still matches.
1026	fn assert_artifact_version(name: &str, expected: &Version) -> Result<()> {
1027	let versions = versions_in_filename(name);
1028	anyhow::ensure!(
1029	versions.is_empty() \|\| versions.iter().any(\|v\| v.core() == expected.core()),
1030	"collected artifact `{name}` carries version {} but the build is {expected}; \
1031	a stale artifact was left in the output dir — clean it so only {expected} remains",
1032	versions
1033	.iter()
1034	.map(ToString::to_string)
1035	.collect::<Vec<_>>()
1036	.join("/"),
1037	);
1038	Ok(())
1039	}
1040
1041	/// sha256 of a file, lowercase hex. Streams in 64 KiB chunks so a multi-GiB
1042	/// bundle never lands in memory whole.
1043	fn sha256_file(path: &Path) -> Result<String> {
1044	let mut file =
1045	std::fs::File::open(path).with_context(\|\| format!("hashing {}", path.display()))?;
1046	let mut hasher = Sha256::new();
1047	std::io::copy(&mut file, &mut hasher)
1048	.with_context(\|\| format!("reading {} to hash", path.display()))?;
1049	Ok(hex_lower(&hasher.finalize()))
1050	}
1051
1052	/// Every regular file under `root`, as `(path relative to root, absolute path)`,
1053	/// sorted by the relative path.
1054	///
1055	/// This walk has to match `bundle::digest_dir` in sando, file for file. That
1056	/// function re-hashes an incoming bundle and refuses it when the bytes disagree
1057	/// with the manifest they arrived with, so a producer that walks differently
1058	/// produces a manifest the consumer will reject for an artifact nothing is wrong
1059	/// with. Three properties carry that agreement, and none is incidental:
1060	///
1061	/// - Recursive. A bundle may carry a directory (migrations, resources), and
1062	/// a top-level-only listing would omit its contents from the manifest while
1063	/// the verifier hashed them.
1064	/// - Symlinks are not followed, and not recorded. Following one would let
1065	/// content from outside the bundle into its identity; recording the link
1066	/// itself would name a file the verifier does not hash.
1067	/// - Relative paths, `/`-separated, sorted. Readdir order is not guaranteed,
1068	/// so an unsorted manifest would differ run to run on one machine, never mind
1069	/// between two.
1070	fn collected_files(root: &Path) -> std::io::Result<Vec<(String, PathBuf)>> {
1071	fn walk(dir: &Path, root: &Path, out: &mut Vec<(String, PathBuf)>) -> std::io::Result<()> {
1072	for entry in std::fs::read_dir(dir)? {
1073	let entry = entry?;
1074	let ft = entry.file_type()?;
1075	let path = entry.path();
1076	if ft.is_dir() {
1077	walk(&path, root, out)?;
1078	} else if ft.is_file() {
1079	let rel = path
1080	.strip_prefix(root)
1081	.unwrap_or(&path)
1082	.components()
1083	.map(\|c\| c.as_os_str().to_string_lossy())
1084	.collect::<Vec<_>>()
1085	.join("/");
1086	out.push((rel, path));
1087	}
1088	// Symlinks and other special files are intentionally ignored,
1089	// matching the verifier.
1090	}
1091	Ok(())
1092	}
1093	let mut out = Vec::new();
1094	walk(root, root, &mut out)?;
1095	out.sort_by(\|a, b\| a.0.cmp(&b.0));
1096	Ok(out)
1097	}
1098
1099	/// Lowercase-hex encode without pulling in a hex crate.
1100	fn hex_lower(bytes: &[u8]) -> String {
1101	use std::fmt::Write as _;
1102	let mut s = String::with_capacity(bytes.len() * 2);
1103	for b in bytes {
1104	let _ = write!(s, "{b:02x}");
1105	}
1106	s
1107	}
1108
1109	/// Refresh every remote's refs and tags, so the tag a release names is present
1110	/// locally however it was pushed. No branch/upstream assumptions — a bare
1111	/// `git pull --ff-only` needs a tracking branch the release path shouldn't
1112	/// depend on.
1113	///
1114	/// Best-effort on purpose. `fetch --all` exits non-zero if ANY remote fails, and
1115	/// the library repos carry three (`astra`, `mnw`, `srht`), so chaining this into
1116	/// the checkout with `&&` meant one unreachable mirror aborted the release and
1117	/// reported it as a missing tag. The checkout below is the step allowed to fail;
1118	/// this one only has to try. See [`git_worktree_pin_cmd`].
1119	///
1120	/// `repo` is interpolated UNQUOTED so a leading `~` is expanded by the remote
1121	/// host's shell (the checkout path is trusted topology config, not user input),
1122	/// matching how the recipes `cd` into it.
1123	pub fn git_fetch_cmd(repo: &str) -> String {
1124	format!("git -C {repo} fetch --all --tags --prune")
1125	}
1126
1127	/// Probe for the one failure a tracked `Cargo.lock` hits inside Bento's
1128	/// worktree. Exits 0 when the crate tracks a lock AND the checkout sits under a
1129	/// `.cargo/config.toml` declaring `[patch]`; 1 otherwise.
1130	///
1131	/// Both halves are needed and cargo reports neither. Under a `[patch]` block
1132	/// cargo re-resolves and rewrites the lock's `[[patch.unused]]` entries, so
1133	/// `cargo publish --dry-run` refuses the tree with "1 files in the working
1134	/// directory contain changes that were not yet committed into git: Cargo.lock"
1135	/// -- naming the lock and nothing about why it moved. pter 0.2.1 lost half an
1136	/// hour to that message on build 325.
1137	///
1138	/// `~/Code/.bento` is under `~/Code` deliberately, so that the patch block
1139	/// reaches the build (see [`crate::topology::Host::worktree_root`]). The patch
1140	/// block is therefore not the half to remove, which is why this is worth
1141	/// saying rather than leaving cargo to be cryptic about it.
1142	///
1143	/// `repo` is interpolated unquoted for the `~`, matching [`git_fetch_cmd`];
1144	/// `pwd -P` then hands the loop an absolute path to walk up from.
1145	pub fn tracked_lock_under_patch_cmd(repo: &str) -> String {
1146	format!(
1147	"git -C {repo} ls-files --error-unmatch Cargo.lock >/dev/null 2>&1 \|\| exit 1; \
1148	d=$(cd {repo} && pwd -P) \|\| exit 1; \
1149	while [ -n \"$d\" ] && [ \"$d\" != / ]; do \
1150	for c in \"$d/.cargo/config.toml\" \"$d/.cargo/config\"; do \
1151	[ -f \"$c\" ] && grep -q '^\\[patch' \"$c\" && exit 0; \
1152	done; d=$(dirname \"$d\"); done; exit 1"
1153	)
1154	}
1155
1156	/// What to say when [`tracked_lock_under_patch_cmd`] answers yes. Names both
1157	/// facts, because the error cargo would otherwise print names neither, and
1158	/// closes off the two wrong fixes that are both one flag away.
1159	pub fn tracked_lock_under_patch_problem(repo: &str) -> String {
1160	format!(
1161	"`Cargo.lock` is tracked and {repo} sits under a `.cargo/config.toml` \
1162	declaring `[patch]`. Cargo re-resolves there and rewrites the lock, so \
1163	`cargo publish --dry-run` will refuse the tree as dirty and name only \
1164	the lock. Untrack it: `git rm --cached Cargo.lock`. Every other library \
1165	in this tree already does. Not `--allow-dirty`, which publishes a lock \
1166	nobody reviewed, and not committing the rewritten lock, which resolves \
1167	differently on the next machine and fails there instead. The build \
1168	worktree is under `~/Code` on purpose so the patch block applies to it; \
1169	that is not the half to change."
1170	)
1171	}
1172
1173	/// Does `tag` resolve to a commit in this checkout? Run only when the checkout
1174	/// has already failed, to say WHY: an absent tag is an untagged or unpushed
1175	/// release, while a tag that resolves fine means the checkout was refused for a
1176	/// local reason (a dirty tree, most often) and the operator needs to hear that
1177	/// instead.
1178	pub fn git_tag_exists_cmd(repo: &str, tag: &str) -> String {
1179	format!("git -C {repo} rev-parse -q --verify \"refs/tags/{tag}^{{commit}}\"")
1180	}
1181
1182	/// Which repository `repo` belongs to, and where `repo` sits inside it, in one
1183	/// call: `--show-toplevel` then `--show-prefix`, one per line.
1184	///
1185	/// Both halves are needed to build in a worktree of a repo holding several
1186	/// products. The worktree is made of the repository (`~/Code/MNW`), and the
1187	/// recipe has to be pointed at the app inside it (`<worktree>/pom`).
1188	pub fn git_toplevel_and_prefix_cmd(repo: &str) -> String {
1189	format!("git -C {repo} rev-parse --show-toplevel --show-prefix")
1190	}
1191
1192	/// Read [`git_toplevel_and_prefix_cmd`]'s two lines.
1193	///
1194	/// The prefix is empty for a repo holding one product, where `repo` IS the
1195	/// repository root — and git prints an empty second line for it, so a missing
1196	/// line is a malformed answer rather than that case.
1197	pub fn parse_toplevel_and_prefix(out: &str) -> Option<(String, String)> {
1198	let mut lines = out.split('\n');
1199	let toplevel = lines.next()?.trim().to_string();
1200	let prefix = lines.next()?.trim().to_string();
1201	(!toplevel.is_empty()).then_some((toplevel, prefix))
1202	}
1203
1204	/// The repository's own directory name, which is what names its worktrees:
1205	/// `MNW` for `/home/max/Code/MNW`.
1206	///
1207	/// Splits on `/` only. Git reports `--show-toplevel` with forward slashes on
1208	/// every platform, Windows included, so this is the separator to read.
1209	pub fn repo_dir_name(toplevel: &str) -> &str {
1210	toplevel
1211	.trim_end_matches('/')
1212	.rsplit('/')
1213	.next()
1214	.unwrap_or(toplevel)
1215	}
1216
1217	/// Where the app being released sits inside its worktree: the worktree root for
1218	/// a repo holding one product, `<worktree>/pom` for one holding several.
1219	pub fn app_dir_in_worktree(worktree: &str, prefix: &str) -> String {
1220	let prefix = prefix.trim_matches('/');
1221	if prefix.is_empty() {
1222	worktree.to_string()
1223	} else {
1224	format!("{}/{prefix}", worktree.trim_end_matches('/'))
1225	}
1226	}
1227
1228	/// Does this worktree already exist? Run before deciding whether to create one.
1229	///
1230	/// `rev-parse --git-dir` rather than a shell test, because the one non-unix
1231	/// build host has no `test`: every command Bento renders for a host is a git
1232	/// command or something a recipe wrote.
1233	pub fn git_worktree_probe_cmd(worktree: &str) -> String {
1234	format!("git -C \"{worktree}\" rev-parse --git-dir")
1235	}
1236
1237	/// Forget worktrees whose directories are gone. Run before creating one: a
1238	/// directory somebody deleted by hand is still registered in the repository, and
1239	/// `worktree add` refuses the path as in use rather than rebuilding it.
1240	pub fn git_worktree_prune_cmd(toplevel: &str) -> String {
1241	format!("git -C \"{toplevel}\" worktree prune")
1242	}
1243
1244	/// Create this app's build worktree, detached at the release tag. Git creates
1245	/// the leading directories, so the worktree root needs no preparation.
1246	pub fn git_worktree_add_cmd(toplevel: &str, worktree: &str, tag: &str) -> String {
1247	format!("git -C \"{toplevel}\" worktree add --detach --force \"{worktree}\" \"{tag}\"")
1248	}
1249
1250	/// Put an existing build worktree at the release tag.
1251	///
1252	/// `--force` discards whatever the last release left in it — a rewritten
1253	/// `Cargo.lock`, most often — and that is safe here in a way it never was in the
1254	/// ordinary checkout: nothing but Bento writes in this tree, so there is no edit
1255	/// of anybody's to lose. Owning the tree is what buys the forcing.
1256	pub fn git_worktree_pin_cmd(worktree: &str, tag: &str) -> String {
1257	format!("git -C \"{worktree}\" checkout --detach --force \"{tag}\"")
1258	}
1259
1260	/// The operator-facing explanation for a worktree that could not be put at the
1261	/// tag.
1262	///
1263	/// An absent tag is an untagged or unpushed release and is the common case, so
1264	/// it is answered plainly. Anything else is git's own stderr, which says more
1265	/// about a path that is not a worktree, or a worktree another release holds,
1266	/// than a guess would.
1267	pub fn worktree_failure_reason(tag: &str, tag_exists: bool, stderr: &str) -> String {
1268	if !tag_exists {
1269	return format!("tag {tag} does not exist there (is it created and pushed?)");
1270	}
1271	let stderr = stderr.trim();
1272	if stderr.is_empty() {
1273	format!("tag {tag} exists, and git said nothing about why")
1274	} else {
1275	stderr.to_string()
1276	}
1277	}
1278
1279	/// The command a host runs to report the commit it has checked out, for the
1280	/// release preflight barrier.
1281	pub fn git_rev_parse_cmd(repo: &str) -> String {
1282	format!("git -C {repo} rev-parse HEAD")
1283	}
1284
1285	/// Expand a leading `~/` to `$HOME`. Paths in the topology are written with `~`.
1286	pub fn expand_tilde(p: &str) -> PathBuf {
1287	if let Some(rest) = p.strip_prefix("~/")
1288	&& let Ok(home) = std::env::var("HOME")
1289	{
1290	return Path::new(&home).join(rest);
1291	}
1292	PathBuf::from(p)
1293	}
1294
1295	/// Reject a glob that carries shell command metacharacters. Path and wildcard
1296	/// characters (`/ . * ? [ ] ~` etc.) are fine — the pattern reaches a login
1297	/// shell to be expanded — but a `;` or `$(...)` must not ride along and run.
1298	/// Not a privilege boundary (a recipe already runs arbitrary shell via `sh_ok`)
1299	/// but it keeps a malformed pattern from turning into a command. Shared by
1300	/// `collect` and `resolve_artifact`.
1301	fn ensure_glob_safe(glob: &str) -> Result<()> {
1302	anyhow::ensure!(
1303	!glob.chars().any(\|c\| matches!(
1304	c,
1305	';' \| '&' \| '\|' \| '$' \| '`' \| '\'' \| '"' \| '\\' \| ' ' \| '\n' \| '(' \| ')' \| '<' \| '>'
1306	)),
1307	"glob `{glob}` contains shell metacharacters"
1308	);
1309	Ok(())
1310	}
1311
1312	/// Decide the single artifact a glob resolves to from a newline-separated
1313	/// listing of the paths that matched it.
1314	///
1315	/// The recipes used to select an artifact with `ls -t <glob> \| head -1` and
1316	/// guard only on an empty string, so a non-zero `ls` slipped past quietly and a
1317	/// stale newest-by-mtime file could win. This is the strict replacement: it
1318	/// demands exactly one match. Zero matches fail when `required` (return `""`
1319	/// when optional); more than one is always an error rather than an arbitrary
1320	/// newest-wins pick, because an ambiguous match means the build left stale
1321	/// artifacts behind and the wrong one could ship.
1322	fn resolve_artifact_match(listing: &str, glob: &str, required: bool) -> Result<String> {
1323	let matches: Vec<&str> = listing
1324	.lines()
1325	.map(str::trim)
1326	.filter(\|l\| !l.is_empty())
1327	.collect();
1328	match matches.as_slice() {
1329	[] if required => anyhow::bail!("no artifact matched glob `{glob}`"),
1330	[] => Ok(String::new()),
1331	[one] => Ok((*one).to_string()),
1332	many => anyhow::bail!(
1333	"glob `{glob}` is ambiguous: {} artifacts matched ({}). \
1334	The build left more than one behind; clean stale artifacts so exactly one remains.",
1335	many.len(),
1336	many.join(", ")
1337	),
1338	}
1339	}
1340
1341	/// Build a Rhai engine with the host API bound to `ctx`. Sandboxed: recipes
1342	/// touch the outside world only through these functions.
1343	pub fn build_engine(ctx: &Arc<RecipeCtx>) -> Engine {
1344	let mut engine = Engine::new();
1345	// Defensive caps — recipes are first-party but bound the blast radius.
1346	engine.set_max_operations(5_000_000);
1347	engine.set_max_call_levels(64);
1348	engine.set_max_string_size(0);
1349
1350	// --- step(name) ---
1351	{
1352	let ctx = ctx.clone();
1353	engine.register_fn(
1354	"step",
1355	move \|name: &str\| -> Result<(), Box<EvalAltResult>> {
1356	let step: Step = name.parse().map_err(rhai_err)?;
1357	ctx.begin_step(step).map_err(rhai_err)
1358	},
1359	);
1360	}
1361
1362	// --- sh(host, cmd) -> #{ code, stdout_tail } ---
1363	//
1364	// The branch-on-exit-code primitive: the recipe OWNS the outcome. A non-zero
1365	// exit is returned, not raised, and does NOT fail the step or bar publish —
1366	// use this only when the recipe inspects `code` and decides. For a command
1367	// that must succeed (build/sign/etc.), use `sh_ok`, which fails the step (and
1368	// therefore bars publish via the failed-step ledger) on a non-zero exit.
1369	{
1370	let ctx = ctx.clone();
1371	engine.register_fn(
1372	"sh",
1373	move \|host: &str, cmd: &str\| -> Result<Map, Box<EvalAltResult>> {
1374	let (code, tail) = ctx.run(host, cmd).map_err(rhai_err)?;
1375	let mut m = Map::new();
1376	m.insert("code".into(), (code as i64).into());
1377	m.insert("stdout_tail".into(), tail.into());
1378	Ok(m)
1379	},
1380	);
1381	}
1382
1383	// --- sh_ok(host, cmd): run + assert exit 0 (the must-succeed primitive) ---
1384	//
1385	// A non-zero exit fails the current step (added to the publish-barring
1386	// ledger) and aborts the recipe, so an artifact is never shipped after a
1387	// must-succeed command failed.
1388	{
1389	let ctx = ctx.clone();
1390	engine.register_fn(
1391	"sh_ok",
1392	move \|host: &str, cmd: &str\| -> Result<(), Box<EvalAltResult>> {
1393	let (code, _) = ctx.run(host, cmd).map_err(rhai_err)?;
1394	if code != 0 {
1395	// Attribute the failure to the current step explicitly so the
1396	// ledger bars publish even if a future caller swallowed the error.
1397	ctx.fail_current_step();
1398	return Err(rhai_err(format!(
1399	"command on `{host}` exited {code}: {cmd}"
1400	)));
1401	}
1402	Ok(())
1403	},
1404	);
1405	}
1406
1407	// --- resolve_artifact(host, glob) -> path: the ONE artifact matching glob ---
1408	//
1409	// The artifact-selection primitive. Replaces `sh(host, "ls -t <glob> \| head
1410	// -1").stdout_tail.trim()` guarded on an empty string, which let a non-zero
1411	// `ls` pass quietly and a stale newest-by-mtime file win. This resolves the
1412	// glob on the host and demands exactly one match: zero matches or more than
1413	// one both throw (an ambiguous match means the build left stale artifacts,
1414	// and silently picking the newest is how the wrong bytes ship). Use
1415	// `resolve_artifact_opt` for an artifact that may legitimately be absent.
1416	{
1417	let ctx = ctx.clone();
1418	engine.register_fn(
1419	"resolve_artifact",
1420	move \|host: &str, glob: &str\| -> Result<String, Box<EvalAltResult>> {
1421	ctx.resolve_artifact(host, glob, true).map_err(rhai_err)
1422	},
1423	);
1424	}
1425
1426	// --- resolve_artifact_opt(host, glob) -> path \| "": zero-or-one match ---
1427	//
1428	// Same strict resolution as `resolve_artifact` but tolerates zero matches
1429	// (returns ""); more than one is still an error. For optional outputs like a
1430	// `.deb` or an updater bundle a recipe collects only when present.
1431	{
1432	let ctx = ctx.clone();
1433	engine.register_fn(
1434	"resolve_artifact_opt",
1435	move \|host: &str, glob: &str\| -> Result<String, Box<EvalAltResult>> {
1436	ctx.resolve_artifact(host, glob, false).map_err(rhai_err)
1437	},
1438	);
1439	}
1440
1441	// --- log(msg): operator-visible line into the current step's tail ---
1442	{
1443	let ctx = ctx.clone();
1444	engine.register_fn("log", move \|msg: &str\| -> Result<(), Box<EvalAltResult>> {
1445	let sink = ctx.ensure_step().map_err(rhai_err)?;
1446	let line = format!("[recipe] {msg}\n");
1447	ctx.rt.block_on(async {
1448	use ops_core::remote::LogSink;
1449	sink.lock().await.write_chunk(line.as_bytes()).await;
1450	});
1451	Ok(())
1452	});
1453	}
1454
1455	// --- version_of(app) -> string ---
1456	{
1457	let ctx = ctx.clone();
1458	engine.register_fn(
1459	"version_of",
1460	move \|app: &str\| -> Result<String, Box<EvalAltResult>> {
1461	// Only the current app is in scope; cross-app reads aren't needed.
1462	if app != ctx.app.as_str() {
1463	return Err(rhai_err(format!(
1464	"version_of: `{app}` is not the app being built"
1465	)));
1466	}
1467	Ok(ctx.version.to_string())
1468	},
1469	);
1470	}
1471
1472	// --- version() -> string: the version being built (no-arg form) ---
1473	{
1474	let ctx = ctx.clone();
1475	engine.register_fn("version", move \|\| -> String { ctx.version.to_string() });
1476	}
1477
1478	// --- build_host() -> string: the host this target builds on ---
1479	{
1480	let ctx = ctx.clone();
1481	engine.register_fn("build_host", move \|\| -> String { ctx.build_host.clone() });
1482	}
1483
1484	// --- repo() -> string: the app's checkout path on this target's build host
1485	// (`~`-prefixed on a unix host). Host-correct rather than one path per
1486	// app, so a recipe for a host whose checkout is elsewhere still calls
1487	// this instead of hard-coding the path — which is what kept the Windows
1488	// recipes off `checkout_sha`. ---
1489	{
1490	let ctx = ctx.clone();
1491	engine.register_fn("repo", move \|\| -> String {
1492	ctx.repo_for(&ctx.build_host).to_string()
1493	});
1494	}
1495
1496	// --- checkout_sha(host) -> sha: pin this host to the release tag and report
1497	// its commit. Replaces a recipe's `git pull --ff-only`, which builds
1498	// whatever `main` is at pull time; the daemon also runs the same pin as
1499	// a cross-host preflight barrier before any target builds. ---
1500	{
1501	let ctx = ctx.clone();
1502	engine.register_fn(
1503	"checkout_sha",
1504	move \|host: &str\| -> Result<String, Box<EvalAltResult>> {
1505	ctx.checkout_sha(host).map_err(rhai_err)
1506	},
1507	);
1508	}
1509
1510	// --- crate_preflight() -> string: verify this crate is safe to publish,
1511	// or abort the run. Everything it checks is immutable once published:
1512	// crates.io versions can be yanked but never edited, so a wrong
1513	// repository URL is permanent. pter 0.1.0 shipped with a dead one. ---
1514	{
1515	let ctx = ctx.clone();
1516	engine.register_fn(
1517	"crate_preflight",
1518	move \|\| -> Result<String, Box<EvalAltResult>> {
1519	// `repo`, not `repo_for(...)`: `cargo metadata` runs on the
1520	// daemon's own box, so this is the one checkout that is always
1521	// the local one. It is not a missed call site.
1522	let repo = expand_tilde(&ctx.repo);
1523
1524	let out = std::process::Command::new("cargo")
1525	.args(["metadata", "--no-deps", "--format-version", "1"])
1526	.current_dir(&repo)
1527	.output()
1528	.map_err(\|e\| format!("running cargo metadata in {}: {e}", repo.display()))?;
1529	if !out.status.success() {
1530	return Err(format!(
1531	"cargo metadata failed in {}: {}",
1532	repo.display(),
1533	String::from_utf8_lossy(&out.stderr).trim()
1534	)
1535	.into());
1536	}
1537	let meta = crate_meta_from_json(&String::from_utf8_lossy(&out.stdout))
1538	.map_err(\|e\| e.to_string())?;
1539
1540	// The real question is not whether a page renders but whether a
1541	// stranger with no credentials can fetch the source, so ask git.
1542	let clonable = meta.repository.as_ref().is_some_and(\|url\| {
1543	std::process::Command::new("git")
1544	.args(["ls-remote", url])
1545	.env("GIT_TERMINAL_PROMPT", "0")
1546	.output()
1547	.is_ok_and(\|o\| o.status.success())
1548	});
1549
1550	// Ask the publishing host whether cargo has credentials, rather
1551	// than moving the token anywhere. It stays in cargo's own 0600
1552	// store; a shell line carrying it would be visible in `ps`.
1553	// An exit code answers "are there credentials"; an Err answers
1554	// "the question could not be asked". Collapsing the second into
1555	// the first reported a capability denial as "no crates.io
1556	// credentials", which sent a real diagnosis three rounds the
1557	// wrong way. A check that cannot run is not a failed check.
1558	let creds =
1559	ctx.run(
1560	&ctx.build_host.clone(),
1561	"cargo login --help >/dev/null 2>&1 && \
1562	test -s \"${CARGO_HOME:-$HOME/.cargo}/credentials.toml\" \
1563	\|\| test -s \"${CARGO_HOME:-$HOME/.cargo}/credentials\"",
1564	)
1565	.map_err(\|e\| {
1566	format!(
1567	"could not check crates.io credentials on `{}`: {e}",
1568	ctx.build_host
1569	)
1570	})?
1571	.0 == 0;
1572
1573	// Asked of the build host rather than the daemon: the tree
1574	// that gets published is the worktree over there, and it is the
1575	// one whose `[patch]` ancestry decides this. An Err is "the
1576	// question could not be asked" and is not a finding -- same
1577	// rule as the credentials check above, for the same reason it
1578	// was written that way.
1579	let build_repo = ctx.repo_for(&ctx.build_host).to_string();
1580	let patched_lock =
1581	ctx.run(
1582	&ctx.build_host.clone(),
1583	&tracked_lock_under_patch_cmd(&build_repo),
1584	)
1585	.map_err(\|e\| {
1586	format!(
1587	"could not check for a tracked Cargo.lock on `{}`: {e}",
1588	ctx.build_host
1589	)
1590	})?
1591	.0 == 0;
1592
1593	let published = RecipeCtx::published_versions(&meta.name);
1594	let mut problems = crate_publish_problems(&meta, clonable, &published, creds);
1595	if patched_lock {
1596	problems.push(tracked_lock_under_patch_problem(&build_repo));
1597	}
1598	if !problems.is_empty() {
1599	return Err(format!(
1600	"{} {} is not safe to publish:\n - {}",
1601	meta.name,
1602	meta.version,
1603	problems.join("\n - ")
1604	)
1605	.into());
1606	}
1607	Ok(format!("{} {} passed preflight", meta.name, meta.version))
1608	},
1609	);
1610	}
1611
1612	// --- feature_flags() -> string: `--features a,b`, or "" when the app
1613	// declares none. Returns the whole flag rather than a bare list so an
1614	// app with no features cannot produce a dangling `--features`. ---
1615	{
1616	let ctx = ctx.clone();
1617	engine.register_fn("feature_flags", move \|\| -> String {
1618	if ctx.features.is_empty() {
1619	String::new()
1620	} else {
1621	format!("--features {}", ctx.features.join(","))
1622	}
1623	});
1624	}
1625
1626	// --- target() / platform() / arch(): the target axis, for one per-platform
1627	// recipe to branch on arch (bundle paths differ between x86_64/aarch64). ---
1628	{
1629	let ctx = ctx.clone();
1630	engine.register_fn("target", move \|\| -> String { ctx.target.to_string() });
1631	}
1632	{
1633	let ctx = ctx.clone();
1634	engine.register_fn("platform", move \|\| -> String {
1635	ctx.target.platform.as_str().to_string()
1636	});
1637	}
1638	{
1639	let ctx = ctx.clone();
1640	engine.register_fn("arch", move \|\| -> String {
1641	ctx.target.arch.as_str().to_string()
1642	});
1643	}
1644
1645	// --- secret(key) -> string (file under secrets_root; never logged) ---
1646	{
1647	let ctx = ctx.clone();
1648	engine.register_fn("secret", move \|key: &str\| -> Result<String, Box<EvalAltResult>> {
1649	// Guard against traversal out of secrets_root. Require every path
1650	// component to be `Normal` (rejects `..`, `.`, absolute roots and
1651	// drive prefixes) and forbid backslashes (a literal filename char on
1652	// Linux, but a separator elsewhere) — the per-component strength of
1653	// Sando's `safe()`. A multi-segment key like `app/token` is still
1654	// allowed; `foo..bar` (a legit filename) is no longer falsely blocked.
1655	let safe = !key.is_empty()
1656	&& !key.contains('\\')
1657	&& std::path::Path::new(key)
1658	.components()
1659	.all(\|c\| matches!(c, std::path::Component::Normal(_)));
1660	if !safe {
1661	return Err(rhai_err(
1662	"secret key must be a relative path under secrets_root (no `..`, `.`, absolute paths, or backslashes)",
1663	));
1664	}
1665	let path = ctx.cfg.secrets_root.join(key);
1666	std::fs::read_to_string(&path)
1667	.map(\|s\| s.trim_end().to_string())
1668	.map_err(\|e\| rhai_err(format!("secret `{key}`: {e}")))
1669	});
1670	}
1671
1672	// --- env(host, key) -> string ---
1673	{
1674	let ctx = ctx.clone();
1675	engine.register_fn(
1676	"env",
1677	move \|host: &str, key: &str\| -> Result<String, Box<EvalAltResult>> {
1678	// The key is interpolated into a `${...}` shell expansion, so it must
1679	// be a bare shell identifier — anything else (quotes, `}`, `$`, `;`)
1680	// could break out and run arbitrary commands on the host. Validate
1681	// before building the command; this is the one env read that can't
1682	// sh-quote its argument (a quoted var name doesn't expand).
1683	if key.is_empty()
1684	\|\| !key
1685	.chars()
1686	.next()
1687	.is_some_and(\|c\| c == '_' \|\| c.is_ascii_alphabetic())
1688	\|\| !key.chars().all(\|c\| c == '_' \|\| c.is_ascii_alphanumeric())
1689	{
1690	return Err(rhai_err(format!(
1691	"env name `{key}` must be a shell identifier ([A-Za-z_][A-Za-z0-9_]*)"
1692	)));
1693	}
1694	// Read via the shell so it works on remote hosts too.
1695	let (code, tail) = ctx
1696	.run(host, &format!("printf '%s' \"${{{key}}}\""))
1697	.map_err(rhai_err)?;
1698	if code != 0 {
1699	return Err(rhai_err(format!("env `{key}` on `{host}` failed")));
1700	}
1701	Ok(tail.trim().to_string())
1702	},
1703	);
1704	}
1705
1706	// --- collect(host, glob, app, version): pull artifacts to dist_root ---
1707	{
1708	let ctx = ctx.clone();
1709	engine.register_fn(
1710	"collect",
1711	move \|host: &str,
1712	glob: &str,
1713	app: &str,
1714	version: &str\|
1715	-> Result<(), Box<EvalAltResult>> {
1716	ctx.collect(host, glob, app, version).map_err(rhai_err)
1717	},
1718	);
1719	}
1720
1721	// --- publish(channel, app, target, version, artifact, meta) ---
1722	{
1723	let ctx = ctx.clone();
1724	engine.register_fn(
1725	"publish",
1726	move \|channel: &str,
1727	app: &str,
1728	target: &str,
1729	version: &str,
1730	artifact: &str,
1731	meta: Map\|
1732	-> Result<String, Box<EvalAltResult>> {
1733	ctx.publish(channel, app, target, version, artifact, &meta)
1734	.map_err(rhai_err)
1735	},
1736	);
1737	}
1738
1739	// --- deploy(binary) -> summary: install a service binary and restart its
1740	// unit. The terminal step for `kind = "service"`, the counterpart of
1741	// `publish` for something that is run rather than distributed.
1742	//
1743	// Takes only the binary's path on the build host: where it lands, on
1744	// which machine, and which unit restarts all come from the `[[deploy]]`
1745	// entry for the target already being built. A recipe cannot deploy the
1746	// aarch64 binary to the x86_64 box by naming the wrong host, because it
1747	// never names a host at all.
1748	{
1749	let ctx = ctx.clone();
1750	engine.register_fn(
1751	"deploy",
1752	move \|binary: &str\| -> Result<String, Box<EvalAltResult>> {
1753	ctx.deploy(binary).map_err(rhai_err)
1754	},
1755	);
1756	}
1757
1758	// --- deploy_host() -> string: the service host's ssh destination, so a
1759	// recipe can run its own assertions there (`sh_ok(deploy_host(), ...)`).
1760	// Commands run through it while the `deploy` step is open, so they are
1761	// gated on the deploy grant like the install itself. ---
1762	{
1763	let ctx = ctx.clone();
1764	engine.register_fn(
1765	"deploy_host",
1766	move \|\| -> Result<String, Box<EvalAltResult>> {
1767	ctx.deploy_target()
1768	.map(\|d\| d.host.clone())
1769	.map_err(rhai_err)
1770	},
1771	);
1772	}
1773
1774	// --- service_name() / install_path() / health_url(): the rest of the
1775	// `[[deploy]]` entry, so a recipe asserts against the configured values
1776	// rather than repeating them as literals that can drift. `health_url`
1777	// is "" when unset. ---
1778	{
1779	let ctx = ctx.clone();
1780	engine.register_fn(
1781	"service_name",
1782	move \|\| -> Result<String, Box<EvalAltResult>> {
1783	ctx.deploy_target()
1784	.map(\|d\| d.service.clone())
1785	.map_err(rhai_err)
1786	},
1787	);
1788	}
1789	{
1790	let ctx = ctx.clone();
1791	engine.register_fn(
1792	"install_path",
1793	move \|\| -> Result<String, Box<EvalAltResult>> {
1794	ctx.deploy_target()
1795	.map(\|d\| d.install_path.clone())
1796	.map_err(rhai_err)
1797	},
1798	);
1799	}
1800	{
1801	let ctx = ctx.clone();
1802	engine.register_fn(
1803	"health_url",
1804	move \|\| -> Result<String, Box<EvalAltResult>> {
1805	ctx.deploy_target()
1806	.map(\|d\| d.health_url.clone().unwrap_or_default())
1807	.map_err(rhai_err)
1808	},
1809	);
1810	}
1811
1812	// --- glibc_check(binary) -> string: assert the build host did not produce
1813	// a binary the service host's glibc is too old to exec. Aborts the run
1814	// if it did; returns "needs X, host has Y" for the log if it did not.
1815	//
1816	// WHICH RECIPES CALL THIS, AND WHY THE OTHERS MUST NOT. The rule is not
1817	// a style preference and it is not optional: this reads the recipe's
1818	// `[[deploy]]` entry to learn which machine runs the bytes, so a recipe
1819	// with no `[[deploy]]` cannot call it at all.
1820	//
1821	// - A service that installs ITSELF (`[[deploy]]` present: magicmirror,
1822	// wam, mnw-cli) SHOULD call it. Bento is both builder and installer
1823	// there, so it knows the service host, and nothing downstream will
1824	// check on its behalf.
1825	// - A service HANDED OFF to Sando (`[[deploy]]` absent: pom) MUST NOT,
1826	// and the absence is the Sando/Bento boundary rather than an omission.
1827	// Which machine runs the bytes is environment knowledge, which is
1828	// Sando's half. Sando covers it on the far side, more strongly: it runs
1829	// the node's own loader against the rsynced bytes before the symlink
1830	// swap (`sando_daemon::deploy::ldd_guard_script`), and since 0.2.12
1831	// also compares the bundle's glibc floor against the node's declared
1832	// `libc` before the rsync (`check_bundle_fits_node`).
1833	//
1834	// So a new service recipe takes its answer from whether it carries a
1835	// `[[deploy]]` table, not from whichever sibling recipe it was copied
1836	// from. Wiki `sando-bento-boundary`, `host-base-images`. ---
1837	{
1838	let ctx = ctx.clone();
1839	engine.register_fn(
1840	"glibc_check",
1841	move \|binary: &str\| -> Result<String, Box<EvalAltResult>> {
1842	let (needs, has) = ctx.glibc_check(binary).map_err(rhai_err)?;
1843	Ok(format!(
1844	"glibc: binary needs {needs}, service host has {has}"
1845	))
1846	},
1847	);
1848	}
1849
1850	// --- macOS signing helpers. They dispatch through the named host's
1851	// executor like any other step; when that host is the mac (transport =
1852	// "agent"), codesign/notarize/staple ride the in-session `AgentRpc`
1853	// transport — the only security session where the Developer ID key is
1854	// usable (design §7 "THE WALL"). Capability-gated by the host's `sign`
1855	// grant. ---
1856	register_macos_fns(&mut engine, ctx);
1857
1858	engine
1859	}
1860
1861	/// Highest `GLIBC_x.y` version referenced by a built binary, and the glibc a
1862	/// host actually has, both parsed from the text the commands print.
1863	///
1864	/// Native-per-architecture builds remove the cross-compile hazard `deploy.sh`
1865	/// was written against, but not this one: fw13 tracks a newer glibc than the
1866	/// Ubuntu 24.04 box in Hetzner, so a binary built here can reference a symbol
1867	/// version that box does not have and fail at exec — after the unit has already
1868	/// been restarted onto it. Comparing the two before the install is what makes
1869	/// that a failed step instead of a downed service.
1870	fn max_glibc_symbol(objdump_out: &str) -> Option<(u64, u64)> {
1871	objdump_out
1872	.split(\|c: char\| !(c.is_ascii_digit() \|\| c == '.' \|\| c == '_' \|\| c.is_ascii_alphabetic()))
1873	.filter_map(\|tok\| tok.strip_prefix("GLIBC_"))
1874	.filter_map(parse_glibc_version)
1875	.max()
1876	}
1877
1878	/// Parse `2.39` (or `2.39.1`, keeping major/minor) into a comparable pair.
1879	fn parse_glibc_version(s: &str) -> Option<(u64, u64)> {
1880	let mut parts = s.split('.');
1881	let major = parts.next()?.parse().ok()?;
1882	let minor = parts.next()?.parse().ok()?;
1883	Some((major, minor))
1884	}
1885
1886	/// The glibc version out of `ldd --version`'s first line, whose tail is the
1887	/// version however the distro decorates the rest (`ldd (Ubuntu GLIBC
1888	/// 2.39-0ubuntu8.8) 2.39`).
1889	fn glibc_from_ldd(ldd_out: &str) -> Option<(u64, u64)> {
1890	let first = ldd_out.lines().find(\|l\| !l.trim().is_empty())?;
1891	parse_glibc_version(first.split_whitespace().last()?)
1892	}
1893
1894	impl RecipeCtx {
1895	/// This target's install destination, or an error naming why there is none.
1896	fn deploy_target(&self) -> Result<&DeployTarget> {
1897	self.deploy.as_ref().ok_or_else(\|\| {
1898	anyhow::anyhow!(
1899	"no deploy destination for {} {}: the app is `kind = \"{}\"`, and only a \
1900	service declares [[deploy]] entries",
1901	self.app,
1902	self.target,
1903	match self.kind {
1904	Kind::App => "app",
1905	Kind::Library => "library",
1906	Kind::Service => "service",
1907	}
1908	)
1909	})
1910	}
1911
1912	/// Compare the built binary's glibc requirement against the service host's.
1913	/// Returns the two versions for the recipe to log.
1914	fn glibc_check(self: &Arc<Self>, binary: &str) -> Result<(String, String)> {
1915	let d = self.deploy_target()?.clone();
1916	// `objdump -T` on the build host; no symbols at all (a static binary)
1917	// means nothing to check, which is a pass rather than a failure.
1918	let (code, out) = self.run(
1919	&self.build_host.clone(),
1920	&format!(
1921	"objdump -T {binary} 2>/dev/null \| grep -o 'GLIBC_[0-9.]*' \| sort -uV \|\| true"
1922	),
1923	)?;
1924	anyhow::ensure!(code == 0, "reading glibc symbols from {binary} failed");
1925	let Some(needs) = max_glibc_symbol(&out) else {
1926	return Ok(("none".into(), "n/a".into()));
1927	};
1928	let (code, ldd) = self.run(&d.host, "ldd --version")?;
1929	anyhow::ensure!(
1930	code == 0,
1931	"could not read glibc version on service host `{}`",
1932	d.host
1933	);
1934	let has = glibc_from_ldd(&ldd).ok_or_else(\|\| {
1935	anyhow::anyhow!(
1936	"could not parse glibc version from `ldd --version` on `{}`",
1937	d.host
1938	)
1939	})?;
1940	anyhow::ensure!(
1941	needs <= has,
1942	"binary needs glibc {}.{} but `{}` has {}.{} — it would fail to exec after the \
1943	unit restarted onto it. Build on a host no newer than the service host.",
1944	needs.0,
1945	needs.1,
1946	d.host,
1947	has.0,
1948	has.1,
1949	);
1950	Ok((
1951	format!("{}.{}", needs.0, needs.1),
1952	format!("{}.{}", has.0, has.1),
1953	))
1954	}
1955
1956	/// Install `binary` (a path on the BUILD host) onto the service host and
1957	/// restart its unit, via the privileged installer the host holds a scoped
1958	/// sudo grant for.
1959	///
1960	/// Bento never runs the install itself. It stages the bytes and calls a
1961	/// root script whose arguments are re-checked on the far side — the same
1962	/// shape as Sando's `install-companion.sh`, and for the same reason: the
1963	/// sudoers grant is then ONE auditable script rather than a broad
1964	/// `install`+`systemctl` grant on a production box.
1965	///
1966	/// Only the binary moves. Config is deliberately untouched: pom's
1967	/// `pom-astra.toml` / `pom-hetzner.toml` differ per instance, and prod's
1968	/// carried a `[targets.mnw.tests]` block the repo did not have. A deploy
1969	/// that copies config over is how that block gets silently deleted.
1970	fn deploy(self: &Arc<Self>, binary: &str) -> Result<String> {
1971	anyhow::ensure!(
1972	!self.is_cancelled(),
1973	"build superseded by a newer request; refusing to deploy"
1974	);
1975	// A failed earlier step bars a deploy exactly as it bars a publish. An
1976	// artifact that failed its gates must not reach a production host just
1977	// because the recipe kept running.
1978	let failed = self.failed_steps.lock().unwrap().clone();
1979	anyhow::ensure!(
1980	failed.is_empty(),
1981	"refusing to deploy {} {}: {} failed earlier in this run",
1982	self.app,
1983	self.version,
1984	failed
1985	.iter()
1986	.map(ToString::to_string)
1987	.collect::<Vec<_>>()
1988	.join(", "),
1989	);
1990	let d = self.deploy_target()?.clone();
1991	ensure_glob_safe(binary)?;
1992
1993	// Stage under a fixed root the installer also insists on, so "what was
1994	// checked" and "what is installed" cannot drift apart.
1995	let staged = format!("{DEPLOY_STAGING_ROOT}/{}", self.app);
1996	let staged_bin = format!("{staged}/{}", self.app);
1997	let deploy_exec = self.exec(&d.host)?;
1998	anyhow::ensure!(
1999	deploy_exec.capabilities().permits(&Action::Deploy),
2000	"service host `{}` is not granted the `deploy` capability",
2001	d.host
2002	);
2003
2004	self.run_ok(&d.host, &format!("mkdir -p {staged}"))?;
2005	if self.build_host_ssh == d.host {
2006	// Same box: the binary is already there. Routing it through the
2007	// daemon would be two transfers to end up where it started. This is
2008	// pom's aarch64 leg — astra builds it and astra runs it.
2009	self.run_ok(&d.host, &format!("cp -f {binary} {staged_bin}"))?;
2010	} else {
2011	// Build host -> daemon -> service host. Two hops because an executor
2012	// reaches one host; a direct host-to-host transport would mean the
2013	// build host holding a credential for the production box.
2014	let tmp = tempfile::tempdir().context("staging dir for deploy")?;
2015	let local = tmp.path().join(self.app.as_str());
2016	self.pull_for_deploy(binary, &local)?;
2017	let (dest, opts) = (PathBuf::from(&staged), SyncOpts::default());
2018	let dir = tmp.path().to_path_buf();
2019	self.run_bounded(&format!("stage {} on `{}`", self.app, d.host), async move {
2020	deploy_exec.push_dir(&dir, &dest, &opts).await
2021	})
2022	.with_context(\|\| format!("staging {} onto `{}`", self.app, d.host))?;
2023	}
2024
2025	// The privileged half. Every argument is re-validated by the script,
2026	// which is the thing actually holding the sudo grant.
2027	self.run_ok(
2028	&d.host,
2029	&format!(
2030	"{} {staged_bin} {} {}",
2031	self.cfg.deploy_installer, d.install_path, d.service
2032	),
2033	)?;
2034	Ok(format!(
2035	"{} {} installed at {} on `{}`; {} restarted",
2036	self.app, self.version, d.install_path, d.host, d.service
2037	))
2038	}
2039
2040	/// Fetch one file off a host into a daemon-local path for re-pushing.
2041	///
2042	/// A local build host is read directly: `fw13` is the daemon's own box, so
2043	/// the file is already on this filesystem. Routing it through the
2044	/// artifact-pull gate instead would demand a `pull_root` covering every repo
2045	/// a service could be built in — today that is `~/Code/Apps`, and pom lives
2046	/// in `~/Code/MNW`. Widening it to `~/Code` would put `_private`, the
2047	/// secrets root, inside the collectable tree. This is pom's x86_64 leg.
2048	fn pull_for_deploy(self: &Arc<Self>, remote: &str, local: &Path) -> Result<()> {
2049	let host = self.build_host.clone();
2050	let remote_path = expand_tilde(remote);
2051	if self.build_host_ssh == "local" \|\| self.build_host_ssh.is_empty() {
2052	std::fs::copy(&remote_path, local).with_context(\|\| {
2053	format!("staging {} from the daemon host", remote_path.display())
2054	})?;
2055	return Ok(());
2056	}
2057	let sync = self.host_sync(&host)?;
2058	let (src, dst, opts) = (remote_path, local.to_path_buf(), SyncOpts::default());
2059	self.run_bounded(&format!("fetch {remote} from `{host}`"), async move {
2060	sync.pull_file(&src, &dst, &opts).await
2061	})
2062	.with_context(\|\| format!("fetching {remote} from `{host}` to deploy"))
2063	}
2064
2065	/// `run`, failing the step on a non-zero exit. The Rust-side twin of the
2066	/// recipe's `sh_ok`, for commands the deploy machinery issues itself.
2067	fn run_ok(self: &Arc<Self>, host: &str, cmd: &str) -> Result<String> {
2068	let (code, tail) = self.run(host, cmd)?;
2069	if code != 0 {
2070	self.fail_current_step();
2071	anyhow::bail!("command on `{host}` exited {code}: {cmd}\n{tail}");
2072	}
2073	Ok(tail)
2074	}
2075
2076	/// Where this run's collected files land locally.
2077	///
2078	/// Per target, not per version. Every target used to share one
2079	/// `dist_root/<app>/<version>/`, so the hash loop below (which lists the
2080	/// directory) attributed a sibling's AppImage to the mac build's artifact
2081	/// record. It is also the layout the archive uses, and the two have to agree
2082	/// or the local copy and the deposited one are different shapes.
2083	fn collect_dest(&self, app: &str, version: &str) -> PathBuf {
2084	self.cfg
2085	.dist_root
2086	.join(app)
2087	.join(version)
2088	.join(crate::archive::target_slug(self.target))
2089	}
2090
2091	fn collect(self: &Arc<Self>, host: &str, glob: &str, app: &str, version: &str) -> Result<()> {
2092	let dest = self.collect_dest(app, version);
2093	let dest_s = dest.to_string_lossy().into_owned();
2094	// The glob reaches a remote login shell intact (that's what expands it),
2095	// so command metacharacters stay barred. Path/wildcard chars are fine.
2096	ensure_glob_safe(glob)?;
2097	std::fs::create_dir_all(&dest)
2098	.with_context(\|\| format!("creating collect dest {dest_s}"))?;
2099	// The SYNC transport, not the host's exec executor: artifacts move over
2100	// ssh/rsync even from an agent host, whose `/pull` is confined to a
2101	// narrow `pull_root` that deliberately excludes the repo checkout these
2102	// artifacts are built in (see `state::build_sync`). The daemon still
2103	// runs the transfer itself, as it always has.
2104	let sync = self.host_sync(host)?;
2105	let opts = SyncOpts::precompressed();
2106	// Bounded by the collect step's deadline (rsync of a multi-GiB artifact
2107	// can wedge on a stalled transport) and interruptible on supersession.
2108	let dest_pull = dest.clone();
2109	self.run_bounded(&format!("collect {glob} from `{host}`"), async move {
2110	sync.pull_glob(glob, &dest_pull, &opts).await
2111	})
2112	.with_context(\|\| format!("collect {glob} from `{host}`"))?;
2113	// Assert the version and hash every collected file. This is where a
2114	// stale artifact is caught: a file whose name embeds a different version
2115	// fails the collect (rather than silently winning a later glob), and the
2116	// sha256 recorded here is what `publish` writes into the release ledger
2117	// and what the artifact record's manifest is built from.
2118	//
2119	// Recursive, and keyed by path relative to the collect dir. That is not
2120	// a preference: Sando's intake re-hashes the bundle with its own walker,
2121	// which recurses and keys the same way, and refuses a bundle whose bytes
2122	// do not match the manifest it was handed. A top-level `read_dir` keyed
2123	// by file name agrees with that walker for a flat directory and diverges
2124	// the moment a bundle carries a subdirectory — the honest artifact would
2125	// be refused for a manifest that omitted everything nested. The two
2126	// walkers have to be the same walk. See `bundle::digest_dir` in sando.
2127	for (rel, path) in
2128	collected_files(&dest).with_context(\|\| format!("listing collect dest {dest_s}"))?
2129	{
2130	// The version check stays on the file NAME rather than the relative
2131	// path: it is looking for a stale `app_1.2.3.AppImage` beside the
2132	// one this release built, and a directory component is not that.
2133	let name = path
2134	.file_name()
2135	.map_or_else(\|\| rel.clone(), \|n\| n.to_string_lossy().into_owned());
2136	assert_artifact_version(&name, &self.version)?;
2137	let digest = sha256_file(&path)?;
2138	self.artifact_hashes.lock().unwrap().insert(rel, digest);
2139	}
2140	// Deposit at the archive path, so this target's bytes have one address
2141	// whichever host produced them. A no-op when no archive is configured.
2142	//
2143	// Inside `collect`, not after the recipe: a failure here fails the
2144	// collect step, before sign and publish, rather than putting a red mark
2145	// on a release that has already shipped. And it is a failure, not a
2146	// warning — a deposit that is quietly skipped leaves the archive path
2147	// wrong for exactly the release nobody was watching, which is the thing
2148	// having one address is for.
2149	let (cfg, app_id, version, target) = (
2150	self.cfg.clone(),
2151	self.app.clone(),
2152	self.version.clone(),
2153	self.target,
2154	);
2155	let dest_archive = dest.clone();
2156	self.run_bounded("deposit in the archive", async move {
2157	crate::archive::deposit(&cfg, &dest_archive, &app_id, &version, target).await
2158	})?;
2159	// Best-effort size accounting for the event.
2160	events::emit(
2161	&self.events,
2162	Event::ArtifactCollected {
2163	app: self.app.clone(),
2164	target: self.target,
2165	path: dest_s,
2166	bytes: dir_size(&dest).unwrap_or(0),
2167	},
2168	);
2169	Ok(())
2170	}
2171
2172	/// The all-targets-green gate: err unless every declared target OTHER than
2173	/// the one publishing has a latest `target_runs` row of `ok` for this
2174	/// `(app, version)`. A sibling with no run, a running run, or a failed
2175	/// latest run all block the publish, naming what is not green.
2176	fn assert_siblings_green(self: &Arc<Self>, declared: &[Target]) -> Result<()> {
2177	let me = self.clone();
2178	let (app_s, ver_s) = (self.app.to_string(), self.version.to_string());
2179	let rows: Vec<(String, String)> = self.rt.block_on(async move {
2180	sqlx::query_as(
2181	"SELECT target, status FROM target_runs tr
2182	WHERE app = ?1 AND version = ?2
2183	AND id = (SELECT MAX(id) FROM target_runs
2184	WHERE app = ?1 AND version = ?2 AND target = tr.target)",
2185	)
2186	.bind(app_s)
2187	.bind(ver_s)
2188	.fetch_all(&me.pool)
2189	.await
2190	.unwrap_or_default()
2191	});
2192	let status_of = \|t: &Target\| -> Option<String> {
2193	let key = t.to_string();
2194	rows.iter()
2195	.find(\|(name, _)\| name == &key)
2196	.map(\|(_, s)\| s.clone())
2197	};
2198	let not_green: Vec<String> = declared
2199	.iter()
2200	.filter(\|t\| **t != self.target) // the publishing target is the last mile
2201	.filter(\|t\| status_of(t).as_deref() != Some("ok"))
2202	.map(\|t\| format!("{t} ({})", status_of(t).unwrap_or_else(\|\| "no run".into())))
2203	.collect();
2204	anyhow::ensure!(
2205	not_green.is_empty(),
2206	"all-targets-green gate: refusing to publish {} {} — not green: {}",
2207	self.app,
2208	self.version,
2209	not_green.join(", "),
2210	);
2211	Ok(())
2212	}
2213
2214	fn publish(
2215	self: &Arc<Self>,
2216	channel: &str,
2217	app: &str,
2218	target: &str,
2219	version: &str,
2220	artifact: &str,
2221	meta: &Map,
2222	) -> Result<String> {
2223	// Never let a superseded build ship. This is the last and most important
2224	// cooperative-cancel checkpoint: even if a long-running step finished
2225	// after supersession, the artifact must not reach the backend.
2226	anyhow::ensure!(
2227	!self.is_cancelled(),
2228	"build superseded by a newer request; refusing to publish"
2229	);
2230	// Opt-in all-targets-green gate: refuse a partial release. Every OTHER
2231	// declared target of this (app, version) must have a successful latest
2232	// run before this one ships, so macOS can't publish while windows is red
2233	// or still building. The publishing target itself is the last mile (it
2234	// reached publish, so its steps passed) and is not required to be green
2235	// in the ledger yet.
2236	if let Some(declared) = self.all_green_required.clone() {
2237	self.assert_siblings_green(&declared)?;
2238	}
2239	let backend = self
2240	.ota
2241	.get(channel)
2242	.ok_or_else(\|\| anyhow::anyhow!("unknown publish channel `{channel}`"))?;
2243	let target: Target = target.parse().map_err(\|e: String\| anyhow::anyhow!(e))?;
2244	let version = Version::parse(version).map_err(\|e\| anyhow::anyhow!(e))?;
2245	let app = AppId::new(app);
2246
2247	// The backend must actually handle this target (e.g. the desktop updater
2248	// disclaims iOS) — otherwise publish would push an artifact through a
2249	// backend that does not support it.
2250	anyhow::ensure!(
2251	backend.supports(target),
2252	"publish channel `{channel}` does not support target {target}",
2253	);
2254
2255	// Monotonicity: never publish a version that is not strictly newer than
2256	// the latest already published for this (app, target, channel). Without
2257	// this an older build could republish over a live newer release. The
2258	// `releases` column is TEXT, so compare by parsed semver precedence
2259	// (Version: Ord), not lexically.
2260	{
2261	let (app_s, target_s, chan_s) =
2262	(app.to_string(), target.to_string(), channel.to_string());
2263	let me = self.clone();
2264	let latest: Option<Version> = self.rt.block_on(async move {
2265	let rows: Vec<(String,)> = sqlx::query_as(
2266	"SELECT version FROM releases WHERE app = ? AND target = ? AND channel = ?",
2267	)
2268	.bind(app_s)
2269	.bind(target_s)
2270	.bind(chan_s)
2271	.fetch_all(&me.pool)
2272	.await
2273	.unwrap_or_default();
2274	rows.into_iter()
2275	.filter_map(\|(v,)\| Version::parse(&v).ok())
2276	.max()
2277	});
2278	if let Some(latest) = latest {
2279	anyhow::ensure!(
2280	version > latest,
2281	"refusing to publish {app} {version} to `{channel}` ({target}): \
2282	not newer than the last published {latest}",
2283	);
2284	}
2285	}
2286
2287	// Step-success ledger (the Bento analogue of Sando's gate fail-closed),
2288	// minted as an unforgeable PublishAuthority. `backend.publish` cannot be
2289	// called without one, so the unverified/post-failure ship path is sealed
2290	// at the type level rather than guarded by a separate runtime check.
2291	let authority = {
2292	let failed = self.failed_steps.lock().unwrap();
2293	let gatekeeper = *self.gatekeeper_ok.lock().unwrap();
2294	PublishAuthority::prove(target, failed.as_slice(), gatekeeper)?
2295	};
2296	let notes = meta
2297	.get("notes")
2298	.and_then(\|v\| v.clone().into_string().ok())
2299	.unwrap_or_default();
2300	// Resolve the artifact relative to the collected dist dir if not absolute.
2301	let artifact_path = {
2302	let p = PathBuf::from(artifact);
2303	if p.is_absolute() {
2304	p
2305	} else {
2306	self.collect_dest(app.as_str(), &version.to_string())
2307	.join(artifact)
2308	}
2309	};
2310	let rel = Release {
2311	app: &app,
2312	target,
2313	version: &version,
2314	notes,
2315	};
2316	let receipt = backend
2317	.publish(&rel, &artifact_path, &authority)
2318	.with_context(\|\| format!("publish to `{channel}`"))?;
2319	// Record for idempotency / monotonicity. This write is CHECKED, not
2320	// fire-and-forget: a swallowed failure here would silently re-arm the
2321	// monotonicity guard (which reads this same table), letting an older
2322	// version republish over a live release. Concurrent same-(app,target)
2323	// publishers can't race the read-then-insert because the latest-wins slot
2324	// (state::ActiveSlot) serializes them and a superseded run is cancelled
2325	// before it reaches publish.
2326	// The artifact's hash, recorded so the release ledger says exactly which
2327	// bytes shipped. Prefer the digest computed at `collect`; fall back to
2328	// hashing the file now (an absolute-path artifact never routed through
2329	// `collect`). A hash failure must not fail an already-published release,
2330	// so degrade to NULL rather than erroring.
2331	let artifact_hash: Option<String> = artifact_path
2332	.file_name()
2333	.and_then(\|n\| n.to_str())
2334	.and_then(\|n\| self.artifact_hashes.lock().unwrap().get(n).cloned())
2335	.or_else(\|\| sha256_file(&artifact_path).ok());
2336	let me = self.clone();
2337	let (app_s, target_s, ver_s, chan_s) = (
2338	app.to_string(),
2339	target.to_string(),
2340	version.to_string(),
2341	channel.to_string(),
2342	);
2343	self.rt
2344	.block_on(async move {
2345	sqlx::query(
2346	"INSERT OR IGNORE INTO releases (app, target, version, channel, artifact_hash, published_at)
2347	VALUES (?, ?, ?, ?, ?, ?)",
2348	)
2349	.bind(app_s)
2350	.bind(target_s)
2351	.bind(ver_s)
2352	.bind(chan_s)
2353	.bind(artifact_hash)
2354	.bind(Self::now())
2355	.execute(&me.pool)
2356	.await
2357	})
2358	.context("recording release in the idempotency ledger (artifact published but ledger write failed)")?;
2359	events::emit(
2360	&self.events,
2361	Event::PublishOk {
2362	app: self.app.clone(),
2363	target: self.target,
2364	channel: channel.to_string(),
2365	},
2366	);
2367	Ok(receipt)
2368	}
2369	}
2370
2371	fn dir_size(p: &Path) -> Option<i64> {
2372	let mut total = 0i64;
2373	for entry in std::fs::read_dir(p).ok()? {
2374	let entry = entry.ok()?;
2375	let md = entry.metadata().ok()?;
2376	if md.is_file() {
2377	total += md.len() as i64;
2378	} else if md.is_dir() {
2379	// Recurse so a bundle dir (a `.app`) reports its real size, not ~0.
2380	total += dir_size(&entry.path()).unwrap_or(0);
2381	}
2382	}
2383	Some(total)
2384	}
2385
2386	/// macOS signing/notarization host functions. Thin wrappers over the right
2387	/// shell incantations, dispatched through the named host's executor. On the mac
2388	/// host (`transport = "agent"`) they run via the in-session `ops-agent`, the only
2389	/// context where codesign can use the Developer ID key (design §7 "THE WALL"); a
2390	/// plain SSH session cannot. Each is gated by the host's `sign` capability.
2391	fn register_macos_fns(engine: &mut Engine, ctx: &Arc<RecipeCtx>) {
2392	{
2393	let ctx = ctx.clone();
2394	engine.register_fn(
2395	"verify_gatekeeper",
2396	move \|host: &str, path: &str\| -> Result<bool, Box<EvalAltResult>> {
2397	// spctl has no JSON mode, so assess on-host and decide there,
2398	// emitting an unambiguous sentinel as the final line. We match the
2399	// sentinel rather than substring-hunting `source=Notarized...` in a
2400	// 2000-char tail: truncation only drops the front, so the sentinel
2401	// is always present, and it can't be spoofed by spctl's own prose.
2402	// The full assess output is still streamed to the step log.
2403	let q = ops_core::remote::sh_quote(path);
2404	let cmd = format!(
2405	"out=$(spctl --assess -vv --type install {q} 2>&1); printf '%s\\n' \"$out\"; \
2406	printf '%s' \"$out\" \| grep -q 'source=Notarized Developer ID' \
2407	&& echo BENTO_GATEKEEPER_OK \|\| echo BENTO_GATEKEEPER_FAIL",
2408	);
2409	let (_, tail) = ctx.run(host, &cmd).map_err(rhai_err)?;
2410	let accepted = tail.contains("BENTO_GATEKEEPER_OK");
2411	// Record the verdict for the publish gate. A rejection also
2412	// fails the step, so the matrix shows red and `publish` is barred
2413	// even if the recipe ignores the returned bool.
2414	*ctx.gatekeeper_ok.lock().unwrap() = Some(accepted);
2415	if !accepted {
2416	ctx.fail_current_step();
2417	}
2418	Ok(accepted)
2419	},
2420	);
2421	}
2422	{
2423	let ctx = ctx.clone();
2424	engine.register_fn(
2425	"codesign",
2426	move \|host: &str, identity: &str, path: &str\| -> Result<(), Box<EvalAltResult>> {
2427	let cmd = format!(
2428	"codesign --force --options runtime --timestamp --sign {} {}",
2429	ops_core::remote::sh_quote(identity),
2430	ops_core::remote::sh_quote(path),
2431	);
2432	let (code, _) = ctx.run(host, &cmd).map_err(rhai_err)?;
2433	if code != 0 {
2434	return Err(rhai_err("codesign failed"));
2435	}
2436	Ok(())
2437	},
2438	);
2439	}
2440	{
2441	let ctx = ctx.clone();
2442	engine.register_fn(
2443	"staple",
2444	move \|host: &str, path: &str\| -> Result<(), Box<EvalAltResult>> {
2445	let (code, _) = ctx
2446	.run(
2447	host,
2448	&format!("xcrun stapler staple {}", ops_core::remote::sh_quote(path)),
2449	)
2450	.map_err(rhai_err)?;
2451	if code != 0 {
2452	return Err(rhai_err("stapler failed"));
2453	}
2454	Ok(())
2455	},
2456	);
2457	}
2458	{
2459	let ctx = ctx.clone();
2460	engine.register_fn(
2461	"notarize",
2462	move \|host: &str, path: &str\| -> Result<String, Box<EvalAltResult>> {
2463	ctx.notarize(host, path).map_err(rhai_err)
2464	},
2465	);
2466	}
2467	{
2468	let ctx = ctx.clone();
2469	engine.register_fn(
2470	"keychain_open",
2471	move \|host: &str, name: &str\| -> Result<(), Box<EvalAltResult>> {
2472	// The full build-keychain lifecycle lives in dist/build-keychain.sh
2473	// (design §7); this drives it by name so the recipe stays short.
2474	let (code, _) = ctx
2475	.run(
2476	host,
2477	&format!(
2478	". ~/.tauri/passwords.env && ./dist/build-keychain.sh open {}",
2479	ops_core::remote::sh_quote(name)
2480	),
2481	)
2482	.map_err(rhai_err)?;
2483	if code != 0 {
2484	return Err(rhai_err("keychain_open failed"));
2485	}
2486	Ok(())
2487	},
2488	);
2489	}
2490	{
2491	let ctx = ctx.clone();
2492	engine.register_fn(
2493	"keychain_close",
2494	move \|host: &str, name: &str\| -> Result<(), Box<EvalAltResult>> {
2495	let _ = ctx.run(
2496	host,
2497	&format!(
2498	"./dist/build-keychain.sh close {}",
2499	ops_core::remote::sh_quote(name)
2500	),
2501	);
2502	Ok(())
2503	},
2504	);
2505	}
2506	}
2507
2508	impl RecipeCtx {
2509	/// `xcrun notarytool submit --wait` with bounded retry (the one flaky,
2510	/// network-bound step). Emits `NotarizeRetry` per attempt.
2511	fn notarize(self: &Arc<Self>, host: &str, path: &str) -> Result<String> {
2512	const MAX_ATTEMPTS: u32 = 3;
2513	let backoff = self
2514	.cfg
2515	.notarize_backoff_secs
2516	.map_or(std::time::Duration::from_secs(15), \|s\| {
2517	std::time::Duration::from_secs(s)
2518	});
2519	let cmd = format!(
2520	". ~/.tauri/passwords.env && xcrun notarytool submit {} \
2521	--key \"$NOTARY_KEY\" --key-id \"$NOTARY_KEY_ID\" --issuer \"$NOTARY_ISSUER\" \
2522	--wait --output-format json",
2523	ops_core::remote::sh_quote(path),
2524	);
2525	let mut last = String::new();
2526	for attempt in 1..=MAX_ATTEMPTS {
2527	let (code, tail) = self.run(host, &cmd)?;
2528	if code == 0 && notary_accepted(&tail) {
2529	return Ok(tail);
2530	}
2531	last = tail;
2532	if attempt < MAX_ATTEMPTS {
2533	events::emit(
2534	&self.events,
2535	Event::NotarizeRetry {
2536	app: self.app.clone(),
2537	target: self.target,
2538	attempt,
2539	reason: format!("exit {code}"),
2540	},
2541	);
2542	self.rt.block_on(tokio::time::sleep(backoff));
2543	}
2544	}
2545	anyhow::bail!("notarization failed after {MAX_ATTEMPTS} attempts: {last}")
2546	}
2547	}
2548
2549	/// True iff `notarytool --output-format json` output reports `status: Accepted`.
2550	/// Isolates the JSON object (`{`..`}`) from any shell-sourcing noise and reads
2551	/// the typed `status` field, rather than substring-matching `"status":"Accepted"`
2552	/// in a possibly-truncated tail — which could match the literal inside an error
2553	/// message or miss it across a whitespace variant. Fails closed: any parse or
2554	/// field miss returns false.
2555	fn notary_accepted(output: &str) -> bool {
2556	let (Some(start), Some(end)) = (output.find('{'), output.rfind('}')) else {
2557	return false;
2558	};
2559	if start > end {
2560	return false;
2561	}
2562	serde_json::from_str::<serde_json::Value>(&output[start..=end])
2563	.ok()
2564	.and_then(\|v\| {
2565	v.get("status")
2566	.and_then(\|s\| s.as_str())
2567	.map(\|s\| s.eq_ignore_ascii_case("accepted"))
2568	})
2569	.unwrap_or(false)
2570	}
2571
2572	#[cfg(test)]
2573	mod tests {
2574	use super::*;
2575
2576	/// Build the shared cross-crate bundle fixture under `root`.
2577	///
2578	/// A binary at the top and two files in a subdirectory — the shape a service
2579	/// that ships its migrations has, which is the case the flat walk used to
2580	/// get wrong.
2581	pub(crate) fn write_bundle_fixture(root: &Path) {
2582	std::fs::create_dir_all(root.join("migrations")).unwrap();
2583	std::fs::write(root.join("pom"), b"binary-bytes").unwrap();
2584	std::fs::write(root.join("migrations/001_init.sql"), b"create table a;").unwrap();
2585	std::fs::write(root.join("migrations/002_next.sql"), b"alter table a;").unwrap();
2586	}
2587
2588	/// The manifest text the fixture must produce, in BOTH crates.
2589	///
2590	/// Sando's `bundle::digest_dir` has the identical constant and the identical
2591	/// fixture. That is the whole point: bento writes this text into the artifact
2592	/// record, sando recomputes it from the bytes that arrive, and an artifact is
2593	/// refused when they differ. Two walks, one answer, pinned from both ends —
2594	/// if either crate's walk drifts, its own test fails and names the drift
2595	/// rather than a release failing intake for a bundle nothing is wrong with.
2596	pub(crate) const BUNDLE_FIXTURE_MANIFEST: &str = concat!(
2597	"e4c908e219c533fa7ad7ea1634398f9bf51637ba20717769ada545bab26d7368 migrations/001_init.sql\n",
2598	"b026fd51bae096b34672cefdb781b6585b13efb53bc301d50c305f422552a380 migrations/002_next.sql\n",
2599	"71227a7f160afca3fb3c39f448735886dda7bd366252580c2222fb87d4bb4d85 pom\n",
2600	);
2601
2602	/// The producer half of the contract above: what `collect` hashes, turned
2603	/// into a manifest, is exactly the text the verifier will recompute.
2604	///
2605	/// Nested files are included and addressed by relative path. Before this,
2606	/// `collect` listed only the top level, so `migrations/` contributed nothing
2607	/// to the manifest while sando's walker hashed both files in it — and the
2608	/// honest bundle was refused for a manifest that had omitted them.
2609	#[test]
2610	fn a_collected_bundle_manifests_exactly_as_the_verifier_will_read_it() {
2611	let dir = tempfile::tempdir().unwrap();
2612	write_bundle_fixture(dir.path());
2613
2614	let files = collected_files(dir.path()).unwrap();
2615	assert_eq!(
2616	files.iter().map(\|(r, _)\| r.as_str()).collect::<Vec<_>>(),
2617	vec!["migrations/001_init.sql", "migrations/002_next.sql", "pom"],
2618	"recursive, relative, sorted"
2619	);
2620
2621	let hashes: Vec<(String, String)> = files
2622	.into_iter()
2623	.map(\|(rel, path)\| (rel, sha256_file(&path).unwrap()))
2624	.collect();
2625	let manifest = ops_artifact::Manifest::new(hashes).unwrap();
2626	assert_eq!(manifest.to_text(), BUNDLE_FIXTURE_MANIFEST);
2627	}
2628
2629	/// A symlink is neither followed nor named. Following one would let bytes
2630	/// from outside the bundle into its identity; naming it would put a path in
2631	/// the manifest the verifier does not hash, which reads as a corrupt bundle.
2632	#[test]
2633	#[cfg(unix)]
2634	fn a_symlink_in_the_collect_dir_is_not_part_of_the_bundle() {
2635	let dir = tempfile::tempdir().unwrap();
2636	write_bundle_fixture(dir.path());
2637	let outside = dir.path().join("..").join("secret.env");
2638	std::fs::write(&outside, b"TOKEN=1").ok();
2639	std::os::unix::fs::symlink(&outside, dir.path().join("link.env")).unwrap();
2640
2641	let files = collected_files(dir.path()).unwrap();
2642	assert!(
2643	!files.iter().any(\|(rel, _)\| rel.contains("link.env")),
2644	"{files:?}"
2645	);
2646	}
2647
2648	/// Run 3 S1: once the cooperative cancel flag is set (a newer build
2649	/// superseded this run), a step boundary refuses to proceed — the blocking
2650	/// recipe stops at the next `step()` instead of running on and publishing.
2651	#[tokio::test]
2652	async fn begin_step_bails_when_cancelled() {
2653	let dir = tempfile::tempdir().unwrap();
2654	let cfg = Arc::new(Config::for_tests(dir.path()));
2655	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2656	let cancel = Arc::new(AtomicBool::new(true));
2657	let ctx = Arc::new(RecipeCtx::new(
2658	AppId::new("demo"),
2659	Version::parse("0.1.0").unwrap(),
2660	"linux/x86_64".parse().unwrap(),
2661	"fw13".into(),
2662	"local".into(),
2663	"v0.1.0".into(),
2664	"/tmp".into(),
2665	vec![],
2666	Kind::App,
2667	1,
2668	Arc::new(std::collections::HashMap::new()),
2669	Arc::new(std::collections::HashMap::new()),
2670	None,
2671	pool,
2672	crate::events::channel(),
2673	cfg,
2674	Arc::new(OtaRegistry::standard("https://makenot.work")),
2675	tokio::runtime::Handle::current(),
2676	cancel.clone(),
2677	None,
2678	));
2679	// Cancelled: begin_step refuses before touching the DB (the ensure! is
2680	// ahead of any block_on, so this is safe to call from the async test).
2681	let err = ctx.begin_step(Step::Build).unwrap_err();
2682	assert!(err.to_string().contains("supersede"), "got: {err}");
2683	assert!(ctx.is_cancelled());
2684	}
2685
2686	/// `feature_flags()` returns a whole flag or nothing at all. An app with
2687	/// no declared features must not yield a bare `--features`, which would
2688	/// swallow the next word of the build command as its argument.
2689	#[tokio::test]
2690	async fn feature_flags_renders_whole_flag_or_empty() {
2691	async fn flags_for(features: Vec<String>) -> String {
2692	let dir = tempfile::tempdir().unwrap();
2693	let cfg = Arc::new(Config::for_tests(dir.path()));
2694	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2695	let ctx = Arc::new(RecipeCtx::new(
2696	AppId::new("demo"),
2697	Version::parse("0.1.0").unwrap(),
2698	"linux/x86_64".parse().unwrap(),
2699	"fw13".into(),
2700	"local".into(),
2701	"v0.1.0".into(),
2702	"/tmp".into(),
2703	features,
2704	Kind::App,
2705	1,
2706	Arc::new(std::collections::HashMap::new()),
2707	Arc::new(std::collections::HashMap::new()),
2708	None,
2709	pool,
2710	crate::events::channel(),
2711	cfg,
2712	Arc::new(OtaRegistry::standard("https://makenot.work")),
2713	tokio::runtime::Handle::current(),
2714	Arc::new(AtomicBool::new(false)),
2715	None,
2716	));
2717	let engine = build_engine(&ctx);
2718	engine.eval::<String>("feature_flags()").unwrap()
2719	}
2720
2721	assert_eq!(flags_for(vec![]).await, "");
2722	assert_eq!(
2723	flags_for(vec!["supernote".into()]).await,
2724	"--features supernote"
2725	);
2726	assert_eq!(
2727	flags_for(vec!["supernote".into(), "extra".into()]).await,
2728	"--features supernote,extra"
2729	);
2730	}
2731
2732	/// `repo()` answers for the host this target builds on, not for the daemon.
2733	///
2734	/// This is what lets a Windows recipe call `repo()` and `checkout_sha(h)`
2735	/// instead of hard-coding `C:/Users/me/...` — and hard-coding it is what
2736	/// kept those recipes off the release-tag pin, since `checkout_sha` builds
2737	/// its git commands from the app's path and takes no override.
2738	#[tokio::test]
2739	async fn repo_resolves_per_build_host() {
2740	async fn repo_on(build_host: &str) -> String {
2741	let dir = tempfile::tempdir().unwrap();
2742	let cfg = Arc::new(Config::for_tests(dir.path()));
2743	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2744	let ctx = Arc::new(
2745	RecipeCtx::new(
2746	AppId::new("demo"),
2747	Version::parse("0.1.0").unwrap(),
2748	"linux/x86_64".parse().unwrap(),
2749	build_host.into(),
2750	"local".into(),
2751	"v0.1.0".into(),
2752	"~/Code/Apps/demo".into(),
2753	vec![],
2754	Kind::App,
2755	1,
2756	Arc::new(std::collections::HashMap::new()),
2757	Arc::new(std::collections::HashMap::new()),
2758	None,
2759	pool,
2760	crate::events::channel(),
2761	cfg,
2762	Arc::new(OtaRegistry::standard("https://makenot.work")),
2763	tokio::runtime::Handle::current(),
2764	Arc::new(AtomicBool::new(false)),
2765	None,
2766	)
2767	.with_repo_by_host(HashMap::from([(
2768	"windows-x86".to_string(),
2769	"C:/Users/me/Code/Apps/demo".to_string(),
2770	)])),
2771	);
2772	build_engine(&ctx).eval::<String>("repo()").unwrap()
2773	}
2774
2775	assert_eq!(repo_on("windows-x86").await, "C:/Users/me/Code/Apps/demo");
2776	assert_eq!(repo_on("fw13").await, "~/Code/Apps/demo");
2777	}
2778
2779	/// `secret(key)` reads a file under `secrets_root`, trims its trailing
2780	/// newline (the shape of a here-doc'd token file), and refuses any key that
2781	/// could escape the root. Covers the host-fn registered in `build_engine`.
2782	#[tokio::test]
2783	async fn secret_reads_under_root_and_blocks_traversal() {
2784	let dir = tempfile::tempdir().unwrap();
2785	let cfg = Config::for_tests(dir.path());
2786	// Seed a secret and one in a nested subdir; a trailing newline that the
2787	// read must strip.
2788	std::fs::create_dir_all(&cfg.secrets_root).unwrap();
2789	std::fs::write(cfg.secrets_root.join("token"), "s3cr3t\n").unwrap();
2790	std::fs::create_dir_all(cfg.secrets_root.join("app")).unwrap();
2791	std::fs::write(cfg.secrets_root.join("app").join("key"), "nested").unwrap();
2792	// Plant a file OUTSIDE the root that a traversal key would reach.
2793	std::fs::write(dir.path().join("outside"), "leak").unwrap();
2794
2795	let cfg = Arc::new(cfg);
2796	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2797	let ctx = Arc::new(RecipeCtx::new(
2798	AppId::new("demo"),
2799	Version::parse("0.1.0").unwrap(),
2800	"linux/x86_64".parse().unwrap(),
2801	"fw13".into(),
2802	"local".into(),
2803	"v0.1.0".into(),
2804	"/tmp".into(),
2805	vec![],
2806	Kind::App,
2807	1,
2808	Arc::new(std::collections::HashMap::new()),
2809	Arc::new(std::collections::HashMap::new()),
2810	None,
2811	pool,
2812	crate::events::channel(),
2813	cfg,
2814	Arc::new(OtaRegistry::standard("https://makenot.work")),
2815	tokio::runtime::Handle::current(),
2816	Arc::new(AtomicBool::new(false)),
2817	None,
2818	));
2819	let engine = build_engine(&ctx);
2820
2821	// Happy path: read + trim.
2822	assert_eq!(
2823	engine.eval::<String>(r#"secret("token")"#).unwrap(),
2824	"s3cr3t"
2825	);
2826	// A multi-segment relative key is allowed.
2827	assert_eq!(
2828	engine.eval::<String>(r#"secret("app/key")"#).unwrap(),
2829	"nested"
2830	);
2831
2832	// Traversal, absolute paths, and empty keys are refused BEFORE any read,
2833	// so the file one `..` above the root is never disclosed.
2834	for bad in [
2835	r#"secret("../outside")"#,
2836	r#"secret("/etc/passwd")"#,
2837	r#"secret("")"#,
2838	] {
2839	let err = engine.eval::<String>(bad).unwrap_err().to_string();
2840	assert!(
2841	err.contains("relative path under secrets_root"),
2842	"`{bad}` should hit the traversal guard, got: {err}"
2843	);
2844	}
2845	// A missing key surfaces the filesystem error, not a panic, and does not
2846	// trip the traversal guard (it is a legitimate relative path).
2847	let err = engine
2848	.eval::<String>(r#"secret("nope")"#)
2849	.unwrap_err()
2850	.to_string();
2851	assert!(err.contains("secret `nope`"), "got: {err}");
2852	}
2853
2854	/// The two failures that actually shipped, as regression cases.
2855	#[test]
2856	fn preflight_catches_a_dead_repository_url() {
2857	// pter 0.1.0: repository pointed at a URL that does not exist. It
2858	// published clean and the link is now permanent for that version.
2859	let meta = CrateMeta {
2860	name: "pter".into(),
2861	version: "0.1.0".into(),
2862	repository: Some("https://github.com/maxjacobson/pter".into()),
2863	description: Some("d".into()),
2864	licensed: true,
2865	};
2866	let problems = crate_publish_problems(&meta, false, &[], true);
2867	assert_eq!(problems.len(), 1, "{problems:?}");
2868	assert!(
2869	problems[0].contains("not publicly clonable"),
2870	"{problems:?}"
2871	);
2872
2873	// Same metadata, reachable URL: nothing to report.
2874	assert!(crate_publish_problems(&meta, true, &[], true).is_empty());
2875	}
2876
2877	#[test]
2878	fn preflight_requires_the_fields_crates_io_bakes_in() {
2879	let bare = CrateMeta {
2880	name: "x".into(),
2881	version: "0.1.0".into(),
2882	repository: None,
2883	description: None,
2884	licensed: false,
2885	};
2886	let problems = crate_publish_problems(&bare, false, &[], true);
2887	assert_eq!(problems.len(), 3, "{problems:?}");
2888	assert!(problems.iter().any(\|p\| p.contains("repository")));
2889	assert!(problems.iter().any(\|p\| p.contains("description")));
2890	assert!(problems.iter().any(\|p\| p.contains("license")));
2891	}
2892
2893	// A library's verify is a crate preflight, not a Gatekeeper check on a
2894	// signed bundle. Gating it on `gatekeeper` asked a Linux host for a macOS
2895	// code-signing capability it can never hold, so the step was denied before
2896	// it ran a command; the denial then surfaced as "no crates.io credentials",
2897	// which is not what went wrong. The only way to satisfy the old gate was to
2898	// declare the capability falsely in the topology.
2899	#[test]
2900	fn a_library_verify_is_not_gated_on_gatekeeper() {
2901	assert_eq!(
2902	action_for(Step::Verify, Kind::Library),
2903	Action::Build,
2904	"a crate preflight runs the build toolchain; that is what it needs",
2905	);
2906	assert_eq!(
2907	action_for(Step::Verify, Kind::App),
2908	Action::Observe(ObserveKind::Custom("gatekeeper".into())),
2909	"an app's verify still proves the bundle is signed and notarized",
2910	);
2911	}
2912
2913	// The capability the default host grant actually carries. Without this the
2914	// fix above is only true by inspection.
2915	#[test]
2916	fn a_default_host_can_run_a_library_verify_and_not_an_app_one() {
2917	let caps =
2918	ops_exec::CapabilitySet::from_tokens(["build", "package"], ["build-log", "artifact"]);
2919	assert!(caps.permits(&action_for(Step::Verify, Kind::Library)));
2920	assert!(!caps.permits(&action_for(Step::Verify, Kind::App)));
2921	}
2922
2923	// Every other step is a property of the step alone; verify is the one that
2924	// depends on what is being released.
2925	#[test]
2926	fn no_other_step_changes_with_the_kind() {
2927	for step in [
2928	Step::Checkout,
2929	Step::Prebuild,
2930	Step::Build,
2931	Step::Sign,
2932	Step::Notarize,
2933	Step::Staple,
2934	Step::Package,
2935	Step::Publish,
2936	Step::Collect,
2937	] {
2938	assert_eq!(
2939	action_for(step, Kind::App),
2940	action_for(step, Kind::Library),
2941	"{step:?} should not depend on the kind",
2942	);
2943	}
2944	}
2945
2946	#[test]
2947	fn preflight_rejects_republishing_the_same_version() {
2948	let meta = CrateMeta {
2949	name: "makeover".into(),
2950	version: "0.10.0".into(),
2951	repository: Some("https://git.sr.ht/~maxmj/makeover".into()),
2952	description: Some("d".into()),
2953	licensed: true,
2954	};
2955	let problems =
2956	crate_publish_problems(&meta, true, &["0.9.0".into(), "0.10.0".into()], true);
2957	assert_eq!(problems.len(), 1, "{problems:?}");
2958	assert!(problems[0].contains("already published"), "{problems:?}");
2959
2960	// An unreleased version against the same history is fine.
2961	let mut next = meta.clone();
2962	next.version = "0.11.0".into();
2963	assert!(crate_publish_problems(&next, true, &["0.10.0".into()], true).is_empty());
2964	}
2965
2966	/// Missing credentials must surface at preflight, not at the upload. The
2967	/// publish step is the irreversible one and runs last, after a full build
2968	/// and verify; discovering there that cargo cannot authenticate wastes the
2969	/// whole run.
2970	#[test]
2971	fn preflight_reports_missing_credentials_up_front() {
2972	let meta = CrateMeta {
2973	name: "makeover".into(),
2974	version: "0.11.0".into(),
2975	repository: Some("https://git.sr.ht/~maxmj/makeover".into()),
2976	description: Some("d".into()),
2977	licensed: true,
2978	};
2979	// Metadata is perfect; only the token is absent.
2980	let problems = crate_publish_problems(&meta, true, &[], false);
2981	assert_eq!(problems.len(), 1, "{problems:?}");
2982	assert!(problems[0].contains("credentials"), "{problems:?}");
2983	assert!(
2984	problems[0].contains("cargo login"),
2985	"should say how to fix it"
2986	);
2987
2988	// Present: nothing to report.
2989	assert!(crate_publish_problems(&meta, true, &[], true).is_empty());
2990	}
2991
2992	#[test]
2993	fn crate_meta_reads_cargo_metadata_json() {
2994	let raw = r#"{"packages":[{"name":"makeover","version":"0.10.0",
2995	"repository":"https://git.sr.ht/~maxmj/makeover","description":"themes",
2996	"license":"MIT"}]}"#;
2997	let m = crate_meta_from_json(raw).unwrap();
2998	assert_eq!(m.name, "makeover");
2999	assert_eq!(m.version, "0.10.0");
3000	assert!(m.licensed);
3001	assert_eq!(
3002	m.repository.as_deref(),
3003	Some("https://git.sr.ht/~maxmj/makeover")
3004	);
3005
3006	// license_file alone also counts as licensed; empty strings do not
3007	// count as present.
3008	let lf = r#"{"packages":[{"name":"x","version":"0.1.0","license":"",
3009	"license_file":"LICENSE","description":""}]}"#;
3010	let m = crate_meta_from_json(lf).unwrap();
3011	assert!(m.licensed);
3012	assert!(m.description.is_none());
3013	}
3014
3015	/// Reads the ambient `HOME` rather than setting one. `set_var` is
3016	/// process-global and unsynchronized, so a test that overwrote HOME changed
3017	/// it for every other test in the binary — which is what silently disabled
3018	/// `topology::live_config_smoke` (it skips when `$HOME/.config/bento` is
3019	/// absent, and `/home/test` always is).
3020	#[test]
3021	fn expand_tilde_handles_home() {
3022	let home = PathBuf::from(std::env::var("HOME").expect("HOME is set"));
3023	assert_eq!(expand_tilde("~/Code/x"), home.join("Code/x"));
3024	assert_eq!(expand_tilde("/abs/path"), PathBuf::from("/abs/path"));
3025	}
3026
3027	// ---- artifact resolution (the M3 silent-`sh` fix) ----
3028
3029	#[test]
3030	fn resolve_artifact_match_wants_exactly_one() {
3031	// Exactly one match: the path, trimmed of the listing's line noise.
3032	assert_eq!(
3033	resolve_artifact_match(" /d/App.AppImage \n", "*.AppImage", true).unwrap(),
3034	"/d/App.AppImage"
3035	);
3036	}
3037
3038	#[test]
3039	fn resolve_artifact_match_zero_depends_on_required() {
3040	// Required + zero matches is the case the old empty-string guard caught;
3041	// keep failing it.
3042	let err = resolve_artifact_match("", "*.dmg", true).unwrap_err();
3043	assert!(err.to_string().contains("no artifact matched"), "{err}");
3044	// Optional + zero matches resolves to empty (recipe skips the collect).
3045	assert_eq!(
3046	resolve_artifact_match("\n \n", "*.deb", false).unwrap(),
3047	""
3048	);
3049	}
3050
3051	#[test]
3052	fn resolve_artifact_match_rejects_ambiguous() {
3053	// Two matches must throw rather than silently pick one — this is the
3054	// stale-newest-mtime hole the audit flagged. Applies even when optional.
3055	for required in [true, false] {
3056	let err =
3057	resolve_artifact_match("/d/old.deb\n/d/new.deb\n", "*.deb", required).unwrap_err();
3058	let msg = err.to_string();
3059	assert!(msg.contains("ambiguous"), "{msg}");
3060	assert!(msg.contains("old.deb") && msg.contains("new.deb"), "{msg}");
3061	}
3062	}
3063
3064	#[test]
3065	fn ensure_glob_safe_allows_paths_bars_commands() {
3066	// Path and wildcard characters pass.
3067	assert!(ensure_glob_safe("~/Code/app/dist/*.AppImage").is_ok());
3068	assert!(ensure_glob_safe("/t/App_1.2.3-x86_64.dmg").is_ok());
3069	// A command substitution or separator does not.
3070	for bad in ["*.dmg; rm -rf /", "$(evil)", "a\|b", "a b"] {
3071	assert!(ensure_glob_safe(bad).is_err(), "should reject {bad:?}");
3072	}
3073	}
3074
3075	// ---- version resolution ----
3076
3077	#[test]
3078	fn version_from_tauri_json_reads_version() {
3079	assert_eq!(
3080	version_from_tauri_json(r#"{"version":"0.4.2"}"#).unwrap(),
3081	"0.4.2"
3082	);
3083	assert!(version_from_tauri_json(r#"{"productName":"X"}"#).is_err());
3084	}
3085
3086	#[test]
3087	fn version_from_cargo_toml_prefers_package_then_workspace() {
3088	// A leaf crate's [package].version.
3089	assert_eq!(
3090	version_from_cargo_toml("[package]\nname = \"x\"\nversion = \"0.5.0\"\n").unwrap(),
3091	"0.5.0"
3092	);
3093	// A workspace that sets [workspace.package].version.
3094	assert_eq!(
3095	version_from_cargo_toml("[workspace.package]\nversion = \"1.2.3\"\n").unwrap(),
3096	"1.2.3"
3097	);
3098	// No version anywhere -> error, not a panic.
3099	assert!(version_from_cargo_toml("[workspace]\nmembers = []\n").is_err());
3100	}
3101
3102	#[test]
3103	fn version_from_repo_default_and_explicit_paths() {
3104	let tmp = tempfile::tempdir().unwrap();
3105	let root = tmp.path();
3106
3107	// Tauri app: default path reads src-tauri/tauri.conf.json.
3108	let tauri = root.join("tauri");
3109	std::fs::create_dir_all(tauri.join("src-tauri")).unwrap();
3110	std::fs::write(
3111	tauri.join("src-tauri/tauri.conf.json"),
3112	r#"{"version":"0.4.2"}"#,
3113	)
3114	.unwrap();
3115	assert_eq!(
3116	version_from_repo(tauri.to_str().unwrap(), None)
3117	.unwrap()
3118	.to_string(),
3119	"0.4.2"
3120	);
3121
3122	// Workspace egui app: no tauri.conf.json, explicit version_path at a member crate.
3123	let ws = root.join("ws");
3124	std::fs::create_dir_all(ws.join("crates/app")).unwrap();
3125	std::fs::write(
3126	ws.join("Cargo.toml"),
3127	"[workspace]\nmembers = [\"crates/app\"]\n",
3128	)
3129	.unwrap();
3130	std::fs::write(
3131	ws.join("crates/app/Cargo.toml"),
3132	"[package]\nname = \"app\"\nversion = \"0.5.0\"\n",
3133	)
3134	.unwrap();
3135	assert_eq!(
3136	version_from_repo(ws.to_str().unwrap(), Some("crates/app/Cargo.toml"))
3137	.unwrap()
3138	.to_string(),
3139	"0.5.0"
3140	);
3141	}
3142
3143	// ---- version-source cross-check (drift preflight) ----
3144
3145	fn ver(s: &str) -> Version {
3146	Version::parse(s).unwrap()
3147	}
3148
3149	#[test]
3150	fn version_consistency_passes_when_all_sources_agree() {
3151	let tmp = tempfile::tempdir().unwrap();
3152	let repo = tmp.path();
3153	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3154	std::fs::write(
3155	repo.join("src-tauri/tauri.conf.json"),
3156	r#"{"version":"0.5.0"}"#,
3157	)
3158	.unwrap();
3159	std::fs::write(
3160	repo.join("Cargo.toml"),
3161	"[package]\nname = \"app\"\nversion = \"0.5.0\"\n",
3162	)
3163	.unwrap();
3164	check_version_consistency(repo.to_str().unwrap(), None, &ver("0.5.0")).unwrap();
3165	}
3166
3167	#[test]
3168	fn version_consistency_flags_tauri_vs_cargo_drift() {
3169	// The concrete finding: tauri.conf.json bumped to 0.5.0 but the root
3170	// Cargo.toml left at 0.4.0. version_from_repo (one file) would miss it.
3171	let tmp = tempfile::tempdir().unwrap();
3172	let repo = tmp.path();
3173	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3174	std::fs::write(
3175	repo.join("src-tauri/tauri.conf.json"),
3176	r#"{"version":"0.5.0"}"#,
3177	)
3178	.unwrap();
3179	std::fs::write(
3180	repo.join("Cargo.toml"),
3181	"[package]\nname = \"app\"\nversion = \"0.4.0\"\n",
3182	)
3183	.unwrap();
3184	let err =
3185	check_version_consistency(repo.to_str().unwrap(), None, &ver("0.5.0")).unwrap_err();
3186	let msg = format!("{err:#}");
3187	assert!(msg.contains("Cargo.toml says 0.4.0"), "{msg}");
3188	}
3189
3190	#[test]
3191	fn version_consistency_flags_explicit_version_the_repo_does_not_reflect() {
3192	let tmp = tempfile::tempdir().unwrap();
3193	let repo = tmp.path();
3194	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3195	std::fs::write(
3196	repo.join("src-tauri/tauri.conf.json"),
3197	r#"{"version":"0.5.0"}"#,
3198	)
3199	.unwrap();
3200	let err =
3201	check_version_consistency(repo.to_str().unwrap(), None, &ver("9.9.9")).unwrap_err();
3202	assert!(format!("{err:#}").contains("building 9.9.9"));
3203	}
3204
3205	#[test]
3206	fn version_consistency_single_source_never_invents_drift() {
3207	// A virtual-workspace root Cargo.toml (no version) alongside the member
3208	// crate the version_path points at: only one real source, so no drift.
3209	let tmp = tempfile::tempdir().unwrap();
3210	let repo = tmp.path();
3211	std::fs::create_dir_all(repo.join("crates/app")).unwrap();
3212	std::fs::write(
3213	repo.join("Cargo.toml"),
3214	"[workspace]\nmembers = [\"crates/app\"]\n",
3215	)
3216	.unwrap();
3217	std::fs::write(
3218	repo.join("crates/app/Cargo.toml"),
3219	"[package]\nname = \"app\"\nversion = \"0.5.0\"\n",
3220	)
3221	.unwrap();
3222	check_version_consistency(
3223	repo.to_str().unwrap(),
3224	Some("crates/app/Cargo.toml"),
3225	&ver("0.5.0"),
3226	)
3227	.unwrap();
3228	}
3229
3230	// ---- artifact filename version assertion + hashing ----
3231
3232	#[test]
3233	fn versions_in_filename_extracts_only_real_semvers() {
3234	assert_eq!(
3235	versions_in_filename("GoingsOn_0.5.0_aarch64.dmg"),
3236	vec![ver("0.5.0")]
3237	);
3238	assert_eq!(
3239	versions_in_filename("AudioFiles-0.4.0-x86_64.AppImage"),
3240	vec![ver("0.4.0")]
3241	);
3242	// No three-part token ⇒ nothing (an updater manifest, a bare signature).
3243	assert!(versions_in_filename("latest.json").is_empty());
3244	assert!(versions_in_filename("app.sig").is_empty());
3245	}
3246
3247	#[test]
3248	fn assert_artifact_version_rejects_a_stale_artifact() {
3249	// The 0.4.0 file sitting in the output dir against a 0.5.0 build.
3250	let err =
3251	assert_artifact_version("AudioFiles-0.4.0-x86_64.AppImage", &ver("0.5.0")).unwrap_err();
3252	assert!(format!("{err:#}").contains("stale artifact"), "{err:#}");
3253	// The matching version passes, and a versionless file is not asserted.
3254	assert_artifact_version("GoingsOn_0.5.0_aarch64.dmg", &ver("0.5.0")).unwrap();
3255	assert_artifact_version("latest.json", &ver("0.5.0")).unwrap();
3256	}
3257
3258	/// The comparison that decides whether a binary can exec on the box that is
3259	/// about to be restarted onto it. Both sides are parsed out of text a tool
3260	/// printed, so both parsers are worth pinning: fw13 tracks a newer glibc
3261	/// than the Ubuntu 24.04 host in Hetzner, and getting this backwards means a
3262	/// dead unit rather than a failed step.
3263	#[test]
3264	fn glibc_versions_parse_from_what_the_tools_actually_print() {
3265	// `objdump -T \| grep -o 'GLIBC_[0-9.]*'` output: highest wins, and the
3266	// comparison is numeric (2.9 must not beat 2.34 lexically).
3267	let objdump = "GLIBC_2.2.5\nGLIBC_2.34\nGLIBC_2.9\nGLIBC_2.17\n";
3268	assert_eq!(max_glibc_symbol(objdump), Some((2, 34)));
3269	// A static binary references none: nothing to check.
3270	assert_eq!(max_glibc_symbol(""), None);
3271
3272	// `ldd --version` first line, however the distro decorates it.
3273	assert_eq!(
3274	glibc_from_ldd("ldd (Ubuntu GLIBC 2.39-0ubuntu8.8) 2.39\nCopyright...\n"),
3275	Some((2, 39))
3276	);
3277	assert_eq!(
3278	glibc_from_ldd("ldd (GNU libc) 2.41\nCopyright (C) 2025\n"),
3279	Some((2, 41))
3280	);
3281	assert_eq!(glibc_from_ldd(""), None);
3282	}
3283
3284	/// A binary needing MORE than the host has is the failure this check exists
3285	/// for; equal and less are both fine (glibc symbol versioning is backward
3286	/// compatible, so an older requirement runs on a newer host).
3287	#[test]
3288	fn glibc_requirement_is_satisfied_by_equal_or_newer_only() {
3289	let needs = max_glibc_symbol("GLIBC_2.41").unwrap();
3290	assert!(needs > glibc_from_ldd("ldd (Ubuntu GLIBC 2.39) 2.39").unwrap());
3291	assert!(needs <= glibc_from_ldd("ldd (GNU libc) 2.41").unwrap());
3292	assert!(needs <= glibc_from_ldd("ldd (GNU libc) 2.42").unwrap());
3293	assert!(needs <= glibc_from_ldd("ldd (GNU libc) 3.0").unwrap());
3294	}
3295
3296	/// Every deploy host function fails with the app's KIND as the reason when
3297	/// there is no destination, rather than with a missing-host error from
3298	/// somewhere deeper. A recipe calling `deploy()` on a library is a recipe
3299	/// written against the wrong kind, and the message should say so.
3300	#[tokio::test]
3301	async fn deploy_host_fns_explain_a_missing_destination_by_kind() {
3302	let dir = tempfile::tempdir().unwrap();
3303	let cfg = Arc::new(Config::for_tests(dir.path()));
3304	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3305	let ctx = Arc::new(RecipeCtx::new(
3306	AppId::new("demo"),
3307	Version::parse("0.1.0").unwrap(),
3308	"linux/x86_64".parse().unwrap(),
3309	"fw13".into(),
3310	"local".into(),
3311	"v0.1.0".into(),
3312	"/tmp".into(),
3313	vec![],
3314	Kind::Library,
3315	1,
3316	Arc::new(std::collections::HashMap::new()),
3317	Arc::new(std::collections::HashMap::new()),
3318	None,
3319	pool,
3320	crate::events::channel(),
3321	cfg,
3322	Arc::new(OtaRegistry::standard("https://makenot.work")),
3323	tokio::runtime::Handle::current(),
3324	Arc::new(AtomicBool::new(false)),
3325	None,
3326	));
3327	let engine = build_engine(&ctx);
3328	for call in [
3329	"deploy_host()",
3330	"service_name()",
3331	"install_path()",
3332	"health_url()",
3333	r#"deploy("/tmp/x")"#,
3334	] {
3335	let err = engine.eval::<String>(call).unwrap_err().to_string();
3336	assert!(
3337	err.contains("library") && err.contains("no deploy destination"),
3338	"`{call}` must fail on the kind, got: {err}"
3339	);
3340	}
3341	}
3342
3343	/// A service host is addressed on the DEPLOY plane whatever step is open.
3344	///
3345	/// The subtle one. Actions are normally derived from the step, which is
3346	/// right for a build host — the step is what that host is being asked to do.
3347	/// A service host is granted `deploy`/`restart` and must never be granted
3348	/// `build`, so the same rule would have `glibc_check` ask it for `build`
3349	/// during a `verify` step and get denied for a reason unrelated to what was
3350	/// attempted. `verify` is the step that check belongs in, so without this
3351	/// routing the glibc gate cannot run at all.
3352	#[tokio::test]
3353	async fn a_service_host_is_addressed_on_the_deploy_plane_in_any_step() {
3354	let dir = tempfile::tempdir().unwrap();
3355	let cfg = Arc::new(Config::for_tests(dir.path()));
3356	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3357	sqlx::query(
3358	"INSERT INTO builds (id, app, version, status, created_at) \
3359	VALUES (1, 'demo', '0.1.0', 'running', '2026-07-30T00:00:00Z')",
3360	)
3361	.execute(&pool)
3362	.await
3363	.unwrap();
3364	sqlx::query(
3365	"INSERT INTO target_runs (id, build_id, app, version, target, status, started_at) \
3366	VALUES (1, 1, 'demo', '0.1.0', 'linux/x86_64', 'running', '2026-07-30T00:00:00Z')",
3367	)
3368	.execute(&pool)
3369	.await
3370	.unwrap();
3371
3372	let deploy = crate::topology::DeployTarget {
3373	target: "linux/x86_64".parse().unwrap(),
3374	host: "local".into(),
3375	port: None,
3376	install_path: "/usr/local/bin/demo".into(),
3377	service: "demo.service".into(),
3378	health_url: None,
3379	};
3380	let mut execs: crate::state::ExecutorMap = std::collections::HashMap::new();
3381	execs.insert("local".into(), crate::state::build_deploy_executor(&deploy));
3382	// The service host's grant is exactly deploy + restart. If this ever
3383	// widens to include `build`, the test below stops proving anything.
3384	assert!(!execs["local"].capabilities().permits(&Action::Build));
3385	assert!(execs["local"].capabilities().permits(&Action::Deploy));
3386
3387	let ctx = Arc::new(RecipeCtx::new(
3388	AppId::new("demo"),
3389	Version::parse("0.1.0").unwrap(),
3390	"linux/x86_64".parse().unwrap(),
3391	"fw13".into(),
3392	"local".into(),
3393	"v0.1.0".into(),
3394	"/tmp".into(),
3395	vec![],
3396	Kind::Service,
3397	1,
3398	Arc::new(execs),
3399	Arc::new(std::collections::HashMap::new()),
3400	Some(deploy),
3401	pool,
3402	crate::events::channel(),
3403	cfg,
3404	Arc::new(OtaRegistry::standard("https://makenot.work")),
3405	tokio::runtime::Handle::current(),
3406	Arc::new(AtomicBool::new(false)),
3407	None,
3408	));
3409
3410	let ctx_blocking = ctx.clone();
3411	tokio::task::spawn_blocking(move \|\| {
3412	// `verify` on a service derives Action::Build — which the service
3413	// host does not grant. The command must still run.
3414	ctx_blocking.begin_step(Step::Verify).unwrap();
3415	assert_eq!(
3416	action_for(Step::Verify, Kind::Service),
3417	Action::Build,
3418	"the step's own action is the one that would be denied",
3419	);
3420	let (code, out) = ctx_blocking
3421	.run("local", "echo reached-the-service-host")
3422	.expect("a service host must be reachable during a verify step");
3423	assert_eq!(code, 0, "{out}");
3424	assert!(out.contains("reached-the-service-host"), "{out}");
3425	})
3426	.await
3427	.unwrap();
3428	}
3429
3430	/// A step that finalized `Failed` bars the deploy, exactly as it bars a
3431	/// publish. Without this, a recipe that inspects `sh(...).code` and carries
3432	/// on regardless still lands a binary on a production host — the precise
3433	/// hazard a pipeline exists to remove. The check is the ledger, not the
3434	/// control flow, so it holds whether or not the recipe noticed.
3435	#[tokio::test]
3436	async fn a_failed_step_bars_the_deploy() {
3437	let dir = tempfile::tempdir().unwrap();
3438	let cfg = Arc::new(Config::for_tests(dir.path()));
3439	let pool = crate::db::open(&cfg.db_path).await.unwrap();
3440	let deploy = crate::topology::DeployTarget {
3441	target: "linux/x86_64".parse().unwrap(),
3442	host: "local".into(),
3443	port: None,
3444	install_path: "/usr/local/bin/demo".into(),
3445	service: "demo.service".into(),
3446	health_url: None,
3447	};
3448	// A real build + target run, so the step rows this test finalizes have
3449	// the parents the schema requires.
3450	sqlx::query(
3451	"INSERT INTO builds (id, app, version, status, created_at) \
3452	VALUES (1, 'demo', '0.1.0', 'running', '2026-07-30T00:00:00Z')",
3453	)
3454	.execute(&pool)
3455	.await
3456	.unwrap();
3457	sqlx::query(
3458	"INSERT INTO target_runs (id, build_id, app, version, target, status, started_at) \
3459	VALUES (1, 1, 'demo', '0.1.0', 'linux/x86_64', 'running', '2026-07-30T00:00:00Z')",
3460	)
3461	.execute(&pool)
3462	.await
3463	.unwrap();
3464
3465	let mut execs: crate::state::ExecutorMap = std::collections::HashMap::new();
3466	execs.insert("local".into(), crate::state::build_deploy_executor(&deploy));
3467	let ctx = Arc::new(RecipeCtx::new(
3468	AppId::new("demo"),
3469	Version::parse("0.1.0").unwrap(),
3470	"linux/x86_64".parse().unwrap(),
3471	"fw13".into(),
3472	"local".into(),
3473	"v0.1.0".into(),
3474	"/tmp".into(),
3475	vec![],
3476	Kind::Service,
3477	1,
3478	Arc::new(execs),
3479	Arc::new(std::collections::HashMap::new()),
3480	Some(deploy),
3481	pool,
3482	crate::events::channel(),
3483	cfg,
3484	Arc::new(OtaRegistry::standard("https://makenot.work")),
3485	tokio::runtime::Handle::current(),
3486	Arc::new(AtomicBool::new(false)),
3487	None,
3488	));
3489
3490	// A gate ran, failed, and the recipe did not abort — the swallowed
3491	// failure. Finalizing it is what puts it in the ledger.
3492	let ctx_blocking = ctx.clone();
3493	tokio::task::spawn_blocking(move \|\| {
3494	ctx_blocking.begin_step(Step::Prebuild).unwrap();
3495	ctx_blocking.fail_current_step();
3496	ctx_blocking.finish_step(Status::Ok).unwrap();
3497
3498	let err = ctx_blocking.deploy("/tmp/demo").unwrap_err().to_string();
3499	assert!(
3500	err.contains("refusing to deploy") && err.contains("prebuild"),
3501	"must refuse and name the failed step, got: {err}"
3502	);
3503	})
3504	.await
3505	.unwrap();
3506	}
3507
3508	#[test]
3509	fn every_step_has_a_nonzero_default_budget() {
3510	// A zero/missing budget would deadline-fail a step instantly. Cover the
3511	// whole matrix so a new Step variant can't silently get a 0 budget.
3512	for step in Step::ALL {
3513	assert!(
3514	default_step_budget(step) >= std::time::Duration::from_mins(1),
3515	"{step} budget must be a sane ceiling",
3516	);
3517	}
3518	}
3519
3520	#[test]
3521	fn sha256_file_is_lowercase_hex_of_contents() {
3522	let tmp = tempfile::tempdir().unwrap();
3523	let f = tmp.path().join("a.bin");
3524	std::fs::write(&f, b"abc").unwrap();
3525	// Known SHA-256 of "abc".
3526	assert_eq!(
3527	sha256_file(&f).unwrap(),
3528	"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
3529	);
3530	}
3531
3532	// ---- publish step-success gate ----
3533
3534	fn target(s: &str) -> Target {
3535	s.parse().unwrap()
3536	}
3537
3538	#[test]
3539	fn publish_gate_blocks_macos_without_verification() {
3540	// Never verified -> blocked, with a message pointing at verify_gatekeeper.
3541	let err = PublishAuthority::prove(target("macos/aarch64"), &[], None).unwrap_err();
3542	assert!(format!("{err:#}").contains("never verified"), "{err:#}");
3543	}
3544
3545	#[test]
3546	fn publish_gate_blocks_macos_when_gatekeeper_rejected() {
3547	let err = PublishAuthority::prove(target("macos/aarch64"), &[], Some(false)).unwrap_err();
3548	assert!(
3549	format!("{err:#}").contains("Gatekeeper rejected"),
3550	"{err:#}"
3551	);
3552	}
3553
3554	#[test]
3555	fn publish_gate_allows_macos_when_gatekeeper_accepted() {
3556	PublishAuthority::prove(target("macos/aarch64"), &[], Some(true)).unwrap();
3557	// iOS is gated the same way.
3558	PublishAuthority::prove(target("ios/universal"), &[], Some(true)).unwrap();
3559	assert!(PublishAuthority::prove(target("ios/universal"), &[], None).is_err());
3560	}
3561
3562	#[test]
3563	fn publish_gate_does_not_require_gatekeeper_for_non_apple_targets() {
3564	// Linux/Windows aren't notarized; no gatekeeper proof needed.
3565	PublishAuthority::prove(target("linux/x86_64"), &[], None).unwrap();
3566	PublishAuthority::prove(target("windows/x86_64"), &[], None).unwrap();
3567	}
3568
3569	#[test]
3570	fn publish_gate_blocks_when_any_prior_step_failed() {
3571	// A failed step bars publish on every target, even a verified macOS one.
3572	let err =
3573	PublishAuthority::prove(target("linux/x86_64"), &[Step::Build], None).unwrap_err();
3574	assert!(
3575	format!("{err:#}").contains("prior step(s) failed"),
3576	"{err:#}"
3577	);
3578	assert!(
3579	format!("{err:#}").contains("build"),
3580	"names the failed step: {err:#}"
3581	);
3582
3583	let err = PublishAuthority::prove(target("macos/aarch64"), &[Step::Sign], Some(true))
3584	.unwrap_err();
3585	assert!(
3586	format!("{err:#}").contains("prior step(s) failed"),
3587	"{err:#}"
3588	);
3589	}
3590
3591	#[test]
3592	fn notary_accepted_parses_status_field() {
3593	assert!(notary_accepted(
3594	r#"{"id":"abc","status":"Accepted","message":"ok"}"#
3595	));
3596	// Embedded in shell-sourcing noise: the object is isolated and parsed.
3597	assert!(notary_accepted(
3598	"sourcing env...\n{\n \"status\": \"Accepted\"\n}\nbye"
3599	));
3600	// Whitespace variant that a tight substring `"status":"Accepted"` misses.
3601	assert!(notary_accepted(r#"{ "status" : "Accepted" }"#));
3602	}
3603
3604	#[test]
3605	fn notary_accepted_rejects_non_accepted_and_garbage() {
3606	assert!(!notary_accepted(r#"{"status":"Invalid"}"#));
3607	assert!(!notary_accepted(r#"{"status":"In Progress"}"#));
3608	assert!(!notary_accepted("no json here"));
3609	assert!(!notary_accepted("")); // empty / truncated -> fail closed
3610	// A truncated tail whose opening brace was cut off cannot parse -> closed.
3611	assert!(!notary_accepted(r#""status":"Accepted"}"#));
3612	// The literal appearing inside an error string must NOT pass as success.
3613	assert!(!notary_accepted(
3614	r#"{"status":"Invalid","message":"expected status:Accepted"}"#
3615	));
3616	}
3617
3618	/// Both shapes of repo: one holding several products, and one holding a
3619	/// single crate, where git prints an empty prefix line.
3620	#[test]
3621	fn toplevel_and_prefix_read_both_shapes_of_repo() {
3622	let (top, prefix) =
3623	parse_toplevel_and_prefix("/home/max/Code/MNW\npom/\n").expect("two lines");
3624	assert_eq!(top, "/home/max/Code/MNW");
3625	assert_eq!(prefix, "pom/");
3626	// A repo holding one product: git prints an empty second line.
3627	let (top, prefix) =
3628	parse_toplevel_and_prefix("/home/max/Code/Libraries/pter\n\n").expect("two lines");
3629	assert_eq!(top, "/home/max/Code/Libraries/pter");
3630	assert_eq!(prefix, "");
3631	assert!(
3632	parse_toplevel_and_prefix("").is_none(),
3633	"no answer is not an answer"
3634	);
3635	}
3636
3637	#[test]
3638	fn repo_dir_name_is_the_last_segment_on_every_platform() {
3639	assert_eq!(repo_dir_name("/home/max/Code/MNW"), "MNW");
3640	assert_eq!(repo_dir_name("/home/max/Code/MNW/"), "MNW");
3641	// Git reports forward slashes on Windows too.
3642	assert_eq!(repo_dir_name("C:/Users/me/Code/Apps/goingson"), "goingson");
3643	}
3644
3645	/// The app's directory inside its worktree, for both repo shapes.
3646	#[test]
3647	fn app_dir_in_worktree_follows_the_prefix() {
3648	assert_eq!(
3649	app_dir_in_worktree("/home/max/Code/.bento/MNW/pom", "pom/"),
3650	"/home/max/Code/.bento/MNW/pom/pom"
3651	);
3652	assert_eq!(
3653	app_dir_in_worktree("/home/max/Code/.bento/pter/pter", ""),
3654	"/home/max/Code/.bento/pter/pter"
3655	);
3656	}
3657
3658	/// A missing tag is the common failure and gets a plain answer; anything
3659	/// else is git's own stderr, which says more than a guess.
3660	#[test]
3661	fn worktree_failure_reason_names_the_tag_or_repeats_git() {
3662	let missing = worktree_failure_reason("pom-v0.4.5", false, "irrelevant");
3663	assert!(missing.contains("does not exist"), "{missing}");
3664	let held = worktree_failure_reason(
3665	"pom-v0.4.5",
3666	true,
3667	"fatal: '/home/max/Code/.bento/MNW/pom' already exists",
3668	);
3669	assert!(held.contains("already exists"), "{held}");
3670	let silent = worktree_failure_reason("pom-v0.4.5", true, " ");
3671	assert!(silent.contains("said nothing"), "{silent}");
3672	}
3673
3674	/// Run the probe for real rather than asserting on its text: it is shell,
3675	/// and the thing worth knowing is whether `sh` agrees, not whether the
3676	/// string looks right.
3677	fn probe(repo: &std::path::Path) -> bool {
3678	std::process::Command::new("sh")
3679	.arg("-c")
3680	.arg(tracked_lock_under_patch_cmd(&repo.display().to_string()))
3681	.status()
3682	.unwrap()
3683	.success()
3684	}
3685
3686	/// A crate under a `[patch]` root, with and without its lock tracked.
3687	///
3688	/// pter 0.2.1 is the case: the tracked half was true, the patch half was
3689	/// true, and cargo reported only "Cargo.lock" (build 325). Untracking the
3690	/// lock in `a9969a9` is what fixed it, and this asserts the probe agrees
3691	/// with that fix in both directions.
3692	#[test]
3693	fn a_tracked_lock_is_only_a_problem_under_a_patch_block() {
3694	let root = tempfile::tempdir().unwrap();
3695	let repo = root.path().join("crate");
3696	std::fs::create_dir_all(&repo).unwrap();
3697	let git = \|args: &[&str]\| {
3698	std::process::Command::new("git")
3699	.args(args)
3700	.current_dir(&repo)
3701	.env("GIT_AUTHOR_NAME", "t")
3702	.env("GIT_AUTHOR_EMAIL", "t@t")
3703	.env("GIT_COMMITTER_NAME", "t")
3704	.env("GIT_COMMITTER_EMAIL", "t@t")
3705	.output()
3706	.unwrap()
3707	};
3708	git(&["init", "-q", "."]);
3709	std::fs::write(repo.join("Cargo.lock"), "# lock\n").unwrap();
3710
3711	// Lock present but untracked, no patch anywhere: nothing to say.
3712	assert!(!probe(&repo));
3713
3714	// Tracked, still no patch root. Every library that commits a lock and
3715	// builds outside `~/Code` lives here, and it publishes fine.
3716	git(&["add", "Cargo.lock"]);
3717	git(&["commit", "-qm", "lock"]);
3718	assert!(!probe(&repo));
3719
3720	// The ancestor declares `[patch]`. Both halves now hold.
3721	std::fs::create_dir_all(root.path().join(".cargo")).unwrap();
3722	std::fs::write(
3723	root.path().join(".cargo/config.toml"),
3724	"[patch.\"https://makenot.work/git/max/docengine.git\"]\ndocengine = { path = \"x\" }\n",
3725	)
3726	.unwrap();
3727	assert!(probe(&repo));
3728
3729	// Untracking the lock is the fix, and the probe has to agree that it is.
3730	git(&["rm", "-q", "--cached", "Cargo.lock"]);
3731	assert!(!probe(&repo));
3732	}
3733
3734	/// The message exists because cargo's names neither fact and both wrong
3735	/// fixes are one flag away. Assert it still says all four things.
3736	#[test]
3737	fn the_tracked_lock_message_names_both_facts_and_refuses_both_wrong_fixes() {
3738	let msg = tracked_lock_under_patch_problem("~/Code/.bento/pter/pter");
3739	assert!(msg.contains("Cargo.lock"), "{msg}");
3740	assert!(msg.contains("[patch]"), "{msg}");
3741	assert!(msg.contains("git rm --cached"), "{msg}");
3742	assert!(msg.contains("--allow-dirty"), "{msg}");
3743	assert!(msg.contains("~/Code/.bento/pter/pter"), "{msg}");
3744	}
3745	}
3746