max / makenotwork

127.7 KB · 3122 lines History Blame Raw

1	//! Rhai recipe engine + host-function API.
2	//!
3	//! A `(app, target)` resolves to a `.rhai` recipe composed from a shared step
4	//! vocabulary. The daemon embeds Rhai and registers the host functions recipes
5	//! call; the recipe is the orchestration, the host functions are the
6	//! privileged primitives (run a command, read a secret, collect artifacts,
7	//! publish). Recipes are otherwise sandboxed — no arbitrary FS/network except
8	//! through these functions — matching the Balanced Breakfast plugin model.
9	//!
10	//! Rhai is synchronous; the engine runs each recipe on a blocking thread
11	//! (`spawn_blocking`, see [`crate::runner`]) and host functions bridge to async
12	//! work via `Handle::block_on`. That is sound only off a runtime worker thread,
13	//! which `spawn_blocking` guarantees.
14
15	use crate::config::Config;
16	use crate::domain::{AppId, Status, Step, StepRunId, Target, Version};
17	use crate::events::{self, Event, EventTx};
18	use crate::ota::{OtaRegistry, PublishAuthority, Release};
19	use crate::state::ExecutorMap;
20	use crate::topology::{DeployTarget, Kind};
21	use anyhow::{Context as _, Result};
22	use ops_core::live_log::LiveLog;
23	use ops_exec::{Action, Executor, ObserveKind, Step as OpStep, SyncOpts};
24	use rhai::{Engine, EvalAltResult, Map};
25	use sha2::{Digest, Sha256};
26	use sqlx::SqlitePool;
27	use std::collections::HashMap;
28	use std::path::{Path, PathBuf};
29	use std::sync::atomic::{AtomicBool, Ordering};
30	use std::sync::{Arc, Mutex};
31	use tokio::runtime::Handle;
32	use tokio::sync::Mutex as AsyncMutex;
33
34	/// The capability label for a command, derived from the open recipe step. A
35	/// recipe's `sh("mbp", …)` under `step("sign")` becomes an `Action::Sign`, gated
36	/// by the mac host's `sign` grant — so recipes stay unchanged while every command
37	/// is capability-checked at its transport. `Verify` is read-only (an observe).
38	/// The capability a step's commands are gated on.
39	///
40	/// `Verify` depends on what is being released, which is the one place this is
41	/// not a property of the step alone. An app's verify is a Gatekeeper check on a
42	/// signed bundle, and the `gatekeeper` observe is granted implicitly to hosts
43	/// that can `sign` (`CapabilitySet::from_tokens`) precisely so that pairing
44	/// holds. A library's verify is a crate preflight: it runs `cargo` on the build
45	/// host and asks the registry a question. Gating that on Gatekeeper asks a Linux
46	/// host for a macOS code-signing capability it can never honestly hold, and the
47	/// only way to satisfy it would be to declare the capability falsely.
48	fn action_for(step: Step, kind: Kind) -> Action {
49	match step {
50	Step::Checkout \| Step::Prebuild \| Step::Build => Action::Build,
51	Step::Sign => Action::Sign,
52	Step::Notarize => Action::Notarize,
53	Step::Staple => Action::Staple,
54	Step::Package => Action::Package,
55	Step::Verify => match kind {
56	Kind::App => Action::Observe(ObserveKind::Custom("gatekeeper".into())),
57	// Running the build toolchain to inspect a crate or a service
58	// binary, which is what `build` means on a host. Neither has a
59	// bundle for Gatekeeper to have an opinion about.
60	Kind::Library \| Kind::Service => Action::Build,
61	},
62	// Publish/Collect run on the daemon, not through a host executor; this
63	// label only applies if a recipe runs a bare `sh` while one is open.
64	Step::Publish \| Step::Collect => Action::Package,
65	// The one step that dispatches to a host OUTSIDE the build topology.
66	// Every command a recipe runs while `deploy` is open — the install, the
67	// restart, the health assertion — carries this action, so it reaches the
68	// service host only through the deploy grant and reaches a build host
69	// not at all (no build host is granted `deploy`).
70	Step::Deploy => Action::Deploy,
71	}
72	}
73
74	/// The currently-open step within a recipe run: its DB row id, which step it
75	/// is, and the live-log sink that `sh`/`log` stream into.
76	struct StepState {
77	run_id: StepRunId,
78	step: Step,
79	log: Arc<AsyncMutex<LiveLog>>,
80	/// Set when something in the step recorded a hard failure the recipe did
81	/// not abort on (e.g. `verify_gatekeeper` rejected the artifact but the
82	/// recipe ignored the bool). Forces the step's recorded status to `Failed`
83	/// and bars `publish` (the step-success ledger).
84	failed: bool,
85	/// Wall-clock deadline for this step. A command that runs past it fails the
86	/// step (and unwinds the recipe) rather than wedging under the old
87	/// whole-build guillotine, which a legitimate 5-target fan-out plus notary
88	/// queueing could trip — mismarking every target failed while the blocking
89	/// recipe bodies kept signing.
90	deadline: std::time::Instant,
91	}
92
93	/// Per-step wall-clock budget: a generous ceiling that catches a wedged command
94	/// (a hung ssh, a stuck notary poll) without killing legitimately slow work.
95	/// The old design bounded the whole build at 2h; this bounds each step so one
96	/// slow step can't be blamed on another and a fan-out of slow-but-fine targets
97	/// isn't guillotined. `Config::step_timeout_secs` overrides these per-kind
98	/// defaults for every step; see [`RecipeCtx::step_budget`].
99	fn default_step_budget(step: Step) -> std::time::Duration {
100	use std::time::Duration;
101	let mins = match step {
102	Step::Checkout => 10,
103	// clippy + full test suite, cold, on a workspace.
104	Step::Prebuild => 45,
105	// cargo tauri build, cold, universal bundles.
106	Step::Build => 90,
107	Step::Sign => 15,
108	// Apple's notary queue + this step's bounded retries.
109	Step::Notarize => 60,
110	Step::Staple => 10,
111	Step::Package => 30,
112	Step::Verify => 10,
113	// rsync of multi-GiB artifacts off the build host.
114	Step::Collect => 30,
115	Step::Publish => 20,
116	// A binary push, an install, a unit restart, and a health poll. Minutes
117	// of work; the ceiling is for a wedged transport, not slow work.
118	Step::Deploy => 15,
119	};
120	Duration::from_secs(mins * 60)
121	}
122
123	/// Where a service's binary is staged on the host that will run it, before the
124	/// privileged installer moves it into place.
125	///
126	/// A fixed, unguessable-by-accident path rather than a recipe-chosen one,
127	/// because the installer refuses any source outside it. That refusal is the
128	/// only thing standing between the NOPASSWD sudo grant and `install`-as-root to
129	/// an arbitrary path, so both ends have to name the same constant. `/var/tmp`
130	/// rather than `/tmp` so a staged binary survives a `PrivateTmp` unit and a
131	/// systemd tmpfiles sweep between staging and install.
132	pub const DEPLOY_STAGING_ROOT: &str = "/var/tmp/bento-deploy";
133
134	/// Everything a recipe's host functions need, shared (Arc) into each closure.
135	pub struct RecipeCtx {
136	pub app: AppId,
137	pub version: Version,
138	pub target: Target,
139	/// Name of the host this target builds on (resolved from the topology by the
140	/// runner). Recipes read it via `build_host()` so one per-platform recipe can
141	/// dispatch to the right host across arches (linux x86_64 -> fw13, aarch64 ->
142	/// astra) without hard-coding a host name.
143	pub build_host: String,
144	/// The build host's SSH destination (topology `ssh`), as opposed to its
145	/// name. Deploy compares it against the service host's to tell "build and
146	/// run on the same box" from "two hosts that need a transfer" — a question
147	/// the host NAMES cannot answer, since a build host and a deploy
148	/// destination are declared in different files and need not agree on one.
149	pub build_host_ssh: String,
150	/// This release's git tag, rendered from the app's `tag_format`. Held here
151	/// rather than derived from the version because a repo holding several
152	/// products spells it per product (`pom-v0.4.1`), and the recipe, the
153	/// preflight barrier and the failure message all have to agree on it.
154	pub tag: String,
155	/// The app's checkout path (topology `repo`, `~`-prefixed). Recipes read it
156	/// via `repo()` to `cd` into the checkout — commands don't auto-cd, and each
157	/// `sh` is a fresh shell.
158	pub repo: String,
159	/// Cargo features this app's release builds enable (topology `features`).
160	/// Recipes read it via `feature_flags()`.
161	pub features: Vec<String>,
162	/// App or library. Decides which capability a `verify` step is gated on;
163	/// see [`action_for`].
164	pub kind: Kind,
165	pub target_run_id: i64,
166	/// Capability-scoped executor per build host. Recipe commands dispatch
167	/// through these — the transport (local / ssh / in-session agent) and the
168	/// capability gate are the executor's, not the engine's.
169	pub execs: Arc<ExecutorMap>,
170	/// Sync transport per build host, used only by `collect` to pull artifacts
171	/// back. Never the agent, even for an agent host — see `state::build_sync`.
172	/// Not an execution path.
173	pub syncs: Arc<ExecutorMap>,
174	/// Where this target installs, for a `kind = "service"` app. `None` for an
175	/// app or a library, which makes every deploy host function fail with that
176	/// as the reason rather than with a missing-host error.
177	///
178	/// The runner resolves it from the app manifest's `[[deploy]]` table and
179	/// registers its executor into `execs` under the destination's host string,
180	/// so `sh_ok(deploy_host(), ...)` reaches the service host through the same
181	/// capability gate as everything else.
182	pub deploy: Option<DeployTarget>,
183	pub pool: SqlitePool,
184	pub events: EventTx,
185	pub cfg: Arc<Config>,
186	pub ota: Arc<OtaRegistry>,
187	pub rt: Handle,
188	current: Mutex<Option<StepState>>,
189	/// Gatekeeper verdict recorded by `verify_gatekeeper`: `None` = never run,
190	/// `Some(false)` = ran and rejected, `Some(true)` = accepted. `publish`
191	/// requires `Some(true)` for a macOS/iOS artifact (the proof it is signed +
192	/// notarized).
193	gatekeeper_ok: Mutex<Option<bool>>,
194	/// Steps finalized as `Failed` during this run. A non-empty ledger bars
195	/// `publish` — an artifact is never shipped after a step failed, even if the
196	/// recipe ignored the failure and ran on.
197	failed_steps: Mutex<Vec<Step>>,
198	/// Set when a newer build supersedes this run. Checked at step boundaries
199	/// and before publish so a superseded recipe stops promptly rather than
200	/// running to completion (the blocking Rhai body can't be `abort()`ed).
201	cancel: Arc<AtomicBool>,
202	/// The all-targets-green publish gate (topology `require_all_targets`).
203	/// `Some(declared)` ⇒ `publish` refuses unless every one of `declared` has a
204	/// successful latest run for this `(app, version)`. `None` ⇒ gate off, each
205	/// target publishes independently.
206	all_green_required: Option<Vec<Target>>,
207	/// sha256 of each artifact hashed at `collect`, keyed by file name. `publish`
208	/// reads it to record `releases.artifact_hash` for the bytes it ships, so the
209	/// hash is the one computed when the artifact landed rather than a re-read
210	/// that could see a different file. Absent ⇒ `publish` hashes on demand.
211	artifact_hashes: Mutex<HashMap<String, String>>,
212	}
213
214	impl RecipeCtx {
215	/// Versions of `name` already on crates.io. A network failure yields an
216	/// empty list: preflight then cannot claim a version is a duplicate, and
217	/// `cargo publish` still refuses one, so the check degrades to advisory
218	/// rather than blocking a release on registry availability.
219	fn published_versions(name: &str) -> Vec<String> {
220	let url = format!("https://crates.io/api/v1/crates/{name}");
221	let Ok(out) = std::process::Command::new("curl")
222	.args([
223	"-sS",
224	"--max-time",
225	"15",
226	"-H",
227	"User-Agent: bento-preflight",
228	&url,
229	])
230	.output()
231	else {
232	return Vec::new();
233	};
234	let Ok(v) = serde_json::from_slice::<serde_json::Value>(&out.stdout) else {
235	return Vec::new();
236	};
237	v.get("versions")
238	.and_then(\|x\| x.as_array())
239	.map(\|a\| {
240	a.iter()
241	.filter_map(\|x\| x.get("num").and_then(\|n\| n.as_str()).map(str::to_string))
242	.collect()
243	})
244	.unwrap_or_default()
245	}
246
247	#[allow(clippy::too_many_arguments)]
248	pub fn new(
249	app: AppId,
250	version: Version,
251	target: Target,
252	build_host: String,
253	build_host_ssh: String,
254	tag: String,
255	repo: String,
256	features: Vec<String>,
257	kind: Kind,
258	target_run_id: i64,
259	execs: Arc<ExecutorMap>,
260	syncs: Arc<ExecutorMap>,
261	deploy: Option<DeployTarget>,
262	pool: SqlitePool,
263	events: EventTx,
264	cfg: Arc<Config>,
265	ota: Arc<OtaRegistry>,
266	rt: Handle,
267	cancel: Arc<AtomicBool>,
268	all_green_required: Option<Vec<Target>>,
269	) -> Self {
270	Self {
271	app,
272	version,
273	target,
274	build_host,
275	build_host_ssh,
276	tag,
277	repo,
278	features,
279	kind,
280	target_run_id,
281	execs,
282	syncs,
283	deploy,
284	pool,
285	events,
286	cfg,
287	ota,
288	rt,
289	current: Mutex::new(None),
290	gatekeeper_ok: Mutex::new(None),
291	failed_steps: Mutex::new(Vec::new()),
292	cancel,
293	all_green_required,
294	artifact_hashes: Mutex::new(HashMap::new()),
295	}
296	}
297
298	/// Whether a newer build has superseded this run.
299	fn is_cancelled(&self) -> bool {
300	self.cancel.load(Ordering::SeqCst)
301	}
302
303	fn now() -> String {
304	chrono::Utc::now().to_rfc3339()
305	}
306
307	/// `<logs_root>/<app>/<version>/<target-with-slash-as-dash>/<step>.log`.
308	fn log_path(&self, step: Step) -> PathBuf {
309	let target_dir = self.target.to_string().replace('/', "-");
310	self.cfg
311	.logs_root
312	.join(self.app.as_str())
313	.join(self.version.to_string())
314	.join(target_dir)
315	.join(format!("{}.log", step.as_str()))
316	}
317
318	/// Close the previous step (as `Ok`), open a new one: insert its DB row,
319	/// open a live log whose chunks broadcast `StepLogChunk`, emit `StepStart`.
320	fn begin_step(self: &Arc<Self>, step: Step) -> Result<()> {
321	anyhow::ensure!(
322	!self.is_cancelled(),
323	"build superseded by a newer request; aborting before `{}`",
324	step.as_str()
325	);
326	self.finish_step(Status::Ok)?;
327	let me = self.clone();
328	let run_id = self.rt.block_on(async move {
329	let started = Self::now();
330	let log_ref = me.log_path(step).to_string_lossy().into_owned();
331	// The step row and the target's current_step pointer are one logical
332	// state — write them atomically so a failure can't leave a `running`
333	// step row while current_step still names the previous step.
334	let mut tx = me.pool.begin().await.context("begin step tx")?;
335	let id: i64 = sqlx::query_scalar(
336	"INSERT INTO step_runs (target_run_id, step, status, log_ref, started_at)
337	VALUES (?, ?, 'running', ?, ?) RETURNING id",
338	)
339	.bind(me.target_run_id)
340	.bind(step.as_str())
341	.bind(&log_ref)
342	.bind(&started)
343	.fetch_one(&mut *tx)
344	.await
345	.context("insert step_run")?;
346	sqlx::query("UPDATE target_runs SET current_step = ? WHERE id = ?")
347	.bind(step.as_str())
348	.bind(me.target_run_id)
349	.execute(&mut *tx)
350	.await
351	.context("update current_step")?;
352	tx.commit().await.context("commit step tx")?;
353	anyhow::Ok(StepRunId(id))
354	})?;
355
356	// Live log: each chunk fans out as a StepLogChunk event keyed by run_id.
357	let events = self.events.clone();
358	let cb_run_id = run_id;
359	let log = self.rt.block_on(LiveLog::open(
360	self.log_path(step),
361	Box::new(move \|seq, text\| {
362	events::emit(
363	&events,
364	Event::StepLogChunk {
365	run_id: cb_run_id,
366	seq,
367	text: text.to_string(),
368	},
369	);
370	}),
371	));
372
373	events::emit(
374	&self.events,
375	Event::StepStart {
376	run_id,
377	app: self.app.clone(),
378	version: self.version.clone(),
379	target: self.target,
380	step,
381	},
382	);
383
384	*self.current.lock().unwrap() = Some(StepState {
385	run_id,
386	step,
387	log: Arc::new(AsyncMutex::new(log)),
388	failed: false,
389	deadline: std::time::Instant::now() + self.step_budget(step),
390	});
391	Ok(())
392	}
393
394	/// This build's budget for `step`: the `Config` override if set, else the
395	/// per-kind default.
396	fn step_budget(&self, step: Step) -> std::time::Duration {
397	self.cfg
398	.step_timeout_secs
399	.map_or_else(\|\| default_step_budget(step), std::time::Duration::from_secs)
400	}
401
402	/// The current step's wall-clock deadline (or a default if no step is open,
403	/// which only happens before the first `step()` — commands then run under an
404	/// implicit `Build` step opened by `ensure_step`).
405	fn step_deadline(&self) -> std::time::Instant {
406	self.current.lock().unwrap().as_ref().map_or_else(
407	\|\| std::time::Instant::now() + self.step_budget(Step::Build),
408	\|s\| s.deadline,
409	)
410	}
411
412	/// Drive `fut` on the runtime, but stop early on two conditions the recipe
413	/// bodies otherwise cannot observe (they run synchronously on a blocking
414	/// thread): the current step's deadline, and supersession by a newer build.
415	/// Either turns into an error that fails the step and unwinds the recipe, so
416	/// a wedged command no longer runs unbounded and a superseded build stops
417	/// mid-step instead of only at the next step boundary.
418	fn run_bounded<F, T>(&self, what: &str, fut: F) -> Result<T>
419	where
420	F: std::future::Future<Output = Result<T>>,
421	{
422	let deadline = self.step_deadline();
423	let cancel = self.cancel.clone();
424	self.rt.block_on(async move {
425	tokio::pin!(fut);
426	let watch = async {
427	// Poll the cooperative cancel flag; the finalizer and a
428	// superseding build both set it. Cheap next to a build step.
429	while !cancel.load(Ordering::SeqCst) {
430	tokio::time::sleep(std::time::Duration::from_millis(250)).await;
431	}
432	};
433	tokio::select! {
434	r = &mut fut => r,
435	() = tokio::time::sleep_until(deadline.into()) => {
436	Err(anyhow::anyhow!("`{what}` exceeded its per-step deadline"))
437	}
438	() = watch => {
439	Err(anyhow::anyhow!("build superseded by a newer request; aborting `{what}`"))
440	}
441	}
442	})
443	}
444
445	/// Flag the currently-open step as failed (no-op if none is open). Forces
446	/// its recorded status to `Failed` at `finish_step` and adds it to the
447	/// publish-barring ledger, even though the recipe kept running.
448	fn fail_current_step(&self) {
449	if let Some(st) = self.current.lock().unwrap().as_mut() {
450	st.failed = true;
451	}
452	}
453
454	/// Finalize the open step (if any): close its log, stamp the DB row, emit
455	/// `StepDone`. Idempotent when no step is open. A step flagged via
456	/// [`fail_current_step`] is recorded `Failed` regardless of the requested
457	/// status, and added to the ledger `publish` consults.
458	pub fn finish_step(self: &Arc<Self>, status: Status) -> Result<()> {
459	let st = self.current.lock().unwrap().take();
460	let Some(st) = st else { return Ok(()) };
461	let status = if st.failed { Status::Failed } else { status };
462	if status == Status::Failed {
463	self.failed_steps.lock().unwrap().push(st.step);
464	}
465	let me = self.clone();
466	self.rt.block_on(async move {
467	// Drop all log refs so the sink can be owned + flushed.
468	if let Ok(m) = Arc::try_unwrap(st.log) {
469	m.into_inner().close().await;
470	}
471	if let Err(e) = sqlx::query(
472	"UPDATE step_runs SET status = ?, finished_at = ? WHERE id = ?",
473	)
474	.bind(status.as_str())
475	.bind(Self::now())
476	.bind(st.run_id.0)
477	.execute(&me.pool)
478	.await
479	{
480	tracing::error!(step = st.step.as_str(), error = %e, "could not stamp step_run status");
481	}
482	});
483	events::emit(
484	&self.events,
485	Event::StepDone {
486	run_id: st.run_id,
487	app: self.app.clone(),
488	target: self.target,
489	step: st.step,
490	status,
491	},
492	);
493	Ok(())
494	}
495
496	/// Ensure a step is open; default to `Build` if a recipe runs a command
497	/// before declaring one.
498	fn ensure_step(self: &Arc<Self>) -> Result<Arc<AsyncMutex<LiveLog>>> {
499	if self.current.lock().unwrap().is_none() {
500	self.begin_step(Step::Build)?;
501	}
502	Ok(self.current.lock().unwrap().as_ref().unwrap().log.clone())
503	}
504
505	/// The step currently open, or `Build` as a default for failure
506	/// attribution before any step was declared.
507	pub fn current_step(&self) -> Step {
508	self.current
509	.lock()
510	.unwrap()
511	.as_ref()
512	.map_or(Step::Build, \|s\| s.step)
513	}
514
515	/// The capability-scoped executor for `name`, or an error if the host isn't
516	/// in the topology.
517	fn exec(&self, name: &str) -> Result<Arc<dyn ops_exec::Executor>> {
518	self.execs
519	.get(name)
520	.cloned()
521	.ok_or_else(\|\| anyhow::anyhow!("unknown build host `{name}` (not in topology)"))
522	}
523
524	/// The ssh string for `name` (for `collect`'s remote scp source).
525	/// The transport that moves artifacts off `name`. Distinct from
526	/// [`RecipeCtx::exec_for`]'s executor: an agent host signs over `AgentRpc`
527	/// but is collected from over ssh (`state::build_sync`).
528	fn host_sync(&self, name: &str) -> Result<Arc<dyn Executor>> {
529	self.syncs
530	.get(name)
531	.cloned()
532	.ok_or_else(\|\| anyhow::anyhow!("unknown build host `{name}` (not in topology)"))
533	}
534
535	/// Run `cmd` on `host` through its capability-scoped executor, streaming into
536	/// the current step's log. The command's [`Action`] is derived from the open
537	/// step (see [`action_for`]) and gated at the transport before dispatch — so a
538	/// `build` step on a host without the `build` grant is denied, and the macOS
539	/// sign steps ride the in-session `AgentRpc` transport automatically. Returns
540	/// exit code + a tail of stdout for the recipe to branch on.
541	fn run(self: &Arc<Self>, host: &str, cmd: &str) -> Result<(i32, String)> {
542	// The service host is addressed as a service host whatever step is open.
543	// Deriving the action from the step is right for a build host, where the
544	// step IS the work; on a service host it would ask for `build` during a
545	// `verify` and be denied for a reason unrelated to what was attempted.
546	let action = match &self.deploy {
547	Some(d) if d.host == host => Action::Deploy,
548	_ => action_for(self.current_step(), self.kind),
549	};
550	self.run_as(host, cmd, action)
551	}
552
553	/// `run`, with the [`Action`] stated rather than resolved. Used where the
554	/// caller already knows which plane it is on.
555	fn run_as(self: &Arc<Self>, host: &str, cmd: &str, action: Action) -> Result<(i32, String)> {
556	let sink = self.ensure_step()?;
557	let exec = self.exec(host)?;
558	let cur = self.current_step();
559	let step = OpStep::shell(action, cmd.to_string());
560	// Bounded by the step's deadline and interruptible on supersession, so a
561	// hung command fails its step instead of running unbounded, and a
562	// superseded build stops mid-step rather than only at the next boundary.
563	let label = format!("{cur} command on `{host}`");
564	let out = self.run_bounded(&label, async move {
565	let mut guard = sink.lock().await;
566	exec.run_streaming(&step, &mut *guard).await
567	})?;
568	let code = out.status.code().unwrap_or(-1);
569	let stdout = String::from_utf8_lossy(&out.stdout);
570	let tail: String = stdout
571	.chars()
572	.rev()
573	.take(2000)
574	.collect::<Vec<_>>()
575	.into_iter()
576	.rev()
577	.collect();
578	Ok((code, tail))
579	}
580
581	/// Pin `host` to the release tag `v<version>` and return the commit it now
582	/// has checked out. Fetch + checkout stream into the current step's log;
583	/// the sha comes from a separate `rev-parse` so its stdout is only the sha.
584	fn checkout_sha(self: &Arc<Self>, host: &str) -> Result<String> {
585	// A failing mirror is not a failing release: fetch is advisory, and only
586	// the checkout decides. Its output still streams into the step log, so an
587	// unreachable remote stays visible without being fatal.
588	let _ = self.run(host, &git_fetch_cmd(&self.repo))?;
589	let (code, _) = self.run(host, &git_checkout_tag_cmd(&self.repo, &self.tag))?;
590	if code != 0 {
591	let (probe, _) = self.run(host, &git_tag_exists_cmd(&self.repo, &self.tag))?;
592	anyhow::bail!(
593	"checkout of {} failed on `{host}`: {}",
594	self.tag,
595	checkout_failure_reason(&self.tag, probe == 0)
596	);
597	}
598	let (code, tail) = self.run(host, &git_rev_parse_cmd(&self.repo))?;
599	anyhow::ensure!(code == 0, "rev-parse failed on `{host}`");
600	Ok(tail.trim().to_string())
601	}
602
603	/// Resolve `glob` on `host` to the single artifact it names. The `for` loop
604	/// lists each existing match on its own line (and prints nothing — rather
605	/// than a literal unexpanded pattern — when the glob matches no file), so
606	/// the count is unambiguous. `required` controls whether zero matches is an
607	/// error; more than one always is. See `resolve_artifact_match`.
608	fn resolve_artifact(
609	self: &Arc<Self>,
610	host: &str,
611	glob: &str,
612	required: bool,
613	) -> Result<String> {
614	ensure_glob_safe(glob)?;
615	// `[ -e ]` guards against a non-matching glob surviving as its literal
616	// self, and lists one path per line for the count.
617	let cmd = format!("for __f in {glob}; do [ -e \"$__f\" ] && printf '%s\\n' \"$__f\"; done");
618	let (code, tail) = self.run(host, &cmd)?;
619	anyhow::ensure!(
620	code == 0,
621	"resolving artifact glob `{glob}` on `{host}` exited {code}"
622	);
623	resolve_artifact_match(&tail, glob, required)
624	}
625	}
626
627	// ----- error bridging: anyhow -> Rhai runtime error -----
628
629	// Rhai host functions return `Result<_, Box<EvalAltResult>>` by convention, so
630	// this bridge must yield the boxed form to be usable with `.map_err(rhai_err)`.
631	#[allow(
632	clippy::unnecessary_box_returns,
633	reason = "rhai's error type is used boxed throughout its host-function API"
634	)]
635	fn rhai_err(e: impl std::fmt::Display) -> Box<EvalAltResult> {
636	Box::new(EvalAltResult::ErrorRuntime(
637	e.to_string().into(),
638	rhai::Position::NONE,
639	))
640	}
641
642	/// A crate's publish-relevant metadata, read from `cargo metadata`.
643	#[derive(Debug, Clone)]
644	pub struct CrateMeta {
645	pub name: String,
646	pub version: String,
647	pub repository: Option<String>,
648	pub description: Option<String>,
649	pub licensed: bool,
650	}
651
652	/// Parse the fields that matter for publishing out of `cargo metadata` JSON.
653	pub fn crate_meta_from_json(raw: &str) -> Result<CrateMeta> {
654	let v: serde_json::Value = serde_json::from_str(raw).context("parsing cargo metadata")?;
655	let p = v
656	.get("packages")
657	.and_then(\|p\| p.as_array())
658	.and_then(\|a\| a.first())
659	.context("cargo metadata reported no package")?;
660	let str_field = \|k: &str\| {
661	p.get(k)
662	.and_then(\|x\| x.as_str())
663	.filter(\|s\| !s.is_empty())
664	.map(str::to_string)
665	};
666	Ok(CrateMeta {
667	name: str_field("name").context("package has no name")?,
668	version: str_field("version").context("package has no version")?,
669	repository: str_field("repository"),
670	description: str_field("description"),
671	licensed: str_field("license").is_some() \|\| str_field("license_file").is_some(),
672	})
673	}
674
675	/// Everything wrong with a crate's metadata, as messages. Empty means publishable.
676	///
677	/// Checks only what crates.io records permanently. A published version cannot
678	/// be edited, only yanked, and yanking does not correct a wrong URL — so these
679	/// are the last moment any of it can be fixed.
680	pub fn crate_publish_problems(
681	meta: &CrateMeta,
682	repo_clonable: bool,
683	published: &[String],
684	credentials_present: bool,
685	) -> Vec<String> {
686	let mut out = Vec::new();
687	if !credentials_present {
688	out.push(
689	"no crates.io credentials on the publishing host: `cargo login` there first. \
690	Checked now rather than at the upload, so this fails in seconds instead of \
691	after a full build and verify."
692	.to_string(),
693	);
694	}
695	match &meta.repository {
696	None => out.push(
697	"no `repository` field: the crates.io page will show no source link, permanently"
698	.to_string(),
699	),
700	Some(url) if !repo_clonable => out.push(format!(
701	"`repository` is not publicly clonable: {url} \
702	(wrong URL, or the repo is private)"
703	)),
704	Some(_) => {}
705	}
706	if meta.description.is_none() {
707	out.push("no `description`: crates.io requires one".to_string());
708	}
709	if !meta.licensed {
710	out.push("no `license` or `license-file`".to_string());
711	}
712	if published.iter().any(\|v\| v == &meta.version) {
713	out.push(format!(
714	"version {} is already published; bump it",
715	meta.version
716	));
717	}
718	out
719	}
720
721	/// Read the app's version from its checkout on the daemon host. With
722	/// `version_path` set (topology `version_path`), read exactly that file — a
723	/// `.json` as a Tauri config, anything else as a `Cargo.toml`. Unset (the Tauri
724	/// default), try `src-tauri/tauri.conf.json` then the root `Cargo.toml`. Used by
725	/// the runner's default-version path.
726	pub fn version_from_repo(repo: &str, version_path: Option<&str>) -> Result<Version> {
727	let root = expand_tilde(repo);
728	if let Some(vp) = version_path {
729	let path = root.join(vp);
730	let raw = std::fs::read_to_string(&path)
731	.with_context(\|\| format!("reading version file {}", path.display()))?;
732	let ver = if std::path::Path::new(vp)
733	.extension()
734	.is_some_and(\|e\| e.eq_ignore_ascii_case("json"))
735	{
736	version_from_tauri_json(&raw)?
737	} else {
738	version_from_cargo_toml(&raw)?
739	};
740	return Version::parse(&ver).map_err(\|e\| anyhow::anyhow!(e));
741	}
742	let tauri_conf = root.join("src-tauri").join("tauri.conf.json");
743	if tauri_conf.exists() {
744	let raw = std::fs::read_to_string(&tauri_conf)
745	.with_context(\|\| format!("reading {}", tauri_conf.display()))?;
746	return Version::parse(&version_from_tauri_json(&raw)?).map_err(\|e\| anyhow::anyhow!(e));
747	}
748	let cargo_toml = root.join("Cargo.toml");
749	let raw = std::fs::read_to_string(&cargo_toml).with_context(\|\| {
750	format!(
751	"reading {} (no tauri.conf.json either)",
752	cargo_toml.display()
753	)
754	})?;
755	Version::parse(&version_from_cargo_toml(&raw)?).map_err(\|e\| anyhow::anyhow!(e))
756	}
757
758	/// Extract `version` from raw `tauri.conf.json` text.
759	fn version_from_tauri_json(raw: &str) -> Result<String> {
760	let v: serde_json::Value = serde_json::from_str(raw).context("parsing tauri.conf.json")?;
761	v.get("version")
762	.and_then(\|x\| x.as_str())
763	.map(str::to_owned)
764	.context("no `version` in tauri.conf.json")
765	}
766
767	/// Extract the version from raw `Cargo.toml` text — `[package].version` (a leaf
768	/// crate) or `[workspace.package].version` (a workspace that sets it).
769	fn version_from_cargo_toml(raw: &str) -> Result<String> {
770	let doc: toml::Value = toml::from_str(raw).context("parsing Cargo.toml")?;
771	doc.get("package")
772	.and_then(\|p\| p.get("version"))
773	.or_else(\|\| {
774	doc.get("workspace")
775	.and_then(\|w\| w.get("package"))
776	.and_then(\|p\| p.get("version"))
777	})
778	.and_then(\|v\| v.as_str())
779	.map(str::to_owned)
780	.context("no `[package].version` or `[workspace.package].version` in Cargo.toml")
781	}
782
783	/// Cross-check every version source in a repo and confirm they all agree with
784	/// the version being built, before a single host pulls or compiles.
785	///
786	/// `version_from_repo` reads exactly one file, so a `tauri.conf.json` at 0.5.0
787	/// and a root `Cargo.toml` still at 0.4.0 build happily and file artifacts under
788	/// whichever the runner happened to read. This reads every source present —
789	/// `version_path` (when set), `src-tauri/tauri.conf.json`, and the root
790	/// `Cargo.toml` — and fails loudly when any disagree, naming each file and its
791	/// version. A source that is absent is skipped (a library crate with only a
792	/// `Cargo.toml` has nothing to disagree with); the check never invents drift.
793	///
794	/// Scope: the JSON/TOML sources bentod itself reads. The iOS `gen/apple/project.yml`
795	/// path (rewritten by a build-time `sed`) is out of scope here — it is asserted at
796	/// its own build step — but the same drift class motivated this guard.
797	pub fn check_version_consistency(
798	repo: &str,
799	version_path: Option<&str>,
800	expected: &Version,
801	) -> Result<()> {
802	let root = expand_tilde(repo);
803	// (human-readable source label, parsed version) for every source present.
804	let mut found: Vec<(String, Version)> = Vec::new();
805
806	let mut consider = \|rel: &str, raw: &str, as_json: bool\| -> Result<()> {
807	let ver = if as_json {
808	version_from_tauri_json(raw)
809	} else {
810	version_from_cargo_toml(raw)
811	}?;
812	let parsed = Version::parse(&ver).map_err(\|e\| anyhow::anyhow!(e))?;
813	found.push((rel.to_string(), parsed));
814	Ok(())
815	};
816
817	if let Some(vp) = version_path {
818	let path = root.join(vp);
819	let raw = std::fs::read_to_string(&path)
820	.with_context(\|\| format!("reading version file {}", path.display()))?;
821	let is_json = std::path::Path::new(vp)
822	.extension()
823	.is_some_and(\|e\| e.eq_ignore_ascii_case("json"));
824	consider(vp, &raw, is_json)?;
825	}
826	let tauri_conf = root.join("src-tauri").join("tauri.conf.json");
827	if version_path != Some("src-tauri/tauri.conf.json") && tauri_conf.exists() {
828	let raw = std::fs::read_to_string(&tauri_conf)
829	.with_context(\|\| format!("reading {}", tauri_conf.display()))?;
830	consider("src-tauri/tauri.conf.json", &raw, true)?;
831	}
832	let cargo_toml = root.join("Cargo.toml");
833	if version_path != Some("Cargo.toml") && cargo_toml.exists() {
834	// A Cargo.toml with neither `[package].version` nor
835	// `[workspace.package].version` (a pure virtual workspace) carries no
836	// version to check — skip it rather than fail.
837	if let Ok(raw) = std::fs::read_to_string(&cargo_toml)
838	&& version_from_cargo_toml(&raw).is_ok()
839	{
840	consider("Cargo.toml", &raw, false)?;
841	}
842	}
843
844	let disagree: Vec<&(String, Version)> = found.iter().filter(\|(_, v)\| v != expected).collect();
845	anyhow::ensure!(
846	disagree.is_empty(),
847	"version drift in {repo}: building {expected} but {}",
848	disagree
849	.iter()
850	.map(\|(src, v)\| format!("{src} says {v}"))
851	.collect::<Vec<_>>()
852	.join(", ")
853	);
854	Ok(())
855	}
856
857	/// Every `X.Y.Z`-shaped version embedded in an artifact file name. Each maximal
858	/// run of digits-and-dots contributes its first three numeric fields:
859	/// `GoingsOn_0.5.0_aarch64.dmg` and `demo-9.9.9.bin` both yield one version (the
860	/// trailing `.bin`/`.dmg` dot is tolerated), while `latest.json` yields `[]` and
861	/// the `64` in `x86_64` is not three fields. Only the `major.minor.patch` core is
862	/// taken; a prerelease/build suffix is separated by `-`/`+` and not needed here.
863	fn versions_in_filename(name: &str) -> Vec<Version> {
864	name.split(\|c: char\| !(c.is_ascii_digit() \|\| c == '.'))
865	.filter_map(\|run\| {
866	let f: Vec<&str> = run.split('.').filter(\|s\| !s.is_empty()).collect();
867	if f.len() >= 3 && f[..3].iter().all(\|s\| s.chars().all(\|c\| c.is_ascii_digit())) {
868	Version::parse(&format!("{}.{}.{}", f[0], f[1], f[2])).ok()
869	} else {
870	None
871	}
872	})
873	.collect()
874	}
875
876	/// Fail when a collected file's name embeds a version whose `major.minor.patch`
877	/// is not the one being built. This is the guard against a stale checked-in
878	/// artifact winning a glob: `ls -t <glob>` once let
879	/// `AudioFiles-0.4.0-x86_64.AppImage` ship against 0.5.0. A file whose name
880	/// carries no version (an updater `latest.json`, a `.sig`) is not asserted —
881	/// there is nothing to compare. Compared on the core so a prerelease build's
882	/// plain `X.Y.Z` in the filename still matches.
883	fn assert_artifact_version(name: &str, expected: &Version) -> Result<()> {
884	let versions = versions_in_filename(name);
885	anyhow::ensure!(
886	versions.is_empty() \|\| versions.iter().any(\|v\| v.core() == expected.core()),
887	"collected artifact `{name}` carries version {} but the build is {expected}; \
888	a stale artifact was left in the output dir — clean it so only {expected} remains",
889	versions
890	.iter()
891	.map(ToString::to_string)
892	.collect::<Vec<_>>()
893	.join("/"),
894	);
895	Ok(())
896	}
897
898	/// sha256 of a file, lowercase hex. Streams in 64 KiB chunks so a multi-GiB
899	/// bundle never lands in memory whole.
900	fn sha256_file(path: &Path) -> Result<String> {
901	let mut file =
902	std::fs::File::open(path).with_context(\|\| format!("hashing {}", path.display()))?;
903	let mut hasher = Sha256::new();
904	std::io::copy(&mut file, &mut hasher)
905	.with_context(\|\| format!("reading {} to hash", path.display()))?;
906	Ok(hex_lower(&hasher.finalize()))
907	}
908
909	/// Lowercase-hex encode without pulling in a hex crate.
910	fn hex_lower(bytes: &[u8]) -> String {
911	use std::fmt::Write as _;
912	let mut s = String::with_capacity(bytes.len() * 2);
913	for b in bytes {
914	let _ = write!(s, "{b:02x}");
915	}
916	s
917	}
918
919	/// Refresh every remote's refs and tags, so the tag a release names is present
920	/// locally however it was pushed. No branch/upstream assumptions — a bare
921	/// `git pull --ff-only` needs a tracking branch the release path shouldn't
922	/// depend on.
923	///
924	/// Best-effort on purpose. `fetch --all` exits non-zero if ANY remote fails, and
925	/// the library repos carry three (`astra`, `mnw`, `srht`), so chaining this into
926	/// the checkout with `&&` meant one unreachable mirror aborted the release and
927	/// reported it as a missing tag. The checkout below is the step allowed to fail;
928	/// this one only has to try. See [`git_checkout_tag_cmd`].
929	///
930	/// `repo` is interpolated UNQUOTED so a leading `~` is expanded by the remote
931	/// host's shell (the checkout path is trusted topology config, not user input),
932	/// matching how the recipes `cd` into it.
933	pub fn git_fetch_cmd(repo: &str) -> String {
934	format!("git -C {repo} fetch --all --tags --prune")
935	}
936
937	/// Pin the host's checkout to the release tag. Runs after [`git_fetch_cmd`], and
938	/// is the operation whose exit code decides whether the release proceeds.
939	pub fn git_checkout_tag_cmd(repo: &str, tag: &str) -> String {
940	format!("git -C {repo} checkout \"{tag}\"")
941	}
942
943	/// Does `tag` resolve to a commit in this checkout? Run only when the checkout
944	/// has already failed, to say WHY: an absent tag is an untagged or unpushed
945	/// release, while a tag that resolves fine means the checkout was refused for a
946	/// local reason (a dirty tree, most often) and the operator needs to hear that
947	/// instead.
948	pub fn git_tag_exists_cmd(repo: &str, tag: &str) -> String {
949	format!("git -C {repo} rev-parse -q --verify \"refs/tags/{tag}^{{commit}}\"")
950	}
951
952	/// The operator-facing explanation for a failed tag checkout, given whether the
953	/// tag turned out to exist locally.
954	pub fn checkout_failure_reason(tag: &str, tag_exists: bool) -> String {
955	if tag_exists {
956	format!(
957	"tag {tag} exists but could not be checked out \
958	(uncommitted changes in the checkout?)"
959	)
960	} else {
961	format!("tag {tag} does not exist there (is it created and pushed?)")
962	}
963	}
964
965	/// Uncommitted changes to TRACKED files, one `XY path` line each, empty when the
966	/// tree is clean.
967	///
968	/// `--untracked-files=no` on purpose: an untracked file is not built into the
969	/// binary and a build host accumulates them (editor scratch, stray logs), so
970	/// failing a release on one would be noise. A modified tracked file is the
971	/// opposite — it is exactly what `cargo build` would pick up instead of the
972	/// tagged content.
973	///
974	/// Scoped to `repo` with `-- .` rather than asking about the whole repository,
975	/// which matters only for the repos holding more than one product. `repo` for
976	/// pom is `~/Code/MNW/pom` inside the MNW monorepo, and an edit in `server/` is
977	/// not something pom's build can compile. Refusing pom's release for it would be
978	/// a gate that fires on unrelated work, which is how a gate gets bypassed. For a
979	/// single-product repo `repo` is the root and this is the whole tree, unchanged.
980	pub fn git_dirty_cmd(repo: &str) -> String {
981	format!("git -C {repo} status --porcelain --untracked-files=no -- .")
982	}
983
984	/// The branch a host's checkout is on, empty (and non-zero) on a detached HEAD.
985	/// Read BEFORE the release pins the tag, so the checkout can be put back
986	/// afterwards — see [`git_restore_branch_cmd`].
987	pub fn git_current_branch_cmd(repo: &str) -> String {
988	format!("git -C {repo} symbolic-ref -q --short HEAD")
989	}
990
991	/// Put a checkout back on the branch it was on before the release pinned it to
992	/// the tag.
993	///
994	/// The pin itself is correct and deliberate: a release must build the tagged
995	/// commit, not the branch tip. What was missing is the other half. Leaving the
996	/// tree detached is invisible — git does not warn, and commits made afterwards
997	/// succeed normally while belonging to no branch. makeover shipped 2.3.0 from
998	/// exactly that state on 2026-07-28: three commits, including the published one,
999	/// existed only as a detached HEAD on one machine, on no branch and no remote.
1000	pub fn git_restore_branch_cmd(repo: &str, branch: &str) -> String {
1001	format!("git -C {repo} checkout \"{branch}\"")
1002	}
1003
1004	/// The command a host runs to report the commit it has checked out, for the
1005	/// release preflight barrier.
1006	pub fn git_rev_parse_cmd(repo: &str) -> String {
1007	format!("git -C {repo} rev-parse HEAD")
1008	}
1009
1010	/// Expand a leading `~/` to `$HOME`. Paths in the topology are written with `~`.
1011	pub fn expand_tilde(p: &str) -> PathBuf {
1012	if let Some(rest) = p.strip_prefix("~/")
1013	&& let Ok(home) = std::env::var("HOME")
1014	{
1015	return Path::new(&home).join(rest);
1016	}
1017	PathBuf::from(p)
1018	}
1019
1020	/// Reject a glob that carries shell command metacharacters. Path and wildcard
1021	/// characters (`/ . * ? [ ] ~` etc.) are fine — the pattern reaches a login
1022	/// shell to be expanded — but a `;` or `$(...)` must not ride along and run.
1023	/// Not a privilege boundary (a recipe already runs arbitrary shell via `sh_ok`)
1024	/// but it keeps a malformed pattern from turning into a command. Shared by
1025	/// `collect` and `resolve_artifact`.
1026	fn ensure_glob_safe(glob: &str) -> Result<()> {
1027	anyhow::ensure!(
1028	!glob.chars().any(\|c\| matches!(
1029	c,
1030	';' \| '&' \| '\|' \| '$' \| '`' \| '\'' \| '"' \| '\\' \| ' ' \| '\n' \| '(' \| ')' \| '<' \| '>'
1031	)),
1032	"glob `{glob}` contains shell metacharacters"
1033	);
1034	Ok(())
1035	}
1036
1037	/// Decide the single artifact a glob resolves to from a newline-separated
1038	/// listing of the paths that matched it.
1039	///
1040	/// The recipes used to select an artifact with `ls -t <glob> \| head -1` and
1041	/// guard only on an empty string, so a non-zero `ls` slipped past quietly and a
1042	/// stale newest-by-mtime file could win. This is the strict replacement: it
1043	/// demands exactly one match. Zero matches fail when `required` (return `""`
1044	/// when optional); more than one is always an error rather than an arbitrary
1045	/// newest-wins pick, because an ambiguous match means the build left stale
1046	/// artifacts behind and the wrong one could ship.
1047	fn resolve_artifact_match(listing: &str, glob: &str, required: bool) -> Result<String> {
1048	let matches: Vec<&str> = listing
1049	.lines()
1050	.map(str::trim)
1051	.filter(\|l\| !l.is_empty())
1052	.collect();
1053	match matches.as_slice() {
1054	[] if required => anyhow::bail!("no artifact matched glob `{glob}`"),
1055	[] => Ok(String::new()),
1056	[one] => Ok((*one).to_string()),
1057	many => anyhow::bail!(
1058	"glob `{glob}` is ambiguous: {} artifacts matched ({}). \
1059	The build left more than one behind; clean stale artifacts so exactly one remains.",
1060	many.len(),
1061	many.join(", ")
1062	),
1063	}
1064	}
1065
1066	/// Build a Rhai engine with the host API bound to `ctx`. Sandboxed: recipes
1067	/// touch the outside world only through these functions.
1068	pub fn build_engine(ctx: &Arc<RecipeCtx>) -> Engine {
1069	let mut engine = Engine::new();
1070	// Defensive caps — recipes are first-party but bound the blast radius.
1071	engine.set_max_operations(5_000_000);
1072	engine.set_max_call_levels(64);
1073	engine.set_max_string_size(0);
1074
1075	// --- step(name) ---
1076	{
1077	let ctx = ctx.clone();
1078	engine.register_fn(
1079	"step",
1080	move \|name: &str\| -> Result<(), Box<EvalAltResult>> {
1081	let step: Step = name.parse().map_err(rhai_err)?;
1082	ctx.begin_step(step).map_err(rhai_err)
1083	},
1084	);
1085	}
1086
1087	// --- sh(host, cmd) -> #{ code, stdout_tail } ---
1088	//
1089	// The branch-on-exit-code primitive: the recipe OWNS the outcome. A non-zero
1090	// exit is returned, not raised, and does NOT fail the step or bar publish —
1091	// use this only when the recipe inspects `code` and decides. For a command
1092	// that must succeed (build/sign/etc.), use `sh_ok`, which fails the step (and
1093	// therefore bars publish via the failed-step ledger) on a non-zero exit.
1094	{
1095	let ctx = ctx.clone();
1096	engine.register_fn(
1097	"sh",
1098	move \|host: &str, cmd: &str\| -> Result<Map, Box<EvalAltResult>> {
1099	let (code, tail) = ctx.run(host, cmd).map_err(rhai_err)?;
1100	let mut m = Map::new();
1101	m.insert("code".into(), (code as i64).into());
1102	m.insert("stdout_tail".into(), tail.into());
1103	Ok(m)
1104	},
1105	);
1106	}
1107
1108	// --- sh_ok(host, cmd): run + assert exit 0 (the must-succeed primitive) ---
1109	//
1110	// A non-zero exit fails the current step (added to the publish-barring
1111	// ledger) and aborts the recipe, so an artifact is never shipped after a
1112	// must-succeed command failed.
1113	{
1114	let ctx = ctx.clone();
1115	engine.register_fn(
1116	"sh_ok",
1117	move \|host: &str, cmd: &str\| -> Result<(), Box<EvalAltResult>> {
1118	let (code, _) = ctx.run(host, cmd).map_err(rhai_err)?;
1119	if code != 0 {
1120	// Attribute the failure to the current step explicitly so the
1121	// ledger bars publish even if a future caller swallowed the error.
1122	ctx.fail_current_step();
1123	return Err(rhai_err(format!(
1124	"command on `{host}` exited {code}: {cmd}"
1125	)));
1126	}
1127	Ok(())
1128	},
1129	);
1130	}
1131
1132	// --- resolve_artifact(host, glob) -> path: the ONE artifact matching glob ---
1133	//
1134	// The artifact-selection primitive. Replaces `sh(host, "ls -t <glob> \| head
1135	// -1").stdout_tail.trim()` guarded on an empty string, which let a non-zero
1136	// `ls` pass quietly and a stale newest-by-mtime file win. This resolves the
1137	// glob on the host and demands exactly one match: zero matches or more than
1138	// one both throw (an ambiguous match means the build left stale artifacts,
1139	// and silently picking the newest is how the wrong bytes ship). Use
1140	// `resolve_artifact_opt` for an artifact that may legitimately be absent.
1141	{
1142	let ctx = ctx.clone();
1143	engine.register_fn(
1144	"resolve_artifact",
1145	move \|host: &str, glob: &str\| -> Result<String, Box<EvalAltResult>> {
1146	ctx.resolve_artifact(host, glob, true).map_err(rhai_err)
1147	},
1148	);
1149	}
1150
1151	// --- resolve_artifact_opt(host, glob) -> path \| "": zero-or-one match ---
1152	//
1153	// Same strict resolution as `resolve_artifact` but tolerates zero matches
1154	// (returns ""); more than one is still an error. For optional outputs like a
1155	// `.deb` or an updater bundle a recipe collects only when present.
1156	{
1157	let ctx = ctx.clone();
1158	engine.register_fn(
1159	"resolve_artifact_opt",
1160	move \|host: &str, glob: &str\| -> Result<String, Box<EvalAltResult>> {
1161	ctx.resolve_artifact(host, glob, false).map_err(rhai_err)
1162	},
1163	);
1164	}
1165
1166	// --- log(msg): operator-visible line into the current step's tail ---
1167	{
1168	let ctx = ctx.clone();
1169	engine.register_fn("log", move \|msg: &str\| -> Result<(), Box<EvalAltResult>> {
1170	let sink = ctx.ensure_step().map_err(rhai_err)?;
1171	let line = format!("[recipe] {msg}\n");
1172	ctx.rt.block_on(async {
1173	use ops_core::remote::LogSink;
1174	sink.lock().await.write_chunk(line.as_bytes()).await;
1175	});
1176	Ok(())
1177	});
1178	}
1179
1180	// --- version_of(app) -> string ---
1181	{
1182	let ctx = ctx.clone();
1183	engine.register_fn(
1184	"version_of",
1185	move \|app: &str\| -> Result<String, Box<EvalAltResult>> {
1186	// Only the current app is in scope; cross-app reads aren't needed.
1187	if app != ctx.app.as_str() {
1188	return Err(rhai_err(format!(
1189	"version_of: `{app}` is not the app being built"
1190	)));
1191	}
1192	Ok(ctx.version.to_string())
1193	},
1194	);
1195	}
1196
1197	// --- version() -> string: the version being built (no-arg form) ---
1198	{
1199	let ctx = ctx.clone();
1200	engine.register_fn("version", move \|\| -> String { ctx.version.to_string() });
1201	}
1202
1203	// --- build_host() -> string: the host this target builds on ---
1204	{
1205	let ctx = ctx.clone();
1206	engine.register_fn("build_host", move \|\| -> String { ctx.build_host.clone() });
1207	}
1208
1209	// --- repo() -> string: the app's checkout path (`~`-prefixed) ---
1210	{
1211	let ctx = ctx.clone();
1212	engine.register_fn("repo", move \|\| -> String { ctx.repo.clone() });
1213	}
1214
1215	// --- checkout_sha(host) -> sha: pin this host to the release tag and report
1216	// its commit. Replaces a recipe's `git pull --ff-only`, which builds
1217	// whatever `main` is at pull time; the daemon also runs the same pin as
1218	// a cross-host preflight barrier before any target builds. ---
1219	{
1220	let ctx = ctx.clone();
1221	engine.register_fn(
1222	"checkout_sha",
1223	move \|host: &str\| -> Result<String, Box<EvalAltResult>> {
1224	ctx.checkout_sha(host).map_err(rhai_err)
1225	},
1226	);
1227	}
1228
1229	// --- crate_preflight() -> string: verify this crate is safe to publish,
1230	// or abort the run. Everything it checks is immutable once published:
1231	// crates.io versions can be yanked but never edited, so a wrong
1232	// repository URL is permanent. pter 0.1.0 shipped with a dead one. ---
1233	{
1234	let ctx = ctx.clone();
1235	engine.register_fn(
1236	"crate_preflight",
1237	move \|\| -> Result<String, Box<EvalAltResult>> {
1238	let repo = expand_tilde(&ctx.repo);
1239
1240	let out = std::process::Command::new("cargo")
1241	.args(["metadata", "--no-deps", "--format-version", "1"])
1242	.current_dir(&repo)
1243	.output()
1244	.map_err(\|e\| format!("running cargo metadata in {}: {e}", repo.display()))?;
1245	if !out.status.success() {
1246	return Err(format!(
1247	"cargo metadata failed in {}: {}",
1248	repo.display(),
1249	String::from_utf8_lossy(&out.stderr).trim()
1250	)
1251	.into());
1252	}
1253	let meta = crate_meta_from_json(&String::from_utf8_lossy(&out.stdout))
1254	.map_err(\|e\| e.to_string())?;
1255
1256	// The real question is not whether a page renders but whether a
1257	// stranger with no credentials can fetch the source, so ask git.
1258	let clonable = meta.repository.as_ref().is_some_and(\|url\| {
1259	std::process::Command::new("git")
1260	.args(["ls-remote", url])
1261	.env("GIT_TERMINAL_PROMPT", "0")
1262	.output()
1263	.is_ok_and(\|o\| o.status.success())
1264	});
1265
1266	// Ask the publishing host whether cargo has credentials, rather
1267	// than moving the token anywhere. It stays in cargo's own 0600
1268	// store; a shell line carrying it would be visible in `ps`.
1269	// An exit code answers "are there credentials"; an Err answers
1270	// "the question could not be asked". Collapsing the second into
1271	// the first reported a capability denial as "no crates.io
1272	// credentials", which sent a real diagnosis three rounds the
1273	// wrong way. A check that cannot run is not a failed check.
1274	let creds =
1275	ctx.run(
1276	&ctx.build_host.clone(),
1277	"cargo login --help >/dev/null 2>&1 && \
1278	test -s \"${CARGO_HOME:-$HOME/.cargo}/credentials.toml\" \
1279	\|\| test -s \"${CARGO_HOME:-$HOME/.cargo}/credentials\"",
1280	)
1281	.map_err(\|e\| {
1282	format!(
1283	"could not check crates.io credentials on `{}`: {e}",
1284	ctx.build_host
1285	)
1286	})?
1287	.0 == 0;
1288
1289	let published = RecipeCtx::published_versions(&meta.name);
1290	let problems = crate_publish_problems(&meta, clonable, &published, creds);
1291	if !problems.is_empty() {
1292	return Err(format!(
1293	"{} {} is not safe to publish:\n - {}",
1294	meta.name,
1295	meta.version,
1296	problems.join("\n - ")
1297	)
1298	.into());
1299	}
1300	Ok(format!("{} {} passed preflight", meta.name, meta.version))
1301	},
1302	);
1303	}
1304
1305	// --- feature_flags() -> string: `--features a,b`, or "" when the app
1306	// declares none. Returns the whole flag rather than a bare list so an
1307	// app with no features cannot produce a dangling `--features`. ---
1308	{
1309	let ctx = ctx.clone();
1310	engine.register_fn("feature_flags", move \|\| -> String {
1311	if ctx.features.is_empty() {
1312	String::new()
1313	} else {
1314	format!("--features {}", ctx.features.join(","))
1315	}
1316	});
1317	}
1318
1319	// --- target() / platform() / arch(): the target axis, for one per-platform
1320	// recipe to branch on arch (bundle paths differ between x86_64/aarch64). ---
1321	{
1322	let ctx = ctx.clone();
1323	engine.register_fn("target", move \|\| -> String { ctx.target.to_string() });
1324	}
1325	{
1326	let ctx = ctx.clone();
1327	engine.register_fn("platform", move \|\| -> String {
1328	ctx.target.platform.as_str().to_string()
1329	});
1330	}
1331	{
1332	let ctx = ctx.clone();
1333	engine.register_fn("arch", move \|\| -> String {
1334	ctx.target.arch.as_str().to_string()
1335	});
1336	}
1337
1338	// --- secret(key) -> string (file under secrets_root; never logged) ---
1339	{
1340	let ctx = ctx.clone();
1341	engine.register_fn("secret", move \|key: &str\| -> Result<String, Box<EvalAltResult>> {
1342	// Guard against traversal out of secrets_root. Require every path
1343	// component to be `Normal` (rejects `..`, `.`, absolute roots and
1344	// drive prefixes) and forbid backslashes (a literal filename char on
1345	// Linux, but a separator elsewhere) — the per-component strength of
1346	// Sando's `safe()`. A multi-segment key like `app/token` is still
1347	// allowed; `foo..bar` (a legit filename) is no longer falsely blocked.
1348	let safe = !key.is_empty()
1349	&& !key.contains('\\')
1350	&& std::path::Path::new(key)
1351	.components()
1352	.all(\|c\| matches!(c, std::path::Component::Normal(_)));
1353	if !safe {
1354	return Err(rhai_err(
1355	"secret key must be a relative path under secrets_root (no `..`, `.`, absolute paths, or backslashes)",
1356	));
1357	}
1358	let path = ctx.cfg.secrets_root.join(key);
1359	std::fs::read_to_string(&path)
1360	.map(\|s\| s.trim_end().to_string())
1361	.map_err(\|e\| rhai_err(format!("secret `{key}`: {e}")))
1362	});
1363	}
1364
1365	// --- env(host, key) -> string ---
1366	{
1367	let ctx = ctx.clone();
1368	engine.register_fn(
1369	"env",
1370	move \|host: &str, key: &str\| -> Result<String, Box<EvalAltResult>> {
1371	// The key is interpolated into a `${...}` shell expansion, so it must
1372	// be a bare shell identifier — anything else (quotes, `}`, `$`, `;`)
1373	// could break out and run arbitrary commands on the host. Validate
1374	// before building the command; this is the one env read that can't
1375	// sh-quote its argument (a quoted var name doesn't expand).
1376	if key.is_empty()
1377	\|\| !key
1378	.chars()
1379	.next()
1380	.is_some_and(\|c\| c == '_' \|\| c.is_ascii_alphabetic())
1381	\|\| !key.chars().all(\|c\| c == '_' \|\| c.is_ascii_alphanumeric())
1382	{
1383	return Err(rhai_err(format!(
1384	"env name `{key}` must be a shell identifier ([A-Za-z_][A-Za-z0-9_]*)"
1385	)));
1386	}
1387	// Read via the shell so it works on remote hosts too.
1388	let (code, tail) = ctx
1389	.run(host, &format!("printf '%s' \"${{{key}}}\""))
1390	.map_err(rhai_err)?;
1391	if code != 0 {
1392	return Err(rhai_err(format!("env `{key}` on `{host}` failed")));
1393	}
1394	Ok(tail.trim().to_string())
1395	},
1396	);
1397	}
1398
1399	// --- collect(host, glob, app, version): pull artifacts to dist_root ---
1400	{
1401	let ctx = ctx.clone();
1402	engine.register_fn(
1403	"collect",
1404	move \|host: &str,
1405	glob: &str,
1406	app: &str,
1407	version: &str\|
1408	-> Result<(), Box<EvalAltResult>> {
1409	ctx.collect(host, glob, app, version).map_err(rhai_err)
1410	},
1411	);
1412	}
1413
1414	// --- publish(channel, app, target, version, artifact, meta) ---
1415	{
1416	let ctx = ctx.clone();
1417	engine.register_fn(
1418	"publish",
1419	move \|channel: &str,
1420	app: &str,
1421	target: &str,
1422	version: &str,
1423	artifact: &str,
1424	meta: Map\|
1425	-> Result<String, Box<EvalAltResult>> {
1426	ctx.publish(channel, app, target, version, artifact, &meta)
1427	.map_err(rhai_err)
1428	},
1429	);
1430	}
1431
1432	// --- deploy(binary) -> summary: install a service binary and restart its
1433	// unit. The terminal step for `kind = "service"`, the counterpart of
1434	// `publish` for something that is run rather than distributed.
1435	//
1436	// Takes only the binary's path on the build host: where it lands, on
1437	// which machine, and which unit restarts all come from the `[[deploy]]`
1438	// entry for the target already being built. A recipe cannot deploy the
1439	// aarch64 binary to the x86_64 box by naming the wrong host, because it
1440	// never names a host at all.
1441	{
1442	let ctx = ctx.clone();
1443	engine.register_fn(
1444	"deploy",
1445	move \|binary: &str\| -> Result<String, Box<EvalAltResult>> {
1446	ctx.deploy(binary).map_err(rhai_err)
1447	},
1448	);
1449	}
1450
1451	// --- deploy_host() -> string: the service host's ssh destination, so a
1452	// recipe can run its own assertions there (`sh_ok(deploy_host(), ...)`).
1453	// Commands run through it while the `deploy` step is open, so they are
1454	// gated on the deploy grant like the install itself. ---
1455	{
1456	let ctx = ctx.clone();
1457	engine.register_fn(
1458	"deploy_host",
1459	move \|\| -> Result<String, Box<EvalAltResult>> {
1460	ctx.deploy_target()
1461	.map(\|d\| d.host.clone())
1462	.map_err(rhai_err)
1463	},
1464	);
1465	}
1466
1467	// --- service_name() / install_path() / health_url(): the rest of the
1468	// `[[deploy]]` entry, so a recipe asserts against the configured values
1469	// rather than repeating them as literals that can drift. `health_url`
1470	// is "" when unset. ---
1471	{
1472	let ctx = ctx.clone();
1473	engine.register_fn(
1474	"service_name",
1475	move \|\| -> Result<String, Box<EvalAltResult>> {
1476	ctx.deploy_target()
1477	.map(\|d\| d.service.clone())
1478	.map_err(rhai_err)
1479	},
1480	);
1481	}
1482	{
1483	let ctx = ctx.clone();
1484	engine.register_fn(
1485	"install_path",
1486	move \|\| -> Result<String, Box<EvalAltResult>> {
1487	ctx.deploy_target()
1488	.map(\|d\| d.install_path.clone())
1489	.map_err(rhai_err)
1490	},
1491	);
1492	}
1493	{
1494	let ctx = ctx.clone();
1495	engine.register_fn(
1496	"health_url",
1497	move \|\| -> Result<String, Box<EvalAltResult>> {
1498	ctx.deploy_target()
1499	.map(\|d\| d.health_url.clone().unwrap_or_default())
1500	.map_err(rhai_err)
1501	},
1502	);
1503	}
1504
1505	// --- glibc_check(binary) -> string: assert the build host did not produce
1506	// a binary the service host's glibc is too old to exec. Aborts the run
1507	// if it did; returns "needs X, host has Y" for the log if it did not. ---
1508	{
1509	let ctx = ctx.clone();
1510	engine.register_fn(
1511	"glibc_check",
1512	move \|binary: &str\| -> Result<String, Box<EvalAltResult>> {
1513	let (needs, has) = ctx.glibc_check(binary).map_err(rhai_err)?;
1514	Ok(format!(
1515	"glibc: binary needs {needs}, service host has {has}"
1516	))
1517	},
1518	);
1519	}
1520
1521	// --- macOS signing helpers. They dispatch through the named host's
1522	// executor like any other step; when that host is the mac (transport =
1523	// "agent"), codesign/notarize/staple ride the in-session `AgentRpc`
1524	// transport — the only security session where the Developer ID key is
1525	// usable (design §7 "THE WALL"). Capability-gated by the host's `sign`
1526	// grant. ---
1527	register_macos_fns(&mut engine, ctx);
1528
1529	engine
1530	}
1531
1532	/// Highest `GLIBC_x.y` version referenced by a built binary, and the glibc a
1533	/// host actually has, both parsed from the text the commands print.
1534	///
1535	/// Native-per-architecture builds remove the cross-compile hazard `deploy.sh`
1536	/// was written against, but not this one: fw13 tracks a newer glibc than the
1537	/// Ubuntu 24.04 box in Hetzner, so a binary built here can reference a symbol
1538	/// version that box does not have and fail at exec — after the unit has already
1539	/// been restarted onto it. Comparing the two before the install is what makes
1540	/// that a failed step instead of a downed service.
1541	fn max_glibc_symbol(objdump_out: &str) -> Option<(u64, u64)> {
1542	objdump_out
1543	.split(\|c: char\| !(c.is_ascii_digit() \|\| c == '.' \|\| c == '_' \|\| c.is_ascii_alphabetic()))
1544	.filter_map(\|tok\| tok.strip_prefix("GLIBC_"))
1545	.filter_map(parse_glibc_version)
1546	.max()
1547	}
1548
1549	/// Parse `2.39` (or `2.39.1`, keeping major/minor) into a comparable pair.
1550	fn parse_glibc_version(s: &str) -> Option<(u64, u64)> {
1551	let mut parts = s.split('.');
1552	let major = parts.next()?.parse().ok()?;
1553	let minor = parts.next()?.parse().ok()?;
1554	Some((major, minor))
1555	}
1556
1557	/// The glibc version out of `ldd --version`'s first line, whose tail is the
1558	/// version however the distro decorates the rest (`ldd (Ubuntu GLIBC
1559	/// 2.39-0ubuntu8.8) 2.39`).
1560	fn glibc_from_ldd(ldd_out: &str) -> Option<(u64, u64)> {
1561	let first = ldd_out.lines().find(\|l\| !l.trim().is_empty())?;
1562	parse_glibc_version(first.split_whitespace().last()?)
1563	}
1564
1565	impl RecipeCtx {
1566	/// This target's install destination, or an error naming why there is none.
1567	fn deploy_target(&self) -> Result<&DeployTarget> {
1568	self.deploy.as_ref().ok_or_else(\|\| {
1569	anyhow::anyhow!(
1570	"no deploy destination for {} {}: the app is `kind = \"{}\"`, and only a \
1571	service declares [[deploy]] entries",
1572	self.app,
1573	self.target,
1574	match self.kind {
1575	Kind::App => "app",
1576	Kind::Library => "library",
1577	Kind::Service => "service",
1578	}
1579	)
1580	})
1581	}
1582
1583	/// Compare the built binary's glibc requirement against the service host's.
1584	/// Returns the two versions for the recipe to log.
1585	fn glibc_check(self: &Arc<Self>, binary: &str) -> Result<(String, String)> {
1586	let d = self.deploy_target()?.clone();
1587	// `objdump -T` on the build host; no symbols at all (a static binary)
1588	// means nothing to check, which is a pass rather than a failure.
1589	let (code, out) = self.run(
1590	&self.build_host.clone(),
1591	&format!(
1592	"objdump -T {binary} 2>/dev/null \| grep -o 'GLIBC_[0-9.]*' \| sort -uV \|\| true"
1593	),
1594	)?;
1595	anyhow::ensure!(code == 0, "reading glibc symbols from {binary} failed");
1596	let Some(needs) = max_glibc_symbol(&out) else {
1597	return Ok(("none".into(), "n/a".into()));
1598	};
1599	let (code, ldd) = self.run(&d.host, "ldd --version")?;
1600	anyhow::ensure!(
1601	code == 0,
1602	"could not read glibc version on service host `{}`",
1603	d.host
1604	);
1605	let has = glibc_from_ldd(&ldd).ok_or_else(\|\| {
1606	anyhow::anyhow!(
1607	"could not parse glibc version from `ldd --version` on `{}`",
1608	d.host
1609	)
1610	})?;
1611	anyhow::ensure!(
1612	needs <= has,
1613	"binary needs glibc {}.{} but `{}` has {}.{} — it would fail to exec after the \
1614	unit restarted onto it. Build on a host no newer than the service host.",
1615	needs.0,
1616	needs.1,
1617	d.host,
1618	has.0,
1619	has.1,
1620	);
1621	Ok((
1622	format!("{}.{}", needs.0, needs.1),
1623	format!("{}.{}", has.0, has.1),
1624	))
1625	}
1626
1627	/// Install `binary` (a path on the BUILD host) onto the service host and
1628	/// restart its unit, via the privileged installer the host holds a scoped
1629	/// sudo grant for.
1630	///
1631	/// Bento never runs the install itself. It stages the bytes and calls a
1632	/// root script whose arguments are re-checked on the far side — the same
1633	/// shape as Sando's `install-companion.sh`, and for the same reason: the
1634	/// sudoers grant is then ONE auditable script rather than a broad
1635	/// `install`+`systemctl` grant on a production box.
1636	///
1637	/// Only the binary moves. Config is deliberately untouched: pom's
1638	/// `pom-astra.toml` / `pom-hetzner.toml` differ per instance, and prod's
1639	/// carried a `[targets.mnw.tests]` block the repo did not have. A deploy
1640	/// that copies config over is how that block gets silently deleted.
1641	fn deploy(self: &Arc<Self>, binary: &str) -> Result<String> {
1642	anyhow::ensure!(
1643	!self.is_cancelled(),
1644	"build superseded by a newer request; refusing to deploy"
1645	);
1646	// A failed earlier step bars a deploy exactly as it bars a publish. An
1647	// artifact that failed its gates must not reach a production host just
1648	// because the recipe kept running.
1649	let failed = self.failed_steps.lock().unwrap().clone();
1650	anyhow::ensure!(
1651	failed.is_empty(),
1652	"refusing to deploy {} {}: {} failed earlier in this run",
1653	self.app,
1654	self.version,
1655	failed
1656	.iter()
1657	.map(ToString::to_string)
1658	.collect::<Vec<_>>()
1659	.join(", "),
1660	);
1661	let d = self.deploy_target()?.clone();
1662	ensure_glob_safe(binary)?;
1663
1664	// Stage under a fixed root the installer also insists on, so "what was
1665	// checked" and "what is installed" cannot drift apart.
1666	let staged = format!("{DEPLOY_STAGING_ROOT}/{}", self.app);
1667	let staged_bin = format!("{staged}/{}", self.app);
1668	let deploy_exec = self.exec(&d.host)?;
1669	anyhow::ensure!(
1670	deploy_exec.capabilities().permits(&Action::Deploy),
1671	"service host `{}` is not granted the `deploy` capability",
1672	d.host
1673	);
1674
1675	self.run_ok(&d.host, &format!("mkdir -p {staged}"))?;
1676	if self.build_host_ssh == d.host {
1677	// Same box: the binary is already there. Routing it through the
1678	// daemon would be two transfers to end up where it started. This is
1679	// pom's aarch64 leg — astra builds it and astra runs it.
1680	self.run_ok(&d.host, &format!("cp -f {binary} {staged_bin}"))?;
1681	} else {
1682	// Build host -> daemon -> service host. Two hops because an executor
1683	// reaches one host; a direct host-to-host transport would mean the
1684	// build host holding a credential for the production box.
1685	let tmp = tempfile::tempdir().context("staging dir for deploy")?;
1686	let local = tmp.path().join(self.app.as_str());
1687	self.pull_for_deploy(binary, &local)?;
1688	let (dest, opts) = (PathBuf::from(&staged), SyncOpts::default());
1689	let dir = tmp.path().to_path_buf();
1690	self.run_bounded(&format!("stage {} on `{}`", self.app, d.host), async move {
1691	deploy_exec.push_dir(&dir, &dest, &opts).await
1692	})
1693	.with_context(\|\| format!("staging {} onto `{}`", self.app, d.host))?;
1694	}
1695
1696	// The privileged half. Every argument is re-validated by the script,
1697	// which is the thing actually holding the sudo grant.
1698	self.run_ok(
1699	&d.host,
1700	&format!(
1701	"{} {staged_bin} {} {}",
1702	self.cfg.deploy_installer, d.install_path, d.service
1703	),
1704	)?;
1705	Ok(format!(
1706	"{} {} installed at {} on `{}`; {} restarted",
1707	self.app, self.version, d.install_path, d.host, d.service
1708	))
1709	}
1710
1711	/// Fetch one file off a host into a daemon-local path for re-pushing.
1712	///
1713	/// A local build host is read directly: `fw13` is the daemon's own box, so
1714	/// the file is already on this filesystem. Routing it through the
1715	/// artifact-pull gate instead would demand a `pull_root` covering every repo
1716	/// a service could be built in — today that is `~/Code/Apps`, and pom lives
1717	/// in `~/Code/MNW`. Widening it to `~/Code` would put `_private`, the
1718	/// secrets root, inside the collectable tree. This is pom's x86_64 leg.
1719	fn pull_for_deploy(self: &Arc<Self>, remote: &str, local: &Path) -> Result<()> {
1720	let host = self.build_host.clone();
1721	let remote_path = expand_tilde(remote);
1722	if self.build_host_ssh == "local" \|\| self.build_host_ssh.is_empty() {
1723	std::fs::copy(&remote_path, local).with_context(\|\| {
1724	format!("staging {} from the daemon host", remote_path.display())
1725	})?;
1726	return Ok(());
1727	}
1728	let sync = self.host_sync(&host)?;
1729	let (src, dst, opts) = (remote_path, local.to_path_buf(), SyncOpts::default());
1730	self.run_bounded(&format!("fetch {remote} from `{host}`"), async move {
1731	sync.pull_file(&src, &dst, &opts).await
1732	})
1733	.with_context(\|\| format!("fetching {remote} from `{host}` to deploy"))
1734	}
1735
1736	/// `run`, failing the step on a non-zero exit. The Rust-side twin of the
1737	/// recipe's `sh_ok`, for commands the deploy machinery issues itself.
1738	fn run_ok(self: &Arc<Self>, host: &str, cmd: &str) -> Result<String> {
1739	let (code, tail) = self.run(host, cmd)?;
1740	if code != 0 {
1741	self.fail_current_step();
1742	anyhow::bail!("command on `{host}` exited {code}: {cmd}\n{tail}");
1743	}
1744	Ok(tail)
1745	}
1746
1747	fn collect(self: &Arc<Self>, host: &str, glob: &str, app: &str, version: &str) -> Result<()> {
1748	let dest = self.cfg.dist_root.join(app).join(version);
1749	let dest_s = dest.to_string_lossy().into_owned();
1750	// The glob reaches a remote login shell intact (that's what expands it),
1751	// so command metacharacters stay barred. Path/wildcard chars are fine.
1752	ensure_glob_safe(glob)?;
1753	std::fs::create_dir_all(&dest)
1754	.with_context(\|\| format!("creating collect dest {dest_s}"))?;
1755	// The SYNC transport, not the host's exec executor: artifacts move over
1756	// ssh/rsync even from an agent host, whose `/pull` is confined to a
1757	// narrow `pull_root` that deliberately excludes the repo checkout these
1758	// artifacts are built in (see `state::build_sync`). The daemon still
1759	// runs the transfer itself, as it always has.
1760	let sync = self.host_sync(host)?;
1761	let opts = SyncOpts::precompressed();
1762	// Bounded by the collect step's deadline (rsync of a multi-GiB artifact
1763	// can wedge on a stalled transport) and interruptible on supersession.
1764	let dest_pull = dest.clone();
1765	self.run_bounded(&format!("collect {glob} from `{host}`"), async move {
1766	sync.pull_glob(glob, &dest_pull, &opts).await
1767	})
1768	.with_context(\|\| format!("collect {glob} from `{host}`"))?;
1769	// Assert the version and hash every collected file. This is where a
1770	// stale artifact is caught: a file whose name embeds a different version
1771	// fails the collect (rather than silently winning a later glob), and the
1772	// sha256 recorded here is what `publish` writes into the release ledger.
1773	for entry in
1774	std::fs::read_dir(&dest).with_context(\|\| format!("listing collect dest {dest_s}"))?
1775	{
1776	let entry = entry?;
1777	let name = entry.file_name().to_string_lossy().into_owned();
1778	assert_artifact_version(&name, &self.version)?;
1779	if entry.file_type().is_ok_and(\|t\| t.is_file()) {
1780	let digest = sha256_file(&entry.path())?;
1781	self.artifact_hashes.lock().unwrap().insert(name, digest);
1782	}
1783	}
1784	// Best-effort size accounting for the event.
1785	events::emit(
1786	&self.events,
1787	Event::ArtifactCollected {
1788	app: self.app.clone(),
1789	target: self.target,
1790	path: dest_s,
1791	bytes: dir_size(&dest).unwrap_or(0),
1792	},
1793	);
1794	Ok(())
1795	}
1796
1797	/// The all-targets-green gate: err unless every declared target OTHER than
1798	/// the one publishing has a latest `target_runs` row of `ok` for this
1799	/// `(app, version)`. A sibling with no run, a running run, or a failed
1800	/// latest run all block the publish, naming what is not green.
1801	fn assert_siblings_green(self: &Arc<Self>, declared: &[Target]) -> Result<()> {
1802	let me = self.clone();
1803	let (app_s, ver_s) = (self.app.to_string(), self.version.to_string());
1804	let rows: Vec<(String, String)> = self.rt.block_on(async move {
1805	sqlx::query_as(
1806	"SELECT target, status FROM target_runs tr
1807	WHERE app = ?1 AND version = ?2
1808	AND id = (SELECT MAX(id) FROM target_runs
1809	WHERE app = ?1 AND version = ?2 AND target = tr.target)",
1810	)
1811	.bind(app_s)
1812	.bind(ver_s)
1813	.fetch_all(&me.pool)
1814	.await
1815	.unwrap_or_default()
1816	});
1817	let status_of = \|t: &Target\| -> Option<String> {
1818	let key = t.to_string();
1819	rows.iter()
1820	.find(\|(name, _)\| name == &key)
1821	.map(\|(_, s)\| s.clone())
1822	};
1823	let not_green: Vec<String> = declared
1824	.iter()
1825	.filter(\|t\| **t != self.target) // the publishing target is the last mile
1826	.filter(\|t\| status_of(t).as_deref() != Some("ok"))
1827	.map(\|t\| format!("{t} ({})", status_of(t).unwrap_or_else(\|\| "no run".into())))
1828	.collect();
1829	anyhow::ensure!(
1830	not_green.is_empty(),
1831	"all-targets-green gate: refusing to publish {} {} — not green: {}",
1832	self.app,
1833	self.version,
1834	not_green.join(", "),
1835	);
1836	Ok(())
1837	}
1838
1839	fn publish(
1840	self: &Arc<Self>,
1841	channel: &str,
1842	app: &str,
1843	target: &str,
1844	version: &str,
1845	artifact: &str,
1846	meta: &Map,
1847	) -> Result<String> {
1848	// Never let a superseded build ship. This is the last and most important
1849	// cooperative-cancel checkpoint: even if a long-running step finished
1850	// after supersession, the artifact must not reach the backend.
1851	anyhow::ensure!(
1852	!self.is_cancelled(),
1853	"build superseded by a newer request; refusing to publish"
1854	);
1855	// Opt-in all-targets-green gate: refuse a partial release. Every OTHER
1856	// declared target of this (app, version) must have a successful latest
1857	// run before this one ships, so macOS can't publish while windows is red
1858	// or still building. The publishing target itself is the last mile (it
1859	// reached publish, so its steps passed) and is not required to be green
1860	// in the ledger yet.
1861	if let Some(declared) = self.all_green_required.clone() {
1862	self.assert_siblings_green(&declared)?;
1863	}
1864	let backend = self
1865	.ota
1866	.get(channel)
1867	.ok_or_else(\|\| anyhow::anyhow!("unknown publish channel `{channel}`"))?;
1868	let target: Target = target.parse().map_err(\|e: String\| anyhow::anyhow!(e))?;
1869	let version = Version::parse(version).map_err(\|e\| anyhow::anyhow!(e))?;
1870	let app = AppId::new(app);
1871
1872	// The backend must actually handle this target (e.g. the desktop updater
1873	// disclaims iOS) — otherwise publish would push an artifact through a
1874	// backend that does not support it.
1875	anyhow::ensure!(
1876	backend.supports(target),
1877	"publish channel `{channel}` does not support target {target}",
1878	);
1879
1880	// Monotonicity: never publish a version that is not strictly newer than
1881	// the latest already published for this (app, target, channel). Without
1882	// this an older build could republish over a live newer release. The
1883	// `releases` column is TEXT, so compare by parsed semver precedence
1884	// (Version: Ord), not lexically.
1885	{
1886	let (app_s, target_s, chan_s) =
1887	(app.to_string(), target.to_string(), channel.to_string());
1888	let me = self.clone();
1889	let latest: Option<Version> = self.rt.block_on(async move {
1890	let rows: Vec<(String,)> = sqlx::query_as(
1891	"SELECT version FROM releases WHERE app = ? AND target = ? AND channel = ?",
1892	)
1893	.bind(app_s)
1894	.bind(target_s)
1895	.bind(chan_s)
1896	.fetch_all(&me.pool)
1897	.await
1898	.unwrap_or_default();
1899	rows.into_iter()
1900	.filter_map(\|(v,)\| Version::parse(&v).ok())
1901	.max()
1902	});
1903	if let Some(latest) = latest {
1904	anyhow::ensure!(
1905	version > latest,
1906	"refusing to publish {app} {version} to `{channel}` ({target}): \
1907	not newer than the last published {latest}",
1908	);
1909	}
1910	}
1911
1912	// Step-success ledger (the Bento analogue of Sando's gate fail-closed),
1913	// minted as an unforgeable PublishAuthority. `backend.publish` cannot be
1914	// called without one, so the unverified/post-failure ship path is sealed
1915	// at the type level rather than guarded by a separate runtime check.
1916	let authority = {
1917	let failed = self.failed_steps.lock().unwrap();
1918	let gatekeeper = *self.gatekeeper_ok.lock().unwrap();
1919	PublishAuthority::prove(target, failed.as_slice(), gatekeeper)?
1920	};
1921	let notes = meta
1922	.get("notes")
1923	.and_then(\|v\| v.clone().into_string().ok())
1924	.unwrap_or_default();
1925	// Resolve the artifact relative to the collected dist dir if not absolute.
1926	let artifact_path = {
1927	let p = PathBuf::from(artifact);
1928	if p.is_absolute() {
1929	p
1930	} else {
1931	self.cfg
1932	.dist_root
1933	.join(app.as_str())
1934	.join(version.to_string())
1935	.join(artifact)
1936	}
1937	};
1938	let rel = Release {
1939	app: &app,
1940	target,
1941	version: &version,
1942	notes,
1943	};
1944	let receipt = backend
1945	.publish(&rel, &artifact_path, &authority)
1946	.with_context(\|\| format!("publish to `{channel}`"))?;
1947	// Record for idempotency / monotonicity. This write is CHECKED, not
1948	// fire-and-forget: a swallowed failure here would silently re-arm the
1949	// monotonicity guard (which reads this same table), letting an older
1950	// version republish over a live release. Concurrent same-(app,target)
1951	// publishers can't race the read-then-insert because the latest-wins slot
1952	// (state::ActiveSlot) serializes them and a superseded run is cancelled
1953	// before it reaches publish.
1954	// The artifact's hash, recorded so the release ledger says exactly which
1955	// bytes shipped. Prefer the digest computed at `collect`; fall back to
1956	// hashing the file now (an absolute-path artifact never routed through
1957	// `collect`). A hash failure must not fail an already-published release,
1958	// so degrade to NULL rather than erroring.
1959	let artifact_hash: Option<String> = artifact_path
1960	.file_name()
1961	.and_then(\|n\| n.to_str())
1962	.and_then(\|n\| self.artifact_hashes.lock().unwrap().get(n).cloned())
1963	.or_else(\|\| sha256_file(&artifact_path).ok());
1964	let me = self.clone();
1965	let (app_s, target_s, ver_s, chan_s) = (
1966	app.to_string(),
1967	target.to_string(),
1968	version.to_string(),
1969	channel.to_string(),
1970	);
1971	self.rt
1972	.block_on(async move {
1973	sqlx::query(
1974	"INSERT OR IGNORE INTO releases (app, target, version, channel, artifact_hash, published_at)
1975	VALUES (?, ?, ?, ?, ?, ?)",
1976	)
1977	.bind(app_s)
1978	.bind(target_s)
1979	.bind(ver_s)
1980	.bind(chan_s)
1981	.bind(artifact_hash)
1982	.bind(Self::now())
1983	.execute(&me.pool)
1984	.await
1985	})
1986	.context("recording release in the idempotency ledger (artifact published but ledger write failed)")?;
1987	events::emit(
1988	&self.events,
1989	Event::PublishOk {
1990	app: self.app.clone(),
1991	target: self.target,
1992	channel: channel.to_string(),
1993	},
1994	);
1995	Ok(receipt)
1996	}
1997	}
1998
1999	fn dir_size(p: &Path) -> Option<i64> {
2000	let mut total = 0i64;
2001	for entry in std::fs::read_dir(p).ok()? {
2002	let entry = entry.ok()?;
2003	let md = entry.metadata().ok()?;
2004	if md.is_file() {
2005	total += md.len() as i64;
2006	} else if md.is_dir() {
2007	// Recurse so a bundle dir (a `.app`) reports its real size, not ~0.
2008	total += dir_size(&entry.path()).unwrap_or(0);
2009	}
2010	}
2011	Some(total)
2012	}
2013
2014	/// macOS signing/notarization host functions. Thin wrappers over the right
2015	/// shell incantations, dispatched through the named host's executor. On the mac
2016	/// host (`transport = "agent"`) they run via the in-session `ops-agent`, the only
2017	/// context where codesign can use the Developer ID key (design §7 "THE WALL"); a
2018	/// plain SSH session cannot. Each is gated by the host's `sign` capability.
2019	fn register_macos_fns(engine: &mut Engine, ctx: &Arc<RecipeCtx>) {
2020	{
2021	let ctx = ctx.clone();
2022	engine.register_fn(
2023	"verify_gatekeeper",
2024	move \|host: &str, path: &str\| -> Result<bool, Box<EvalAltResult>> {
2025	// spctl has no JSON mode, so assess on-host and decide there,
2026	// emitting an unambiguous sentinel as the final line. We match the
2027	// sentinel rather than substring-hunting `source=Notarized...` in a
2028	// 2000-char tail: truncation only drops the front, so the sentinel
2029	// is always present, and it can't be spoofed by spctl's own prose.
2030	// The full assess output is still streamed to the step log.
2031	let q = ops_core::remote::sh_quote(path);
2032	let cmd = format!(
2033	"out=$(spctl --assess -vv --type install {q} 2>&1); printf '%s\\n' \"$out\"; \
2034	printf '%s' \"$out\" \| grep -q 'source=Notarized Developer ID' \
2035	&& echo BENTO_GATEKEEPER_OK \|\| echo BENTO_GATEKEEPER_FAIL",
2036	);
2037	let (_, tail) = ctx.run(host, &cmd).map_err(rhai_err)?;
2038	let accepted = tail.contains("BENTO_GATEKEEPER_OK");
2039	// Record the verdict for the publish gate. A rejection also
2040	// fails the step, so the matrix shows red and `publish` is barred
2041	// even if the recipe ignores the returned bool.
2042	*ctx.gatekeeper_ok.lock().unwrap() = Some(accepted);
2043	if !accepted {
2044	ctx.fail_current_step();
2045	}
2046	Ok(accepted)
2047	},
2048	);
2049	}
2050	{
2051	let ctx = ctx.clone();
2052	engine.register_fn(
2053	"codesign",
2054	move \|host: &str, identity: &str, path: &str\| -> Result<(), Box<EvalAltResult>> {
2055	let cmd = format!(
2056	"codesign --force --options runtime --timestamp --sign {} {}",
2057	ops_core::remote::sh_quote(identity),
2058	ops_core::remote::sh_quote(path),
2059	);
2060	let (code, _) = ctx.run(host, &cmd).map_err(rhai_err)?;
2061	if code != 0 {
2062	return Err(rhai_err("codesign failed"));
2063	}
2064	Ok(())
2065	},
2066	);
2067	}
2068	{
2069	let ctx = ctx.clone();
2070	engine.register_fn(
2071	"staple",
2072	move \|host: &str, path: &str\| -> Result<(), Box<EvalAltResult>> {
2073	let (code, _) = ctx
2074	.run(
2075	host,
2076	&format!("xcrun stapler staple {}", ops_core::remote::sh_quote(path)),
2077	)
2078	.map_err(rhai_err)?;
2079	if code != 0 {
2080	return Err(rhai_err("stapler failed"));
2081	}
2082	Ok(())
2083	},
2084	);
2085	}
2086	{
2087	let ctx = ctx.clone();
2088	engine.register_fn(
2089	"notarize",
2090	move \|host: &str, path: &str\| -> Result<String, Box<EvalAltResult>> {
2091	ctx.notarize(host, path).map_err(rhai_err)
2092	},
2093	);
2094	}
2095	{
2096	let ctx = ctx.clone();
2097	engine.register_fn(
2098	"keychain_open",
2099	move \|host: &str, name: &str\| -> Result<(), Box<EvalAltResult>> {
2100	// The full build-keychain lifecycle lives in dist/build-keychain.sh
2101	// (design §7); this drives it by name so the recipe stays short.
2102	let (code, _) = ctx
2103	.run(
2104	host,
2105	&format!(
2106	". ~/.tauri/passwords.env && ./dist/build-keychain.sh open {}",
2107	ops_core::remote::sh_quote(name)
2108	),
2109	)
2110	.map_err(rhai_err)?;
2111	if code != 0 {
2112	return Err(rhai_err("keychain_open failed"));
2113	}
2114	Ok(())
2115	},
2116	);
2117	}
2118	{
2119	let ctx = ctx.clone();
2120	engine.register_fn(
2121	"keychain_close",
2122	move \|host: &str, name: &str\| -> Result<(), Box<EvalAltResult>> {
2123	let _ = ctx.run(
2124	host,
2125	&format!(
2126	"./dist/build-keychain.sh close {}",
2127	ops_core::remote::sh_quote(name)
2128	),
2129	);
2130	Ok(())
2131	},
2132	);
2133	}
2134	}
2135
2136	impl RecipeCtx {
2137	/// `xcrun notarytool submit --wait` with bounded retry (the one flaky,
2138	/// network-bound step). Emits `NotarizeRetry` per attempt.
2139	fn notarize(self: &Arc<Self>, host: &str, path: &str) -> Result<String> {
2140	const MAX_ATTEMPTS: u32 = 3;
2141	let backoff = self
2142	.cfg
2143	.notarize_backoff_secs
2144	.map_or(std::time::Duration::from_secs(15), \|s\| {
2145	std::time::Duration::from_secs(s)
2146	});
2147	let cmd = format!(
2148	". ~/.tauri/passwords.env && xcrun notarytool submit {} \
2149	--key \"$NOTARY_KEY\" --key-id \"$NOTARY_KEY_ID\" --issuer \"$NOTARY_ISSUER\" \
2150	--wait --output-format json",
2151	ops_core::remote::sh_quote(path),
2152	);
2153	let mut last = String::new();
2154	for attempt in 1..=MAX_ATTEMPTS {
2155	let (code, tail) = self.run(host, &cmd)?;
2156	if code == 0 && notary_accepted(&tail) {
2157	return Ok(tail);
2158	}
2159	last = tail;
2160	if attempt < MAX_ATTEMPTS {
2161	events::emit(
2162	&self.events,
2163	Event::NotarizeRetry {
2164	app: self.app.clone(),
2165	target: self.target,
2166	attempt,
2167	reason: format!("exit {code}"),
2168	},
2169	);
2170	self.rt.block_on(tokio::time::sleep(backoff));
2171	}
2172	}
2173	anyhow::bail!("notarization failed after {MAX_ATTEMPTS} attempts: {last}")
2174	}
2175	}
2176
2177	/// True iff `notarytool --output-format json` output reports `status: Accepted`.
2178	/// Isolates the JSON object (`{`..`}`) from any shell-sourcing noise and reads
2179	/// the typed `status` field, rather than substring-matching `"status":"Accepted"`
2180	/// in a possibly-truncated tail — which could match the literal inside an error
2181	/// message or miss it across a whitespace variant. Fails closed: any parse or
2182	/// field miss returns false.
2183	fn notary_accepted(output: &str) -> bool {
2184	let (Some(start), Some(end)) = (output.find('{'), output.rfind('}')) else {
2185	return false;
2186	};
2187	if start > end {
2188	return false;
2189	}
2190	serde_json::from_str::<serde_json::Value>(&output[start..=end])
2191	.ok()
2192	.and_then(\|v\| {
2193	v.get("status")
2194	.and_then(\|s\| s.as_str())
2195	.map(\|s\| s.eq_ignore_ascii_case("accepted"))
2196	})
2197	.unwrap_or(false)
2198	}
2199
2200	#[cfg(test)]
2201	mod tests {
2202	use super::*;
2203
2204	/// Run 3 S1: once the cooperative cancel flag is set (a newer build
2205	/// superseded this run), a step boundary refuses to proceed — the blocking
2206	/// recipe stops at the next `step()` instead of running on and publishing.
2207	#[tokio::test]
2208	async fn begin_step_bails_when_cancelled() {
2209	let dir = tempfile::tempdir().unwrap();
2210	let cfg = Arc::new(Config::for_tests(dir.path()));
2211	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2212	let cancel = Arc::new(AtomicBool::new(true));
2213	let ctx = Arc::new(RecipeCtx::new(
2214	AppId::new("demo"),
2215	Version::parse("0.1.0").unwrap(),
2216	"linux/x86_64".parse().unwrap(),
2217	"fw13".into(),
2218	"local".into(),
2219	"v0.1.0".into(),
2220	"/tmp".into(),
2221	vec![],
2222	Kind::App,
2223	1,
2224	Arc::new(std::collections::HashMap::new()),
2225	Arc::new(std::collections::HashMap::new()),
2226	None,
2227	pool,
2228	crate::events::channel(),
2229	cfg,
2230	Arc::new(OtaRegistry::standard("https://makenot.work")),
2231	tokio::runtime::Handle::current(),
2232	cancel.clone(),
2233	None,
2234	));
2235	// Cancelled: begin_step refuses before touching the DB (the ensure! is
2236	// ahead of any block_on, so this is safe to call from the async test).
2237	let err = ctx.begin_step(Step::Build).unwrap_err();
2238	assert!(err.to_string().contains("supersede"), "got: {err}");
2239	assert!(ctx.is_cancelled());
2240	}
2241
2242	/// `feature_flags()` returns a whole flag or nothing at all. An app with
2243	/// no declared features must not yield a bare `--features`, which would
2244	/// swallow the next word of the build command as its argument.
2245	#[tokio::test]
2246	async fn feature_flags_renders_whole_flag_or_empty() {
2247	async fn flags_for(features: Vec<String>) -> String {
2248	let dir = tempfile::tempdir().unwrap();
2249	let cfg = Arc::new(Config::for_tests(dir.path()));
2250	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2251	let ctx = Arc::new(RecipeCtx::new(
2252	AppId::new("demo"),
2253	Version::parse("0.1.0").unwrap(),
2254	"linux/x86_64".parse().unwrap(),
2255	"fw13".into(),
2256	"local".into(),
2257	"v0.1.0".into(),
2258	"/tmp".into(),
2259	features,
2260	Kind::App,
2261	1,
2262	Arc::new(std::collections::HashMap::new()),
2263	Arc::new(std::collections::HashMap::new()),
2264	None,
2265	pool,
2266	crate::events::channel(),
2267	cfg,
2268	Arc::new(OtaRegistry::standard("https://makenot.work")),
2269	tokio::runtime::Handle::current(),
2270	Arc::new(AtomicBool::new(false)),
2271	None,
2272	));
2273	let engine = build_engine(&ctx);
2274	engine.eval::<String>("feature_flags()").unwrap()
2275	}
2276
2277	assert_eq!(flags_for(vec![]).await, "");
2278	assert_eq!(
2279	flags_for(vec!["supernote".into()]).await,
2280	"--features supernote"
2281	);
2282	assert_eq!(
2283	flags_for(vec!["supernote".into(), "extra".into()]).await,
2284	"--features supernote,extra"
2285	);
2286	}
2287
2288	/// `secret(key)` reads a file under `secrets_root`, trims its trailing
2289	/// newline (the shape of a here-doc'd token file), and refuses any key that
2290	/// could escape the root. Covers the host-fn registered in `build_engine`.
2291	#[tokio::test]
2292	async fn secret_reads_under_root_and_blocks_traversal() {
2293	let dir = tempfile::tempdir().unwrap();
2294	let cfg = Config::for_tests(dir.path());
2295	// Seed a secret and one in a nested subdir; a trailing newline that the
2296	// read must strip.
2297	std::fs::create_dir_all(&cfg.secrets_root).unwrap();
2298	std::fs::write(cfg.secrets_root.join("token"), "s3cr3t\n").unwrap();
2299	std::fs::create_dir_all(cfg.secrets_root.join("app")).unwrap();
2300	std::fs::write(cfg.secrets_root.join("app").join("key"), "nested").unwrap();
2301	// Plant a file OUTSIDE the root that a traversal key would reach.
2302	std::fs::write(dir.path().join("outside"), "leak").unwrap();
2303
2304	let cfg = Arc::new(cfg);
2305	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2306	let ctx = Arc::new(RecipeCtx::new(
2307	AppId::new("demo"),
2308	Version::parse("0.1.0").unwrap(),
2309	"linux/x86_64".parse().unwrap(),
2310	"fw13".into(),
2311	"local".into(),
2312	"v0.1.0".into(),
2313	"/tmp".into(),
2314	vec![],
2315	Kind::App,
2316	1,
2317	Arc::new(std::collections::HashMap::new()),
2318	Arc::new(std::collections::HashMap::new()),
2319	None,
2320	pool,
2321	crate::events::channel(),
2322	cfg,
2323	Arc::new(OtaRegistry::standard("https://makenot.work")),
2324	tokio::runtime::Handle::current(),
2325	Arc::new(AtomicBool::new(false)),
2326	None,
2327	));
2328	let engine = build_engine(&ctx);
2329
2330	// Happy path: read + trim.
2331	assert_eq!(
2332	engine.eval::<String>(r#"secret("token")"#).unwrap(),
2333	"s3cr3t"
2334	);
2335	// A multi-segment relative key is allowed.
2336	assert_eq!(
2337	engine.eval::<String>(r#"secret("app/key")"#).unwrap(),
2338	"nested"
2339	);
2340
2341	// Traversal, absolute paths, and empty keys are refused BEFORE any read,
2342	// so the file one `..` above the root is never disclosed.
2343	for bad in [
2344	r#"secret("../outside")"#,
2345	r#"secret("/etc/passwd")"#,
2346	r#"secret("")"#,
2347	] {
2348	let err = engine.eval::<String>(bad).unwrap_err().to_string();
2349	assert!(
2350	err.contains("relative path under secrets_root"),
2351	"`{bad}` should hit the traversal guard, got: {err}"
2352	);
2353	}
2354	// A missing key surfaces the filesystem error, not a panic, and does not
2355	// trip the traversal guard (it is a legitimate relative path).
2356	let err = engine
2357	.eval::<String>(r#"secret("nope")"#)
2358	.unwrap_err()
2359	.to_string();
2360	assert!(err.contains("secret `nope`"), "got: {err}");
2361	}
2362
2363	/// The two failures that actually shipped, as regression cases.
2364	#[test]
2365	fn preflight_catches_a_dead_repository_url() {
2366	// pter 0.1.0: repository pointed at a URL that does not exist. It
2367	// published clean and the link is now permanent for that version.
2368	let meta = CrateMeta {
2369	name: "pter".into(),
2370	version: "0.1.0".into(),
2371	repository: Some("https://github.com/maxjacobson/pter".into()),
2372	description: Some("d".into()),
2373	licensed: true,
2374	};
2375	let problems = crate_publish_problems(&meta, false, &[], true);
2376	assert_eq!(problems.len(), 1, "{problems:?}");
2377	assert!(
2378	problems[0].contains("not publicly clonable"),
2379	"{problems:?}"
2380	);
2381
2382	// Same metadata, reachable URL: nothing to report.
2383	assert!(crate_publish_problems(&meta, true, &[], true).is_empty());
2384	}
2385
2386	#[test]
2387	fn preflight_requires_the_fields_crates_io_bakes_in() {
2388	let bare = CrateMeta {
2389	name: "x".into(),
2390	version: "0.1.0".into(),
2391	repository: None,
2392	description: None,
2393	licensed: false,
2394	};
2395	let problems = crate_publish_problems(&bare, false, &[], true);
2396	assert_eq!(problems.len(), 3, "{problems:?}");
2397	assert!(problems.iter().any(\|p\| p.contains("repository")));
2398	assert!(problems.iter().any(\|p\| p.contains("description")));
2399	assert!(problems.iter().any(\|p\| p.contains("license")));
2400	}
2401
2402	// A library's verify is a crate preflight, not a Gatekeeper check on a
2403	// signed bundle. Gating it on `gatekeeper` asked a Linux host for a macOS
2404	// code-signing capability it can never hold, so the step was denied before
2405	// it ran a command; the denial then surfaced as "no crates.io credentials",
2406	// which is not what went wrong. The only way to satisfy the old gate was to
2407	// declare the capability falsely in the topology.
2408	#[test]
2409	fn a_library_verify_is_not_gated_on_gatekeeper() {
2410	assert_eq!(
2411	action_for(Step::Verify, Kind::Library),
2412	Action::Build,
2413	"a crate preflight runs the build toolchain; that is what it needs",
2414	);
2415	assert_eq!(
2416	action_for(Step::Verify, Kind::App),
2417	Action::Observe(ObserveKind::Custom("gatekeeper".into())),
2418	"an app's verify still proves the bundle is signed and notarized",
2419	);
2420	}
2421
2422	// The capability the default host grant actually carries. Without this the
2423	// fix above is only true by inspection.
2424	#[test]
2425	fn a_default_host_can_run_a_library_verify_and_not_an_app_one() {
2426	let caps =
2427	ops_exec::CapabilitySet::from_tokens(["build", "package"], ["build-log", "artifact"]);
2428	assert!(caps.permits(&action_for(Step::Verify, Kind::Library)));
2429	assert!(!caps.permits(&action_for(Step::Verify, Kind::App)));
2430	}
2431
2432	// Every other step is a property of the step alone; verify is the one that
2433	// depends on what is being released.
2434	#[test]
2435	fn no_other_step_changes_with_the_kind() {
2436	for step in [
2437	Step::Checkout,
2438	Step::Prebuild,
2439	Step::Build,
2440	Step::Sign,
2441	Step::Notarize,
2442	Step::Staple,
2443	Step::Package,
2444	Step::Publish,
2445	Step::Collect,
2446	] {
2447	assert_eq!(
2448	action_for(step, Kind::App),
2449	action_for(step, Kind::Library),
2450	"{step:?} should not depend on the kind",
2451	);
2452	}
2453	}
2454
2455	#[test]
2456	fn preflight_rejects_republishing_the_same_version() {
2457	let meta = CrateMeta {
2458	name: "makeover".into(),
2459	version: "0.10.0".into(),
2460	repository: Some("https://git.sr.ht/~maxmj/makeover".into()),
2461	description: Some("d".into()),
2462	licensed: true,
2463	};
2464	let problems =
2465	crate_publish_problems(&meta, true, &["0.9.0".into(), "0.10.0".into()], true);
2466	assert_eq!(problems.len(), 1, "{problems:?}");
2467	assert!(problems[0].contains("already published"), "{problems:?}");
2468
2469	// An unreleased version against the same history is fine.
2470	let mut next = meta.clone();
2471	next.version = "0.11.0".into();
2472	assert!(crate_publish_problems(&next, true, &["0.10.0".into()], true).is_empty());
2473	}
2474
2475	/// Missing credentials must surface at preflight, not at the upload. The
2476	/// publish step is the irreversible one and runs last, after a full build
2477	/// and verify; discovering there that cargo cannot authenticate wastes the
2478	/// whole run.
2479	#[test]
2480	fn preflight_reports_missing_credentials_up_front() {
2481	let meta = CrateMeta {
2482	name: "makeover".into(),
2483	version: "0.11.0".into(),
2484	repository: Some("https://git.sr.ht/~maxmj/makeover".into()),
2485	description: Some("d".into()),
2486	licensed: true,
2487	};
2488	// Metadata is perfect; only the token is absent.
2489	let problems = crate_publish_problems(&meta, true, &[], false);
2490	assert_eq!(problems.len(), 1, "{problems:?}");
2491	assert!(problems[0].contains("credentials"), "{problems:?}");
2492	assert!(
2493	problems[0].contains("cargo login"),
2494	"should say how to fix it"
2495	);
2496
2497	// Present: nothing to report.
2498	assert!(crate_publish_problems(&meta, true, &[], true).is_empty());
2499	}
2500
2501	#[test]
2502	fn crate_meta_reads_cargo_metadata_json() {
2503	let raw = r#"{"packages":[{"name":"makeover","version":"0.10.0",
2504	"repository":"https://git.sr.ht/~maxmj/makeover","description":"themes",
2505	"license":"MIT"}]}"#;
2506	let m = crate_meta_from_json(raw).unwrap();
2507	assert_eq!(m.name, "makeover");
2508	assert_eq!(m.version, "0.10.0");
2509	assert!(m.licensed);
2510	assert_eq!(
2511	m.repository.as_deref(),
2512	Some("https://git.sr.ht/~maxmj/makeover")
2513	);
2514
2515	// license_file alone also counts as licensed; empty strings do not
2516	// count as present.
2517	let lf = r#"{"packages":[{"name":"x","version":"0.1.0","license":"",
2518	"license_file":"LICENSE","description":""}]}"#;
2519	let m = crate_meta_from_json(lf).unwrap();
2520	assert!(m.licensed);
2521	assert!(m.description.is_none());
2522	}
2523
2524	#[test]
2525	fn expand_tilde_handles_home() {
2526	unsafe { std::env::set_var("HOME", "/home/test") };
2527	assert_eq!(expand_tilde("~/Code/x"), PathBuf::from("/home/test/Code/x"));
2528	assert_eq!(expand_tilde("/abs/path"), PathBuf::from("/abs/path"));
2529	}
2530
2531	// ---- artifact resolution (the M3 silent-`sh` fix) ----
2532
2533	#[test]
2534	fn resolve_artifact_match_wants_exactly_one() {
2535	// Exactly one match: the path, trimmed of the listing's line noise.
2536	assert_eq!(
2537	resolve_artifact_match(" /d/App.AppImage \n", "*.AppImage", true).unwrap(),
2538	"/d/App.AppImage"
2539	);
2540	}
2541
2542	#[test]
2543	fn resolve_artifact_match_zero_depends_on_required() {
2544	// Required + zero matches is the case the old empty-string guard caught;
2545	// keep failing it.
2546	let err = resolve_artifact_match("", "*.dmg", true).unwrap_err();
2547	assert!(err.to_string().contains("no artifact matched"), "{err}");
2548	// Optional + zero matches resolves to empty (recipe skips the collect).
2549	assert_eq!(
2550	resolve_artifact_match("\n \n", "*.deb", false).unwrap(),
2551	""
2552	);
2553	}
2554
2555	#[test]
2556	fn resolve_artifact_match_rejects_ambiguous() {
2557	// Two matches must throw rather than silently pick one — this is the
2558	// stale-newest-mtime hole the audit flagged. Applies even when optional.
2559	for required in [true, false] {
2560	let err =
2561	resolve_artifact_match("/d/old.deb\n/d/new.deb\n", "*.deb", required).unwrap_err();
2562	let msg = err.to_string();
2563	assert!(msg.contains("ambiguous"), "{msg}");
2564	assert!(msg.contains("old.deb") && msg.contains("new.deb"), "{msg}");
2565	}
2566	}
2567
2568	#[test]
2569	fn ensure_glob_safe_allows_paths_bars_commands() {
2570	// Path and wildcard characters pass.
2571	assert!(ensure_glob_safe("~/Code/app/dist/*.AppImage").is_ok());
2572	assert!(ensure_glob_safe("/t/App_1.2.3-x86_64.dmg").is_ok());
2573	// A command substitution or separator does not.
2574	for bad in ["*.dmg; rm -rf /", "$(evil)", "a\|b", "a b"] {
2575	assert!(ensure_glob_safe(bad).is_err(), "should reject {bad:?}");
2576	}
2577	}
2578
2579	// ---- version resolution ----
2580
2581	#[test]
2582	fn version_from_tauri_json_reads_version() {
2583	assert_eq!(
2584	version_from_tauri_json(r#"{"version":"0.4.2"}"#).unwrap(),
2585	"0.4.2"
2586	);
2587	assert!(version_from_tauri_json(r#"{"productName":"X"}"#).is_err());
2588	}
2589
2590	#[test]
2591	fn version_from_cargo_toml_prefers_package_then_workspace() {
2592	// A leaf crate's [package].version.
2593	assert_eq!(
2594	version_from_cargo_toml("[package]\nname = \"x\"\nversion = \"0.5.0\"\n").unwrap(),
2595	"0.5.0"
2596	);
2597	// A workspace that sets [workspace.package].version.
2598	assert_eq!(
2599	version_from_cargo_toml("[workspace.package]\nversion = \"1.2.3\"\n").unwrap(),
2600	"1.2.3"
2601	);
2602	// No version anywhere -> error, not a panic.
2603	assert!(version_from_cargo_toml("[workspace]\nmembers = []\n").is_err());
2604	}
2605
2606	#[test]
2607	fn version_from_repo_default_and_explicit_paths() {
2608	let tmp = tempfile::tempdir().unwrap();
2609	let root = tmp.path();
2610
2611	// Tauri app: default path reads src-tauri/tauri.conf.json.
2612	let tauri = root.join("tauri");
2613	std::fs::create_dir_all(tauri.join("src-tauri")).unwrap();
2614	std::fs::write(
2615	tauri.join("src-tauri/tauri.conf.json"),
2616	r#"{"version":"0.4.2"}"#,
2617	)
2618	.unwrap();
2619	assert_eq!(
2620	version_from_repo(tauri.to_str().unwrap(), None)
2621	.unwrap()
2622	.to_string(),
2623	"0.4.2"
2624	);
2625
2626	// Workspace egui app: no tauri.conf.json, explicit version_path at a member crate.
2627	let ws = root.join("ws");
2628	std::fs::create_dir_all(ws.join("crates/app")).unwrap();
2629	std::fs::write(
2630	ws.join("Cargo.toml"),
2631	"[workspace]\nmembers = [\"crates/app\"]\n",
2632	)
2633	.unwrap();
2634	std::fs::write(
2635	ws.join("crates/app/Cargo.toml"),
2636	"[package]\nname = \"app\"\nversion = \"0.5.0\"\n",
2637	)
2638	.unwrap();
2639	assert_eq!(
2640	version_from_repo(ws.to_str().unwrap(), Some("crates/app/Cargo.toml"))
2641	.unwrap()
2642	.to_string(),
2643	"0.5.0"
2644	);
2645	}
2646
2647	// ---- version-source cross-check (drift preflight) ----
2648
2649	fn ver(s: &str) -> Version {
2650	Version::parse(s).unwrap()
2651	}
2652
2653	#[test]
2654	fn version_consistency_passes_when_all_sources_agree() {
2655	let tmp = tempfile::tempdir().unwrap();
2656	let repo = tmp.path();
2657	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2658	std::fs::write(
2659	repo.join("src-tauri/tauri.conf.json"),
2660	r#"{"version":"0.5.0"}"#,
2661	)
2662	.unwrap();
2663	std::fs::write(
2664	repo.join("Cargo.toml"),
2665	"[package]\nname = \"app\"\nversion = \"0.5.0\"\n",
2666	)
2667	.unwrap();
2668	check_version_consistency(repo.to_str().unwrap(), None, &ver("0.5.0")).unwrap();
2669	}
2670
2671	#[test]
2672	fn version_consistency_flags_tauri_vs_cargo_drift() {
2673	// The concrete finding: tauri.conf.json bumped to 0.5.0 but the root
2674	// Cargo.toml left at 0.4.0. version_from_repo (one file) would miss it.
2675	let tmp = tempfile::tempdir().unwrap();
2676	let repo = tmp.path();
2677	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2678	std::fs::write(
2679	repo.join("src-tauri/tauri.conf.json"),
2680	r#"{"version":"0.5.0"}"#,
2681	)
2682	.unwrap();
2683	std::fs::write(
2684	repo.join("Cargo.toml"),
2685	"[package]\nname = \"app\"\nversion = \"0.4.0\"\n",
2686	)
2687	.unwrap();
2688	let err =
2689	check_version_consistency(repo.to_str().unwrap(), None, &ver("0.5.0")).unwrap_err();
2690	let msg = format!("{err:#}");
2691	assert!(msg.contains("Cargo.toml says 0.4.0"), "{msg}");
2692	}
2693
2694	#[test]
2695	fn version_consistency_flags_explicit_version_the_repo_does_not_reflect() {
2696	let tmp = tempfile::tempdir().unwrap();
2697	let repo = tmp.path();
2698	std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2699	std::fs::write(
2700	repo.join("src-tauri/tauri.conf.json"),
2701	r#"{"version":"0.5.0"}"#,
2702	)
2703	.unwrap();
2704	let err =
2705	check_version_consistency(repo.to_str().unwrap(), None, &ver("9.9.9")).unwrap_err();
2706	assert!(format!("{err:#}").contains("building 9.9.9"));
2707	}
2708
2709	#[test]
2710	fn version_consistency_single_source_never_invents_drift() {
2711	// A virtual-workspace root Cargo.toml (no version) alongside the member
2712	// crate the version_path points at: only one real source, so no drift.
2713	let tmp = tempfile::tempdir().unwrap();
2714	let repo = tmp.path();
2715	std::fs::create_dir_all(repo.join("crates/app")).unwrap();
2716	std::fs::write(
2717	repo.join("Cargo.toml"),
2718	"[workspace]\nmembers = [\"crates/app\"]\n",
2719	)
2720	.unwrap();
2721	std::fs::write(
2722	repo.join("crates/app/Cargo.toml"),
2723	"[package]\nname = \"app\"\nversion = \"0.5.0\"\n",
2724	)
2725	.unwrap();
2726	check_version_consistency(
2727	repo.to_str().unwrap(),
2728	Some("crates/app/Cargo.toml"),
2729	&ver("0.5.0"),
2730	)
2731	.unwrap();
2732	}
2733
2734	// ---- artifact filename version assertion + hashing ----
2735
2736	#[test]
2737	fn versions_in_filename_extracts_only_real_semvers() {
2738	assert_eq!(
2739	versions_in_filename("GoingsOn_0.5.0_aarch64.dmg"),
2740	vec![ver("0.5.0")]
2741	);
2742	assert_eq!(
2743	versions_in_filename("AudioFiles-0.4.0-x86_64.AppImage"),
2744	vec![ver("0.4.0")]
2745	);
2746	// No three-part token ⇒ nothing (an updater manifest, a bare signature).
2747	assert!(versions_in_filename("latest.json").is_empty());
2748	assert!(versions_in_filename("app.sig").is_empty());
2749	}
2750
2751	#[test]
2752	fn assert_artifact_version_rejects_a_stale_artifact() {
2753	// The 0.4.0 file sitting in the output dir against a 0.5.0 build.
2754	let err =
2755	assert_artifact_version("AudioFiles-0.4.0-x86_64.AppImage", &ver("0.5.0")).unwrap_err();
2756	assert!(format!("{err:#}").contains("stale artifact"), "{err:#}");
2757	// The matching version passes, and a versionless file is not asserted.
2758	assert_artifact_version("GoingsOn_0.5.0_aarch64.dmg", &ver("0.5.0")).unwrap();
2759	assert_artifact_version("latest.json", &ver("0.5.0")).unwrap();
2760	}
2761
2762	/// The comparison that decides whether a binary can exec on the box that is
2763	/// about to be restarted onto it. Both sides are parsed out of text a tool
2764	/// printed, so both parsers are worth pinning: fw13 tracks a newer glibc
2765	/// than the Ubuntu 24.04 host in Hetzner, and getting this backwards means a
2766	/// dead unit rather than a failed step.
2767	#[test]
2768	fn glibc_versions_parse_from_what_the_tools_actually_print() {
2769	// `objdump -T \| grep -o 'GLIBC_[0-9.]*'` output: highest wins, and the
2770	// comparison is numeric (2.9 must not beat 2.34 lexically).
2771	let objdump = "GLIBC_2.2.5\nGLIBC_2.34\nGLIBC_2.9\nGLIBC_2.17\n";
2772	assert_eq!(max_glibc_symbol(objdump), Some((2, 34)));
2773	// A static binary references none: nothing to check.
2774	assert_eq!(max_glibc_symbol(""), None);
2775
2776	// `ldd --version` first line, however the distro decorates it.
2777	assert_eq!(
2778	glibc_from_ldd("ldd (Ubuntu GLIBC 2.39-0ubuntu8.8) 2.39\nCopyright...\n"),
2779	Some((2, 39))
2780	);
2781	assert_eq!(
2782	glibc_from_ldd("ldd (GNU libc) 2.41\nCopyright (C) 2025\n"),
2783	Some((2, 41))
2784	);
2785	assert_eq!(glibc_from_ldd(""), None);
2786	}
2787
2788	/// A binary needing MORE than the host has is the failure this check exists
2789	/// for; equal and less are both fine (glibc symbol versioning is backward
2790	/// compatible, so an older requirement runs on a newer host).
2791	#[test]
2792	fn glibc_requirement_is_satisfied_by_equal_or_newer_only() {
2793	let needs = max_glibc_symbol("GLIBC_2.41").unwrap();
2794	assert!(needs > glibc_from_ldd("ldd (Ubuntu GLIBC 2.39) 2.39").unwrap());
2795	assert!(needs <= glibc_from_ldd("ldd (GNU libc) 2.41").unwrap());
2796	assert!(needs <= glibc_from_ldd("ldd (GNU libc) 2.42").unwrap());
2797	assert!(needs <= glibc_from_ldd("ldd (GNU libc) 3.0").unwrap());
2798	}
2799
2800	/// Every deploy host function fails with the app's KIND as the reason when
2801	/// there is no destination, rather than with a missing-host error from
2802	/// somewhere deeper. A recipe calling `deploy()` on a library is a recipe
2803	/// written against the wrong kind, and the message should say so.
2804	#[tokio::test]
2805	async fn deploy_host_fns_explain_a_missing_destination_by_kind() {
2806	let dir = tempfile::tempdir().unwrap();
2807	let cfg = Arc::new(Config::for_tests(dir.path()));
2808	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2809	let ctx = Arc::new(RecipeCtx::new(
2810	AppId::new("demo"),
2811	Version::parse("0.1.0").unwrap(),
2812	"linux/x86_64".parse().unwrap(),
2813	"fw13".into(),
2814	"local".into(),
2815	"v0.1.0".into(),
2816	"/tmp".into(),
2817	vec![],
2818	Kind::Library,
2819	1,
2820	Arc::new(std::collections::HashMap::new()),
2821	Arc::new(std::collections::HashMap::new()),
2822	None,
2823	pool,
2824	crate::events::channel(),
2825	cfg,
2826	Arc::new(OtaRegistry::standard("https://makenot.work")),
2827	tokio::runtime::Handle::current(),
2828	Arc::new(AtomicBool::new(false)),
2829	None,
2830	));
2831	let engine = build_engine(&ctx);
2832	for call in [
2833	"deploy_host()",
2834	"service_name()",
2835	"install_path()",
2836	"health_url()",
2837	r#"deploy("/tmp/x")"#,
2838	] {
2839	let err = engine.eval::<String>(call).unwrap_err().to_string();
2840	assert!(
2841	err.contains("library") && err.contains("no deploy destination"),
2842	"`{call}` must fail on the kind, got: {err}"
2843	);
2844	}
2845	}
2846
2847	/// A service host is addressed on the DEPLOY plane whatever step is open.
2848	///
2849	/// The subtle one. Actions are normally derived from the step, which is
2850	/// right for a build host — the step is what that host is being asked to do.
2851	/// A service host is granted `deploy`/`restart` and must never be granted
2852	/// `build`, so the same rule would have `glibc_check` ask it for `build`
2853	/// during a `verify` step and get denied for a reason unrelated to what was
2854	/// attempted. `verify` is the step that check belongs in, so without this
2855	/// routing the glibc gate cannot run at all.
2856	#[tokio::test]
2857	async fn a_service_host_is_addressed_on_the_deploy_plane_in_any_step() {
2858	let dir = tempfile::tempdir().unwrap();
2859	let cfg = Arc::new(Config::for_tests(dir.path()));
2860	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2861	sqlx::query(
2862	"INSERT INTO builds (id, app, version, status, created_at) \
2863	VALUES (1, 'demo', '0.1.0', 'running', '2026-07-30T00:00:00Z')",
2864	)
2865	.execute(&pool)
2866	.await
2867	.unwrap();
2868	sqlx::query(
2869	"INSERT INTO target_runs (id, build_id, app, version, target, status, started_at) \
2870	VALUES (1, 1, 'demo', '0.1.0', 'linux/x86_64', 'running', '2026-07-30T00:00:00Z')",
2871	)
2872	.execute(&pool)
2873	.await
2874	.unwrap();
2875
2876	let deploy = crate::topology::DeployTarget {
2877	target: "linux/x86_64".parse().unwrap(),
2878	host: "local".into(),
2879	port: None,
2880	install_path: "/usr/local/bin/demo".into(),
2881	service: "demo.service".into(),
2882	health_url: None,
2883	};
2884	let mut execs: crate::state::ExecutorMap = std::collections::HashMap::new();
2885	execs.insert("local".into(), crate::state::build_deploy_executor(&deploy));
2886	// The service host's grant is exactly deploy + restart. If this ever
2887	// widens to include `build`, the test below stops proving anything.
2888	assert!(!execs["local"].capabilities().permits(&Action::Build));
2889	assert!(execs["local"].capabilities().permits(&Action::Deploy));
2890
2891	let ctx = Arc::new(RecipeCtx::new(
2892	AppId::new("demo"),
2893	Version::parse("0.1.0").unwrap(),
2894	"linux/x86_64".parse().unwrap(),
2895	"fw13".into(),
2896	"local".into(),
2897	"v0.1.0".into(),
2898	"/tmp".into(),
2899	vec![],
2900	Kind::Service,
2901	1,
2902	Arc::new(execs),
2903	Arc::new(std::collections::HashMap::new()),
2904	Some(deploy),
2905	pool,
2906	crate::events::channel(),
2907	cfg,
2908	Arc::new(OtaRegistry::standard("https://makenot.work")),
2909	tokio::runtime::Handle::current(),
2910	Arc::new(AtomicBool::new(false)),
2911	None,
2912	));
2913
2914	let ctx_blocking = ctx.clone();
2915	tokio::task::spawn_blocking(move \|\| {
2916	// `verify` on a service derives Action::Build — which the service
2917	// host does not grant. The command must still run.
2918	ctx_blocking.begin_step(Step::Verify).unwrap();
2919	assert_eq!(
2920	action_for(Step::Verify, Kind::Service),
2921	Action::Build,
2922	"the step's own action is the one that would be denied",
2923	);
2924	let (code, out) = ctx_blocking
2925	.run("local", "echo reached-the-service-host")
2926	.expect("a service host must be reachable during a verify step");
2927	assert_eq!(code, 0, "{out}");
2928	assert!(out.contains("reached-the-service-host"), "{out}");
2929	})
2930	.await
2931	.unwrap();
2932	}
2933
2934	/// A step that finalized `Failed` bars the deploy, exactly as it bars a
2935	/// publish. Without this, a recipe that inspects `sh(...).code` and carries
2936	/// on regardless still lands a binary on a production host — the precise
2937	/// hazard a pipeline exists to remove. The check is the ledger, not the
2938	/// control flow, so it holds whether or not the recipe noticed.
2939	#[tokio::test]
2940	async fn a_failed_step_bars_the_deploy() {
2941	let dir = tempfile::tempdir().unwrap();
2942	let cfg = Arc::new(Config::for_tests(dir.path()));
2943	let pool = crate::db::open(&cfg.db_path).await.unwrap();
2944	let deploy = crate::topology::DeployTarget {
2945	target: "linux/x86_64".parse().unwrap(),
2946	host: "local".into(),
2947	port: None,
2948	install_path: "/usr/local/bin/demo".into(),
2949	service: "demo.service".into(),
2950	health_url: None,
2951	};
2952	// A real build + target run, so the step rows this test finalizes have
2953	// the parents the schema requires.
2954	sqlx::query(
2955	"INSERT INTO builds (id, app, version, status, created_at) \
2956	VALUES (1, 'demo', '0.1.0', 'running', '2026-07-30T00:00:00Z')",
2957	)
2958	.execute(&pool)
2959	.await
2960	.unwrap();
2961	sqlx::query(
2962	"INSERT INTO target_runs (id, build_id, app, version, target, status, started_at) \
2963	VALUES (1, 1, 'demo', '0.1.0', 'linux/x86_64', 'running', '2026-07-30T00:00:00Z')",
2964	)
2965	.execute(&pool)
2966	.await
2967	.unwrap();
2968
2969	let mut execs: crate::state::ExecutorMap = std::collections::HashMap::new();
2970	execs.insert("local".into(), crate::state::build_deploy_executor(&deploy));
2971	let ctx = Arc::new(RecipeCtx::new(
2972	AppId::new("demo"),
2973	Version::parse("0.1.0").unwrap(),
2974	"linux/x86_64".parse().unwrap(),
2975	"fw13".into(),
2976	"local".into(),
2977	"v0.1.0".into(),
2978	"/tmp".into(),
2979	vec![],
2980	Kind::Service,
2981	1,
2982	Arc::new(execs),
2983	Arc::new(std::collections::HashMap::new()),
2984	Some(deploy),
2985	pool,
2986	crate::events::channel(),
2987	cfg,
2988	Arc::new(OtaRegistry::standard("https://makenot.work")),
2989	tokio::runtime::Handle::current(),
2990	Arc::new(AtomicBool::new(false)),
2991	None,
2992	));
2993
2994	// A gate ran, failed, and the recipe did not abort — the swallowed
2995	// failure. Finalizing it is what puts it in the ledger.
2996	let ctx_blocking = ctx.clone();
2997	tokio::task::spawn_blocking(move \|\| {
2998	ctx_blocking.begin_step(Step::Prebuild).unwrap();
2999	ctx_blocking.fail_current_step();
3000	ctx_blocking.finish_step(Status::Ok).unwrap();
3001
3002	let err = ctx_blocking.deploy("/tmp/demo").unwrap_err().to_string();
3003	assert!(
3004	err.contains("refusing to deploy") && err.contains("prebuild"),
3005	"must refuse and name the failed step, got: {err}"
3006	);
3007	})
3008	.await
3009	.unwrap();
3010	}
3011
3012	#[test]
3013	fn every_step_has_a_nonzero_default_budget() {
3014	// A zero/missing budget would deadline-fail a step instantly. Cover the
3015	// whole matrix so a new Step variant can't silently get a 0 budget.
3016	for step in Step::ALL {
3017	assert!(
3018	default_step_budget(step) >= std::time::Duration::from_mins(1),
3019	"{step} budget must be a sane ceiling",
3020	);
3021	}
3022	}
3023
3024	#[test]
3025	fn sha256_file_is_lowercase_hex_of_contents() {
3026	let tmp = tempfile::tempdir().unwrap();
3027	let f = tmp.path().join("a.bin");
3028	std::fs::write(&f, b"abc").unwrap();
3029	// Known SHA-256 of "abc".
3030	assert_eq!(
3031	sha256_file(&f).unwrap(),
3032	"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
3033	);
3034	}
3035
3036	// ---- publish step-success gate ----
3037
3038	fn target(s: &str) -> Target {
3039	s.parse().unwrap()
3040	}
3041
3042	#[test]
3043	fn publish_gate_blocks_macos_without_verification() {
3044	// Never verified -> blocked, with a message pointing at verify_gatekeeper.
3045	let err = PublishAuthority::prove(target("macos/aarch64"), &[], None).unwrap_err();
3046	assert!(format!("{err:#}").contains("never verified"), "{err:#}");
3047	}
3048
3049	#[test]
3050	fn publish_gate_blocks_macos_when_gatekeeper_rejected() {
3051	let err = PublishAuthority::prove(target("macos/aarch64"), &[], Some(false)).unwrap_err();
3052	assert!(
3053	format!("{err:#}").contains("Gatekeeper rejected"),
3054	"{err:#}"
3055	);
3056	}
3057
3058	#[test]
3059	fn publish_gate_allows_macos_when_gatekeeper_accepted() {
3060	PublishAuthority::prove(target("macos/aarch64"), &[], Some(true)).unwrap();
3061	// iOS is gated the same way.
3062	PublishAuthority::prove(target("ios/universal"), &[], Some(true)).unwrap();
3063	assert!(PublishAuthority::prove(target("ios/universal"), &[], None).is_err());
3064	}
3065
3066	#[test]
3067	fn publish_gate_does_not_require_gatekeeper_for_non_apple_targets() {
3068	// Linux/Windows aren't notarized; no gatekeeper proof needed.
3069	PublishAuthority::prove(target("linux/x86_64"), &[], None).unwrap();
3070	PublishAuthority::prove(target("windows/x86_64"), &[], None).unwrap();
3071	}
3072
3073	#[test]
3074	fn publish_gate_blocks_when_any_prior_step_failed() {
3075	// A failed step bars publish on every target, even a verified macOS one.
3076	let err =
3077	PublishAuthority::prove(target("linux/x86_64"), &[Step::Build], None).unwrap_err();
3078	assert!(
3079	format!("{err:#}").contains("prior step(s) failed"),
3080	"{err:#}"
3081	);
3082	assert!(
3083	format!("{err:#}").contains("build"),
3084	"names the failed step: {err:#}"
3085	);
3086
3087	let err = PublishAuthority::prove(target("macos/aarch64"), &[Step::Sign], Some(true))
3088	.unwrap_err();
3089	assert!(
3090	format!("{err:#}").contains("prior step(s) failed"),
3091	"{err:#}"
3092	);
3093	}
3094
3095	#[test]
3096	fn notary_accepted_parses_status_field() {
3097	assert!(notary_accepted(
3098	r#"{"id":"abc","status":"Accepted","message":"ok"}"#
3099	));
3100	// Embedded in shell-sourcing noise: the object is isolated and parsed.
3101	assert!(notary_accepted(
3102	"sourcing env...\n{\n \"status\": \"Accepted\"\n}\nbye"
3103	));
3104	// Whitespace variant that a tight substring `"status":"Accepted"` misses.
3105	assert!(notary_accepted(r#"{ "status" : "Accepted" }"#));
3106	}
3107
3108	#[test]
3109	fn notary_accepted_rejects_non_accepted_and_garbage() {
3110	assert!(!notary_accepted(r#"{"status":"Invalid"}"#));
3111	assert!(!notary_accepted(r#"{"status":"In Progress"}"#));
3112	assert!(!notary_accepted("no json here"));
3113	assert!(!notary_accepted("")); // empty / truncated -> fail closed
3114	// A truncated tail whose opening brace was cut off cannot parse -> closed.
3115	assert!(!notary_accepted(r#""status":"Accepted"}"#));
3116	// The literal appearing inside an error string must NOT pass as success.
3117	assert!(!notary_accepted(
3118	r#"{"status":"Invalid","message":"expected status:Accepted"}"#
3119	));
3120	}
3121	}
3122