Skip to main content

max / makenotwork

123.9 KB · 3310 lines History Blame Raw
1 //! Build orchestration: fan a `(app, version)` out across its targets, each
2 //! running its recipe concurrently on the host that can build it.
3 //!
4 //! A build inserts one `builds` row, then spawns one task per target. Each
5 //! target task registers itself in the single-slot guard (a newer build for
6 //! the same `(app, target)` aborts the in-flight one — latest wins, but other
7 //! targets keep running, which is the fan-out), then runs the recipe.
8
9 use crate::domain::{AppId, Status, Step, Target, Version};
10 use crate::engine::{self, RecipeCtx};
11 use crate::events::{self, Event};
12 use crate::state::AppState;
13 use anyhow::{Context, Result};
14 use ops_exec::{Action, LogSink, Step as OpStep};
15 use std::path::PathBuf;
16 use std::sync::Arc;
17
18 /// A [`LogSink`] that drops what it's handed — the release preflight runs git on
19 /// each host for its exit code and (via a separate `rev-parse`) the sha in
20 /// `RunOutput`, not for a streamed log, so there is no step to stream into.
21 struct DiscardSink;
22
23 #[async_trait::async_trait]
24 impl LogSink for DiscardSink {
25 async fn write_chunk(&mut self, _bytes: &[u8]) {}
26 }
27
28 /// Preflight barrier: pin every host that will build a target for this release
29 /// to the tag `v<version>` and refuse the build unless they all report the SAME
30 /// commit. Recipes used to `git pull --ff-only` per host, so `mbp`/`astra`/`fw13`
31 /// each built whatever `main` was at pull time and the artifacts were filed
32 /// under the daemon host's version. This runs before any target task spawns, so
33 /// a mixed-source release is stopped before a single artifact is built.
34 ///
35 /// Returns the branch each host was on before it was pinned, for
36 /// [`restore_branches`] to put back once the build settles. A host that is
37 /// ALREADY detached has no branch to return to, and is refused: that state means
38 /// some earlier release never cleaned up, and any commits made in the meantime
39 /// are sitting on no branch at all.
40 async fn pin_release(
41 state: &AppState,
42 app: &AppId,
43 version: &Version,
44 targets: &[Target],
45 ) -> Result<Vec<(String, String)>> {
46 let cfg = state
47 .topo
48 .app(app)
49 .ok_or_else(|| anyhow::anyhow!("unknown app `{app}`"))?;
50 let repo = cfg.repo.clone();
51 // Spelled per app: a repo holding one product tags `v0.4.1`, a repo holding
52 // several tags `pom-v0.4.1`. See `AppConfig::tag_format`.
53 let tag = cfg.tag_for(version);
54
55 // The distinct hosts across the requested targets (order-stable).
56 let mut hosts: Vec<String> = Vec::new();
57 for t in targets {
58 if let Some(h) = state.topo.host_for(*t)
59 && !hosts.contains(&h.name)
60 {
61 hosts.push(h.name.clone());
62 }
63 }
64
65 let mut shas: Vec<(String, String)> = Vec::new();
66 let mut branches: Vec<(String, String)> = Vec::new();
67 for host in &hosts {
68 let exec = state
69 .executors
70 .get(host)
71 .ok_or_else(|| anyhow::anyhow!("no executor for host `{host}`"))?;
72 let mut sink = DiscardSink;
73
74 // Read the branch BEFORE pinning, while there is still one to read.
75 let branch_cmd = OpStep::shell(Action::Build, engine::git_current_branch_cmd(&repo));
76 let out = exec
77 .run_streaming(&branch_cmd, &mut sink)
78 .await
79 .with_context(|| format!("release preflight: reading branch on `{host}`"))?;
80 let branch = String::from_utf8_lossy(&out.stdout).trim().to_string();
81 anyhow::ensure!(
82 out.status.success() && !branch.is_empty(),
83 "release preflight: `{repo}` on `{host}` is on a detached HEAD, so there is \
84 no branch to restore it to after the release. An earlier release left it \
85 that way; any commits made since are on no branch and may exist nowhere \
86 else. Reattach it (`git checkout <branch>`, fast-forwarding if the commits \
87 should be kept) before releasing."
88 );
89 branches.push((host.clone(), branch));
90
91 // A dirty tree defeats the pin silently, which is worse than not pinning
92 // at all. `git checkout <tag>` does not fail on local modifications to
93 // files whose content is unchanged in the tag — it succeeds and KEEPS
94 // them. So a host with edits in the working tree builds those edits
95 // while reporting the tagged sha, and a second host with a clean tree
96 // builds something else. The rev-parse barrier below cannot see it:
97 // both hosts genuinely are on the same commit. Two architectures, two
98 // different binaries, one tag containing neither.
99 let dirty_cmd = OpStep::shell(Action::Build, engine::git_dirty_cmd(&repo));
100 let out = exec
101 .run_streaming(&dirty_cmd, &mut sink)
102 .await
103 .with_context(|| format!("release preflight: reading tree state on `{host}`"))?;
104 let dirty = String::from_utf8_lossy(&out.stdout);
105 let dirty: Vec<&str> = dirty
106 .lines()
107 .map(str::trim)
108 .filter(|l| !l.is_empty())
109 .collect();
110 anyhow::ensure!(
111 dirty.is_empty(),
112 "release preflight: `{repo}` on `{host}` has uncommitted changes to tracked \
113 files, so the build would not be the tagged commit:\n {}\nCommit or stash \
114 them before releasing.",
115 dirty.join("\n "),
116 );
117
118 // Advisory: `fetch --all` is non-zero if any one of a repo's remotes is
119 // unreachable, and blaming the tag for a dead mirror is what made this
120 // preflight misreport. Only the checkout below is allowed to fail.
121 let fetch = OpStep::shell(Action::Build, engine::git_fetch_cmd(&repo));
122 let _ = exec.run_streaming(&fetch, &mut sink).await;
123
124 let checkout = OpStep::shell(Action::Build, engine::git_checkout_tag_cmd(&repo, &tag));
125 let out = exec
126 .run_streaming(&checkout, &mut sink)
127 .await
128 .with_context(|| format!("release preflight: checkout on `{host}`"))?;
129 if !out.status.success() {
130 // Ask git why before telling the operator. The two causes need
131 // opposite responses: tag-and-push, versus clean the working tree.
132 let probe = OpStep::shell(Action::Build, engine::git_tag_exists_cmd(&repo, &tag));
133 let tag_exists = exec
134 .run_streaming(&probe, &mut sink)
135 .await
136 .is_ok_and(|o| o.status.success());
137 anyhow::bail!(
138 "release preflight: `git checkout {tag}` failed on `{host}`: {}",
139 engine::checkout_failure_reason(&tag, tag_exists)
140 );
141 }
142
143 let rev = OpStep::shell(Action::Build, engine::git_rev_parse_cmd(&repo));
144 let out = exec
145 .run_streaming(&rev, &mut sink)
146 .await
147 .with_context(|| format!("release preflight: rev-parse on `{host}`"))?;
148 anyhow::ensure!(
149 out.status.success(),
150 "release preflight: `git rev-parse HEAD` failed on `{host}`"
151 );
152 let sha = String::from_utf8_lossy(&out.stdout).trim().to_string();
153 shas.push((host.clone(), sha));
154 }
155
156 // The barrier: every host must be on the same commit.
157 if let Some((first_host, first_sha)) = shas.first() {
158 let mismatch: Vec<String> = shas
159 .iter()
160 .filter(|(_, sha)| sha != first_sha)
161 .map(|(h, sha)| format!("{h}={}", short(sha)))
162 .collect();
163 anyhow::ensure!(
164 mismatch.is_empty(),
165 "release preflight: build hosts are on different commits for v{version} \
166 ({}={}, {}); refusing to build a release from mixed sources",
167 first_host,
168 short(first_sha),
169 mismatch.join(", "),
170 );
171 }
172 Ok(branches)
173 }
174
175 /// Put every host's checkout back on the branch [`pin_release`] found it on.
176 ///
177 /// Best-effort and non-fatal: the release itself is already decided by the time
178 /// this runs, and failing a green build because a `git checkout` did not take
179 /// would be worse than the detached tree it is cleaning up. A failure is logged
180 /// loudly instead, because the state it leaves behind is the silent one.
181 async fn restore_branches(state: &AppState, app: &AppId, branches: &[(String, String)]) {
182 let Some(cfg) = state.topo.app(app) else {
183 return;
184 };
185 for (host, branch) in branches {
186 let Some(exec) = state.executors.get(host) else {
187 continue;
188 };
189 // Serialize against a target still holding this host for its recipe run,
190 // so the restore can't move the tree mid-build.
191 let _host_guard = match state.host_locks.get(host).cloned() {
192 Some(lock) => Some(lock.lock_owned().await),
193 None => None,
194 };
195 let mut sink = DiscardSink;
196 let step = OpStep::shell(
197 Action::Build,
198 engine::git_restore_branch_cmd(&cfg.repo, branch),
199 );
200 match exec.run_streaming(&step, &mut sink).await {
201 Ok(out) if out.status.success() => {
202 tracing::debug!(%host, %branch, "restored checkout to its branch");
203 }
204 Ok(_) => tracing::error!(
205 %host, %branch, repo = %cfg.repo,
206 "could not restore the checkout to its branch; it is left DETACHED at the \
207 release tag, and commits made there will belong to no branch"
208 ),
209 Err(e) => tracing::error!(
210 %host, %branch, repo = %cfg.repo, error = %e,
211 "could not restore the checkout to its branch; it is left DETACHED at the \
212 release tag, and commits made there will belong to no branch"
213 ),
214 }
215 }
216 }
217
218 /// First 12 chars of a sha for a readable error.
219 fn short(sha: &str) -> &str {
220 sha.get(..12).unwrap_or(sha)
221 }
222
223 /// Resolve the version to build: explicit, or read from `tauri.conf.json`.
224 ///
225 /// Returns typed errors so the route maps user mistakes (unknown app, bad
226 /// version string) to 400, and only genuine daemon-side failures (reading the
227 /// app's `tauri.conf.json`) to 500.
228 pub fn resolve_version(
229 state: &AppState,
230 app: &AppId,
231 explicit: Option<String>,
232 ) -> crate::error::Result<Version> {
233 use crate::error::Error;
234 if let Some(v) = explicit {
235 return Version::parse(&v).map_err(Error::BadRequest);
236 }
237 let cfg = state
238 .topo
239 .app(app)
240 .ok_or_else(|| Error::BadRequest(format!("unknown app `{app}`")))?;
241 engine::version_from_repo(&cfg.repo, cfg.version_path.as_deref()).map_err(Error::Other)
242 }
243
244 /// Validate + default the target list against what the app ships. Unknown app,
245 /// an unshipped target, or a target no host can build are all client errors
246 /// (400), not server errors (500).
247 pub fn resolve_targets(
248 state: &AppState,
249 app: &AppId,
250 requested: Vec<Target>,
251 ) -> crate::error::Result<Vec<Target>> {
252 use crate::error::Error;
253 let cfg = state
254 .topo
255 .app(app)
256 .ok_or_else(|| Error::BadRequest(format!("unknown app `{app}`")))?;
257 if requested.is_empty() {
258 return Ok(cfg.targets.clone());
259 }
260 for t in &requested {
261 if !cfg.targets.contains(t) {
262 return Err(Error::BadRequest(format!(
263 "app `{app}` does not ship target {t}"
264 )));
265 }
266 if state.topo.host_for(*t).is_none() {
267 return Err(Error::BadRequest(format!("no host can build {t}")));
268 }
269 }
270 Ok(requested)
271 }
272
273 /// Insert the build row and spawn per-target tasks. Returns the build id.
274 pub async fn start_build(
275 state: AppState,
276 app: AppId,
277 version: Version,
278 targets: Vec<Target>,
279 ) -> Result<i64> {
280 // Preflight: every version source in the repo must agree with the version
281 // being built, before any host pulls or compiles. `version_from_repo` reads
282 // one file, so a tauri.conf.json/Cargo.toml (or explicit-version) mismatch
283 // would otherwise sail through and file artifacts under the wrong version.
284 if let Some(cfg) = state.topo.app(&app) {
285 engine::check_version_consistency(&cfg.repo, cfg.version_path.as_deref(), &version)
286 .context("version preflight")?;
287 }
288
289 // Pin every build host to the release tag and verify they agree, before any
290 // target task spawns. Off in tests (their repos aren't git checkouts).
291 let pinned_branches = if state.cfg.pin_release_sha {
292 pin_release(&state, &app, &version, &targets)
293 .await
294 .context("release preflight")?
295 } else {
296 Vec::new()
297 };
298
299 let build_id: i64 = sqlx::query_scalar(
300 "INSERT INTO builds (app, version, status, created_at) VALUES (?, ?, 'running', ?) RETURNING id",
301 )
302 .bind(app.as_str())
303 .bind(version.to_string())
304 .bind(chrono::Utc::now().to_rfc3339())
305 .fetch_one(&state.pool)
306 .await
307 .context("insert build")?;
308
309 events::emit(
310 &state.events,
311 Event::BuildRequested {
312 app: app.clone(),
313 version: version.clone(),
314 targets: targets.clone(),
315 },
316 );
317 crate::metrics::build_started();
318
319 // Spawn every target into a JoinSet so finalize_build can await completion
320 // event-driven (no DB polling) and observe each task's outcome (panic vs
321 // clean) for supervision.
322 let mut set: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
323 for target in targets {
324 let app = app.clone();
325 let version = version.clone();
326 let key = (app.clone(), target);
327 let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
328
329 // Latest-wins, done as ONE critical section: supersede the prior occupant
330 // (cooperatively cancel its recipe + abort its task) and install ours
331 // without ever releasing the lock, so two concurrent /build+/retry for the
332 // same (app, target) can't both pass "nothing to abort" and run together.
333 let mut active = state.active.lock().await;
334 if let Some(prev) = active.remove(&key)
335 && !prev.abort.is_finished()
336 {
337 // The recipe runs on a blocking thread; abort() alone can't stop it,
338 // so set the cooperative flag the engine checks at step boundaries.
339 prev.cancel.store(true, std::sync::atomic::Ordering::SeqCst);
340 prev.abort.abort();
341 events::emit(
342 &state.events,
343 Event::TargetAborted {
344 app: app.clone(),
345 target,
346 },
347 );
348 }
349 let abort = set.spawn(run_target(
350 state.clone(),
351 build_id,
352 app,
353 version,
354 target,
355 cancel.clone(),
356 ));
357 active.insert(
358 key,
359 crate::state::ActiveSlot {
360 build_id,
361 abort,
362 cancel,
363 },
364 );
365 crate::metrics::set_in_flight(active.len());
366 }
367
368 // Mark the build done once all target tasks settle, and put the pinned
369 // checkouts back on their branches. Spawned so /build returns immediately.
370 tokio::spawn(finalize_build(state, build_id, set, app, pinned_branches));
371 Ok(build_id)
372 }
373
374 /// Run one target's recipe end to end, updating its `target_runs` row. `cancel`
375 /// is set when a newer build supersedes this one; the engine checks it at step
376 /// boundaries and before publish so a superseded recipe can't advance or ship.
377 async fn run_target(
378 state: AppState,
379 build_id: i64,
380 app: AppId,
381 version: Version,
382 target: Target,
383 cancel: Arc<std::sync::atomic::AtomicBool>,
384 ) {
385 let started = std::time::Instant::now();
386 let target_run_id: i64 = match sqlx::query_scalar(
387 "INSERT INTO target_runs (build_id, app, version, target, status, started_at)
388 VALUES (?, ?, ?, ?, 'running', ?) RETURNING id",
389 )
390 .bind(build_id)
391 .bind(app.as_str())
392 .bind(version.to_string())
393 .bind(target.to_string())
394 .bind(chrono::Utc::now().to_rfc3339())
395 .fetch_one(&state.pool)
396 .await
397 {
398 Ok(id) => id,
399 Err(e) => {
400 tracing::error!(%app, %target, error = %e, "could not create target_run");
401 return;
402 }
403 };
404
405 events::emit(
406 &state.events,
407 Event::TargetStart {
408 app: app.clone(),
409 version: version.clone(),
410 target,
411 },
412 );
413
414 let recipe_src = match read_recipe(&state, &app, target) {
415 Ok(s) => s,
416 Err(e) => {
417 fail_target(
418 &state,
419 target_run_id,
420 &app,
421 &version,
422 target,
423 Step::Checkout,
424 &format!("{e:#}"),
425 )
426 .await;
427 crate::metrics::target_finished(
428 &target.to_string(),
429 "failed",
430 started.elapsed().as_secs_f64(),
431 );
432 return;
433 }
434 };
435
436 // Pre-flight the build host before running the recipe. For local/ssh hosts
437 // this is a no-op; for the agent host (macOS in-session signing) it hits
438 // `/health` so a dead ops-agent fails here with a clear message — matching
439 // what the driver does — instead of erroring opaquely on the recipe's first
440 // dispatch to that host.
441 if let Some(host) = state.topo.host_for(target)
442 && let Some(exec) = state.executors.get(&host.name)
443 && let Err(e) = exec.preflight().await
444 {
445 fail_target(
446 &state,
447 target_run_id,
448 &app,
449 &version,
450 target,
451 Step::Checkout,
452 &format!("{e:#}"),
453 )
454 .await;
455 crate::metrics::target_finished(
456 &target.to_string(),
457 "failed",
458 started.elapsed().as_secs_f64(),
459 );
460 return;
461 }
462
463 // Host + checkout path for this target, resolved from the topology. Both are
464 // exposed to the recipe (`build_host()` / `repo()`); a missing host here is
465 // unreachable (resolve_targets rejected unbuildable targets before /build),
466 // but fall back rather than panic.
467 let build_host = state
468 .topo
469 .host_for(target)
470 .map(|h| h.name.clone())
471 .unwrap_or_default();
472 let build_host_ssh = state
473 .topo
474 .host_for(target)
475 .map(|h| h.ssh.clone())
476 .unwrap_or_default();
477 let repo = state
478 .topo
479 .app(&app)
480 .map(|a| a.repo.clone())
481 .unwrap_or_default();
482 let features = state
483 .topo
484 .app(&app)
485 .map(|a| a.features.clone())
486 .unwrap_or_default();
487 let tag = state
488 .topo
489 .app(&app)
490 .map_or_else(|| format!("v{version}"), |a| a.tag_for(&version));
491 // The opt-in all-targets-green publish gate: pass the declared target set
492 // (so `publish` can require every sibling green) only when the app turns it
493 // on; otherwise None leaves independent per-target publishing unchanged.
494 let all_green_required = state
495 .topo
496 .app(&app)
497 .and_then(|a| a.require_all_targets.then(|| a.targets.clone()));
498
499 // A service's install destination for THIS target, plus an executor for it.
500 //
501 // The executor is added to a per-run copy of the map rather than to the
502 // daemon-wide one, so a deploy grant on a production host exists only for
503 // the duration of the run that needs it and only for the app that declared
504 // it. Nothing else can address that host: it is not in the topology, so no
505 // other app's recipe can name it.
506 let deploy = state
507 .topo
508 .app(&app)
509 .and_then(|a| a.deploy_for(target))
510 .cloned();
511 let execs = match &deploy {
512 Some(d) => {
513 let mut map = (*state.executors).clone();
514 map.insert(d.host.clone(), crate::state::build_deploy_executor(d));
515 Arc::new(map)
516 }
517 None => state.executors.clone(),
518 };
519
520 let ctx = Arc::new(RecipeCtx::new(
521 app.clone(),
522 version.clone(),
523 target,
524 build_host,
525 build_host_ssh,
526 tag.clone(),
527 repo,
528 features,
529 // Gates the `verify` step's capability: a library's crate preflight is
530 // not an app's Gatekeeper check. See engine::action_for.
531 state.topo.app(&app).map(|a| a.kind).unwrap_or_default(),
532 target_run_id,
533 execs,
534 state.syncs.clone(),
535 deploy,
536 state.pool.clone(),
537 state.events.clone(),
538 state.cfg.clone(),
539 state.ota.clone(),
540 tokio::runtime::Handle::current(),
541 cancel,
542 all_green_required,
543 ));
544
545 // Serialize per host: hold this host's lock for the whole recipe run so a
546 // second target on the same box (goingson macos + ios both on mbp) can't
547 // build concurrently in one checkout and corrupt the shared target/ +
548 // keychain. Targets on different hosts hold different locks and still fan
549 // out. Acquired at an await point, so a supersede-abort while queued drops
550 // the task cleanly before it ever takes the lock. Not held during the
551 // read-only preflight above.
552 let _host_guard = match state.host_locks.get(&ctx.build_host).cloned() {
553 Some(lock) => Some(lock.lock_owned().await),
554 None => None,
555 };
556
557 // Rhai is synchronous; run the recipe (and its final step finalization) on
558 // a blocking thread so host functions can `block_on` without sitting on a
559 // runtime worker.
560 let ctx_run = ctx.clone();
561 let outcome = tokio::task::spawn_blocking(move || {
562 let engine = engine::build_engine(&ctx_run);
563 let res = engine.run(&recipe_src);
564 let last_step = ctx_run.current_step();
565 match &res {
566 Ok(()) => {
567 let _ = ctx_run.finish_step(Status::Ok);
568 }
569 Err(_) => {
570 let _ = ctx_run.finish_step(Status::Failed);
571 }
572 }
573 res.map_err(|e| (last_step, e.to_string()))
574 })
575 .await;
576
577 match outcome {
578 Ok(Ok(())) => {
579 let artifacts = collected_artifacts(&state, &app, &version);
580 if let Err(e) = sqlx::query(
581 "UPDATE target_runs SET status = 'ok', current_step = NULL, finished_at = ? WHERE id = ?",
582 )
583 .bind(chrono::Utc::now().to_rfc3339())
584 .bind(target_run_id)
585 .execute(&state.pool)
586 .await
587 {
588 // A swallowed terminal write would leave the row `running`;
589 // finalize_build reconciles it, but log so the cause is visible.
590 tracing::error!(%app, %target, error = %e, "could not stamp target_run ok");
591 }
592 crate::metrics::target_finished(
593 &target.to_string(),
594 "ok",
595 started.elapsed().as_secs_f64(),
596 );
597 events::emit(
598 &state.events,
599 Event::TargetOk {
600 app,
601 version,
602 target,
603 artifacts,
604 },
605 );
606 }
607 Ok(Err((step, msg))) => {
608 fail_target(&state, target_run_id, &app, &version, target, step, &msg).await;
609 crate::metrics::target_finished(
610 &target.to_string(),
611 "failed",
612 started.elapsed().as_secs_f64(),
613 );
614 }
615 Err(join_err) => {
616 // Task was aborted (superseded) or panicked.
617 let status = if join_err.is_cancelled() {
618 "aborted"
619 } else {
620 "failed"
621 };
622 crate::metrics::target_finished(
623 &target.to_string(),
624 status,
625 started.elapsed().as_secs_f64(),
626 );
627 let msg = if join_err.is_cancelled() {
628 "aborted (superseded)".to_string()
629 } else {
630 format!("recipe task panicked: {join_err}")
631 };
632 fail_target(
633 &state,
634 target_run_id,
635 &app,
636 &version,
637 target,
638 Step::Build,
639 &msg,
640 )
641 .await;
642 }
643 }
644 }
645
646 async fn fail_target(
647 state: &AppState,
648 target_run_id: i64,
649 app: &AppId,
650 version: &Version,
651 target: Target,
652 step: Step,
653 error: &str,
654 ) {
655 if let Err(e) = sqlx::query(
656 "UPDATE target_runs SET status = 'failed', current_step = NULL, error = ?, finished_at = ? WHERE id = ?",
657 )
658 .bind(error)
659 .bind(chrono::Utc::now().to_rfc3339())
660 .bind(target_run_id)
661 .execute(&state.pool)
662 .await
663 {
664 tracing::error!(%app, %target, error = %e, "could not stamp target_run failed");
665 }
666 events::emit(
667 &state.events,
668 Event::TargetFailed {
669 app: app.clone(),
670 version: version.clone(),
671 target,
672 step,
673 error: error.to_string(),
674 },
675 );
676 }
677
678 /// Await every target task of a build, then stamp the build's terminal status.
679 ///
680 /// Event-driven: it joins the `JoinSet` rather than polling the DB. Each task's
681 /// outcome is supervised — a panicked task is logged (and its still-`running`
682 /// row is reconciled below), an aborted (superseded) task is expected.
683 ///
684 /// Per-step deadlines (see `engine::step_budget`) are the real bound on a wedged
685 /// step now; this overall deadline is only a generous last-resort backstop for a
686 /// hang outside a bounded command. When it trips it sets each still-running
687 /// target's cooperative cancel FIRST — the blocking recipe bodies observe that
688 /// at their next bounded command or step boundary and stop signing — then aborts
689 /// the async wrappers. The old code aborted only the wrappers, which could not
690 /// reach the blocking bodies, so a build was marked failed while codesign kept
691 /// running on the mac.
692 async fn finalize_build(
693 state: AppState,
694 build_id: i64,
695 mut set: tokio::task::JoinSet<()>,
696 app: AppId,
697 pinned_branches: Vec<(String, String)>,
698 ) {
699 const MAX_WAIT: std::time::Duration = std::time::Duration::from_hours(6);
700 let deadline = tokio::time::Instant::now() + MAX_WAIT;
701 loop {
702 match tokio::time::timeout_at(deadline, set.join_next()).await {
703 Ok(Some(Ok(()))) => {}
704 Ok(Some(Err(e))) => {
705 // A panic in run_target itself (outside its inner spawn_blocking,
706 // which is already caught). Cancellation = a superseding build.
707 if !e.is_cancelled() {
708 tracing::error!(build_id, error = %e, "target task panicked");
709 }
710 }
711 Ok(None) => break, // all targets settled
712 Err(_elapsed) => {
713 tracing::error!(
714 build_id,
715 "finalize_build backstop deadline hit; cancelling then aborting remaining targets"
716 );
717 // Set the cooperative cancel on this build's still-running slots
718 // BEFORE aborting, so the blocking recipe bodies actually stop
719 // (abort() alone cannot reach a spawn_blocking body).
720 {
721 let active = state.active.lock().await;
722 for slot in active.values().filter(|s| s.build_id == build_id) {
723 slot.cancel.store(true, std::sync::atomic::Ordering::SeqCst);
724 }
725 }
726 set.abort_all();
727 while set.join_next().await.is_some() {}
728 break;
729 }
730 }
731 }
732
733 // Reap this build's latest-wins slots (only our own — a superseding build
734 // owns a different build_id and is left intact).
735 let app_still_building = {
736 let mut active = state.active.lock().await;
737 active.retain(|_, slot| slot.build_id != build_id);
738 crate::metrics::set_in_flight(active.len());
739 active.keys().any(|(a, _)| a == &app)
740 };
741
742 // Undo the preflight's pin. Skipped if a superseding build for this same app
743 // still holds the checkout — it pinned the tree to ITS tag, and restoring the
744 // branch here would move that build off the commit it is releasing.
745 if !pinned_branches.is_empty() {
746 if app_still_building {
747 tracing::debug!(
748 build_id, %app,
749 "leaving the checkout pinned; a superseding build for this app is still running"
750 );
751 } else {
752 restore_branches(&state, &app, &pinned_branches).await;
753 }
754 }
755
756 let now = chrono::Utc::now().to_rfc3339();
757 // Any row still `running` is a panicked/aborted task that never stamped its
758 // terminal status — reconcile it so the build finalizes truthfully.
759 if let Err(e) = sqlx::query(
760 "UPDATE target_runs SET status = 'failed', \
761 error = COALESCE(error, 'target task ended before stamping its status'), \
762 finished_at = ? WHERE build_id = ? AND status = 'running'",
763 )
764 .bind(&now)
765 .bind(build_id)
766 .execute(&state.pool)
767 .await
768 {
769 tracing::error!(build_id, error = %e, "could not reconcile straggling target_runs");
770 }
771
772 let failed: i64 = sqlx::query_scalar(
773 "SELECT COUNT(*) FROM target_runs WHERE build_id = ? AND status = 'failed'",
774 )
775 .bind(build_id)
776 .fetch_one(&state.pool)
777 .await
778 .unwrap_or(0);
779 let status = if failed == 0 { "ok" } else { "failed" };
780 if let Err(e) = sqlx::query("UPDATE builds SET status = ?, finished_at = ? WHERE id = ?")
781 .bind(status)
782 .bind(&now)
783 .bind(build_id)
784 .execute(&state.pool)
785 .await
786 {
787 tracing::error!(build_id, error = %e, "could not stamp build terminal status");
788 }
789 }
790
791 /// Read the recipe text for `(app, target)` from the app's checkout on the
792 /// daemon host.
793 ///
794 /// Apps use one recipe per platform (`<recipe_dir>/<platform>.rhai`), since
795 /// what they produce differs by platform. A library produces one crate whatever
796 /// host uploads it, so it uses a single `publish.rhai` — a `linux.rhai` naming
797 /// a registry upload would imply a per-platform artifact that does not exist.
798 fn read_recipe(state: &AppState, app: &AppId, target: Target) -> Result<String> {
799 let cfg = state
800 .topo
801 .app(app)
802 .ok_or_else(|| anyhow::anyhow!("unknown app `{app}`"))?;
803 let file = match cfg.kind {
804 crate::topology::Kind::Library => "publish.rhai".to_string(),
805 // A service is built per target like an app — the binary genuinely
806 // differs per platform, and so does where it lands — so it takes the
807 // same per-platform recipe naming rather than a single deploy.rhai.
808 crate::topology::Kind::App | crate::topology::Kind::Service => {
809 format!("{}.rhai", target.platform.as_str())
810 }
811 };
812 let path: PathBuf = engine::expand_tilde(&cfg.repo)
813 .join(&cfg.recipe_dir)
814 .join(file);
815 std::fs::read_to_string(&path).with_context(|| format!("reading recipe {}", path.display()))
816 }
817
818 fn collected_artifacts(state: &AppState, app: &AppId, version: &Version) -> Vec<String> {
819 let dir = state
820 .cfg
821 .dist_root
822 .join(app.as_str())
823 .join(version.to_string());
824 let Ok(rd) = std::fs::read_dir(&dir) else {
825 return Vec::new();
826 };
827 rd.filter_map(std::result::Result::ok)
828 .map(|e| e.file_name().to_string_lossy().into_owned())
829 .collect()
830 }
831
832 #[cfg(test)]
833 mod tests {
834 use super::*;
835 use crate::config::Config;
836 use crate::ota::OtaRegistry;
837 use crate::topology::Topology;
838 use async_trait::async_trait;
839 use ops_exec::{CapabilitySet, Executor, LogSink, RunOutput, SyncOpts};
840 use sqlx::SqlitePool;
841 use std::collections::HashMap;
842 use std::os::unix::process::ExitStatusExt;
843 use std::sync::Arc;
844 use tokio::sync::Mutex;
845
846 /// A no-transport [`Executor`] for the paths that don't need a real command
847 /// to run: its `preflight` is programmable (the agent host hits `/health`
848 /// there, and a dead `ops-agent` must fail the target before the recipe
849 /// dispatches), and every actual op is a success no-op.
850 struct FakeExec {
851 caps: CapabilitySet,
852 preflight_err: Option<String>,
853 }
854
855 impl FakeExec {
856 fn preflight_fails(msg: &str) -> Arc<dyn Executor> {
857 Arc::new(Self {
858 caps: CapabilitySet::default(),
859 preflight_err: Some(msg.to_string()),
860 })
861 }
862 }
863
864 #[async_trait]
865 impl Executor for FakeExec {
866 async fn run_streaming(
867 &self,
868 _step: &ops_exec::Step,
869 _sink: &mut dyn LogSink,
870 ) -> anyhow::Result<RunOutput> {
871 Ok(RunOutput {
872 status: std::process::ExitStatus::from_raw(0),
873 stdout: Vec::new(),
874 stderr: Vec::new(),
875 })
876 }
877 async fn pull_file(
878 &self,
879 _r: &std::path::Path,
880 _l: &std::path::Path,
881 _o: &SyncOpts,
882 ) -> anyhow::Result<()> {
883 Ok(())
884 }
885 async fn pull_dir(
886 &self,
887 _r: &std::path::Path,
888 _l: &std::path::Path,
889 _o: &SyncOpts,
890 ) -> anyhow::Result<()> {
891 Ok(())
892 }
893 async fn pull_glob(
894 &self,
895 _g: &str,
896 _l: &std::path::Path,
897 _o: &SyncOpts,
898 ) -> anyhow::Result<()> {
899 Ok(())
900 }
901 async fn push_dir(
902 &self,
903 _l: &std::path::Path,
904 _r: &std::path::Path,
905 _o: &SyncOpts,
906 ) -> anyhow::Result<()> {
907 Ok(())
908 }
909 async fn preflight(&self) -> anyhow::Result<()> {
910 match &self.preflight_err {
911 Some(m) => anyhow::bail!("{m}"),
912 None => Ok(()),
913 }
914 }
915 fn capabilities(&self) -> &CapabilitySet {
916 &self.caps
917 }
918 }
919
920 /// A recording, programmable [`Executor`] for the macOS sign chain. Every
921 /// dispatched shell command is captured for assertion, and its exit code +
922 /// stdout is chosen by the first rule whose needle the command contains — so
923 /// a test can make `notarytool` report `Accepted`, make `codesign` fail, or
924 /// make `spctl` emit the Gatekeeper sentinel without a real Mac or SSH. A
925 /// rule may carry a *sequence* of responses (one per successive match) to
926 /// drive the notarize retry loop; the last entry repeats once the sequence
927 /// is exhausted. Unmatched commands succeed as empty no-ops (so a plain
928 /// build `sh_ok` passes), and the sync/preflight ops are no-ops.
929 struct ScriptedExec {
930 caps: CapabilitySet,
931 rules: Vec<ScriptRule>,
932 log: Arc<std::sync::Mutex<Vec<String>>>,
933 }
934
935 struct ScriptRule {
936 needle: String,
937 responses: Vec<(i32, String)>,
938 calls: std::sync::atomic::AtomicUsize,
939 }
940
941 impl ScriptedExec {
942 fn new() -> Self {
943 Self {
944 // A mac host's real grant. Nothing in the dispatch path gates on
945 // it (this fake never calls `gate`), but keep it coherent so
946 // `capabilities()` is not a lie.
947 caps: CapabilitySet::from_tokens(
948 ["build", "sign", "notarize", "staple"],
949 ["build-log", "artifact"],
950 ),
951 rules: Vec::new(),
952 log: Arc::new(std::sync::Mutex::new(Vec::new())),
953 }
954 }
955
956 /// Respond to every command containing `needle` with `(code, stdout)`.
957 fn on(mut self, needle: &str, code: i32, stdout: &str) -> Self {
958 self.rules.push(ScriptRule {
959 needle: needle.to_string(),
960 responses: vec![(code, stdout.to_string())],
961 calls: std::sync::atomic::AtomicUsize::new(0),
962 });
963 self
964 }
965
966 /// Respond to successive `needle` matches with successive responses; the
967 /// last repeats once the list is exhausted. Drives the notarize retry.
968 fn on_seq(mut self, needle: &str, responses: &[(i32, &str)]) -> Self {
969 self.rules.push(ScriptRule {
970 needle: needle.to_string(),
971 responses: responses
972 .iter()
973 .map(|(c, s)| (*c, (*s).to_string()))
974 .collect(),
975 calls: std::sync::atomic::AtomicUsize::new(0),
976 });
977 self
978 }
979
980 /// Every shell command this executor was asked to run, in order.
981 fn commands(&self) -> Vec<String> {
982 self.log.lock().unwrap().clone()
983 }
984 }
985
986 #[async_trait]
987 impl Executor for ScriptedExec {
988 async fn run_streaming(
989 &self,
990 step: &ops_exec::Step,
991 _sink: &mut dyn LogSink,
992 ) -> anyhow::Result<RunOutput> {
993 let cmd = step.argv.last().cloned().unwrap_or_default();
994 self.log.lock().unwrap().push(cmd.clone());
995 let (code, stdout) = self.rules.iter().find(|r| cmd.contains(&r.needle)).map_or(
996 (0, String::new()),
997 |r| {
998 let i = r.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
999 r.responses[i.min(r.responses.len() - 1)].clone()
1000 },
1001 );
1002 Ok(RunOutput {
1003 // Shift into the wait-status word's exit-code byte so
1004 // `ExitStatus::code()` reports `code` exactly (a bare
1005 // `from_raw(1)` reads as a signal, yielding `None`).
1006 status: std::process::ExitStatus::from_raw(code << 8),
1007 stdout: stdout.into_bytes(),
1008 stderr: Vec::new(),
1009 })
1010 }
1011 async fn pull_file(
1012 &self,
1013 _r: &std::path::Path,
1014 _l: &std::path::Path,
1015 _o: &SyncOpts,
1016 ) -> anyhow::Result<()> {
1017 Ok(())
1018 }
1019 async fn pull_dir(
1020 &self,
1021 _r: &std::path::Path,
1022 _l: &std::path::Path,
1023 _o: &SyncOpts,
1024 ) -> anyhow::Result<()> {
1025 Ok(())
1026 }
1027 async fn pull_glob(
1028 &self,
1029 _g: &str,
1030 _l: &std::path::Path,
1031 _o: &SyncOpts,
1032 ) -> anyhow::Result<()> {
1033 Ok(())
1034 }
1035 async fn push_dir(
1036 &self,
1037 _l: &std::path::Path,
1038 _r: &std::path::Path,
1039 _o: &SyncOpts,
1040 ) -> anyhow::Result<()> {
1041 Ok(())
1042 }
1043 async fn preflight(&self) -> anyhow::Result<()> {
1044 Ok(())
1045 }
1046 fn capabilities(&self) -> &CapabilitySet {
1047 &self.caps
1048 }
1049 }
1050
1051 /// Assemble an [`AppState`] from the three per-test inputs, filling in the
1052 /// executors/syncs (built from `topo`) and the fixed test scaffolding
1053 /// (metrics handle, event bus, standard OTA registry, empty active map, no
1054 /// token). Every runner test builds the same struct around a different repo
1055 /// + recipe; this is that struct in one place.
1056 fn test_state(pool: SqlitePool, topo: Topology, cfg: Config) -> AppState {
1057 let executors = Arc::new(crate::state::build_executors(&topo));
1058 let syncs = Arc::new(crate::state::build_syncs(&topo));
1059 let host_locks = crate::state::build_host_locks(&topo);
1060 AppState {
1061 pool,
1062 topo: Arc::new(topo),
1063 cfg: Arc::new(cfg),
1064 prom: crate::metrics::test_handle(),
1065 events: crate::events::channel(),
1066 ota: Arc::new(OtaRegistry::standard("https://makenot.work")),
1067 executors,
1068 syncs,
1069 active: Arc::new(Mutex::new(HashMap::new())),
1070 api_token: None,
1071 host_locks,
1072 }
1073 }
1074
1075 /// A `kind = "service"` release end to end: build, `glibc_check`, `deploy`,
1076 /// and a health assertion the recipe makes itself against the service host.
1077 ///
1078 /// The whole point of the deploy step is that it dispatches to a host
1079 /// OUTSIDE the build topology, so this exercises the resolution
1080 /// (target -> `[[deploy]]` entry -> executor registered for the run), the
1081 /// staging, and the call into the privileged installer — with a fake
1082 /// installer standing in for the root script, which is the one part a test
1083 /// cannot run for real.
1084 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1085 async fn service_recipe_builds_then_deploys_and_verifies() {
1086 let tmp = tempfile::tempdir().unwrap();
1087 let root = tmp.path();
1088 let repo = root.join("svc");
1089 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1090 std::fs::write(repo.join("Cargo.toml"), "[package]\nversion = \"0.4.0\"\n").unwrap();
1091
1092 // Stand-in for the root installer: same three arguments, records what it
1093 // was asked to do instead of writing to /usr/local/bin and restarting a
1094 // unit. `install` + a marker file, so the test can assert the binary
1095 // that arrived is the binary that was built.
1096 let installer = root.join("install-service.sh");
1097 std::fs::write(
1098 &installer,
1099 "#!/bin/sh\nset -eu\ninstall -m 0755 \"$1\" \"$2\"\necho \"restarted $3\" >> \"$2.log\"\n",
1100 )
1101 .unwrap();
1102 std::fs::set_permissions(
1103 &installer,
1104 <std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o755),
1105 )
1106 .unwrap();
1107 let install_path = root.join("bin/svc");
1108 std::fs::create_dir_all(root.join("bin")).unwrap();
1109
1110 std::fs::write(
1111 repo.join("dist/recipes/linux.rhai"),
1112 r#"
1113 let v = version();
1114 step("build");
1115 sh_ok("fw13", "mkdir -p REPO/target/release && echo built-BIN > REPO/target/release/svc");
1116 step("verify");
1117 log(glibc_check("REPO/target/release/svc"));
1118 step("deploy");
1119 log(deploy("REPO/target/release/svc"));
1120 // The recipe owns what "healthy" means, and asserts it itself
1121 // against the host it just restarted.
1122 sh_ok(deploy_host(), "test -x " + install_path());
1123 "#
1124 .replace("REPO", repo.to_str().unwrap())
1125 .replace("BIN", "0.4.0"),
1126 )
1127 .unwrap();
1128
1129 std::fs::write(
1130 repo.join("bento.toml"),
1131 format!(
1132 r#"kind = "service"
1133 targets = ["linux/x86_64"]
1134 version_path = "Cargo.toml"
1135
1136 [[deploy]]
1137 target = "linux/x86_64"
1138 host = "local"
1139 install_path = "{}"
1140 service = "svc.service"
1141 health_url = "http://localhost:9100/api/health"
1142 "#,
1143 install_path.display()
1144 ),
1145 )
1146 .unwrap();
1147
1148 let mut cfg = Config::for_tests(root);
1149 cfg.deploy_installer = installer.display().to_string();
1150 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1151 let topo = Topology::from_str_for_tests(&format!(
1152 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
1153 pull_root = \"{repo}\"\n\n[app.svc]\nrepo = \"{repo}\"\n",
1154 repo = repo.display()
1155 ))
1156 .unwrap();
1157 let state = test_state(pool.clone(), topo, cfg);
1158
1159 let build_id = start_build(
1160 state.clone(),
1161 AppId::new("svc"),
1162 Version::parse("0.4.0").unwrap(),
1163 vec!["linux/x86_64".parse().unwrap()],
1164 )
1165 .await
1166 .unwrap();
1167
1168 let mut status = String::new();
1169 for _ in 0..100 {
1170 status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1171 .bind(build_id)
1172 .fetch_optional(&pool)
1173 .await
1174 .unwrap()
1175 .unwrap_or_else(|| "running".to_string());
1176 if status != "running" {
1177 break;
1178 }
1179 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1180 }
1181 assert_eq!(status, "ok", "service run should succeed");
1182
1183 let steps: Vec<(String, String)> = sqlx::query_as(
1184 "SELECT step, status FROM step_runs WHERE target_run_id IN \
1185 (SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id",
1186 )
1187 .bind(build_id)
1188 .fetch_all(&pool)
1189 .await
1190 .unwrap();
1191 assert_eq!(
1192 steps.iter().map(|(s, _)| s.as_str()).collect::<Vec<_>>(),
1193 vec!["build", "verify", "deploy"],
1194 "a service ends at deploy, not collect"
1195 );
1196 assert!(steps.iter().all(|(_, st)| st == "ok"), "{steps:?}");
1197
1198 // The bytes that were built are the bytes that landed, and the unit was
1199 // restarted only after the install succeeded.
1200 assert_eq!(
1201 std::fs::read_to_string(&install_path).unwrap().trim(),
1202 "built-0.4.0"
1203 );
1204 assert!(
1205 std::fs::read_to_string(install_path.with_extension("").with_file_name("svc.log"))
1206 .unwrap()
1207 .contains("restarted svc.service")
1208 );
1209 }
1210
1211 /// Stand up a tmp app repo + topology and run a real local recipe end to
1212 /// end: step transitions, streamed `sh_ok`, `version_of`, `log`, and a
1213 /// `collect` that pulls a built artifact into dist_root.
1214 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1215 async fn local_linux_recipe_runs_end_to_end() {
1216 let tmp = tempfile::tempdir().unwrap();
1217 let root = tmp.path();
1218
1219 // Fake app checkout: tauri.conf.json + a linux recipe.
1220 let repo = root.join("app");
1221 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1222 std::fs::write(
1223 repo.join("src-tauri/tauri.conf.json"),
1224 r#"{"version":"0.0.1"}"#,
1225 )
1226 .unwrap();
1227 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1228 // Build writes an artifact into the repo; collect pulls it to dist_root.
1229 std::fs::write(
1230 repo.join("dist/recipes/linux.rhai"),
1231 r#"
1232 step("build");
1233 let v = version_of("demo");
1234 log("building demo " + v);
1235 sh_ok("fw13", "echo compiling; mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
1236 step("collect");
1237 collect("fw13", "REPO/out/demo.bin", "demo", v);
1238 "#
1239 .replace("REPO", repo.to_str().unwrap()),
1240 )
1241 .unwrap();
1242
1243 let cfg = Config::for_tests(root);
1244 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1245 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1246 let topo = Topology::from_str_for_tests(&format!(
1247 r#"
1248 [[host]]
1249 name = "fw13"
1250 ssh = "local"
1251 targets = ["linux/x86_64"]
1252 pull_root = "{repo}"
1253
1254 [app.demo]
1255 repo = "{repo}"
1256 "#,
1257 repo = repo.display()
1258 ))
1259 .unwrap();
1260
1261 let state = test_state(pool.clone(), topo, cfg);
1262
1263 let app = AppId::new("demo");
1264 let version = Version::parse("0.0.1").unwrap();
1265 let build_id = start_build(
1266 state.clone(),
1267 app,
1268 version,
1269 vec!["linux/x86_64".parse().unwrap()],
1270 )
1271 .await
1272 .unwrap();
1273
1274 // Wait for the target run to settle. The row may not be inserted on the
1275 // first poll (run_target is spawned, not awaited), so treat a missing row
1276 // as still-pending rather than an error.
1277 let mut status = String::new();
1278 for _ in 0..100 {
1279 status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1280 .bind(build_id)
1281 .fetch_optional(&pool)
1282 .await
1283 .unwrap()
1284 .unwrap_or_else(|| "running".to_string());
1285 if status != "running" {
1286 break;
1287 }
1288 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1289 }
1290 assert_eq!(status, "ok", "target run should succeed");
1291
1292 // Both steps recorded and finished ok.
1293 let steps: Vec<(String, String)> =
1294 sqlx::query_as("SELECT step, status FROM step_runs WHERE target_run_id IN (SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id")
1295 .bind(build_id)
1296 .fetch_all(&pool)
1297 .await
1298 .unwrap();
1299 let names: Vec<&str> = steps.iter().map(|(s, _)| s.as_str()).collect();
1300 assert_eq!(names, vec!["build", "collect"]);
1301 assert!(steps.iter().all(|(_, st)| st == "ok"));
1302
1303 // Artifact landed in dist_root and a step log was written.
1304 let artifact = state.cfg.dist_root.join("demo/0.0.1/demo.bin");
1305 assert!(artifact.exists(), "collect should copy the artifact");
1306 let log = state
1307 .cfg
1308 .logs_root
1309 .join("demo/0.0.1/linux-x86_64/build.log");
1310 assert!(log.exists(), "build step log should exist");
1311 assert!(std::fs::read_to_string(&log).unwrap().contains("compiling"));
1312 }
1313
1314 /// Multi-target fan-out, which is what the daemon exists for: every other
1315 /// runner test drives exactly one target, so nothing covered `start_build`'s
1316 /// `JoinSet` fan-out or `finalize_build`'s rollup. One target fails and one
1317 /// succeeds — the failure must not abort its sibling, both must land their
1318 /// own terminal row, and the build must finalize `failed` because any failed
1319 /// target fails the build.
1320 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1321 async fn multi_target_fan_out_rolls_up_partial_failure() {
1322 let tmp = tempfile::tempdir().unwrap();
1323 let root = tmp.path();
1324
1325 let repo = root.join("app");
1326 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1327 std::fs::write(
1328 repo.join("src-tauri/tauri.conf.json"),
1329 r#"{"version":"0.0.1"}"#,
1330 )
1331 .unwrap();
1332 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1333
1334 // Linux succeeds and collects a real artifact. The artifact is the proof
1335 // this target ran to completion rather than being torn down when its
1336 // sibling failed.
1337 std::fs::write(
1338 repo.join("dist/recipes/linux.rhai"),
1339 r#"
1340 step("build");
1341 let v = version_of("demo");
1342 sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
1343 step("collect");
1344 collect("fw13", "REPO/out/demo.bin", "demo", v);
1345 "#
1346 .replace("REPO", repo.to_str().unwrap()),
1347 )
1348 .unwrap();
1349 // Windows fails in its first step.
1350 std::fs::write(
1351 repo.join("dist/recipes/windows.rhai"),
1352 r#"
1353 step("build");
1354 sh_ok("winbox", "echo nope 1>&2; exit 1");
1355 "#,
1356 )
1357 .unwrap();
1358
1359 let cfg = Config::for_tests(root);
1360 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1361 std::fs::write(
1362 repo.join("bento.toml"),
1363 "targets = [\"linux/x86_64\", \"windows/x86_64\"]\n",
1364 )
1365 .unwrap();
1366 // Two hosts so each target resolves its own, mirroring the real
1367 // per-architecture topology.
1368 let topo = Topology::from_str_for_tests(&format!(
1369 r#"
1370 [[host]]
1371 name = "fw13"
1372 ssh = "local"
1373 targets = ["linux/x86_64"]
1374 pull_root = "{repo}"
1375
1376 [[host]]
1377 name = "winbox"
1378 ssh = "local"
1379 targets = ["windows/x86_64"]
1380
1381 [app.demo]
1382 repo = "{repo}"
1383 "#,
1384 repo = repo.display()
1385 ))
1386 .unwrap();
1387
1388 let state = test_state(pool.clone(), topo, cfg);
1389
1390 let build_id = start_build(
1391 state.clone(),
1392 AppId::new("demo"),
1393 Version::parse("0.0.1").unwrap(),
1394 vec![
1395 "linux/x86_64".parse().unwrap(),
1396 "windows/x86_64".parse().unwrap(),
1397 ],
1398 )
1399 .await
1400 .unwrap();
1401
1402 // Poll the build row, not the target rows: the build is terminal only
1403 // once finalize_build has joined every task and stamped it.
1404 let mut build_status = String::new();
1405 for _ in 0..200 {
1406 build_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
1407 .bind(build_id)
1408 .fetch_one(&pool)
1409 .await
1410 .unwrap();
1411 if build_status != "running" {
1412 break;
1413 }
1414 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1415 }
1416
1417 let runs: Vec<(String, String)> = sqlx::query_as(
1418 "SELECT target, status FROM target_runs WHERE build_id = ? ORDER BY target",
1419 )
1420 .bind(build_id)
1421 .fetch_all(&pool)
1422 .await
1423 .unwrap();
1424 assert_eq!(
1425 runs,
1426 vec![
1427 ("linux/x86_64".to_string(), "ok".to_string()),
1428 ("windows/x86_64".to_string(), "failed".to_string()),
1429 ],
1430 "each target lands its own terminal row; one failing does not take the other down",
1431 );
1432
1433 // The surviving target finished its work, not merely its row.
1434 assert!(
1435 state.cfg.dist_root.join("demo/0.0.1/demo.bin").exists(),
1436 "the succeeding target ran to completion and collected its artifact",
1437 );
1438
1439 // The failure is attributed to the target that failed, and only it.
1440 let err: Option<String> = sqlx::query_scalar(
1441 "SELECT error FROM target_runs WHERE build_id = ? AND target = 'windows/x86_64'",
1442 )
1443 .bind(build_id)
1444 .fetch_one(&pool)
1445 .await
1446 .unwrap();
1447 assert!(
1448 err.is_some_and(|e| !e.is_empty()),
1449 "a failed target records why",
1450 );
1451
1452 assert_eq!(
1453 build_status, "failed",
1454 "any failed target fails the build; a partial release must not read as ok",
1455 );
1456 let finished: Option<String> =
1457 sqlx::query_scalar("SELECT finished_at FROM builds WHERE id = ?")
1458 .bind(build_id)
1459 .fetch_one(&pool)
1460 .await
1461 .unwrap();
1462 assert!(finished.is_some(), "finalize_build stamps the finish time");
1463
1464 // finalize_build reaps its own latest-wins slots, so nothing is left
1465 // in flight to block a later build of the same targets.
1466 assert!(
1467 state.active.lock().await.is_empty(),
1468 "finalize_build reaps the slots it owned",
1469 );
1470 }
1471
1472 /// Latest-wins supersession, driven through `start_build` (not the leaf
1473 /// `begin_step` bail that's already covered). Two builds of the SAME
1474 /// (app, target) race: the second must cooperatively cancel + abort the
1475 /// first and take the single slot, the first must terminate non-`ok`, and
1476 /// the second must run to completion.
1477 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1478 async fn a_newer_build_supersedes_the_in_flight_one_for_the_same_target() {
1479 let tmp = tempfile::tempdir().unwrap();
1480 let root = tmp.path();
1481
1482 let repo = root.join("app");
1483 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1484 std::fs::write(
1485 repo.join("src-tauri/tauri.conf.json"),
1486 r#"{"version":"0.0.1"}"#,
1487 )
1488 .unwrap();
1489 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1490 // A sleep long enough that the first build is still mid-recipe when the
1491 // second arrives; the engine checks the cancel flag at the step boundary
1492 // before "done", so the superseded run bails rather than finishing.
1493 std::fs::write(
1494 repo.join("dist/recipes/linux.rhai"),
1495 r#"
1496 step("build");
1497 sh_ok("fw13", "sleep 2");
1498 "#,
1499 )
1500 .unwrap();
1501
1502 let cfg = Config::for_tests(root);
1503 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1504 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1505 let topo = Topology::from_str_for_tests(&format!(
1506 r#"
1507 [[host]]
1508 name = "fw13"
1509 ssh = "local"
1510 targets = ["linux/x86_64"]
1511
1512 [app.demo]
1513 repo = "{}"
1514 "#,
1515 repo.display()
1516 ))
1517 .unwrap();
1518 let state = test_state(pool.clone(), topo, cfg);
1519
1520 let target = "linux/x86_64".parse().unwrap();
1521 let first = start_build(
1522 state.clone(),
1523 AppId::new("demo"),
1524 Version::parse("0.0.1").unwrap(),
1525 vec![target],
1526 )
1527 .await
1528 .unwrap();
1529 // Let the first build register its slot and enter the recipe before the
1530 // second supersedes it (start_build inserts the slot synchronously, but
1531 // the recipe runs on a spawned task).
1532 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
1533 let second = start_build(
1534 state.clone(),
1535 AppId::new("demo"),
1536 Version::parse("0.0.1").unwrap(),
1537 vec![target],
1538 )
1539 .await
1540 .unwrap();
1541 assert_ne!(first, second);
1542
1543 // Latest wins: exactly one slot for the key, owned by the second build.
1544 {
1545 let active = state.active.lock().await;
1546 assert_eq!(active.len(), 1, "supersession must not leave two slots");
1547 assert_eq!(
1548 active.values().next().unwrap().build_id,
1549 second,
1550 "the surviving slot belongs to the newer build",
1551 );
1552 }
1553
1554 // Wait for the second (surviving) build to finalize.
1555 let mut second_status = String::new();
1556 for _ in 0..200 {
1557 second_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
1558 .bind(second)
1559 .fetch_one(&pool)
1560 .await
1561 .unwrap();
1562 if second_status != "running" {
1563 break;
1564 }
1565 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1566 }
1567 assert_eq!(
1568 second_status, "ok",
1569 "the superseding build runs to completion"
1570 );
1571
1572 // The first build's target run terminated without succeeding — it was
1573 // cancelled/aborted, never stamped `ok`.
1574 let first_status: String =
1575 sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
1576 .bind(first)
1577 .fetch_one(&pool)
1578 .await
1579 .unwrap();
1580 assert_ne!(
1581 first_status, "ok",
1582 "the superseded build must not complete successfully",
1583 );
1584
1585 // Both builds reaped their slots; nothing left in flight.
1586 assert!(
1587 state.active.lock().await.is_empty(),
1588 "every finalized build reaps its own slot",
1589 );
1590 }
1591
1592 /// A failing `preflight` (the agent host's `/health` probe when `ops-agent`
1593 /// is down) must fail the target BEFORE the recipe dispatches — the whole
1594 /// point of preflighting. Covered only via a fake here, since a real
1595 /// LocalExec/SshExec preflight is a no-op that always passes.
1596 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1597 async fn a_failing_preflight_fails_the_target_before_the_recipe_runs() {
1598 let tmp = tempfile::tempdir().unwrap();
1599 let root = tmp.path();
1600
1601 let repo = root.join("app");
1602 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1603 std::fs::write(
1604 repo.join("src-tauri/tauri.conf.json"),
1605 r#"{"version":"0.0.1"}"#,
1606 )
1607 .unwrap();
1608 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1609 // If the recipe ran it would drop this marker; preflight failing first
1610 // means it never does.
1611 let marker = root.join("recipe-ran");
1612 std::fs::write(
1613 repo.join("dist/recipes/linux.rhai"),
1614 r#"
1615 step("build");
1616 sh_ok("fw13", "touch MARKER");
1617 "#
1618 .replace("MARKER", marker.to_str().unwrap()),
1619 )
1620 .unwrap();
1621
1622 let cfg = Config::for_tests(root);
1623 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1624 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1625 let topo = Topology::from_str_for_tests(&format!(
1626 r#"
1627 [[host]]
1628 name = "fw13"
1629 ssh = "local"
1630 targets = ["linux/x86_64"]
1631
1632 [app.demo]
1633 repo = "{}"
1634 "#,
1635 repo.display()
1636 ))
1637 .unwrap();
1638 let mut state = test_state(pool.clone(), topo, cfg);
1639 // Swap fw13's executor for one whose preflight fails.
1640 let mut execs = HashMap::new();
1641 execs.insert(
1642 "fw13".to_string(),
1643 FakeExec::preflight_fails("ops-agent not reachable at /health"),
1644 );
1645 state.executors = Arc::new(execs);
1646
1647 let build_id = start_build(
1648 state.clone(),
1649 AppId::new("demo"),
1650 Version::parse("0.0.1").unwrap(),
1651 vec!["linux/x86_64".parse().unwrap()],
1652 )
1653 .await
1654 .unwrap();
1655
1656 let mut status = String::new();
1657 let mut error = String::new();
1658 for _ in 0..100 {
1659 let row: Option<(String, Option<String>)> =
1660 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1661 .bind(build_id)
1662 .fetch_optional(&pool)
1663 .await
1664 .unwrap();
1665 if let Some((s, e)) = row {
1666 status = s;
1667 error = e.unwrap_or_default();
1668 if status != "running" {
1669 break;
1670 }
1671 }
1672 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1673 }
1674 assert_eq!(status, "failed", "a failed preflight must fail the target");
1675 assert!(
1676 error.contains("ops-agent not reachable"),
1677 "the failure must carry the preflight error, got: {error}"
1678 );
1679 assert!(
1680 !marker.exists(),
1681 "the recipe must NOT run when preflight fails"
1682 );
1683 }
1684
1685 /// A target whose recipe file is absent must fail at the `checkout` boundary
1686 /// with a readable "reading recipe" error, before any step runs — exercising
1687 /// `read_recipe`'s error branch and the `fail_target` path in `run_target`
1688 /// that neither the happy-path nor the failing-preflight test reaches. The
1689 /// app is configured to ship linux, but no `linux.rhai` is written.
1690 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1691 async fn a_target_with_no_recipe_file_fails_at_checkout() {
1692 let tmp = tempfile::tempdir().unwrap();
1693 let root = tmp.path();
1694
1695 let repo = root.join("app");
1696 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1697 std::fs::write(
1698 repo.join("src-tauri/tauri.conf.json"),
1699 r#"{"version":"0.0.1"}"#,
1700 )
1701 .unwrap();
1702 // The recipe dir exists but is empty — no linux.rhai.
1703 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1704
1705 let cfg = Config::for_tests(root);
1706 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1707 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1708 let topo = Topology::from_str_for_tests(&format!(
1709 r#"
1710 [[host]]
1711 name = "fw13"
1712 ssh = "local"
1713 targets = ["linux/x86_64"]
1714
1715 [app.demo]
1716 repo = "{}"
1717 "#,
1718 repo.display()
1719 ))
1720 .unwrap();
1721 let state = test_state(pool.clone(), topo, cfg);
1722
1723 let build_id = start_build(
1724 state.clone(),
1725 AppId::new("demo"),
1726 Version::parse("0.0.1").unwrap(),
1727 vec!["linux/x86_64".parse().unwrap()],
1728 )
1729 .await
1730 .unwrap();
1731
1732 // Poll the build row: it is terminal only once finalize_build joins the
1733 // one (failing) target task.
1734 let mut build_status = String::new();
1735 for _ in 0..100 {
1736 build_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
1737 .bind(build_id)
1738 .fetch_one(&pool)
1739 .await
1740 .unwrap();
1741 if build_status != "running" {
1742 break;
1743 }
1744 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1745 }
1746
1747 // The target failed, and the error points at the unreadable recipe.
1748 let (status, error): (String, Option<String>) =
1749 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1750 .bind(build_id)
1751 .fetch_one(&pool)
1752 .await
1753 .unwrap();
1754 assert_eq!(status, "failed", "a missing recipe must fail the target");
1755 assert!(
1756 error.is_some_and(|e| e.contains("reading recipe")),
1757 "the failure must name the recipe it could not read",
1758 );
1759
1760 // The recipe never ran, so no step_runs row was ever created — the
1761 // failure is at the checkout boundary, ahead of any step.
1762 let step_count: i64 = sqlx::query_scalar(
1763 "SELECT COUNT(*) FROM step_runs WHERE target_run_id IN \
1764 (SELECT id FROM target_runs WHERE build_id = ?)",
1765 )
1766 .bind(build_id)
1767 .fetch_one(&pool)
1768 .await
1769 .unwrap();
1770 assert_eq!(
1771 step_count, 0,
1772 "no step should run when the recipe is absent"
1773 );
1774
1775 assert_eq!(
1776 build_status, "failed",
1777 "the build rolls up the target failure",
1778 );
1779 assert!(
1780 state.active.lock().await.is_empty(),
1781 "finalize_build reaps the slot even on the recipe-read failure path",
1782 );
1783 }
1784
1785 /// Run 2 S5: a publish whose version is not strictly newer than the latest
1786 /// already published for the same (app, target, channel) is refused — an
1787 /// older build cannot republish over a live newer release.
1788 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1789 async fn publish_rejects_a_non_monotonic_version() {
1790 let tmp = tempfile::tempdir().unwrap();
1791 let root = tmp.path();
1792 let repo = root.join("app");
1793 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1794 std::fs::write(
1795 repo.join("src-tauri/tauri.conf.json"),
1796 r#"{"version":"0.2.0"}"#,
1797 )
1798 .unwrap();
1799 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1800 let artifact = repo.join("out/app.tar.gz");
1801 // Build an artifact, publish 0.2.0 (records a release), then try to
1802 // publish the older 0.1.0 — the second publish must fail.
1803 std::fs::write(
1804 repo.join("dist/recipes/linux.rhai"),
1805 r#"
1806 step("build");
1807 sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
1808 step("publish");
1809 publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
1810 publish("tauri-mnw", "demo", "linux/x86_64", "0.1.0", "ARTIFACT", #{});
1811 "#
1812 .replace("ARTIFACT", artifact.to_str().unwrap())
1813 .replace("REPO", repo.to_str().unwrap()),
1814 )
1815 .unwrap();
1816
1817 let cfg = Config::for_tests(root);
1818 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1819 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1820 let topo = Topology::from_str_for_tests(&format!(
1821 r#"
1822 [[host]]
1823 name = "fw13"
1824 ssh = "local"
1825 targets = ["linux/x86_64"]
1826
1827 [app.demo]
1828 repo = "{}"
1829 "#,
1830 repo.display()
1831 ))
1832 .unwrap();
1833 let state = test_state(pool.clone(), topo, cfg);
1834 let build_id = start_build(
1835 state.clone(),
1836 AppId::new("demo"),
1837 Version::parse("0.2.0").unwrap(),
1838 vec!["linux/x86_64".parse().unwrap()],
1839 )
1840 .await
1841 .unwrap();
1842
1843 let mut status = String::new();
1844 let mut error = String::new();
1845 for _ in 0..100 {
1846 let row: Option<(String, Option<String>)> =
1847 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1848 .bind(build_id)
1849 .fetch_optional(&pool)
1850 .await
1851 .unwrap();
1852 if let Some((s, e)) = row {
1853 status = s;
1854 error = e.unwrap_or_default();
1855 if status != "running" {
1856 break;
1857 }
1858 }
1859 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1860 }
1861 assert_eq!(status, "failed", "non-monotonic publish must fail the run");
1862 assert!(
1863 error.contains("not newer"),
1864 "expected monotonicity error, got: {error}"
1865 );
1866 // Exactly one release was recorded (0.2.0); 0.1.0 never landed.
1867 let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
1868 .fetch_one(&pool)
1869 .await
1870 .unwrap();
1871 assert_eq!(
1872 count, 1,
1873 "only the first (newer) publish should record a release"
1874 );
1875 // The recorded release carries the artifact's sha256 (64 hex chars),
1876 // not a NULL — the ledger says which bytes shipped.
1877 let hash: Option<String> =
1878 sqlx::query_scalar("SELECT artifact_hash FROM releases WHERE version = '0.2.0'")
1879 .fetch_one(&pool)
1880 .await
1881 .unwrap();
1882 assert!(
1883 hash.as_deref().is_some_and(|h| h.len() == 64),
1884 "publish must record the artifact sha256, got {hash:?}"
1885 );
1886 }
1887
1888 /// The artifact-identity fix at collect: a stale, wrongly-versioned artifact
1889 /// left in the output dir fails the collect instead of silently winning a
1890 /// later glob. Here the build is 0.0.1 but the recipe produces a 9.9.9 file.
1891 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1892 async fn collect_rejects_a_stale_versioned_artifact() {
1893 let tmp = tempfile::tempdir().unwrap();
1894 let root = tmp.path();
1895 let repo = root.join("app");
1896 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1897 std::fs::write(
1898 repo.join("src-tauri/tauri.conf.json"),
1899 r#"{"version":"0.0.1"}"#,
1900 )
1901 .unwrap();
1902 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1903 std::fs::write(
1904 repo.join("dist/recipes/linux.rhai"),
1905 r#"
1906 step("build");
1907 let v = version_of("demo");
1908 sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo-9.9.9.bin");
1909 step("collect");
1910 collect("fw13", "REPO/out/demo-9.9.9.bin", "demo", v);
1911 "#
1912 .replace("REPO", repo.to_str().unwrap()),
1913 )
1914 .unwrap();
1915
1916 let cfg = Config::for_tests(root);
1917 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1918 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1919 let topo = Topology::from_str_for_tests(&format!(
1920 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
1921 pull_root = \"{repo}\"\n\n[app.demo]\nrepo = \"{repo}\"\n",
1922 repo = repo.display()
1923 ))
1924 .unwrap();
1925 let state = test_state(pool.clone(), topo, cfg);
1926 let build_id = start_build(
1927 state.clone(),
1928 AppId::new("demo"),
1929 Version::parse("0.0.1").unwrap(),
1930 vec!["linux/x86_64".parse().unwrap()],
1931 )
1932 .await
1933 .unwrap();
1934
1935 let mut status = String::new();
1936 let mut error = String::new();
1937 for _ in 0..100 {
1938 let row: Option<(String, Option<String>)> =
1939 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1940 .bind(build_id)
1941 .fetch_optional(&pool)
1942 .await
1943 .unwrap();
1944 if let Some((s, e)) = row {
1945 status = s;
1946 error = e.unwrap_or_default();
1947 if status != "running" {
1948 break;
1949 }
1950 }
1951 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1952 }
1953 assert_eq!(
1954 status, "failed",
1955 "a mismatched-version artifact must fail collect"
1956 );
1957 assert!(
1958 error.contains("stale artifact"),
1959 "expected a stale-artifact error, got: {error}"
1960 );
1961 }
1962
1963 /// A command that runs past its step's deadline fails THAT step (and unwinds
1964 /// the recipe) rather than wedging under the old whole-build guillotine. The
1965 /// per-step budget is overridden to 1s here; the recipe then sleeps 30s, so
1966 /// the step deadline — not the sleep — decides the outcome, quickly.
1967 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1968 async fn a_step_that_exceeds_its_deadline_fails() {
1969 let tmp = tempfile::tempdir().unwrap();
1970 let root = tmp.path();
1971 let repo = root.join("app");
1972 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1973 std::fs::write(
1974 repo.join("src-tauri/tauri.conf.json"),
1975 r#"{"version":"0.0.1"}"#,
1976 )
1977 .unwrap();
1978 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1979 std::fs::write(
1980 repo.join("dist/recipes/linux.rhai"),
1981 r#"
1982 step("build");
1983 sh_ok("fw13", "sleep 30");
1984 "#,
1985 )
1986 .unwrap();
1987
1988 let mut cfg = Config::for_tests(root);
1989 cfg.step_timeout_secs = Some(1); // every step's budget -> 1s
1990 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1991 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1992 let topo = Topology::from_str_for_tests(&format!(
1993 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
1994 [app.demo]\nrepo = \"{}\"\n",
1995 repo.display()
1996 ))
1997 .unwrap();
1998 let state = test_state(pool.clone(), topo, cfg);
1999 let started = std::time::Instant::now();
2000 let build_id = start_build(
2001 state.clone(),
2002 AppId::new("demo"),
2003 Version::parse("0.0.1").unwrap(),
2004 vec!["linux/x86_64".parse().unwrap()],
2005 )
2006 .await
2007 .unwrap();
2008
2009 let mut status = String::new();
2010 let mut error = String::new();
2011 for _ in 0..100 {
2012 let row: Option<(String, Option<String>)> =
2013 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2014 .bind(build_id)
2015 .fetch_optional(&pool)
2016 .await
2017 .unwrap();
2018 if let Some((s, e)) = row {
2019 status = s;
2020 error = e.unwrap_or_default();
2021 if status != "running" {
2022 break;
2023 }
2024 }
2025 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2026 }
2027 assert_eq!(status, "failed", "a step past its deadline must fail");
2028 assert!(
2029 error.contains("per-step deadline"),
2030 "expected a deadline error, got: {error}"
2031 );
2032 // The deadline (1s), not the 30s sleep, decided it — proof the command
2033 // was actually interrupted rather than run to completion.
2034 assert!(
2035 started.elapsed() < std::time::Duration::from_secs(20),
2036 "the step deadline must fire well before the sleep would finish"
2037 );
2038 }
2039
2040 /// A demo app that publishes linux and has the all-targets-green gate on; it
2041 /// declares linux + macos, so publishing linux is gated on macos being green.
2042 async fn gate_state(root: &std::path::Path) -> (AppState, sqlx::SqlitePool) {
2043 let repo = root.join("app");
2044 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2045 std::fs::write(
2046 repo.join("src-tauri/tauri.conf.json"),
2047 r#"{"version":"0.2.0"}"#,
2048 )
2049 .unwrap();
2050 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2051 let artifact = repo.join("out/app.tar.gz");
2052 std::fs::write(
2053 repo.join("dist/recipes/linux.rhai"),
2054 r#"
2055 step("build");
2056 sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
2057 step("publish");
2058 publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
2059 "#
2060 .replace("ARTIFACT", artifact.to_str().unwrap())
2061 .replace("REPO", repo.to_str().unwrap()),
2062 )
2063 .unwrap();
2064 std::fs::write(
2065 repo.join("bento.toml"),
2066 "targets = [\"linux/x86_64\", \"macos/aarch64\"]\nrequire_all_targets = true\n",
2067 )
2068 .unwrap();
2069 let cfg = Config::for_tests(root);
2070 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2071 let topo = Topology::from_str_for_tests(&format!(
2072 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2073 [[host]]\nname = \"mbp\"\nssh = \"mbp\"\ntargets = [\"macos/aarch64\"]\n\
2074 [app.demo]\nrepo = \"{}\"\n",
2075 repo.display()
2076 ))
2077 .unwrap();
2078 (test_state(pool.clone(), topo, cfg), pool)
2079 }
2080
2081 async fn await_target(pool: &sqlx::SqlitePool, build_id: i64) -> (String, String) {
2082 for _ in 0..100 {
2083 let row: Option<(String, Option<String>)> =
2084 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2085 .bind(build_id)
2086 .fetch_optional(pool)
2087 .await
2088 .unwrap();
2089 if let Some((s, e)) = row
2090 && s != "running"
2091 {
2092 return (s, e.unwrap_or_default());
2093 }
2094 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2095 }
2096 panic!("target never settled");
2097 }
2098
2099 /// The all-targets-green gate blocks a partial release: linux tries to
2100 /// publish while macos has no successful run, so publish is refused and the
2101 /// target fails at its publish step.
2102 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2103 async fn all_green_gate_blocks_publish_when_a_sibling_is_not_green() {
2104 let tmp = tempfile::tempdir().unwrap();
2105 let (state, pool) = gate_state(tmp.path()).await;
2106 let build_id = start_build(
2107 state,
2108 AppId::new("demo"),
2109 Version::parse("0.2.0").unwrap(),
2110 vec!["linux/x86_64".parse().unwrap()],
2111 )
2112 .await
2113 .unwrap();
2114 let (status, error) = await_target(&pool, build_id).await;
2115 assert_eq!(status, "failed", "a partial release must be blocked");
2116 assert!(
2117 error.contains("all-targets-green gate"),
2118 "expected the gate to name itself, got: {error}"
2119 );
2120 // Nothing shipped.
2121 let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
2122 .fetch_one(&pool)
2123 .await
2124 .unwrap();
2125 assert_eq!(count, 0, "a gated-off publish records no release");
2126 }
2127
2128 /// With every sibling green, the gate lets the publish through. macos is
2129 /// pre-recorded `ok` for this version, so linux's publish proceeds.
2130 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2131 async fn all_green_gate_allows_publish_when_every_sibling_is_green() {
2132 let tmp = tempfile::tempdir().unwrap();
2133 let (state, pool) = gate_state(tmp.path()).await;
2134 // Pre-record a green macos run at 0.2.0 (as if its target already built).
2135 let bid: i64 = sqlx::query_scalar(
2136 "INSERT INTO builds (app, version, status, created_at) VALUES ('demo','0.2.0','ok','2026-07-23T00:00:00Z') RETURNING id",
2137 )
2138 .fetch_one(&pool)
2139 .await
2140 .unwrap();
2141 sqlx::query(
2142 "INSERT INTO target_runs (build_id, app, version, target, status, started_at)
2143 VALUES (?, 'demo', '0.2.0', 'macos/aarch64', 'ok', '2026-07-23T00:00:00Z')",
2144 )
2145 .bind(bid)
2146 .execute(&pool)
2147 .await
2148 .unwrap();
2149
2150 let build_id = start_build(
2151 state,
2152 AppId::new("demo"),
2153 Version::parse("0.2.0").unwrap(),
2154 vec!["linux/x86_64".parse().unwrap()],
2155 )
2156 .await
2157 .unwrap();
2158 let (status, error) = await_target(&pool, build_id).await;
2159 assert_eq!(
2160 status, "ok",
2161 "all siblings green -> publish proceeds ({error})"
2162 );
2163 let count: i64 =
2164 sqlx::query_scalar("SELECT COUNT(*) FROM releases WHERE target = 'linux/x86_64'")
2165 .fetch_one(&pool)
2166 .await
2167 .unwrap();
2168 assert_eq!(count, 1, "linux published once the gate was satisfied");
2169 }
2170
2171 /// A committed + tagged git repo whose linux recipe pins the host to the tag
2172 /// via `checkout_sha`. `tag` is created only when `Some`.
2173 fn init_git_app(repo: &std::path::Path, tauri_version: &str, tag: Option<&str>) {
2174 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2175 std::fs::write(
2176 repo.join("src-tauri/tauri.conf.json"),
2177 format!("{{\"version\":\"{tauri_version}\"}}"),
2178 )
2179 .unwrap();
2180 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2181 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2182 std::fs::write(
2183 repo.join("dist/recipes/linux.rhai"),
2184 "step(\"checkout\");\nlet s = checkout_sha(build_host());\nlog(\"pinned \" + s);\n\
2185 step(\"build\");\nsh_ok(build_host(), \"true\");\n",
2186 )
2187 .unwrap();
2188 // Isolate from the dev's global git config (which forces signed tags).
2189 let run = |args: &[&str]| {
2190 let out = std::process::Command::new("git")
2191 .args(args)
2192 .current_dir(repo)
2193 .env("GIT_CONFIG_GLOBAL", "/dev/null")
2194 .env("GIT_CONFIG_SYSTEM", "/dev/null")
2195 .output()
2196 .expect("git runs");
2197 assert!(
2198 out.status.success(),
2199 "git {args:?}: {}",
2200 String::from_utf8_lossy(&out.stderr)
2201 );
2202 };
2203 run(&["init", "-q"]);
2204 run(&["-c", "user.email=t@t", "-c", "user.name=t", "add", "-A"]);
2205 run(&[
2206 "-c",
2207 "user.email=t@t",
2208 "-c",
2209 "user.name=t",
2210 "commit",
2211 "-q",
2212 "-m",
2213 "init",
2214 ]);
2215 if let Some(t) = tag {
2216 run(&["tag", t]);
2217 }
2218 }
2219
2220 fn one_host_topo(repo: &std::path::Path) -> Topology {
2221 Topology::from_str_for_tests(&format!(
2222 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2223 [app.demo]\nrepo = \"{}\"\n",
2224 repo.display()
2225 ))
2226 .unwrap()
2227 }
2228
2229 /// The release preflight (pin on) pins the host to the tag and the build
2230 /// proceeds. Exercises both the barrier and the `checkout_sha` host function.
2231 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2232 async fn release_preflight_pins_and_builds_when_the_tag_is_present() {
2233 let tmp = tempfile::tempdir().unwrap();
2234 let repo = tmp.path().join("demo");
2235 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2236 let mut cfg = Config::for_tests(tmp.path());
2237 cfg.pin_release_sha = true;
2238 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2239 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2240 let build_id = start_build(
2241 state,
2242 AppId::new("demo"),
2243 Version::parse("0.0.1").unwrap(),
2244 vec!["linux/x86_64".parse().unwrap()],
2245 )
2246 .await
2247 .unwrap();
2248 let (status, error) = await_target(&pool, build_id).await;
2249 assert_eq!(status, "ok", "a pinned build should succeed ({error})");
2250 }
2251
2252 /// The preflight refuses a host whose tree has uncommitted changes to
2253 /// tracked files.
2254 ///
2255 /// This is the hole the rev-parse barrier cannot see. `git checkout <tag>`
2256 /// does NOT fail on local modifications to files whose content is unchanged
2257 /// in the tag — it succeeds and keeps them. So a dirty host builds its edits
2258 /// while honestly reporting the tagged sha, and a clean host builds
2259 /// something else: two architectures, two different binaries, one tag
2260 /// containing neither combination. Caught on pom's first release, where
2261 /// fw13 had uncommitted changes in the serve path and astra did not.
2262 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2263 async fn release_preflight_refuses_a_dirty_working_tree() {
2264 let tmp = tempfile::tempdir().unwrap();
2265 let repo = tmp.path().join("demo");
2266 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2267
2268 // Modify a TRACKED file, exactly as an editor session would.
2269 let tracked = repo.join("src-tauri/tauri.conf.json");
2270 let body = std::fs::read_to_string(&tracked).unwrap();
2271 std::fs::write(&tracked, format!("{body}\n")).unwrap();
2272
2273 let mut cfg = Config::for_tests(tmp.path());
2274 cfg.pin_release_sha = true;
2275 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2276 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2277 let err = start_build(
2278 state,
2279 AppId::new("demo"),
2280 Version::parse("0.0.1").unwrap(),
2281 vec!["linux/x86_64".parse().unwrap()],
2282 )
2283 .await
2284 .unwrap_err();
2285 let msg = format!("{err:#}");
2286 assert!(
2287 msg.contains("uncommitted changes") && msg.contains("tauri.conf.json"),
2288 "the error must name the host and the files, got: {msg}"
2289 );
2290 let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2291 .fetch_one(&pool)
2292 .await
2293 .unwrap();
2294 assert_eq!(builds, 0, "a refused preflight writes no build row");
2295 }
2296
2297 /// An UNTRACKED file does not fail a release. A build host accumulates
2298 /// editor scratch and stray logs, none of which reach the binary, so failing
2299 /// on them would be noise that trains an operator to bypass the gate.
2300 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2301 async fn release_preflight_tolerates_untracked_files() {
2302 let tmp = tempfile::tempdir().unwrap();
2303 let repo = tmp.path().join("demo");
2304 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2305 std::fs::write(repo.join("scratch.log"), "noise").unwrap();
2306
2307 let mut cfg = Config::for_tests(tmp.path());
2308 cfg.pin_release_sha = true;
2309 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2310 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2311 let build_id = start_build(
2312 state,
2313 AppId::new("demo"),
2314 Version::parse("0.0.1").unwrap(),
2315 vec!["linux/x86_64".parse().unwrap()],
2316 )
2317 .await
2318 .unwrap();
2319 let (status, error) = await_target(&pool, build_id).await;
2320 assert_eq!(
2321 status, "ok",
2322 "untracked files must not fail a release ({error})"
2323 );
2324 }
2325
2326 /// The preflight refuses the build (before any row is written) when the
2327 /// release tag does not exist on the host — a missing/unpushed tag can't
2328 /// silently fall back to whatever `main` is.
2329 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2330 async fn release_preflight_refuses_when_the_release_tag_is_missing() {
2331 let tmp = tempfile::tempdir().unwrap();
2332 let repo = tmp.path().join("demo");
2333 // App is 0.0.2 (so the version check passes) but only v0.0.1 is tagged.
2334 init_git_app(&repo, "0.0.2", Some("v0.0.1"));
2335 let mut cfg = Config::for_tests(tmp.path());
2336 cfg.pin_release_sha = true;
2337 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2338 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2339 let err = start_build(
2340 state,
2341 AppId::new("demo"),
2342 Version::parse("0.0.2").unwrap(),
2343 vec!["linux/x86_64".parse().unwrap()],
2344 )
2345 .await
2346 .unwrap_err();
2347 let msg = format!("{err:#}");
2348 assert!(
2349 msg.contains("release preflight") && msg.contains("v0.0.2"),
2350 "expected a preflight tag error, got: {msg}"
2351 );
2352 assert!(
2353 msg.contains("does not exist"),
2354 "the error should name the absent tag as the cause, got: {msg}"
2355 );
2356 // Refused before anything was recorded.
2357 let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2358 .fetch_one(&pool)
2359 .await
2360 .unwrap();
2361 assert_eq!(builds, 0, "a refused preflight writes no build row");
2362 }
2363
2364 /// The branch a release was launched from is still checked out afterwards.
2365 ///
2366 /// The preflight pins every host to `v<version>`, which detaches HEAD. That
2367 /// is correct during the build and wrong to leave behind: git does not warn,
2368 /// and later commits succeed while belonging to no branch. makeover shipped
2369 /// 2.3.0 from exactly that state on 2026-07-28 — the published commit existed
2370 /// only as a detached HEAD on one machine, on no branch and no remote.
2371 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2372 async fn a_release_leaves_the_checkout_on_its_branch() {
2373 let tmp = tempfile::tempdir().unwrap();
2374 let repo = tmp.path().join("demo");
2375 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2376 let branch_before = current_branch(&repo);
2377 assert!(!branch_before.is_empty(), "test repo starts on a branch");
2378
2379 let mut cfg = Config::for_tests(tmp.path());
2380 cfg.pin_release_sha = true;
2381 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2382 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2383 let build_id = start_build(
2384 state,
2385 AppId::new("demo"),
2386 Version::parse("0.0.1").unwrap(),
2387 vec!["linux/x86_64".parse().unwrap()],
2388 )
2389 .await
2390 .unwrap();
2391 let (status, error) = await_target(&pool, build_id).await;
2392 assert_eq!(status, "ok", "the build itself should pass ({error})");
2393
2394 // finalize_build restores after the targets settle, so give it a moment.
2395 for _ in 0..50 {
2396 if current_branch(&repo) == branch_before {
2397 break;
2398 }
2399 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2400 }
2401 assert_eq!(
2402 current_branch(&repo),
2403 branch_before,
2404 "the release must put the checkout back on its branch, not leave it detached"
2405 );
2406 }
2407
2408 /// A checkout that is ALREADY detached is refused, rather than released from
2409 /// and left detached again. It means an earlier release never cleaned up, and
2410 /// anything committed since is on no branch — which is precisely the state
2411 /// that has to be looked at by a human before more releases pile onto it.
2412 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2413 async fn a_detached_checkout_is_refused_before_it_is_pinned_again() {
2414 let tmp = tempfile::tempdir().unwrap();
2415 let repo = tmp.path().join("demo");
2416 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2417 // Detach, standing in for a checkout an earlier release left on its tag.
2418 let out = std::process::Command::new("git")
2419 .args(["checkout", "--detach", "-q", "HEAD"])
2420 .current_dir(&repo)
2421 .env("GIT_CONFIG_GLOBAL", "/dev/null")
2422 .env("GIT_CONFIG_SYSTEM", "/dev/null")
2423 .output()
2424 .expect("git runs");
2425 assert!(out.status.success());
2426 assert!(current_branch(&repo).is_empty(), "repo is detached");
2427
2428 let mut cfg = Config::for_tests(tmp.path());
2429 cfg.pin_release_sha = true;
2430 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2431 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2432 let err = start_build(
2433 state,
2434 AppId::new("demo"),
2435 Version::parse("0.0.1").unwrap(),
2436 vec!["linux/x86_64".parse().unwrap()],
2437 )
2438 .await
2439 .unwrap_err();
2440 let msg = format!("{err:#}");
2441 assert!(
2442 msg.contains("detached HEAD"),
2443 "the refusal should name the detached checkout, got: {msg}"
2444 );
2445 let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
2446 .fetch_one(&pool)
2447 .await
2448 .unwrap();
2449 assert_eq!(builds, 0, "a refused preflight writes no build row");
2450 }
2451
2452 /// The branch `repo` is on, empty on a detached HEAD.
2453 fn current_branch(repo: &std::path::Path) -> String {
2454 let out = std::process::Command::new("git")
2455 .args(["symbolic-ref", "-q", "--short", "HEAD"])
2456 .current_dir(repo)
2457 .env("GIT_CONFIG_GLOBAL", "/dev/null")
2458 .env("GIT_CONFIG_SYSTEM", "/dev/null")
2459 .output()
2460 .expect("git runs");
2461 String::from_utf8_lossy(&out.stdout).trim().to_string()
2462 }
2463
2464 /// An unreachable remote does not fail a release whose tag is present.
2465 ///
2466 /// The preflight used to run `fetch --all --tags --prune && checkout`, and
2467 /// `fetch --all` is non-zero if ANY remote fails. Every library repo carries
2468 /// three (`astra`, `mnw`, `srht`), so one dead mirror aborted the release and
2469 /// blamed it on a missing tag — which is how makeover v2.1.1 failed on
2470 /// 2026-07-26 four minutes after its tag was created. Fetch is advisory now;
2471 /// only the checkout decides.
2472 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2473 async fn a_dead_remote_does_not_fail_a_release_whose_tag_exists() {
2474 let tmp = tempfile::tempdir().unwrap();
2475 let repo = tmp.path().join("demo");
2476 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2477 // A remote that cannot possibly be fetched, standing in for an offline
2478 // astra or an srht mirror the repo was never pushed to.
2479 let out = std::process::Command::new("git")
2480 .args([
2481 "remote",
2482 "add",
2483 "srht",
2484 &tmp.path().join("nowhere.git").display().to_string(),
2485 ])
2486 .current_dir(&repo)
2487 .env("GIT_CONFIG_GLOBAL", "/dev/null")
2488 .env("GIT_CONFIG_SYSTEM", "/dev/null")
2489 .output()
2490 .expect("git runs");
2491 assert!(out.status.success());
2492
2493 let mut cfg = Config::for_tests(tmp.path());
2494 cfg.pin_release_sha = true;
2495 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2496 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2497 let build_id = start_build(
2498 state,
2499 AppId::new("demo"),
2500 Version::parse("0.0.1").unwrap(),
2501 vec!["linux/x86_64".parse().unwrap()],
2502 )
2503 .await
2504 .expect("a dead mirror must not refuse the release");
2505 let (status, error) = await_target(&pool, build_id).await;
2506 assert_eq!(status, "ok", "the tag is present, so this builds ({error})");
2507 }
2508
2509 /// The audit fix: a `build` step dispatched to a host whose executor lacks the
2510 /// `build` capability is denied at the transport BEFORE the command runs. This
2511 /// is the structural guarantee behind "never build on prod" — a recipe naming
2512 /// the wrong host can't compile there.
2513 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2514 async fn build_on_a_host_without_the_build_grant_is_denied() {
2515 let tmp = tempfile::tempdir().unwrap();
2516 let root = tmp.path();
2517
2518 let repo = root.join("app");
2519 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2520 std::fs::write(
2521 repo.join("src-tauri/tauri.conf.json"),
2522 r#"{"version":"0.0.1"}"#,
2523 )
2524 .unwrap();
2525 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2526 // The recipe (wrongly) tries to compile on `prod`, a host with no build
2527 // grant. A marker file would appear if the command actually ran.
2528 let marker = root.join("ran-on-prod");
2529 std::fs::write(
2530 repo.join("dist/recipes/linux.rhai"),
2531 r#"
2532 step("build");
2533 sh_ok("prod", "touch MARKER");
2534 "#
2535 .replace("MARKER", marker.to_str().unwrap()),
2536 )
2537 .unwrap();
2538
2539 let cfg = Config::for_tests(root);
2540 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2541 // fw13 builds linux; `prod` is local-but-restart-only (no build/package).
2542 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2543 let topo = Topology::from_str_for_tests(
2544 r#"
2545 [[host]]
2546 name = "fw13"
2547 ssh = "local"
2548 targets = ["linux/x86_64"]
2549
2550 [[host]]
2551 name = "prod"
2552 ssh = "local"
2553 actuate = ["restart"]
2554 observe = []
2555
2556 [app.demo]
2557 repo = "REPO"
2558 "#
2559 .replace("REPO", repo.to_str().unwrap())
2560 .as_str(),
2561 )
2562 .unwrap();
2563
2564 let state = test_state(pool.clone(), topo, cfg);
2565
2566 let build_id = start_build(
2567 state.clone(),
2568 AppId::new("demo"),
2569 Version::parse("0.0.1").unwrap(),
2570 vec!["linux/x86_64".parse().unwrap()],
2571 )
2572 .await
2573 .unwrap();
2574
2575 let mut status = String::new();
2576 let mut error = String::new();
2577 for _ in 0..100 {
2578 let row: Option<(String, Option<String>)> =
2579 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2580 .bind(build_id)
2581 .fetch_optional(&pool)
2582 .await
2583 .unwrap();
2584 if let Some((s, e)) = row {
2585 status = s;
2586 error = e.unwrap_or_default();
2587 if status != "running" {
2588 break;
2589 }
2590 }
2591 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2592 }
2593
2594 assert_eq!(status, "failed", "build on an ungranted host must fail");
2595 assert!(
2596 error.contains("capability denied") && error.contains("build"),
2597 "failure must be a capability denial, got: {error}"
2598 );
2599 assert!(!marker.exists(), "denied build step must NOT have executed");
2600 }
2601
2602 // ---- macOS sign / notarize / staple execution chain (via ScriptedExec) ----
2603
2604 /// Stand up a single-macOS-target app whose recipe is `recipe_body` (with the
2605 /// literal `ARTIFACT` replaced by a real, non-empty file on disk), dispatch
2606 /// every host command through `scripted`, run the build to a terminal state,
2607 /// and return the pool plus the final `(status, error)`. The returned
2608 /// [`tempfile::TempDir`] must be kept alive by the caller: it holds the
2609 /// sqlite DB the pool reads.
2610 async fn run_macos_recipe(
2611 scripted: Arc<ScriptedExec>,
2612 recipe_body: &str,
2613 backoff_secs: Option<u64>,
2614 ) -> (tempfile::TempDir, SqlitePool, String, String) {
2615 let tmp = tempfile::tempdir().unwrap();
2616 let root = tmp.path();
2617 let repo = root.join("app");
2618 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2619 std::fs::write(
2620 repo.join("src-tauri/tauri.conf.json"),
2621 r#"{"version":"0.0.1"}"#,
2622 )
2623 .unwrap();
2624 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2625
2626 // A real, non-empty artifact so `publish`'s size floor is satisfied; the
2627 // build step is faked, so nothing else creates it.
2628 let artifact = repo.join("out/demo.dmg");
2629 std::fs::create_dir_all(artifact.parent().unwrap()).unwrap();
2630 std::fs::write(&artifact, b"dmg-bytes").unwrap();
2631
2632 std::fs::write(
2633 repo.join("dist/recipes/macos.rhai"),
2634 recipe_body.replace("ARTIFACT", artifact.to_str().unwrap()),
2635 )
2636 .unwrap();
2637
2638 let cfg = Config {
2639 notarize_backoff_secs: backoff_secs,
2640 ..Config::for_tests(root)
2641 };
2642 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2643 std::fs::write(repo.join("bento.toml"), "targets = [\"macos/aarch64\"]\n").unwrap();
2644 let topo = Topology::from_str_for_tests(&format!(
2645 r#"
2646 [[host]]
2647 name = "mbp"
2648 ssh = "local"
2649 targets = ["macos/aarch64"]
2650
2651 [app.demo]
2652 repo = "{}"
2653 "#,
2654 repo.display()
2655 ))
2656 .unwrap();
2657
2658 let mut state = test_state(pool.clone(), topo, cfg);
2659 // Route every host command through the scripted executor.
2660 let mut execs = HashMap::new();
2661 execs.insert("mbp".to_string(), scripted as Arc<dyn Executor>);
2662 state.executors = Arc::new(execs);
2663
2664 let build_id = start_build(
2665 state.clone(),
2666 AppId::new("demo"),
2667 Version::parse("0.0.1").unwrap(),
2668 vec!["macos/aarch64".parse().unwrap()],
2669 )
2670 .await
2671 .unwrap();
2672
2673 let mut status = String::new();
2674 let mut error = String::new();
2675 for _ in 0..100 {
2676 let row: Option<(String, Option<String>)> =
2677 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2678 .bind(build_id)
2679 .fetch_optional(&pool)
2680 .await
2681 .unwrap();
2682 if let Some((s, e)) = row {
2683 status = s;
2684 error = e.unwrap_or_default();
2685 if status != "running" {
2686 break;
2687 }
2688 }
2689 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2690 }
2691 (tmp, pool, status, error)
2692 }
2693
2694 async fn release_count(pool: &SqlitePool) -> i64 {
2695 sqlx::query_scalar("SELECT COUNT(*) FROM releases")
2696 .fetch_one(pool)
2697 .await
2698 .unwrap()
2699 }
2700
2701 /// The whole macOS release chain end to end through a fake host: codesign,
2702 /// notarize (Accepted first try), staple, verify_gatekeeper, then publish.
2703 /// The recorded commands lock the exact incantations each host function
2704 /// dispatches, and a `releases` row proves the publish gate opened for a
2705 /// signed + notarized + Gatekeeper-accepted artifact.
2706 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2707 async fn a_macos_recipe_signs_notarizes_staples_verifies_and_publishes() {
2708 let scripted = Arc::new(
2709 ScriptedExec::new()
2710 .on("codesign", 0, "")
2711 .on("notarytool", 0, r#"{"status":"Accepted"}"#)
2712 .on("stapler staple", 0, "")
2713 .on(
2714 "spctl",
2715 0,
2716 "source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
2717 ),
2718 );
2719 let recipe = r#"
2720 let h = build_host();
2721 step("build");
2722 sh_ok(h, "echo built");
2723 step("sign");
2724 codesign(h, "Developer ID Application: Test", "ARTIFACT");
2725 notarize(h, "ARTIFACT");
2726 staple(h, "ARTIFACT");
2727 step("verify");
2728 if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
2729 step("publish");
2730 publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2731 "#;
2732 let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, None).await;
2733
2734 assert_eq!(
2735 status, "ok",
2736 "signed+notarized macOS build should publish: {error}"
2737 );
2738 assert_eq!(
2739 release_count(&pool).await,
2740 1,
2741 "publish must record a release"
2742 );
2743
2744 // The exact shell incantations each host function dispatched.
2745 let cmds = scripted.commands();
2746 let has = |needle: &str| cmds.iter().any(|c| c.contains(needle));
2747 assert!(
2748 has("codesign --force --options runtime --timestamp --sign"),
2749 "codesign runtime+timestamp incantation, got: {cmds:?}"
2750 );
2751 assert!(
2752 has("xcrun notarytool submit"),
2753 "notarytool submit: {cmds:?}"
2754 );
2755 assert!(
2756 has("--wait --output-format json"),
2757 "notarytool --wait json: {cmds:?}"
2758 );
2759 assert!(has("xcrun stapler staple"), "stapler staple: {cmds:?}");
2760 assert!(has("spctl --assess"), "gatekeeper assess: {cmds:?}");
2761 }
2762
2763 /// A failing `codesign` fails the sign step and aborts the recipe before
2764 /// publish — an unsigned artifact never ships.
2765 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2766 async fn a_codesign_failure_fails_the_sign_step_and_blocks_publish() {
2767 let scripted = Arc::new(ScriptedExec::new().on("codesign", 1, ""));
2768 let recipe = r#"
2769 let h = build_host();
2770 step("build");
2771 sh_ok(h, "echo built");
2772 step("sign");
2773 codesign(h, "Developer ID Application: Test", "ARTIFACT");
2774 step("publish");
2775 publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2776 "#;
2777 let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
2778
2779 assert_eq!(status, "failed", "a failed codesign must fail the target");
2780 assert!(
2781 error.contains("codesign failed"),
2782 "error names the codesign failure, got: {error}"
2783 );
2784 assert_eq!(
2785 release_count(&pool).await,
2786 0,
2787 "nothing may publish after a codesign failure"
2788 );
2789 }
2790
2791 /// Even if the recipe ignores `verify_gatekeeper`'s returned `false`, the
2792 /// publish gate refuses the artifact: `verify_gatekeeper` both records the
2793 /// rejection and fails its step, and `publish` proves neither passed. This
2794 /// is the defense-in-depth the pure `PublishAuthority::prove` tests assert in
2795 /// isolation, here exercised through the real host-function path.
2796 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2797 async fn a_gatekeeper_rejection_bars_publish_even_if_the_recipe_ignores_it() {
2798 let scripted = Arc::new(
2799 ScriptedExec::new()
2800 .on("codesign", 0, "")
2801 .on("notarytool", 0, r#"{"status":"Accepted"}"#)
2802 .on("stapler staple", 0, "")
2803 // Gatekeeper says no: the sentinel is FAIL, not OK.
2804 .on("spctl", 0, "source=Unnotarized\nBENTO_GATEKEEPER_FAIL"),
2805 );
2806 let recipe = r#"
2807 let h = build_host();
2808 step("build");
2809 sh_ok(h, "echo built");
2810 step("sign");
2811 codesign(h, "Developer ID Application: Test", "ARTIFACT");
2812 notarize(h, "ARTIFACT");
2813 staple(h, "ARTIFACT");
2814 step("verify");
2815 verify_gatekeeper(h, "ARTIFACT");
2816 step("publish");
2817 publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2818 "#;
2819 let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
2820
2821 assert_eq!(
2822 status, "failed",
2823 "a Gatekeeper-rejected artifact must not publish"
2824 );
2825 assert!(
2826 !error.is_empty(),
2827 "the barred publish must surface an error"
2828 );
2829 assert_eq!(
2830 release_count(&pool).await,
2831 0,
2832 "no release for a rejected artifact"
2833 );
2834 }
2835
2836 /// The one flaky, network-bound step: `notarize` retries a non-`Accepted`
2837 /// result and succeeds on a later attempt. Two notarytool calls (reject then
2838 /// accept) then a recorded release prove the retry ran and the chain
2839 /// completed. Backoff is 0 so the retry sleep doesn't stall the test.
2840 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2841 async fn notarize_retries_a_non_accepted_result_then_succeeds() {
2842 let scripted = Arc::new(
2843 ScriptedExec::new()
2844 .on("codesign", 0, "")
2845 .on_seq(
2846 "notarytool",
2847 &[
2848 (0, r#"{"status":"In Progress"}"#),
2849 (0, r#"{"status":"Accepted"}"#),
2850 ],
2851 )
2852 .on("stapler staple", 0, "")
2853 .on(
2854 "spctl",
2855 0,
2856 "source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
2857 ),
2858 );
2859 let recipe = r#"
2860 let h = build_host();
2861 step("build");
2862 sh_ok(h, "echo built");
2863 step("sign");
2864 codesign(h, "Developer ID Application: Test", "ARTIFACT");
2865 notarize(h, "ARTIFACT");
2866 staple(h, "ARTIFACT");
2867 step("verify");
2868 if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
2869 step("publish");
2870 publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2871 "#;
2872 let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
2873
2874 assert_eq!(
2875 status, "ok",
2876 "notarize should succeed on the retry: {error}"
2877 );
2878 assert_eq!(
2879 release_count(&pool).await,
2880 1,
2881 "the retried build still publishes"
2882 );
2883 let notary_calls = scripted
2884 .commands()
2885 .iter()
2886 .filter(|c| c.contains("notarytool"))
2887 .count();
2888 assert_eq!(
2889 notary_calls, 2,
2890 "notarytool ran once, was rejected, then ran again"
2891 );
2892 }
2893
2894 /// `notarize` gives up after its bounded retries: three notarytool attempts,
2895 /// all non-`Accepted`, fail the target and bar publish.
2896 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2897 async fn notarize_fails_the_target_after_exhausting_its_retries() {
2898 let scripted = Arc::new(
2899 ScriptedExec::new()
2900 .on("codesign", 0, "")
2901 // Every attempt reports a still-pending status, never Accepted.
2902 .on("notarytool", 0, r#"{"status":"In Progress"}"#),
2903 );
2904 let recipe = r#"
2905 let h = build_host();
2906 step("build");
2907 sh_ok(h, "echo built");
2908 step("sign");
2909 codesign(h, "Developer ID Application: Test", "ARTIFACT");
2910 notarize(h, "ARTIFACT");
2911 step("publish");
2912 publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2913 "#;
2914 let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
2915
2916 assert_eq!(
2917 status, "failed",
2918 "exhausted notarization must fail the target"
2919 );
2920 assert!(
2921 error.contains("notarization failed after 3 attempts"),
2922 "error names the exhausted retry, got: {error}"
2923 );
2924 assert_eq!(
2925 release_count(&pool).await,
2926 0,
2927 "an unnotarized artifact never publishes"
2928 );
2929 let notary_calls = scripted
2930 .commands()
2931 .iter()
2932 .filter(|c| c.contains("notarytool"))
2933 .count();
2934 assert_eq!(notary_calls, 3, "the retry is bounded at three attempts");
2935 }
2936 // ---- item (5): SSH and Agent transports end to end, via a recording fake ----
2937 //
2938 // Every other test declares `ssh = "local"`, so the runner's two-plane
2939 // routing is only exercised at construction
2940 // (state::agent_host_syncs_over_ssh_never_the_agent): steps run over the
2941 // EXEC transport (state.executors -- SshExec, or the in-session AgentRpc for
2942 // a mac host) while artifacts move over the SYNC transport (state.syncs --
2943 // always ssh, NEVER the agent, whose confined `/pull` would 404 or force
2944 // `pull_root` wide enough to expose `~/.tauri/passwords.env`). These drive a
2945 // real recipe through `start_build` on a NON-local topology, replace the
2946 // real ssh/agent transports with a recording fake, and assert which plane
2947 // handled which operation -- the runtime form of that construction-time
2948 // invariant, on the ssh string every other test pins to "local".
2949
2950 /// Records shell commands and artifact pulls on separate logs, so a test can
2951 /// prove the exec transport built/signed and the sync transport collected --
2952 /// and that neither did the other's job -- without a live host or ssh.
2953 struct RecordingExec {
2954 caps: CapabilitySet,
2955 commands: Arc<std::sync::Mutex<Vec<String>>>,
2956 pulls: Arc<std::sync::Mutex<Vec<String>>>,
2957 }
2958
2959 impl RecordingExec {
2960 fn new() -> Arc<Self> {
2961 Arc::new(Self {
2962 // A mac build host's real grant. Nothing in this fake gates on it
2963 // (the real transports do), but keep it coherent so
2964 // `capabilities()` is not a lie.
2965 caps: CapabilitySet::from_tokens(
2966 ["build", "sign", "notarize", "staple"],
2967 ["build-log", "artifact"],
2968 ),
2969 commands: Arc::new(std::sync::Mutex::new(Vec::new())),
2970 pulls: Arc::new(std::sync::Mutex::new(Vec::new())),
2971 })
2972 }
2973 fn commands(&self) -> Vec<String> {
2974 self.commands.lock().unwrap().clone()
2975 }
2976 fn pulls(&self) -> Vec<String> {
2977 self.pulls.lock().unwrap().clone()
2978 }
2979 }
2980
2981 #[async_trait]
2982 impl Executor for RecordingExec {
2983 async fn run_streaming(
2984 &self,
2985 step: &ops_exec::Step,
2986 _sink: &mut dyn LogSink,
2987 ) -> anyhow::Result<RunOutput> {
2988 self.commands
2989 .lock()
2990 .unwrap()
2991 .push(step.argv.last().cloned().unwrap_or_default());
2992 Ok(RunOutput {
2993 status: std::process::ExitStatus::from_raw(0),
2994 stdout: Vec::new(),
2995 stderr: Vec::new(),
2996 })
2997 }
2998 async fn pull_file(
2999 &self,
3000 r: &std::path::Path,
3001 _l: &std::path::Path,
3002 _o: &SyncOpts,
3003 ) -> anyhow::Result<()> {
3004 self.pulls
3005 .lock()
3006 .unwrap()
3007 .push(r.to_string_lossy().into_owned());
3008 Ok(())
3009 }
3010 async fn pull_dir(
3011 &self,
3012 r: &std::path::Path,
3013 _l: &std::path::Path,
3014 _o: &SyncOpts,
3015 ) -> anyhow::Result<()> {
3016 self.pulls
3017 .lock()
3018 .unwrap()
3019 .push(r.to_string_lossy().into_owned());
3020 Ok(())
3021 }
3022 async fn pull_glob(
3023 &self,
3024 g: &str,
3025 _l: &std::path::Path,
3026 _o: &SyncOpts,
3027 ) -> anyhow::Result<()> {
3028 self.pulls.lock().unwrap().push(g.to_string());
3029 Ok(())
3030 }
3031 async fn push_dir(
3032 &self,
3033 _l: &std::path::Path,
3034 _r: &std::path::Path,
3035 _o: &SyncOpts,
3036 ) -> anyhow::Result<()> {
3037 Ok(())
3038 }
3039 async fn preflight(&self) -> anyhow::Result<()> {
3040 Ok(())
3041 }
3042 fn capabilities(&self) -> &CapabilitySet {
3043 &self.caps
3044 }
3045 }
3046
3047 /// Stand up a single-target app (host named `h1`) whose recipe is
3048 /// `recipe_body` (with `REPO` replaced by the checkout path), inject
3049 /// `exec_fake` as the host's EXEC transport and `sync_fake` as its SYNC
3050 /// transport, run the build to a terminal state, and return the tmpdir plus
3051 /// `(status, error)`. Unlike `test_state`'s `build_executors`, this replaces
3052 /// BOTH planes so no real ssh/agent transport is dialed. The returned
3053 /// [`tempfile::TempDir`] holds the sqlite DB and must outlive the caller's
3054 /// assertions.
3055 async fn run_two_plane(
3056 host_toml: &str,
3057 target: &str,
3058 recipe_file: &str,
3059 recipe_body: &str,
3060 exec_fake: Arc<dyn Executor>,
3061 sync_fake: Arc<dyn Executor>,
3062 ) -> (tempfile::TempDir, String, String) {
3063 let tmp = tempfile::tempdir().unwrap();
3064 let root = tmp.path();
3065 let repo = root.join("app");
3066 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
3067 std::fs::write(
3068 repo.join("src-tauri/tauri.conf.json"),
3069 r#"{"version":"0.0.1"}"#,
3070 )
3071 .unwrap();
3072 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
3073 std::fs::write(
3074 repo.join("dist/recipes").join(recipe_file),
3075 recipe_body.replace("REPO", repo.to_str().unwrap()),
3076 )
3077 .unwrap();
3078 std::fs::write(
3079 repo.join("bento.toml"),
3080 format!("targets = [\"{target}\"]\n"),
3081 )
3082 .unwrap();
3083
3084 let cfg = Config::for_tests(root);
3085 let pool = crate::db::open(&cfg.db_path).await.unwrap();
3086 let topo = Topology::from_str_for_tests(&format!(
3087 "{}\n[app.demo]\nrepo = \"{}\"\n",
3088 host_toml.replace("REPO", repo.to_str().unwrap()),
3089 repo.display()
3090 ))
3091 .unwrap();
3092
3093 let mut state = test_state(pool.clone(), topo, cfg);
3094 state.executors = Arc::new(HashMap::from([("h1".to_string(), exec_fake)]));
3095 state.syncs = Arc::new(HashMap::from([("h1".to_string(), sync_fake)]));
3096
3097 let build_id = start_build(
3098 state.clone(),
3099 AppId::new("demo"),
3100 Version::parse("0.0.1").unwrap(),
3101 vec![target.parse().unwrap()],
3102 )
3103 .await
3104 .unwrap();
3105
3106 let mut status = String::new();
3107 let mut error = String::new();
3108 for _ in 0..100 {
3109 let row: Option<(String, Option<String>)> =
3110 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
3111 .bind(build_id)
3112 .fetch_optional(&pool)
3113 .await
3114 .unwrap();
3115 if let Some((s, e)) = row {
3116 status = s;
3117 error = e.unwrap_or_default();
3118 if status != "running" {
3119 break;
3120 }
3121 }
3122 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
3123 }
3124 (tmp, status, error)
3125 }
3126
3127 /// An agent (macOS) host signs over the AGENT transport but is collected
3128 /// from over SSH -- driven through the whole runner, not just `build_sync`.
3129 /// The sign chain's commands land on the exec plane and the artifact pull on
3130 /// the sync plane; crucially, the agent plane is asked to move NOTHING (a
3131 /// regression to one transport would route collect at `AgentRpc::pull_glob`,
3132 /// refused by design, or widen `pull_root` over the secret-bearing home dir).
3133 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3134 async fn an_agent_host_signs_over_the_agent_and_collects_over_ssh_end_to_end() {
3135 let agent = RecordingExec::new(); // exec plane (AgentRpc in prod)
3136 let ssh = RecordingExec::new(); // sync plane (SshExec in prod)
3137 let host = r#"
3138 [[host]]
3139 name = "h1"
3140 ssh = "mbp"
3141 targets = ["macos/aarch64"]
3142 transport = "agent"
3143 agent_url = "http://mbp:8765"
3144 actuate = ["build", "sign", "notarize", "staple"]
3145 observe = ["build-log", "gatekeeper", "artifact"]
3146 pull_root = "REPO"
3147 "#;
3148 let recipe = r#"
3149 let h = build_host();
3150 step("build");
3151 sh_ok(h, "echo built");
3152 step("sign");
3153 codesign(h, "Developer ID Application: Test", "REPO/out/demo.dmg");
3154 step("collect");
3155 collect(h, "REPO/out/*.dmg", "demo", "0.0.1");
3156 "#;
3157 let (_tmp, status, error) = run_two_plane(
3158 host,
3159 "macos/aarch64",
3160 "macos.rhai",
3161 recipe,
3162 agent.clone(),
3163 ssh.clone(),
3164 )
3165 .await;
3166 assert_eq!(status, "ok", "the recipe should complete: {error}");
3167
3168 // The build + sign commands ran on the AGENT (exec) transport.
3169 let agent_cmds = agent.commands();
3170 assert!(
3171 agent_cmds.iter().any(|c| c.contains("codesign")),
3172 "codesign rides the agent exec transport: {agent_cmds:?}"
3173 );
3174 assert!(
3175 agent_cmds.iter().any(|c| c.contains("echo built")),
3176 "the build step rides the agent exec transport: {agent_cmds:?}"
3177 );
3178 // ...and the agent moved NO artifacts. This is the load-bearing half:
3179 // AgentRpc::pull_glob is refused by design, so collect must not touch it.
3180 assert!(
3181 agent.pulls().is_empty(),
3182 "the agent transport must never collect artifacts: {:?}",
3183 agent.pulls()
3184 );
3185
3186 // The artifact was collected over the SSH (sync) transport...
3187 let ssh_pulls = ssh.pulls();
3188 assert!(
3189 ssh_pulls
3190 .iter()
3191 .any(|p| p.contains("demo.dmg") || p.contains("*.dmg")),
3192 "collect rides the ssh sync transport: {ssh_pulls:?}"
3193 );
3194 // ...and the sync transport was never asked to run a build/sign command.
3195 assert!(
3196 ssh.commands().is_empty(),
3197 "the sync transport must never run host commands: {:?}",
3198 ssh.commands()
3199 );
3200 }
3201
3202 /// A plain (non-agent) host whose `ssh` is a remote alias, not "local" --
3203 /// the case every other test avoids. The recipe runs end to end through the
3204 /// fake, proving the runner drives a non-local host and still splits exec
3205 /// (build) from sync (collect) across the two transport maps.
3206 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3207 async fn a_non_local_ssh_host_runs_a_recipe_end_to_end() {
3208 let exec = RecordingExec::new();
3209 let sync = RecordingExec::new();
3210 let host = r#"
3211 [[host]]
3212 name = "h1"
3213 ssh = "astra"
3214 targets = ["linux/x86_64"]
3215 pull_root = "REPO"
3216 "#;
3217 let recipe = r#"
3218 let h = build_host();
3219 step("build");
3220 sh_ok(h, "echo compiling");
3221 step("collect");
3222 collect(h, "REPO/out/demo.bin", "demo", "0.0.1");
3223 "#;
3224 let (_tmp, status, error) = run_two_plane(
3225 host,
3226 "linux/x86_64",
3227 "linux.rhai",
3228 recipe,
3229 exec.clone(),
3230 sync.clone(),
3231 )
3232 .await;
3233 assert_eq!(status, "ok", "the recipe should complete: {error}");
3234
3235 assert!(
3236 exec.commands().iter().any(|c| c.contains("echo compiling")),
3237 "the build command rides the exec transport: {:?}",
3238 exec.commands()
3239 );
3240 assert!(
3241 exec.pulls().is_empty(),
3242 "the exec transport must not collect: {:?}",
3243 exec.pulls()
3244 );
3245 assert!(
3246 sync.pulls().iter().any(|p| p.contains("demo.bin")),
3247 "collect rides the sync transport: {:?}",
3248 sync.pulls()
3249 );
3250 assert!(
3251 sync.commands().is_empty(),
3252 "the sync transport must not run commands: {:?}",
3253 sync.commands()
3254 );
3255 }
3256 }
3257
3258 /// Every recipe of every configured app must parse.
3259 ///
3260 /// A recipe is read and compiled at release time, on the build host, after the
3261 /// checkout has already run — so a typo in one is discovered at the worst
3262 /// possible moment. Compiling them here costs nothing and moves that discovery
3263 /// to `cargo test`. Skips when the live config is absent (CI, another host),
3264 /// like [`crate::topology`]'s live-config smoke test.
3265 #[cfg(test)]
3266 mod live_recipe_smoke {
3267 use crate::topology::{Kind, Topology};
3268 use std::path::Path;
3269
3270 #[test]
3271 fn live_recipes_compile_if_present() {
3272 let Some(home) = std::env::var_os("HOME") else {
3273 return;
3274 };
3275 let path = Path::new(&home).join(".config/bento/bento.toml");
3276 if !path.exists() {
3277 return;
3278 }
3279 let topo = Topology::load(&path).expect("live bento.toml must load");
3280 // Syntax only: the host functions are bound per run against a live
3281 // context, and Rhai resolves calls at eval time regardless.
3282 let engine = rhai::Engine::new();
3283 let mut checked = 0;
3284 for (name, cfg) in &topo.app {
3285 let dir = crate::engine::expand_tilde(&cfg.repo).join(&cfg.recipe_dir);
3286 let files: Vec<String> = match cfg.kind {
3287 Kind::Library => vec!["publish.rhai".into()],
3288 Kind::App | Kind::Service => cfg
3289 .targets
3290 .iter()
3291 .map(|t| format!("{}.rhai", t.platform.as_str()))
3292 .collect(),
3293 };
3294 for file in files {
3295 let p = dir.join(&file);
3296 // An app can ship a target whose recipe is still pending (BB's
3297 // macOS/iOS); a missing file is that, not a parse failure.
3298 let Ok(src) = std::fs::read_to_string(&p) else {
3299 continue;
3300 };
3301 engine
3302 .compile(&src)
3303 .unwrap_or_else(|e| panic!("recipe {name}/{file} does not parse: {e}"));
3304 checked += 1;
3305 }
3306 }
3307 assert!(checked > 0, "live config resolved no readable recipes");
3308 }
3309 }
3310