Skip to main content

max / makenotwork

107.4 KB · 3058 lines History Blame Raw
1 //! Tests for [`super`].
2
3 use super::*;
4 use crate::config::Config;
5 use crate::ota::OtaRegistry;
6 use crate::topology::Topology;
7 use async_trait::async_trait;
8 use ops_exec::{CapabilitySet, Executor, LogSink, RunOutput, SyncOpts};
9 use sqlx::SqlitePool;
10 use std::collections::HashMap;
11 use std::os::unix::process::ExitStatusExt;
12 use std::sync::Arc;
13 use tokio::sync::Mutex;
14
15 /// A no-transport [`Executor`] for the paths that don't need a real command
16 /// to run: its `preflight` is programmable (the agent host hits `/health`
17 /// there, and a dead `ops-agent` must fail the target before the recipe
18 /// dispatches), and every actual op is a success no-op.
19 struct FakeExec {
20 caps: CapabilitySet,
21 preflight_err: Option<String>,
22 }
23
24 impl FakeExec {
25 fn preflight_fails(msg: &str) -> Arc<dyn Executor> {
26 Arc::new(Self {
27 caps: CapabilitySet::default(),
28 preflight_err: Some(msg.to_string()),
29 })
30 }
31 }
32
33 #[async_trait]
34 impl Executor for FakeExec {
35 async fn run_streaming(
36 &self,
37 _step: &ops_exec::Step,
38 _sink: &mut dyn LogSink,
39 ) -> anyhow::Result<RunOutput> {
40 Ok(RunOutput {
41 status: std::process::ExitStatus::from_raw(0),
42 stdout: Vec::new(),
43 stderr: Vec::new(),
44 })
45 }
46 async fn pull_file(
47 &self,
48 _r: &std::path::Path,
49 _l: &std::path::Path,
50 _o: &SyncOpts,
51 ) -> anyhow::Result<()> {
52 Ok(())
53 }
54 async fn pull_dir(
55 &self,
56 _r: &std::path::Path,
57 _l: &std::path::Path,
58 _o: &SyncOpts,
59 ) -> anyhow::Result<()> {
60 Ok(())
61 }
62 async fn pull_glob(&self, _g: &str, _l: &std::path::Path, _o: &SyncOpts) -> anyhow::Result<()> {
63 Ok(())
64 }
65 async fn push_dir(
66 &self,
67 _l: &std::path::Path,
68 _r: &std::path::Path,
69 _o: &SyncOpts,
70 ) -> anyhow::Result<()> {
71 Ok(())
72 }
73 async fn preflight(&self) -> anyhow::Result<()> {
74 match &self.preflight_err {
75 Some(m) => anyhow::bail!("{m}"),
76 None => Ok(()),
77 }
78 }
79 fn capabilities(&self) -> &CapabilitySet {
80 &self.caps
81 }
82 }
83
84 /// A recording, programmable [`Executor`] for the macOS sign chain. Every
85 /// dispatched shell command is captured for assertion, and its exit code +
86 /// stdout is chosen by the first rule whose needle the command contains — so
87 /// a test can make `notarytool` report `Accepted`, make `codesign` fail, or
88 /// make `spctl` emit the Gatekeeper sentinel without a real Mac or SSH. A
89 /// rule may carry a *sequence* of responses (one per successive match) to
90 /// drive the notarize retry loop; the last entry repeats once the sequence
91 /// is exhausted. Unmatched commands succeed as empty no-ops (so a plain
92 /// build `sh_ok` passes), and the sync/preflight ops are no-ops.
93 struct ScriptedExec {
94 caps: CapabilitySet,
95 rules: Vec<ScriptRule>,
96 log: Arc<std::sync::Mutex<Vec<String>>>,
97 }
98
99 struct ScriptRule {
100 needle: String,
101 responses: Vec<(i32, String)>,
102 calls: std::sync::atomic::AtomicUsize,
103 }
104
105 impl ScriptedExec {
106 fn new() -> Self {
107 Self {
108 // A mac host's real grant. Nothing in the dispatch path gates on
109 // it (this fake never calls `gate`), but keep it coherent so
110 // `capabilities()` is not a lie.
111 caps: CapabilitySet::from_tokens(
112 ["build", "sign", "notarize", "staple"],
113 ["build-log", "artifact"],
114 ),
115 rules: Vec::new(),
116 log: Arc::new(std::sync::Mutex::new(Vec::new())),
117 }
118 }
119
120 /// Respond to every command containing `needle` with `(code, stdout)`.
121 fn on(mut self, needle: &str, code: i32, stdout: &str) -> Self {
122 self.rules.push(ScriptRule {
123 needle: needle.to_string(),
124 responses: vec![(code, stdout.to_string())],
125 calls: std::sync::atomic::AtomicUsize::new(0),
126 });
127 self
128 }
129
130 /// Respond to successive `needle` matches with successive responses; the
131 /// last repeats once the list is exhausted. Drives the notarize retry.
132 fn on_seq(mut self, needle: &str, responses: &[(i32, &str)]) -> Self {
133 self.rules.push(ScriptRule {
134 needle: needle.to_string(),
135 responses: responses
136 .iter()
137 .map(|(c, s)| (*c, (*s).to_string()))
138 .collect(),
139 calls: std::sync::atomic::AtomicUsize::new(0),
140 });
141 self
142 }
143
144 /// Every shell command this executor was asked to run, in order.
145 fn commands(&self) -> Vec<String> {
146 self.log.lock().unwrap().clone()
147 }
148 }
149
150 #[async_trait]
151 impl Executor for ScriptedExec {
152 async fn run_streaming(
153 &self,
154 step: &ops_exec::Step,
155 _sink: &mut dyn LogSink,
156 ) -> anyhow::Result<RunOutput> {
157 let cmd = step.argv.last().cloned().unwrap_or_default();
158 self.log.lock().unwrap().push(cmd.clone());
159 let (code, stdout) =
160 self.rules
161 .iter()
162 .find(|r| cmd.contains(&r.needle))
163 .map_or((0, String::new()), |r| {
164 let i = r.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
165 r.responses[i.min(r.responses.len() - 1)].clone()
166 });
167 Ok(RunOutput {
168 // Shift into the wait-status word's exit-code byte so
169 // `ExitStatus::code()` reports `code` exactly (a bare
170 // `from_raw(1)` reads as a signal, yielding `None`).
171 status: std::process::ExitStatus::from_raw(code << 8),
172 stdout: stdout.into_bytes(),
173 stderr: Vec::new(),
174 })
175 }
176 async fn pull_file(
177 &self,
178 _r: &std::path::Path,
179 _l: &std::path::Path,
180 _o: &SyncOpts,
181 ) -> anyhow::Result<()> {
182 Ok(())
183 }
184 async fn pull_dir(
185 &self,
186 _r: &std::path::Path,
187 _l: &std::path::Path,
188 _o: &SyncOpts,
189 ) -> anyhow::Result<()> {
190 Ok(())
191 }
192 async fn pull_glob(&self, _g: &str, _l: &std::path::Path, _o: &SyncOpts) -> anyhow::Result<()> {
193 Ok(())
194 }
195 async fn push_dir(
196 &self,
197 _l: &std::path::Path,
198 _r: &std::path::Path,
199 _o: &SyncOpts,
200 ) -> anyhow::Result<()> {
201 Ok(())
202 }
203 async fn preflight(&self) -> anyhow::Result<()> {
204 Ok(())
205 }
206 fn capabilities(&self) -> &CapabilitySet {
207 &self.caps
208 }
209 }
210
211 /// Assemble an [`AppState`] from the three per-test inputs, filling in the
212 /// executors/syncs (built from `topo`) and the fixed test scaffolding
213 /// (metrics handle, event bus, standard OTA registry, empty active map, no
214 /// token). Every runner test builds the same struct around a different repo
215 /// + recipe; this is that struct in one place.
216 fn test_state(pool: SqlitePool, topo: Topology, cfg: Config) -> AppState {
217 let executors = Arc::new(crate::state::build_executors(&topo));
218 let syncs = Arc::new(crate::state::build_syncs(&topo));
219 let host_locks = crate::state::build_host_locks(&topo);
220 AppState {
221 pool,
222 topo: Arc::new(topo),
223 cfg: Arc::new(cfg),
224 prom: crate::metrics::test_handle(),
225 events: crate::events::channel(),
226 ota: Arc::new(OtaRegistry::standard("https://makenot.work")),
227 executors,
228 syncs,
229 active: Arc::new(Mutex::new(HashMap::new())),
230 api_token: None,
231 host_locks,
232 distribution: Arc::new(Mutex::new(HashMap::new())),
233 http: crate::tls::builder().build().unwrap(),
234 // Port 1 refuses instantly. A test must never probe production, and
235 // a refusal is also faster than any timeout would be.
236 mnw_base_url: "http://127.0.0.1:1".into(),
237 }
238 }
239
240 /// A `kind = "service"` release end to end: build, `glibc_check`, `deploy`,
241 /// and a health assertion the recipe makes itself against the service host.
242 ///
243 /// The whole point of the deploy step is that it dispatches to a host
244 /// OUTSIDE the build topology, so this exercises the resolution
245 /// (target -> `[[deploy]]` entry -> executor registered for the run), the
246 /// staging, and the call into the privileged installer — with a fake
247 /// installer standing in for the root script, which is the one part a test
248 /// cannot run for real.
249 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
250 async fn service_recipe_builds_then_deploys_and_verifies() {
251 let tmp = tempfile::tempdir().unwrap();
252 let root = tmp.path();
253 let repo = root.join("svc");
254 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
255 std::fs::write(repo.join("Cargo.toml"), "[package]\nversion = \"0.4.0\"\n").unwrap();
256
257 // Stand-in for the root installer: same three arguments, records what it
258 // was asked to do instead of writing to /usr/local/bin and restarting a
259 // unit. `install` + a marker file, so the test can assert the binary
260 // that arrived is the binary that was built.
261 let installer = root.join("install-service.sh");
262 std::fs::write(
263 &installer,
264 "#!/bin/sh\nset -eu\ninstall -m 0755 \"$1\" \"$2\"\necho \"restarted $3\" >> \"$2.log\"\n",
265 )
266 .unwrap();
267 std::fs::set_permissions(
268 &installer,
269 <std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o755),
270 )
271 .unwrap();
272 let install_path = root.join("bin/svc");
273 std::fs::create_dir_all(root.join("bin")).unwrap();
274
275 std::fs::write(
276 repo.join("dist/recipes/linux.rhai"),
277 r#"
278 let v = version();
279 step("build");
280 sh_ok("fw13", "mkdir -p REPO/target/release && echo built-BIN > REPO/target/release/svc");
281 step("verify");
282 log(glibc_check("REPO/target/release/svc"));
283 step("deploy");
284 log(deploy("REPO/target/release/svc"));
285 // The recipe owns what "healthy" means, and asserts it itself
286 // against the host it just restarted.
287 sh_ok(deploy_host(), "test -x " + install_path());
288 "#
289 .replace("REPO", repo.to_str().unwrap())
290 .replace("BIN", "0.4.0"),
291 )
292 .unwrap();
293
294 std::fs::write(
295 repo.join("bento.toml"),
296 format!(
297 r#"kind = "service"
298 targets = ["linux/x86_64"]
299 version_path = "Cargo.toml"
300
301 [[deploy]]
302 target = "linux/x86_64"
303 host = "local"
304 install_path = "{}"
305 service = "svc.service"
306 health_url = "http://localhost:9100/api/health"
307 "#,
308 install_path.display()
309 ),
310 )
311 .unwrap();
312
313 let mut cfg = Config::for_tests(root);
314 cfg.deploy_installer = installer.display().to_string();
315 let pool = crate::db::open(&cfg.db_path).await.unwrap();
316 let topo = Topology::from_str_for_tests(&format!(
317 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
318 pull_root = \"{repo}\"\n\n[app.svc]\nrepo = \"{repo}\"\n",
319 repo = repo.display()
320 ))
321 .unwrap();
322 let state = test_state(pool.clone(), topo, cfg);
323
324 let build_id = start_build(
325 state.clone(),
326 AppId::new("svc"),
327 Version::parse("0.4.0").unwrap(),
328 vec!["linux/x86_64".parse().unwrap()],
329 )
330 .await
331 .unwrap();
332
333 let mut status = String::new();
334 for _ in 0..100 {
335 status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
336 .bind(build_id)
337 .fetch_optional(&pool)
338 .await
339 .unwrap()
340 .unwrap_or_else(|| "running".to_string());
341 if status != "running" {
342 break;
343 }
344 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
345 }
346 assert_eq!(status, "ok", "service run should succeed");
347
348 let steps: Vec<(String, String)> = sqlx::query_as(
349 "SELECT step, status FROM step_runs WHERE target_run_id IN \
350 (SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id",
351 )
352 .bind(build_id)
353 .fetch_all(&pool)
354 .await
355 .unwrap();
356 assert_eq!(
357 steps.iter().map(|(s, _)| s.as_str()).collect::<Vec<_>>(),
358 vec!["build", "verify", "deploy"],
359 "a service ends at deploy, not collect"
360 );
361 assert!(steps.iter().all(|(_, st)| st == "ok"), "{steps:?}");
362
363 // The bytes that were built are the bytes that landed, and the unit was
364 // restarted only after the install succeeded.
365 assert_eq!(
366 std::fs::read_to_string(&install_path).unwrap().trim(),
367 "built-0.4.0"
368 );
369 assert!(
370 std::fs::read_to_string(install_path.with_extension("").with_file_name("svc.log"))
371 .unwrap()
372 .contains("restarted svc.service")
373 );
374 }
375
376 /// Stand up a tmp app repo + topology and run a real local recipe end to
377 /// end: step transitions, streamed `sh_ok`, `version_of`, `log`, and a
378 /// `collect` that pulls a built artifact into dist_root.
379 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
380 async fn local_linux_recipe_runs_end_to_end() {
381 let tmp = tempfile::tempdir().unwrap();
382 let root = tmp.path();
383
384 // Fake app checkout: tauri.conf.json + a linux recipe.
385 let repo = root.join("app");
386 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
387 std::fs::write(
388 repo.join("src-tauri/tauri.conf.json"),
389 r#"{"version":"0.0.1"}"#,
390 )
391 .unwrap();
392 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
393 // Build writes an artifact into the repo; collect pulls it to dist_root.
394 std::fs::write(
395 repo.join("dist/recipes/linux.rhai"),
396 r#"
397 step("build");
398 let v = version_of("demo");
399 log("building demo " + v);
400 sh_ok("fw13", "echo compiling; mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
401 step("collect");
402 collect("fw13", "REPO/out/demo.bin", "demo", v);
403 "#
404 .replace("REPO", repo.to_str().unwrap()),
405 )
406 .unwrap();
407
408 let mut cfg = Config::for_tests(root);
409 // Archive to a local directory, so the deposit a real release makes to
410 // astra runs on the same code path here.
411 cfg.archive = Some(crate::config::Archive {
412 host: "local".into(),
413 root: root.join("archive"),
414 });
415 let pool = crate::db::open(&cfg.db_path).await.unwrap();
416 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
417 let topo = Topology::from_str_for_tests(&format!(
418 r#"
419 [[host]]
420 name = "fw13"
421 ssh = "local"
422 targets = ["linux/x86_64"]
423 pull_root = "{repo}"
424
425 [app.demo]
426 repo = "{repo}"
427 "#,
428 repo = repo.display()
429 ))
430 .unwrap();
431
432 let state = test_state(pool.clone(), topo, cfg);
433
434 let app = AppId::new("demo");
435 let version = Version::parse("0.0.1").unwrap();
436 let build_id = start_build(
437 state.clone(),
438 app,
439 version,
440 vec!["linux/x86_64".parse().unwrap()],
441 )
442 .await
443 .unwrap();
444
445 // Wait for the target run to settle. The row may not be inserted on the
446 // first poll (run_target is spawned, not awaited), so treat a missing row
447 // as still-pending rather than an error.
448 let mut status = String::new();
449 for _ in 0..100 {
450 status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
451 .bind(build_id)
452 .fetch_optional(&pool)
453 .await
454 .unwrap()
455 .unwrap_or_else(|| "running".to_string());
456 if status != "running" {
457 break;
458 }
459 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
460 }
461 assert_eq!(status, "ok", "target run should succeed");
462
463 // Both steps recorded and finished ok.
464 let steps: Vec<(String, String)> =
465 sqlx::query_as("SELECT step, status FROM step_runs WHERE target_run_id IN (SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id")
466 .bind(build_id)
467 .fetch_all(&pool)
468 .await
469 .unwrap();
470 let names: Vec<&str> = steps.iter().map(|(s, _)| s.as_str()).collect();
471 assert_eq!(names, vec!["build", "collect"]);
472 assert!(steps.iter().all(|(_, st)| st == "ok"));
473
474 // Artifact landed in dist_root, under its own target, and a step log was
475 // written. Both trees are keyed the same way, `<app>/<version>/<target>/`.
476 let artifact = state.cfg.dist_root.join("demo/0.0.1/linux-x86_64/demo.bin");
477 assert!(artifact.exists(), "collect should copy the artifact");
478 // ...and the same bytes reached the archive, at the same path under its
479 // own root. This is the answer to "where is demo 0.0.1 for linux".
480 let archived = root.join("archive/demo/0.0.1/linux-x86_64/demo.bin");
481 assert!(
482 archived.exists(),
483 "collect should deposit into the archive: {}",
484 archived.display()
485 );
486 assert_eq!(
487 std::fs::read(&archived).unwrap(),
488 std::fs::read(&artifact).unwrap()
489 );
490 // Read the path off the ledger rather than rebuilding it: the log is
491 // named for its step run id, and the point of that is that the row
492 // resolves to exactly one file.
493 let (run_id, log_ref): (i64, String) = sqlx::query_as(
494 "SELECT id, log_ref FROM step_runs WHERE step = 'build' AND target_run_id IN \
495 (SELECT id FROM target_runs WHERE build_id = ?)",
496 )
497 .bind(build_id)
498 .fetch_one(&pool)
499 .await
500 .unwrap();
501 let log = std::path::PathBuf::from(&log_ref);
502 assert_eq!(
503 log,
504 state
505 .cfg
506 .logs_root
507 .join(format!("demo/0.0.1/linux-x86_64/build.{run_id}.log")),
508 "log path should be keyed on the step run id"
509 );
510 assert!(log.exists(), "build step log should exist");
511 let body = std::fs::read_to_string(&log).unwrap();
512 assert!(body.contains("compiling"));
513 assert!(
514 body.starts_with(&format!(
515 "=== bento demo 0.0.1 linux/x86_64 step=build run_id={run_id} "
516 )),
517 "log should open with a run header naming its run: {body}"
518 );
519 }
520
521 /// Multi-target fan-out, which is what the daemon exists for: every other
522 /// runner test drives exactly one target, so nothing covered `start_build`'s
523 /// `JoinSet` fan-out or `finalize_build`'s rollup. One target fails and one
524 /// succeeds — the failure must not abort its sibling, both must land their
525 /// own terminal row, and the build must finalize `failed` because any failed
526 /// target fails the build.
527 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
528 async fn multi_target_fan_out_rolls_up_partial_failure() {
529 let tmp = tempfile::tempdir().unwrap();
530 let root = tmp.path();
531
532 let repo = root.join("app");
533 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
534 std::fs::write(
535 repo.join("src-tauri/tauri.conf.json"),
536 r#"{"version":"0.0.1"}"#,
537 )
538 .unwrap();
539 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
540
541 // Linux succeeds and collects a real artifact. The artifact is the proof
542 // this target ran to completion rather than being torn down when its
543 // sibling failed.
544 std::fs::write(
545 repo.join("dist/recipes/linux.rhai"),
546 r#"
547 step("build");
548 let v = version_of("demo");
549 sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo.bin");
550 step("collect");
551 collect("fw13", "REPO/out/demo.bin", "demo", v);
552 "#
553 .replace("REPO", repo.to_str().unwrap()),
554 )
555 .unwrap();
556 // Windows fails in its first step.
557 std::fs::write(
558 repo.join("dist/recipes/windows.rhai"),
559 r#"
560 step("build");
561 sh_ok("winbox", "echo nope 1>&2; exit 1");
562 "#,
563 )
564 .unwrap();
565
566 let cfg = Config::for_tests(root);
567 let pool = crate::db::open(&cfg.db_path).await.unwrap();
568 std::fs::write(
569 repo.join("bento.toml"),
570 "targets = [\"linux/x86_64\", \"windows/x86_64\"]\n",
571 )
572 .unwrap();
573 // Two hosts so each target resolves its own, mirroring the real
574 // per-architecture topology.
575 let topo = Topology::from_str_for_tests(&format!(
576 r#"
577 [[host]]
578 name = "fw13"
579 ssh = "local"
580 targets = ["linux/x86_64"]
581 pull_root = "{repo}"
582
583 [[host]]
584 name = "winbox"
585 ssh = "local"
586 targets = ["windows/x86_64"]
587
588 [app.demo]
589 repo = "{repo}"
590 "#,
591 repo = repo.display()
592 ))
593 .unwrap();
594
595 let state = test_state(pool.clone(), topo, cfg);
596
597 let build_id = start_build(
598 state.clone(),
599 AppId::new("demo"),
600 Version::parse("0.0.1").unwrap(),
601 vec![
602 "linux/x86_64".parse().unwrap(),
603 "windows/x86_64".parse().unwrap(),
604 ],
605 )
606 .await
607 .unwrap();
608
609 // Poll the build row, not the target rows: the build is terminal only
610 // once finalize_build has joined every task and stamped it.
611 let mut build_status = String::new();
612 for _ in 0..200 {
613 build_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
614 .bind(build_id)
615 .fetch_one(&pool)
616 .await
617 .unwrap();
618 if build_status != "running" {
619 break;
620 }
621 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
622 }
623
624 let runs: Vec<(String, String)> =
625 sqlx::query_as("SELECT target, status FROM target_runs WHERE build_id = ? ORDER BY target")
626 .bind(build_id)
627 .fetch_all(&pool)
628 .await
629 .unwrap();
630 assert_eq!(
631 runs,
632 vec![
633 ("linux/x86_64".to_string(), "ok".to_string()),
634 ("windows/x86_64".to_string(), "failed".to_string()),
635 ],
636 "each target lands its own terminal row; one failing does not take the other down",
637 );
638
639 // The surviving target finished its work, not merely its row.
640 assert!(
641 state
642 .cfg
643 .dist_root
644 .join("demo/0.0.1/linux-x86_64/demo.bin")
645 .exists(),
646 "the succeeding target ran to completion and collected its artifact",
647 );
648
649 // The failure is attributed to the target that failed, and only it.
650 let err: Option<String> = sqlx::query_scalar(
651 "SELECT error FROM target_runs WHERE build_id = ? AND target = 'windows/x86_64'",
652 )
653 .bind(build_id)
654 .fetch_one(&pool)
655 .await
656 .unwrap();
657 assert!(
658 err.is_some_and(|e| !e.is_empty()),
659 "a failed target records why",
660 );
661
662 assert_eq!(
663 build_status, "failed",
664 "any failed target fails the build; a partial release must not read as ok",
665 );
666 let finished: Option<String> =
667 sqlx::query_scalar("SELECT finished_at FROM builds WHERE id = ?")
668 .bind(build_id)
669 .fetch_one(&pool)
670 .await
671 .unwrap();
672 assert!(finished.is_some(), "finalize_build stamps the finish time");
673
674 // finalize_build reaps its own latest-wins slots, so nothing is left
675 // in flight to block a later build of the same targets.
676 assert!(
677 state.active.lock().await.is_empty(),
678 "finalize_build reaps the slots it owned",
679 );
680 }
681
682 /// Latest-wins supersession, driven through `start_build` (not the leaf
683 /// `begin_step` bail that's already covered). Two builds of the SAME
684 /// (app, target) race: the second must cooperatively cancel + abort the
685 /// first and take the single slot, the first must terminate non-`ok`, and
686 /// the second must run to completion.
687 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
688 async fn a_newer_build_supersedes_the_in_flight_one_for_the_same_target() {
689 let tmp = tempfile::tempdir().unwrap();
690 let root = tmp.path();
691
692 let repo = root.join("app");
693 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
694 std::fs::write(
695 repo.join("src-tauri/tauri.conf.json"),
696 r#"{"version":"0.0.1"}"#,
697 )
698 .unwrap();
699 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
700 // A sleep long enough that the first build is still mid-recipe when the
701 // second arrives; the engine checks the cancel flag at the step boundary
702 // before "done", so the superseded run bails rather than finishing.
703 std::fs::write(
704 repo.join("dist/recipes/linux.rhai"),
705 r#"
706 step("build");
707 sh_ok("fw13", "sleep 2");
708 "#,
709 )
710 .unwrap();
711
712 let cfg = Config::for_tests(root);
713 let pool = crate::db::open(&cfg.db_path).await.unwrap();
714 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
715 let topo = Topology::from_str_for_tests(&format!(
716 r#"
717 [[host]]
718 name = "fw13"
719 ssh = "local"
720 targets = ["linux/x86_64"]
721
722 [app.demo]
723 repo = "{}"
724 "#,
725 repo.display()
726 ))
727 .unwrap();
728 let state = test_state(pool.clone(), topo, cfg);
729
730 let target = "linux/x86_64".parse().unwrap();
731 let first = start_build(
732 state.clone(),
733 AppId::new("demo"),
734 Version::parse("0.0.1").unwrap(),
735 vec![target],
736 )
737 .await
738 .unwrap();
739 // Let the first build register its slot and enter the recipe before the
740 // second supersedes it (start_build inserts the slot synchronously, but
741 // the recipe runs on a spawned task).
742 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
743 let second = start_build(
744 state.clone(),
745 AppId::new("demo"),
746 Version::parse("0.0.1").unwrap(),
747 vec![target],
748 )
749 .await
750 .unwrap();
751 assert_ne!(first, second);
752
753 // Latest wins: exactly one slot for the key, owned by the second build.
754 {
755 let active = state.active.lock().await;
756 assert_eq!(active.len(), 1, "supersession must not leave two slots");
757 assert_eq!(
758 active.values().next().unwrap().build_id,
759 second,
760 "the surviving slot belongs to the newer build",
761 );
762 }
763
764 // Wait for the second (surviving) build to finalize.
765 let mut second_status = String::new();
766 for _ in 0..200 {
767 second_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
768 .bind(second)
769 .fetch_one(&pool)
770 .await
771 .unwrap();
772 if second_status != "running" {
773 break;
774 }
775 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
776 }
777 assert_eq!(
778 second_status, "ok",
779 "the superseding build runs to completion"
780 );
781
782 // The first build's target run terminated without succeeding — it was
783 // cancelled/aborted, never stamped `ok`.
784 let first_status: String =
785 sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?")
786 .bind(first)
787 .fetch_one(&pool)
788 .await
789 .unwrap();
790 assert_ne!(
791 first_status, "ok",
792 "the superseded build must not complete successfully",
793 );
794
795 // Both builds reaped their slots; nothing left in flight.
796 assert!(
797 state.active.lock().await.is_empty(),
798 "every finalized build reaps its own slot",
799 );
800 }
801
802 /// A failing `preflight` (the agent host's `/health` probe when `ops-agent`
803 /// is down) must fail the target BEFORE the recipe dispatches — the whole
804 /// point of preflighting. Covered only via a fake here, since a real
805 /// LocalExec/SshExec preflight is a no-op that always passes.
806 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
807 async fn a_failing_preflight_fails_the_target_before_the_recipe_runs() {
808 let tmp = tempfile::tempdir().unwrap();
809 let root = tmp.path();
810
811 let repo = root.join("app");
812 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
813 std::fs::write(
814 repo.join("src-tauri/tauri.conf.json"),
815 r#"{"version":"0.0.1"}"#,
816 )
817 .unwrap();
818 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
819 // If the recipe ran it would drop this marker; preflight failing first
820 // means it never does.
821 let marker = root.join("recipe-ran");
822 std::fs::write(
823 repo.join("dist/recipes/linux.rhai"),
824 r#"
825 step("build");
826 sh_ok("fw13", "touch MARKER");
827 "#
828 .replace("MARKER", marker.to_str().unwrap()),
829 )
830 .unwrap();
831
832 let cfg = Config::for_tests(root);
833 let pool = crate::db::open(&cfg.db_path).await.unwrap();
834 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
835 let topo = Topology::from_str_for_tests(&format!(
836 r#"
837 [[host]]
838 name = "fw13"
839 ssh = "local"
840 targets = ["linux/x86_64"]
841
842 [app.demo]
843 repo = "{}"
844 "#,
845 repo.display()
846 ))
847 .unwrap();
848 let mut state = test_state(pool.clone(), topo, cfg);
849 // Swap fw13's executor for one whose preflight fails.
850 let mut execs = HashMap::new();
851 execs.insert(
852 "fw13".to_string(),
853 FakeExec::preflight_fails("ops-agent not reachable at /health"),
854 );
855 state.executors = Arc::new(execs);
856
857 let build_id = start_build(
858 state.clone(),
859 AppId::new("demo"),
860 Version::parse("0.0.1").unwrap(),
861 vec!["linux/x86_64".parse().unwrap()],
862 )
863 .await
864 .unwrap();
865
866 let mut status = String::new();
867 let mut error = String::new();
868 for _ in 0..100 {
869 let row: Option<(String, Option<String>)> =
870 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
871 .bind(build_id)
872 .fetch_optional(&pool)
873 .await
874 .unwrap();
875 if let Some((s, e)) = row {
876 status = s;
877 error = e.unwrap_or_default();
878 if status != "running" {
879 break;
880 }
881 }
882 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
883 }
884 assert_eq!(status, "failed", "a failed preflight must fail the target");
885 assert!(
886 error.contains("ops-agent not reachable"),
887 "the failure must carry the preflight error, got: {error}"
888 );
889 assert!(
890 !marker.exists(),
891 "the recipe must NOT run when preflight fails"
892 );
893 }
894
895 /// A target whose recipe file is absent must fail at the `checkout` boundary
896 /// with a readable "reading recipe" error, before any step runs — exercising
897 /// `read_recipe`'s error branch and the `fail_target` path in `run_target`
898 /// that neither the happy-path nor the failing-preflight test reaches. The
899 /// app is configured to ship linux, but no `linux.rhai` is written.
900 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
901 async fn a_target_with_no_recipe_file_fails_at_checkout() {
902 let tmp = tempfile::tempdir().unwrap();
903 let root = tmp.path();
904
905 let repo = root.join("app");
906 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
907 std::fs::write(
908 repo.join("src-tauri/tauri.conf.json"),
909 r#"{"version":"0.0.1"}"#,
910 )
911 .unwrap();
912 // The recipe dir exists but is empty — no linux.rhai.
913 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
914
915 let cfg = Config::for_tests(root);
916 let pool = crate::db::open(&cfg.db_path).await.unwrap();
917 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
918 let topo = Topology::from_str_for_tests(&format!(
919 r#"
920 [[host]]
921 name = "fw13"
922 ssh = "local"
923 targets = ["linux/x86_64"]
924
925 [app.demo]
926 repo = "{}"
927 "#,
928 repo.display()
929 ))
930 .unwrap();
931 let state = test_state(pool.clone(), topo, cfg);
932
933 let build_id = start_build(
934 state.clone(),
935 AppId::new("demo"),
936 Version::parse("0.0.1").unwrap(),
937 vec!["linux/x86_64".parse().unwrap()],
938 )
939 .await
940 .unwrap();
941
942 // Poll the build row: it is terminal only once finalize_build joins the
943 // one (failing) target task.
944 let mut build_status = String::new();
945 for _ in 0..100 {
946 build_status = sqlx::query_scalar("SELECT status FROM builds WHERE id = ?")
947 .bind(build_id)
948 .fetch_one(&pool)
949 .await
950 .unwrap();
951 if build_status != "running" {
952 break;
953 }
954 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
955 }
956
957 // The target failed, and the error points at the unreadable recipe.
958 let (status, error): (String, Option<String>) =
959 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
960 .bind(build_id)
961 .fetch_one(&pool)
962 .await
963 .unwrap();
964 assert_eq!(status, "failed", "a missing recipe must fail the target");
965 assert!(
966 error.is_some_and(|e| e.contains("reading recipe")),
967 "the failure must name the recipe it could not read",
968 );
969
970 // The recipe never ran, so no step_runs row was ever created — the
971 // failure is at the checkout boundary, ahead of any step.
972 let step_count: i64 = sqlx::query_scalar(
973 "SELECT COUNT(*) FROM step_runs WHERE target_run_id IN \
974 (SELECT id FROM target_runs WHERE build_id = ?)",
975 )
976 .bind(build_id)
977 .fetch_one(&pool)
978 .await
979 .unwrap();
980 assert_eq!(
981 step_count, 0,
982 "no step should run when the recipe is absent"
983 );
984
985 assert_eq!(
986 build_status, "failed",
987 "the build rolls up the target failure",
988 );
989 assert!(
990 state.active.lock().await.is_empty(),
991 "finalize_build reaps the slot even on the recipe-read failure path",
992 );
993 }
994
995 /// Run 2 S5: a publish whose version is not strictly newer than the latest
996 /// already published for the same (app, target, channel) is refused — an
997 /// older build cannot republish over a live newer release.
998 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
999 async fn publish_rejects_a_non_monotonic_version() {
1000 let tmp = tempfile::tempdir().unwrap();
1001 let root = tmp.path();
1002 let repo = root.join("app");
1003 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1004 std::fs::write(
1005 repo.join("src-tauri/tauri.conf.json"),
1006 r#"{"version":"0.2.0"}"#,
1007 )
1008 .unwrap();
1009 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1010 let artifact = repo.join("out/app.tar.gz");
1011 // Build an artifact, publish 0.2.0 (records a release), then try to
1012 // publish the older 0.1.0 — the second publish must fail.
1013 std::fs::write(
1014 repo.join("dist/recipes/linux.rhai"),
1015 r#"
1016 step("build");
1017 sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
1018 step("publish");
1019 publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
1020 publish("tauri-mnw", "demo", "linux/x86_64", "0.1.0", "ARTIFACT", #{});
1021 "#
1022 .replace("ARTIFACT", artifact.to_str().unwrap())
1023 .replace("REPO", repo.to_str().unwrap()),
1024 )
1025 .unwrap();
1026
1027 let cfg = Config::for_tests(root);
1028 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1029 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1030 let topo = Topology::from_str_for_tests(&format!(
1031 r#"
1032 [[host]]
1033 name = "fw13"
1034 ssh = "local"
1035 targets = ["linux/x86_64"]
1036
1037 [app.demo]
1038 repo = "{}"
1039 "#,
1040 repo.display()
1041 ))
1042 .unwrap();
1043 let state = test_state(pool.clone(), topo, cfg);
1044 let build_id = start_build(
1045 state.clone(),
1046 AppId::new("demo"),
1047 Version::parse("0.2.0").unwrap(),
1048 vec!["linux/x86_64".parse().unwrap()],
1049 )
1050 .await
1051 .unwrap();
1052
1053 let mut status = String::new();
1054 let mut error = String::new();
1055 for _ in 0..100 {
1056 let row: Option<(String, Option<String>)> =
1057 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1058 .bind(build_id)
1059 .fetch_optional(&pool)
1060 .await
1061 .unwrap();
1062 if let Some((s, e)) = row {
1063 status = s;
1064 error = e.unwrap_or_default();
1065 if status != "running" {
1066 break;
1067 }
1068 }
1069 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1070 }
1071 assert_eq!(status, "failed", "non-monotonic publish must fail the run");
1072 assert!(
1073 error.contains("not newer"),
1074 "expected monotonicity error, got: {error}"
1075 );
1076 // Exactly one release was recorded (0.2.0); 0.1.0 never landed.
1077 let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
1078 .fetch_one(&pool)
1079 .await
1080 .unwrap();
1081 assert_eq!(
1082 count, 1,
1083 "only the first (newer) publish should record a release"
1084 );
1085 // The recorded release carries the artifact's sha256 (64 hex chars),
1086 // not a NULL — the ledger says which bytes shipped.
1087 let hash: Option<String> =
1088 sqlx::query_scalar("SELECT artifact_hash FROM releases WHERE version = '0.2.0'")
1089 .fetch_one(&pool)
1090 .await
1091 .unwrap();
1092 assert!(
1093 hash.as_deref().is_some_and(|h| h.len() == 64),
1094 "publish must record the artifact sha256, got {hash:?}"
1095 );
1096 }
1097
1098 /// The artifact-identity fix at collect: a stale, wrongly-versioned artifact
1099 /// left in the output dir fails the collect instead of silently winning a
1100 /// later glob. Here the build is 0.0.1 but the recipe produces a 9.9.9 file.
1101 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1102 async fn collect_rejects_a_stale_versioned_artifact() {
1103 let tmp = tempfile::tempdir().unwrap();
1104 let root = tmp.path();
1105 let repo = root.join("app");
1106 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1107 std::fs::write(
1108 repo.join("src-tauri/tauri.conf.json"),
1109 r#"{"version":"0.0.1"}"#,
1110 )
1111 .unwrap();
1112 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1113 std::fs::write(
1114 repo.join("dist/recipes/linux.rhai"),
1115 r#"
1116 step("build");
1117 let v = version_of("demo");
1118 sh_ok("fw13", "mkdir -p REPO/out && echo bin > REPO/out/demo-9.9.9.bin");
1119 step("collect");
1120 collect("fw13", "REPO/out/demo-9.9.9.bin", "demo", v);
1121 "#
1122 .replace("REPO", repo.to_str().unwrap()),
1123 )
1124 .unwrap();
1125
1126 let cfg = Config::for_tests(root);
1127 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1128 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1129 let topo = Topology::from_str_for_tests(&format!(
1130 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
1131 pull_root = \"{repo}\"\n\n[app.demo]\nrepo = \"{repo}\"\n",
1132 repo = repo.display()
1133 ))
1134 .unwrap();
1135 let state = test_state(pool.clone(), topo, cfg);
1136 let build_id = start_build(
1137 state.clone(),
1138 AppId::new("demo"),
1139 Version::parse("0.0.1").unwrap(),
1140 vec!["linux/x86_64".parse().unwrap()],
1141 )
1142 .await
1143 .unwrap();
1144
1145 let mut status = String::new();
1146 let mut error = String::new();
1147 for _ in 0..100 {
1148 let row: Option<(String, Option<String>)> =
1149 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1150 .bind(build_id)
1151 .fetch_optional(&pool)
1152 .await
1153 .unwrap();
1154 if let Some((s, e)) = row {
1155 status = s;
1156 error = e.unwrap_or_default();
1157 if status != "running" {
1158 break;
1159 }
1160 }
1161 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1162 }
1163 assert_eq!(
1164 status, "failed",
1165 "a mismatched-version artifact must fail collect"
1166 );
1167 assert!(
1168 error.contains("stale artifact"),
1169 "expected a stale-artifact error, got: {error}"
1170 );
1171 }
1172
1173 /// A command that runs past its step's deadline fails THAT step and unwinds
1174 /// the recipe. The per-step budget is overridden to 1s here; the recipe then
1175 /// sleeps 30s, so the step deadline, not the sleep, decides the outcome.
1176 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1177 async fn a_step_that_exceeds_its_deadline_fails() {
1178 let tmp = tempfile::tempdir().unwrap();
1179 let root = tmp.path();
1180 let repo = root.join("app");
1181 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1182 std::fs::write(
1183 repo.join("src-tauri/tauri.conf.json"),
1184 r#"{"version":"0.0.1"}"#,
1185 )
1186 .unwrap();
1187 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1188 std::fs::write(
1189 repo.join("dist/recipes/linux.rhai"),
1190 r#"
1191 step("build");
1192 sh_ok("fw13", "sleep 30");
1193 "#,
1194 )
1195 .unwrap();
1196
1197 let mut cfg = Config::for_tests(root);
1198 cfg.step_timeout_secs = Some(1); // every step's budget -> 1s
1199 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1200 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
1201 let topo = Topology::from_str_for_tests(&format!(
1202 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
1203 [app.demo]\nrepo = \"{}\"\n",
1204 repo.display()
1205 ))
1206 .unwrap();
1207 let state = test_state(pool.clone(), topo, cfg);
1208 let started = std::time::Instant::now();
1209 let build_id = start_build(
1210 state.clone(),
1211 AppId::new("demo"),
1212 Version::parse("0.0.1").unwrap(),
1213 vec!["linux/x86_64".parse().unwrap()],
1214 )
1215 .await
1216 .unwrap();
1217
1218 let mut status = String::new();
1219 let mut error = String::new();
1220 for _ in 0..100 {
1221 let row: Option<(String, Option<String>)> =
1222 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1223 .bind(build_id)
1224 .fetch_optional(&pool)
1225 .await
1226 .unwrap();
1227 if let Some((s, e)) = row {
1228 status = s;
1229 error = e.unwrap_or_default();
1230 if status != "running" {
1231 break;
1232 }
1233 }
1234 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1235 }
1236 assert_eq!(status, "failed", "a step past its deadline must fail");
1237 assert!(
1238 error.contains("per-step deadline"),
1239 "expected a deadline error, got: {error}"
1240 );
1241 // The deadline (1s), not the 30s sleep, decided it — proof the command
1242 // was actually interrupted rather than run to completion.
1243 assert!(
1244 started.elapsed() < std::time::Duration::from_secs(20),
1245 "the step deadline must fire well before the sleep would finish"
1246 );
1247 }
1248
1249 /// A demo app that publishes linux and has the all-targets-green gate on; it
1250 /// declares linux + macos, so publishing linux is gated on macos being green.
1251 async fn gate_state(root: &std::path::Path) -> (AppState, sqlx::SqlitePool) {
1252 let repo = root.join("app");
1253 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1254 std::fs::write(
1255 repo.join("src-tauri/tauri.conf.json"),
1256 r#"{"version":"0.2.0"}"#,
1257 )
1258 .unwrap();
1259 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1260 let artifact = repo.join("out/app.tar.gz");
1261 std::fs::write(
1262 repo.join("dist/recipes/linux.rhai"),
1263 r#"
1264 step("build");
1265 sh_ok("fw13", "mkdir -p REPO/out && echo bin > ARTIFACT");
1266 step("publish");
1267 publish("tauri-mnw", "demo", "linux/x86_64", "0.2.0", "ARTIFACT", #{});
1268 "#
1269 .replace("ARTIFACT", artifact.to_str().unwrap())
1270 .replace("REPO", repo.to_str().unwrap()),
1271 )
1272 .unwrap();
1273 std::fs::write(
1274 repo.join("bento.toml"),
1275 "targets = [\"linux/x86_64\", \"macos/aarch64\"]\nrequire_all_targets = true\n",
1276 )
1277 .unwrap();
1278 let cfg = Config::for_tests(root);
1279 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1280 let topo = Topology::from_str_for_tests(&format!(
1281 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
1282 [[host]]\nname = \"mbp\"\nssh = \"mbp\"\ntargets = [\"macos/aarch64\"]\n\
1283 [app.demo]\nrepo = \"{}\"\n",
1284 repo.display()
1285 ))
1286 .unwrap();
1287 (test_state(pool.clone(), topo, cfg), pool)
1288 }
1289
1290 async fn await_target(pool: &sqlx::SqlitePool, build_id: i64) -> (String, String) {
1291 for _ in 0..100 {
1292 let row: Option<(String, Option<String>)> =
1293 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
1294 .bind(build_id)
1295 .fetch_optional(pool)
1296 .await
1297 .unwrap();
1298 if let Some((s, e)) = row
1299 && s != "running"
1300 {
1301 return (s, e.unwrap_or_default());
1302 }
1303 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
1304 }
1305 panic!("target never settled");
1306 }
1307
1308 /// The all-targets-green gate blocks a partial release: linux tries to
1309 /// publish while macos has no successful run, so publish is refused and the
1310 /// target fails at its publish step.
1311 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1312 async fn all_green_gate_blocks_publish_when_a_sibling_is_not_green() {
1313 let tmp = tempfile::tempdir().unwrap();
1314 let (state, pool) = gate_state(tmp.path()).await;
1315 let build_id = start_build(
1316 state,
1317 AppId::new("demo"),
1318 Version::parse("0.2.0").unwrap(),
1319 vec!["linux/x86_64".parse().unwrap()],
1320 )
1321 .await
1322 .unwrap();
1323 let (status, error) = await_target(&pool, build_id).await;
1324 assert_eq!(status, "failed", "a partial release must be blocked");
1325 assert!(
1326 error.contains("all-targets-green gate"),
1327 "expected the gate to name itself, got: {error}"
1328 );
1329 // Nothing shipped.
1330 let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM releases")
1331 .fetch_one(&pool)
1332 .await
1333 .unwrap();
1334 assert_eq!(count, 0, "a gated-off publish records no release");
1335 }
1336
1337 /// With every sibling green, the gate lets the publish through. macos is
1338 /// pre-recorded `ok` for this version, so linux's publish proceeds.
1339 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1340 async fn all_green_gate_allows_publish_when_every_sibling_is_green() {
1341 let tmp = tempfile::tempdir().unwrap();
1342 let (state, pool) = gate_state(tmp.path()).await;
1343 // Pre-record a green macos run at 0.2.0 (as if its target already built).
1344 let bid: i64 = sqlx::query_scalar(
1345 "INSERT INTO builds (app, version, status, created_at) VALUES ('demo','0.2.0','ok','2026-07-23T00:00:00Z') RETURNING id",
1346 )
1347 .fetch_one(&pool)
1348 .await
1349 .unwrap();
1350 sqlx::query(
1351 "INSERT INTO target_runs (build_id, app, version, target, status, started_at)
1352 VALUES (?, 'demo', '0.2.0', 'macos/aarch64', 'ok', '2026-07-23T00:00:00Z')",
1353 )
1354 .bind(bid)
1355 .execute(&pool)
1356 .await
1357 .unwrap();
1358
1359 let build_id = start_build(
1360 state,
1361 AppId::new("demo"),
1362 Version::parse("0.2.0").unwrap(),
1363 vec!["linux/x86_64".parse().unwrap()],
1364 )
1365 .await
1366 .unwrap();
1367 let (status, error) = await_target(&pool, build_id).await;
1368 assert_eq!(
1369 status, "ok",
1370 "all siblings green -> publish proceeds ({error})"
1371 );
1372 let count: i64 =
1373 sqlx::query_scalar("SELECT COUNT(*) FROM releases WHERE target = 'linux/x86_64'")
1374 .fetch_one(&pool)
1375 .await
1376 .unwrap();
1377 assert_eq!(count, 1, "linux published once the gate was satisfied");
1378 }
1379
1380 /// A committed + tagged git repo whose linux recipe pins the host to the tag
1381 /// via `checkout_sha`. `tag` is created only when `Some`.
1382 fn init_git_app(repo: &std::path::Path, tauri_version: &str, tag: Option<&str>) {
1383 init_git_app_with_recipe(
1384 repo,
1385 tauri_version,
1386 tag,
1387 "step(\"checkout\");\nlet s = checkout_sha(build_host());\nlog(\"pinned \" + s);\n\
1388 step(\"build\");\nsh_ok(build_host(), \"true\");\n",
1389 );
1390 }
1391
1392 /// As [`init_git_app`], with the linux recipe spelled by the caller.
1393 fn init_git_app_with_recipe(
1394 repo: &std::path::Path,
1395 tauri_version: &str,
1396 tag: Option<&str>,
1397 recipe: &str,
1398 ) {
1399 init_git_app_shipping(repo, tauri_version, tag, recipe, "[\"linux/x86_64\"]");
1400 }
1401
1402 /// As [`init_git_app_with_recipe`], with the manifest's target list spelled
1403 /// by the caller — a second target is what puts a second HOST in the
1404 /// preflight, which is the only way to reach its unwind path.
1405 fn init_git_app_shipping(
1406 repo: &std::path::Path,
1407 tauri_version: &str,
1408 tag: Option<&str>,
1409 recipe: &str,
1410 targets: &str,
1411 ) {
1412 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
1413 std::fs::write(
1414 repo.join("src-tauri/tauri.conf.json"),
1415 format!("{{\"version\":\"{tauri_version}\"}}"),
1416 )
1417 .unwrap();
1418 std::fs::write(repo.join("bento.toml"), format!("targets = {targets}\n")).unwrap();
1419 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
1420 std::fs::write(repo.join("dist/recipes/linux.rhai"), recipe).unwrap();
1421 // Isolate from the dev's global git config (which forces signed tags).
1422 let run = |args: &[&str]| {
1423 let out = std::process::Command::new("git")
1424 .args(args)
1425 .current_dir(repo)
1426 .env("GIT_CONFIG_GLOBAL", "/dev/null")
1427 .env("GIT_CONFIG_SYSTEM", "/dev/null")
1428 .output()
1429 .expect("git runs");
1430 assert!(
1431 out.status.success(),
1432 "git {args:?}: {}",
1433 String::from_utf8_lossy(&out.stderr)
1434 );
1435 };
1436 run(&["init", "-q"]);
1437 run(&["-c", "user.email=t@t", "-c", "user.name=t", "add", "-A"]);
1438 run(&[
1439 "-c",
1440 "user.email=t@t",
1441 "-c",
1442 "user.name=t",
1443 "commit",
1444 "-q",
1445 "-m",
1446 "init",
1447 ]);
1448 if let Some(t) = tag {
1449 run(&["tag", t]);
1450 }
1451 }
1452
1453 /// A worktree root beside the checkout, the shape production uses: a hidden
1454 /// directory in the tree the repos live in, not inside any of them.
1455 fn worktree_root(repo: &std::path::Path) -> std::path::PathBuf {
1456 repo.parent().expect("repo has a parent").join(".bento")
1457 }
1458
1459 /// Where a release of `demo` out of `repo` builds: the worktree, plus the
1460 /// app's own prefix inside it for a repo holding several products.
1461 fn build_dir(repo: &std::path::Path, prefix: &str) -> std::path::PathBuf {
1462 let repo_dir = repo.file_name().expect("repo has a name");
1463 worktree_root(repo).join(repo_dir).join("demo").join(prefix)
1464 }
1465
1466 fn one_host_topo(repo: &std::path::Path) -> Topology {
1467 Topology::from_str_for_tests(&format!(
1468 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
1469 worktree_root = \"{}\"\n\
1470 [app.demo]\nrepo = \"{}\"\n",
1471 worktree_root(repo).display(),
1472 repo.display()
1473 ))
1474 .unwrap()
1475 }
1476
1477 /// The release preflight (pin on) pins the host to the tag and the build
1478 /// proceeds. Exercises both the barrier and the `checkout_sha` host function.
1479 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1480 async fn release_preflight_pins_and_builds_when_the_tag_is_present() {
1481 let tmp = tempfile::tempdir().unwrap();
1482 let repo = tmp.path().join("demo");
1483 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
1484 let mut cfg = Config::for_tests(tmp.path());
1485 cfg.pin_release_sha = true;
1486 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1487 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
1488 let build_id = start_build(
1489 state,
1490 AppId::new("demo"),
1491 Version::parse("0.0.1").unwrap(),
1492 vec!["linux/x86_64".parse().unwrap()],
1493 )
1494 .await
1495 .unwrap();
1496 let (status, error) = await_target(&pool, build_id).await;
1497 assert_eq!(status, "ok", "a pinned build should succeed ({error})");
1498 assert!(
1499 build_dir(&repo, "")
1500 .join("src-tauri/tauri.conf.json")
1501 .exists(),
1502 "the release must have built in the worktree, not the checkout"
1503 );
1504 assert_eq!(
1505 head_sha(&build_dir(&repo, "")),
1506 tag_sha(&repo, "v0.0.1"),
1507 "and that worktree must be at the release tag"
1508 );
1509 }
1510
1511 /// A pinned build writes an artifact record beside what it collected: the
1512 /// manifest of those bytes, the commit the preflight pinned, and the steps
1513 /// as artifact-scoped gates.
1514 ///
1515 /// The three facts already existed and met nowhere — the per-file sha256 at
1516 /// `collect`, the sha at the preflight, the step outcomes in `step_runs` —
1517 /// which is how gates came to vouch for one thing while the deploy shipped
1518 /// another.
1519 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1520 async fn a_pinned_build_writes_an_artifact_record_for_what_it_collected() {
1521 let tmp = tempfile::tempdir().unwrap();
1522 let repo = tmp.path().join("demo");
1523 std::fs::create_dir_all(&repo).unwrap();
1524 // The recipe builds and collects out of `repo()`, which is the worktree.
1525 // That also proves the collect: an artifact built in a tree the sync gate
1526 // does not trust is refused, and the worktree root is an artifact root
1527 // for exactly this reason (`Host::artifact_roots`).
1528 let recipe = r#"
1529 step("checkout");
1530 let sha = checkout_sha(build_host());
1531 log("pinned " + sha);
1532 step("build");
1533 sh_ok(build_host(), "mkdir -p " + repo() + "/out && echo bin > " + repo() + "/out/demo.bin");
1534 step("collect");
1535 collect(build_host(), repo() + "/out/demo.bin", "demo", version());
1536 "#;
1537 init_git_app_with_recipe(&repo, "0.0.1", Some("v0.0.1"), recipe);
1538
1539 let mut cfg = Config::for_tests(tmp.path());
1540 cfg.pin_release_sha = true;
1541 let dist_root = cfg.dist_root.clone();
1542 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1543 // No `pull_root`: the worktree root is the artifact root here, which is
1544 // the arrangement production uses.
1545 let topo = Topology::from_str_for_tests(&format!(
1546 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
1547 worktree_root = \"{wt}\"\n[app.demo]\nrepo = \"{repo}\"\n",
1548 wt = worktree_root(&repo).display(),
1549 repo = repo.display()
1550 ))
1551 .unwrap();
1552 let state = test_state(pool.clone(), topo, cfg);
1553 let build_id = start_build(
1554 state,
1555 AppId::new("demo"),
1556 Version::parse("0.0.1").unwrap(),
1557 vec!["linux/x86_64".parse().unwrap()],
1558 )
1559 .await
1560 .unwrap();
1561 let (status, error) = await_target(&pool, build_id).await;
1562 assert_eq!(status, "ok", "the build should succeed ({error})");
1563
1564 let path = crate::artifact_record::record_path(
1565 &dist_root,
1566 &AppId::new("demo"),
1567 &Version::parse("0.0.1").unwrap(),
1568 "linux/x86_64".parse().unwrap(),
1569 );
1570 let json = std::fs::read_to_string(&path)
1571 .unwrap_or_else(|e| panic!("record at {}: {e}", path.display()));
1572 // Parsing revalidates, so this also asserts the digest matches the
1573 // manifest and no environment-scoped gate slipped in.
1574 let record = ops_artifact::ArtifactRecord::parse(&json).unwrap();
1575
1576 assert_eq!(record.producer, "bento");
1577 assert_eq!(record.manifest.entries().len(), 1);
1578 assert_eq!(record.manifest.entries()[0].path, "demo.bin");
1579 assert_eq!(record.digest, record.manifest.digest());
1580
1581 // The provenance names the commit the preflight pinned, not a rebuild of
1582 // whatever the branch is now.
1583 let head = std::process::Command::new("git")
1584 .args(["rev-parse", "v0.0.1^{commit}"])
1585 .current_dir(&repo)
1586 .env("GIT_CONFIG_GLOBAL", "/dev/null")
1587 .output()
1588 .unwrap();
1589 let head = String::from_utf8_lossy(&head.stdout).trim().to_string();
1590 assert_eq!(record.provenance.git_sha, head);
1591 assert_eq!(record.provenance.target, "linux/x86_64");
1592 assert_eq!(record.provenance.build_host, "fw13");
1593 assert!(!record.provenance.toolchain.is_empty());
1594
1595 let gates: Vec<&str> = record.gates.iter().map(|g| g.gate.as_str()).collect();
1596 assert_eq!(gates, ["checkout", "build", "collect"]);
1597 assert!(record.all_gates_passed());
1598 assert!(
1599 record
1600 .gates
1601 .iter()
1602 .all(|g| g.scope == ops_artifact::Scope::Artifact),
1603 "a build host cannot vouch for an environment"
1604 );
1605 }
1606
1607 /// The barrier refuses a target no host can build, instead of leaving it out
1608 /// of the pin and letting the remaining hosts vouch for the release.
1609 ///
1610 /// The hole this closes is silence reading as agreement: the comparison used
1611 /// to run over the hosts that reported, so a target dropped for want of a
1612 /// host still built while the other hosts' unanimity looked like proof the
1613 /// whole release came from one commit.
1614 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1615 async fn release_preflight_refuses_a_target_no_host_can_build() {
1616 let tmp = tempfile::tempdir().unwrap();
1617 let repo = tmp.path().join("demo");
1618 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
1619 let mut cfg = Config::for_tests(tmp.path());
1620 cfg.pin_release_sha = true;
1621 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1622 // The topology declares fw13 (linux/x86_64) only, so macOS has no host.
1623 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
1624 let err = start_build(
1625 state,
1626 AppId::new("demo"),
1627 Version::parse("0.0.1").unwrap(),
1628 vec!["macos/aarch64".parse().unwrap()],
1629 )
1630 .await
1631 .unwrap_err();
1632 let msg = format!("{err:#}");
1633 assert!(
1634 msg.contains("no host can build macos/aarch64"),
1635 "the error must name the unbuildable target, got: {msg}"
1636 );
1637 let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
1638 .fetch_one(&pool)
1639 .await
1640 .unwrap();
1641 assert_eq!(builds, 0, "a refused preflight writes no build row");
1642 }
1643
1644 /// A dirty checkout neither refuses a release nor reaches it.
1645 ///
1646 /// `git checkout <tag>` does NOT fail on local modifications to files whose
1647 /// content is unchanged in the tag: it succeeds and keeps them, so a dirty
1648 /// host would build its edits while honestly reporting the tagged sha.
1649 ///
1650 /// The worktree answers that instead of gating on it. The release is built
1651 /// from a tree the edit is not in, so the edit cannot reach the binary and a
1652 /// release is never refused because somebody has unsaved work.
1653 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1654 async fn a_dirty_checkout_neither_refuses_a_release_nor_reaches_it() {
1655 let tmp = tempfile::tempdir().unwrap();
1656 let repo = tmp.path().join("demo");
1657 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
1658
1659 // Modify a TRACKED file, exactly as an editor session would.
1660 let tracked = repo.join("src-tauri/tauri.conf.json");
1661 // Still valid JSON and still version 0.0.1: the version preflight reads
1662 // this file on the daemon's box before anything is pinned, so an edit
1663 // that broke it would fail the release for a reason this test is not
1664 // about.
1665 let edited = "{\"version\":\"0.0.1\",\"unsaved\":true}".to_string();
1666 std::fs::write(&tracked, &edited).unwrap();
1667
1668 let mut cfg = Config::for_tests(tmp.path());
1669 cfg.pin_release_sha = true;
1670 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1671 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
1672 let build_id = start_build(
1673 state,
1674 AppId::new("demo"),
1675 Version::parse("0.0.1").unwrap(),
1676 vec!["linux/x86_64".parse().unwrap()],
1677 )
1678 .await
1679 .unwrap();
1680 let (status, error) = await_target(&pool, build_id).await;
1681 assert_eq!(
1682 status, "ok",
1683 "an edit elsewhere must not stop a release ({error})"
1684 );
1685 assert_eq!(
1686 std::fs::read_to_string(&tracked).unwrap(),
1687 edited,
1688 "and the edit must still be there afterwards"
1689 );
1690 assert_ne!(
1691 std::fs::read_to_string(build_dir(&repo, "").join("src-tauri/tauri.conf.json")).unwrap(),
1692 edited,
1693 "what was built is the tag's content, not the edit"
1694 );
1695 }
1696
1697 /// An UNTRACKED file does not fail a release. A build host accumulates
1698 /// editor scratch and stray logs, none of which reach the binary, so failing
1699 /// on them would be noise that trains an operator to bypass the gate.
1700 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1701 async fn release_preflight_tolerates_untracked_files() {
1702 let tmp = tempfile::tempdir().unwrap();
1703 let repo = tmp.path().join("demo");
1704 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
1705 std::fs::write(repo.join("scratch.log"), "noise").unwrap();
1706
1707 let mut cfg = Config::for_tests(tmp.path());
1708 cfg.pin_release_sha = true;
1709 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1710 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
1711 let build_id = start_build(
1712 state,
1713 AppId::new("demo"),
1714 Version::parse("0.0.1").unwrap(),
1715 vec!["linux/x86_64".parse().unwrap()],
1716 )
1717 .await
1718 .unwrap();
1719 let (status, error) = await_target(&pool, build_id).await;
1720 assert_eq!(
1721 status, "ok",
1722 "untracked files must not fail a release ({error})"
1723 );
1724 }
1725
1726 /// The preflight refuses the build (before any row is written) when the
1727 /// release tag does not exist on the host — a missing/unpushed tag can't
1728 /// silently fall back to whatever `main` is.
1729 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1730 async fn release_preflight_refuses_when_the_release_tag_is_missing() {
1731 let tmp = tempfile::tempdir().unwrap();
1732 let repo = tmp.path().join("demo");
1733 // App is 0.0.2 (so the version check passes) but only v0.0.1 is tagged.
1734 init_git_app(&repo, "0.0.2", Some("v0.0.1"));
1735 let mut cfg = Config::for_tests(tmp.path());
1736 cfg.pin_release_sha = true;
1737 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1738 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
1739 let err = start_build(
1740 state,
1741 AppId::new("demo"),
1742 Version::parse("0.0.2").unwrap(),
1743 vec!["linux/x86_64".parse().unwrap()],
1744 )
1745 .await
1746 .unwrap_err();
1747 let msg = format!("{err:#}");
1748 assert!(
1749 msg.contains("release preflight") && msg.contains("v0.0.2"),
1750 "expected a preflight tag error, got: {msg}"
1751 );
1752 assert!(
1753 msg.contains("does not exist"),
1754 "the error should name the absent tag as the cause, got: {msg}"
1755 );
1756 // Refused before anything was recorded.
1757 let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
1758 .fetch_one(&pool)
1759 .await
1760 .unwrap();
1761 assert_eq!(builds, 0, "a refused preflight writes no build row");
1762 }
1763
1764 /// A release does not touch the checkout at all: same branch, same commit,
1765 /// same working tree.
1766 ///
1767 /// Checking the tag out there would move a tree somebody works in under them
1768 /// for the length of a build, and leave it detached whenever the restore
1769 /// failed. The worktree makes the question moot, so this asserts the strong
1770 /// form rather than a recovery.
1771 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1772 async fn a_release_never_moves_the_checkout() {
1773 let tmp = tempfile::tempdir().unwrap();
1774 let repo = tmp.path().join("demo");
1775 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
1776 let branch_before = current_branch(&repo);
1777 let head_before = head_sha(&repo);
1778 assert!(!branch_before.is_empty(), "test repo starts on a branch");
1779
1780 let mut cfg = Config::for_tests(tmp.path());
1781 cfg.pin_release_sha = true;
1782 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1783 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
1784 let build_id = start_build(
1785 state,
1786 AppId::new("demo"),
1787 Version::parse("0.0.1").unwrap(),
1788 vec!["linux/x86_64".parse().unwrap()],
1789 )
1790 .await
1791 .unwrap();
1792 let (status, error) = await_target(&pool, build_id).await;
1793 assert_eq!(status, "ok", "the build itself should pass ({error})");
1794
1795 assert_eq!(
1796 current_branch(&repo),
1797 branch_before,
1798 "the checkout must still be on its branch"
1799 );
1800 assert_eq!(head_sha(&repo), head_before, "and at the same commit");
1801 assert_eq!(repo_status(&repo), "", "and with the same working tree");
1802 }
1803
1804 /// A detached checkout does not refuse a release.
1805 ///
1806 /// Releases do not move that tree, so its HEAD is not their business.
1807 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1808 async fn a_detached_checkout_no_longer_refuses_a_release() {
1809 let tmp = tempfile::tempdir().unwrap();
1810 let repo = tmp.path().join("demo");
1811 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
1812 // Detach, standing in for a checkout an earlier release left on its tag.
1813 let out = std::process::Command::new("git")
1814 .args(["checkout", "--detach", "-q", "HEAD"])
1815 .current_dir(&repo)
1816 .env("GIT_CONFIG_GLOBAL", "/dev/null")
1817 .env("GIT_CONFIG_SYSTEM", "/dev/null")
1818 .output()
1819 .expect("git runs");
1820 assert!(out.status.success());
1821 assert!(current_branch(&repo).is_empty(), "repo is detached");
1822
1823 let head_before = head_sha(&repo);
1824
1825 let mut cfg = Config::for_tests(tmp.path());
1826 cfg.pin_release_sha = true;
1827 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1828 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
1829 let build_id = start_build(
1830 state,
1831 AppId::new("demo"),
1832 Version::parse("0.0.1").unwrap(),
1833 vec!["linux/x86_64".parse().unwrap()],
1834 )
1835 .await
1836 .unwrap();
1837 let (status, error) = await_target(&pool, build_id).await;
1838 assert_eq!(
1839 status, "ok",
1840 "a detached checkout is not a reason to refuse ({error})"
1841 );
1842 assert!(
1843 current_branch(&repo).is_empty() && head_sha(&repo) == head_before,
1844 "and the release left it exactly as detached as it found it"
1845 );
1846 }
1847
1848 /// A build that rewrites a tracked file does not stop the next release.
1849 ///
1850 /// `cargo build` rewrites `Cargo.lock` under the `[patch]` block, so a build
1851 /// leaves the tree it ran in dirty. The dirt lands in Bento's own worktree
1852 /// and the next release forces past it, which is safe precisely because
1853 /// nothing else writes there. So this releases the same app twice with a
1854 /// build that dirties the tree, and the second one is the assertion.
1855 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1856 async fn a_build_that_dirties_its_tree_does_not_refuse_the_next_release() {
1857 let tmp = tempfile::tempdir().unwrap();
1858 let repo = tmp.path().join("demo");
1859 std::fs::create_dir_all(&repo).unwrap();
1860 std::fs::write(repo.join("Cargo.lock"), "version = 4\n").unwrap();
1861 init_git_app_with_recipe(
1862 &repo,
1863 "0.0.1",
1864 Some("v0.0.1"),
1865 "step(\"build\");\nsh_ok(build_host(), \"echo churn >> \" + repo() + \"/Cargo.lock\");\n",
1866 );
1867 // The tag and the branch disagree about the lockfile.
1868 let git = git_in(&repo);
1869 std::fs::write(repo.join("Cargo.lock"), "version = 4\n# moved on\n").unwrap();
1870 git(&["add", "-A"]);
1871 git(&["commit", "-q", "-m", "lock moves on"]);
1872
1873 let mut cfg = Config::for_tests(tmp.path());
1874 cfg.pin_release_sha = true;
1875 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1876 let topo = one_host_topo(&repo);
1877
1878 for attempt in 1..=2 {
1879 let state = test_state(pool.clone(), topo.clone(), cfg.clone());
1880 let build_id = start_build(
1881 state,
1882 AppId::new("demo"),
1883 Version::parse("0.0.1").unwrap(),
1884 vec!["linux/x86_64".parse().unwrap()],
1885 )
1886 .await
1887 .unwrap_or_else(|e| panic!("release {attempt} refused: {e:#}"));
1888 let (status, error) = await_target(&pool, build_id).await;
1889 assert_eq!(status, "ok", "release {attempt} should build ({error})");
1890 }
1891
1892 assert_eq!(
1893 std::fs::read_to_string(build_dir(&repo, "").join("Cargo.lock")).unwrap(),
1894 "version = 4\nchurn\n",
1895 "the second release started from the tag's lockfile, not the first's leavings"
1896 );
1897 assert_eq!(repo_status(&repo), "", "and the checkout was never in it");
1898 }
1899
1900 /// A working copy ahead of the tag does not fail the version check.
1901 ///
1902 /// The check reads the tag, not the checkout, so releasing v0.0.1 while
1903 /// `main` has moved to 0.0.2 is not version drift: the tag says 0.0.1.
1904 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1905 async fn a_working_copy_ahead_of_the_tag_does_not_fail_the_version_check() {
1906 let tmp = tempfile::tempdir().unwrap();
1907 let repo = tmp.path().join("demo");
1908 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
1909 // main moves on, untagged, exactly as it does the day after a release.
1910 let git = git_in(&repo);
1911 std::fs::write(
1912 repo.join("src-tauri/tauri.conf.json"),
1913 "{\"version\":\"0.0.2\"}",
1914 )
1915 .unwrap();
1916 git(&["add", "-A"]);
1917 git(&["commit", "-q", "-m", "0.0.2"]);
1918
1919 let mut cfg = Config::for_tests(tmp.path());
1920 cfg.pin_release_sha = true;
1921 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1922 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
1923 let build_id = start_build(
1924 state,
1925 AppId::new("demo"),
1926 Version::parse("0.0.1").unwrap(),
1927 vec!["linux/x86_64".parse().unwrap()],
1928 )
1929 .await
1930 .expect("releasing the tag behind main must not be version drift");
1931 let (status, error) = await_target(&pool, build_id).await;
1932 assert_eq!(status, "ok", "({error})");
1933 }
1934
1935 /// A tag that disagrees with itself is refused, naming the file.
1936 ///
1937 /// The drift worth catching: `tauri.conf.json` at 0.0.1 and `Cargo.toml`
1938 /// still at 0.0.2, committed and tagged that way. Reading the checkout would
1939 /// compare against whatever the working copy happens to say instead.
1940 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1941 async fn a_tag_whose_manifest_disagrees_with_it_is_refused() {
1942 let tmp = tempfile::tempdir().unwrap();
1943 let repo = tmp.path().join("demo");
1944 std::fs::create_dir_all(&repo).unwrap();
1945 std::fs::write(
1946 repo.join("Cargo.toml"),
1947 "[package]\nname = \"demo\"\nversion = \"0.0.2\"\n",
1948 )
1949 .unwrap();
1950 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
1951
1952 let mut cfg = Config::for_tests(tmp.path());
1953 cfg.pin_release_sha = true;
1954 let pool = crate::db::open(&cfg.db_path).await.unwrap();
1955 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
1956 let err = start_build(
1957 state,
1958 AppId::new("demo"),
1959 Version::parse("0.0.1").unwrap(),
1960 vec!["linux/x86_64".parse().unwrap()],
1961 )
1962 .await
1963 .unwrap_err();
1964 let msg = format!("{err:#}");
1965 assert!(
1966 msg.contains("version drift") && msg.contains("Cargo.toml says 0.0.2"),
1967 "the refusal must name the file and what it says, got: {msg}"
1968 );
1969 assert!(
1970 msg.contains("v0.0.1"),
1971 "and say which tag it read, got: {msg}"
1972 );
1973 let builds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM builds")
1974 .fetch_one(&pool)
1975 .await
1976 .unwrap();
1977 assert_eq!(builds, 0, "a refused version preflight writes no build row");
1978 }
1979
1980 /// A preflight that refuses on a LATER host leaves every checkout alone.
1981 ///
1982 /// The hosts are prepared one at a time, so when the second is refused the
1983 /// first has already been through the whole preflight. There is no unwind to
1984 /// get right: the checkouts were never moved, and the worktrees a refusal
1985 /// leaves behind are where the next release would have put them.
1986 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1987 async fn a_refused_preflight_leaves_every_checkout_untouched() {
1988 let tmp = tempfile::tempdir().unwrap();
1989 // fw13's checkout is tagged and clean, so it pins. mbp's has no v0.0.1,
1990 // so it is refused after fw13 has already been moved.
1991 let repo = tmp.path().join("demo");
1992 init_git_app_shipping(
1993 &repo,
1994 "0.0.1",
1995 Some("v0.0.1"),
1996 "step(\"build\");\nsh_ok(build_host(), \"true\");\n",
1997 "[\"linux/x86_64\", \"macos/aarch64\"]",
1998 );
1999 let other = tmp.path().join("demo-mbp");
2000 init_git_app_shipping(
2001 &other,
2002 "0.0.1",
2003 None,
2004 "step(\"build\");\nsh_ok(build_host(), \"true\");\n",
2005 "[\"linux/x86_64\", \"macos/aarch64\"]",
2006 );
2007 let branch_before = current_branch(&repo);
2008 assert!(
2009 !branch_before.is_empty(),
2010 "fw13's checkout starts on a branch"
2011 );
2012
2013 let mut cfg = Config::for_tests(tmp.path());
2014 cfg.pin_release_sha = true;
2015 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2016 let topo = Topology::from_str_for_tests(&format!(
2017 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2018 worktree_root = \"{wt}\"\n\
2019 [[host]]\nname = \"mbp\"\nssh = \"local\"\ntargets = [\"macos/aarch64\"]\n\
2020 worktree_root = \"{wt}\"\n\
2021 [app.demo]\nrepo = \"{}\"\n[app.demo.repo_by_host]\nmbp = \"{}\"\n",
2022 repo.display(),
2023 other.display(),
2024 wt = worktree_root(&repo).display(),
2025 ))
2026 .unwrap();
2027 let state = test_state(pool.clone(), topo, cfg);
2028 let err = start_build(
2029 state,
2030 AppId::new("demo"),
2031 Version::parse("0.0.1").unwrap(),
2032 vec![
2033 "linux/x86_64".parse().unwrap(),
2034 "macos/aarch64".parse().unwrap(),
2035 ],
2036 )
2037 .await
2038 .unwrap_err();
2039 let msg = format!("{err:#}");
2040 assert!(
2041 msg.contains("does not exist"),
2042 "the refusal should still be mbp's missing tag, got: {msg}"
2043 );
2044 assert_eq!(
2045 current_branch(&repo),
2046 branch_before,
2047 "a refused preflight must not have moved the host it prepared first"
2048 );
2049 assert_eq!(repo_status(&repo), "", "nor left anything in its tree");
2050 }
2051
2052 /// An edit elsewhere in a repo holding several products does not touch a
2053 /// release of one of them.
2054 ///
2055 /// An edit in `server/` is not something pom's build compiles, but `git
2056 /// checkout <tag>` acts on the whole repository, and MNW is one `.git` over
2057 /// the server, sando, multithreaded and pom.
2058 ///
2059 /// A worktree is made of the repository too, so the release still gets the
2060 /// whole tagged tree, in its own copy, and the edit stays where its author
2061 /// left it.
2062 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2063 async fn an_edit_elsewhere_in_the_repo_survives_a_release_and_stays_out_of_it() {
2064 let tmp = tempfile::tempdir().unwrap();
2065 let root = tmp.path().join("monorepo");
2066 let app = root.join("pom");
2067 init_git_app(&app, "0.0.1", None);
2068 let git = |args: &[&str]| {
2069 let out = std::process::Command::new("git")
2070 .args(args)
2071 .current_dir(&root)
2072 .env("GIT_CONFIG_GLOBAL", "/dev/null")
2073 .env("GIT_CONFIG_SYSTEM", "/dev/null")
2074 .env("GIT_AUTHOR_NAME", "t")
2075 .env("GIT_AUTHOR_EMAIL", "t@t")
2076 .env("GIT_COMMITTER_NAME", "t")
2077 .env("GIT_COMMITTER_EMAIL", "t@t")
2078 .output()
2079 .expect("git runs");
2080 assert!(
2081 out.status.success(),
2082 "git {args:?}: {}",
2083 String::from_utf8_lossy(&out.stderr)
2084 );
2085 };
2086 // `init_git_app` made `pom/` its own repo; the monorepo is the parent, so
2087 // drop that and re-init one `.git` over both products.
2088 std::fs::remove_dir_all(app.join(".git")).unwrap();
2089 std::fs::create_dir_all(root.join("server")).unwrap();
2090 std::fs::write(root.join("server/Cargo.lock"), "version = 4\n").unwrap();
2091 git(&["init", "-q"]);
2092 git(&["add", "-A"]);
2093 git(&["commit", "-q", "-m", "init"]);
2094 git(&["tag", "v0.0.1"]);
2095 // The tag and the branch differ in `server/`, and the working copy of
2096 // that file is modified — so the checkout cannot carry the edit across.
2097 std::fs::write(root.join("server/Cargo.lock"), "version = 4\n# moved on\n").unwrap();
2098 git(&["add", "-A"]);
2099 git(&["commit", "-q", "-m", "server moves"]);
2100 std::fs::write(
2101 root.join("server/Cargo.lock"),
2102 "version = 4\n# local edit\n",
2103 )
2104 .unwrap();
2105
2106 let mut cfg = Config::for_tests(tmp.path());
2107 cfg.pin_release_sha = true;
2108 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2109 // The worktree root sits beside the monorepo, not inside it.
2110 let topo = Topology::from_str_for_tests(&format!(
2111 "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\
2112 worktree_root = \"{wt}\"\n[app.demo]\nrepo = \"{repo}\"\n",
2113 wt = worktree_root(&root).display(),
2114 repo = app.display()
2115 ))
2116 .unwrap();
2117 let state = test_state(pool.clone(), topo, cfg);
2118 let build_id = start_build(
2119 state,
2120 AppId::new("demo"),
2121 Version::parse("0.0.1").unwrap(),
2122 vec!["linux/x86_64".parse().unwrap()],
2123 )
2124 .await
2125 .unwrap();
2126 let (status, error) = await_target(&pool, build_id).await;
2127 assert_eq!(
2128 status, "ok",
2129 "an edit in server/ must not stop pom's release ({error})"
2130 );
2131 assert_eq!(
2132 std::fs::read_to_string(root.join("server/Cargo.lock")).unwrap(),
2133 "version = 4\n# local edit\n",
2134 "the edit must be exactly where its author left it"
2135 );
2136 // The worktree is of the whole repository, at the tag: `server/` is
2137 // there, and it is the tagged content rather than either the branch tip
2138 // or the edit.
2139 let worktree = worktree_root(&root).join("monorepo").join("demo");
2140 assert_eq!(
2141 std::fs::read_to_string(worktree.join("server/Cargo.lock")).unwrap(),
2142 "version = 4\n",
2143 "the release built the tag's server/, not the working copy's"
2144 );
2145 }
2146
2147 /// The branch `repo` is on, empty on a detached HEAD.
2148 /// Tracked files with local changes, as `git status --porcelain` writes
2149 /// them, joined by newlines and empty for a clean tree.
2150 fn repo_status(repo: &std::path::Path) -> String {
2151 let out = std::process::Command::new("git")
2152 .args(["status", "--porcelain", "--untracked-files=no"])
2153 .current_dir(repo)
2154 .env("GIT_CONFIG_GLOBAL", "/dev/null")
2155 .env("GIT_CONFIG_SYSTEM", "/dev/null")
2156 .output()
2157 .expect("git runs");
2158 String::from_utf8_lossy(&out.stdout).trim_end().to_string()
2159 }
2160
2161 /// Run git in `dir`, isolated from the dev's global config and with an
2162 /// identity, panicking with git's own stderr on failure.
2163 fn git_in(dir: &std::path::Path) -> impl Fn(&[&str]) + '_ {
2164 move |args: &[&str]| {
2165 let out = std::process::Command::new("git")
2166 .args(args)
2167 .current_dir(dir)
2168 .env("GIT_CONFIG_GLOBAL", "/dev/null")
2169 .env("GIT_CONFIG_SYSTEM", "/dev/null")
2170 .env("GIT_AUTHOR_NAME", "t")
2171 .env("GIT_AUTHOR_EMAIL", "t@t")
2172 .env("GIT_COMMITTER_NAME", "t")
2173 .env("GIT_COMMITTER_EMAIL", "t@t")
2174 .output()
2175 .expect("git runs");
2176 assert!(
2177 out.status.success(),
2178 "git {args:?}: {}",
2179 String::from_utf8_lossy(&out.stderr)
2180 );
2181 }
2182 }
2183
2184 /// The commit a checkout (or worktree) is on.
2185 fn head_sha(repo: &std::path::Path) -> String {
2186 git_read(repo, &["rev-parse", "HEAD"])
2187 }
2188
2189 /// The commit a tag names.
2190 fn tag_sha(repo: &std::path::Path, tag: &str) -> String {
2191 git_read(repo, &["rev-parse", &format!("{tag}^{{commit}}")])
2192 }
2193
2194 fn git_read(dir: &std::path::Path, args: &[&str]) -> String {
2195 let out = std::process::Command::new("git")
2196 .args(args)
2197 .current_dir(dir)
2198 .env("GIT_CONFIG_GLOBAL", "/dev/null")
2199 .env("GIT_CONFIG_SYSTEM", "/dev/null")
2200 .output()
2201 .expect("git runs");
2202 String::from_utf8_lossy(&out.stdout).trim().to_string()
2203 }
2204
2205 fn current_branch(repo: &std::path::Path) -> String {
2206 let out = std::process::Command::new("git")
2207 .args(["symbolic-ref", "-q", "--short", "HEAD"])
2208 .current_dir(repo)
2209 .env("GIT_CONFIG_GLOBAL", "/dev/null")
2210 .env("GIT_CONFIG_SYSTEM", "/dev/null")
2211 .output()
2212 .expect("git runs");
2213 String::from_utf8_lossy(&out.stdout).trim().to_string()
2214 }
2215
2216 /// An unreachable remote does not fail a release whose tag is present.
2217 ///
2218 /// `fetch --all` is non-zero if ANY remote fails, and every library repo
2219 /// carries three (`astra`, `mnw`, `srht`), so a dead mirror would abort the
2220 /// release and blame it on a missing tag. Fetch is advisory; only the
2221 /// checkout decides.
2222 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2223 async fn a_dead_remote_does_not_fail_a_release_whose_tag_exists() {
2224 let tmp = tempfile::tempdir().unwrap();
2225 let repo = tmp.path().join("demo");
2226 init_git_app(&repo, "0.0.1", Some("v0.0.1"));
2227 // A remote that cannot possibly be fetched, standing in for an offline
2228 // astra or an srht mirror the repo was never pushed to.
2229 let out = std::process::Command::new("git")
2230 .args([
2231 "remote",
2232 "add",
2233 "srht",
2234 &tmp.path().join("nowhere.git").display().to_string(),
2235 ])
2236 .current_dir(&repo)
2237 .env("GIT_CONFIG_GLOBAL", "/dev/null")
2238 .env("GIT_CONFIG_SYSTEM", "/dev/null")
2239 .output()
2240 .expect("git runs");
2241 assert!(out.status.success());
2242
2243 let mut cfg = Config::for_tests(tmp.path());
2244 cfg.pin_release_sha = true;
2245 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2246 let state = test_state(pool.clone(), one_host_topo(&repo), cfg);
2247 let build_id = start_build(
2248 state,
2249 AppId::new("demo"),
2250 Version::parse("0.0.1").unwrap(),
2251 vec!["linux/x86_64".parse().unwrap()],
2252 )
2253 .await
2254 .expect("a dead mirror must not refuse the release");
2255 let (status, error) = await_target(&pool, build_id).await;
2256 assert_eq!(status, "ok", "the tag is present, so this builds ({error})");
2257 }
2258
2259 /// The audit fix: a `build` step dispatched to a host whose executor lacks the
2260 /// `build` capability is denied at the transport BEFORE the command runs. This
2261 /// is the structural guarantee behind "never build on prod" — a recipe naming
2262 /// the wrong host can't compile there.
2263 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2264 async fn build_on_a_host_without_the_build_grant_is_denied() {
2265 let tmp = tempfile::tempdir().unwrap();
2266 let root = tmp.path();
2267
2268 let repo = root.join("app");
2269 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2270 std::fs::write(
2271 repo.join("src-tauri/tauri.conf.json"),
2272 r#"{"version":"0.0.1"}"#,
2273 )
2274 .unwrap();
2275 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2276 // The recipe (wrongly) tries to compile on `prod`, a host with no build
2277 // grant. A marker file would appear if the command actually ran.
2278 let marker = root.join("ran-on-prod");
2279 std::fs::write(
2280 repo.join("dist/recipes/linux.rhai"),
2281 r#"
2282 step("build");
2283 sh_ok("prod", "touch MARKER");
2284 "#
2285 .replace("MARKER", marker.to_str().unwrap()),
2286 )
2287 .unwrap();
2288
2289 let cfg = Config::for_tests(root);
2290 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2291 // fw13 builds linux; `prod` is local-but-restart-only (no build/package).
2292 std::fs::write(repo.join("bento.toml"), "targets = [\"linux/x86_64\"]\n").unwrap();
2293 let topo = Topology::from_str_for_tests(
2294 r#"
2295 [[host]]
2296 name = "fw13"
2297 ssh = "local"
2298 targets = ["linux/x86_64"]
2299
2300 [[host]]
2301 name = "prod"
2302 ssh = "local"
2303 actuate = ["restart"]
2304 observe = []
2305
2306 [app.demo]
2307 repo = "REPO"
2308 "#
2309 .replace("REPO", repo.to_str().unwrap())
2310 .as_str(),
2311 )
2312 .unwrap();
2313
2314 let state = test_state(pool.clone(), topo, cfg);
2315
2316 let build_id = start_build(
2317 state.clone(),
2318 AppId::new("demo"),
2319 Version::parse("0.0.1").unwrap(),
2320 vec!["linux/x86_64".parse().unwrap()],
2321 )
2322 .await
2323 .unwrap();
2324
2325 let mut status = String::new();
2326 let mut error = String::new();
2327 for _ in 0..100 {
2328 let row: Option<(String, Option<String>)> =
2329 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2330 .bind(build_id)
2331 .fetch_optional(&pool)
2332 .await
2333 .unwrap();
2334 if let Some((s, e)) = row {
2335 status = s;
2336 error = e.unwrap_or_default();
2337 if status != "running" {
2338 break;
2339 }
2340 }
2341 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2342 }
2343
2344 assert_eq!(status, "failed", "build on an ungranted host must fail");
2345 assert!(
2346 error.contains("capability denied") && error.contains("build"),
2347 "failure must be a capability denial, got: {error}"
2348 );
2349 assert!(!marker.exists(), "denied build step must NOT have executed");
2350 }
2351
2352 // ---- macOS sign / notarize / staple execution chain (via ScriptedExec) ----
2353
2354 /// Stand up a single-macOS-target app whose recipe is `recipe_body` (with the
2355 /// literal `ARTIFACT` replaced by a real, non-empty file on disk), dispatch
2356 /// every host command through `scripted`, run the build to a terminal state,
2357 /// and return the pool plus the final `(status, error)`. The returned
2358 /// [`tempfile::TempDir`] must be kept alive by the caller: it holds the
2359 /// sqlite DB the pool reads.
2360 async fn run_macos_recipe(
2361 scripted: Arc<ScriptedExec>,
2362 recipe_body: &str,
2363 backoff_secs: Option<u64>,
2364 ) -> (tempfile::TempDir, SqlitePool, String, String) {
2365 let tmp = tempfile::tempdir().unwrap();
2366 let root = tmp.path();
2367 let repo = root.join("app");
2368 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2369 std::fs::write(
2370 repo.join("src-tauri/tauri.conf.json"),
2371 r#"{"version":"0.0.1"}"#,
2372 )
2373 .unwrap();
2374 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2375
2376 // A real, non-empty artifact so `publish`'s size floor is satisfied; the
2377 // build step is faked, so nothing else creates it.
2378 let artifact = repo.join("out/demo.dmg");
2379 std::fs::create_dir_all(artifact.parent().unwrap()).unwrap();
2380 std::fs::write(&artifact, b"dmg-bytes").unwrap();
2381
2382 std::fs::write(
2383 repo.join("dist/recipes/macos.rhai"),
2384 recipe_body.replace("ARTIFACT", artifact.to_str().unwrap()),
2385 )
2386 .unwrap();
2387
2388 let cfg = Config {
2389 notarize_backoff_secs: backoff_secs,
2390 ..Config::for_tests(root)
2391 };
2392 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2393 std::fs::write(repo.join("bento.toml"), "targets = [\"macos/aarch64\"]\n").unwrap();
2394 let topo = Topology::from_str_for_tests(&format!(
2395 r#"
2396 [[host]]
2397 name = "mbp"
2398 ssh = "local"
2399 targets = ["macos/aarch64"]
2400
2401 [app.demo]
2402 repo = "{}"
2403 "#,
2404 repo.display()
2405 ))
2406 .unwrap();
2407
2408 let mut state = test_state(pool.clone(), topo, cfg);
2409 // Route every host command through the scripted executor.
2410 let mut execs = HashMap::new();
2411 execs.insert("mbp".to_string(), scripted as Arc<dyn Executor>);
2412 state.executors = Arc::new(execs);
2413
2414 let build_id = start_build(
2415 state.clone(),
2416 AppId::new("demo"),
2417 Version::parse("0.0.1").unwrap(),
2418 vec!["macos/aarch64".parse().unwrap()],
2419 )
2420 .await
2421 .unwrap();
2422
2423 let mut status = String::new();
2424 let mut error = String::new();
2425 for _ in 0..100 {
2426 let row: Option<(String, Option<String>)> =
2427 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2428 .bind(build_id)
2429 .fetch_optional(&pool)
2430 .await
2431 .unwrap();
2432 if let Some((s, e)) = row {
2433 status = s;
2434 error = e.unwrap_or_default();
2435 if status != "running" {
2436 break;
2437 }
2438 }
2439 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2440 }
2441 (tmp, pool, status, error)
2442 }
2443
2444 async fn release_count(pool: &SqlitePool) -> i64 {
2445 sqlx::query_scalar("SELECT COUNT(*) FROM releases")
2446 .fetch_one(pool)
2447 .await
2448 .unwrap()
2449 }
2450
2451 /// The whole macOS release chain end to end through a fake host: codesign,
2452 /// notarize (Accepted first try), staple, verify_gatekeeper, then publish.
2453 /// The recorded commands lock the exact incantations each host function
2454 /// dispatches, and a `releases` row proves the publish gate opened for a
2455 /// signed + notarized + Gatekeeper-accepted artifact.
2456 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2457 async fn a_macos_recipe_signs_notarizes_staples_verifies_and_publishes() {
2458 let scripted = Arc::new(
2459 ScriptedExec::new()
2460 .on("codesign", 0, "")
2461 .on("notarytool", 0, r#"{"status":"Accepted"}"#)
2462 .on("stapler staple", 0, "")
2463 .on(
2464 "spctl",
2465 0,
2466 "source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
2467 ),
2468 );
2469 let recipe = r#"
2470 let h = build_host();
2471 step("build");
2472 sh_ok(h, "echo built");
2473 step("sign");
2474 codesign(h, "Developer ID Application: Test", "ARTIFACT");
2475 notarize(h, "ARTIFACT");
2476 staple(h, "ARTIFACT");
2477 step("verify");
2478 if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
2479 step("publish");
2480 publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2481 "#;
2482 let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, None).await;
2483
2484 assert_eq!(
2485 status, "ok",
2486 "signed+notarized macOS build should publish: {error}"
2487 );
2488 assert_eq!(
2489 release_count(&pool).await,
2490 1,
2491 "publish must record a release"
2492 );
2493
2494 // The exact shell incantations each host function dispatched.
2495 let cmds = scripted.commands();
2496 let has = |needle: &str| cmds.iter().any(|c| c.contains(needle));
2497 assert!(
2498 has("codesign --force --options runtime --timestamp --sign"),
2499 "codesign runtime+timestamp incantation, got: {cmds:?}"
2500 );
2501 assert!(
2502 has("xcrun notarytool submit"),
2503 "notarytool submit: {cmds:?}"
2504 );
2505 assert!(
2506 has("--wait --output-format json"),
2507 "notarytool --wait json: {cmds:?}"
2508 );
2509 assert!(has("xcrun stapler staple"), "stapler staple: {cmds:?}");
2510 assert!(has("spctl --assess"), "gatekeeper assess: {cmds:?}");
2511 }
2512
2513 /// A failing `codesign` fails the sign step and aborts the recipe before
2514 /// publish — an unsigned artifact never ships.
2515 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2516 async fn a_codesign_failure_fails_the_sign_step_and_blocks_publish() {
2517 let scripted = Arc::new(ScriptedExec::new().on("codesign", 1, ""));
2518 let recipe = r#"
2519 let h = build_host();
2520 step("build");
2521 sh_ok(h, "echo built");
2522 step("sign");
2523 codesign(h, "Developer ID Application: Test", "ARTIFACT");
2524 step("publish");
2525 publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2526 "#;
2527 let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
2528
2529 assert_eq!(status, "failed", "a failed codesign must fail the target");
2530 assert!(
2531 error.contains("codesign failed"),
2532 "error names the codesign failure, got: {error}"
2533 );
2534 assert_eq!(
2535 release_count(&pool).await,
2536 0,
2537 "nothing may publish after a codesign failure"
2538 );
2539 }
2540
2541 /// Even if the recipe ignores `verify_gatekeeper`'s returned `false`, the
2542 /// publish gate refuses the artifact: `verify_gatekeeper` both records the
2543 /// rejection and fails its step, and `publish` proves neither passed. This
2544 /// is the defense-in-depth the pure `PublishAuthority::prove` tests assert in
2545 /// isolation, here exercised through the real host-function path.
2546 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2547 async fn a_gatekeeper_rejection_bars_publish_even_if_the_recipe_ignores_it() {
2548 let scripted = Arc::new(
2549 ScriptedExec::new()
2550 .on("codesign", 0, "")
2551 .on("notarytool", 0, r#"{"status":"Accepted"}"#)
2552 .on("stapler staple", 0, "")
2553 // Gatekeeper says no: the sentinel is FAIL, not OK.
2554 .on("spctl", 0, "source=Unnotarized\nBENTO_GATEKEEPER_FAIL"),
2555 );
2556 let recipe = r#"
2557 let h = build_host();
2558 step("build");
2559 sh_ok(h, "echo built");
2560 step("sign");
2561 codesign(h, "Developer ID Application: Test", "ARTIFACT");
2562 notarize(h, "ARTIFACT");
2563 staple(h, "ARTIFACT");
2564 step("verify");
2565 verify_gatekeeper(h, "ARTIFACT");
2566 step("publish");
2567 publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2568 "#;
2569 let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
2570
2571 assert_eq!(
2572 status, "failed",
2573 "a Gatekeeper-rejected artifact must not publish"
2574 );
2575 assert!(
2576 !error.is_empty(),
2577 "the barred publish must surface an error"
2578 );
2579 assert_eq!(
2580 release_count(&pool).await,
2581 0,
2582 "no release for a rejected artifact"
2583 );
2584 }
2585
2586 /// The one flaky, network-bound step: `notarize` retries a non-`Accepted`
2587 /// result and succeeds on a later attempt. Two notarytool calls (reject then
2588 /// accept) then a recorded release prove the retry ran and the chain
2589 /// completed. Backoff is 0 so the retry sleep doesn't stall the test.
2590 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2591 async fn notarize_retries_a_non_accepted_result_then_succeeds() {
2592 let scripted = Arc::new(
2593 ScriptedExec::new()
2594 .on("codesign", 0, "")
2595 .on_seq(
2596 "notarytool",
2597 &[
2598 (0, r#"{"status":"In Progress"}"#),
2599 (0, r#"{"status":"Accepted"}"#),
2600 ],
2601 )
2602 .on("stapler staple", 0, "")
2603 .on(
2604 "spctl",
2605 0,
2606 "source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
2607 ),
2608 );
2609 let recipe = r#"
2610 let h = build_host();
2611 step("build");
2612 sh_ok(h, "echo built");
2613 step("sign");
2614 codesign(h, "Developer ID Application: Test", "ARTIFACT");
2615 notarize(h, "ARTIFACT");
2616 staple(h, "ARTIFACT");
2617 step("verify");
2618 if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
2619 step("publish");
2620 publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2621 "#;
2622 let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
2623
2624 assert_eq!(
2625 status, "ok",
2626 "notarize should succeed on the retry: {error}"
2627 );
2628 assert_eq!(
2629 release_count(&pool).await,
2630 1,
2631 "the retried build still publishes"
2632 );
2633 let notary_calls = scripted
2634 .commands()
2635 .iter()
2636 .filter(|c| c.contains("notarytool"))
2637 .count();
2638 assert_eq!(
2639 notary_calls, 2,
2640 "notarytool ran once, was rejected, then ran again"
2641 );
2642 }
2643
2644 /// `notarize` gives up after its bounded retries: three notarytool attempts,
2645 /// all non-`Accepted`, fail the target and bar publish.
2646 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2647 async fn notarize_fails_the_target_after_exhausting_its_retries() {
2648 let scripted = Arc::new(
2649 ScriptedExec::new()
2650 .on("codesign", 0, "")
2651 // Every attempt reports a still-pending status, never Accepted.
2652 .on("notarytool", 0, r#"{"status":"In Progress"}"#),
2653 );
2654 let recipe = r#"
2655 let h = build_host();
2656 step("build");
2657 sh_ok(h, "echo built");
2658 step("sign");
2659 codesign(h, "Developer ID Application: Test", "ARTIFACT");
2660 notarize(h, "ARTIFACT");
2661 step("publish");
2662 publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2663 "#;
2664 let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
2665
2666 assert_eq!(
2667 status, "failed",
2668 "exhausted notarization must fail the target"
2669 );
2670 assert!(
2671 error.contains("notarization failed after 3 attempts"),
2672 "error names the exhausted retry, got: {error}"
2673 );
2674 assert_eq!(
2675 release_count(&pool).await,
2676 0,
2677 "an unnotarized artifact never publishes"
2678 );
2679 let notary_calls = scripted
2680 .commands()
2681 .iter()
2682 .filter(|c| c.contains("notarytool"))
2683 .count();
2684 assert_eq!(notary_calls, 3, "the retry is bounded at three attempts");
2685 }
2686 // ---- item (5): SSH and Agent transports end to end, via a recording fake ----
2687 //
2688 // Every other test declares `ssh = "local"`, so the runner's two-plane
2689 // routing is only exercised at construction
2690 // (state::agent_host_syncs_over_ssh_never_the_agent): steps run over the
2691 // EXEC transport (state.executors -- SshExec, or the in-session AgentRpc for
2692 // a mac host) while artifacts move over the SYNC transport (state.syncs --
2693 // always ssh, NEVER the agent, whose confined `/pull` would 404 or force
2694 // `pull_root` wide enough to expose `~/.tauri/passwords.env`). These drive a
2695 // real recipe through `start_build` on a NON-local topology, replace the
2696 // real ssh/agent transports with a recording fake, and assert which plane
2697 // handled which operation -- the runtime form of that construction-time
2698 // invariant, on the ssh string every other test pins to "local".
2699
2700 /// Records shell commands and artifact pulls on separate logs, so a test can
2701 /// prove the exec transport built/signed and the sync transport collected --
2702 /// and that neither did the other's job -- without a live host or ssh.
2703 struct RecordingExec {
2704 caps: CapabilitySet,
2705 commands: Arc<std::sync::Mutex<Vec<String>>>,
2706 pulls: Arc<std::sync::Mutex<Vec<String>>>,
2707 }
2708
2709 impl RecordingExec {
2710 fn new() -> Arc<Self> {
2711 Arc::new(Self {
2712 // A mac build host's real grant. Nothing in this fake gates on it
2713 // (the real transports do), but keep it coherent so
2714 // `capabilities()` is not a lie.
2715 caps: CapabilitySet::from_tokens(
2716 ["build", "sign", "notarize", "staple"],
2717 ["build-log", "artifact"],
2718 ),
2719 commands: Arc::new(std::sync::Mutex::new(Vec::new())),
2720 pulls: Arc::new(std::sync::Mutex::new(Vec::new())),
2721 })
2722 }
2723 fn commands(&self) -> Vec<String> {
2724 self.commands.lock().unwrap().clone()
2725 }
2726 fn pulls(&self) -> Vec<String> {
2727 self.pulls.lock().unwrap().clone()
2728 }
2729 }
2730
2731 #[async_trait]
2732 impl Executor for RecordingExec {
2733 async fn run_streaming(
2734 &self,
2735 step: &ops_exec::Step,
2736 _sink: &mut dyn LogSink,
2737 ) -> anyhow::Result<RunOutput> {
2738 self.commands
2739 .lock()
2740 .unwrap()
2741 .push(step.argv.last().cloned().unwrap_or_default());
2742 Ok(RunOutput {
2743 status: std::process::ExitStatus::from_raw(0),
2744 stdout: Vec::new(),
2745 stderr: Vec::new(),
2746 })
2747 }
2748 async fn pull_file(
2749 &self,
2750 r: &std::path::Path,
2751 _l: &std::path::Path,
2752 _o: &SyncOpts,
2753 ) -> anyhow::Result<()> {
2754 self.pulls
2755 .lock()
2756 .unwrap()
2757 .push(r.to_string_lossy().into_owned());
2758 Ok(())
2759 }
2760 async fn pull_dir(
2761 &self,
2762 r: &std::path::Path,
2763 _l: &std::path::Path,
2764 _o: &SyncOpts,
2765 ) -> anyhow::Result<()> {
2766 self.pulls
2767 .lock()
2768 .unwrap()
2769 .push(r.to_string_lossy().into_owned());
2770 Ok(())
2771 }
2772 async fn pull_glob(&self, g: &str, _l: &std::path::Path, _o: &SyncOpts) -> anyhow::Result<()> {
2773 self.pulls.lock().unwrap().push(g.to_string());
2774 Ok(())
2775 }
2776 async fn push_dir(
2777 &self,
2778 _l: &std::path::Path,
2779 _r: &std::path::Path,
2780 _o: &SyncOpts,
2781 ) -> anyhow::Result<()> {
2782 Ok(())
2783 }
2784 async fn preflight(&self) -> anyhow::Result<()> {
2785 Ok(())
2786 }
2787 fn capabilities(&self) -> &CapabilitySet {
2788 &self.caps
2789 }
2790 }
2791
2792 /// Stand up a single-target app (host named `h1`) whose recipe is
2793 /// `recipe_body` (with `REPO` replaced by the checkout path), inject
2794 /// `exec_fake` as the host's EXEC transport and `sync_fake` as its SYNC
2795 /// transport, run the build to a terminal state, and return the tmpdir plus
2796 /// `(status, error)`. Unlike `test_state`'s `build_executors`, this replaces
2797 /// BOTH planes so no real ssh/agent transport is dialed. The returned
2798 /// [`tempfile::TempDir`] holds the sqlite DB and must outlive the caller's
2799 /// assertions.
2800 async fn run_two_plane(
2801 host_toml: &str,
2802 target: &str,
2803 recipe_file: &str,
2804 recipe_body: &str,
2805 exec_fake: Arc<dyn Executor>,
2806 sync_fake: Arc<dyn Executor>,
2807 ) -> (tempfile::TempDir, String, String) {
2808 let tmp = tempfile::tempdir().unwrap();
2809 let root = tmp.path();
2810 let repo = root.join("app");
2811 std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2812 std::fs::write(
2813 repo.join("src-tauri/tauri.conf.json"),
2814 r#"{"version":"0.0.1"}"#,
2815 )
2816 .unwrap();
2817 std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2818 std::fs::write(
2819 repo.join("dist/recipes").join(recipe_file),
2820 recipe_body.replace("REPO", repo.to_str().unwrap()),
2821 )
2822 .unwrap();
2823 std::fs::write(
2824 repo.join("bento.toml"),
2825 format!("targets = [\"{target}\"]\n"),
2826 )
2827 .unwrap();
2828
2829 let cfg = Config::for_tests(root);
2830 let pool = crate::db::open(&cfg.db_path).await.unwrap();
2831 let topo = Topology::from_str_for_tests(&format!(
2832 "{}\n[app.demo]\nrepo = \"{}\"\n",
2833 host_toml.replace("REPO", repo.to_str().unwrap()),
2834 repo.display()
2835 ))
2836 .unwrap();
2837
2838 let mut state = test_state(pool.clone(), topo, cfg);
2839 state.executors = Arc::new(HashMap::from([("h1".to_string(), exec_fake)]));
2840 state.syncs = Arc::new(HashMap::from([("h1".to_string(), sync_fake)]));
2841
2842 let build_id = start_build(
2843 state.clone(),
2844 AppId::new("demo"),
2845 Version::parse("0.0.1").unwrap(),
2846 vec![target.parse().unwrap()],
2847 )
2848 .await
2849 .unwrap();
2850
2851 let mut status = String::new();
2852 let mut error = String::new();
2853 for _ in 0..100 {
2854 let row: Option<(String, Option<String>)> =
2855 sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2856 .bind(build_id)
2857 .fetch_optional(&pool)
2858 .await
2859 .unwrap();
2860 if let Some((s, e)) = row {
2861 status = s;
2862 error = e.unwrap_or_default();
2863 if status != "running" {
2864 break;
2865 }
2866 }
2867 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2868 }
2869 (tmp, status, error)
2870 }
2871
2872 /// An agent (macOS) host signs over the AGENT transport but is collected
2873 /// from over SSH -- driven through the whole runner, not just `build_sync`.
2874 /// The sign chain's commands land on the exec plane and the artifact pull on
2875 /// the sync plane; crucially, the agent plane is asked to move NOTHING (a
2876 /// regression to one transport would route collect at `AgentRpc::pull_glob`,
2877 /// refused by design, or widen `pull_root` over the secret-bearing home dir).
2878 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2879 async fn an_agent_host_signs_over_the_agent_and_collects_over_ssh_end_to_end() {
2880 let agent = RecordingExec::new(); // exec plane (AgentRpc in prod)
2881 let ssh = RecordingExec::new(); // sync plane (SshExec in prod)
2882 let host = r#"
2883 [[host]]
2884 name = "h1"
2885 ssh = "mbp"
2886 targets = ["macos/aarch64"]
2887 transport = "agent"
2888 agent_url = "http://mbp:8765"
2889 actuate = ["build", "sign", "notarize", "staple"]
2890 observe = ["build-log", "gatekeeper", "artifact"]
2891 pull_root = "REPO"
2892 "#;
2893 let recipe = r#"
2894 let h = build_host();
2895 step("build");
2896 sh_ok(h, "echo built");
2897 step("sign");
2898 codesign(h, "Developer ID Application: Test", "REPO/out/demo.dmg");
2899 step("collect");
2900 collect(h, "REPO/out/*.dmg", "demo", "0.0.1");
2901 "#;
2902 let (_tmp, status, error) = run_two_plane(
2903 host,
2904 "macos/aarch64",
2905 "macos.rhai",
2906 recipe,
2907 agent.clone(),
2908 ssh.clone(),
2909 )
2910 .await;
2911 assert_eq!(status, "ok", "the recipe should complete: {error}");
2912
2913 // The build + sign commands ran on the AGENT (exec) transport.
2914 let agent_cmds = agent.commands();
2915 assert!(
2916 agent_cmds.iter().any(|c| c.contains("codesign")),
2917 "codesign rides the agent exec transport: {agent_cmds:?}"
2918 );
2919 assert!(
2920 agent_cmds.iter().any(|c| c.contains("echo built")),
2921 "the build step rides the agent exec transport: {agent_cmds:?}"
2922 );
2923 // ...and the agent moved NO artifacts. This is the load-bearing half:
2924 // AgentRpc::pull_glob is refused by design, so collect must not touch it.
2925 assert!(
2926 agent.pulls().is_empty(),
2927 "the agent transport must never collect artifacts: {:?}",
2928 agent.pulls()
2929 );
2930
2931 // The artifact was collected over the SSH (sync) transport...
2932 let ssh_pulls = ssh.pulls();
2933 assert!(
2934 ssh_pulls
2935 .iter()
2936 .any(|p| p.contains("demo.dmg") || p.contains("*.dmg")),
2937 "collect rides the ssh sync transport: {ssh_pulls:?}"
2938 );
2939 // ...and the sync transport was never asked to run a build/sign command.
2940 assert!(
2941 ssh.commands().is_empty(),
2942 "the sync transport must never run host commands: {:?}",
2943 ssh.commands()
2944 );
2945 }
2946
2947 /// A plain (non-agent) host whose `ssh` is a remote alias, not "local" --
2948 /// the case every other test avoids. The recipe runs end to end through the
2949 /// fake, proving the runner drives a non-local host and still splits exec
2950 /// (build) from sync (collect) across the two transport maps.
2951 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2952 async fn a_non_local_ssh_host_runs_a_recipe_end_to_end() {
2953 let exec = RecordingExec::new();
2954 let sync = RecordingExec::new();
2955 let host = r#"
2956 [[host]]
2957 name = "h1"
2958 ssh = "astra"
2959 targets = ["linux/x86_64"]
2960 pull_root = "REPO"
2961 "#;
2962 let recipe = r#"
2963 let h = build_host();
2964 step("build");
2965 sh_ok(h, "echo compiling");
2966 step("collect");
2967 collect(h, "REPO/out/demo.bin", "demo", "0.0.1");
2968 "#;
2969 let (_tmp, status, error) = run_two_plane(
2970 host,
2971 "linux/x86_64",
2972 "linux.rhai",
2973 recipe,
2974 exec.clone(),
2975 sync.clone(),
2976 )
2977 .await;
2978 assert_eq!(status, "ok", "the recipe should complete: {error}");
2979
2980 assert!(
2981 exec.commands().iter().any(|c| c.contains("echo compiling")),
2982 "the build command rides the exec transport: {:?}",
2983 exec.commands()
2984 );
2985 assert!(
2986 exec.pulls().is_empty(),
2987 "the exec transport must not collect: {:?}",
2988 exec.pulls()
2989 );
2990 assert!(
2991 sync.pulls().iter().any(|p| p.contains("demo.bin")),
2992 "collect rides the sync transport: {:?}",
2993 sync.pulls()
2994 );
2995 assert!(
2996 sync.commands().is_empty(),
2997 "the sync transport must not run commands: {:?}",
2998 sync.commands()
2999 );
3000 }
3001
3002 /// Every recipe of every configured app must parse.
3003 ///
3004 /// A recipe is read and compiled at release time, on the build host, after the
3005 /// checkout has already run — so a typo in one is discovered at the worst
3006 /// possible moment. Compiling them here costs nothing and moves that discovery
3007 /// to `cargo test`. Skips when the live config is absent (CI, another host),
3008 /// like [`crate::topology`]'s live-config smoke test.
3009 #[cfg(test)]
3010 mod live_recipe_smoke {
3011 use crate::topology::Topology;
3012 use std::path::{Path, PathBuf};
3013
3014 #[test]
3015 fn live_recipes_compile_if_present() {
3016 let Some(home) = std::env::var_os("HOME") else {
3017 return;
3018 };
3019 let path = Path::new(&home).join(".config/bento/bento.toml");
3020 if !path.exists() {
3021 return;
3022 }
3023 let topo = Topology::load(&path).expect("live bento.toml must load");
3024 // Syntax only: the host functions are bound per run against a live
3025 // context, and Rhai resolves calls at eval time regardless.
3026 let engine = rhai::Engine::new();
3027 let mut checked = 0;
3028 for (name, cfg) in &topo.app {
3029 let dir = crate::engine::expand_tilde(&cfg.repo).join(&cfg.recipe_dir);
3030 // Every recipe in the directory, not just the ones the app's current
3031 // targets name. A recipe for a target that is temporarily not
3032 // shipped (windows, dropped from the manifests until its host is
3033 // real) still has to parse, and it is the one nothing else is
3034 // watching.
3035 let Ok(entries) = std::fs::read_dir(&dir) else {
3036 continue;
3037 };
3038 let mut files: Vec<PathBuf> = entries
3039 .filter_map(Result::ok)
3040 .map(|e| e.path())
3041 .filter(|p| p.extension().is_some_and(|x| x == "rhai"))
3042 .collect();
3043 files.sort();
3044 for p in files {
3045 let file = p.file_name().unwrap_or_default().to_string_lossy();
3046 let Ok(src) = std::fs::read_to_string(&p) else {
3047 continue;
3048 };
3049 engine
3050 .compile(&src)
3051 .unwrap_or_else(|e| panic!("recipe {name}/{file} does not parse: {e}"));
3052 checked += 1;
3053 }
3054 }
3055 assert!(checked > 0, "live config resolved no readable recipes");
3056 }
3057 }
3058