Skip to main content

max / makenotwork

bento: cover the macOS sign/notarize/staple execution chain Add a recording, programmable ScriptedExec and five integration tests driving the full codesign -> notarize -> staple -> verify_gatekeeper -> publish chain through a fake host: the happy path locks each shell incantation and proves the publish gate opens, and the negatives cover a codesign failure, a Gatekeeper rejection the recipe ignores, and the notarize retry loop both succeeding on a retry and exhausting its bound. Only notary_accepted and PublishAuthority::prove were tested before; the execution path was not. Add a Config.notarize_backoff_secs test seam (mirrors step_timeout_secs) so the retry loop doesn't sleep 15s in tests.
Co-Authored-By
Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-25 13:37 UTC
Signed with PGP, not checked
Commit: d3cf72a53d14224e4b27ab4b49a9077b89c39c4b
Parent: c430bc1
3 files changed, +480 insertions, -2 deletions
M bento/daemon/src/config.rs +7
@@ -26,6 +26,12 @@
26 26 /// test that needs a short deadline.
27 27 #[serde(default)]
28 28 pub step_timeout_secs: Option<u64>,
29 + /// Seconds to wait between `notarize` retry attempts. Unset (the default)
30 + /// uses the 15-second production backoff; a test drives the retry loop with
31 + /// `Some(0)` so it doesn't actually sleep. Same test-seam shape as
32 + /// `step_timeout_secs`.
33 + #[serde(default)]
34 + pub notarize_backoff_secs: Option<u64>,
29 35 /// Pin every build host to the release tag `v<version>` and verify they all
30 36 /// report the same commit BEFORE any target builds — so `mbp`/`astra`/`fw13`
31 37 /// can't each build whatever `main` happened to be at pull time. On by
@@ -61,6 +67,7 @@
61 67 dist_root: root.join("dist"),
62 68 logs_root: root.join("logs"),
63 69 step_timeout_secs: None,
70 + notarize_backoff_secs: None,
64 71 // Test repos are plain dirs, not git checkouts; the barrier is
65 72 // exercised by its own tests that build a real tagged repo.
66 73 pin_release_sha: false,
M bento/daemon/src/engine.rs +7 -2
@@ -1692,6 +1692,12 @@
1692 1692 /// network-bound step). Emits `NotarizeRetry` per attempt.
1693 1693 fn notarize(self: &Arc<Self>, host: &str, path: &str) -> Result<String> {
1694 1694 const MAX_ATTEMPTS: u32 = 3;
1695 + let backoff = self
1696 + .cfg
1697 + .notarize_backoff_secs
1698 + .map_or(std::time::Duration::from_secs(15), |s| {
1699 + std::time::Duration::from_secs(s)
1700 + });
1695 1701 let cmd = format!(
1696 1702 ". ~/.tauri/passwords.env && xcrun notarytool submit {} \
1697 1703 --key \"$NOTARY_KEY\" --key-id \"$NOTARY_KEY_ID\" --issuer \"$NOTARY_ISSUER\" \
@@ -1715,8 +1721,7 @@
1715 1721 reason: format!("exit {code}"),
1716 1722 },
1717 1723 );
1718 - self.rt
1719 - .block_on(tokio::time::sleep(std::time::Duration::from_secs(15)));
1724 + self.rt.block_on(tokio::time::sleep(backoff));
1720 1725 }
1721 1726 }
1722 1727 anyhow::bail!("notarization failed after {MAX_ATTEMPTS} attempts: {last}")
M bento/daemon/src/runner.rs +466
@@ -748,6 +748,137 @@
748 748 }
749 749 }
750 750
751 + /// A recording, programmable [`Executor`] for the macOS sign chain. Every
752 + /// dispatched shell command is captured for assertion, and its exit code +
753 + /// stdout is chosen by the first rule whose needle the command contains — so
754 + /// a test can make `notarytool` report `Accepted`, make `codesign` fail, or
755 + /// make `spctl` emit the Gatekeeper sentinel without a real Mac or SSH. A
756 + /// rule may carry a *sequence* of responses (one per successive match) to
757 + /// drive the notarize retry loop; the last entry repeats once the sequence
758 + /// is exhausted. Unmatched commands succeed as empty no-ops (so a plain
759 + /// build `sh_ok` passes), and the sync/preflight ops are no-ops.
760 + struct ScriptedExec {
761 + caps: CapabilitySet,
762 + rules: Vec<ScriptRule>,
763 + log: Arc<std::sync::Mutex<Vec<String>>>,
764 + }
765 +
766 + struct ScriptRule {
767 + needle: String,
768 + responses: Vec<(i32, String)>,
769 + calls: std::sync::atomic::AtomicUsize,
770 + }
771 +
772 + impl ScriptedExec {
773 + fn new() -> Self {
774 + Self {
775 + // A mac host's real grant. Nothing in the dispatch path gates on
776 + // it (this fake never calls `gate`), but keep it coherent so
777 + // `capabilities()` is not a lie.
778 + caps: CapabilitySet::from_tokens(
779 + ["build", "sign", "notarize", "staple"],
780 + ["build-log", "artifact"],
781 + ),
782 + rules: Vec::new(),
783 + log: Arc::new(std::sync::Mutex::new(Vec::new())),
784 + }
785 + }
786 +
787 + /// Respond to every command containing `needle` with `(code, stdout)`.
788 + fn on(mut self, needle: &str, code: i32, stdout: &str) -> Self {
789 + self.rules.push(ScriptRule {
790 + needle: needle.to_string(),
791 + responses: vec![(code, stdout.to_string())],
792 + calls: std::sync::atomic::AtomicUsize::new(0),
793 + });
794 + self
795 + }
796 +
797 + /// Respond to successive `needle` matches with successive responses; the
798 + /// last repeats once the list is exhausted. Drives the notarize retry.
799 + fn on_seq(mut self, needle: &str, responses: &[(i32, &str)]) -> Self {
800 + self.rules.push(ScriptRule {
801 + needle: needle.to_string(),
802 + responses: responses
803 + .iter()
804 + .map(|(c, s)| (*c, (*s).to_string()))
805 + .collect(),
806 + calls: std::sync::atomic::AtomicUsize::new(0),
807 + });
808 + self
809 + }
810 +
811 + /// Every shell command this executor was asked to run, in order.
812 + fn commands(&self) -> Vec<String> {
813 + self.log.lock().unwrap().clone()
814 + }
815 + }
816 +
817 + #[async_trait]
818 + impl Executor for ScriptedExec {
819 + async fn run_streaming(
820 + &self,
821 + step: &ops_exec::Step,
822 + _sink: &mut dyn LogSink,
823 + ) -> anyhow::Result<RunOutput> {
824 + let cmd = step.argv.last().cloned().unwrap_or_default();
825 + self.log.lock().unwrap().push(cmd.clone());
826 + let (code, stdout) = self.rules.iter().find(|r| cmd.contains(&r.needle)).map_or(
827 + (0, String::new()),
828 + |r| {
829 + let i = r.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
830 + r.responses[i.min(r.responses.len() - 1)].clone()
831 + },
832 + );
833 + Ok(RunOutput {
834 + // Shift into the wait-status word's exit-code byte so
835 + // `ExitStatus::code()` reports `code` exactly (a bare
836 + // `from_raw(1)` reads as a signal, yielding `None`).
837 + status: std::process::ExitStatus::from_raw(code << 8),
838 + stdout: stdout.into_bytes(),
839 + stderr: Vec::new(),
840 + })
841 + }
842 + async fn pull_file(
843 + &self,
844 + _r: &std::path::Path,
845 + _l: &std::path::Path,
846 + _o: &SyncOpts,
847 + ) -> anyhow::Result<()> {
848 + Ok(())
849 + }
850 + async fn pull_dir(
851 + &self,
852 + _r: &std::path::Path,
853 + _l: &std::path::Path,
854 + _o: &SyncOpts,
855 + ) -> anyhow::Result<()> {
856 + Ok(())
857 + }
858 + async fn pull_glob(
859 + &self,
860 + _g: &str,
861 + _l: &std::path::Path,
862 + _o: &SyncOpts,
863 + ) -> anyhow::Result<()> {
864 + Ok(())
865 + }
866 + async fn push_dir(
867 + &self,
868 + _l: &std::path::Path,
869 + _r: &std::path::Path,
870 + _o: &SyncOpts,
871 + ) -> anyhow::Result<()> {
872 + Ok(())
873 + }
874 + async fn preflight(&self) -> anyhow::Result<()> {
875 + Ok(())
876 + }
877 + fn capabilities(&self) -> &CapabilitySet {
878 + &self.caps
879 + }
880 + }
881 +
751 882 /// Assemble an [`AppState`] from the three per-test inputs, filling in the
752 883 /// executors/syncs (built from `topo`) and the fixed test scaffolding
753 884 /// (metrics handle, event bus, standard OTA registry, empty active map, no
@@ -1939,6 +2070,341 @@
1939 2070 );
1940 2071 assert!(!marker.exists(), "denied build step must NOT have executed");
1941 2072 }
2073 +
2074 + // ---- macOS sign / notarize / staple execution chain (via ScriptedExec) ----
2075 +
2076 + /// Stand up a single-macOS-target app whose recipe is `recipe_body` (with the
2077 + /// literal `ARTIFACT` replaced by a real, non-empty file on disk), dispatch
2078 + /// every host command through `scripted`, run the build to a terminal state,
2079 + /// and return the pool plus the final `(status, error)`. The returned
2080 + /// [`tempfile::TempDir`] must be kept alive by the caller: it holds the
2081 + /// sqlite DB the pool reads.
2082 + async fn run_macos_recipe(
2083 + scripted: Arc<ScriptedExec>,
2084 + recipe_body: &str,
2085 + backoff_secs: Option<u64>,
2086 + ) -> (tempfile::TempDir, SqlitePool, String, String) {
2087 + let tmp = tempfile::tempdir().unwrap();
2088 + let root = tmp.path();
2089 + let repo = root.join("app");
2090 + std::fs::create_dir_all(repo.join("src-tauri")).unwrap();
2091 + std::fs::write(
2092 + repo.join("src-tauri/tauri.conf.json"),
2093 + r#"{"version":"0.0.1"}"#,
2094 + )
2095 + .unwrap();
2096 + std::fs::create_dir_all(repo.join("dist/recipes")).unwrap();
2097 +
2098 + // A real, non-empty artifact so `publish`'s size floor is satisfied; the
2099 + // build step is faked, so nothing else creates it.
2100 + let artifact = repo.join("out/demo.dmg");
2101 + std::fs::create_dir_all(artifact.parent().unwrap()).unwrap();
2102 + std::fs::write(&artifact, b"dmg-bytes").unwrap();
2103 +
2104 + std::fs::write(
2105 + repo.join("dist/recipes/macos.rhai"),
2106 + recipe_body.replace("ARTIFACT", artifact.to_str().unwrap()),
2107 + )
2108 + .unwrap();
2109 +
2110 + let cfg = Config {
2111 + notarize_backoff_secs: backoff_secs,
2112 + ..Config::for_tests(root)
2113 + };
2114 + let pool = crate::db::open(&cfg.db_path).await.unwrap();
2115 + std::fs::write(repo.join("bento.toml"), "targets = [\"macos/aarch64\"]\n").unwrap();
2116 + let topo = Topology::from_str_for_tests(&format!(
2117 + r#"
2118 + [[host]]
2119 + name = "mbp"
2120 + ssh = "local"
2121 + targets = ["macos/aarch64"]
2122 +
2123 + [app.demo]
2124 + repo = "{}"
2125 + "#,
2126 + repo.display()
2127 + ))
2128 + .unwrap();
2129 +
2130 + let mut state = test_state(pool.clone(), topo, cfg);
2131 + // Route every host command through the scripted executor.
2132 + let mut execs = HashMap::new();
2133 + execs.insert("mbp".to_string(), scripted as Arc<dyn Executor>);
2134 + state.executors = Arc::new(execs);
2135 +
2136 + let build_id = start_build(
2137 + state.clone(),
2138 + AppId::new("demo"),
2139 + Version::parse("0.0.1").unwrap(),
2140 + vec!["macos/aarch64".parse().unwrap()],
2141 + )
2142 + .await
2143 + .unwrap();
2144 +
2145 + let mut status = String::new();
2146 + let mut error = String::new();
2147 + for _ in 0..100 {
2148 + let row: Option<(String, Option<String>)> =
2149 + sqlx::query_as("SELECT status, error FROM target_runs WHERE build_id = ?")
2150 + .bind(build_id)
2151 + .fetch_optional(&pool)
2152 + .await
2153 + .unwrap();
2154 + if let Some((s, e)) = row {
2155 + status = s;
2156 + error = e.unwrap_or_default();
2157 + if status != "running" {
2158 + break;
2159 + }
2160 + }
2161 + tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2162 + }
2163 + (tmp, pool, status, error)
2164 + }
2165 +
2166 + async fn release_count(pool: &SqlitePool) -> i64 {
2167 + sqlx::query_scalar("SELECT COUNT(*) FROM releases")
2168 + .fetch_one(pool)
2169 + .await
2170 + .unwrap()
2171 + }
2172 +
2173 + /// The whole macOS release chain end to end through a fake host: codesign,
2174 + /// notarize (Accepted first try), staple, verify_gatekeeper, then publish.
2175 + /// The recorded commands lock the exact incantations each host function
2176 + /// dispatches, and a `releases` row proves the publish gate opened for a
2177 + /// signed + notarized + Gatekeeper-accepted artifact.
2178 + #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2179 + async fn a_macos_recipe_signs_notarizes_staples_verifies_and_publishes() {
2180 + let scripted = Arc::new(
2181 + ScriptedExec::new()
2182 + .on("codesign", 0, "")
2183 + .on("notarytool", 0, r#"{"status":"Accepted"}"#)
2184 + .on("stapler staple", 0, "")
2185 + .on(
2186 + "spctl",
2187 + 0,
2188 + "source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
2189 + ),
2190 + );
2191 + let recipe = r#"
2192 + let h = build_host();
2193 + step("build");
2194 + sh_ok(h, "echo built");
2195 + step("sign");
2196 + codesign(h, "Developer ID Application: Test", "ARTIFACT");
2197 + notarize(h, "ARTIFACT");
2198 + staple(h, "ARTIFACT");
2199 + step("verify");
2200 + if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
2201 + step("publish");
2202 + publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2203 + "#;
2204 + let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, None).await;
2205 +
2206 + assert_eq!(
2207 + status, "ok",
2208 + "signed+notarized macOS build should publish: {error}"
2209 + );
2210 + assert_eq!(
2211 + release_count(&pool).await,
2212 + 1,
2213 + "publish must record a release"
2214 + );
2215 +
2216 + // The exact shell incantations each host function dispatched.
2217 + let cmds = scripted.commands();
2218 + let has = |needle: &str| cmds.iter().any(|c| c.contains(needle));
2219 + assert!(
2220 + has("codesign --force --options runtime --timestamp --sign"),
2221 + "codesign runtime+timestamp incantation, got: {cmds:?}"
2222 + );
2223 + assert!(
2224 + has("xcrun notarytool submit"),
2225 + "notarytool submit: {cmds:?}"
2226 + );
2227 + assert!(
2228 + has("--wait --output-format json"),
2229 + "notarytool --wait json: {cmds:?}"
2230 + );
2231 + assert!(has("xcrun stapler staple"), "stapler staple: {cmds:?}");
2232 + assert!(has("spctl --assess"), "gatekeeper assess: {cmds:?}");
2233 + }
2234 +
2235 + /// A failing `codesign` fails the sign step and aborts the recipe before
2236 + /// publish — an unsigned artifact never ships.
2237 + #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2238 + async fn a_codesign_failure_fails_the_sign_step_and_blocks_publish() {
2239 + let scripted = Arc::new(ScriptedExec::new().on("codesign", 1, ""));
2240 + let recipe = r#"
2241 + let h = build_host();
2242 + step("build");
2243 + sh_ok(h, "echo built");
2244 + step("sign");
2245 + codesign(h, "Developer ID Application: Test", "ARTIFACT");
2246 + step("publish");
2247 + publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2248 + "#;
2249 + let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
2250 +
2251 + assert_eq!(status, "failed", "a failed codesign must fail the target");
2252 + assert!(
2253 + error.contains("codesign failed"),
2254 + "error names the codesign failure, got: {error}"
2255 + );
2256 + assert_eq!(
2257 + release_count(&pool).await,
2258 + 0,
2259 + "nothing may publish after a codesign failure"
2260 + );
2261 + }
2262 +
2263 + /// Even if the recipe ignores `verify_gatekeeper`'s returned `false`, the
2264 + /// publish gate refuses the artifact: `verify_gatekeeper` both records the
2265 + /// rejection and fails its step, and `publish` proves neither passed. This
2266 + /// is the defense-in-depth the pure `PublishAuthority::prove` tests assert in
2267 + /// isolation, here exercised through the real host-function path.
2268 + #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2269 + async fn a_gatekeeper_rejection_bars_publish_even_if_the_recipe_ignores_it() {
2270 + let scripted = Arc::new(
2271 + ScriptedExec::new()
2272 + .on("codesign", 0, "")
2273 + .on("notarytool", 0, r#"{"status":"Accepted"}"#)
2274 + .on("stapler staple", 0, "")
2275 + // Gatekeeper says no: the sentinel is FAIL, not OK.
2276 + .on("spctl", 0, "source=Unnotarized\nBENTO_GATEKEEPER_FAIL"),
2277 + );
2278 + let recipe = r#"
2279 + let h = build_host();
2280 + step("build");
2281 + sh_ok(h, "echo built");
2282 + step("sign");
2283 + codesign(h, "Developer ID Application: Test", "ARTIFACT");
2284 + notarize(h, "ARTIFACT");
2285 + staple(h, "ARTIFACT");
2286 + step("verify");
2287 + verify_gatekeeper(h, "ARTIFACT");
2288 + step("publish");
2289 + publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2290 + "#;
2291 + let (_tmp, pool, status, error) = run_macos_recipe(scripted, recipe, None).await;
2292 +
2293 + assert_eq!(
2294 + status, "failed",
2295 + "a Gatekeeper-rejected artifact must not publish"
2296 + );
2297 + assert!(
2298 + !error.is_empty(),
2299 + "the barred publish must surface an error"
2300 + );
2301 + assert_eq!(
2302 + release_count(&pool).await,
2303 + 0,
2304 + "no release for a rejected artifact"
2305 + );
2306 + }
2307 +
2308 + /// The one flaky, network-bound step: `notarize` retries a non-`Accepted`
2309 + /// result and succeeds on a later attempt. Two notarytool calls (reject then
2310 + /// accept) then a recorded release prove the retry ran and the chain
2311 + /// completed. Backoff is 0 so the retry sleep doesn't stall the test.
2312 + #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2313 + async fn notarize_retries_a_non_accepted_result_then_succeeds() {
2314 + let scripted = Arc::new(
2315 + ScriptedExec::new()
2316 + .on("codesign", 0, "")
2317 + .on_seq(
2318 + "notarytool",
2319 + &[
2320 + (0, r#"{"status":"In Progress"}"#),
2321 + (0, r#"{"status":"Accepted"}"#),
2322 + ],
2323 + )
2324 + .on("stapler staple", 0, "")
2325 + .on(
2326 + "spctl",
2327 + 0,
2328 + "source=Notarized Developer ID\nBENTO_GATEKEEPER_OK",
2329 + ),
2330 + );
2331 + let recipe = r#"
2332 + let h = build_host();
2333 + step("build");
2334 + sh_ok(h, "echo built");
2335 + step("sign");
2336 + codesign(h, "Developer ID Application: Test", "ARTIFACT");
2337 + notarize(h, "ARTIFACT");
2338 + staple(h, "ARTIFACT");
2339 + step("verify");
2340 + if !verify_gatekeeper(h, "ARTIFACT") { throw "not Gatekeeper-accepted"; }
2341 + step("publish");
2342 + publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2343 + "#;
2344 + let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
2345 +
2346 + assert_eq!(
2347 + status, "ok",
2348 + "notarize should succeed on the retry: {error}"
2349 + );
2350 + assert_eq!(
2351 + release_count(&pool).await,
2352 + 1,
2353 + "the retried build still publishes"
2354 + );
2355 + let notary_calls = scripted
2356 + .commands()
2357 + .iter()
2358 + .filter(|c| c.contains("notarytool"))
2359 + .count();
2360 + assert_eq!(
2361 + notary_calls, 2,
2362 + "notarytool ran once, was rejected, then ran again"
2363 + );
2364 + }
2365 +
2366 + /// `notarize` gives up after its bounded retries: three notarytool attempts,
2367 + /// all non-`Accepted`, fail the target and bar publish.
2368 + #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2369 + async fn notarize_fails_the_target_after_exhausting_its_retries() {
2370 + let scripted = Arc::new(
2371 + ScriptedExec::new()
2372 + .on("codesign", 0, "")
2373 + // Every attempt reports a still-pending status, never Accepted.
2374 + .on("notarytool", 0, r#"{"status":"In Progress"}"#),
2375 + );
2376 + let recipe = r#"
2377 + let h = build_host();
2378 + step("build");
2379 + sh_ok(h, "echo built");
2380 + step("sign");
2381 + codesign(h, "Developer ID Application: Test", "ARTIFACT");
2382 + notarize(h, "ARTIFACT");
2383 + step("publish");
2384 + publish("tauri-mnw", "demo", "macos/aarch64", "0.0.1", "ARTIFACT", #{});
2385 + "#;
2386 + let (_tmp, pool, status, error) = run_macos_recipe(scripted.clone(), recipe, Some(0)).await;
2387 +
2388 + assert_eq!(
2389 + status, "failed",
2390 + "exhausted notarization must fail the target"
2391 + );
2392 + assert!(
2393 + error.contains("notarization failed after 3 attempts"),
2394 + "error names the exhausted retry, got: {error}"
2395 + );
2396 + assert_eq!(
2397 + release_count(&pool).await,
2398 + 0,
2399 + "an unnotarized artifact never publishes"
2400 + );
2401 + let notary_calls = scripted
2402 + .commands()
2403 + .iter()
2404 + .filter(|c| c.contains("notarytool"))
2405 + .count();
2406 + assert_eq!(notary_calls, 3, "the retry is bounded at three attempts");
2407 + }
1942 2408 }
1943 2409
1944 2410 /// Every recipe of every configured app must parse.