//! [`RecipeCtx`]: everything a recipe's host functions are handed, and the //! step lifecycle they drive. //! //! The four private run-state fields are reached from the sibling modules //! through the accessors at the bottom of this file, which hand out values and //! never the guard. use super::git::{ git_fetch_cmd, git_rev_parse_cmd, git_tag_exists_cmd, git_worktree_pin_cmd, worktree_failure_reason, }; use super::{StepState, action_for, default_step_budget}; use crate::config::Config; use crate::domain::{AppId, Status, Step, StepRunId, Target, Version}; use crate::events::{self, Event, EventTx}; use crate::ota::OtaRegistry; use crate::state::ExecutorMap; use crate::topology::{DeployTarget, Kind}; use anyhow::{Context as _, Result}; use ops_core::live_log::LiveLog; use ops_exec::{Action, Executor, Step as OpStep}; use sqlx::SqlitePool; use std::collections::HashMap; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use tokio::runtime::Handle; use tokio::sync::Mutex as AsyncMutex; /// Everything a recipe's host functions need, shared (Arc) into each closure. pub struct RecipeCtx { pub app: AppId, pub version: Version, pub target: Target, /// Name of the host this target builds on (resolved from the topology by the /// runner). Recipes read it via `build_host()` so one per-platform recipe can /// dispatch to the right host across arches (linux x86_64 -> fw13, aarch64 -> /// astra) without hard-coding a host name. pub build_host: String, /// The build host's SSH destination (topology `ssh`), as opposed to its /// name. Deploy compares it against the service host's to tell "build and /// run on the same box" from "two hosts that need a transfer" — a question /// the host NAMES cannot answer, since a build host and a deploy /// destination are declared in different files and need not agree on one. pub build_host_ssh: String, /// This release's git tag, rendered from the app's `tag_format`. Held here /// rather than derived from the version because a repo holding several /// products spells it per product (`pom-v0.4.1`), and the recipe, the /// preflight barrier and the failure message all have to agree on it. pub tag: String, /// The app's default checkout path (topology `repo`, `~`-prefixed). Read it /// through [`RecipeCtx::repo_for`] for anything that runs on a build host; /// this field alone is the daemon-local answer. pub repo: String, /// Per-host checkout overrides (topology `repo_by_host`). Empty for every app /// that has not declared one, which is all of them but the Windows-shipping /// ones. pub repo_by_host: HashMap, /// Cargo features this app's release builds enable (topology `features`). /// Recipes read it via `feature_flags()`. pub features: Vec, /// App or library. Decides which capability a `verify` step is gated on; /// see `action_for`. pub kind: Kind, pub target_run_id: i64, /// Capability-scoped executor per build host. Recipe commands dispatch /// through these — the transport (local / ssh / in-session agent) and the /// capability gate are the executor's, not the engine's. pub execs: Arc, /// Sync transport per build host, used only by `collect` to pull artifacts /// back. Never the agent, even for an agent host — see `state::build_sync`. /// Not an execution path. pub syncs: Arc, /// Where this target installs, for a `kind = "service"` app. `None` for an /// app or a library, which makes every deploy host function fail with that /// as the reason rather than with a missing-host error. /// /// The runner resolves it from the app manifest's `[[deploy]]` table and /// registers its executor into `execs` under the destination's host string, /// so `sh_ok(deploy_host(), ...)` reaches the service host through the same /// capability gate as everything else. pub deploy: Option, pub pool: SqlitePool, pub events: EventTx, pub cfg: Arc, pub ota: Arc, pub rt: Handle, current: Mutex>, /// Gatekeeper verdict recorded by `verify_gatekeeper`: `None` = never run, /// `Some(false)` = ran and rejected, `Some(true)` = accepted. `publish` /// requires `Some(true)` for a macOS/iOS artifact (the proof it is signed + /// notarized). gatekeeper_ok: Mutex>, /// Steps finalized as `Failed` during this run. A non-empty ledger bars /// `publish` — an artifact is never shipped after a step failed, even if the /// recipe ignored the failure and ran on. failed_steps: Mutex>, /// Set when a newer build supersedes this run. Checked at step boundaries /// and before publish so a superseded recipe stops promptly rather than /// running to completion (the blocking Rhai body can't be `abort()`ed). cancel: Arc, /// The all-targets-green publish gate (topology `require_all_targets`). /// `Some(declared)` ⇒ `publish` refuses unless every one of `declared` has a /// successful latest run for this `(app, version)`. `None` ⇒ gate off, each /// target publishes independently. all_green_required: Option>, /// sha256 of each artifact hashed at `collect`, keyed by file name. `publish` /// reads it to record `releases.artifact_hash` for the bytes it ships, so the /// hash is the one computed when the artifact landed rather than a re-read /// that could see a different file. Absent ⇒ `publish` hashes on demand. artifact_hashes: Mutex>, } impl RecipeCtx { #[allow(clippy::too_many_arguments)] pub fn new( app: AppId, version: Version, target: Target, build_host: String, build_host_ssh: String, tag: String, repo: String, features: Vec, kind: Kind, target_run_id: i64, execs: Arc, syncs: Arc, deploy: Option, pool: SqlitePool, events: EventTx, cfg: Arc, ota: Arc, rt: Handle, cancel: Arc, all_green_required: Option>, ) -> Self { Self { app, version, target, build_host, build_host_ssh, tag, repo, repo_by_host: HashMap::new(), features, kind, target_run_id, execs, syncs, deploy, pool, events, cfg, ota, rt, current: Mutex::new(None), gatekeeper_ok: Mutex::new(None), failed_steps: Mutex::new(Vec::new()), cancel, all_green_required, artifact_hashes: Mutex::new(HashMap::new()), } } /// Declare the app's per-host checkout overrides (topology `repo_by_host`). /// /// Separate from `new` because it is empty for every app that has not opted /// in, and `new` already carries twenty arguments no test wants a /// twenty-first of. #[must_use] pub fn with_repo_by_host(mut self, repo_by_host: HashMap) -> Self { self.repo_by_host = repo_by_host; self } /// Where this app is checked out on `host` (topology `AppConfig::repo_for`). /// /// Every git command a recipe or the engine runs on a build host goes /// through this. The bare `repo` field is the daemon-local path. pub fn repo_for(&self, host: &str) -> &str { self.repo_by_host .get(host) .map_or(self.repo.as_str(), String::as_str) } /// Whether a newer build has superseded this run. pub(super) fn is_cancelled(&self) -> bool { self.cancel.load(Ordering::SeqCst) } pub(super) fn now() -> String { chrono::Utc::now().to_rfc3339() } /// `////..log`. /// /// The run id is in the filename because the rest of the key is not unique: /// re-running an app at a version it already built (a retry, or a rebuild of /// an already-published release) reopens the same path, and appending would /// show two runs' output with nothing marking the boundary. The /// step ledger records a run id per step, so keying the file on it makes a /// ledger row resolve to exactly one file and keeps the earlier run readable. fn log_path(&self, step: Step, run_id: StepRunId) -> PathBuf { self.log_dir() .join(format!("{}.{}.log", step.as_str(), run_id.0)) } /// `////`. fn log_dir(&self) -> PathBuf { let target_dir = self.target.to_string().replace('/', "-"); self.cfg .logs_root .join(self.app.as_str()) .join(self.version.to_string()) .join(target_dir) } /// Close the previous step (as `Ok`), open a new one: insert its DB row, /// open a live log whose chunks broadcast `StepLogChunk`, emit `StepStart`. pub(super) fn begin_step(self: &Arc, step: Step) -> Result<()> { anyhow::ensure!( !self.is_cancelled(), "build superseded by a newer request; aborting before `{}`", step.as_str() ); self.finish_step(Status::Ok)?; let me = self.clone(); let started = Self::now(); let started_for_header = started.clone(); let run_id = self.rt.block_on(async move { // The step row and the target's current_step pointer are one logical // state — write them atomically so a failure can't leave a `running` // step row while current_step still names the previous step. let mut tx = me.pool.begin().await.context("begin step tx")?; // log_ref names the run id, which only exists once the row does, so // the path is written back inside the same transaction rather than // guessed beforehand. let id: i64 = sqlx::query_scalar( "INSERT INTO step_runs (target_run_id, step, status, started_at) VALUES (?, ?, 'running', ?) RETURNING id", ) .bind(me.target_run_id) .bind(step.as_str()) .bind(&started) .fetch_one(&mut *tx) .await .context("insert step_run")?; let log_ref = me .log_path(step, StepRunId(id)) .to_string_lossy() .into_owned(); sqlx::query("UPDATE step_runs SET log_ref = ? WHERE id = ?") .bind(&log_ref) .bind(id) .execute(&mut *tx) .await .context("set step_run log_ref")?; sqlx::query("UPDATE target_runs SET current_step = ? WHERE id = ?") .bind(step.as_str()) .bind(me.target_run_id) .execute(&mut *tx) .await .context("update current_step")?; tx.commit().await.context("commit step tx")?; anyhow::Ok(StepRunId(id)) })?; // Live log: each chunk fans out as a StepLogChunk event keyed by run_id. let events = self.events.clone(); let cb_run_id = run_id; let mut log = self.rt.block_on(LiveLog::open( self.log_path(step, run_id), Box::new(move |seq, text| { events::emit( &events, Event::StepLogChunk { run_id: cb_run_id, seq, text: text.to_string(), }, ); }), )); events::emit( &self.events, Event::StepStart { run_id, app: self.app.clone(), version: self.version.clone(), target: self.target, step, }, ); // A log that names its own run: reading one tells you which ledger row // it belongs to without going back to the DB, and a file that somehow // does get appended to still shows where the second run began. Written // after `StepStart` so its chunk event cannot precede the step it // belongs to. let header = format!( "=== bento {app} {version} {target} step={step} run_id={run_id} started={started_for_header} ===\n", app = self.app.as_str(), version = self.version, target = self.target, step = step.as_str(), ); self.rt.block_on(async { use ops_core::remote::LogSink as _; log.write_chunk(header.as_bytes()).await; }); *self.current.lock().unwrap() = Some(StepState { run_id, step, log: Arc::new(AsyncMutex::new(log)), failed: false, deadline: std::time::Instant::now() + self.step_budget(step), }); Ok(()) } /// This build's budget for `step`: the `Config` override if set, else the /// per-kind default. fn step_budget(&self, step: Step) -> std::time::Duration { self.cfg .step_timeout_secs .map_or_else(|| default_step_budget(step), std::time::Duration::from_secs) } /// The current step's wall-clock deadline (or a default if no step is open, /// which only happens before the first `step()` — commands then run under an /// implicit `Build` step opened by `ensure_step`). fn step_deadline(&self) -> std::time::Instant { self.current.lock().unwrap().as_ref().map_or_else( || std::time::Instant::now() + self.step_budget(Step::Build), |s| s.deadline, ) } /// Drive `fut` on the runtime, but stop early on two conditions the recipe /// bodies otherwise cannot observe (they run synchronously on a blocking /// thread): the current step's deadline, and supersession by a newer build. /// Either turns into an error that fails the step and unwinds the recipe, so /// a wedged command cannot run unbounded and a superseded build stops /// mid-step instead of only at the next step boundary. pub(super) fn run_bounded(&self, what: &str, fut: F) -> Result where F: std::future::Future>, { let deadline = self.step_deadline(); let cancel = self.cancel.clone(); self.rt.block_on(async move { tokio::pin!(fut); let watch = async { // Poll the cooperative cancel flag; the finalizer and a // superseding build both set it. Cheap next to a build step. while !cancel.load(Ordering::SeqCst) { tokio::time::sleep(std::time::Duration::from_millis(250)).await; } }; tokio::select! { r = &mut fut => r, () = tokio::time::sleep_until(deadline.into()) => { Err(anyhow::anyhow!("`{what}` exceeded its per-step deadline")) } () = watch => { Err(anyhow::anyhow!("build superseded by a newer request; aborting `{what}`")) } } }) } /// Flag the currently-open step as failed (no-op if none is open). Forces /// its recorded status to `Failed` at `finish_step` and adds it to the /// publish-barring ledger, even though the recipe kept running. pub(super) fn fail_current_step(&self) { if let Some(st) = self.current.lock().unwrap().as_mut() { st.failed = true; } } /// Finalize the open step (if any): close its log, stamp the DB row, emit /// `StepDone`. Idempotent when no step is open. A step flagged via /// `fail_current_step` is recorded `Failed` regardless of the requested /// status, and added to the ledger `publish` consults. pub fn finish_step(self: &Arc, status: Status) -> Result<()> { let st = self.current.lock().unwrap().take(); let Some(st) = st else { return Ok(()) }; let status = if st.failed { Status::Failed } else { status }; if status == Status::Failed { self.failed_steps.lock().unwrap().push(st.step); } let me = self.clone(); self.rt.block_on(async move { // Drop all log refs so the sink can be owned + flushed. if let Ok(m) = Arc::try_unwrap(st.log) { m.into_inner().close().await; } if let Err(e) = sqlx::query( "UPDATE step_runs SET status = ?, finished_at = ? WHERE id = ?", ) .bind(status.as_str()) .bind(Self::now()) .bind(st.run_id.0) .execute(&me.pool) .await { tracing::error!(step = st.step.as_str(), error = %e, "could not stamp step_run status"); } }); events::emit( &self.events, Event::StepDone { run_id: st.run_id, app: self.app.clone(), target: self.target, step: st.step, status, }, ); Ok(()) } /// Ensure a step is open; default to `Build` if a recipe runs a command /// before declaring one. pub(super) fn ensure_step(self: &Arc) -> Result>> { if self.current.lock().unwrap().is_none() { self.begin_step(Step::Build)?; } Ok(self.current.lock().unwrap().as_ref().unwrap().log.clone()) } /// The step currently open, or `Build` as a default for failure /// attribution before any step was declared. pub fn current_step(&self) -> Step { self.current .lock() .unwrap() .as_ref() .map_or(Step::Build, |s| s.step) } /// The capability-scoped executor for `name`, or an error if the host isn't /// in the topology. pub(super) fn exec(&self, name: &str) -> Result> { self.execs .get(name) .cloned() .ok_or_else(|| anyhow::anyhow!("unknown build host `{name}` (not in topology)")) } /// The transport that moves artifacts off `name`, for `collect`'s remote /// scp source. Distinct from `RecipeCtx::exec`'s executor: an agent host /// signs over `AgentRpc` /// but is collected from over ssh (`state::build_sync`). pub(super) fn host_sync(&self, name: &str) -> Result> { self.syncs .get(name) .cloned() .ok_or_else(|| anyhow::anyhow!("unknown build host `{name}` (not in topology)")) } /// Run `cmd` on `host` through its capability-scoped executor, streaming into /// the current step's log. The command's [`Action`] is derived from the open /// step (see [`action_for`]) and gated at the transport before dispatch — so a /// `build` step on a host without the `build` grant is denied, and the macOS /// sign steps ride the in-session `AgentRpc` transport automatically. Returns /// exit code + a tail of stdout for the recipe to branch on. pub(super) fn run(self: &Arc, host: &str, cmd: &str) -> Result<(i32, String)> { // The service host is addressed as a service host whatever step is open. // Deriving the action from the step is right for a build host, where the // step IS the work; on a service host it would ask for `build` during a // `verify` and be denied for a reason unrelated to what was attempted. let action = match &self.deploy { Some(d) if d.host == host => Action::Deploy, _ => action_for(self.current_step(), self.kind), }; self.run_as(host, cmd, action) } /// `run`, with the [`Action`] stated rather than resolved. Used where the /// caller already knows which plane it is on. fn run_as(self: &Arc, host: &str, cmd: &str, action: Action) -> Result<(i32, String)> { let sink = self.ensure_step()?; let exec = self.exec(host)?; let cur = self.current_step(); let step = OpStep::shell(action, cmd.to_string()); // Echo the command before running it. Without this a log says what // happened but not what was asked, and a gate that prints nothing when // it passes (`cargo fmt --all --check`) is indistinguishable from a gate // that never ran. let echo = format!("$ [{host}] {cmd}\n"); // Bounded by the step's deadline and interruptible on supersession, so a // hung command fails its step instead of running unbounded, and a // superseded build stops mid-step rather than only at the next boundary. let label = format!("{cur} command on `{host}`"); let out = self.run_bounded(&label, async move { use ops_core::remote::LogSink as _; let mut guard = sink.lock().await; guard.write_chunk(echo.as_bytes()).await; exec.run_streaming(&step, &mut *guard).await })?; let code = out.status.code().unwrap_or(-1); let stdout = String::from_utf8_lossy(&out.stdout); let tail: String = stdout .chars() .rev() .take(2000) .collect::>() .into_iter() .rev() .collect(); Ok((code, tail)) } /// Assert `host`'s build tree is at the release tag and return the commit, /// which is what a recipe's `checkout` step logs. Fetch + checkout stream /// into the current step's log; the sha comes from a separate `rev-parse` so /// its stdout is only the sha. /// /// The preflight has already put this tree at the tag before any recipe ran — /// this is the same operation again, on purpose, so that a recipe's own /// `checkout` step is a real step with a real log rather than a claim about /// something that happened elsewhere. Re-running it is cheap and, because the /// tree is Bento's own worktree, forcing: a build that has already dirtied it /// must not be able to fail its own retry. pub(super) fn checkout_sha(self: &Arc, host: &str) -> Result { // Every command below runs ON `host`, so the path is that host's, not the // daemon's. Windows is why: its worktree is under `C:/Users/me/Code/...`. let repo = self.repo_for(host).to_string(); // A failing mirror is not a failing release: fetch is advisory, and only // the checkout decides. Its output still streams into the step log, so an // unreachable remote stays visible without being fatal. let _ = self.run(host, &git_fetch_cmd(&repo))?; let (code, err) = self.run(host, &git_worktree_pin_cmd(&repo, &self.tag))?; if code != 0 { let (probe, _) = self.run(host, &git_tag_exists_cmd(&repo, &self.tag))?; anyhow::bail!( "checkout of {} failed on `{host}`: {}", self.tag, worktree_failure_reason(&self.tag, probe == 0, &err) ); } let (code, tail) = self.run(host, &git_rev_parse_cmd(&repo))?; anyhow::ensure!(code == 0, "rev-parse failed on `{host}`"); Ok(tail.trim().to_string()) } /// Every artifact this run collected, `file name -> sha256`. /// /// Already computed at `collect`, which is the only moment the bytes are /// known to be the ones that landed. pub fn artifact_hashes(&self) -> HashMap { self.artifact_hashes.lock().unwrap().clone() } /// The steps finalized as `Failed` so far, as a snapshot. /// /// Cloned rather than borrowed because `PublishAuthority::prove` wants a /// slice and the caller slices this, which is the shape that keeps the lock /// out of the caller's scope. pub(super) fn failed_steps_snapshot(&self) -> Vec { self.failed_steps.lock().unwrap().clone() } /// Record the digest `collect` computed for one artifact, by file name. pub(super) fn record_artifact_hash(&self, name: String, digest: String) { self.artifact_hashes.lock().unwrap().insert(name, digest); } /// One artifact's recorded digest, if `collect` hashed it. /// /// The targeted read beside [`RecipeCtx::artifact_hashes`], which clones the /// whole map; `publish` wants exactly one entry. pub(super) fn artifact_hash(&self, name: &str) -> Option { self.artifact_hashes.lock().unwrap().get(name).cloned() } /// The all-targets-green publish gate, if this app declares one. pub(super) fn all_green_required(&self) -> Option> { self.all_green_required.clone() } /// The gatekeeper verdict: `None` = never run, `Some(false)` = rejected. pub(super) fn gatekeeper_ok(&self) -> Option { *self.gatekeeper_ok.lock().unwrap() } /// Record what the gatekeeper said about this artifact. pub(super) fn set_gatekeeper_ok(&self, accepted: bool) { *self.gatekeeper_ok.lock().unwrap() = Some(accepted); } } #[cfg(test)] mod tests { use super::super::build_engine; use super::*; /// Run 3 S1: once the cooperative cancel flag is set (a newer build /// superseded this run), a step boundary refuses to proceed — the blocking /// recipe stops at the next `step()` instead of running on and publishing. #[tokio::test] async fn begin_step_bails_when_cancelled() { let dir = tempfile::tempdir().unwrap(); let cfg = Arc::new(Config::for_tests(dir.path())); let pool = crate::db::open(&cfg.db_path).await.unwrap(); let cancel = Arc::new(AtomicBool::new(true)); let ctx = Arc::new(RecipeCtx::new( AppId::new("demo"), Version::parse("0.1.0").unwrap(), "linux/x86_64".parse().unwrap(), "fw13".into(), "local".into(), "v0.1.0".into(), "/tmp".into(), vec![], Kind::App, 1, Arc::new(std::collections::HashMap::new()), Arc::new(std::collections::HashMap::new()), None, pool, crate::events::channel(), cfg, Arc::new(OtaRegistry::standard("https://makenot.work")), tokio::runtime::Handle::current(), cancel.clone(), None, )); // Cancelled: begin_step refuses before touching the DB (the ensure! is // ahead of any block_on, so this is safe to call from the async test). let err = ctx.begin_step(Step::Build).unwrap_err(); assert!(err.to_string().contains("supersede"), "got: {err}"); assert!(ctx.is_cancelled()); } /// `feature_flags()` returns a whole flag or nothing at all. An app with /// no declared features must not yield a bare `--features`, which would /// swallow the next word of the build command as its argument. #[tokio::test] async fn feature_flags_renders_whole_flag_or_empty() { async fn flags_for(features: Vec) -> String { let dir = tempfile::tempdir().unwrap(); let cfg = Arc::new(Config::for_tests(dir.path())); let pool = crate::db::open(&cfg.db_path).await.unwrap(); let ctx = Arc::new(RecipeCtx::new( AppId::new("demo"), Version::parse("0.1.0").unwrap(), "linux/x86_64".parse().unwrap(), "fw13".into(), "local".into(), "v0.1.0".into(), "/tmp".into(), features, Kind::App, 1, Arc::new(std::collections::HashMap::new()), Arc::new(std::collections::HashMap::new()), None, pool, crate::events::channel(), cfg, Arc::new(OtaRegistry::standard("https://makenot.work")), tokio::runtime::Handle::current(), Arc::new(AtomicBool::new(false)), None, )); let engine = build_engine(&ctx); engine.eval::("feature_flags()").unwrap() } assert_eq!(flags_for(vec![]).await, ""); assert_eq!( flags_for(vec!["supernote".into()]).await, "--features supernote" ); assert_eq!( flags_for(vec!["supernote".into(), "extra".into()]).await, "--features supernote,extra" ); } /// `repo()` answers for the host this target builds on, not for the daemon. /// /// This is what lets a Windows recipe call `repo()` and `checkout_sha(h)` /// instead of hard-coding `C:/Users/me/...` — and hard-coding it is what /// kept those recipes off the release-tag pin, since `checkout_sha` builds /// its git commands from the app's path and takes no override. #[tokio::test] async fn repo_resolves_per_build_host() { async fn repo_on(build_host: &str) -> String { let dir = tempfile::tempdir().unwrap(); let cfg = Arc::new(Config::for_tests(dir.path())); let pool = crate::db::open(&cfg.db_path).await.unwrap(); let ctx = Arc::new( RecipeCtx::new( AppId::new("demo"), Version::parse("0.1.0").unwrap(), "linux/x86_64".parse().unwrap(), build_host.into(), "local".into(), "v0.1.0".into(), "~/Code/Apps/demo".into(), vec![], Kind::App, 1, Arc::new(std::collections::HashMap::new()), Arc::new(std::collections::HashMap::new()), None, pool, crate::events::channel(), cfg, Arc::new(OtaRegistry::standard("https://makenot.work")), tokio::runtime::Handle::current(), Arc::new(AtomicBool::new(false)), None, ) .with_repo_by_host(HashMap::from([( "windows-x86".to_string(), "C:/Users/me/Code/Apps/demo".to_string(), )])), ); build_engine(&ctx).eval::("repo()").unwrap() } assert_eq!(repo_on("windows-x86").await, "C:/Users/me/Code/Apps/demo"); assert_eq!(repo_on("fw13").await, "~/Code/Apps/demo"); } }