max / makenotwork
- Co-Authored-By
- Claude Opus 5 (1M context) <noreply@anthropic.com>
10 files changed,
+889 insertions,
-9 deletions
| @@ -30,6 +30,9 @@ | |||
| 30 | 30 | semver = { version = "1.0", features = ["serde"] } | |
| 31 | 31 | sha2 = "0.10" | |
| 32 | 32 | async-trait = "0.1" | |
| 33 | + | # Staging dir for a service deploy: the binary lands here on the daemon between | |
| 34 | + | # the pull off the build host and the push onto the service host. | |
| 35 | + | tempfile = "3.20" | |
| 33 | 36 | ||
| 34 | 37 | [dev-dependencies] | |
| 35 | 38 | async-trait = "0.1" |
| @@ -35,3 +35,11 @@ | |||
| 35 | 35 | # the same commit before any target builds (default true). Turn off only to | |
| 36 | 36 | # build from an untagged commit. | |
| 37 | 37 | # pin_release_sha = true | |
| 38 | + | ||
| 39 | + | # The command that installs a service binary and restarts its unit, run on the | |
| 40 | + | # service host with `<staged-binary> <install-path> <unit>` appended. Bento never | |
| 41 | + | # installs or restarts anything itself: it stages bytes and calls this, and what | |
| 42 | + | # it calls is a root script that re-checks its own arguments. That way the | |
| 43 | + | # sudoers grant on a production box is ONE auditable script rather than a broad | |
| 44 | + | # install + systemctl grant. Only affects `kind = "service"` apps. | |
| 45 | + | # deploy_installer = "sudo /usr/local/lib/bento/install-service.sh" |
| @@ -62,3 +62,20 @@ | |||
| 62 | 62 | ||
| 63 | 63 | [app.alloy_tui] | |
| 64 | 64 | repo = "~/Code/Libraries/alloy_tui" | |
| 65 | + | ||
| 66 | + | # Services: binaries Bento installs onto the hosts that run them, rather than | |
| 67 | + | # distributing to users. Same two-file split again — the pointer lives here, and | |
| 68 | + | # `kind = "service"` plus the `[[deploy]]` table (which target lands on which | |
| 69 | + | # machine, at which path, restarting which unit) lives in the repo's bento.toml. | |
| 70 | + | # | |
| 71 | + | # A deploy destination is deliberately NOT a `[[host]]` above. Build hosts and | |
| 72 | + | # service hosts are different grants: a build host gets `build`/`package`, a | |
| 73 | + | # service host gets `deploy`/`restart` and never `build`, so a recipe cannot turn | |
| 74 | + | # a production box into a build host. Its executor is constructed per run from | |
| 75 | + | # the app's own manifest and exists only for that run. | |
| 76 | + | # | |
| 77 | + | # Each service host needs the privileged installer and its scoped sudoers line | |
| 78 | + | # installed once. See pom/deploy/install-service.sh and bento-deploy.sudoers. | |
| 79 | + | ||
| 80 | + | [app.pom] | |
| 81 | + | repo = "~/Code/MNW/pom" |
| @@ -39,12 +39,31 @@ | |||
| 39 | 39 | /// whose repos aren't git checkouts). | |
| 40 | 40 | #[serde(default = "default_true")] | |
| 41 | 41 | pub pin_release_sha: bool, | |
| 42 | + | /// The command that installs a service binary and restarts its unit, run on | |
| 43 | + | /// the service host with `<staged-binary> <install-path> <unit>` appended. | |
| 44 | + | /// | |
| 45 | + | /// Bento never installs or restarts anything itself. It stages bytes and | |
| 46 | + | /// calls this, and what it calls is a root script whose arguments are | |
| 47 | + | /// re-checked on the far side — so the sudoers grant on a production box is | |
| 48 | + | /// ONE auditable script rather than a broad `install` + `systemctl` grant. | |
| 49 | + | /// Same shape as Sando's `install-companion.sh`. | |
| 50 | + | /// | |
| 51 | + | /// Configurable because the sudo path differs per host fleet, and because a | |
| 52 | + | /// test needs to point it somewhere that is not root. It is read from the | |
| 53 | + | /// same trusted daemon config that already names the secrets root, so this | |
| 54 | + | /// adds no trust boundary that was not already there. | |
| 55 | + | #[serde(default = "default_deploy_installer")] | |
| 56 | + | pub deploy_installer: String, | |
| 42 | 57 | } | |
| 43 | 58 | ||
| 44 | 59 | fn default_true() -> bool { | |
| 45 | 60 | true | |
| 46 | 61 | } | |
| 47 | 62 | ||
| 63 | + | fn default_deploy_installer() -> String { | |
| 64 | + | "sudo /usr/local/lib/bento/install-service.sh".into() | |
| 65 | + | } | |
| 66 | + | ||
| 48 | 67 | fn default_logs_root() -> PathBuf { | |
| 49 | 68 | PathBuf::from("/srv/bento/logs") | |
| 50 | 69 | } | |
| @@ -71,6 +90,7 @@ | |||
| 71 | 90 | // Test repos are plain dirs, not git checkouts; the barrier is | |
| 72 | 91 | // exercised by its own tests that build a real tagged repo. | |
| 73 | 92 | pin_release_sha: false, | |
| 93 | + | deploy_installer: default_deploy_installer(), | |
| 74 | 94 | } | |
| 75 | 95 | } | |
| 76 | 96 | } |
| @@ -161,6 +161,13 @@ | |||
| 161 | 161 | /// The canonical release step sequence. A recipe marks transitions by calling | |
| 162 | 162 | /// the `step(name)` host function; not every platform uses every step (Linux | |
| 163 | 163 | /// skips sign/notarize/staple). The TUI renders these as matrix columns. | |
| 164 | + | /// | |
| 165 | + | /// `Deploy` is the terminal step for a [`crate::topology::Kind::Service`]: where | |
| 166 | + | /// an app ends at `collect` and a library at `publish`, a service ends by | |
| 167 | + | /// landing its binary on the host that runs it and restarting the unit. It is | |
| 168 | + | /// last in `ALL` because a service reaches it after every gate the other kinds | |
| 169 | + | /// use, and appending rather than inserting leaves the existing column order | |
| 170 | + | /// (and every stored `step_runs.step` string) untouched. | |
| 164 | 171 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] | |
| 165 | 172 | #[serde(rename_all = "snake_case")] | |
| 166 | 173 | pub enum Step { | |
| @@ -174,11 +181,12 @@ | |||
| 174 | 181 | Package, | |
| 175 | 182 | Publish, | |
| 176 | 183 | Collect, | |
| 184 | + | Deploy, | |
| 177 | 185 | } | |
| 178 | 186 | ||
| 179 | 187 | impl Step { | |
| 180 | 188 | /// All steps in canonical order — the matrix column set. | |
| 181 | - | pub const ALL: [Step; 10] = [ | |
| 189 | + | pub const ALL: [Step; 11] = [ | |
| 182 | 190 | Step::Checkout, | |
| 183 | 191 | Step::Prebuild, | |
| 184 | 192 | Step::Build, | |
| @@ -189,6 +197,7 @@ | |||
| 189 | 197 | Step::Package, | |
| 190 | 198 | Step::Publish, | |
| 191 | 199 | Step::Collect, | |
| 200 | + | Step::Deploy, | |
| 192 | 201 | ]; | |
| 193 | 202 | ||
| 194 | 203 | pub fn as_str(self) -> &'static str { | |
| @@ -203,6 +212,7 @@ | |||
| 203 | 212 | Step::Package => "package", | |
| 204 | 213 | Step::Publish => "publish", | |
| 205 | 214 | Step::Collect => "collect", | |
| 215 | + | Step::Deploy => "deploy", | |
| 206 | 216 | } | |
| 207 | 217 | } | |
| 208 | 218 | } |
| @@ -17,7 +17,7 @@ | |||
| 17 | 17 | use crate::events::{self, Event, EventTx}; | |
| 18 | 18 | use crate::ota::{OtaRegistry, PublishAuthority, Release}; | |
| 19 | 19 | use crate::state::ExecutorMap; | |
| 20 | - | use crate::topology::Kind; | |
| 20 | + | use crate::topology::{DeployTarget, Kind}; | |
| 21 | 21 | use anyhow::{Context as _, Result}; | |
| 22 | 22 | use ops_core::live_log::LiveLog; | |
| 23 | 23 | use ops_exec::{Action, Executor, ObserveKind, Step as OpStep, SyncOpts}; | |
| @@ -54,13 +54,20 @@ | |||
| 54 | 54 | Step::Package => Action::Package, | |
| 55 | 55 | Step::Verify => match kind { | |
| 56 | 56 | Kind::App => Action::Observe(ObserveKind::Custom("gatekeeper".into())), | |
| 57 | - | // Running the build toolchain to inspect a crate, which is what | |
| 58 | - | // `build` means on a host. | |
| 59 | - | Kind::Library => Action::Build, | |
| 57 | + | // Running the build toolchain to inspect a crate or a service | |
| 58 | + | // binary, which is what `build` means on a host. Neither has a | |
| 59 | + | // bundle for Gatekeeper to have an opinion about. | |
| 60 | + | Kind::Library | Kind::Service => Action::Build, | |
| 60 | 61 | }, | |
| 61 | 62 | // Publish/Collect run on the daemon, not through a host executor; this | |
| 62 | 63 | // label only applies if a recipe runs a bare `sh` while one is open. | |
| 63 | 64 | Step::Publish | Step::Collect => Action::Package, | |
| 65 | + | // The one step that dispatches to a host OUTSIDE the build topology. | |
| 66 | + | // Every command a recipe runs while `deploy` is open — the install, the | |
| 67 | + | // restart, the health assertion — carries this action, so it reaches the | |
| 68 | + | // service host only through the deploy grant and reaches a build host | |
| 69 | + | // not at all (no build host is granted `deploy`). | |
| 70 | + | Step::Deploy => Action::Deploy, | |
| 64 | 71 | } | |
| 65 | 72 | } | |
| 66 | 73 | ||
| @@ -106,10 +113,24 @@ | |||
| 106 | 113 | // rsync of multi-GiB artifacts off the build host. | |
| 107 | 114 | Step::Collect => 30, | |
| 108 | 115 | Step::Publish => 20, | |
| 116 | + | // A binary push, an install, a unit restart, and a health poll. Minutes | |
| 117 | + | // of work; the ceiling is for a wedged transport, not slow work. | |
| 118 | + | Step::Deploy => 15, | |
| 109 | 119 | }; | |
| 110 | 120 | Duration::from_secs(mins * 60) | |
| 111 | 121 | } | |
| 112 | 122 | ||
| 123 | + | /// Where a service's binary is staged on the host that will run it, before the | |
| 124 | + | /// privileged installer moves it into place. | |
| 125 | + | /// | |
| 126 | + | /// A fixed, unguessable-by-accident path rather than a recipe-chosen one, | |
| 127 | + | /// because the installer refuses any source outside it. That refusal is the | |
| 128 | + | /// only thing standing between the NOPASSWD sudo grant and `install`-as-root to | |
| 129 | + | /// an arbitrary path, so both ends have to name the same constant. `/var/tmp` | |
| 130 | + | /// rather than `/tmp` so a staged binary survives a `PrivateTmp` unit and a | |
| 131 | + | /// systemd tmpfiles sweep between staging and install. | |
| 132 | + | pub const DEPLOY_STAGING_ROOT: &str = "/var/tmp/bento-deploy"; | |
| 133 | + | ||
| 113 | 134 | /// Everything a recipe's host functions need, shared (Arc) into each closure. | |
| 114 | 135 | pub struct RecipeCtx { | |
| 115 | 136 | pub app: AppId, | |
| @@ -120,6 +141,12 @@ | |||
| 120 | 141 | /// dispatch to the right host across arches (linux x86_64 -> fw13, aarch64 -> | |
| 121 | 142 | /// astra) without hard-coding a host name. | |
| 122 | 143 | pub build_host: String, | |
| 144 | + | /// The build host's SSH destination (topology `ssh`), as opposed to its | |
| 145 | + | /// name. Deploy compares it against the service host's to tell "build and | |
| 146 | + | /// run on the same box" from "two hosts that need a transfer" — a question | |
| 147 | + | /// the host NAMES cannot answer, since a build host and a deploy | |
| 148 | + | /// destination are declared in different files and need not agree on one. | |
| 149 | + | pub build_host_ssh: String, | |
| 123 | 150 | /// The app's checkout path (topology `repo`, `~`-prefixed). Recipes read it | |
| 124 | 151 | /// via `repo()` to `cd` into the checkout — commands don't auto-cd, and each | |
| 125 | 152 | /// `sh` is a fresh shell. | |
| @@ -139,6 +166,15 @@ | |||
| 139 | 166 | /// back. Never the agent, even for an agent host — see `state::build_sync`. | |
| 140 | 167 | /// Not an execution path. | |
| 141 | 168 | pub syncs: Arc<ExecutorMap>, | |
| 169 | + | /// Where this target installs, for a `kind = "service"` app. `None` for an | |
| 170 | + | /// app or a library, which makes every deploy host function fail with that | |
| 171 | + | /// as the reason rather than with a missing-host error. | |
| 172 | + | /// | |
| 173 | + | /// The runner resolves it from the app manifest's `[[deploy]]` table and | |
| 174 | + | /// registers its executor into `execs` under the destination's host string, | |
| 175 | + | /// so `sh_ok(deploy_host(), ...)` reaches the service host through the same | |
| 176 | + | /// capability gate as everything else. | |
| 177 | + | pub deploy: Option<DeployTarget>, | |
| 142 | 178 | pub pool: SqlitePool, | |
| 143 | 179 | pub events: EventTx, | |
| 144 | 180 | pub cfg: Arc<Config>, | |
| @@ -209,12 +245,14 @@ | |||
| 209 | 245 | version: Version, | |
| 210 | 246 | target: Target, | |
| 211 | 247 | build_host: String, | |
| 248 | + | build_host_ssh: String, | |
| 212 | 249 | repo: String, | |
| 213 | 250 | features: Vec<String>, | |
| 214 | 251 | kind: Kind, | |
| 215 | 252 | target_run_id: i64, | |
| 216 | 253 | execs: Arc<ExecutorMap>, | |
| 217 | 254 | syncs: Arc<ExecutorMap>, | |
| 255 | + | deploy: Option<DeployTarget>, | |
| 218 | 256 | pool: SqlitePool, | |
| 219 | 257 | events: EventTx, | |
| 220 | 258 | cfg: Arc<Config>, | |
| @@ -228,12 +266,14 @@ | |||
| 228 | 266 | version, | |
| 229 | 267 | target, | |
| 230 | 268 | build_host, | |
| 269 | + | build_host_ssh, | |
| 231 | 270 | repo, | |
| 232 | 271 | features, | |
| 233 | 272 | kind, | |
| 234 | 273 | target_run_id, | |
| 235 | 274 | execs, | |
| 236 | 275 | syncs, | |
| 276 | + | deploy, | |
| 237 | 277 | pool, | |
| 238 | 278 | events, | |
| 239 | 279 | cfg, | |
| @@ -492,10 +532,24 @@ | |||
| 492 | 532 | /// sign steps ride the in-session `AgentRpc` transport automatically. Returns | |
| 493 | 533 | /// exit code + a tail of stdout for the recipe to branch on. | |
| 494 | 534 | fn run(self: &Arc<Self>, host: &str, cmd: &str) -> Result<(i32, String)> { | |
| 535 | + | // The service host is addressed as a service host whatever step is open. | |
| 536 | + | // Deriving the action from the step is right for a build host, where the | |
| 537 | + | // step IS the work; on a service host it would ask for `build` during a | |
| 538 | + | // `verify` and be denied for a reason unrelated to what was attempted. | |
| 539 | + | let action = match &self.deploy { | |
| 540 | + | Some(d) if d.host == host => Action::Deploy, | |
| 541 | + | _ => action_for(self.current_step(), self.kind), | |
| 542 | + | }; | |
| 543 | + | self.run_as(host, cmd, action) | |
| 544 | + | } | |
| 545 | + | ||
| 546 | + | /// `run`, with the [`Action`] stated rather than resolved. Used where the | |
| 547 | + | /// caller already knows which plane it is on. | |
| 548 | + | fn run_as(self: &Arc<Self>, host: &str, cmd: &str, action: Action) -> Result<(i32, String)> { | |
| 495 | 549 | let sink = self.ensure_step()?; | |
| 496 | 550 | let exec = self.exec(host)?; | |
| 497 | 551 | let cur = self.current_step(); | |
| 498 | - | let step = OpStep::shell(action_for(cur, self.kind), cmd.to_string()); | |
| 552 | + | let step = OpStep::shell(action, cmd.to_string()); | |
| 499 | 553 | // Bounded by the step's deadline and interruptible on supersession, so a | |
| 500 | 554 | // hung command fails its step instead of running unbounded, and a | |
| 501 | 555 | // superseded build stops mid-step rather than only at the next boundary. | |
| @@ -1349,6 +1403,95 @@ | |||
| 1349 | 1403 | ); | |
| 1350 | 1404 | } | |
| 1351 | 1405 | ||
| 1406 | + | // --- deploy(binary) -> summary: install a service binary and restart its | |
| 1407 | + | // unit. The terminal step for `kind = "service"`, the counterpart of | |
| 1408 | + | // `publish` for something that is run rather than distributed. | |
| 1409 | + | // | |
| 1410 | + | // Takes only the binary's path on the build host: where it lands, on | |
| 1411 | + | // which machine, and which unit restarts all come from the `[[deploy]]` | |
| 1412 | + | // entry for the target already being built. A recipe cannot deploy the | |
| 1413 | + | // aarch64 binary to the x86_64 box by naming the wrong host, because it | |
| 1414 | + | // never names a host at all. | |
| 1415 | + | { | |
| 1416 | + | let ctx = ctx.clone(); | |
| 1417 | + | engine.register_fn( | |
| 1418 | + | "deploy", | |
| 1419 | + | move |binary: &str| -> Result<String, Box<EvalAltResult>> { | |
| 1420 | + | ctx.deploy(binary).map_err(rhai_err) | |
| 1421 | + | }, | |
| 1422 | + | ); | |
| 1423 | + | } | |
| 1424 | + | ||
| 1425 | + | // --- deploy_host() -> string: the service host's ssh destination, so a | |
| 1426 | + | // recipe can run its own assertions there (`sh_ok(deploy_host(), ...)`). | |
| 1427 | + | // Commands run through it while the `deploy` step is open, so they are | |
| 1428 | + | // gated on the deploy grant like the install itself. --- | |
| 1429 | + | { | |
| 1430 | + | let ctx = ctx.clone(); | |
| 1431 | + | engine.register_fn( | |
| 1432 | + | "deploy_host", | |
| 1433 | + | move || -> Result<String, Box<EvalAltResult>> { | |
| 1434 | + | ctx.deploy_target() | |
| 1435 | + | .map(|d| d.host.clone()) | |
| 1436 | + | .map_err(rhai_err) | |
| 1437 | + | }, | |
| 1438 | + | ); | |
| 1439 | + | } | |
| 1440 | + | ||
| 1441 | + | // --- service_name() / install_path() / health_url(): the rest of the | |
| 1442 | + | // `[[deploy]]` entry, so a recipe asserts against the configured values | |
| 1443 | + | // rather than repeating them as literals that can drift. `health_url` | |
| 1444 | + | // is "" when unset. --- | |
| 1445 | + | { | |
| 1446 | + | let ctx = ctx.clone(); | |
| 1447 | + | engine.register_fn( | |
| 1448 | + | "service_name", | |
| 1449 | + | move || -> Result<String, Box<EvalAltResult>> { | |
| 1450 | + | ctx.deploy_target() | |
| 1451 | + | .map(|d| d.service.clone()) | |
| 1452 | + | .map_err(rhai_err) | |
| 1453 | + | }, | |
| 1454 | + | ); | |
| 1455 | + | } | |
| 1456 | + | { | |
| 1457 | + | let ctx = ctx.clone(); | |
| 1458 | + | engine.register_fn( | |
| 1459 | + | "install_path", | |
| 1460 | + | move || -> Result<String, Box<EvalAltResult>> { | |
| 1461 | + | ctx.deploy_target() | |
| 1462 | + | .map(|d| d.install_path.clone()) | |
| 1463 | + | .map_err(rhai_err) | |
| 1464 | + | }, | |
| 1465 | + | ); | |
| 1466 | + | } | |
| 1467 | + | { | |
| 1468 | + | let ctx = ctx.clone(); | |
| 1469 | + | engine.register_fn( | |
| 1470 | + | "health_url", | |
| 1471 | + | move || -> Result<String, Box<EvalAltResult>> { | |
| 1472 | + | ctx.deploy_target() | |
| 1473 | + | .map(|d| d.health_url.clone().unwrap_or_default()) | |
| 1474 | + | .map_err(rhai_err) | |
| 1475 | + | }, | |
| 1476 | + | ); | |
| 1477 | + | } | |
| 1478 | + | ||
| 1479 | + | // --- glibc_check(binary) -> string: assert the build host did not produce | |
| 1480 | + | // a binary the service host's glibc is too old to exec. Aborts the run | |
| 1481 | + | // if it did; returns "needs X, host has Y" for the log if it did not. --- | |
| 1482 | + | { | |
| 1483 | + | let ctx = ctx.clone(); | |
| 1484 | + | engine.register_fn( | |
| 1485 | + | "glibc_check", | |
| 1486 | + | move |binary: &str| -> Result<String, Box<EvalAltResult>> { | |
| 1487 | + | let (needs, has) = ctx.glibc_check(binary).map_err(rhai_err)?; | |
| 1488 | + | Ok(format!( | |
| 1489 | + | "glibc: binary needs {needs}, service host has {has}" | |
| 1490 | + | )) | |
| 1491 | + | }, | |
| 1492 | + | ); | |
| 1493 | + | } | |
| 1494 | + | ||
| 1352 | 1495 | // --- macOS signing helpers. They dispatch through the named host's | |
| 1353 | 1496 | // executor like any other step; when that host is the mac (transport = | |
| 1354 | 1497 | // "agent"), codesign/notarize/staple ride the in-session `AgentRpc` | |
| @@ -1360,7 +1503,221 @@ | |||
| 1360 | 1503 | engine | |
| 1361 | 1504 | } | |
| 1362 | 1505 | ||
| 1506 | + | /// Highest `GLIBC_x.y` version referenced by a built binary, and the glibc a | |
| 1507 | + | /// host actually has, both parsed from the text the commands print. | |
| 1508 | + | /// | |
| 1509 | + | /// Native-per-architecture builds remove the cross-compile hazard `deploy.sh` | |
| 1510 | + | /// was written against, but not this one: fw13 tracks a newer glibc than the | |
| 1511 | + | /// Ubuntu 24.04 box in Hetzner, so a binary built here can reference a symbol | |
| 1512 | + | /// version that box does not have and fail at exec — after the unit has already | |
| 1513 | + | /// been restarted onto it. Comparing the two before the install is what makes | |
| 1514 | + | /// that a failed step instead of a downed service. | |
| 1515 | + | fn max_glibc_symbol(objdump_out: &str) -> Option<(u64, u64)> { | |
| 1516 | + | objdump_out | |
| 1517 | + | .split(|c: char| !(c.is_ascii_digit() || c == '.' || c == '_' || c.is_ascii_alphabetic())) | |
| 1518 | + | .filter_map(|tok| tok.strip_prefix("GLIBC_")) | |
| 1519 | + | .filter_map(parse_glibc_version) | |
| 1520 | + | .max() | |
| 1521 | + | } | |
| 1522 | + | ||
| 1523 | + | /// Parse `2.39` (or `2.39.1`, keeping major/minor) into a comparable pair. | |
| 1524 | + | fn parse_glibc_version(s: &str) -> Option<(u64, u64)> { | |
| 1525 | + | let mut parts = s.split('.'); | |
| 1526 | + | let major = parts.next()?.parse().ok()?; | |
| 1527 | + | let minor = parts.next()?.parse().ok()?; | |
| 1528 | + | Some((major, minor)) | |
| 1529 | + | } | |
| 1530 | + | ||
| 1531 | + | /// The glibc version out of `ldd --version`'s first line, whose tail is the | |
| 1532 | + | /// version however the distro decorates the rest (`ldd (Ubuntu GLIBC | |
| 1533 | + | /// 2.39-0ubuntu8.8) 2.39`). | |
| 1534 | + | fn glibc_from_ldd(ldd_out: &str) -> Option<(u64, u64)> { | |
| 1535 | + | let first = ldd_out.lines().find(|l| !l.trim().is_empty())?; | |
| 1536 | + | parse_glibc_version(first.split_whitespace().last()?) | |
| 1537 | + | } | |
| 1538 | + | ||
| 1363 | 1539 | impl RecipeCtx { | |
| 1540 | + | /// This target's install destination, or an error naming why there is none. | |
| 1541 | + | fn deploy_target(&self) -> Result<&DeployTarget> { | |
| 1542 | + | self.deploy.as_ref().ok_or_else(|| { | |
| 1543 | + | anyhow::anyhow!( | |
| 1544 | + | "no deploy destination for {} {}: the app is `kind = \"{}\"`, and only a \ | |
| 1545 | + | service declares [[deploy]] entries", | |
| 1546 | + | self.app, | |
| 1547 | + | self.target, | |
| 1548 | + | match self.kind { | |
| 1549 | + | Kind::App => "app", | |
| 1550 | + | Kind::Library => "library", | |
| 1551 | + | Kind::Service => "service", | |
| 1552 | + | } | |
| 1553 | + | ) | |
| 1554 | + | }) | |
| 1555 | + | } | |
| 1556 | + | ||
| 1557 | + | /// Compare the built binary's glibc requirement against the service host's. | |
| 1558 | + | /// Returns the two versions for the recipe to log. | |
| 1559 | + | fn glibc_check(self: &Arc<Self>, binary: &str) -> Result<(String, String)> { | |
| 1560 | + | let d = self.deploy_target()?.clone(); | |
| 1561 | + | // `objdump -T` on the build host; no symbols at all (a static binary) | |
| 1562 | + | // means nothing to check, which is a pass rather than a failure. | |
| 1563 | + | let (code, out) = self.run( | |
| 1564 | + | &self.build_host.clone(), | |
| 1565 | + | &format!( | |
| 1566 | + | "objdump -T {binary} 2>/dev/null | grep -o 'GLIBC_[0-9.]*' | sort -uV || true" | |
| 1567 | + | ), | |
| 1568 | + | )?; | |
| 1569 | + | anyhow::ensure!(code == 0, "reading glibc symbols from {binary} failed"); | |
| 1570 | + | let Some(needs) = max_glibc_symbol(&out) else { | |
| 1571 | + | return Ok(("none".into(), "n/a".into())); | |
| 1572 | + | }; | |
| 1573 | + | let (code, ldd) = self.run(&d.host, "ldd --version")?; | |
| 1574 | + | anyhow::ensure!( | |
| 1575 | + | code == 0, | |
| 1576 | + | "could not read glibc version on service host `{}`", | |
| 1577 | + | d.host | |
| 1578 | + | ); | |
| 1579 | + | let has = glibc_from_ldd(&ldd).ok_or_else(|| { | |
| 1580 | + | anyhow::anyhow!( | |
| 1581 | + | "could not parse glibc version from `ldd --version` on `{}`", | |
| 1582 | + | d.host | |
| 1583 | + | ) | |
| 1584 | + | })?; | |
| 1585 | + | anyhow::ensure!( | |
| 1586 | + | needs <= has, | |
| 1587 | + | "binary needs glibc {}.{} but `{}` has {}.{} — it would fail to exec after the \ | |
| 1588 | + | unit restarted onto it. Build on a host no newer than the service host.", | |
| 1589 | + | needs.0, | |
| 1590 | + | needs.1, | |
| 1591 | + | d.host, | |
| 1592 | + | has.0, | |
| 1593 | + | has.1, | |
| 1594 | + | ); | |
| 1595 | + | Ok(( | |
| 1596 | + | format!("{}.{}", needs.0, needs.1), | |
| 1597 | + | format!("{}.{}", has.0, has.1), | |
| 1598 | + | )) | |
| 1599 | + | } | |
| 1600 | + | ||
| 1601 | + | /// Install `binary` (a path on the BUILD host) onto the service host and | |
| 1602 | + | /// restart its unit, via the privileged installer the host holds a scoped | |
| 1603 | + | /// sudo grant for. | |
| 1604 | + | /// | |
| 1605 | + | /// Bento never runs the install itself. It stages the bytes and calls a | |
| 1606 | + | /// root script whose arguments are re-checked on the far side — the same | |
| 1607 | + | /// shape as Sando's `install-companion.sh`, and for the same reason: the | |
| 1608 | + | /// sudoers grant is then ONE auditable script rather than a broad | |
| 1609 | + | /// `install`+`systemctl` grant on a production box. | |
| 1610 | + | /// | |
| 1611 | + | /// Only the binary moves. Config is deliberately untouched: pom's | |
| 1612 | + | /// `pom-astra.toml` / `pom-hetzner.toml` differ per instance, and prod's | |
| 1613 | + | /// carried a `[targets.mnw.tests]` block the repo did not have. A deploy | |
| 1614 | + | /// that copies config over is how that block gets silently deleted. | |
| 1615 | + | fn deploy(self: &Arc<Self>, binary: &str) -> Result<String> { | |
| 1616 | + | anyhow::ensure!( | |
| 1617 | + | !self.is_cancelled(), | |
| 1618 | + | "build superseded by a newer request; refusing to deploy" | |
| 1619 | + | ); | |
| 1620 | + | // A failed earlier step bars a deploy exactly as it bars a publish. An | |
| 1621 | + | // artifact that failed its gates must not reach a production host just | |
| 1622 | + | // because the recipe kept running. | |
| 1623 | + | let failed = self.failed_steps.lock().unwrap().clone(); | |
| 1624 | + | anyhow::ensure!( | |
| 1625 | + | failed.is_empty(), | |
| 1626 | + | "refusing to deploy {} {}: {} failed earlier in this run", | |
| 1627 | + | self.app, | |
| 1628 | + | self.version, | |
| 1629 | + | failed | |
| 1630 | + | .iter() | |
| 1631 | + | .map(ToString::to_string) | |
| 1632 | + | .collect::<Vec<_>>() | |
| 1633 | + | .join(", "), | |
| 1634 | + | ); | |
| 1635 | + | let d = self.deploy_target()?.clone(); | |
| 1636 | + | ensure_glob_safe(binary)?; | |
| 1637 | + | ||
| 1638 | + | // Stage under a fixed root the installer also insists on, so "what was | |
| 1639 | + | // checked" and "what is installed" cannot drift apart. | |
| 1640 | + | let staged = format!("{DEPLOY_STAGING_ROOT}/{}", self.app); | |
| 1641 | + | let staged_bin = format!("{staged}/{}", self.app); | |
| 1642 | + | let deploy_exec = self.exec(&d.host)?; | |
| 1643 | + | anyhow::ensure!( | |
| 1644 | + | deploy_exec.capabilities().permits(&Action::Deploy), | |
| 1645 | + | "service host `{}` is not granted the `deploy` capability", | |
| 1646 | + | d.host | |
| 1647 | + | ); | |
| 1648 | + | ||
| 1649 | + | self.run_ok(&d.host, &format!("mkdir -p {staged}"))?; | |
| 1650 | + | if self.build_host_ssh == d.host { | |
| 1651 | + | // Same box: the binary is already there. Routing it through the | |
| 1652 | + | // daemon would be two transfers to end up where it started. This is | |
| 1653 | + | // pom's aarch64 leg — astra builds it and astra runs it. | |
| 1654 | + | self.run_ok(&d.host, &format!("cp -f {binary} {staged_bin}"))?; | |
| 1655 | + | } else { | |
| 1656 | + | // Build host -> daemon -> service host. Two hops because an executor | |
| 1657 | + | // reaches one host; a direct host-to-host transport would mean the | |
| 1658 | + | // build host holding a credential for the production box. | |
| 1659 | + | let tmp = tempfile::tempdir().context("staging dir for deploy")?; | |
| 1660 | + | let local = tmp.path().join(self.app.as_str()); | |
| 1661 | + | self.pull_for_deploy(binary, &local)?; | |
| 1662 | + | let (dest, opts) = (PathBuf::from(&staged), SyncOpts::default()); | |
| 1663 | + | let dir = tmp.path().to_path_buf(); | |
| 1664 | + | self.run_bounded(&format!("stage {} on `{}`", self.app, d.host), async move { | |
| 1665 | + | deploy_exec.push_dir(&dir, &dest, &opts).await | |
| 1666 | + | }) | |
| 1667 | + | .with_context(|| format!("staging {} onto `{}`", self.app, d.host))?; | |
| 1668 | + | } | |
| 1669 | + | ||
| 1670 | + | // The privileged half. Every argument is re-validated by the script, | |
| 1671 | + | // which is the thing actually holding the sudo grant. | |
| 1672 | + | self.run_ok( | |
| 1673 | + | &d.host, | |
| 1674 | + | &format!( | |
| 1675 | + | "{} {staged_bin} {} {}", | |
| 1676 | + | self.cfg.deploy_installer, d.install_path, d.service | |
| 1677 | + | ), | |
| 1678 | + | )?; | |
| 1679 | + | Ok(format!( | |
| 1680 | + | "{} {} installed at {} on `{}`; {} restarted", | |
| 1681 | + | self.app, self.version, d.install_path, d.host, d.service | |
| 1682 | + | )) | |
| 1683 | + | } | |
| 1684 | + | ||
| 1685 | + | /// Fetch one file off a host into a daemon-local path for re-pushing. | |
| 1686 | + | /// | |
| 1687 | + | /// A local build host is read directly: `fw13` is the daemon's own box, so | |
| 1688 | + | /// the file is already on this filesystem. Routing it through the | |
| 1689 | + | /// artifact-pull gate instead would demand a `pull_root` covering every repo | |
| 1690 | + | /// a service could be built in — today that is `~/Code/Apps`, and pom lives | |
| 1691 | + | /// in `~/Code/MNW`. Widening it to `~/Code` would put `_private`, the | |
| 1692 | + | /// secrets root, inside the collectable tree. This is pom's x86_64 leg. | |
| 1693 | + | fn pull_for_deploy(self: &Arc<Self>, remote: &str, local: &Path) -> Result<()> { | |
| 1694 | + | let host = self.build_host.clone(); | |
| 1695 | + | let remote_path = expand_tilde(remote); | |
| 1696 | + | if self.build_host_ssh == "local" || self.build_host_ssh.is_empty() { | |
| 1697 | + | std::fs::copy(&remote_path, local).with_context(|| { | |
| 1698 | + | format!("staging {} from the daemon host", remote_path.display()) | |
| 1699 | + | })?; | |
| 1700 | + | return Ok(()); | |
| 1701 | + | } | |
| 1702 | + | let sync = self.host_sync(&host)?; | |
| 1703 | + | let (src, dst, opts) = (remote_path, local.to_path_buf(), SyncOpts::default()); | |
| 1704 | + | self.run_bounded(&format!("fetch {remote} from `{host}`"), async move { | |
| 1705 | + | sync.pull_file(&src, &dst, &opts).await | |
| 1706 | + | }) | |
| 1707 | + | .with_context(|| format!("fetching {remote} from `{host}` to deploy")) | |
| 1708 | + | } | |
| 1709 | + | ||
| 1710 | + | /// `run`, failing the step on a non-zero exit. The Rust-side twin of the | |
| 1711 | + | /// recipe's `sh_ok`, for commands the deploy machinery issues itself. | |
| 1712 | + | fn run_ok(self: &Arc<Self>, host: &str, cmd: &str) -> Result<String> { | |
| 1713 | + | let (code, tail) = self.run(host, cmd)?; | |
| 1714 | + | if code != 0 { | |
| 1715 | + | self.fail_current_step(); | |
| 1716 | + | anyhow::bail!("command on `{host}` exited {code}: {cmd}\n{tail}"); | |
| 1717 | + | } | |
| 1718 | + | Ok(tail) | |
| 1719 | + | } | |
| 1720 | + | ||
| 1364 | 1721 | fn collect(self: &Arc<Self>, host: &str, glob: &str, app: &str, version: &str) -> Result<()> { | |
| 1365 | 1722 | let dest = self.cfg.dist_root.join(app).join(version); | |
| 1366 | 1723 | let dest_s = dest.to_string_lossy().into_owned(); | |
| @@ -1832,12 +2189,14 @@ | |||
| 1832 | 2189 | Version::parse("0.1.0").unwrap(), | |
| 1833 | 2190 | "linux/x86_64".parse().unwrap(), | |
| 1834 | 2191 | "fw13".into(), | |
| 2192 | + | "local".into(), | |
| 1835 | 2193 | "/tmp".into(), | |
| 1836 | 2194 | vec![], | |
| 1837 | 2195 | Kind::App, | |
| 1838 | 2196 | 1, | |
| 1839 | 2197 | Arc::new(std::collections::HashMap::new()), | |
| 1840 | 2198 | Arc::new(std::collections::HashMap::new()), | |
| 2199 | + | None, | |
| 1841 | 2200 | pool, | |
| 1842 | 2201 | crate::events::channel(), | |
| 1843 | 2202 | cfg, | |
| @@ -1867,12 +2226,14 @@ | |||
| 1867 | 2226 | Version::parse("0.1.0").unwrap(), | |
| 1868 | 2227 | "linux/x86_64".parse().unwrap(), | |
| 1869 | 2228 | "fw13".into(), | |
| 2229 | + | "local".into(), | |
| 1870 | 2230 | "/tmp".into(), | |
| 1871 | 2231 | features, | |
| 1872 | 2232 | Kind::App, | |
| 1873 | 2233 | 1, | |
| 1874 | 2234 | Arc::new(std::collections::HashMap::new()), | |
| 1875 | 2235 | Arc::new(std::collections::HashMap::new()), | |
| 2236 | + | None, | |
| 1876 | 2237 | pool, | |
| 1877 | 2238 | crate::events::channel(), | |
| 1878 | 2239 | cfg, | |
| @@ -1919,12 +2280,14 @@ | |||
| 1919 | 2280 | Version::parse("0.1.0").unwrap(), | |
| 1920 | 2281 | "linux/x86_64".parse().unwrap(), | |
| 1921 | 2282 | "fw13".into(), | |
| 2283 | + | "local".into(), | |
| 1922 | 2284 | "/tmp".into(), | |
| 1923 | 2285 | vec![], | |
| 1924 | 2286 | Kind::App, | |
| 1925 | 2287 | 1, | |
| 1926 | 2288 | Arc::new(std::collections::HashMap::new()), | |
| 1927 | 2289 | Arc::new(std::collections::HashMap::new()), | |
| 2290 | + | None, | |
| 1928 | 2291 | pool, | |
| 1929 | 2292 | crate::events::channel(), | |
| 1930 | 2293 | cfg, | |
| @@ -2367,6 +2730,253 @@ | |||
| 2367 | 2730 | assert_artifact_version("latest.json", &ver("0.5.0")).unwrap(); | |
| 2368 | 2731 | } | |
| 2369 | 2732 | ||
| 2733 | + | /// The comparison that decides whether a binary can exec on the box that is | |
| 2734 | + | /// about to be restarted onto it. Both sides are parsed out of text a tool | |
| 2735 | + | /// printed, so both parsers are worth pinning: fw13 tracks a newer glibc | |
| 2736 | + | /// than the Ubuntu 24.04 host in Hetzner, and getting this backwards means a |
Lines truncated
| @@ -439,6 +439,11 @@ | |||
| 439 | 439 | .host_for(target) | |
| 440 | 440 | .map(|h| h.name.clone()) | |
| 441 | 441 | .unwrap_or_default(); | |
| 442 | + | let build_host_ssh = state | |
| 443 | + | .topo | |
| 444 | + | .host_for(target) | |
| 445 | + | .map(|h| h.ssh.clone()) | |
| 446 | + | .unwrap_or_default(); | |
| 442 | 447 | let repo = state | |
| 443 | 448 | .topo | |
| 444 | 449 | .app(&app) | |
| @@ -457,19 +462,42 @@ | |||
| 457 | 462 | .app(&app) | |
| 458 | 463 | .and_then(|a| a.require_all_targets.then(|| a.targets.clone())); | |
| 459 | 464 | ||
| 465 | + | // A service's install destination for THIS target, plus an executor for it. | |
| 466 | + | // | |
| 467 | + | // The executor is added to a per-run copy of the map rather than to the | |
| 468 | + | // daemon-wide one, so a deploy grant on a production host exists only for | |
| 469 | + | // the duration of the run that needs it and only for the app that declared | |
| 470 | + | // it. Nothing else can address that host: it is not in the topology, so no | |
| 471 | + | // other app's recipe can name it. | |
| 472 | + | let deploy = state | |
| 473 | + | .topo | |
| 474 | + | .app(&app) | |
| 475 | + | .and_then(|a| a.deploy_for(target)) | |
| 476 | + | .cloned(); | |
| 477 | + | let execs = match &deploy { | |
| 478 | + | Some(d) => { | |
| 479 | + | let mut map = (*state.executors).clone(); | |
| 480 | + | map.insert(d.host.clone(), crate::state::build_deploy_executor(d)); | |
| 481 | + | Arc::new(map) | |
| 482 | + | } | |
| 483 | + | None => state.executors.clone(), | |
| 484 | + | }; | |
| 485 | + | ||
| 460 | 486 | let ctx = Arc::new(RecipeCtx::new( | |
| 461 | 487 | app.clone(), | |
| 462 | 488 | version.clone(), | |
| 463 | 489 | target, | |
| 464 | 490 | build_host, | |
| 491 | + | build_host_ssh, | |
| 465 | 492 | repo, | |
| 466 | 493 | features, | |
| 467 | 494 | // Gates the `verify` step's capability: a library's crate preflight is | |
| 468 | 495 | // not an app's Gatekeeper check. See engine::action_for. | |
| 469 | 496 | state.topo.app(&app).map(|a| a.kind).unwrap_or_default(), | |
| 470 | 497 | target_run_id, | |
| 471 | - | state.executors.clone(), | |
| 498 | + | execs, | |
| 472 | 499 | state.syncs.clone(), | |
| 500 | + | deploy, | |
| 473 | 501 | state.pool.clone(), | |
| 474 | 502 | state.events.clone(), | |
| 475 | 503 | state.cfg.clone(), | |
| @@ -739,7 +767,12 @@ | |||
| 739 | 767 | .ok_or_else(|| anyhow::anyhow!("unknown app `{app}`"))?; | |
| 740 | 768 | let file = match cfg.kind { | |
| 741 | 769 | crate::topology::Kind::Library => "publish.rhai".to_string(), | |
| 742 | - | crate::topology::Kind::App => format!("{}.rhai", target.platform.as_str()), | |
| 770 | + | // A service is built per target like an app — the binary genuinely | |
| 771 | + | // differs per platform, and so does where it lands — so it takes the | |
| 772 | + | // same per-platform recipe naming rather than a single deploy.rhai. | |
| 773 | + | crate::topology::Kind::App | crate::topology::Kind::Service => { | |
| 774 | + | format!("{}.rhai", target.platform.as_str()) | |
| 775 | + | } | |
| 743 | 776 | }; | |
| 744 | 777 | let path: PathBuf = engine::expand_tilde(&cfg.repo) | |
| 745 | 778 | .join(&cfg.recipe_dir) | |
| @@ -1004,6 +1037,142 @@ | |||
| 1004 | 1037 | } | |
| 1005 | 1038 | } | |
| 1006 | 1039 | ||
| 1040 | + | /// A `kind = "service"` release end to end: build, `glibc_check`, `deploy`, | |
| 1041 | + | /// and a health assertion the recipe makes itself against the service host. | |
| 1042 | + | /// | |
| 1043 | + | /// The whole point of the deploy step is that it dispatches to a host | |
| 1044 | + | /// OUTSIDE the build topology, so this exercises the resolution | |
| 1045 | + | /// (target -> `[[deploy]]` entry -> executor registered for the run), the | |
| 1046 | + | /// staging, and the call into the privileged installer — with a fake | |
| 1047 | + | /// installer standing in for the root script, which is the one part a test | |
| 1048 | + | /// cannot run for real. | |
| 1049 | + | #[tokio::test(flavor = "multi_thread", worker_threads = 2)] | |
| 1050 | + | async fn service_recipe_builds_then_deploys_and_verifies() { | |
| 1051 | + | let tmp = tempfile::tempdir().unwrap(); | |
| 1052 | + | let root = tmp.path(); | |
| 1053 | + | let repo = root.join("svc"); | |
| 1054 | + | std::fs::create_dir_all(repo.join("dist/recipes")).unwrap(); | |
| 1055 | + | std::fs::write(repo.join("Cargo.toml"), "[package]\nversion = \"0.4.0\"\n").unwrap(); | |
| 1056 | + | ||
| 1057 | + | // Stand-in for the root installer: same three arguments, records what it | |
| 1058 | + | // was asked to do instead of writing to /usr/local/bin and restarting a | |
| 1059 | + | // unit. `install` + a marker file, so the test can assert the binary | |
| 1060 | + | // that arrived is the binary that was built. | |
| 1061 | + | let installer = root.join("install-service.sh"); | |
| 1062 | + | std::fs::write( | |
| 1063 | + | &installer, | |
| 1064 | + | "#!/bin/sh\nset -eu\ninstall -m 0755 \"$1\" \"$2\"\necho \"restarted $3\" >> \"$2.log\"\n", | |
| 1065 | + | ) | |
| 1066 | + | .unwrap(); | |
| 1067 | + | std::fs::set_permissions( | |
| 1068 | + | &installer, | |
| 1069 | + | <std::fs::Permissions as std::os::unix::fs::PermissionsExt>::from_mode(0o755), | |
| 1070 | + | ) | |
| 1071 | + | .unwrap(); | |
| 1072 | + | let install_path = root.join("bin/svc"); | |
| 1073 | + | std::fs::create_dir_all(root.join("bin")).unwrap(); | |
| 1074 | + | ||
| 1075 | + | std::fs::write( | |
| 1076 | + | repo.join("dist/recipes/linux.rhai"), | |
| 1077 | + | r#" | |
| 1078 | + | let v = version(); | |
| 1079 | + | step("build"); | |
| 1080 | + | sh_ok("fw13", "mkdir -p REPO/target/release && echo built-BIN > REPO/target/release/svc"); | |
| 1081 | + | step("verify"); | |
| 1082 | + | log(glibc_check("REPO/target/release/svc")); | |
| 1083 | + | step("deploy"); | |
| 1084 | + | log(deploy("REPO/target/release/svc")); | |
| 1085 | + | // The recipe owns what "healthy" means, and asserts it itself | |
| 1086 | + | // against the host it just restarted. | |
| 1087 | + | sh_ok(deploy_host(), "test -x " + install_path()); | |
| 1088 | + | "# | |
| 1089 | + | .replace("REPO", repo.to_str().unwrap()) | |
| 1090 | + | .replace("BIN", "0.4.0"), | |
| 1091 | + | ) | |
| 1092 | + | .unwrap(); | |
| 1093 | + | ||
| 1094 | + | std::fs::write( | |
| 1095 | + | repo.join("bento.toml"), | |
| 1096 | + | format!( | |
| 1097 | + | r#"kind = "service" | |
| 1098 | + | targets = ["linux/x86_64"] | |
| 1099 | + | version_path = "Cargo.toml" | |
| 1100 | + | ||
| 1101 | + | [[deploy]] | |
| 1102 | + | target = "linux/x86_64" | |
| 1103 | + | host = "local" | |
| 1104 | + | install_path = "{}" | |
| 1105 | + | service = "svc.service" | |
| 1106 | + | health_url = "http://localhost:9100/api/health" | |
| 1107 | + | "#, | |
| 1108 | + | install_path.display() | |
| 1109 | + | ), | |
| 1110 | + | ) | |
| 1111 | + | .unwrap(); | |
| 1112 | + | ||
| 1113 | + | let mut cfg = Config::for_tests(root); | |
| 1114 | + | cfg.deploy_installer = installer.display().to_string(); | |
| 1115 | + | let pool = crate::db::open(&cfg.db_path).await.unwrap(); | |
| 1116 | + | let topo = Topology::from_str_for_tests(&format!( | |
| 1117 | + | "[[host]]\nname = \"fw13\"\nssh = \"local\"\ntargets = [\"linux/x86_64\"]\n\ | |
| 1118 | + | pull_root = \"{repo}\"\n\n[app.svc]\nrepo = \"{repo}\"\n", | |
| 1119 | + | repo = repo.display() | |
| 1120 | + | )) | |
| 1121 | + | .unwrap(); | |
| 1122 | + | let state = test_state(pool.clone(), topo, cfg); | |
| 1123 | + | ||
| 1124 | + | let build_id = start_build( | |
| 1125 | + | state.clone(), | |
| 1126 | + | AppId::new("svc"), | |
| 1127 | + | Version::parse("0.4.0").unwrap(), | |
| 1128 | + | vec!["linux/x86_64".parse().unwrap()], | |
| 1129 | + | ) | |
| 1130 | + | .await | |
| 1131 | + | .unwrap(); | |
| 1132 | + | ||
| 1133 | + | let mut status = String::new(); | |
| 1134 | + | for _ in 0..100 { | |
| 1135 | + | status = sqlx::query_scalar("SELECT status FROM target_runs WHERE build_id = ?") | |
| 1136 | + | .bind(build_id) | |
| 1137 | + | .fetch_optional(&pool) | |
| 1138 | + | .await | |
| 1139 | + | .unwrap() | |
| 1140 | + | .unwrap_or_else(|| "running".to_string()); | |
| 1141 | + | if status != "running" { | |
| 1142 | + | break; | |
| 1143 | + | } | |
| 1144 | + | tokio::time::sleep(std::time::Duration::from_millis(50)).await; | |
| 1145 | + | } | |
| 1146 | + | assert_eq!(status, "ok", "service run should succeed"); | |
| 1147 | + | ||
| 1148 | + | let steps: Vec<(String, String)> = sqlx::query_as( | |
| 1149 | + | "SELECT step, status FROM step_runs WHERE target_run_id IN \ | |
| 1150 | + | (SELECT id FROM target_runs WHERE build_id = ?) ORDER BY id", | |
| 1151 | + | ) | |
| 1152 | + | .bind(build_id) | |
| 1153 | + | .fetch_all(&pool) | |
| 1154 | + | .await | |
| 1155 | + | .unwrap(); | |
| 1156 | + | assert_eq!( | |
| 1157 | + | steps.iter().map(|(s, _)| s.as_str()).collect::<Vec<_>>(), | |
| 1158 | + | vec!["build", "verify", "deploy"], | |
| 1159 | + | "a service ends at deploy, not collect" | |
| 1160 | + | ); | |
| 1161 | + | assert!(steps.iter().all(|(_, st)| st == "ok"), "{steps:?}"); | |
| 1162 | + | ||
| 1163 | + | // The bytes that were built are the bytes that landed, and the unit was | |
| 1164 | + | // restarted only after the install succeeded. | |
| 1165 | + | assert_eq!( | |
| 1166 | + | std::fs::read_to_string(&install_path).unwrap().trim(), | |
| 1167 | + | "built-0.4.0" | |
| 1168 | + | ); | |
| 1169 | + | assert!( | |
| 1170 | + | std::fs::read_to_string(install_path.with_extension("").with_file_name("svc.log")) | |
| 1171 | + | .unwrap() | |
| 1172 | + | .contains("restarted svc.service") | |
| 1173 | + | ); | |
| 1174 | + | } | |
| 1175 | + | ||
| 1007 | 1176 | /// Stand up a tmp app repo + topology and run a real local recipe end to | |
| 1008 | 1177 | /// end: step transitions, streamed `sh_ok`, `version_of`, `log`, and a | |
| 1009 | 1178 | /// `collect` that pulls a built artifact into dist_root. | |
| @@ -3007,7 +3176,7 @@ | |||
| 3007 | 3176 | let dir = crate::engine::expand_tilde(&cfg.repo).join(&cfg.recipe_dir); | |
| 3008 | 3177 | let files: Vec<String> = match cfg.kind { | |
| 3009 | 3178 | Kind::Library => vec!["publish.rhai".into()], | |
| 3010 | - | Kind::App => cfg | |
| 3179 | + | Kind::App | Kind::Service => cfg | |
| 3011 | 3180 | .targets | |
| 3012 | 3181 | .iter() | |
| 3013 | 3182 | .map(|t| format!("{}.rhai", t.platform.as_str())) |
| @@ -147,6 +147,26 @@ | |||
| 147 | 147 | } | |
| 148 | 148 | } | |
| 149 | 149 | ||
| 150 | + | /// Build the executor for a service's deploy destination. | |
| 151 | + | /// | |
| 152 | + | /// Deliberately NOT a build host, and never in the topology's host list: it is | |
| 153 | + | /// granted `deploy` + `restart` and nothing else, so the same executor that | |
| 154 | + | /// installs pom's binary cannot be handed a `build` step, and a compromised | |
| 155 | + | /// recipe cannot turn the production box into a build host. `Action::Deploy` and | |
| 156 | + | /// `Action::Restart` already exist in `ops_exec` for Sando's promotions; this is | |
| 157 | + | /// the same grant reaching the same kind of destination. | |
| 158 | + | /// | |
| 159 | + | /// It gets no `pull_root`, so the artifact-collection plane is closed on it in | |
| 160 | + | /// both directions: `collect()` from a deploy host is refused fail-closed, which | |
| 161 | + | /// is right — a service host produces nothing Bento should be fetching. | |
| 162 | + | pub fn build_deploy_executor(d: &crate::topology::DeployTarget) -> Arc<dyn Executor> { | |
| 163 | + | let caps = CapabilitySet::from_tokens(["deploy", "restart"], ["build-log"]); | |
| 164 | + | if d.host == "local" || d.host.is_empty() { | |
| 165 | + | return Arc::new(LocalExec::new(caps)); | |
| 166 | + | } | |
| 167 | + | Arc::new(SshExec::new(d.host.clone(), caps).with_port(d.port)) | |
| 168 | + | } | |
| 169 | + | ||
| 150 | 170 | /// Build the full host name -> exec-executor map from the topology. | |
| 151 | 171 | pub fn build_executors(topo: &Topology) -> ExecutorMap { | |
| 152 | 172 | topo.hosts |
| @@ -117,6 +117,7 @@ | |||
| 117 | 117 | value: match app.kind { | |
| 118 | 118 | Kind::App => "app".into(), | |
| 119 | 119 | Kind::Library => "library".into(), | |
| 120 | + | Kind::Service => "service".into(), | |
| 120 | 121 | }, | |
| 121 | 122 | }, | |
| 122 | 123 | )); |
| @@ -69,6 +69,53 @@ | |||
| 69 | 69 | /// uploads from one host — so it runs a single `publish.rhai` rather than a | |
| 70 | 70 | /// recipe per platform, and its `targets` name the host that does it. | |
| 71 | 71 | Library, | |
| 72 | + | /// A long-running binary Bento ships onto the hosts that run it, rather than | |
| 73 | + | /// distributing to users. Built per target like an app, but it ends at | |
| 74 | + | /// `deploy` instead of `collect`: no bundle, no signature for a human to | |
| 75 | + | /// check, no registry. Where each target lands is `[[deploy]]` in the app's own manifest. | |
| 76 | + | /// | |
| 77 | + | /// A separate kind rather than an app with an extra step, because the two | |
| 78 | + | /// differ in what `verify` means. An app's verify is Gatekeeper (is this | |
| 79 | + | /// bundle notarized); a service's is running the toolchain against the | |
| 80 | + | /// binary it just built — the same thing a library's crate preflight does. | |
| 81 | + | Service, | |
| 82 | + | } | |
| 83 | + | ||
| 84 | + | /// Where one target of a [`Kind::Service`] gets installed, from a `[[deploy]]` | |
| 85 | + | /// table in the app's own `bento.toml`. | |
| 86 | + | /// | |
| 87 | + | /// It lives in the repo's manifest rather than the daemon's file for the same | |
| 88 | + | /// reason `targets` does: which host runs the service is a fact about the | |
| 89 | + | /// service, and a change to it should arrive in the same commit as the change | |
| 90 | + | /// that needed it, not drift in a config on one machine. | |
| 91 | + | /// | |
| 92 | + | /// The binding is target -> host, so the recipe never names a machine: it calls | |
| 93 | + | /// `deploy()` and the target it is already running for decides where that goes. | |
| 94 | + | /// That is what keeps the aarch64 build from ever being installable on the | |
| 95 | + | /// x86_64 box, without the recipe having to be careful. | |
| 96 | + | #[derive(Debug, Clone, Deserialize)] | |
| 97 | + | pub struct DeployTarget { | |
| 98 | + | /// Which built target this entry installs. Must be one the app ships. | |
| 99 | + | pub target: Target, | |
| 100 | + | /// SSH destination of the host that runs the service: a tailnet alias | |
| 101 | + | /// (`astra`) or `user@host`. Reached over the same `SshExec` every other | |
| 102 | + | /// remote host uses, and `local` runs on the daemon's own box. | |
| 103 | + | pub host: String, | |
| 104 | + | /// Optional SSH port, when the service host is not on 22. | |
| 105 | + | #[serde(default)] | |
| 106 | + | pub port: Option<u16>, | |
| 107 | + | /// Absolute path of the unit's `ExecStart` binary on that host, e.g. | |
| 108 | + | /// `/usr/local/bin/pom`. The privileged installer is what actually writes | |
| 109 | + | /// here; this is the argument it is given. | |
| 110 | + | pub install_path: String, | |
| 111 | + | /// The systemd unit to restart once the binary is in place, e.g. | |
| 112 | + | /// `pom.service`. | |
| 113 | + | pub service: String, | |
| 114 | + | /// URL the recipe can assert answers 200 after the restart. Read by the | |
| 115 | + | /// recipe via `health_url()`; Bento does not poll it on the recipe's behalf, | |
| 116 | + | /// because what counts as healthy is the service's business. | |
| 117 | + | #[serde(default)] | |
| 118 | + | pub health_url: Option<String>, | |
| 72 | 119 | } | |
| 73 | 120 | ||
| 74 | 121 | /// `bento.toml` at the root of an app's repo: everything about how that app | |
| @@ -89,6 +136,9 @@ | |||
| 89 | 136 | #[serde(default)] | |
| 90 | 137 | require_all_targets: bool, | |
| 91 | 138 | targets: Vec<Target>, | |
| 139 | + | /// Service install destinations, one per target. Empty for any other kind. | |
| 140 | + | #[serde(default, rename = "deploy")] | |
| 141 | + | deploy: Vec<DeployTarget>, | |
| 92 | 142 | } | |
| 93 | 143 | ||
| 94 | 144 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)] | |
| @@ -192,6 +242,16 @@ | |||
| 192 | 242 | pub require_all_targets: bool, | |
| 193 | 243 | /// Targets this app ships. | |
| 194 | 244 | pub targets: Vec<Target>, | |
| 245 | + | /// Service install destinations, one per target (see [`DeployTarget`]). | |
| 246 | + | /// Empty unless `kind = "service"`. | |
| 247 | + | pub deploy: Vec<DeployTarget>, | |
| 248 | + | } | |
| 249 | + | ||
| 250 | + | impl AppConfig { | |
| 251 | + | /// The install destination for `target`, if this app declares one. | |
| 252 | + | pub fn deploy_for(&self, target: Target) -> Option<&DeployTarget> { | |
| 253 | + | self.deploy.iter().find(|d| d.target == target) | |
| 254 | + | } | |
| 195 | 255 | } | |
| 196 | 256 | ||
| 197 | 257 | fn default_branch() -> String { | |
| @@ -204,6 +264,77 @@ | |||
| 204 | 264 | /// Where an app's manifest lives inside its repo. | |
| 205 | 265 | pub const APP_MANIFEST: &str = "bento.toml"; | |
| 206 | 266 | ||
| 267 | + | /// Check one app's `[[deploy]]` tables at load time. | |
| 268 | + | /// | |
| 269 | + | /// Every one of these fails a release later and more expensively if it is only | |
| 270 | + | /// caught when the recipe runs — a bad `install_path` is a root `install` to the | |
| 271 | + | /// wrong place, and a target with no entry is a build that silently deploys | |
| 272 | + | /// nothing. The privileged installer on each host re-checks its own arguments | |
| 273 | + | /// (it is the thing holding the sudo grant, so it cannot trust a caller); this | |
| 274 | + | /// is the earlier, friendlier half of the same rule. | |
| 275 | + | fn validate_deploy(name: &str, app: &AppConfig) -> Result<()> { | |
| 276 | + | if app.deploy.is_empty() { | |
| 277 | + | anyhow::ensure!( | |
| 278 | + | app.kind != Kind::Service, | |
| 279 | + | "app `{name}` is a service but declares no [[deploy]] entries — \ | |
| 280 | + | a service that lands nowhere has no release" | |
| 281 | + | ); | |
| 282 | + | return Ok(()); | |
| 283 | + | } | |
| 284 | + | anyhow::ensure!( | |
| 285 | + | app.kind == Kind::Service, | |
| 286 | + | "app `{name}` declares [[deploy]] entries but is not `kind = \"service\"`; \ | |
| 287 | + | only a service is installed onto a host" | |
| 288 | + | ); | |
| 289 | + | let mut seen = Vec::new(); | |
| 290 | + | for d in &app.deploy { | |
| 291 | + | anyhow::ensure!( | |
| 292 | + | app.targets.contains(&d.target), | |
| 293 | + | "app `{name}`: [[deploy]] names target {} which the app does not ship", | |
| 294 | + | d.target | |
| 295 | + | ); | |
| 296 | + | anyhow::ensure!( | |
| 297 | + | !seen.contains(&d.target), | |
| 298 | + | "app `{name}`: two [[deploy]] entries for target {} — \ | |
| 299 | + | one target installs to one place", | |
| 300 | + | d.target | |
| 301 | + | ); | |
| 302 | + | seen.push(d.target); | |
| 303 | + | anyhow::ensure!( | |
| 304 | + | !d.host.trim().is_empty(), | |
| 305 | + | "app `{name}`: [[deploy]] for {} has an empty host", | |
| 306 | + | d.target | |
| 307 | + | ); | |
| 308 | + | // Absolute, and no `..` to walk out of wherever it appears to point. | |
| 309 | + | anyhow::ensure!( | |
| 310 | + | d.install_path.starts_with('/') | |
| 311 | + | && !Path::new(&d.install_path) | |
| 312 | + | .components() | |
| 313 | + | .any(|c| c == std::path::Component::ParentDir), | |
| 314 | + | "app `{name}`: install_path `{}` must be an absolute path with no `..`", | |
| 315 | + | d.install_path | |
| 316 | + | ); | |
| 317 | + | // A bare unit name. Anything with a slash or whitespace is either a | |
| 318 | + | // path or an attempt to smuggle a second argument into `systemctl`. | |
| 319 | + | anyhow::ensure!( | |
| 320 | + | d.service.ends_with(".service") | |
| 321 | + | && !d.service.contains('/') | |
| 322 | + | && !d.service.chars().any(char::is_whitespace), | |
| 323 | + | "app `{name}`: service `{}` must be a bare unit name ending in `.service`", | |
| 324 | + | d.service | |
| 325 | + | ); | |
| 326 | + | } | |
| 327 | + | // A service that ships a target it cannot install is a build with no ending. | |
| 328 | + | for t in &app.targets { | |
| 329 | + | anyhow::ensure!( | |
| 330 | + | seen.contains(t), | |
| 331 | + | "app `{name}`: target {t} has no [[deploy]] entry — \ | |
| 332 | + | every target a service ships must say where it lands" | |
| 333 | + | ); | |
| 334 | + | } | |
| 335 | + | Ok(()) | |
| 336 | + | } | |
| 337 | + | ||
| 207 | 338 | impl Topology { | |
| 208 | 339 | pub fn load(path: &Path) -> Result<Self> { | |
| 209 | 340 | let raw = std::fs::read_to_string(path) | |
| @@ -243,6 +374,7 @@ | |||
| 243 | 374 | features: m.features, | |
| 244 | 375 | require_all_targets: m.require_all_targets, | |
| 245 | 376 | targets: m.targets, | |
| 377 | + | deploy: m.deploy, | |
| 246 | 378 | }, | |
| 247 | 379 | ); | |
| 248 | 380 | } | |
| @@ -279,6 +411,7 @@ | |||
| 279 | 411 | anyhow::bail!("app `{name}` ships target {t} but no host declares it"); | |
| 280 | 412 | } | |
| 281 | 413 | } | |
| 414 | + | validate_deploy(name, app)?; | |
| 282 | 415 | } | |
| 283 | 416 | // Capability/transport coherence: a host that declares buildable targets | |
| 284 | 417 | // must be granted `build` (otherwise its own recipes would be denied at | |
| @@ -430,6 +563,138 @@ | |||
| 430 | 563 | assert!(load(bad).is_err()); | |
| 431 | 564 | } | |
| 432 | 565 | ||
| 566 | + | /// A service's `[[deploy]]` entries resolve, and the target -> destination | |
| 567 | + | /// binding is what the runner reads. The recipe never names a host, so this | |
| 568 | + | /// mapping is the only thing deciding which box each binary lands on. | |
| 569 | + | #[test] | |
| 570 | + | fn service_deploy_entries_resolve_per_target() { | |
| 571 | + | let (t, _dir) = load_with( | |
| 572 | + | HOSTS, | |
| 573 | + | r#"kind = "service" | |
| 574 | + | targets = ["linux/x86_64", "macos/aarch64"] | |
| 575 | + | ||
| 576 | + | [[deploy]] | |
| 577 | + | target = "linux/x86_64" | |
| 578 | + | host = "root@prod" | |
| 579 | + | port = 2200 | |
| 580 | + | install_path = "/usr/local/bin/demo" | |
| 581 | + | service = "demo.service" | |
| 582 | + | health_url = "http://prod:9100/api/health" | |
| 583 | + | ||
| 584 | + | [[deploy]] | |
| 585 | + | target = "macos/aarch64" | |
| 586 | + | host = "mbp" | |
| 587 | + | install_path = "/usr/local/bin/demo" | |
| 588 | + | service = "demo.service" | |
| 589 | + | "#, | |
| 590 | + | ) | |
| 591 | + | .unwrap(); | |
| 592 | + | let app = t.app(&"goingson".into()).unwrap(); | |
| 593 | + | assert_eq!(app.kind, Kind::Service); | |
| 594 | + | let x86 = app.deploy_for("linux/x86_64".parse().unwrap()).unwrap(); | |
| 595 | + | assert_eq!(x86.host, "root@prod"); | |
| 596 | + | assert_eq!(x86.port, Some(2200)); | |
| 597 | + | assert_eq!( | |
| 598 | + | x86.health_url.as_deref(), | |
| 599 | + | Some("http://prod:9100/api/health") | |
| 600 | + | ); | |
| 601 | + | let mac = app.deploy_for("macos/aarch64".parse().unwrap()).unwrap(); | |
| 602 | + | assert_eq!(mac.host, "mbp"); | |
| 603 | + | assert_eq!(mac.port, None); | |
| 604 | + | } | |
| 605 | + | ||
| 606 | + | /// The kind and the `[[deploy]]` table have to agree in both directions. An | |
| 607 | + | /// app with deploy entries would build them and never install them; a | |
| 608 | + | /// service without any would have no terminal step at all. | |
| 609 | + | #[test] | |
| 610 | + | fn kind_and_deploy_entries_must_agree() { | |
| 611 | + | let deploy = "\n[[deploy]]\ntarget = \"linux/x86_64\"\nhost = \"h\"\n\ | |
| 612 | + | install_path = \"/usr/local/bin/d\"\nservice = \"d.service\"\n"; | |
| 613 | + | // Deploy entries on a plain app. | |
| 614 | + | assert!( | |
| 615 | + | load_with(HOSTS, &format!("targets = [\"linux/x86_64\"]\n{deploy}")).is_err(), | |
| 616 | + | "only a service installs onto a host" | |
| 617 | + | ); | |
| 618 | + | // A service with none. | |
| 619 | + | assert!( | |
| 620 | + | load_with(HOSTS, "kind = \"service\"\ntargets = [\"linux/x86_64\"]\n").is_err(), | |
| 621 | + | "a service that lands nowhere has no release" | |
| 622 | + | ); | |
| 623 | + | } | |
| 624 | + | ||
| 625 | + | /// Every target a service ships must say where it lands. Without this a | |
| 626 | + | /// half-configured service builds both arches and silently installs one. | |
| 627 | + | #[test] | |
| 628 | + | fn service_target_without_a_deploy_entry_is_rejected() { | |
| 629 | + | let err = load_with( | |
| 630 | + | HOSTS, | |
| 631 | + | "kind = \"service\"\ntargets = [\"linux/x86_64\", \"macos/aarch64\"]\n\ | |
| 632 | + | [[deploy]]\ntarget = \"linux/x86_64\"\nhost = \"h\"\n\ | |
| 633 | + | install_path = \"/usr/local/bin/d\"\nservice = \"d.service\"\n", | |
| 634 | + | ) | |
| 635 | + | .unwrap_err(); | |
| 636 | + | assert!( | |
| 637 | + | format!("{err:#}").contains("macos/aarch64"), | |
| 638 | + | "must name the target with no destination: {err:#}" | |
| 639 | + | ); | |
| 640 | + | } | |
| 641 | + | ||
| 642 | + | /// The install path and unit name reach a root script on a production host. | |
| 643 | + | /// It re-checks them itself (it holds the sudo grant, so it cannot trust a | |
| 644 | + | /// caller), but a config that could only ever be refused should fail here, | |
| 645 | + | /// where the fix is one file away rather than mid-deploy. | |
| 646 | + | #[test] | |
| 647 | + | fn deploy_rejects_paths_and_units_the_installer_would_refuse() { | |
| 648 | + | let entry = |install: &str, service: &str| { | |
| 649 | + | format!( | |
| 650 | + | "kind = \"service\"\ntargets = [\"linux/x86_64\"]\n\ | |
| 651 | + | [[deploy]]\ntarget = \"linux/x86_64\"\nhost = \"h\"\n\ | |
| 652 | + | install_path = \"{install}\"\nservice = \"{service}\"\n" | |
| 653 | + | ) | |
| 654 | + | }; | |
| 655 | + | // Relative, and absolute-with-`..` — both are a root `install` somewhere | |
| 656 | + | // other than where the config appears to say. | |
| 657 | + | assert!(load_with(HOSTS, &entry("usr/local/bin/d", "d.service")).is_err()); | |
| 658 | + | assert!(load_with(HOSTS, &entry("/opt/../etc/systemd/system/x", "d.service")).is_err()); | |
| 659 | + | // A unit name that is really a path, or that smuggles a second argument | |
| 660 | + | // past `systemctl restart`. | |
| 661 | + | assert!(load_with(HOSTS, &entry("/usr/local/bin/d", "/etc/x.service")).is_err()); | |
| 662 | + | assert!(load_with(HOSTS, &entry("/usr/local/bin/d", "d.service x")).is_err()); | |
| 663 | + | assert!(load_with(HOSTS, &entry("/usr/local/bin/d", "d")).is_err()); | |
| 664 | + | // The shape that should pass. | |
| 665 | + | assert!(load_with(HOSTS, &entry("/usr/local/bin/d", "d.service")).is_ok()); | |
| 666 | + | } | |
| 667 | + | ||
| 668 | + | /// Two entries for one target: the second silently wins in a `find`, so the | |
| 669 | + | /// binary lands somewhere the config's first answer says it does not. | |
| 670 | + | #[test] | |
| 671 | + | fn duplicate_deploy_entries_for_one_target_are_rejected() { | |
| 672 | + | let e = "[[deploy]]\ntarget = \"linux/x86_64\"\nhost = \"h\"\n\ | |
| 673 | + | install_path = \"/usr/local/bin/d\"\nservice = \"d.service\"\n"; | |
| 674 | + | assert!( | |
| 675 | + | load_with( | |
| 676 | + | HOSTS, | |
| 677 | + | &format!("kind = \"service\"\ntargets = [\"linux/x86_64\"]\n{e}{e}") | |
| 678 | + | ) | |
| 679 | + | .is_err() | |
| 680 | + | ); | |
| 681 | + | } | |
| 682 | + | ||
| 683 | + | /// A deploy entry for a target the app does not build. It would never run, | |
| 684 | + | /// and it reads as coverage that does not exist. | |
| 685 | + | #[test] | |
| 686 | + | fn deploy_entry_for_an_unshipped_target_is_rejected() { | |
| 687 | + | assert!( | |
| 688 | + | load_with( | |
| 689 | + | HOSTS, | |
| 690 | + | "kind = \"service\"\ntargets = [\"linux/x86_64\"]\n\ | |
| 691 | + | [[deploy]]\ntarget = \"macos/aarch64\"\nhost = \"h\"\n\ | |
| 692 | + | install_path = \"/usr/local/bin/d\"\nservice = \"d.service\"\n", | |
| 693 | + | ) | |
| 694 | + | .is_err() | |
| 695 | + | ); | |
| 696 | + | } | |
| 697 | + | ||
| 433 | 698 | #[test] | |
| 434 | 699 | fn build_host_without_build_capability_is_rejected() { | |
| 435 | 700 | let bad = r#" |