Skip to main content

max / makenotwork

20.6 KB · 490 lines History Blame Raw
1 //! Deploying a service target, and the glibc floor that says whether the host
2 //! can run what was built.
3
4 use super::DEPLOY_STAGING_ROOT;
5 use super::RecipeCtx;
6 use super::collect::ensure_glob_safe;
7 use super::git::expand_tilde;
8 use crate::topology::{DeployTarget, Kind};
9 use anyhow::{Context as _, Result};
10 use ops_exec::{Action, SyncOpts};
11 use std::path::{Path, PathBuf};
12 use std::sync::Arc;
13
14 /// Highest `GLIBC_x.y` version referenced by a built binary, and the glibc a
15 /// host actually has, both parsed from the text the commands print.
16 ///
17 /// Native-per-architecture builds remove the cross-compile hazard `deploy.sh`
18 /// was written against, but not this one: fw13 tracks a newer glibc than the
19 /// Ubuntu 24.04 box in Hetzner, so a binary built here can reference a symbol
20 /// version that box does not have and fail at exec — after the unit has already
21 /// been restarted onto it. Comparing the two before the install is what makes
22 /// that a failed step instead of a downed service.
23 pub(super) fn max_glibc_symbol(objdump_out: &str) -> Option<(u64, u64)> {
24 objdump_out
25 .split(|c: char| !(c.is_ascii_digit() || c == '.' || c == '_' || c.is_ascii_alphabetic()))
26 .filter_map(|tok| tok.strip_prefix("GLIBC_"))
27 .filter_map(parse_glibc_version)
28 .max()
29 }
30
31 /// Parse `2.39` (or `2.39.1`, keeping major/minor) into a comparable pair.
32 pub(super) fn parse_glibc_version(s: &str) -> Option<(u64, u64)> {
33 let mut parts = s.split('.');
34 let major = parts.next()?.parse().ok()?;
35 let minor = parts.next()?.parse().ok()?;
36 Some((major, minor))
37 }
38
39 /// The glibc version out of `ldd --version`'s first line, whose tail is the
40 /// version however the distro decorates the rest (`ldd (Ubuntu GLIBC
41 /// 2.39-0ubuntu8.8) 2.39`).
42 pub(super) fn glibc_from_ldd(ldd_out: &str) -> Option<(u64, u64)> {
43 let first = ldd_out.lines().find(|l| !l.trim().is_empty())?;
44 parse_glibc_version(first.split_whitespace().last()?)
45 }
46
47 impl RecipeCtx {
48 /// This target's install destination, or an error naming why there is none.
49 pub(super) fn deploy_target(&self) -> Result<&DeployTarget> {
50 self.deploy.as_ref().ok_or_else(|| {
51 anyhow::anyhow!(
52 "no deploy destination for {} {}: the app is `kind = \"{}\"`, and only a \
53 service declares [[deploy]] entries",
54 self.app,
55 self.target,
56 match self.kind {
57 Kind::App => "app",
58 Kind::Library => "library",
59 Kind::Service => "service",
60 }
61 )
62 })
63 }
64
65 /// Compare the built binary's glibc requirement against the service host's.
66 /// Returns the two versions for the recipe to log.
67 pub(super) fn glibc_check(self: &Arc<Self>, binary: &str) -> Result<(String, String)> {
68 let d = self.deploy_target()?.clone();
69 // `objdump -T` on the build host; no symbols at all (a static binary)
70 // means nothing to check, which is a pass rather than a failure.
71 let (code, out) = self.run(
72 &self.build_host.clone(),
73 &format!(
74 "objdump -T {binary} 2>/dev/null | grep -o 'GLIBC_[0-9.]*' | sort -uV || true"
75 ),
76 )?;
77 anyhow::ensure!(code == 0, "reading glibc symbols from {binary} failed");
78 let Some(needs) = max_glibc_symbol(&out) else {
79 return Ok(("none".into(), "n/a".into()));
80 };
81 let (code, ldd) = self.run(&d.host, "ldd --version")?;
82 anyhow::ensure!(
83 code == 0,
84 "could not read glibc version on service host `{}`",
85 d.host
86 );
87 let has = glibc_from_ldd(&ldd).ok_or_else(|| {
88 anyhow::anyhow!(
89 "could not parse glibc version from `ldd --version` on `{}`",
90 d.host
91 )
92 })?;
93 anyhow::ensure!(
94 needs <= has,
95 "binary needs glibc {}.{} but `{}` has {}.{} — it would fail to exec after the \
96 unit restarted onto it. Build on a host no newer than the service host.",
97 needs.0,
98 needs.1,
99 d.host,
100 has.0,
101 has.1,
102 );
103 Ok((
104 format!("{}.{}", needs.0, needs.1),
105 format!("{}.{}", has.0, has.1),
106 ))
107 }
108
109 /// Install `binary` (a path on the BUILD host) onto the service host and
110 /// restart its unit, via the privileged installer the host holds a scoped
111 /// sudo grant for.
112 ///
113 /// Bento never runs the install itself. It stages the bytes and calls a
114 /// root script whose arguments are re-checked on the far side — the same
115 /// shape as Sando's `install-companion.sh`, and for the same reason: the
116 /// sudoers grant is then ONE auditable script rather than a broad
117 /// `install`+`systemctl` grant on a production box.
118 ///
119 /// Only the binary moves. Config is deliberately untouched: pom's
120 /// `pom-astra.toml` / `pom-hetzner.toml` differ per instance, and prod's
121 /// carried a `[targets.mnw.tests]` block the repo did not have. A deploy
122 /// that copies config over is how that block gets silently deleted.
123 pub(super) fn deploy(self: &Arc<Self>, binary: &str) -> Result<String> {
124 anyhow::ensure!(
125 !self.is_cancelled(),
126 "build superseded by a newer request; refusing to deploy"
127 );
128 // A failed earlier step bars a deploy exactly as it bars a publish. An
129 // artifact that failed its gates must not reach a production host just
130 // because the recipe kept running.
131 let failed = self.failed_steps_snapshot();
132 anyhow::ensure!(
133 failed.is_empty(),
134 "refusing to deploy {} {}: {} failed earlier in this run",
135 self.app,
136 self.version,
137 failed
138 .iter()
139 .map(ToString::to_string)
140 .collect::<Vec<_>>()
141 .join(", "),
142 );
143 let d = self.deploy_target()?.clone();
144 ensure_glob_safe(binary)?;
145
146 // Stage under a fixed root the installer also insists on, so "what was
147 // checked" and "what is installed" cannot drift apart.
148 let staged = format!("{DEPLOY_STAGING_ROOT}/{}", self.app);
149 let staged_bin = format!("{staged}/{}", self.app);
150 let deploy_exec = self.exec(&d.host)?;
151 anyhow::ensure!(
152 deploy_exec.capabilities().permits(&Action::Deploy),
153 "service host `{}` is not granted the `deploy` capability",
154 d.host
155 );
156
157 self.run_ok(&d.host, &format!("mkdir -p {staged}"))?;
158 if self.build_host_ssh == d.host {
159 // Same box: the binary is already there. Routing it through the
160 // daemon would be two transfers to end up where it started. This is
161 // pom's aarch64 leg — astra builds it and astra runs it.
162 self.run_ok(&d.host, &format!("cp -f {binary} {staged_bin}"))?;
163 } else {
164 // Build host -> daemon -> service host. Two hops because an executor
165 // reaches one host; a direct host-to-host transport would mean the
166 // build host holding a credential for the production box.
167 let tmp = tempfile::tempdir().context("staging dir for deploy")?;
168 let local = tmp.path().join(self.app.as_str());
169 self.pull_for_deploy(binary, &local)?;
170 let (dest, opts) = (PathBuf::from(&staged), SyncOpts::default());
171 let dir = tmp.path().to_path_buf();
172 self.run_bounded(&format!("stage {} on `{}`", self.app, d.host), async move {
173 deploy_exec.push_dir(&dir, &dest, &opts).await
174 })
175 .with_context(|| format!("staging {} onto `{}`", self.app, d.host))?;
176 }
177
178 // The privileged half. Every argument is re-validated by the script,
179 // which is the thing actually holding the sudo grant.
180 self.run_ok(
181 &d.host,
182 &format!(
183 "{} {staged_bin} {} {}",
184 self.cfg.deploy_installer, d.install_path, d.service
185 ),
186 )?;
187 Ok(format!(
188 "{} {} installed at {} on `{}`; {} restarted",
189 self.app, self.version, d.install_path, d.host, d.service
190 ))
191 }
192
193 /// Fetch one file off a host into a daemon-local path for re-pushing.
194 ///
195 /// A local build host is read directly: `fw13` is the daemon's own box, so
196 /// the file is already on this filesystem. Routing it through the
197 /// artifact-pull gate instead would demand a `pull_root` covering every repo
198 /// a service could be built in — today that is `~/Code/Apps`, and pom lives
199 /// in `~/Code/MNW`. Widening it to `~/Code` would put `_private`, the
200 /// secrets root, inside the collectable tree. This is pom's x86_64 leg.
201 fn pull_for_deploy(self: &Arc<Self>, remote: &str, local: &Path) -> Result<()> {
202 let host = self.build_host.clone();
203 let remote_path = expand_tilde(remote);
204 if self.build_host_ssh == "local" || self.build_host_ssh.is_empty() {
205 std::fs::copy(&remote_path, local).with_context(|| {
206 format!("staging {} from the daemon host", remote_path.display())
207 })?;
208 return Ok(());
209 }
210 let sync = self.host_sync(&host)?;
211 let (src, dst, opts) = (remote_path, local.to_path_buf(), SyncOpts::default());
212 self.run_bounded(&format!("fetch {remote} from `{host}`"), async move {
213 sync.pull_file(&src, &dst, &opts).await
214 })
215 .with_context(|| format!("fetching {remote} from `{host}` to deploy"))
216 }
217
218 /// `run`, failing the step on a non-zero exit. The Rust-side twin of the
219 /// recipe's `sh_ok`, for commands the deploy machinery issues itself.
220 fn run_ok(self: &Arc<Self>, host: &str, cmd: &str) -> Result<String> {
221 let (code, tail) = self.run(host, cmd)?;
222 if code != 0 {
223 self.fail_current_step();
224 anyhow::bail!("command on `{host}` exited {code}: {cmd}\n{tail}");
225 }
226 Ok(tail)
227 }
228 }
229
230 #[cfg(test)]
231 mod tests {
232 use super::super::action_for;
233 use super::super::build_engine;
234 use super::*;
235 use crate::config::Config;
236 use crate::domain::{AppId, Status, Step, Version};
237 use crate::ota::OtaRegistry;
238 use std::sync::atomic::AtomicBool;
239
240 /// The comparison that decides whether a binary can exec on the box that is
241 /// about to be restarted onto it. Both sides are parsed out of text a tool
242 /// printed, so both parsers are worth pinning: fw13 tracks a newer glibc
243 /// than the Ubuntu 24.04 host in Hetzner, and getting this backwards means a
244 /// dead unit rather than a failed step.
245 #[test]
246 fn glibc_versions_parse_from_what_the_tools_actually_print() {
247 // `objdump -T | grep -o 'GLIBC_[0-9.]*'` output: highest wins, and the
248 // comparison is numeric (2.9 must not beat 2.34 lexically).
249 let objdump = "GLIBC_2.2.5\nGLIBC_2.34\nGLIBC_2.9\nGLIBC_2.17\n";
250 assert_eq!(max_glibc_symbol(objdump), Some((2, 34)));
251 // A static binary references none: nothing to check.
252 assert_eq!(max_glibc_symbol(""), None);
253
254 // `ldd --version` first line, however the distro decorates it.
255 assert_eq!(
256 glibc_from_ldd("ldd (Ubuntu GLIBC 2.39-0ubuntu8.8) 2.39\nCopyright...\n"),
257 Some((2, 39))
258 );
259 assert_eq!(
260 glibc_from_ldd("ldd (GNU libc) 2.41\nCopyright (C) 2025\n"),
261 Some((2, 41))
262 );
263 assert_eq!(glibc_from_ldd(""), None);
264 }
265
266 /// A binary needing MORE than the host has is the failure this check exists
267 /// for; equal and less are both fine (glibc symbol versioning is backward
268 /// compatible, so an older requirement runs on a newer host).
269 #[test]
270 fn glibc_requirement_is_satisfied_by_equal_or_newer_only() {
271 let needs = max_glibc_symbol("GLIBC_2.41").unwrap();
272 assert!(needs > glibc_from_ldd("ldd (Ubuntu GLIBC 2.39) 2.39").unwrap());
273 assert!(needs <= glibc_from_ldd("ldd (GNU libc) 2.41").unwrap());
274 assert!(needs <= glibc_from_ldd("ldd (GNU libc) 2.42").unwrap());
275 assert!(needs <= glibc_from_ldd("ldd (GNU libc) 3.0").unwrap());
276 }
277
278 /// Every deploy host function fails with the app's KIND as the reason when
279 /// there is no destination, rather than with a missing-host error from
280 /// somewhere deeper. A recipe calling `deploy()` on a library is a recipe
281 /// written against the wrong kind, and the message should say so.
282 #[tokio::test]
283 async fn deploy_host_fns_explain_a_missing_destination_by_kind() {
284 let dir = tempfile::tempdir().unwrap();
285 let cfg = Arc::new(Config::for_tests(dir.path()));
286 let pool = crate::db::open(&cfg.db_path).await.unwrap();
287 let ctx = Arc::new(RecipeCtx::new(
288 AppId::new("demo"),
289 Version::parse("0.1.0").unwrap(),
290 "linux/x86_64".parse().unwrap(),
291 "fw13".into(),
292 "local".into(),
293 "v0.1.0".into(),
294 "/tmp".into(),
295 vec![],
296 Kind::Library,
297 1,
298 Arc::new(std::collections::HashMap::new()),
299 Arc::new(std::collections::HashMap::new()),
300 None,
301 pool,
302 crate::events::channel(),
303 cfg,
304 Arc::new(OtaRegistry::standard("https://makenot.work")),
305 tokio::runtime::Handle::current(),
306 Arc::new(AtomicBool::new(false)),
307 None,
308 ));
309 let engine = build_engine(&ctx);
310 for call in [
311 "deploy_host()",
312 "service_name()",
313 "install_path()",
314 "health_url()",
315 r#"deploy("/tmp/x")"#,
316 ] {
317 let err = engine.eval::<String>(call).unwrap_err().to_string();
318 assert!(
319 err.contains("library") && err.contains("no deploy destination"),
320 "`{call}` must fail on the kind, got: {err}"
321 );
322 }
323 }
324
325 /// A service host is addressed on the DEPLOY plane whatever step is open.
326 ///
327 /// The subtle one. Actions are normally derived from the step, which is
328 /// right for a build host — the step is what that host is being asked to do.
329 /// A service host is granted `deploy`/`restart` and must never be granted
330 /// `build`, so the same rule would have `glibc_check` ask it for `build`
331 /// during a `verify` step and get denied for a reason unrelated to what was
332 /// attempted. `verify` is the step that check belongs in, so without this
333 /// routing the glibc gate cannot run at all.
334 #[tokio::test]
335 async fn a_service_host_is_addressed_on_the_deploy_plane_in_any_step() {
336 let dir = tempfile::tempdir().unwrap();
337 let cfg = Arc::new(Config::for_tests(dir.path()));
338 let pool = crate::db::open(&cfg.db_path).await.unwrap();
339 sqlx::query(
340 "INSERT INTO builds (id, app, version, status, created_at) \
341 VALUES (1, 'demo', '0.1.0', 'running', '2026-07-30T00:00:00Z')",
342 )
343 .execute(&pool)
344 .await
345 .unwrap();
346 sqlx::query(
347 "INSERT INTO target_runs (id, build_id, app, version, target, status, started_at) \
348 VALUES (1, 1, 'demo', '0.1.0', 'linux/x86_64', 'running', '2026-07-30T00:00:00Z')",
349 )
350 .execute(&pool)
351 .await
352 .unwrap();
353
354 let deploy = crate::topology::DeployTarget {
355 target: "linux/x86_64".parse().unwrap(),
356 host: "local".into(),
357 port: None,
358 install_path: "/usr/local/bin/demo".into(),
359 service: "demo.service".into(),
360 health_url: None,
361 };
362 let mut execs: crate::state::ExecutorMap = std::collections::HashMap::new();
363 execs.insert("local".into(), crate::state::build_deploy_executor(&deploy));
364 // The service host's grant is exactly deploy + restart. If this ever
365 // widens to include `build`, the test below stops proving anything.
366 assert!(!execs["local"].capabilities().permits(&Action::Build));
367 assert!(execs["local"].capabilities().permits(&Action::Deploy));
368
369 let ctx = Arc::new(RecipeCtx::new(
370 AppId::new("demo"),
371 Version::parse("0.1.0").unwrap(),
372 "linux/x86_64".parse().unwrap(),
373 "fw13".into(),
374 "local".into(),
375 "v0.1.0".into(),
376 "/tmp".into(),
377 vec![],
378 Kind::Service,
379 1,
380 Arc::new(execs),
381 Arc::new(std::collections::HashMap::new()),
382 Some(deploy),
383 pool,
384 crate::events::channel(),
385 cfg,
386 Arc::new(OtaRegistry::standard("https://makenot.work")),
387 tokio::runtime::Handle::current(),
388 Arc::new(AtomicBool::new(false)),
389 None,
390 ));
391
392 let ctx_blocking = ctx.clone();
393 tokio::task::spawn_blocking(move || {
394 // `verify` on a service derives Action::Build — which the service
395 // host does not grant. The command must still run.
396 ctx_blocking.begin_step(Step::Verify).unwrap();
397 assert_eq!(
398 action_for(Step::Verify, Kind::Service),
399 Action::Build,
400 "the step's own action is the one that would be denied",
401 );
402 let (code, out) = ctx_blocking
403 .run("local", "echo reached-the-service-host")
404 .expect("a service host must be reachable during a verify step");
405 assert_eq!(code, 0, "{out}");
406 assert!(out.contains("reached-the-service-host"), "{out}");
407 })
408 .await
409 .unwrap();
410 }
411
412 /// A step that finalized `Failed` bars the deploy, exactly as it bars a
413 /// publish. Without this, a recipe that inspects `sh(...).code` and carries
414 /// on regardless still lands a binary on a production host — the precise
415 /// hazard a pipeline exists to remove. The check is the ledger, not the
416 /// control flow, so it holds whether or not the recipe noticed.
417 #[tokio::test]
418 async fn a_failed_step_bars_the_deploy() {
419 let dir = tempfile::tempdir().unwrap();
420 let cfg = Arc::new(Config::for_tests(dir.path()));
421 let pool = crate::db::open(&cfg.db_path).await.unwrap();
422 let deploy = crate::topology::DeployTarget {
423 target: "linux/x86_64".parse().unwrap(),
424 host: "local".into(),
425 port: None,
426 install_path: "/usr/local/bin/demo".into(),
427 service: "demo.service".into(),
428 health_url: None,
429 };
430 // A real build + target run, so the step rows this test finalizes have
431 // the parents the schema requires.
432 sqlx::query(
433 "INSERT INTO builds (id, app, version, status, created_at) \
434 VALUES (1, 'demo', '0.1.0', 'running', '2026-07-30T00:00:00Z')",
435 )
436 .execute(&pool)
437 .await
438 .unwrap();
439 sqlx::query(
440 "INSERT INTO target_runs (id, build_id, app, version, target, status, started_at) \
441 VALUES (1, 1, 'demo', '0.1.0', 'linux/x86_64', 'running', '2026-07-30T00:00:00Z')",
442 )
443 .execute(&pool)
444 .await
445 .unwrap();
446
447 let mut execs: crate::state::ExecutorMap = std::collections::HashMap::new();
448 execs.insert("local".into(), crate::state::build_deploy_executor(&deploy));
449 let ctx = Arc::new(RecipeCtx::new(
450 AppId::new("demo"),
451 Version::parse("0.1.0").unwrap(),
452 "linux/x86_64".parse().unwrap(),
453 "fw13".into(),
454 "local".into(),
455 "v0.1.0".into(),
456 "/tmp".into(),
457 vec![],
458 Kind::Service,
459 1,
460 Arc::new(execs),
461 Arc::new(std::collections::HashMap::new()),
462 Some(deploy),
463 pool,
464 crate::events::channel(),
465 cfg,
466 Arc::new(OtaRegistry::standard("https://makenot.work")),
467 tokio::runtime::Handle::current(),
468 Arc::new(AtomicBool::new(false)),
469 None,
470 ));
471
472 // A gate ran, failed, and the recipe did not abort — the swallowed
473 // failure. Finalizing it is what puts it in the ledger.
474 let ctx_blocking = ctx.clone();
475 tokio::task::spawn_blocking(move || {
476 ctx_blocking.begin_step(Step::Prebuild).unwrap();
477 ctx_blocking.fail_current_step();
478 ctx_blocking.finish_step(Status::Ok).unwrap();
479
480 let err = ctx_blocking.deploy("/tmp/demo").unwrap_err().to_string();
481 assert!(
482 err.contains("refusing to deploy") && err.contains("prebuild"),
483 "must refuse and name the failed step, got: {err}"
484 );
485 })
486 .await
487 .unwrap();
488 }
489 }
490