Skip to main content

max / makenotwork

40.9 KB · 1060 lines History Blame Raw
1 //! Build orchestration: resolve a sha to a worktree, read the server version,
2 //! shell out to `cargo build --release`, record a `versions` row.
3 //!
4 //! Runs as a tokio task spawned from `POST /rebuild`; the HTTP request
5 //! returns the version id immediately and the task drives the rest.
6
7 use crate::config::Config;
8 use crate::deploy;
9 use crate::domain::{GitSha, RunId, TierId, Version};
10 use crate::gates::{self, GateCtx};
11 use crate::git;
12 use crate::topology::Topology;
13 use anyhow::{Context, Result};
14 use chrono::Utc;
15 use sqlx::SqlitePool;
16 use std::path::{Path, PathBuf};
17 use std::sync::Arc;
18 use tokio::process::Command;
19
20 #[derive(Debug, Clone)]
21 pub struct BuildArtifact {
22 pub version: Version,
23 pub git_sha: GitSha,
24 pub worktree: PathBuf,
25 /// One entry per `cfg.bin_names` in declared order. First is the primary
26 /// (referenced by the systemd unit's ExecStart). Paths are inside the
27 /// worktree's `target/release/`.
28 pub binary_paths: Vec<PathBuf>,
29 /// `(companion name, built binary path)` for each `cfg.companions`, built
30 /// from the same worktree/sha as the server. Staged into the release bundle
31 /// under `companions/<name>/<bin>` and installed by the nodes that opt in.
32 pub companion_paths: Vec<(String, PathBuf)>,
33 }
34
35 /// The live kernel hostname (`/proc/sys/kernel/hostname`, trimmed). Linux-only,
36 /// which Sando is. The build-host guard reads this rather than `$HOSTNAME`
37 /// (not reliably exported) so the check reflects the actual machine.
38 fn runtime_hostname() -> Result<String> {
39 let raw = std::fs::read_to_string("/proc/sys/kernel/hostname")
40 .context("reading /proc/sys/kernel/hostname for the build-host guard")?;
41 Ok(raw.trim().to_string())
42 }
43
44 /// Pure half of the build-host guard: fail unless the live host matches the
45 /// configured build host. Split out from [`enforce_build_host`] so it is unit-
46 /// testable without depending on the test machine's hostname.
47 fn check_build_host(actual: &str, expected: &str) -> Result<()> {
48 anyhow::ensure!(
49 actual == expected,
50 "refusing to build on host {actual}: configured build host is {expected} \
51 (never build on a prod/serving node)",
52 );
53 Ok(())
54 }
55
56 /// Refuse to build unless this daemon is running on the configured build host.
57 fn enforce_build_host(expected: &str) -> Result<()> {
58 check_build_host(&runtime_hostname()?, expected)
59 }
60
61 pub async fn run(
62 pool: SqlitePool,
63 cfg: Arc<Config>,
64 topo: Arc<Topology>,
65 sha: GitSha,
66 events: crate::events::EventTx,
67 run_id: RunId,
68 ) -> Result<BuildArtifact> {
69 // Build-host guard: refuse to compile anywhere but the configured build
70 // host. The build runs locally (`cargo build` in cfg.workdir), so a sandod
71 // misdeployed onto a prod/serving node would otherwise build there — exactly
72 // the "never build on prod" rule. Enforced before any cargo invocation so a
73 // wrong-host daemon fails fast with a clear message rather than compiling.
74 enforce_build_host(&cfg.build_host)?;
75
76 let worktree = cfg.workdir.join(sha.as_str());
77 let bare = PathBuf::from(&topo.repo.bare_path);
78
79 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Fetching)
80 .await
81 .ok();
82
83 // Pull-based ingestion: if an upstream remote is configured, fetch the
84 // deploy branch so a just-pushed sha is locally resolvable. A fetch
85 // failure is non-fatal — the sha may already be present from a prior
86 // fetch or a direct push; the presence check below is the real gate.
87 if let Some(upstream) = topo.repo.upstream.as_deref()
88 && let Err(e) = git::fetch_upstream(&bare, upstream, &topo.repo.branch).await
89 {
90 tracing::warn!(error = %e, upstream, "upstream fetch failed; proceeding with current bare-repo state");
91 }
92 anyhow::ensure!(
93 git::sha_present(&bare, sha.as_str()).await?,
94 "sha {} not present in bare repo {} after fetch — push the commit to the upstream remote first",
95 sha.as_str(),
96 bare.display(),
97 );
98
99 git::checkout_worktree(&bare, sha.as_str(), &worktree).await?;
100
101 // Check out any auxiliary repos (e.g. synckit) beside the worktree so a
102 // cross-repo path dependency in the server or a companion resolves. Fails the
103 // build if an aux repo can't be assembled — a companion that silently fails to
104 // find its source would fail the compile downstream with a worse message.
105 checkout_aux_repos(&cfg, &topo).await?;
106
107 let server_dir = worktree.join("server");
108 let version = read_pkg_version(&server_dir.join("Cargo.toml"))
109 .await
110 .with_context(|| format!("reading version from {}/Cargo.toml", server_dir.display()))?;
111 crate::runs::set_version(&pool, run_id, &version).await.ok();
112
113 // sqlx compile-time query checking needs a live DB with the current schema.
114 // We point cargo at the scratch DB and prep it (drop public, re-migrate)
115 // before invoking cargo build. The same DB is reset again by
116 // `migration_dry_run` later if it runs as a gate.
117 let mut cargo_cmd = Command::new("cargo");
118 cargo_cmd
119 .arg("build")
120 .arg("--release")
121 .current_dir(&server_dir)
122 .kill_on_drop(true);
123 // Shared build cache across per-sha worktrees: reuse one target dir so an
124 // incremental diff doesn't clean-compile from scratch. Serialized builds
125 // make this contention-free. Unset → cargo's default per-worktree target/.
126 if let Some(target) = cfg.cargo_target_dir.as_deref() {
127 cargo_cmd.env("CARGO_TARGET_DIR", target);
128 }
129 if let Some(scratch_url) = cfg.scratch_db_url.as_deref() {
130 tracing::info!(sha = %sha.as_str(), "preparing scratch DB schema for sqlx compile-time checks");
131 crate::gates::reset_scratch(scratch_url, &cfg.scratch_owner_role)
132 .await
133 .context("scratch DB reset before build")?;
134 crate::gates::run_migrator(scratch_url, &server_dir.join("migrations"))
135 .await
136 .context("applying MNW migrations to scratch DB before build")?;
137 cargo_cmd.env("DATABASE_URL", scratch_url);
138 } else {
139 tracing::warn!("scratch_db_url unset; sqlx will fall back to offline mode and may fail");
140 }
141
142 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Compiling)
143 .await
144 .ok();
145 tracing::info!(sha = %sha, version = %version, dir = %server_dir.display(), "cargo build --release start");
146 crate::events::emit(
147 &events,
148 crate::events::Event::BuildStart {
149 sha: sha.clone(),
150 version: version.clone(),
151 },
152 );
153 let started = std::time::Instant::now();
154 let out = cargo_cmd.output().await.context("spawning cargo build")?;
155 let elapsed_s = started.elapsed().as_secs();
156 if !out.status.success() {
157 tracing::error!(sha = %sha, version = %version, elapsed_s, "cargo build --release failed");
158 crate::events::emit(
159 &events,
160 crate::events::Event::BuildFailed {
161 sha: sha.clone(),
162 version: version.clone(),
163 elapsed_s,
164 },
165 );
166 // Settle the run with the headline compiler diagnostic (not the raw
167 // 4 KB tail) so `GET /runs/{id}` answers "why" without a journald dive.
168 let summary = crate::classify::classify_compile_error(&out.stdout, &out.stderr).summary();
169 if let Err(e) = crate::runs::mark_failed(&pool, run_id, &summary).await {
170 tracing::error!(run_id = %run_id, error = %e, "persisting compile-fail verdict failed; run may show stale 'building' until restart-reconcile");
171 }
172 anyhow::bail!(
173 "cargo build --release failed:\n{}",
174 tail(&out.stderr, 4_000)
175 );
176 }
177 tracing::info!(sha = %sha, version = %version, elapsed_s, "cargo build --release ok");
178 crate::events::emit(
179 &events,
180 crate::events::Event::BuildOk {
181 sha: sha.clone(),
182 version: version.clone(),
183 elapsed_s,
184 },
185 );
186
187 // Binaries land under `<target>/release/`; with a shared target dir that's
188 // not inside the worktree, so resolve it the same way cargo did above.
189 let release_dir = cfg
190 .cargo_target_dir
191 .as_deref()
192 .map_or_else(|| server_dir.join("target/release"), |t| t.join("release"));
193 let mut binary_paths = Vec::with_capacity(cfg.bin_names.len());
194 for name in &cfg.bin_names {
195 let p = release_dir.join(name);
196 anyhow::ensure!(p.exists(), "expected binary at {} after build", p.display());
197 binary_paths.push(p);
198 }
199 // Primary binary path is the one we record in `versions.artifact_path`
200 // (everything downstream — promote, rollback — looks it up by version).
201 let primary = binary_paths[0].clone();
202
203 // Companion crates (e.g. mnw-cli): built from the SAME worktree/sha so a
204 // service that shares the server's internal-API contract cannot drift out of
205 // lockstep (the 2026-07-09 git-hosting outage). A companion build failure
206 // fails the whole pipeline — the server never ships without its companions.
207 let mut companion_paths = Vec::with_capacity(cfg.companions.len());
208 for c in &cfg.companions {
209 let bin = build_companion(&worktree, &cfg, c).await?;
210 companion_paths.push((c.name.clone(), bin));
211 }
212
213 sqlx::query(
214 "INSERT OR IGNORE INTO versions (version, git_sha, built_at, artifact_path)
215 VALUES (?, ?, ?, ?)",
216 )
217 .bind(&version)
218 .bind(&sha)
219 .bind(Utc::now().to_rfc3339())
220 .bind(primary.to_string_lossy().as_ref())
221 .execute(&pool)
222 .await?;
223
224 Ok(BuildArtifact {
225 version,
226 git_sha: sha,
227 worktree,
228 binary_paths,
229 companion_paths,
230 })
231 }
232
233 /// Fetch and check out every configured auxiliary repo at `cfg.workdir/<checkout_dir>`,
234 /// so a cross-repo path dependency built from the main worktree resolves (wiki
235 /// [[sando-overview]]; the synckit split, task sando-18cdb32f).
236 ///
237 /// Each aux repo is a fixed, shared checkout refreshed to `branch` HEAD — not
238 /// per-sha — because the dependent's relative path resolves to that fixed spot
239 /// regardless of the main sha, and builds serialize. A fetch failure is a warning
240 /// (the branch may already be present from a prior build); an unresolvable branch
241 /// after that is fatal, as is a failed worktree — a half-assembled source tree
242 /// must fail the build here, loudly, not as a downstream compile error.
243 pub async fn checkout_aux_repos(cfg: &Config, topo: &Topology) -> Result<()> {
244 for aux in &topo.aux_repos {
245 let bare = PathBuf::from(&aux.bare_path);
246 git::ensure_bare_repo_no_hook(&bare)
247 .await
248 .with_context(|| format!("aux repo {}: init bare {}", aux.name, aux.bare_path))?;
249 if let Err(e) = git::fetch_upstream(&bare, &aux.upstream, &aux.branch).await {
250 tracing::warn!(
251 aux = %aux.name, error = %e,
252 "aux repo fetch failed; proceeding with current bare-repo state",
253 );
254 }
255 let sha = git::resolve_ref(&bare, &aux.branch).await.with_context(|| {
256 format!(
257 "aux repo {}: branch {} not resolvable after fetch — is {} reachable with that branch?",
258 aux.name, aux.branch, aux.upstream,
259 )
260 })?;
261 let dest = cfg.workdir.join(&aux.checkout_dir);
262 git::checkout_worktree(&bare, &sha, &dest)
263 .await
264 .with_context(|| {
265 format!(
266 "aux repo {}: checking out {} ({}) at {}",
267 aux.name,
268 aux.branch,
269 sha,
270 dest.display()
271 )
272 })?;
273 tracing::info!(
274 aux = %aux.name, branch = %aux.branch, sha = %sha, dest = %dest.display(),
275 "aux repo checked out beside worktree",
276 );
277 }
278 Ok(())
279 }
280
281 /// Build one companion crate from the worktree, returning its release binary
282 /// path. Mirrors the server build's target-dir handling (shared
283 /// `cargo_target_dir` when set, for incremental reuse; else the crate's own
284 /// `target/`). A companion is an API client, not a sqlx crate, so it needs no
285 /// scratch DB. A non-zero exit propagates and fails the pipeline.
286 async fn build_companion(
287 worktree: &Path,
288 cfg: &Config,
289 c: &crate::config::Companion,
290 ) -> Result<PathBuf> {
291 let dir = worktree.join(&c.manifest_dir);
292 anyhow::ensure!(
293 dir.join("Cargo.toml").exists(),
294 "companion {}: no Cargo.toml at {}",
295 c.name,
296 dir.display(),
297 );
298 // Match the server build: no `--locked` (the pipeline builds whatever the
299 // sha pins; a stale lock shouldn't block a deploy the server build allows).
300 let mut cmd = Command::new("cargo");
301 cmd.arg("build")
302 .arg("--release")
303 .current_dir(&dir)
304 .kill_on_drop(true);
305 let release_dir = if let Some(target) = cfg.cargo_target_dir.as_deref() {
306 cmd.env("CARGO_TARGET_DIR", target);
307 target.join("release")
308 } else {
309 dir.join("target/release")
310 };
311 tracing::info!(companion = %c.name, dir = %dir.display(), "cargo build --release (companion) start");
312 let started = std::time::Instant::now();
313 let out = cmd
314 .output()
315 .await
316 .context("spawning cargo build for companion")?;
317 if !out.status.success() {
318 anyhow::bail!(
319 "companion {} build failed:\n{}",
320 c.name,
321 tail(&out.stderr, 4_000),
322 );
323 }
324 let bin = release_dir.join(&c.bin);
325 anyhow::ensure!(
326 bin.exists(),
327 "companion {} produced no binary at {} after build",
328 c.name,
329 bin.display(),
330 );
331 tracing::info!(companion = %c.name, elapsed_s = started.elapsed().as_secs(), "companion build ok");
332 Ok(bin)
333 }
334
335 /// Full host-tier pipeline: build, stage the bundle into the host's
336 /// release_root, run the host tier's configured gates, advance tier_state
337 /// for "host" if all pass. Errors propagate back to the spawned task and
338 /// get logged. (Tier was called "mm" pre-Session-1; renamed to "host"
339 /// since sandod runs on whatever machine ends up being the Sando host.)
340 pub async fn build_and_run_host(
341 pool: SqlitePool,
342 cfg: Arc<Config>,
343 topo: Arc<Topology>,
344 sha: GitSha,
345 events: crate::events::EventTx,
346 run_id: RunId,
347 deploy_lock: Arc<tokio::sync::Mutex<()>>,
348 ) -> Result<()> {
349 let art = run(
350 pool.clone(),
351 cfg.clone(),
352 topo.clone(),
353 sha,
354 events.clone(),
355 run_id,
356 )
357 .await?;
358
359 stage_and_gate(pool, cfg, topo, art, events, run_id, deploy_lock).await
360 }
361
362 /// Post-build half of the host pipeline: stage the artifact into the host's
363 /// release_root, run the host tier's gates, and advance `tier_state` for
364 /// "host" iff all pass. Split from [`build_and_run_host`] at the `run()`
365 /// boundary so the staging/gating/advance logic is reachable in tests from a
366 /// synthetic [`BuildArtifact`] — no real `cargo build --release` required.
367 pub async fn stage_and_gate(
368 pool: SqlitePool,
369 cfg: Arc<Config>,
370 topo: Arc<Topology>,
371 art: BuildArtifact,
372 events: crate::events::EventTx,
373 run_id: RunId,
374 deploy_lock: Arc<tokio::sync::Mutex<()>>,
375 ) -> Result<()> {
376 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Staging)
377 .await
378 .ok();
379
380 // Stage the bundle into `staging/<build_id>/` — a private scratch dir, not
381 // yet a release. It is published content-addressed below, once its digest is
382 // known. This is what makes overwrite unexpressible (wiki
383 // [[release-artifact-identity]]): a build never touches another build's dir.
384 let host_release_root = &cfg.release_root;
385 let staging =
386 deploy::stage_local_bundle(host_release_root, run_id.0, &art.binary_paths).await?;
387
388 // Stage every entry from cfg.release_contents into the staged bundle. This is
389 // how non-binary version-coupled content (static assets, docs, error-pages,
390 // ...) makes it into the atomic deploy bundle. Projects opt in via daemon
391 // config — the sando code carries no MNW-specific knowledge.
392 for entry in &cfg.release_contents {
393 stage_entry(&art.worktree, &staging, entry).await?;
394 }
395
396 // Stage companion binaries as `companions/<name>` (the file itself) so they
397 // ride the same atomic bundle rsync to the nodes, and a node can locate its
398 // companion source from the logical name alone — no bin-filename coupling in
399 // the topology. The nodes that opt in install them post-swap (see
400 // deploy::deploy_remote).
401 if !art.companion_paths.is_empty() {
402 let dst_dir = staging.join("companions");
403 tokio::fs::create_dir_all(&dst_dir)
404 .await
405 .with_context(|| format!("create staged companions dir {}", dst_dir.display()))?;
406 for (name, built) in &art.companion_paths {
407 let dst = dst_dir.join(name);
408 tokio::fs::copy(built, &dst).await.with_context(|| {
409 format!(
410 "stage companion {name}: {} -> {}",
411 built.display(),
412 dst.display()
413 )
414 })?;
415 }
416 }
417
418 // Content identity: hash the fully-staged bundle, write its MANIFEST into the
419 // bundle (for node-side verification), then publish it at `releases/<digest16>`.
420 // The digest is now load-bearing — a hashing failure fails the build rather
421 // than shipping an unidentifiable artifact.
422 let digest = crate::bundle::digest_dir(&staging)
423 .await
424 .context("hashing the staged bundle for content addressing")?;
425 tokio::fs::write(
426 staging.join(crate::bundle::MANIFEST_NAME),
427 digest.manifest.as_bytes(),
428 )
429 .await
430 .context("writing bundle MANIFEST")?;
431 let released =
432 deploy::finalize_local_release(host_release_root, &staging, digest.short()).await?;
433
434 let staged_bin = released.join(cfg.primary_bin());
435 sqlx::query("UPDATE versions SET artifact_path = ? WHERE version = ?")
436 .bind(staged_bin.to_string_lossy().as_ref())
437 .bind(&art.version)
438 .execute(&pool)
439 .await?;
440
441 // Record the identity on the build row: the full digest and the
442 // content-addressed dir the bundle was published to. This is what promote
443 // resolves the artifact through, and burn-in/retention key on.
444 {
445 let released_path = released.to_string_lossy();
446 crate::runs::set_identity(&pool, run_id, &digest.full, &released_path)
447 .await
448 .ok();
449 }
450
451 let host = topo
452 .tiers
453 .iter()
454 .find(|t| t.name.as_str() == "host")
455 .context("topology has no `host` tier")?;
456
457 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Gating)
458 .await
459 .ok();
460 let ctx = GateCtx {
461 pool: pool.clone(),
462 cfg: cfg.clone(),
463 tier: TierId::new("host"),
464 version: art.version.clone(),
465 worktree: art.worktree.clone(),
466 events: events.clone(),
467 // Host runs build-time gates (cargo_test / migration_dry_run /
468 // boot_smoke) only — `node_health` never appears here, so there are no
469 // nodes to probe.
470 nodes: Vec::new(),
471 // These gates vouch for this build; record its id so promote can resolve
472 // the artifact through the evidence rather than a version string.
473 build_id: Some(run_id.0),
474 };
475 let failed = gates::run_all(&ctx, &host.gates).await?;
476
477 if failed.is_empty() {
478 // Advance the host tier through the single sealed forward-advance op, under
479 // deploy_lock so this can't interleave with a concurrent `/rollback host`
480 // (the old fetch-then-write here was the one CF3 site outside the lock —
481 // ultra-fuzz Run 2, S1). Held only for the atomic UPDATE, never the gates.
482 {
483 let _deploy_guard = deploy_lock.lock().await;
484 crate::runs::advance_tier(&pool, "host", &art.version, Some(run_id.0)).await?;
485 }
486 // Terminal verdict: unlike the phase pings above (best-effort), a dropped
487 // pass/fail write leaves the run wedged at `building`. Log it loudly if it
488 // fails — the startup reconcile (main) is the backstop that settles such a
489 // row on the next restart.
490 if let Err(e) = crate::runs::mark_passed(&pool, run_id).await {
491 tracing::error!(run_id = %run_id, error = %e, "persisting host-green verdict failed; run may show stale 'building' until restart-reconcile");
492 }
493 tracing::info!(version = %art.version, "host pipeline green; ready to promote to next tier");
494 } else {
495 // Pull the first red gate's typed summary into the run so the API
496 // answers "which gate, and why" — not just "failed".
497 let summary = crate::runs::first_failed_gate_summary(&pool, &art.version)
498 .await
499 .unwrap_or_else(|| "host pipeline red".to_string());
500 if let Err(e) = crate::runs::mark_failed(&pool, run_id, &summary).await {
501 tracing::error!(run_id = %run_id, error = %e, "persisting host-red verdict failed; run may show stale 'building' until restart-reconcile");
502 }
503 tracing::warn!(version = %art.version, "host pipeline red; not advancing tier_state");
504 }
505 Ok(())
506 }
507
508 async fn read_pkg_version(cargo_toml: &Path) -> Result<Version> {
509 let raw = tokio::fs::read_to_string(cargo_toml).await?;
510 let parsed: toml::Value = toml::from_str(&raw)?;
511 let v = parsed
512 .get("package")
513 .and_then(|p| p.get("version"))
514 .and_then(|v| v.as_str())
515 .context("package.version not found")?;
516 Version::parse(v).with_context(|| format!("parsing package.version `{v}`"))
517 }
518
519 fn tail(buf: &[u8], max: usize) -> String {
520 let s = String::from_utf8_lossy(buf);
521 if s.len() <= max {
522 return s.into_owned();
523 }
524 // `s.len() - max` can land mid-codepoint; walk forward to the next char
525 // boundary so the slice never panics (returns slightly fewer than `max`
526 // bytes in that case). `floor_char_boundary` is still unstable, so do it by
527 // hand.
528 let mut start = s.len() - max;
529 while start < s.len() && !s.is_char_boundary(start) {
530 start += 1;
531 }
532 s[start..].to_string()
533 }
534
535 /// Copy `worktree/<entry.src>` into `staged/<entry.dst>`. Handles file or
536 /// directory sources transparently. Missing source policy depends on
537 /// `entry.required`:
538 /// - required=true -> error (build fails)
539 /// - required=false -> log warn + skip (e.g. older shas missing a dir)
540 ///
541 /// Uses `cp -a` to preserve modes/symlinks/etc; parent of dst is created if
542 /// needed so entries like `dst = "docs/assumptions.toml"` work without
543 /// extra config.
544 async fn stage_entry(
545 worktree: &Path,
546 staged: &Path,
547 entry: &crate::config::ReleaseEntry,
548 ) -> Result<()> {
549 let src = worktree.join(&entry.src);
550 let dst = staged.join(&entry.dst);
551 if !src.exists() {
552 if entry.required {
553 anyhow::bail!(
554 "required release_contents source missing: {}",
555 src.display()
556 );
557 }
558 tracing::warn!(src = %src.display(), "release_contents source missing (optional); skipping");
559 return Ok(());
560 }
561 if let Some(parent) = dst.parent() {
562 tokio::fs::create_dir_all(parent)
563 .await
564 .with_context(|| format!("create staged parent {}", parent.display()))?;
565 }
566 // Multiple entries with the same dst (e.g. site-docs/public/ +
567 // site-docs/examples/ both landing under docs/) need additive merging.
568 // `cp -a SRC/. DST/` copies SRC's contents into DST without overwriting
569 // the dst dir itself; that's the merge-friendly form when dst is a dir
570 // that may already exist from a prior entry. For non-dir sources or a
571 // missing dst we fall back to the plain `cp -a SRC DST` form.
572 let merge_into_existing_dir = src.is_dir() && dst.is_dir();
573 let mut cmd = Command::new("cp");
574 cmd.arg("-a");
575 if merge_into_existing_dir {
576 let mut src_arg = src.clone().into_os_string();
577 src_arg.push("/.");
578 cmd.arg(src_arg);
579 let mut dst_arg = dst.clone().into_os_string();
580 dst_arg.push("/");
581 cmd.arg(dst_arg);
582 } else {
583 cmd.arg(&src).arg(&dst);
584 }
585 let out = cmd
586 .output()
587 .await
588 .with_context(|| format!("spawning cp for {} -> {}", src.display(), dst.display()))?;
589 anyhow::ensure!(
590 out.status.success(),
591 "stage {} -> {}: {}",
592 src.display(),
593 dst.display(),
594 String::from_utf8_lossy(&out.stderr),
595 );
596 Ok(())
597 }
598
599 #[cfg(test)]
600 mod tests {
601 use super::{
602 BuildArtifact, check_build_host, checkout_aux_repos, runtime_hostname, stage_and_gate, tail,
603 };
604 use crate::config::{Config, TestTarget};
605 use crate::domain::{GitSha, RunId, Version};
606 use crate::topology::{AuxRepo, BackupConfig, CanaryPolicy, Gate, RepoConfig, Tier, Topology};
607 use sqlx::SqlitePool;
608 use sqlx::sqlite::SqlitePoolOptions;
609 use std::path::PathBuf;
610 use std::sync::Arc;
611
612 /// Post-build pipeline fixture: an in-memory store with the `host` tier
613 /// seeded, a synthetic worktree holding a fake primary binary, and a
614 /// build_runs row in flight. Returns everything `stage_and_gate` needs plus
615 /// the tempdir root (drop it to clean up) and the run/version it seeded.
616 ///
617 /// `gates` is the host tier's gate list: `[]` is the green path;
618 /// `[Gate::ManualConfirm]` is a deterministic red — with no prior operator
619 /// confirmation row that gate blocks, and it shells out to nothing.
620 async fn stage_fixture(
621 gates: Vec<Gate>,
622 ) -> (
623 SqlitePool,
624 Arc<Config>,
625 Arc<Topology>,
626 BuildArtifact,
627 RunId,
628 Version,
629 tempfile::TempDir,
630 ) {
631 let tmp = tempfile::tempdir().unwrap();
632 let release_root = tmp.path().join("release-root");
633 let worktree = tmp.path().join("worktree");
634 let bin_dir = worktree.join("target").join("release");
635 tokio::fs::create_dir_all(&bin_dir).await.unwrap();
636 let bin_path = bin_dir.join("makenotwork");
637 tokio::fs::write(&bin_path, b"#!/bin/false\nfake sando artifact\n")
638 .await
639 .unwrap();
640
641 let pool = SqlitePoolOptions::new()
642 .max_connections(1)
643 .connect("sqlite::memory:")
644 .await
645 .unwrap();
646 sqlx::migrate!("./migrations").run(&pool).await.unwrap();
647
648 // gate_runs and tier_state FK into `tiers`; the pipeline only touches host.
649 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('host', 0, 1, 'sequential')")
650 .execute(&pool)
651 .await
652 .unwrap();
653 sqlx::query("INSERT INTO tier_state (tier) VALUES ('host')")
654 .execute(&pool)
655 .await
656 .unwrap();
657
658 let version = Version::parse("1.2.3").unwrap();
659 let git_sha = GitSha::parse("abc1234").unwrap();
660 // gate_runs.version and the `SET artifact_path` UPDATE both need the row.
661 sqlx::query(
662 "INSERT INTO versions (version, git_sha, built_at, artifact_path)
663 VALUES (?, ?, datetime('now'), '')",
664 )
665 .bind(version.to_string())
666 .bind(git_sha.to_string())
667 .execute(&pool)
668 .await
669 .unwrap();
670
671 let run_id = crate::runs::create(&pool, &git_sha.to_string())
672 .await
673 .unwrap();
674
675 let cfg = Config {
676 listen: "127.0.0.1:0".into(),
677 db_path: PathBuf::from(":memory:"),
678 topology_path: PathBuf::from("/tmp/test-sando.toml"),
679 build_host: "test-host".into(),
680 workdir: tmp.path().to_path_buf(),
681 release_root: release_root.clone(),
682 scratch_db_url: None,
683 scratch_owner_role: "makenotwork".into(),
684 boot_smoke_port: 18181,
685 code_smoke_port: 18182,
686 bin_names: vec!["makenotwork".into()],
687 logs_root: tmp.path().join("logs"),
688 release_contents: vec![],
689 cargo_target_dir: None,
690 gate_timeout_secs: 2400,
691 companions: Vec::new(),
692 test_targets: vec![TestTarget {
693 dir: PathBuf::from("server"),
694 features: vec!["fast-tests".into()],
695 all_features: false,
696 scratch_db: true,
697 }],
698 frontend_builds: vec![],
699 backup_max_age_hours: 48,
700 };
701
702 let topo = Topology {
703 repo: RepoConfig {
704 bare_path: "/tmp/test.git".into(),
705 branch: "main".into(),
706 upstream: None,
707 },
708 backup: BackupConfig {
709 source: "file:///tmp/test-backup.sql".into(),
710 local_path: "/tmp/local-backup.sql".into(),
711 },
712 tiers: vec![Tier {
713 name: "host".into(),
714 provisioned: true,
715 gates,
716 canary: CanaryPolicy::Sequential,
717 nodes: Vec::new(),
718 }],
719 aux_repos: Vec::new(),
720 };
721
722 let art = BuildArtifact {
723 version: version.clone(),
724 git_sha,
725 worktree,
726 binary_paths: vec![bin_path],
727 companion_paths: Vec::new(),
728 };
729
730 (
731 pool,
732 Arc::new(cfg),
733 Arc::new(topo),
734 art,
735 run_id,
736 version,
737 tmp,
738 )
739 }
740
741 // ---- checkout_aux_repos ----
742
743 async fn git_in(dir: &std::path::Path, args: &[&str]) {
744 let out = tokio::process::Command::new("git")
745 .args(["-c", "user.email=t@t", "-c", "user.name=t"])
746 .current_dir(dir)
747 .args(args)
748 .output()
749 .await
750 .unwrap();
751 assert!(
752 out.status.success(),
753 "git {args:?}: {}",
754 String::from_utf8_lossy(&out.stderr)
755 );
756 }
757
758 /// A minimal `Config` whose only field this test path reads is `workdir`.
759 fn cfg_with_workdir(workdir: PathBuf) -> Config {
760 Config {
761 listen: "127.0.0.1:0".into(),
762 db_path: PathBuf::from(":memory:"),
763 topology_path: PathBuf::from("/tmp/test-sando.toml"),
764 build_host: "test-host".into(),
765 workdir,
766 release_root: PathBuf::from("/tmp/rr"),
767 scratch_db_url: None,
768 scratch_owner_role: "makenotwork".into(),
769 boot_smoke_port: 18181,
770 code_smoke_port: 18182,
771 bin_names: vec!["makenotwork".into()],
772 logs_root: PathBuf::from("/tmp/logs"),
773 release_contents: vec![],
774 cargo_target_dir: None,
775 gate_timeout_secs: 2400,
776 companions: Vec::new(),
777 test_targets: vec![],
778 frontend_builds: vec![],
779 backup_max_age_hours: 48,
780 }
781 }
782
783 fn topo_with_aux(aux_repos: Vec<AuxRepo>) -> Topology {
784 Topology {
785 repo: RepoConfig {
786 bare_path: "/tmp/x.git".into(),
787 branch: "main".into(),
788 upstream: None,
789 },
790 backup: BackupConfig {
791 source: "s".into(),
792 local_path: "/tmp/d".into(),
793 },
794 tiers: vec![],
795 aux_repos,
796 }
797 }
798
799 #[tokio::test]
800 async fn checkout_aux_repos_places_repo_beside_worktree_and_refreshes_to_branch_head() {
801 let tmp = tempfile::tempdir().unwrap();
802
803 // An "upstream" source repo with a marker file on main.
804 let src = tmp.path().join("synckit-src");
805 tokio::fs::create_dir_all(&src).await.unwrap();
806 git_in(&src, &["init", "-q", "-b", "main"]).await;
807 tokio::fs::write(src.join("VERSION"), b"v1").await.unwrap();
808 git_in(&src, &["add", "."]).await;
809 git_in(&src, &["commit", "-q", "-m", "one"]).await;
810
811 let workdir = tmp.path().join("work");
812 tokio::fs::create_dir_all(&workdir).await.unwrap();
813 let cfg = cfg_with_workdir(workdir.clone());
814 let topo = topo_with_aux(vec![AuxRepo {
815 name: "synckit".into(),
816 bare_path: tmp
817 .path()
818 .join("synckit.git")
819 .to_string_lossy()
820 .into_owned(),
821 upstream: src.to_string_lossy().into_owned(),
822 branch: "main".into(),
823 checkout_dir: "synckit".into(),
824 }]);
825
826 // First build: the aux repo lands at workdir/synckit at v1.
827 checkout_aux_repos(&cfg, &topo).await.unwrap();
828 let dest = workdir.join("synckit");
829 assert_eq!(
830 tokio::fs::read(dest.join("VERSION")).await.unwrap(),
831 b"v1",
832 "aux repo checked out beside the worktree",
833 );
834
835 // Upstream advances; a later build refreshes the shared checkout to HEAD.
836 tokio::fs::write(src.join("VERSION"), b"v2").await.unwrap();
837 git_in(&src, &["add", "."]).await;
838 git_in(&src, &["commit", "-q", "-m", "two"]).await;
839 checkout_aux_repos(&cfg, &topo).await.unwrap();
840 assert_eq!(
841 tokio::fs::read(dest.join("VERSION")).await.unwrap(),
842 b"v2",
843 "aux checkout refreshed to the new branch HEAD",
844 );
845
846 // The aux bare carries no build-trigger hook.
847 assert!(
848 !tmp.path().join("synckit.git/hooks/post-receive").exists(),
849 "aux bare must be hookless",
850 );
851 }
852
853 #[tokio::test]
854 async fn checkout_aux_repos_is_a_noop_without_aux_repos() {
855 let tmp = tempfile::tempdir().unwrap();
856 let cfg = cfg_with_workdir(tmp.path().to_path_buf());
857 checkout_aux_repos(&cfg, &topo_with_aux(vec![]))
858 .await
859 .unwrap();
860 }
861
862 #[tokio::test]
863 async fn checkout_aux_repos_fails_on_an_unresolvable_branch() {
864 let tmp = tempfile::tempdir().unwrap();
865 let src = tmp.path().join("src");
866 tokio::fs::create_dir_all(&src).await.unwrap();
867 git_in(&src, &["init", "-q", "-b", "main"]).await;
868 tokio::fs::write(src.join("f"), b"x").await.unwrap();
869 git_in(&src, &["add", "."]).await;
870 git_in(&src, &["commit", "-q", "-m", "c"]).await;
871
872 let cfg = cfg_with_workdir(tmp.path().join("work"));
873 let topo = topo_with_aux(vec![AuxRepo {
874 name: "synckit".into(),
875 bare_path: tmp.path().join("s.git").to_string_lossy().into_owned(),
876 upstream: src.to_string_lossy().into_owned(),
877 branch: "nonexistent".into(),
878 checkout_dir: "synckit".into(),
879 }]);
880 let err = checkout_aux_repos(&cfg, &topo).await.unwrap_err();
881 assert!(
882 format!("{err:#}").contains("synckit"),
883 "error names the aux repo: {err:#}",
884 );
885 }
886
887 async fn tier_versions(pool: &SqlitePool, tier: &str) -> (Option<String>, Option<String>) {
888 sqlx::query_as("SELECT current_version, previous_version FROM tier_state WHERE tier = ?")
889 .bind(tier)
890 .fetch_one(pool)
891 .await
892 .unwrap()
893 }
894
895 async fn run_result(pool: &SqlitePool, run_id: RunId) -> (String, Option<String>) {
896 sqlx::query_as("SELECT result, failure_summary FROM build_runs WHERE id = ?")
897 .bind(run_id.0)
898 .fetch_one(pool)
899 .await
900 .unwrap()
901 }
902
903 #[tokio::test]
904 async fn stage_and_gate_stages_advances_and_flips_the_symlink_when_gates_are_green() {
905 let (pool, cfg, topo, art, run_id, version, tmp) = stage_fixture(vec![]).await;
906 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
907
908 stage_and_gate(
909 pool.clone(),
910 cfg.clone(),
911 topo,
912 art,
913 crate::events::channel(),
914 run_id,
915 deploy_lock,
916 )
917 .await
918 .expect("green host pipeline returns Ok");
919
920 // Tier advanced to the built version (previous was NULL -> stays NULL).
921 let (current, previous) = tier_versions(&pool, "host").await;
922 assert_eq!(current.as_deref(), Some(version.to_string().as_str()));
923 assert_eq!(previous, None);
924
925 // Run settled green.
926 let (result, summary) = run_result(&pool, run_id).await;
927 assert_eq!(result, "passed");
928 assert_eq!(summary, None);
929
930 // Identity: the build row carries the bundle digest (64 hex) and the
931 // content-addressed dir it was published to (releases/<digest16>).
932 let (digest, staged_path): (Option<String>, Option<String>) =
933 sqlx::query_as("SELECT bundle_digest, staged_path FROM build_runs WHERE id = ?")
934 .bind(run_id.0)
935 .fetch_one(&pool)
936 .await
937 .unwrap();
938 let digest = digest.expect("bundle_digest recorded");
939 let staged_path = staged_path.expect("staged_path recorded");
940 assert_eq!(digest.len(), 64);
941 let releases = tmp.path().join("release-root").join("releases");
942 assert_eq!(
943 std::path::Path::new(&staged_path),
944 releases.join(&digest[..16]),
945 "bundle is published content-addressed at releases/<digest16>"
946 );
947
948 // versions.artifact_path points at the primary binary inside that dir,
949 // and it exists on disk.
950 let staged_bin: String =
951 sqlx::query_scalar("SELECT artifact_path FROM versions WHERE version = ?")
952 .bind(version.to_string())
953 .fetch_one(&pool)
954 .await
955 .unwrap();
956 let expected_bin = releases.join(&digest[..16]).join("makenotwork");
957 assert_eq!(staged_bin, expected_bin.to_string_lossy());
958 assert!(
959 expected_bin.exists(),
960 "staged binary missing at {expected_bin:?}"
961 );
962
963 // The bundle carries its MANIFEST (for node-side verification), and the
964 // recorded digest recomputes over the published dir (MANIFEST excluded).
965 assert!(
966 releases.join(&digest[..16]).join("MANIFEST").exists(),
967 "MANIFEST written into the bundle"
968 );
969 let recomputed = crate::bundle::digest_dir(std::path::Path::new(&staged_path))
970 .await
971 .unwrap();
972 assert_eq!(
973 digest, recomputed.full,
974 "recorded digest matches the bundle"
975 );
976
977 // The `current` symlink flipped to the content-addressed release.
978 let link = tmp.path().join("release-root").join("current");
979 let target = std::fs::read_link(&link).expect("current is a symlink");
980 assert_eq!(target, PathBuf::from(format!("releases/{}", &digest[..16])));
981 }
982
983 #[tokio::test]
984 async fn stage_and_gate_marks_the_run_failed_and_does_not_advance_when_a_gate_is_red() {
985 // ManualConfirm with no prior confirmation row blocks deterministically.
986 let (pool, cfg, topo, art, run_id, _version, _tmp) =
987 stage_fixture(vec![Gate::ManualConfirm]).await;
988 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
989
990 // A red gate is a pipeline outcome, not an error: the fn records the
991 // failure and returns Ok so the spawned task settles the run cleanly.
992 stage_and_gate(
993 pool.clone(),
994 cfg,
995 topo,
996 art,
997 crate::events::channel(),
998 run_id,
999 deploy_lock,
1000 )
1001 .await
1002 .expect("a red gate settles the run, it does not error out");
1003
1004 // Tier did NOT advance — still the seeded NULL/NULL.
1005 let (current, previous) = tier_versions(&pool, "host").await;
1006 assert_eq!(current, None);
1007 assert_eq!(previous, None);
1008
1009 // Run settled red with a non-empty summary.
1010 let (result, summary) = run_result(&pool, run_id).await;
1011 assert_eq!(result, "failed");
1012 assert!(
1013 summary.as_deref().is_some_and(|s| !s.is_empty()),
1014 "failed run must carry a summary, got {summary:?}"
1015 );
1016 }
1017
1018 #[test]
1019 fn check_build_host_accepts_matching_host() {
1020 assert!(check_build_host("fw13", "fw13").is_ok());
1021 }
1022
1023 #[test]
1024 fn check_build_host_refuses_mismatched_host() {
1025 // A daemon misdeployed onto prod (e.g. a Hetzner host) must refuse.
1026 let err = check_build_host("alpha-west-1", "fw13")
1027 .unwrap_err()
1028 .to_string();
1029 assert!(err.contains("refusing to build"), "{err}");
1030 assert!(
1031 err.contains("alpha-west-1") && err.contains("fw13"),
1032 "{err}"
1033 );
1034 }
1035
1036 #[test]
1037 fn runtime_hostname_reads_a_nonempty_trimmed_name() {
1038 let h = runtime_hostname().expect("hostname readable on Linux");
1039 assert!(!h.is_empty());
1040 assert_eq!(h, h.trim(), "must be trimmed");
1041 }
1042
1043 #[test]
1044 fn tail_does_not_panic_on_multibyte_boundary() {
1045 // Each '€' is 3 bytes; a byte cap landing mid-codepoint must not panic.
1046 let s = "€".repeat(10); // 30 bytes
1047 for max in 1..=30 {
1048 let out = tail(s.as_bytes(), max);
1049 assert!(out.len() <= max, "max={max} got {} bytes", out.len());
1050 // Result is always valid UTF-8 made only of whole '€'s.
1051 assert!(out.chars().all(|c| c == '€'), "max={max}: {out:?}");
1052 }
1053 }
1054
1055 #[test]
1056 fn tail_returns_whole_input_when_under_cap() {
1057 assert_eq!(tail(b"hello", 100), "hello");
1058 }
1059 }
1060