Skip to main content

max / makenotwork

63.4 KB · 1651 lines History Blame Raw
1 //! Build orchestration: resolve a sha to a worktree, read the server version,
2 //! shell out to `cargo build --release`, record a `versions` row.
3 //!
4 //! Runs as a tokio task spawned from `POST /rebuild`; the HTTP request
5 //! returns the version id immediately and the task drives the rest.
6
7 use crate::config::AppConfig;
8 use crate::deploy;
9 use crate::domain::{GitSha, Platform, RunId, TierId, Version};
10 use crate::gates::{self, GateCtx};
11 use crate::git;
12 use crate::topology::Topology;
13 use anyhow::{Context, Result};
14 use chrono::Utc;
15 use sqlx::SqlitePool;
16 use std::path::{Path, PathBuf};
17 use std::sync::Arc;
18 use tokio::process::Command;
19
20 #[derive(Debug, Clone)]
21 pub struct BuildArtifact {
22 pub version: Version,
23 pub git_sha: GitSha,
24 pub worktree: PathBuf,
25 /// One entry per `cfg.bin_names` in declared order. First is the primary
26 /// (referenced by the systemd unit's ExecStart). Paths are inside the
27 /// worktree's `target/release/`.
28 pub binary_paths: Vec<PathBuf>,
29 /// `(companion name, built binary path)` for each `cfg.companions`, built
30 /// from the same worktree/sha as the server. Staged into the release bundle
31 /// under `companions/<name>/<bin>` and installed by the nodes that opt in.
32 pub companion_paths: Vec<(String, PathBuf)>,
33 }
34
35 /// The live kernel hostname (`/proc/sys/kernel/hostname`, trimmed). Linux-only,
36 /// which Sando is. The build-host guard reads this rather than `$HOSTNAME`
37 /// (not reliably exported) so the check reflects the actual machine.
38 fn runtime_hostname() -> Result<String> {
39 let raw = std::fs::read_to_string("/proc/sys/kernel/hostname")
40 .context("reading /proc/sys/kernel/hostname for the build-host guard")?;
41 Ok(raw.trim().to_string())
42 }
43
44 /// Pure half of the build-host guard: fail unless the live host matches the
45 /// configured build host. Split out from [`enforce_build_host`] so it is unit-
46 /// testable without depending on the test machine's hostname.
47 fn check_build_host(actual: &str, expected: &str) -> Result<()> {
48 anyhow::ensure!(
49 actual == expected,
50 "refusing to build on host {actual}: configured build host is {expected} \
51 (never build on a prod/serving node)",
52 );
53 Ok(())
54 }
55
56 /// Refuse to build unless this daemon is running on the configured build host.
57 fn enforce_build_host(expected: &str) -> Result<()> {
58 check_build_host(&runtime_hostname()?, expected)
59 }
60
61 pub async fn run(
62 pool: SqlitePool,
63 cfg: Arc<AppConfig>,
64 topo: Arc<Topology>,
65 sha: GitSha,
66 events: crate::events::EventTx,
67 run_id: RunId,
68 ) -> Result<BuildArtifact> {
69 // Build-host guard: refuse to compile anywhere but the configured build
70 // host. The build runs locally (`cargo build` in cfg.workdir), so a sandod
71 // misdeployed onto a prod/serving node would otherwise build there — exactly
72 // the "never build on prod" rule. Enforced before any cargo invocation so a
73 // wrong-host daemon fails fast with a clear message rather than compiling.
74 let Some(build_host) = cfg.build_host.as_deref() else {
75 anyhow::bail!(
76 "{} declares no build_host, which makes it intake-only: Sando does not compile \
77 it. Ship it with POST /intake, from a builder that does.",
78 cfg.id
79 );
80 };
81 enforce_build_host(build_host)?;
82
83 let repo = topo.repo.as_ref().with_context(|| {
84 format!(
85 "{} declares no [repo]: it is intake-only and Sando has no source to check out",
86 cfg.id
87 )
88 })?;
89 let worktree = cfg.workdir.join(sha.as_str());
90 let bare = PathBuf::from(&repo.bare_path);
91
92 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Fetching)
93 .await
94 .ok();
95
96 // Pull-based ingestion: if an upstream remote is configured, fetch the
97 // deploy branch so a just-pushed sha is locally resolvable. A fetch
98 // failure is non-fatal — the sha may already be present from a prior
99 // fetch or a direct push; the presence check below is the real gate.
100 if let Some(upstream) = repo.upstream.as_deref()
101 && let Err(e) = git::fetch_upstream(&bare, upstream, &repo.branch).await
102 {
103 tracing::warn!(error = %e, upstream, "upstream fetch failed; proceeding with current bare-repo state");
104 }
105 anyhow::ensure!(
106 git::sha_present(&bare, sha.as_str()).await?,
107 "sha {} not present in bare repo {} after fetch — push the commit to the upstream remote first",
108 sha.as_str(),
109 bare.display(),
110 );
111
112 git::checkout_worktree(&bare, sha.as_str(), &worktree).await?;
113
114 // Check out any auxiliary repos (e.g. synckit) beside the worktree so a
115 // cross-repo path dependency in the server or a companion resolves. Fails the
116 // build if an aux repo can't be assembled — a companion that silently fails to
117 // find its source would fail the compile downstream with a worse message.
118 checkout_aux_repos(&cfg, &topo).await?;
119
120 let server_dir = worktree.join("server");
121 let version = read_pkg_version(&server_dir.join("Cargo.toml"))
122 .await
123 .with_context(|| format!("reading version from {}/Cargo.toml", server_dir.display()))?;
124 crate::runs::set_version(&pool, run_id, &version).await.ok();
125
126 // sqlx compile-time query checking needs a live DB with the current schema.
127 // We point cargo at the scratch DB and prep it (drop public, re-migrate)
128 // before invoking cargo build. The same DB is reset again by
129 // `migration_dry_run` later if it runs as a gate.
130 let mut cargo_cmd = Command::new("cargo");
131 cargo_cmd
132 .arg("build")
133 .arg("--release")
134 .current_dir(&server_dir)
135 .kill_on_drop(true);
136 // Shared build cache across per-sha worktrees: reuse one target dir so an
137 // incremental diff doesn't clean-compile from scratch. Serialized builds
138 // make this contention-free. Unset → cargo's default per-worktree target/.
139 if let Some(target) = cfg.cargo_target_dir.as_deref() {
140 cargo_cmd.env("CARGO_TARGET_DIR", target);
141 }
142 if let Some(scratch_url) = cfg.scratch_db_url.as_deref() {
143 tracing::info!(sha = %sha.as_str(), "preparing scratch DB schema for sqlx compile-time checks");
144 crate::gates::reset_scratch(scratch_url, &cfg.scratch_owner_role)
145 .await
146 .context("scratch DB reset before build")?;
147 crate::gates::run_migrator(scratch_url, &server_dir.join("migrations"))
148 .await
149 .context("applying MNW migrations to scratch DB before build")?;
150 cargo_cmd.env("DATABASE_URL", scratch_url);
151 } else {
152 tracing::warn!("scratch_db_url unset; sqlx will fall back to offline mode and may fail");
153 }
154
155 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Compiling)
156 .await
157 .ok();
158 tracing::info!(sha = %sha, version = %version, dir = %server_dir.display(), "cargo build --release start");
159 crate::events::emit(
160 &events,
161 crate::events::Event::BuildStart {
162 sha: sha.clone(),
163 version: version.clone(),
164 },
165 );
166 let started = std::time::Instant::now();
167 let out = cargo_cmd.output().await.context("spawning cargo build")?;
168 let elapsed_s = started.elapsed().as_secs();
169 if !out.status.success() {
170 tracing::error!(sha = %sha, version = %version, elapsed_s, "cargo build --release failed");
171 crate::events::emit(
172 &events,
173 crate::events::Event::BuildFailed {
174 sha: sha.clone(),
175 version: version.clone(),
176 elapsed_s,
177 },
178 );
179 // Settle the run with the headline compiler diagnostic (not the raw
180 // 4 KB tail) so `GET /runs/{id}` answers "why" without a journald dive.
181 let summary = crate::classify::classify_compile_error(&out.stdout, &out.stderr).summary();
182 if let Err(e) = crate::runs::mark_failed(&pool, run_id, &summary).await {
183 tracing::error!(run_id = %run_id, error = %e, "persisting compile-fail verdict failed; run may show stale 'building' until restart-reconcile");
184 }
185 anyhow::bail!(
186 "cargo build --release failed:\n{}",
187 tail(&out.stderr, 4_000)
188 );
189 }
190 tracing::info!(sha = %sha, version = %version, elapsed_s, "cargo build --release ok");
191 crate::events::emit(
192 &events,
193 crate::events::Event::BuildOk {
194 sha: sha.clone(),
195 version: version.clone(),
196 elapsed_s,
197 },
198 );
199
200 // Binaries land under `<target>/release/`; with a shared target dir that's
201 // not inside the worktree, so resolve it the same way cargo did above.
202 let release_dir = cfg
203 .cargo_target_dir
204 .as_deref()
205 .map_or_else(|| server_dir.join("target/release"), |t| t.join("release"));
206 let mut binary_paths = Vec::with_capacity(cfg.bin_names.len());
207 for name in &cfg.bin_names {
208 let p = release_dir.join(name);
209 anyhow::ensure!(p.exists(), "expected binary at {} after build", p.display());
210 binary_paths.push(p);
211 }
212 // Primary binary path is the one we record in `versions.artifact_path`
213 // (everything downstream — promote, rollback — looks it up by version).
214 let primary = binary_paths[0].clone();
215
216 // Companion crates (e.g. mnw-cli): built from the SAME worktree/sha so a
217 // service that shares the server's internal-API contract cannot drift out of
218 // lockstep (the 2026-07-09 git-hosting outage). A companion build failure
219 // fails the whole pipeline — the server never ships without its companions.
220 let mut companion_paths = Vec::with_capacity(cfg.companions.len());
221 for c in &cfg.companions {
222 let bin = build_companion(&worktree, &cfg, c).await?;
223 companion_paths.push((c.name.clone(), bin));
224 }
225
226 sqlx::query(
227 "INSERT OR IGNORE INTO versions (app, version, git_sha, built_at, artifact_path)
228 VALUES (?, ?, ?, ?, ?)",
229 )
230 .bind(&cfg.id)
231 .bind(&version)
232 .bind(&sha)
233 .bind(Utc::now().to_rfc3339())
234 .bind(primary.to_string_lossy().as_ref())
235 .execute(&pool)
236 .await?;
237
238 Ok(BuildArtifact {
239 version,
240 git_sha: sha,
241 worktree,
242 binary_paths,
243 companion_paths,
244 })
245 }
246
247 /// Fetch and check out every configured auxiliary repo at `cfg.workdir/<checkout_dir>`,
248 /// so a cross-repo path dependency built from the main worktree resolves (wiki
249 /// [[sando-overview]]; the synckit split, task sando-18cdb32f).
250 ///
251 /// Each aux repo is a fixed, shared checkout refreshed to `branch` HEAD — not
252 /// per-sha — because the dependent's relative path resolves to that fixed spot
253 /// regardless of the main sha, and builds serialize. A fetch failure is a warning
254 /// (the branch may already be present from a prior build); an unresolvable branch
255 /// after that is fatal, as is a failed worktree — a half-assembled source tree
256 /// must fail the build here, loudly, not as a downstream compile error.
257 /// Where an aux repo's checkout lands. The single derivation: `checkout_aux_repos`
258 /// creates it here and `GateCtx::aux_dirs` resolves `test_target`s against it, so
259 /// the two cannot drift into looking in different places.
260 pub fn aux_checkout_dir(cfg: &AppConfig, aux: &crate::topology::AuxRepo) -> PathBuf {
261 cfg.workdir.join(&aux.checkout_dir)
262 }
263
264 /// Every aux repo's checkout dir, keyed by name — what `GateCtx::aux_dirs` holds.
265 pub fn aux_checkout_dirs(
266 cfg: &AppConfig,
267 topo: &Topology,
268 ) -> std::collections::HashMap<String, PathBuf> {
269 topo.aux_repos
270 .iter()
271 .map(|a| (a.name.clone(), aux_checkout_dir(cfg, a)))
272 .collect()
273 }
274
275 pub async fn checkout_aux_repos(cfg: &AppConfig, topo: &Topology) -> Result<()> {
276 for aux in &topo.aux_repos {
277 let bare = PathBuf::from(&aux.bare_path);
278 git::ensure_bare_repo_no_hook(&bare)
279 .await
280 .with_context(|| format!("aux repo {}: init bare {}", aux.name, aux.bare_path))?;
281 if let Err(e) = git::fetch_upstream(&bare, &aux.upstream, &aux.branch).await {
282 tracing::warn!(
283 aux = %aux.name, error = %e,
284 "aux repo fetch failed; proceeding with current bare-repo state",
285 );
286 }
287 let sha = git::resolve_ref(&bare, &aux.branch).await.with_context(|| {
288 format!(
289 "aux repo {}: branch {} not resolvable after fetch — is {} reachable with that branch?",
290 aux.name, aux.branch, aux.upstream,
291 )
292 })?;
293 let dest = aux_checkout_dir(cfg, aux);
294 git::checkout_worktree(&bare, &sha, &dest)
295 .await
296 .with_context(|| {
297 format!(
298 "aux repo {}: checking out {} ({}) at {}",
299 aux.name,
300 aux.branch,
301 sha,
302 dest.display()
303 )
304 })?;
305 tracing::info!(
306 aux = %aux.name, branch = %aux.branch, sha = %sha, dest = %dest.display(),
307 "aux repo checked out beside worktree",
308 );
309 }
310 Ok(())
311 }
312
313 /// Build one companion crate from the worktree, returning its release binary
314 /// path. Mirrors the server build's target-dir handling (shared
315 /// `cargo_target_dir` when set, for incremental reuse; else the crate's own
316 /// `target/`). A companion is an API client, not a sqlx crate, so it needs no
317 /// scratch DB. A non-zero exit propagates and fails the pipeline.
318 async fn build_companion(
319 worktree: &Path,
320 cfg: &AppConfig,
321 c: &crate::config::Companion,
322 ) -> Result<PathBuf> {
323 let dir = worktree.join(&c.manifest_dir);
324 anyhow::ensure!(
325 dir.join("Cargo.toml").exists(),
326 "companion {}: no Cargo.toml at {}",
327 c.name,
328 dir.display(),
329 );
330 // Match the server build: no `--locked` (the pipeline builds whatever the
331 // sha pins; a stale lock shouldn't block a deploy the server build allows).
332 let mut cmd = Command::new("cargo");
333 cmd.arg("build")
334 .arg("--release")
335 .current_dir(&dir)
336 .kill_on_drop(true);
337 let release_dir = if let Some(target) = cfg.cargo_target_dir.as_deref() {
338 cmd.env("CARGO_TARGET_DIR", target);
339 target.join("release")
340 } else {
341 dir.join("target/release")
342 };
343 tracing::info!(companion = %c.name, dir = %dir.display(), "cargo build --release (companion) start");
344 let started = std::time::Instant::now();
345 let out = cmd
346 .output()
347 .await
348 .context("spawning cargo build for companion")?;
349 if !out.status.success() {
350 anyhow::bail!(
351 "companion {} build failed:\n{}",
352 c.name,
353 tail(&out.stderr, 4_000),
354 );
355 }
356 let bin = release_dir.join(&c.bin);
357 anyhow::ensure!(
358 bin.exists(),
359 "companion {} produced no binary at {} after build",
360 c.name,
361 bin.display(),
362 );
363 tracing::info!(companion = %c.name, elapsed_s = started.elapsed().as_secs(), "companion build ok");
364 Ok(bin)
365 }
366
367 /// Full host-tier pipeline: build, stage the bundle into the host's
368 /// release_root, run the host tier's configured gates, advance tier_state
369 /// for "host" if all pass. Errors propagate back to the spawned task and
370 /// get logged. (Tier was called "mm" pre-Session-1; renamed to "host"
371 /// since sandod runs on whatever machine ends up being the Sando host.)
372 pub async fn build_and_run_host(
373 pool: SqlitePool,
374 cfg: Arc<AppConfig>,
375 topo: Arc<Topology>,
376 sha: GitSha,
377 events: crate::events::EventTx,
378 run_id: RunId,
379 deploy_lock: Arc<tokio::sync::Mutex<()>>,
380 ) -> Result<()> {
381 let art = run(
382 pool.clone(),
383 cfg.clone(),
384 topo.clone(),
385 sha,
386 events.clone(),
387 run_id,
388 )
389 .await?;
390
391 stage_and_gate(pool, cfg, topo, art, events, run_id, deploy_lock).await
392 }
393
394 /// A bundle assembled on disk and not yet published: the point both the build
395 /// path and the intake path have to reach before anything else can happen to it.
396 ///
397 /// The two paths reach it differently. A build assembles it out of a worktree
398 /// (binaries, `release_contents`, companions); an intake is handed it already
399 /// assembled, with no worktree anywhere. Everything after this point — hashing,
400 /// publishing, recording identity, gating, advancing — is the same work, and
401 /// used to be welded to the assembling half inside `stage_and_gate`.
402 struct StagedBundle {
403 version: Version,
404 /// The staging dir under `release_root/staging/`, pre-publish.
405 staging: PathBuf,
406 /// What the bundle is built to run on, when it is known. A Sando build
407 /// inherits the app's declared platform; an intake takes it from the
408 /// record's provenance.
409 platform: Option<Platform>,
410 }
411
412 /// Post-build half of the host pipeline: stage the artifact into the host's
413 /// release_root, run the host tier's gates, and advance `tier_state` for
414 /// "host" iff all pass. Split from [`build_and_run_host`] at the `run()`
415 /// boundary so the staging/gating/advance logic is reachable in tests from a
416 /// synthetic [`BuildArtifact`] — no real `cargo build --release` required.
417 pub async fn stage_and_gate(
418 pool: SqlitePool,
419 cfg: Arc<AppConfig>,
420 topo: Arc<Topology>,
421 art: BuildArtifact,
422 events: crate::events::EventTx,
423 run_id: RunId,
424 deploy_lock: Arc<tokio::sync::Mutex<()>>,
425 ) -> Result<()> {
426 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Staging)
427 .await
428 .ok();
429 let staged = assemble_from_source(&cfg, &art, run_id).await?;
430 let published = publish(&pool, &cfg, staged, run_id).await?;
431 record_and_gate(
432 pool,
433 cfg,
434 topo,
435 published,
436 events,
437 run_id,
438 deploy_lock,
439 Some(art.worktree),
440 )
441 .await
442 }
443
444 /// Accept an artifact built elsewhere and take it through the same host-tier
445 /// gating and advance a Sando-built one gets.
446 ///
447 /// This is the whole point of the seam. `intake::accept` publishes the bundle
448 /// content-addressed once it has proved the bytes are the ones the record
449 /// vouches for, which lands it at exactly the state [`publish`] leaves a
450 /// Sando-built bundle in — so the two paths join at [`record_and_gate`] and
451 /// nothing downstream knows or cares which one it came from.
452 ///
453 /// `staged` must already sit under `release_root/staging/` (publishing is an
454 /// atomic same-filesystem rename); getting the bytes there is the transport's
455 /// job, not this function's.
456 #[allow(clippy::too_many_arguments)]
457 pub async fn accept_intake(
458 pool: &SqlitePool,
459 cfg: &AppConfig,
460 staged: &Path,
461 record_json: &str,
462 run_id: RunId,
463 ) -> Result<Published> {
464 crate::runs::set_phase(pool, run_id, crate::runs::Phase::Staging)
465 .await
466 .ok();
467
468 let pinned = crate::retention::pinned_dirs(pool, &cfg.id).await?;
469 let accepted = crate::intake::accept(&cfg.release_root, staged, record_json, &pinned)
470 .await
471 .map_err(|e| anyhow::anyhow!("{e}"))?;
472
473 let version = Version::parse(&accepted.record.provenance.version).with_context(|| {
474 format!(
475 "artifact record carries version `{}`, which is not semver",
476 accepted.record.provenance.version
477 )
478 })?;
479 let platform = Platform::parse(&accepted.record.provenance.target).with_context(|| {
480 format!(
481 "artifact record carries target `{}`, which is not `os/arch`",
482 accepted.record.provenance.target
483 )
484 })?;
485 let git_sha = GitSha::parse(&accepted.record.provenance.git_sha).with_context(|| {
486 format!(
487 "artifact record carries git_sha `{}`",
488 accepted.record.provenance.git_sha
489 )
490 })?;
491
492 crate::runs::set_version(pool, run_id, &version).await.ok();
493 upsert_version_row(
494 pool,
495 &cfg.id,
496 &version,
497 &git_sha,
498 &accepted.released.join(cfg.primary_bin()),
499 )
500 .await?;
501
502 let published = Published {
503 version,
504 released: accepted.released,
505 digest_full: accepted.record.digest.to_string(),
506 platform: Some(platform),
507 };
508 record_identity(pool, cfg, &published, run_id).await?;
509 Ok(published)
510 }
511
512 /// Gate an artifact that has already been accepted.
513 ///
514 /// Split from [`accept_intake`] so the two can be answered on different clocks.
515 /// Acceptance is fast and is the producer's business — it either believes the
516 /// bytes or names the file that drifted — so the caller waits for it and gets
517 /// the verdict. Gating is Sando's business and can take an hour, so the caller
518 /// does not.
519 ///
520 /// An intake carries no worktree, and the gates that need one refuse rather than
521 /// resolve against nothing. That is the boundary showing up in the type:
522 /// artifact-scoped gates belong to the builder (wiki [[sando-bento-boundary]]),
523 /// so a tier that asks Sando to re-run them against an accepted artifact is
524 /// misconfigured and should be told so.
525 pub async fn gate_intake(
526 pool: SqlitePool,
527 cfg: Arc<AppConfig>,
528 topo: Arc<Topology>,
529 published: Published,
530 events: crate::events::EventTx,
531 run_id: RunId,
532 deploy_lock: Arc<tokio::sync::Mutex<()>>,
533 ) -> Result<()> {
534 record_and_gate(
535 pool,
536 cfg,
537 topo,
538 published,
539 events,
540 run_id,
541 deploy_lock,
542 None,
543 )
544 .await
545 }
546
547 /// Record the `versions` label row for an artifact that arrived rather than was
548 /// built here. The build path writes its own inside [`run`]; this is the same
549 /// row for the path that never ran a compiler.
550 async fn upsert_version_row(
551 pool: &SqlitePool,
552 app: &crate::domain::AppId,
553 version: &Version,
554 git_sha: &GitSha,
555 artifact_path: &Path,
556 ) -> Result<()> {
557 sqlx::query(
558 "INSERT OR IGNORE INTO versions (app, version, git_sha, built_at, artifact_path)
559 VALUES (?, ?, ?, ?, ?)",
560 )
561 .bind(app)
562 .bind(version)
563 .bind(git_sha)
564 .bind(Utc::now().to_rfc3339())
565 .bind(artifact_path.to_string_lossy().as_ref())
566 .execute(pool)
567 .await?;
568 Ok(())
569 }
570
571 /// Assemble a bundle out of a worktree: binaries, `release_contents`, companions.
572 async fn assemble_from_source(
573 cfg: &AppConfig,
574 art: &BuildArtifact,
575 run_id: RunId,
576 ) -> Result<StagedBundle> {
577 // Stage the bundle into `staging/<build_id>/` — a private scratch dir, not
578 // yet a release. It is published content-addressed below, once its digest is
579 // known. This is what makes overwrite unexpressible (wiki
580 // [[release-artifact-identity]]): a build never touches another build's dir.
581 let staging =
582 deploy::stage_local_bundle(&cfg.release_root, run_id.0, &art.binary_paths).await?;
583
584 // Stage every entry from cfg.release_contents into the staged bundle. This is
585 // how non-binary version-coupled content (static assets, docs, error-pages,
586 // ...) makes it into the atomic deploy bundle. Projects opt in via daemon
587 // config — the sando code carries no MNW-specific knowledge.
588 for entry in &cfg.release_contents {
589 stage_entry(&art.worktree, &staging, entry).await?;
590 }
591
592 // Stage companion binaries as `companions/<name>` (the file itself) so they
593 // ride the same atomic bundle rsync to the nodes, and a node can locate its
594 // companion source from the logical name alone — no bin-filename coupling in
595 // the topology. The nodes that opt in install them post-swap (see
596 // deploy::deploy_remote).
597 if !art.companion_paths.is_empty() {
598 let dst_dir = staging.join("companions");
599 tokio::fs::create_dir_all(&dst_dir)
600 .await
601 .with_context(|| format!("create staged companions dir {}", dst_dir.display()))?;
602 for (name, built) in &art.companion_paths {
603 let dst = dst_dir.join(name);
604 tokio::fs::copy(built, &dst).await.with_context(|| {
605 format!(
606 "stage companion {name}: {} -> {}",
607 built.display(),
608 dst.display()
609 )
610 })?;
611 }
612 }
613
614 Ok(StagedBundle {
615 version: art.version.clone(),
616 staging,
617 platform: cfg.platform.clone(),
618 })
619 }
620
621 /// A bundle that has been hashed and published content-addressed. Both paths
622 /// produce one; nothing downstream can tell them apart.
623 ///
624 /// Public because the intake route now hands one from `accept_intake` to
625 /// `gate_intake`: proving the bytes answers the producer, gating them does not,
626 /// so the two run on different clocks and the value passes between them.
627 #[derive(Debug)]
628 pub struct Published {
629 version: Version,
630 released: PathBuf,
631 digest_full: String,
632 platform: Option<Platform>,
633 }
634
635 /// Hash the assembled bundle, write its MANIFEST, and publish it at
636 /// `releases/<digest16>`.
637 ///
638 /// The intake path does not call this: `intake::accept` does the same three
639 /// steps itself, because it has to hash the bytes to verify them and hashing
640 /// them twice would be the one place the two implementations could disagree.
641 async fn publish(
642 pool: &SqlitePool,
643 cfg: &AppConfig,
644 staged: StagedBundle,
645 run_id: RunId,
646 ) -> Result<Published> {
647 // Content identity: hash the fully-staged bundle, write its MANIFEST into the
648 // bundle (for node-side verification), then publish it at `releases/<digest16>`.
649 // The digest is now load-bearing — a hashing failure fails the build rather
650 // than shipping an unidentifiable artifact.
651 let digest = crate::bundle::digest_dir(&staged.staging)
652 .await
653 .context("hashing the staged bundle for content addressing")?;
654 tokio::fs::write(
655 staged.staging.join(crate::bundle::MANIFEST_NAME),
656 digest.manifest.as_bytes(),
657 )
658 .await
659 .context("writing bundle MANIFEST")?;
660 let pinned = crate::retention::pinned_dirs(pool, &cfg.id).await?;
661 let released =
662 deploy::finalize_local_release(&cfg.release_root, &staged.staging, digest.short(), &pinned)
663 .await?;
664
665 let staged_bin = released.join(cfg.primary_bin());
666 sqlx::query("UPDATE versions SET artifact_path = ? WHERE app = ? AND version = ?")
667 .bind(staged_bin.to_string_lossy().as_ref())
668 .bind(&cfg.id)
669 .bind(&staged.version)
670 .execute(pool)
671 .await?;
672
673 let published = Published {
674 version: staged.version,
675 released,
676 digest_full: digest.full,
677 platform: staged.platform,
678 };
679 record_identity(pool, cfg, &published, run_id).await?;
680 Ok(published)
681 }
682
683 /// Record the identity on the build row: the digest, the content-addressed dir
684 /// the bundle was published to, and what it runs on. This is what promote
685 /// resolves the artifact through, and burn-in/retention key on.
686 async fn record_identity(
687 pool: &SqlitePool,
688 cfg: &AppConfig,
689 published: &Published,
690 run_id: RunId,
691 ) -> Result<()> {
692 let released_path = published.released.to_string_lossy();
693 crate::runs::set_identity(pool, run_id, &published.digest_full, &released_path)
694 .await
695 .ok();
696 // Platform is what lets two bundles of one version be told apart, so a
697 // dropped write here would leave a pom artifact that can be placed nowhere
698 // (a node declaring a platform refuses an artifact that records none). Fail
699 // rather than ship an unplaceable bundle.
700 if let Some(p) = &published.platform {
701 crate::runs::set_platform(pool, run_id, p)
702 .await
703 .with_context(|| format!("recording platform {p} for {}", cfg.id))?;
704 }
705 Ok(())
706 }
707
708 /// The shared tail of both paths: run the host tier's gates against a published
709 /// bundle and advance `tier_state` iff all pass.
710 #[allow(clippy::too_many_arguments)]
711 async fn record_and_gate(
712 pool: SqlitePool,
713 cfg: Arc<AppConfig>,
714 topo: Arc<Topology>,
715 published: Published,
716 events: crate::events::EventTx,
717 run_id: RunId,
718 deploy_lock: Arc<tokio::sync::Mutex<()>>,
719 worktree: Option<PathBuf>,
720 ) -> Result<()> {
721 let host = topo
722 .tiers
723 .iter()
724 .find(|t| t.name.as_str() == "host")
725 .context("topology has no `host` tier")?;
726
727 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Gating)
728 .await
729 .ok();
730 let ctx = GateCtx {
731 pool: pool.clone(),
732 cfg: cfg.clone(),
733 tier: TierId::new("host"),
734 version: published.version.clone(),
735 worktree,
736 // The published bundle. `migration_dry_run` prefers it over the
737 // worktree, so the migrations it proves are the ones inside the digest
738 // rather than ones sitting beside them in a checkout.
739 bundle: Some(published.released.clone()),
740 events: events.clone(),
741 // Host runs build-time gates (cargo_test / migration_dry_run /
742 // boot_smoke) only — `node_health` never appears here, so there are no
743 // nodes to probe.
744 nodes: Vec::new(),
745 // These gates vouch for this build; record its id so promote can resolve
746 // the artifact through the evidence rather than a version string.
747 build_id: Some(run_id.0),
748 // Where checkout_aux_repos put each aux repo, so a test_target naming
749 // one resolves. Shared derivation, so the two cannot disagree.
750 public_url: None,
751 aux_dirs: aux_checkout_dirs(&cfg, &topo),
752 };
753 let failed = gates::run_all(&ctx, &host.gates).await?;
754
755 if failed.is_empty() {
756 // Advance the host tier through the single sealed forward-advance op, under
757 // deploy_lock so this can't interleave with a concurrent `/rollback host`
758 // (the old fetch-then-write here was the one CF3 site outside the lock —
759 // ultra-fuzz Run 2, S1). Held only for the atomic UPDATE, never the gates.
760 {
761 let _deploy_guard = deploy_lock.lock().await;
762 crate::runs::advance_tier(&pool, &cfg.id, "host", &published.version, Some(run_id.0))
763 .await?;
764 }
765 // Terminal verdict: unlike the phase pings above (best-effort), a dropped
766 // pass/fail write leaves the run wedged at `building`. Log it loudly if it
767 // fails — the startup reconcile (main) is the backstop that settles such a
768 // row on the next restart.
769 if let Err(e) = crate::runs::mark_passed(&pool, run_id).await {
770 tracing::error!(run_id = %run_id, error = %e, "persisting host-green verdict failed; run may show stale 'building' until restart-reconcile");
771 }
772 tracing::info!(version = %published.version, "host pipeline green; ready to promote to next tier");
773 } else {
774 // Pull the first red gate's typed summary into the run so the API
775 // answers "which gate, and why" — not just "failed".
776 let summary = crate::runs::first_failed_gate_summary(&pool, &cfg.id, run_id)
777 .await
778 .unwrap_or_else(|| "host pipeline red".to_string());
779 if let Err(e) = crate::runs::mark_failed(&pool, run_id, &summary).await {
780 tracing::error!(run_id = %run_id, error = %e, "persisting host-red verdict failed; run may show stale 'building' until restart-reconcile");
781 }
782 tracing::warn!(version = %published.version, "host pipeline red; not advancing tier_state");
783 }
784 Ok(())
785 }
786
787 async fn read_pkg_version(cargo_toml: &Path) -> Result<Version> {
788 let raw = tokio::fs::read_to_string(cargo_toml).await?;
789 let parsed: toml::Value = toml::from_str(&raw)?;
790 let v = parsed
791 .get("package")
792 .and_then(|p| p.get("version"))
793 .and_then(|v| v.as_str())
794 .context("package.version not found")?;
795 Version::parse(v).with_context(|| format!("parsing package.version `{v}`"))
796 }
797
798 fn tail(buf: &[u8], max: usize) -> String {
799 let s = String::from_utf8_lossy(buf);
800 if s.len() <= max {
801 return s.into_owned();
802 }
803 // `s.len() - max` can land mid-codepoint; walk forward to the next char
804 // boundary so the slice never panics (returns slightly fewer than `max`
805 // bytes in that case). `floor_char_boundary` is still unstable, so do it by
806 // hand.
807 let mut start = s.len() - max;
808 while start < s.len() && !s.is_char_boundary(start) {
809 start += 1;
810 }
811 s[start..].to_string()
812 }
813
814 /// Copy `worktree/<entry.src>` into `staged/<entry.dst>`. Handles file or
815 /// directory sources transparently. Missing source policy depends on
816 /// `entry.required`:
817 /// - required=true -> error (build fails)
818 /// - required=false -> log warn + skip (e.g. older shas missing a dir)
819 ///
820 /// Uses `cp -a` to preserve modes/symlinks/etc; parent of dst is created if
821 /// needed so entries like `dst = "docs/assumptions.toml"` work without
822 /// extra config.
823 async fn stage_entry(
824 worktree: &Path,
825 staged: &Path,
826 entry: &crate::config::ReleaseEntry,
827 ) -> Result<()> {
828 let src = worktree.join(&entry.src);
829 let dst = staged.join(&entry.dst);
830 if !src.exists() {
831 if entry.required {
832 anyhow::bail!(
833 "required release_contents source missing: {}",
834 src.display()
835 );
836 }
837 tracing::warn!(src = %src.display(), "release_contents source missing (optional); skipping");
838 return Ok(());
839 }
840 if let Some(parent) = dst.parent() {
841 tokio::fs::create_dir_all(parent)
842 .await
843 .with_context(|| format!("create staged parent {}", parent.display()))?;
844 }
845 // Multiple entries with the same dst (e.g. site-docs/public/ +
846 // site-docs/examples/ both landing under docs/) need additive merging.
847 // `cp -a SRC/. DST/` copies SRC's contents into DST without overwriting
848 // the dst dir itself; that's the merge-friendly form when dst is a dir
849 // that may already exist from a prior entry. For non-dir sources or a
850 // missing dst we fall back to the plain `cp -a SRC DST` form.
851 let merge_into_existing_dir = src.is_dir() && dst.is_dir();
852 let mut cmd = Command::new("cp");
853 cmd.arg("-a");
854 if merge_into_existing_dir {
855 let mut src_arg = src.clone().into_os_string();
856 src_arg.push("/.");
857 cmd.arg(src_arg);
858 let mut dst_arg = dst.clone().into_os_string();
859 dst_arg.push("/");
860 cmd.arg(dst_arg);
861 } else {
862 cmd.arg(&src).arg(&dst);
863 }
864 let out = cmd
865 .output()
866 .await
867 .with_context(|| format!("spawning cp for {} -> {}", src.display(), dst.display()))?;
868 anyhow::ensure!(
869 out.status.success(),
870 "stage {} -> {}: {}",
871 src.display(),
872 dst.display(),
873 String::from_utf8_lossy(&out.stderr),
874 );
875 Ok(())
876 }
877
878 #[cfg(test)]
879 mod tests {
880 use super::{
881 BuildArtifact, accept_intake, check_build_host, checkout_aux_repos, gate_intake,
882 runtime_hostname, stage_and_gate, tail,
883 };
884 use crate::config::{AppConfig, TestTarget};
885 use crate::domain::{GitSha, RunId, Version};
886 use crate::topology::{AuxRepo, BackupConfig, CanaryPolicy, Gate, RepoConfig, Tier, Topology};
887 use sqlx::SqlitePool;
888 use sqlx::sqlite::SqlitePoolOptions;
889 use std::collections::BTreeMap;
890 use std::path::PathBuf;
891 use std::sync::Arc;
892
893 /// Post-build pipeline fixture: an in-memory store with the `host` tier
894 /// seeded, a synthetic worktree holding a fake primary binary, and a
895 /// build_runs row in flight. Returns everything `stage_and_gate` needs plus
896 /// the tempdir root (drop it to clean up) and the run/version it seeded.
897 ///
898 /// `gates` is the host tier's gate list: `[]` is the green path;
899 /// `[Gate::ManualConfirm]` is a deterministic red — with no prior operator
900 /// confirmation row that gate blocks, and it shells out to nothing.
901 async fn stage_fixture(
902 gates: Vec<Gate>,
903 ) -> (
904 SqlitePool,
905 Arc<AppConfig>,
906 Arc<Topology>,
907 BuildArtifact,
908 RunId,
909 Version,
910 tempfile::TempDir,
911 ) {
912 let tmp = tempfile::tempdir().unwrap();
913 let release_root = tmp.path().join("release-root");
914 let worktree = tmp.path().join("worktree");
915 let bin_dir = worktree.join("target").join("release");
916 tokio::fs::create_dir_all(&bin_dir).await.unwrap();
917 let bin_path = bin_dir.join("makenotwork");
918 tokio::fs::write(&bin_path, b"#!/bin/false\nfake sando artifact\n")
919 .await
920 .unwrap();
921
922 let pool = SqlitePoolOptions::new()
923 .max_connections(1)
924 .connect("sqlite::memory:")
925 .await
926 .unwrap();
927 sqlx::migrate!("./migrations").run(&pool).await.unwrap();
928
929 // gate_runs and tier_state FK into `tiers`; the pipeline only touches host.
930 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('host', 0, 1, 'sequential')")
931 .execute(&pool)
932 .await
933 .unwrap();
934 sqlx::query("INSERT INTO tier_state (tier) VALUES ('host')")
935 .execute(&pool)
936 .await
937 .unwrap();
938
939 let version = Version::parse("1.2.3").unwrap();
940 let git_sha = GitSha::parse("abc1234").unwrap();
941 // gate_runs.version and the `SET artifact_path` UPDATE both need the row.
942 sqlx::query(
943 "INSERT INTO versions (version, git_sha, built_at, artifact_path)
944 VALUES (?, ?, datetime('now'), '')",
945 )
946 .bind(version.to_string())
947 .bind(git_sha.to_string())
948 .execute(&pool)
949 .await
950 .unwrap();
951
952 let run_id = crate::runs::create(
953 &pool,
954 &crate::domain::AppId::default(),
955 &git_sha.to_string(),
956 )
957 .await
958 .unwrap();
959
960 let cfg = AppConfig {
961 page_smoke_cmd: None,
962 platform: None,
963 code_smoke_env: BTreeMap::default(),
964 id: crate::domain::AppId::default(),
965 topology_path: PathBuf::from("/tmp/test-sando.toml"),
966 build_host: Some("test-host".into()),
967 workdir: tmp.path().to_path_buf(),
968 release_root: release_root.clone(),
969 scratch_db_url: None,
970 scratch_owner_role: "makenotwork".into(),
971 boot_smoke_port: 18181,
972 code_smoke_port: 18182,
973 bin_names: vec!["makenotwork".into()],
974 logs_root: tmp.path().join("logs"),
975 release_contents: vec![],
976 cargo_target_dir: None,
977 gate_timeout_secs: 2400,
978 companions: Vec::new(),
979 test_targets: vec![TestTarget {
980 dir: PathBuf::from("server"),
981 aux_repo: None,
982 features: vec!["fast-tests".into()],
983 all_features: false,
984 scratch_db: true,
985 }],
986 migration_checks: vec![],
987 frontend_builds: vec![],
988 backup_max_age_hours: 48,
989 };
990
991 let topo = Topology {
992 repo: Some(RepoConfig {
993 bare_path: "/tmp/test.git".into(),
994 branch: "main".into(),
995 upstream: None,
996 }),
997 backup: vec![BackupConfig {
998 name: "server".into(),
999 source: "file:///tmp/test-backup.sql".into(),
1000 local_path: "/tmp/local-backup.sql".into(),
1001 }],
1002 tiers: vec![Tier {
1003 public_url: None,
1004 name: "host".into(),
1005 provisioned: true,
1006 gates,
1007 canary: CanaryPolicy::Sequential,
1008 nodes: Vec::new(),
1009 }],
1010 aux_repos: Vec::new(),
1011 };
1012
1013 let art = BuildArtifact {
1014 version: version.clone(),
1015 git_sha,
1016 worktree,
1017 binary_paths: vec![bin_path],
1018 companion_paths: Vec::new(),
1019 };
1020
1021 (
1022 pool,
1023 Arc::new(cfg),
1024 Arc::new(topo),
1025 art,
1026 run_id,
1027 version,
1028 tmp,
1029 )
1030 }
1031
1032 // ---- intake through the seam ----
1033
1034 /// The `ArtifactRecord` Bento would have written for a staged bundle.
1035 async fn record_for(staged: &std::path::Path, version: &str, target: &str) -> String {
1036 use ops_artifact::{ArtifactRecord, GateRecord, Manifest, Provenance, Scope, Verdict};
1037 let computed = crate::bundle::digest_dir(staged).await.unwrap();
1038 let manifest = Manifest::parse(&computed.manifest).unwrap();
1039 let at = chrono::DateTime::<chrono::Utc>::from_timestamp(1_754_000_000, 0).unwrap();
1040 ArtifactRecord::new(
1041 "bento",
1042 manifest,
1043 Provenance {
1044 app: "pom".into(),
1045 version: version.into(),
1046 tag: format!("pom-v{version}"),
1047 git_sha: "a".repeat(40),
1048 target: target.into(),
1049 build_host: "astra".into(),
1050 toolchain: "rustc 1.97.0".into(),
1051 built_at: at,
1052 },
1053 vec![GateRecord::new(
1054 "prebuild",
1055 Scope::Artifact,
1056 Verdict::Passed,
1057 "prebuild passed in 90s",
1058 at,
1059 )],
1060 )
1061 .unwrap()
1062 .to_json()
1063 }
1064
1065 #[tokio::test]
1066 async fn an_accepted_artifact_is_published_gated_and_advances_the_tier() {
1067 // The seam: an artifact Sando did not build reaches the same published,
1068 // gated, tier-advanced end state a Sando-built one does. No worktree
1069 // exists anywhere in this test, which is the point — everything from
1070 // `finalize_local_release` onward stopped caring where the bytes came
1071 // from.
1072 let (pool, cfg, topo, _art, run_id, _version, tmp) = stage_fixture(vec![]).await;
1073 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1074
1075 let staged = cfg.release_root.join("staging").join("intake-1");
1076 tokio::fs::create_dir_all(&staged).await.unwrap();
1077 tokio::fs::write(staged.join("makenotwork"), b"bytes built elsewhere")
1078 .await
1079 .unwrap();
1080 let record = record_for(&staged, "1.2.3", "linux/aarch64").await;
1081
1082 // Two calls now, on purpose: the route answers its caller on the first
1083 // and spawns the second. Acceptance is what the producer waits for.
1084 let published = accept_intake(&pool, &cfg, &staged, &record, run_id)
1085 .await
1086 .expect("the bytes are believed");
1087 gate_intake(
1088 pool.clone(),
1089 cfg.clone(),
1090 topo,
1091 published,
1092 crate::events::channel(),
1093 run_id,
1094 deploy_lock,
1095 )
1096 .await
1097 .expect("a green intake settles the run");
1098
1099 // Published content-addressed, and the staging dir is gone: renamed,
1100 // not copied.
1101 let (digest, staged_path, platform): (Option<String>, Option<String>, Option<String>) =
1102 sqlx::query_as(
1103 "SELECT bundle_digest, staged_path, platform FROM build_runs WHERE id = ?",
1104 )
1105 .bind(run_id.0)
1106 .fetch_one(&pool)
1107 .await
1108 .unwrap();
1109 let digest = digest.expect("bundle_digest recorded");
1110 let staged_path = staged_path.expect("staged_path recorded");
1111 assert_eq!(digest.len(), 64);
1112 assert!(
1113 !staged.exists(),
1114 "staging was renamed into the release root"
1115 );
1116 assert_eq!(
1117 std::path::Path::new(&staged_path),
1118 tmp.path()
1119 .join("release-root")
1120 .join("releases")
1121 .join(&digest[..16]),
1122 );
1123
1124 // The platform came off the record's provenance and is on the row. This
1125 // is what makes two bundles of one version tellable apart later.
1126 assert_eq!(platform.as_deref(), Some("linux/aarch64"));
1127
1128 // Green gates advanced the host tier, exactly as a build would have.
1129 let (result, _summary) = run_result(&pool, run_id).await;
1130 assert_eq!(result, "passed");
1131 let (current, _prev) = tier_versions(&pool, "host").await;
1132 assert_eq!(current.as_deref(), Some("1.2.3"));
1133 }
1134
1135 #[tokio::test]
1136 async fn an_intake_whose_bytes_drifted_never_reaches_the_gates() {
1137 // Identity is decided before anything else happens to the bundle, so a
1138 // record vouching for one set of bytes arriving with another fails the
1139 // run rather than gating and shipping.
1140 let (pool, cfg, topo, _art, run_id, _version, _tmp) = stage_fixture(vec![]).await;
1141 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1142
1143 let staged = cfg.release_root.join("staging").join("intake-1");
1144 tokio::fs::create_dir_all(&staged).await.unwrap();
1145 tokio::fs::write(staged.join("makenotwork"), b"bytes built elsewhere")
1146 .await
1147 .unwrap();
1148 let record = record_for(&staged, "1.2.3", "linux/aarch64").await;
1149 tokio::fs::write(staged.join("makenotwork"), b"other bytes entirely")
1150 .await
1151 .unwrap();
1152
1153 // Refused by ACCEPTANCE, not by gating — which is what lets the route
1154 // answer the producer with the refusal instead of a `202`-shaped lie.
1155 let _ = (&topo, &deploy_lock);
1156 let err = accept_intake(&pool, &cfg, &staged, &record, run_id)
1157 .await
1158 .expect_err("a drifted bundle is refused");
1159 assert!(err.to_string().contains("makenotwork"), "{err}");
1160
1161 // Nothing advanced, and the bytes were left where they were.
1162 let (current, _prev) = tier_versions(&pool, "host").await;
1163 assert_eq!(current, None);
1164 assert!(staged.exists(), "a refused intake leaves the bytes alone");
1165 }
1166
1167 #[tokio::test]
1168 async fn a_gate_that_reads_source_refuses_against_an_accepted_artifact() {
1169 // The boundary showing up at runtime. `code_smoke` is artifact-scoped —
1170 // it compiles frontends and boots the binary against a scratch DB — and
1171 // an accepted artifact has no checkout for it to read. It has to say so
1172 // rather than pass on having run nothing, which is what an unwrapped
1173 // `worktree.join(..)` against an empty path would have done.
1174 let (pool, cfg, _topo, _art, run_id, version, _tmp) = stage_fixture(vec![]).await;
1175 let ctx = crate::gates::GateCtx {
1176 pool,
1177 cfg,
1178 tier: crate::domain::TierId::new("host"),
1179 version,
1180 worktree: None,
1181 bundle: Some(PathBuf::from("/r/abc")),
1182 events: crate::events::channel(),
1183 nodes: Vec::new(),
1184 build_id: Some(run_id.0),
1185 public_url: None,
1186 aux_dirs: std::collections::HashMap::default(),
1187 };
1188 let outcome = ctx
1189 .worktree_for(crate::domain::GateKind::CodeSmoke)
1190 .expect_err("no worktree means no source-reading gate");
1191 assert!(!outcome.is_passed());
1192 let crate::outcome::GateStatus::Failed { failure } = &outcome.status else {
1193 panic!("expected a failure, got {:?}", outcome.status)
1194 };
1195 assert!(
1196 matches!(failure, crate::outcome::GateFailure::NeedsSource { .. }),
1197 "{failure:?}"
1198 );
1199 assert!(
1200 failure.summary().contains("built elsewhere"),
1201 "{}",
1202 failure.summary()
1203 );
1204 }
1205
1206 #[tokio::test]
1207 async fn migrations_come_from_the_bundle_before_the_worktree() {
1208 // What the gate proves has to be what ships. Migrations staged into the
1209 // bundle are inside its digest; the same files sitting in a checkout are
1210 // not, and a checkout can be edited between the dry run and the deploy.
1211 // So when both hold a copy, the bundle wins.
1212 let (pool, cfg, _topo, _art, run_id, version, tmp) = stage_fixture(vec![]).await;
1213 let bundle = tmp.path().join("bundle");
1214 let worktree = tmp.path().join("wt");
1215 for root in [&bundle, &worktree] {
1216 tokio::fs::create_dir_all(root.join("server/migrations"))
1217 .await
1218 .unwrap();
1219 }
1220 let ctx = crate::gates::GateCtx {
1221 pool,
1222 cfg,
1223 tier: crate::domain::TierId::new("host"),
1224 version,
1225 worktree: Some(worktree.clone()),
1226 bundle: Some(bundle.clone()),
1227 events: crate::events::channel(),
1228 nodes: Vec::new(),
1229 build_id: Some(run_id.0),
1230 public_url: None,
1231 aux_dirs: std::collections::HashMap::default(),
1232 };
1233 assert_eq!(
1234 ctx.migrations_dir(std::path::Path::new("server/migrations")),
1235 Some(bundle.join("server/migrations")),
1236 );
1237
1238 // The worktree is the fallback for a build whose config has not opted
1239 // into bundling them yet, which is every MNW build before this lands.
1240 let ctx = crate::gates::GateCtx {
1241 bundle: Some(tmp.path().join("empty-bundle")),
1242 ..ctx
1243 };
1244 assert_eq!(
1245 ctx.migrations_dir(std::path::Path::new("server/migrations")),
1246 Some(worktree.join("server/migrations")),
1247 );
1248
1249 // And neither is not silently green: an accepted artifact whose builder
1250 // did not bundle its migrations has nothing to dry-run, and the gate
1251 // has to be told so rather than restore a dump and report success.
1252 let ctx = crate::gates::GateCtx {
1253 worktree: None,
1254 ..ctx
1255 };
1256 assert_eq!(
1257 ctx.migrations_dir(std::path::Path::new("server/migrations")),
1258 None
1259 );
1260 }
1261
1262 // ---- checkout_aux_repos ----
1263
1264 async fn git_in(dir: &std::path::Path, args: &[&str]) {
1265 let out = tokio::process::Command::new("git")
1266 .args(["-c", "user.email=t@t", "-c", "user.name=t"])
1267 .current_dir(dir)
1268 .args(args)
1269 .output()
1270 .await
1271 .unwrap();
1272 assert!(
1273 out.status.success(),
1274 "git {args:?}: {}",
1275 String::from_utf8_lossy(&out.stderr)
1276 );
1277 }
1278
1279 /// A minimal `Config` whose only field this test path reads is `workdir`.
1280 fn cfg_with_workdir(workdir: PathBuf) -> AppConfig {
1281 AppConfig {
1282 page_smoke_cmd: None,
1283 platform: None,
1284 code_smoke_env: BTreeMap::default(),
1285 id: crate::domain::AppId::default(),
1286 topology_path: PathBuf::from("/tmp/test-sando.toml"),
1287 build_host: Some("test-host".into()),
1288 workdir,
1289 release_root: PathBuf::from("/tmp/rr"),
1290 scratch_db_url: None,
1291 scratch_owner_role: "makenotwork".into(),
1292 boot_smoke_port: 18181,
1293 code_smoke_port: 18182,
1294 bin_names: vec!["makenotwork".into()],
1295 logs_root: PathBuf::from("/tmp/logs"),
1296 release_contents: vec![],
1297 cargo_target_dir: None,
1298 gate_timeout_secs: 2400,
1299 companions: Vec::new(),
1300 test_targets: vec![],
1301 migration_checks: vec![],
1302 frontend_builds: vec![],
1303 backup_max_age_hours: 48,
1304 }
1305 }
1306
1307 fn topo_with_aux(aux_repos: Vec<AuxRepo>) -> Topology {
1308 Topology {
1309 repo: Some(RepoConfig {
1310 bare_path: "/tmp/x.git".into(),
1311 branch: "main".into(),
1312 upstream: None,
1313 }),
1314 backup: vec![BackupConfig {
1315 name: "server".into(),
1316 source: "s".into(),
1317 local_path: "/tmp/d".into(),
1318 }],
1319 tiers: vec![],
1320 aux_repos,
1321 }
1322 }
1323
1324 #[tokio::test]
1325 async fn a_gate_looks_where_the_aux_checkout_actually_landed() {
1326 // The two halves of the aux-repo test_target path: checkout_aux_repos
1327 // writes the tree, and GateCtx::target_dir reads it. Nothing but this
1328 // stops one from being changed without the other, and the failure would
1329 // be a warn-and-skip — a green gate that ran one crate fewer.
1330 let tmp = tempfile::tempdir().unwrap();
1331 let src = tmp.path().join("docengine-src");
1332 tokio::fs::create_dir_all(&src).await.unwrap();
1333 git_in(&src, &["init", "-q", "-b", "main"]).await;
1334 tokio::fs::write(src.join("Cargo.toml"), b"[package]\nname = \"docengine\"\n")
1335 .await
1336 .unwrap();
1337 git_in(&src, &["add", "."]).await;
1338 git_in(&src, &["commit", "-q", "-m", "one"]).await;
1339
1340 let workdir = tmp.path().join("work");
1341 tokio::fs::create_dir_all(&workdir).await.unwrap();
1342 let cfg = cfg_with_workdir(workdir.clone());
1343 let topo = topo_with_aux(vec![AuxRepo {
1344 name: "docengine".into(),
1345 bare_path: tmp
1346 .path()
1347 .join("docengine.git")
1348 .to_string_lossy()
1349 .into_owned(),
1350 upstream: src.to_string_lossy().into_owned(),
1351 branch: "main".into(),
1352 // Nested, as the real one is: it must not be mistaken for a path
1353 // under the per-sha worktree.
1354 checkout_dir: "Libraries/docengine".into(),
1355 }]);
1356 checkout_aux_repos(&cfg, &topo).await.unwrap();
1357
1358 let ctx = crate::gates::GateCtx {
1359 pool: sqlx::SqlitePool::connect_lazy("sqlite::memory:").unwrap(),
1360 cfg: Arc::new(cfg.clone()),
1361 tier: crate::domain::TierId::new("host"),
1362 version: "0.1.0".parse().unwrap(),
1363 worktree: Some(workdir.join("abc123")),
1364 bundle: None,
1365 events: crate::events::channel(),
1366 nodes: Vec::new(),
1367 build_id: None,
1368 public_url: None,
1369 aux_dirs: super::aux_checkout_dirs(&cfg, &topo),
1370 };
1371 let target = crate::config::TestTarget {
1372 dir: PathBuf::new(),
1373 aux_repo: Some("docengine".into()),
1374 features: Vec::new(),
1375 all_features: true,
1376 scratch_db: false,
1377 };
1378 let resolved = ctx.target_dir(&target).expect("aux repo is checked out");
1379 assert!(
1380 resolved.join("Cargo.toml").is_file(),
1381 "gate would skip the aux target as absent; resolved {}",
1382 resolved.display(),
1383 );
1384 assert!(
1385 !resolved.starts_with(ctx.worktree.as_ref().unwrap()),
1386 "an aux checkout is a sibling of the worktree, not under it",
1387 );
1388 }
1389
1390 #[tokio::test]
1391 async fn checkout_aux_repos_places_repo_beside_worktree_and_refreshes_to_branch_head() {
1392 let tmp = tempfile::tempdir().unwrap();
1393
1394 // An "upstream" source repo with a marker file on main.
1395 let src = tmp.path().join("synckit-src");
1396 tokio::fs::create_dir_all(&src).await.unwrap();
1397 git_in(&src, &["init", "-q", "-b", "main"]).await;
1398 tokio::fs::write(src.join("VERSION"), b"v1").await.unwrap();
1399 git_in(&src, &["add", "."]).await;
1400 git_in(&src, &["commit", "-q", "-m", "one"]).await;
1401
1402 let workdir = tmp.path().join("work");
1403 tokio::fs::create_dir_all(&workdir).await.unwrap();
1404 let cfg = cfg_with_workdir(workdir.clone());
1405 let topo = topo_with_aux(vec![AuxRepo {
1406 name: "synckit".into(),
1407 bare_path: tmp
1408 .path()
1409 .join("synckit.git")
1410 .to_string_lossy()
1411 .into_owned(),
1412 upstream: src.to_string_lossy().into_owned(),
1413 branch: "main".into(),
1414 checkout_dir: "synckit".into(),
1415 }]);
1416
1417 // First build: the aux repo lands at workdir/synckit at v1.
1418 checkout_aux_repos(&cfg, &topo).await.unwrap();
1419 let dest = workdir.join("synckit");
1420 assert_eq!(
1421 tokio::fs::read(dest.join("VERSION")).await.unwrap(),
1422 b"v1",
1423 "aux repo checked out beside the worktree",
1424 );
1425
1426 // Upstream advances; a later build refreshes the shared checkout to HEAD.
1427 tokio::fs::write(src.join("VERSION"), b"v2").await.unwrap();
1428 git_in(&src, &["add", "."]).await;
1429 git_in(&src, &["commit", "-q", "-m", "two"]).await;
1430 checkout_aux_repos(&cfg, &topo).await.unwrap();
1431 assert_eq!(
1432 tokio::fs::read(dest.join("VERSION")).await.unwrap(),
1433 b"v2",
1434 "aux checkout refreshed to the new branch HEAD",
1435 );
1436
1437 // The aux bare carries no build-trigger hook.
1438 assert!(
1439 !tmp.path().join("synckit.git/hooks/post-receive").exists(),
1440 "aux bare must be hookless",
1441 );
1442 }
1443
1444 #[tokio::test]
1445 async fn checkout_aux_repos_is_a_noop_without_aux_repos() {
1446 let tmp = tempfile::tempdir().unwrap();
1447 let cfg = cfg_with_workdir(tmp.path().to_path_buf());
1448 checkout_aux_repos(&cfg, &topo_with_aux(vec![]))
1449 .await
1450 .unwrap();
1451 }
1452
1453 #[tokio::test]
1454 async fn checkout_aux_repos_fails_on_an_unresolvable_branch() {
1455 let tmp = tempfile::tempdir().unwrap();
1456 let src = tmp.path().join("src");
1457 tokio::fs::create_dir_all(&src).await.unwrap();
1458 git_in(&src, &["init", "-q", "-b", "main"]).await;
1459 tokio::fs::write(src.join("f"), b"x").await.unwrap();
1460 git_in(&src, &["add", "."]).await;
1461 git_in(&src, &["commit", "-q", "-m", "c"]).await;
1462
1463 let cfg = cfg_with_workdir(tmp.path().join("work"));
1464 let topo = topo_with_aux(vec![AuxRepo {
1465 name: "synckit".into(),
1466 bare_path: tmp.path().join("s.git").to_string_lossy().into_owned(),
1467 upstream: src.to_string_lossy().into_owned(),
1468 branch: "nonexistent".into(),
1469 checkout_dir: "synckit".into(),
1470 }]);
1471 let err = checkout_aux_repos(&cfg, &topo).await.unwrap_err();
1472 assert!(
1473 format!("{err:#}").contains("synckit"),
1474 "error names the aux repo: {err:#}",
1475 );
1476 }
1477
1478 async fn tier_versions(pool: &SqlitePool, tier: &str) -> (Option<String>, Option<String>) {
1479 sqlx::query_as("SELECT current_version, previous_version FROM tier_state WHERE tier = ?")
1480 .bind(tier)
1481 .fetch_one(pool)
1482 .await
1483 .unwrap()
1484 }
1485
1486 async fn run_result(pool: &SqlitePool, run_id: RunId) -> (String, Option<String>) {
1487 sqlx::query_as("SELECT result, failure_summary FROM build_runs WHERE id = ?")
1488 .bind(run_id.0)
1489 .fetch_one(pool)
1490 .await
1491 .unwrap()
1492 }
1493
1494 #[tokio::test]
1495 async fn stage_and_gate_stages_advances_and_flips_the_symlink_when_gates_are_green() {
1496 let (pool, cfg, topo, art, run_id, version, tmp) = stage_fixture(vec![]).await;
1497 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1498
1499 stage_and_gate(
1500 pool.clone(),
1501 cfg.clone(),
1502 topo,
1503 art,
1504 crate::events::channel(),
1505 run_id,
1506 deploy_lock,
1507 )
1508 .await
1509 .expect("green host pipeline returns Ok");
1510
1511 // Tier advanced to the built version (previous was NULL -> stays NULL).
1512 let (current, previous) = tier_versions(&pool, "host").await;
1513 assert_eq!(current.as_deref(), Some(version.to_string().as_str()));
1514 assert_eq!(previous, None);
1515
1516 // Run settled green.
1517 let (result, summary) = run_result(&pool, run_id).await;
1518 assert_eq!(result, "passed");
1519 assert_eq!(summary, None);
1520
1521 // Identity: the build row carries the bundle digest (64 hex) and the
1522 // content-addressed dir it was published to (releases/<digest16>).
1523 let (digest, staged_path): (Option<String>, Option<String>) =
1524 sqlx::query_as("SELECT bundle_digest, staged_path FROM build_runs WHERE id = ?")
1525 .bind(run_id.0)
1526 .fetch_one(&pool)
1527 .await
1528 .unwrap();
1529 let digest = digest.expect("bundle_digest recorded");
1530 let staged_path = staged_path.expect("staged_path recorded");
1531 assert_eq!(digest.len(), 64);
1532 let releases = tmp.path().join("release-root").join("releases");
1533 assert_eq!(
1534 std::path::Path::new(&staged_path),
1535 releases.join(&digest[..16]),
1536 "bundle is published content-addressed at releases/<digest16>"
1537 );
1538
1539 // versions.artifact_path points at the primary binary inside that dir,
1540 // and it exists on disk.
1541 let staged_bin: String =
1542 sqlx::query_scalar("SELECT artifact_path FROM versions WHERE version = ?")
1543 .bind(version.to_string())
1544 .fetch_one(&pool)
1545 .await
1546 .unwrap();
1547 let expected_bin = releases.join(&digest[..16]).join("makenotwork");
1548 assert_eq!(staged_bin, expected_bin.to_string_lossy());
1549 assert!(
1550 expected_bin.exists(),
1551 "staged binary missing at {expected_bin:?}"
1552 );
1553
1554 // The bundle carries its MANIFEST (for node-side verification), and the
1555 // recorded digest recomputes over the published dir (MANIFEST excluded).
1556 assert!(
1557 releases.join(&digest[..16]).join("MANIFEST").exists(),
1558 "MANIFEST written into the bundle"
1559 );
1560 let recomputed = crate::bundle::digest_dir(std::path::Path::new(&staged_path))
1561 .await
1562 .unwrap();
1563 assert_eq!(
1564 digest, recomputed.full,
1565 "recorded digest matches the bundle"
1566 );
1567
1568 // The `current` symlink flipped to the content-addressed release.
1569 let link = tmp.path().join("release-root").join("current");
1570 let target = std::fs::read_link(&link).expect("current is a symlink");
1571 assert_eq!(target, PathBuf::from(format!("releases/{}", &digest[..16])));
1572 }
1573
1574 #[tokio::test]
1575 async fn stage_and_gate_marks_the_run_failed_and_does_not_advance_when_a_gate_is_red() {
1576 // ManualConfirm with no prior confirmation row blocks deterministically.
1577 let (pool, cfg, topo, art, run_id, _version, _tmp) =
1578 stage_fixture(vec![Gate::ManualConfirm]).await;
1579 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1580
1581 // A red gate is a pipeline outcome, not an error: the fn records the
1582 // failure and returns Ok so the spawned task settles the run cleanly.
1583 stage_and_gate(
1584 pool.clone(),
1585 cfg,
1586 topo,
1587 art,
1588 crate::events::channel(),
1589 run_id,
1590 deploy_lock,
1591 )
1592 .await
1593 .expect("a red gate settles the run, it does not error out");
1594
1595 // Tier did NOT advance — still the seeded NULL/NULL.
1596 let (current, previous) = tier_versions(&pool, "host").await;
1597 assert_eq!(current, None);
1598 assert_eq!(previous, None);
1599
1600 // Run settled red with a non-empty summary.
1601 let (result, summary) = run_result(&pool, run_id).await;
1602 assert_eq!(result, "failed");
1603 assert!(
1604 summary.as_deref().is_some_and(|s| !s.is_empty()),
1605 "failed run must carry a summary, got {summary:?}"
1606 );
1607 }
1608
1609 #[test]
1610 fn check_build_host_accepts_matching_host() {
1611 assert!(check_build_host("fw13", "fw13").is_ok());
1612 }
1613
1614 #[test]
1615 fn check_build_host_refuses_mismatched_host() {
1616 // A daemon misdeployed onto prod (e.g. a Hetzner host) must refuse.
1617 let err = check_build_host("alpha-west-1", "fw13")
1618 .unwrap_err()
1619 .to_string();
1620 assert!(err.contains("refusing to build"), "{err}");
1621 assert!(
1622 err.contains("alpha-west-1") && err.contains("fw13"),
1623 "{err}"
1624 );
1625 }
1626
1627 #[test]
1628 fn runtime_hostname_reads_a_nonempty_trimmed_name() {
1629 let h = runtime_hostname().expect("hostname readable on Linux");
1630 assert!(!h.is_empty());
1631 assert_eq!(h, h.trim(), "must be trimmed");
1632 }
1633
1634 #[test]
1635 fn tail_does_not_panic_on_multibyte_boundary() {
1636 // Each '€' is 3 bytes; a byte cap landing mid-codepoint must not panic.
1637 let s = "€".repeat(10); // 30 bytes
1638 for max in 1..=30 {
1639 let out = tail(s.as_bytes(), max);
1640 assert!(out.len() <= max, "max={max} got {} bytes", out.len());
1641 // Result is always valid UTF-8 made only of whole '€'s.
1642 assert!(out.chars().all(|c| c == '€'), "max={max}: {out:?}");
1643 }
1644 }
1645
1646 #[test]
1647 fn tail_returns_whole_input_when_under_cap() {
1648 assert_eq!(tail(b"hello", 100), "hello");
1649 }
1650 }
1651