Skip to main content

max / makenotwork

63.2 KB · 1648 lines History Blame Raw
1 //! Build orchestration: resolve a sha to a worktree, read the server version,
2 //! shell out to `cargo build --release`, record a `versions` row.
3 //!
4 //! Runs as a tokio task spawned from `POST /rebuild`; the HTTP request
5 //! returns the version id immediately and the task drives the rest.
6
7 use crate::config::AppConfig;
8 use crate::deploy;
9 use crate::domain::{GitSha, Platform, RunId, TierId, Version};
10 use crate::gates::{self, GateCtx};
11 use crate::git;
12 use crate::topology::Topology;
13 use anyhow::{Context, Result};
14 use chrono::Utc;
15 use sqlx::SqlitePool;
16 use std::path::{Path, PathBuf};
17 use std::sync::Arc;
18 use tokio::process::Command;
19
20 #[derive(Debug, Clone)]
21 pub struct BuildArtifact {
22 pub version: Version,
23 pub git_sha: GitSha,
24 pub worktree: PathBuf,
25 /// One entry per `cfg.bin_names` in declared order. First is the primary
26 /// (referenced by the systemd unit's ExecStart). Paths are inside the
27 /// worktree's `target/release/`.
28 pub binary_paths: Vec<PathBuf>,
29 /// `(companion name, built binary path)` for each `cfg.companions`, built
30 /// from the same worktree/sha as the server. Staged into the release bundle
31 /// under `companions/<name>/<bin>` and installed by the nodes that opt in.
32 pub companion_paths: Vec<(String, PathBuf)>,
33 }
34
35 /// The live kernel hostname (`/proc/sys/kernel/hostname`, trimmed). Linux-only,
36 /// which Sando is. The build-host guard reads this rather than `$HOSTNAME`
37 /// (not reliably exported) so the check reflects the actual machine.
38 fn runtime_hostname() -> Result<String> {
39 let raw = std::fs::read_to_string("/proc/sys/kernel/hostname")
40 .context("reading /proc/sys/kernel/hostname for the build-host guard")?;
41 Ok(raw.trim().to_string())
42 }
43
44 /// Pure half of the build-host guard: fail unless the live host matches the
45 /// configured build host. Split out from [`enforce_build_host`] so it is unit-
46 /// testable without depending on the test machine's hostname.
47 fn check_build_host(actual: &str, expected: &str) -> Result<()> {
48 anyhow::ensure!(
49 actual == expected,
50 "refusing to build on host {actual}: configured build host is {expected} \
51 (never build on a prod/serving node)",
52 );
53 Ok(())
54 }
55
56 /// Refuse to build unless this daemon is running on the configured build host.
57 fn enforce_build_host(expected: &str) -> Result<()> {
58 check_build_host(&runtime_hostname()?, expected)
59 }
60
61 pub async fn run(
62 pool: SqlitePool,
63 cfg: Arc<AppConfig>,
64 topo: Arc<Topology>,
65 sha: GitSha,
66 events: crate::events::EventTx,
67 run_id: RunId,
68 ) -> Result<BuildArtifact> {
69 // Build-host guard: refuse to compile anywhere but the configured build
70 // host. The build runs locally (`cargo build` in cfg.workdir), so a sandod
71 // misdeployed onto a prod/serving node would otherwise build there — exactly
72 // the "never build on prod" rule. Enforced before any cargo invocation so a
73 // wrong-host daemon fails fast with a clear message rather than compiling.
74 let Some(build_host) = cfg.build_host.as_deref() else {
75 anyhow::bail!(
76 "{} declares no build_host, which makes it intake-only: Sando does not compile \
77 it. Ship it with POST /intake, from a builder that does.",
78 cfg.id
79 );
80 };
81 enforce_build_host(build_host)?;
82
83 let repo = topo.repo.as_ref().with_context(|| {
84 format!(
85 "{} declares no [repo]: it is intake-only and Sando has no source to check out",
86 cfg.id
87 )
88 })?;
89 let worktree = cfg.workdir.join(sha.as_str());
90 let bare = PathBuf::from(&repo.bare_path);
91
92 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Fetching)
93 .await
94 .ok();
95
96 // Pull-based ingestion: if an upstream remote is configured, fetch the
97 // deploy branch so a just-pushed sha is locally resolvable. A fetch
98 // failure is non-fatal — the sha may already be present from a prior
99 // fetch or a direct push; the presence check below is the real gate.
100 if let Some(upstream) = repo.upstream.as_deref()
101 && let Err(e) = git::fetch_upstream(&bare, upstream, &repo.branch).await
102 {
103 tracing::warn!(error = %e, upstream, "upstream fetch failed; proceeding with current bare-repo state");
104 }
105 anyhow::ensure!(
106 git::sha_present(&bare, sha.as_str()).await?,
107 "sha {} not present in bare repo {} after fetch — push the commit to the upstream remote first",
108 sha.as_str(),
109 bare.display(),
110 );
111
112 git::checkout_worktree(&bare, sha.as_str(), &worktree).await?;
113
114 // Check out any auxiliary repos (e.g. synckit) beside the worktree so a
115 // cross-repo path dependency in the server or a companion resolves. Fails the
116 // build if an aux repo can't be assembled — a companion that silently fails to
117 // find its source would fail the compile downstream with a worse message.
118 checkout_aux_repos(&cfg, &topo).await?;
119
120 let server_dir = worktree.join("server");
121 let version = read_pkg_version(&server_dir.join("Cargo.toml"))
122 .await
123 .with_context(|| format!("reading version from {}/Cargo.toml", server_dir.display()))?;
124 crate::runs::set_version(&pool, run_id, &version).await.ok();
125
126 // sqlx compile-time query checking needs a live DB with the current schema.
127 // We point cargo at the scratch DB and prep it (drop public, re-migrate)
128 // before invoking cargo build. The same DB is reset again by
129 // `migration_dry_run` later if it runs as a gate.
130 let mut cargo_cmd = Command::new("cargo");
131 cargo_cmd
132 .arg("build")
133 .arg("--release")
134 .current_dir(&server_dir)
135 .kill_on_drop(true);
136 // Shared build cache across per-sha worktrees: reuse one target dir so an
137 // incremental diff doesn't clean-compile from scratch. Serialized builds
138 // make this contention-free. Unset → cargo's default per-worktree target/.
139 if let Some(target) = cfg.cargo_target_dir.as_deref() {
140 cargo_cmd.env("CARGO_TARGET_DIR", target);
141 }
142 if let Some(scratch_url) = cfg.scratch_db_url.as_deref() {
143 tracing::info!(sha = %sha.as_str(), "preparing scratch DB schema for sqlx compile-time checks");
144 crate::gates::reset_scratch(scratch_url, &cfg.scratch_owner_role)
145 .await
146 .context("scratch DB reset before build")?;
147 crate::gates::run_migrator(scratch_url, &server_dir.join("migrations"))
148 .await
149 .context("applying MNW migrations to scratch DB before build")?;
150 cargo_cmd.env("DATABASE_URL", scratch_url);
151 } else {
152 tracing::warn!("scratch_db_url unset; sqlx will fall back to offline mode and may fail");
153 }
154
155 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Compiling)
156 .await
157 .ok();
158 tracing::info!(sha = %sha, version = %version, dir = %server_dir.display(), "cargo build --release start");
159 crate::events::emit(
160 &events,
161 crate::events::Event::BuildStart {
162 sha: sha.clone(),
163 version: version.clone(),
164 },
165 );
166 let started = std::time::Instant::now();
167 let out = cargo_cmd.output().await.context("spawning cargo build")?;
168 let elapsed_s = started.elapsed().as_secs();
169 if !out.status.success() {
170 tracing::error!(sha = %sha, version = %version, elapsed_s, "cargo build --release failed");
171 crate::events::emit(
172 &events,
173 crate::events::Event::BuildFailed {
174 sha: sha.clone(),
175 version: version.clone(),
176 elapsed_s,
177 },
178 );
179 // Settle the run with the headline compiler diagnostic (not the raw
180 // 4 KB tail) so `GET /runs/{id}` answers "why" without a journald dive.
181 let summary = crate::classify::classify_compile_error(&out.stdout, &out.stderr).summary();
182 if let Err(e) = crate::runs::mark_failed(&pool, run_id, &summary).await {
183 tracing::error!(run_id = %run_id, error = %e, "persisting compile-fail verdict failed; run may show stale 'building' until restart-reconcile");
184 }
185 anyhow::bail!(
186 "cargo build --release failed:\n{}",
187 tail(&out.stderr, 4_000)
188 );
189 }
190 tracing::info!(sha = %sha, version = %version, elapsed_s, "cargo build --release ok");
191 crate::events::emit(
192 &events,
193 crate::events::Event::BuildOk {
194 sha: sha.clone(),
195 version: version.clone(),
196 elapsed_s,
197 },
198 );
199
200 // Binaries land under `<target>/release/`; with a shared target dir that's
201 // not inside the worktree, so resolve it the same way cargo did above.
202 let release_dir = cfg
203 .cargo_target_dir
204 .as_deref()
205 .map_or_else(|| server_dir.join("target/release"), |t| t.join("release"));
206 let mut binary_paths = Vec::with_capacity(cfg.bin_names.len());
207 for name in &cfg.bin_names {
208 let p = release_dir.join(name);
209 anyhow::ensure!(p.exists(), "expected binary at {} after build", p.display());
210 binary_paths.push(p);
211 }
212 // Primary binary path is the one we record in `versions.artifact_path`
213 // (everything downstream — promote, rollback — looks it up by version).
214 let primary = binary_paths[0].clone();
215
216 // Companion crates (e.g. mnw-cli): built from the SAME worktree/sha so a
217 // service that shares the server's internal-API contract cannot drift out of
218 // lockstep (the 2026-07-09 git-hosting outage). A companion build failure
219 // fails the whole pipeline — the server never ships without its companions.
220 let mut companion_paths = Vec::with_capacity(cfg.companions.len());
221 for c in &cfg.companions {
222 let bin = build_companion(&worktree, &cfg, c).await?;
223 companion_paths.push((c.name.clone(), bin));
224 }
225
226 sqlx::query(
227 "INSERT OR IGNORE INTO versions (app, version, git_sha, built_at, artifact_path)
228 VALUES (?, ?, ?, ?, ?)",
229 )
230 .bind(&cfg.id)
231 .bind(&version)
232 .bind(&sha)
233 .bind(Utc::now().to_rfc3339())
234 .bind(primary.to_string_lossy().as_ref())
235 .execute(&pool)
236 .await?;
237
238 Ok(BuildArtifact {
239 version,
240 git_sha: sha,
241 worktree,
242 binary_paths,
243 companion_paths,
244 })
245 }
246
247 /// Fetch and check out every configured auxiliary repo at `cfg.workdir/<checkout_dir>`,
248 /// so a cross-repo path dependency built from the main worktree resolves (wiki
249 /// [[sando-overview]]; the synckit split, task sando-18cdb32f).
250 ///
251 /// Each aux repo is a fixed, shared checkout refreshed to `branch` HEAD — not
252 /// per-sha — because the dependent's relative path resolves to that fixed spot
253 /// regardless of the main sha, and builds serialize. A fetch failure is a warning
254 /// (the branch may already be present from a prior build); an unresolvable branch
255 /// after that is fatal, as is a failed worktree — a half-assembled source tree
256 /// must fail the build here, loudly, not as a downstream compile error.
257 /// Where an aux repo's checkout lands. The single derivation: `checkout_aux_repos`
258 /// creates it here and `GateCtx::aux_dirs` resolves `test_target`s against it, so
259 /// the two cannot drift into looking in different places.
260 pub fn aux_checkout_dir(cfg: &AppConfig, aux: &crate::topology::AuxRepo) -> PathBuf {
261 cfg.workdir.join(&aux.checkout_dir)
262 }
263
264 /// Every aux repo's checkout dir, keyed by name — what `GateCtx::aux_dirs` holds.
265 pub fn aux_checkout_dirs(
266 cfg: &AppConfig,
267 topo: &Topology,
268 ) -> std::collections::HashMap<String, PathBuf> {
269 topo.aux_repos
270 .iter()
271 .map(|a| (a.name.clone(), aux_checkout_dir(cfg, a)))
272 .collect()
273 }
274
275 pub async fn checkout_aux_repos(cfg: &AppConfig, topo: &Topology) -> Result<()> {
276 for aux in &topo.aux_repos {
277 let bare = PathBuf::from(&aux.bare_path);
278 git::ensure_bare_repo_no_hook(&bare)
279 .await
280 .with_context(|| format!("aux repo {}: init bare {}", aux.name, aux.bare_path))?;
281 if let Err(e) = git::fetch_upstream(&bare, &aux.upstream, &aux.branch).await {
282 tracing::warn!(
283 aux = %aux.name, error = %e,
284 "aux repo fetch failed; proceeding with current bare-repo state",
285 );
286 }
287 let sha = git::resolve_ref(&bare, &aux.branch).await.with_context(|| {
288 format!(
289 "aux repo {}: branch {} not resolvable after fetch — is {} reachable with that branch?",
290 aux.name, aux.branch, aux.upstream,
291 )
292 })?;
293 let dest = aux_checkout_dir(cfg, aux);
294 git::checkout_worktree(&bare, &sha, &dest)
295 .await
296 .with_context(|| {
297 format!(
298 "aux repo {}: checking out {} ({}) at {}",
299 aux.name,
300 aux.branch,
301 sha,
302 dest.display()
303 )
304 })?;
305 tracing::info!(
306 aux = %aux.name, branch = %aux.branch, sha = %sha, dest = %dest.display(),
307 "aux repo checked out beside worktree",
308 );
309 }
310 Ok(())
311 }
312
313 /// Build one companion crate from the worktree, returning its release binary
314 /// path. Mirrors the server build's target-dir handling (shared
315 /// `cargo_target_dir` when set, for incremental reuse; else the crate's own
316 /// `target/`). A companion is an API client, not a sqlx crate, so it needs no
317 /// scratch DB. A non-zero exit propagates and fails the pipeline.
318 async fn build_companion(
319 worktree: &Path,
320 cfg: &AppConfig,
321 c: &crate::config::Companion,
322 ) -> Result<PathBuf> {
323 let dir = worktree.join(&c.manifest_dir);
324 anyhow::ensure!(
325 dir.join("Cargo.toml").exists(),
326 "companion {}: no Cargo.toml at {}",
327 c.name,
328 dir.display(),
329 );
330 // Match the server build: no `--locked` (the pipeline builds whatever the
331 // sha pins; a stale lock shouldn't block a deploy the server build allows).
332 let mut cmd = Command::new("cargo");
333 cmd.arg("build")
334 .arg("--release")
335 .current_dir(&dir)
336 .kill_on_drop(true);
337 let release_dir = if let Some(target) = cfg.cargo_target_dir.as_deref() {
338 cmd.env("CARGO_TARGET_DIR", target);
339 target.join("release")
340 } else {
341 dir.join("target/release")
342 };
343 tracing::info!(companion = %c.name, dir = %dir.display(), "cargo build --release (companion) start");
344 let started = std::time::Instant::now();
345 let out = cmd
346 .output()
347 .await
348 .context("spawning cargo build for companion")?;
349 if !out.status.success() {
350 anyhow::bail!(
351 "companion {} build failed:\n{}",
352 c.name,
353 tail(&out.stderr, 4_000),
354 );
355 }
356 let bin = release_dir.join(&c.bin);
357 anyhow::ensure!(
358 bin.exists(),
359 "companion {} produced no binary at {} after build",
360 c.name,
361 bin.display(),
362 );
363 tracing::info!(companion = %c.name, elapsed_s = started.elapsed().as_secs(), "companion build ok");
364 Ok(bin)
365 }
366
367 /// Full host-tier pipeline: build, stage the bundle into the host's
368 /// release_root, run the host tier's configured gates, advance tier_state
369 /// for "host" if all pass. Errors propagate back to the spawned task and
370 /// get logged. (Tier was called "mm" pre-Session-1; renamed to "host"
371 /// since sandod runs on whatever machine ends up being the Sando host.)
372 pub async fn build_and_run_host(
373 pool: SqlitePool,
374 cfg: Arc<AppConfig>,
375 topo: Arc<Topology>,
376 sha: GitSha,
377 events: crate::events::EventTx,
378 run_id: RunId,
379 deploy_lock: Arc<tokio::sync::Mutex<()>>,
380 ) -> Result<()> {
381 let art = run(
382 pool.clone(),
383 cfg.clone(),
384 topo.clone(),
385 sha,
386 events.clone(),
387 run_id,
388 )
389 .await?;
390
391 stage_and_gate(pool, cfg, topo, art, events, run_id, deploy_lock).await
392 }
393
394 /// A bundle assembled on disk and not yet published: the point both the build
395 /// path and the intake path have to reach before anything else can happen to it.
396 ///
397 /// The two paths reach it differently. A build assembles it out of a worktree
398 /// (binaries, `release_contents`, companions); an intake is handed it already
399 /// assembled, with no worktree anywhere. Everything after this point — hashing,
400 /// publishing, recording identity, gating, advancing — is the same work, and
401 /// used to be welded to the assembling half inside `stage_and_gate`.
402 struct StagedBundle {
403 version: Version,
404 /// The staging dir under `release_root/staging/`, pre-publish.
405 staging: PathBuf,
406 /// What the bundle is built to run on, when it is known. A Sando build
407 /// inherits the app's declared platform; an intake takes it from the
408 /// record's provenance.
409 platform: Option<Platform>,
410 }
411
412 /// Post-build half of the host pipeline: stage the artifact into the host's
413 /// release_root, run the host tier's gates, and advance `tier_state` for
414 /// "host" iff all pass. Split from [`build_and_run_host`] at the `run()`
415 /// boundary so the staging/gating/advance logic is reachable in tests from a
416 /// synthetic [`BuildArtifact`] — no real `cargo build --release` required.
417 pub async fn stage_and_gate(
418 pool: SqlitePool,
419 cfg: Arc<AppConfig>,
420 topo: Arc<Topology>,
421 art: BuildArtifact,
422 events: crate::events::EventTx,
423 run_id: RunId,
424 deploy_lock: Arc<tokio::sync::Mutex<()>>,
425 ) -> Result<()> {
426 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Staging)
427 .await
428 .ok();
429 let staged = assemble_from_source(&cfg, &art, run_id).await?;
430 let published = publish(&pool, &cfg, staged, run_id).await?;
431 record_and_gate(
432 pool,
433 cfg,
434 topo,
435 published,
436 events,
437 run_id,
438 deploy_lock,
439 Some(art.worktree),
440 )
441 .await
442 }
443
444 /// Accept an artifact built elsewhere and take it through the same host-tier
445 /// gating and advance a Sando-built one gets.
446 ///
447 /// This is the whole point of the seam. `intake::accept` publishes the bundle
448 /// content-addressed once it has proved the bytes are the ones the record
449 /// vouches for, which lands it at exactly the state [`publish`] leaves a
450 /// Sando-built bundle in — so the two paths join at [`record_and_gate`] and
451 /// nothing downstream knows or cares which one it came from.
452 ///
453 /// `staged` must already sit under `release_root/staging/` (publishing is an
454 /// atomic same-filesystem rename); getting the bytes there is the transport's
455 /// job, not this function's.
456 #[allow(clippy::too_many_arguments)]
457 pub async fn accept_intake(
458 pool: &SqlitePool,
459 cfg: &AppConfig,
460 staged: &Path,
461 record_json: &str,
462 run_id: RunId,
463 ) -> Result<Published> {
464 crate::runs::set_phase(pool, run_id, crate::runs::Phase::Staging)
465 .await
466 .ok();
467
468 let accepted = crate::intake::accept(&cfg.release_root, staged, record_json)
469 .await
470 .map_err(|e| anyhow::anyhow!("{e}"))?;
471
472 let version = Version::parse(&accepted.record.provenance.version).with_context(|| {
473 format!(
474 "artifact record carries version `{}`, which is not semver",
475 accepted.record.provenance.version
476 )
477 })?;
478 let platform = Platform::parse(&accepted.record.provenance.target).with_context(|| {
479 format!(
480 "artifact record carries target `{}`, which is not `os/arch`",
481 accepted.record.provenance.target
482 )
483 })?;
484 let git_sha = GitSha::parse(&accepted.record.provenance.git_sha).with_context(|| {
485 format!(
486 "artifact record carries git_sha `{}`",
487 accepted.record.provenance.git_sha
488 )
489 })?;
490
491 crate::runs::set_version(pool, run_id, &version).await.ok();
492 upsert_version_row(
493 pool,
494 &cfg.id,
495 &version,
496 &git_sha,
497 &accepted.released.join(cfg.primary_bin()),
498 )
499 .await?;
500
501 let published = Published {
502 version,
503 released: accepted.released,
504 digest_full: accepted.record.digest.to_string(),
505 platform: Some(platform),
506 };
507 record_identity(pool, cfg, &published, run_id).await?;
508 Ok(published)
509 }
510
511 /// Gate an artifact that has already been accepted.
512 ///
513 /// Split from [`accept_intake`] so the two can be answered on different clocks.
514 /// Acceptance is fast and is the producer's business — it either believes the
515 /// bytes or names the file that drifted — so the caller waits for it and gets
516 /// the verdict. Gating is Sando's business and can take an hour, so the caller
517 /// does not.
518 ///
519 /// An intake carries no worktree, and the gates that need one refuse rather than
520 /// resolve against nothing. That is the boundary showing up in the type:
521 /// artifact-scoped gates belong to the builder (wiki [[sando-bento-boundary]]),
522 /// so a tier that asks Sando to re-run them against an accepted artifact is
523 /// misconfigured and should be told so.
524 pub async fn gate_intake(
525 pool: SqlitePool,
526 cfg: Arc<AppConfig>,
527 topo: Arc<Topology>,
528 published: Published,
529 events: crate::events::EventTx,
530 run_id: RunId,
531 deploy_lock: Arc<tokio::sync::Mutex<()>>,
532 ) -> Result<()> {
533 record_and_gate(
534 pool,
535 cfg,
536 topo,
537 published,
538 events,
539 run_id,
540 deploy_lock,
541 None,
542 )
543 .await
544 }
545
546 /// Record the `versions` label row for an artifact that arrived rather than was
547 /// built here. The build path writes its own inside [`run`]; this is the same
548 /// row for the path that never ran a compiler.
549 async fn upsert_version_row(
550 pool: &SqlitePool,
551 app: &crate::domain::AppId,
552 version: &Version,
553 git_sha: &GitSha,
554 artifact_path: &Path,
555 ) -> Result<()> {
556 sqlx::query(
557 "INSERT OR IGNORE INTO versions (app, version, git_sha, built_at, artifact_path)
558 VALUES (?, ?, ?, ?, ?)",
559 )
560 .bind(app)
561 .bind(version)
562 .bind(git_sha)
563 .bind(Utc::now().to_rfc3339())
564 .bind(artifact_path.to_string_lossy().as_ref())
565 .execute(pool)
566 .await?;
567 Ok(())
568 }
569
570 /// Assemble a bundle out of a worktree: binaries, `release_contents`, companions.
571 async fn assemble_from_source(
572 cfg: &AppConfig,
573 art: &BuildArtifact,
574 run_id: RunId,
575 ) -> Result<StagedBundle> {
576 // Stage the bundle into `staging/<build_id>/` — a private scratch dir, not
577 // yet a release. It is published content-addressed below, once its digest is
578 // known. This is what makes overwrite unexpressible (wiki
579 // [[release-artifact-identity]]): a build never touches another build's dir.
580 let staging =
581 deploy::stage_local_bundle(&cfg.release_root, run_id.0, &art.binary_paths).await?;
582
583 // Stage every entry from cfg.release_contents into the staged bundle. This is
584 // how non-binary version-coupled content (static assets, docs, error-pages,
585 // ...) makes it into the atomic deploy bundle. Projects opt in via daemon
586 // config — the sando code carries no MNW-specific knowledge.
587 for entry in &cfg.release_contents {
588 stage_entry(&art.worktree, &staging, entry).await?;
589 }
590
591 // Stage companion binaries as `companions/<name>` (the file itself) so they
592 // ride the same atomic bundle rsync to the nodes, and a node can locate its
593 // companion source from the logical name alone — no bin-filename coupling in
594 // the topology. The nodes that opt in install them post-swap (see
595 // deploy::deploy_remote).
596 if !art.companion_paths.is_empty() {
597 let dst_dir = staging.join("companions");
598 tokio::fs::create_dir_all(&dst_dir)
599 .await
600 .with_context(|| format!("create staged companions dir {}", dst_dir.display()))?;
601 for (name, built) in &art.companion_paths {
602 let dst = dst_dir.join(name);
603 tokio::fs::copy(built, &dst).await.with_context(|| {
604 format!(
605 "stage companion {name}: {} -> {}",
606 built.display(),
607 dst.display()
608 )
609 })?;
610 }
611 }
612
613 Ok(StagedBundle {
614 version: art.version.clone(),
615 staging,
616 platform: cfg.platform.clone(),
617 })
618 }
619
620 /// A bundle that has been hashed and published content-addressed. Both paths
621 /// produce one; nothing downstream can tell them apart.
622 ///
623 /// Public because the intake route now hands one from `accept_intake` to
624 /// `gate_intake`: proving the bytes answers the producer, gating them does not,
625 /// so the two run on different clocks and the value passes between them.
626 #[derive(Debug)]
627 pub struct Published {
628 version: Version,
629 released: PathBuf,
630 digest_full: String,
631 platform: Option<Platform>,
632 }
633
634 /// Hash the assembled bundle, write its MANIFEST, and publish it at
635 /// `releases/<digest16>`.
636 ///
637 /// The intake path does not call this: `intake::accept` does the same three
638 /// steps itself, because it has to hash the bytes to verify them and hashing
639 /// them twice would be the one place the two implementations could disagree.
640 async fn publish(
641 pool: &SqlitePool,
642 cfg: &AppConfig,
643 staged: StagedBundle,
644 run_id: RunId,
645 ) -> Result<Published> {
646 // Content identity: hash the fully-staged bundle, write its MANIFEST into the
647 // bundle (for node-side verification), then publish it at `releases/<digest16>`.
648 // The digest is now load-bearing — a hashing failure fails the build rather
649 // than shipping an unidentifiable artifact.
650 let digest = crate::bundle::digest_dir(&staged.staging)
651 .await
652 .context("hashing the staged bundle for content addressing")?;
653 tokio::fs::write(
654 staged.staging.join(crate::bundle::MANIFEST_NAME),
655 digest.manifest.as_bytes(),
656 )
657 .await
658 .context("writing bundle MANIFEST")?;
659 let released =
660 deploy::finalize_local_release(&cfg.release_root, &staged.staging, digest.short()).await?;
661
662 let staged_bin = released.join(cfg.primary_bin());
663 sqlx::query("UPDATE versions SET artifact_path = ? WHERE app = ? AND version = ?")
664 .bind(staged_bin.to_string_lossy().as_ref())
665 .bind(&cfg.id)
666 .bind(&staged.version)
667 .execute(pool)
668 .await?;
669
670 let published = Published {
671 version: staged.version,
672 released,
673 digest_full: digest.full,
674 platform: staged.platform,
675 };
676 record_identity(pool, cfg, &published, run_id).await?;
677 Ok(published)
678 }
679
680 /// Record the identity on the build row: the digest, the content-addressed dir
681 /// the bundle was published to, and what it runs on. This is what promote
682 /// resolves the artifact through, and burn-in/retention key on.
683 async fn record_identity(
684 pool: &SqlitePool,
685 cfg: &AppConfig,
686 published: &Published,
687 run_id: RunId,
688 ) -> Result<()> {
689 let released_path = published.released.to_string_lossy();
690 crate::runs::set_identity(pool, run_id, &published.digest_full, &released_path)
691 .await
692 .ok();
693 // Platform is what lets two bundles of one version be told apart, so a
694 // dropped write here would leave a pom artifact that can be placed nowhere
695 // (a node declaring a platform refuses an artifact that records none). Fail
696 // rather than ship an unplaceable bundle.
697 if let Some(p) = &published.platform {
698 crate::runs::set_platform(pool, run_id, p)
699 .await
700 .with_context(|| format!("recording platform {p} for {}", cfg.id))?;
701 }
702 Ok(())
703 }
704
705 /// The shared tail of both paths: run the host tier's gates against a published
706 /// bundle and advance `tier_state` iff all pass.
707 #[allow(clippy::too_many_arguments)]
708 async fn record_and_gate(
709 pool: SqlitePool,
710 cfg: Arc<AppConfig>,
711 topo: Arc<Topology>,
712 published: Published,
713 events: crate::events::EventTx,
714 run_id: RunId,
715 deploy_lock: Arc<tokio::sync::Mutex<()>>,
716 worktree: Option<PathBuf>,
717 ) -> Result<()> {
718 let host = topo
719 .tiers
720 .iter()
721 .find(|t| t.name.as_str() == "host")
722 .context("topology has no `host` tier")?;
723
724 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Gating)
725 .await
726 .ok();
727 let ctx = GateCtx {
728 pool: pool.clone(),
729 cfg: cfg.clone(),
730 tier: TierId::new("host"),
731 version: published.version.clone(),
732 worktree,
733 // The published bundle. `migration_dry_run` prefers it over the
734 // worktree, so the migrations it proves are the ones inside the digest
735 // rather than ones sitting beside them in a checkout.
736 bundle: Some(published.released.clone()),
737 events: events.clone(),
738 // Host runs build-time gates (cargo_test / migration_dry_run /
739 // boot_smoke) only — `node_health` never appears here, so there are no
740 // nodes to probe.
741 nodes: Vec::new(),
742 // These gates vouch for this build; record its id so promote can resolve
743 // the artifact through the evidence rather than a version string.
744 build_id: Some(run_id.0),
745 // Where checkout_aux_repos put each aux repo, so a test_target naming
746 // one resolves. Shared derivation, so the two cannot disagree.
747 public_url: None,
748 aux_dirs: aux_checkout_dirs(&cfg, &topo),
749 };
750 let failed = gates::run_all(&ctx, &host.gates).await?;
751
752 if failed.is_empty() {
753 // Advance the host tier through the single sealed forward-advance op, under
754 // deploy_lock so this can't interleave with a concurrent `/rollback host`
755 // (the old fetch-then-write here was the one CF3 site outside the lock —
756 // ultra-fuzz Run 2, S1). Held only for the atomic UPDATE, never the gates.
757 {
758 let _deploy_guard = deploy_lock.lock().await;
759 crate::runs::advance_tier(&pool, &cfg.id, "host", &published.version, Some(run_id.0))
760 .await?;
761 }
762 // Terminal verdict: unlike the phase pings above (best-effort), a dropped
763 // pass/fail write leaves the run wedged at `building`. Log it loudly if it
764 // fails — the startup reconcile (main) is the backstop that settles such a
765 // row on the next restart.
766 if let Err(e) = crate::runs::mark_passed(&pool, run_id).await {
767 tracing::error!(run_id = %run_id, error = %e, "persisting host-green verdict failed; run may show stale 'building' until restart-reconcile");
768 }
769 tracing::info!(version = %published.version, "host pipeline green; ready to promote to next tier");
770 } else {
771 // Pull the first red gate's typed summary into the run so the API
772 // answers "which gate, and why" — not just "failed".
773 let summary = crate::runs::first_failed_gate_summary(&pool, &cfg.id, run_id)
774 .await
775 .unwrap_or_else(|| "host pipeline red".to_string());
776 if let Err(e) = crate::runs::mark_failed(&pool, run_id, &summary).await {
777 tracing::error!(run_id = %run_id, error = %e, "persisting host-red verdict failed; run may show stale 'building' until restart-reconcile");
778 }
779 tracing::warn!(version = %published.version, "host pipeline red; not advancing tier_state");
780 }
781 Ok(())
782 }
783
784 async fn read_pkg_version(cargo_toml: &Path) -> Result<Version> {
785 let raw = tokio::fs::read_to_string(cargo_toml).await?;
786 let parsed: toml::Value = toml::from_str(&raw)?;
787 let v = parsed
788 .get("package")
789 .and_then(|p| p.get("version"))
790 .and_then(|v| v.as_str())
791 .context("package.version not found")?;
792 Version::parse(v).with_context(|| format!("parsing package.version `{v}`"))
793 }
794
795 fn tail(buf: &[u8], max: usize) -> String {
796 let s = String::from_utf8_lossy(buf);
797 if s.len() <= max {
798 return s.into_owned();
799 }
800 // `s.len() - max` can land mid-codepoint; walk forward to the next char
801 // boundary so the slice never panics (returns slightly fewer than `max`
802 // bytes in that case). `floor_char_boundary` is still unstable, so do it by
803 // hand.
804 let mut start = s.len() - max;
805 while start < s.len() && !s.is_char_boundary(start) {
806 start += 1;
807 }
808 s[start..].to_string()
809 }
810
811 /// Copy `worktree/<entry.src>` into `staged/<entry.dst>`. Handles file or
812 /// directory sources transparently. Missing source policy depends on
813 /// `entry.required`:
814 /// - required=true -> error (build fails)
815 /// - required=false -> log warn + skip (e.g. older shas missing a dir)
816 ///
817 /// Uses `cp -a` to preserve modes/symlinks/etc; parent of dst is created if
818 /// needed so entries like `dst = "docs/assumptions.toml"` work without
819 /// extra config.
820 async fn stage_entry(
821 worktree: &Path,
822 staged: &Path,
823 entry: &crate::config::ReleaseEntry,
824 ) -> Result<()> {
825 let src = worktree.join(&entry.src);
826 let dst = staged.join(&entry.dst);
827 if !src.exists() {
828 if entry.required {
829 anyhow::bail!(
830 "required release_contents source missing: {}",
831 src.display()
832 );
833 }
834 tracing::warn!(src = %src.display(), "release_contents source missing (optional); skipping");
835 return Ok(());
836 }
837 if let Some(parent) = dst.parent() {
838 tokio::fs::create_dir_all(parent)
839 .await
840 .with_context(|| format!("create staged parent {}", parent.display()))?;
841 }
842 // Multiple entries with the same dst (e.g. site-docs/public/ +
843 // site-docs/examples/ both landing under docs/) need additive merging.
844 // `cp -a SRC/. DST/` copies SRC's contents into DST without overwriting
845 // the dst dir itself; that's the merge-friendly form when dst is a dir
846 // that may already exist from a prior entry. For non-dir sources or a
847 // missing dst we fall back to the plain `cp -a SRC DST` form.
848 let merge_into_existing_dir = src.is_dir() && dst.is_dir();
849 let mut cmd = Command::new("cp");
850 cmd.arg("-a");
851 if merge_into_existing_dir {
852 let mut src_arg = src.clone().into_os_string();
853 src_arg.push("/.");
854 cmd.arg(src_arg);
855 let mut dst_arg = dst.clone().into_os_string();
856 dst_arg.push("/");
857 cmd.arg(dst_arg);
858 } else {
859 cmd.arg(&src).arg(&dst);
860 }
861 let out = cmd
862 .output()
863 .await
864 .with_context(|| format!("spawning cp for {} -> {}", src.display(), dst.display()))?;
865 anyhow::ensure!(
866 out.status.success(),
867 "stage {} -> {}: {}",
868 src.display(),
869 dst.display(),
870 String::from_utf8_lossy(&out.stderr),
871 );
872 Ok(())
873 }
874
875 #[cfg(test)]
876 mod tests {
877 use super::{
878 BuildArtifact, accept_intake, check_build_host, checkout_aux_repos, gate_intake,
879 runtime_hostname, stage_and_gate, tail,
880 };
881 use crate::config::{AppConfig, TestTarget};
882 use crate::domain::{GitSha, RunId, Version};
883 use crate::topology::{AuxRepo, BackupConfig, CanaryPolicy, Gate, RepoConfig, Tier, Topology};
884 use sqlx::SqlitePool;
885 use sqlx::sqlite::SqlitePoolOptions;
886 use std::collections::BTreeMap;
887 use std::path::PathBuf;
888 use std::sync::Arc;
889
890 /// Post-build pipeline fixture: an in-memory store with the `host` tier
891 /// seeded, a synthetic worktree holding a fake primary binary, and a
892 /// build_runs row in flight. Returns everything `stage_and_gate` needs plus
893 /// the tempdir root (drop it to clean up) and the run/version it seeded.
894 ///
895 /// `gates` is the host tier's gate list: `[]` is the green path;
896 /// `[Gate::ManualConfirm]` is a deterministic red — with no prior operator
897 /// confirmation row that gate blocks, and it shells out to nothing.
898 async fn stage_fixture(
899 gates: Vec<Gate>,
900 ) -> (
901 SqlitePool,
902 Arc<AppConfig>,
903 Arc<Topology>,
904 BuildArtifact,
905 RunId,
906 Version,
907 tempfile::TempDir,
908 ) {
909 let tmp = tempfile::tempdir().unwrap();
910 let release_root = tmp.path().join("release-root");
911 let worktree = tmp.path().join("worktree");
912 let bin_dir = worktree.join("target").join("release");
913 tokio::fs::create_dir_all(&bin_dir).await.unwrap();
914 let bin_path = bin_dir.join("makenotwork");
915 tokio::fs::write(&bin_path, b"#!/bin/false\nfake sando artifact\n")
916 .await
917 .unwrap();
918
919 let pool = SqlitePoolOptions::new()
920 .max_connections(1)
921 .connect("sqlite::memory:")
922 .await
923 .unwrap();
924 sqlx::migrate!("./migrations").run(&pool).await.unwrap();
925
926 // gate_runs and tier_state FK into `tiers`; the pipeline only touches host.
927 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('host', 0, 1, 'sequential')")
928 .execute(&pool)
929 .await
930 .unwrap();
931 sqlx::query("INSERT INTO tier_state (tier) VALUES ('host')")
932 .execute(&pool)
933 .await
934 .unwrap();
935
936 let version = Version::parse("1.2.3").unwrap();
937 let git_sha = GitSha::parse("abc1234").unwrap();
938 // gate_runs.version and the `SET artifact_path` UPDATE both need the row.
939 sqlx::query(
940 "INSERT INTO versions (version, git_sha, built_at, artifact_path)
941 VALUES (?, ?, datetime('now'), '')",
942 )
943 .bind(version.to_string())
944 .bind(git_sha.to_string())
945 .execute(&pool)
946 .await
947 .unwrap();
948
949 let run_id = crate::runs::create(
950 &pool,
951 &crate::domain::AppId::default(),
952 &git_sha.to_string(),
953 )
954 .await
955 .unwrap();
956
957 let cfg = AppConfig {
958 page_smoke_cmd: None,
959 platform: None,
960 code_smoke_env: BTreeMap::default(),
961 id: crate::domain::AppId::default(),
962 topology_path: PathBuf::from("/tmp/test-sando.toml"),
963 build_host: Some("test-host".into()),
964 workdir: tmp.path().to_path_buf(),
965 release_root: release_root.clone(),
966 scratch_db_url: None,
967 scratch_owner_role: "makenotwork".into(),
968 boot_smoke_port: 18181,
969 code_smoke_port: 18182,
970 bin_names: vec!["makenotwork".into()],
971 logs_root: tmp.path().join("logs"),
972 release_contents: vec![],
973 cargo_target_dir: None,
974 gate_timeout_secs: 2400,
975 companions: Vec::new(),
976 test_targets: vec![TestTarget {
977 dir: PathBuf::from("server"),
978 aux_repo: None,
979 features: vec!["fast-tests".into()],
980 all_features: false,
981 scratch_db: true,
982 }],
983 migration_checks: vec![],
984 frontend_builds: vec![],
985 backup_max_age_hours: 48,
986 };
987
988 let topo = Topology {
989 repo: Some(RepoConfig {
990 bare_path: "/tmp/test.git".into(),
991 branch: "main".into(),
992 upstream: None,
993 }),
994 backup: vec![BackupConfig {
995 name: "server".into(),
996 source: "file:///tmp/test-backup.sql".into(),
997 local_path: "/tmp/local-backup.sql".into(),
998 }],
999 tiers: vec![Tier {
1000 public_url: None,
1001 name: "host".into(),
1002 provisioned: true,
1003 gates,
1004 canary: CanaryPolicy::Sequential,
1005 nodes: Vec::new(),
1006 }],
1007 aux_repos: Vec::new(),
1008 };
1009
1010 let art = BuildArtifact {
1011 version: version.clone(),
1012 git_sha,
1013 worktree,
1014 binary_paths: vec![bin_path],
1015 companion_paths: Vec::new(),
1016 };
1017
1018 (
1019 pool,
1020 Arc::new(cfg),
1021 Arc::new(topo),
1022 art,
1023 run_id,
1024 version,
1025 tmp,
1026 )
1027 }
1028
1029 // ---- intake through the seam ----
1030
1031 /// The `ArtifactRecord` Bento would have written for a staged bundle.
1032 async fn record_for(staged: &std::path::Path, version: &str, target: &str) -> String {
1033 use ops_artifact::{ArtifactRecord, GateRecord, Manifest, Provenance, Scope, Verdict};
1034 let computed = crate::bundle::digest_dir(staged).await.unwrap();
1035 let manifest = Manifest::parse(&computed.manifest).unwrap();
1036 let at = chrono::DateTime::<chrono::Utc>::from_timestamp(1_754_000_000, 0).unwrap();
1037 ArtifactRecord::new(
1038 "bento",
1039 manifest,
1040 Provenance {
1041 app: "pom".into(),
1042 version: version.into(),
1043 tag: format!("pom-v{version}"),
1044 git_sha: "a".repeat(40),
1045 target: target.into(),
1046 build_host: "astra".into(),
1047 toolchain: "rustc 1.97.0".into(),
1048 built_at: at,
1049 },
1050 vec![GateRecord::new(
1051 "prebuild",
1052 Scope::Artifact,
1053 Verdict::Passed,
1054 "prebuild passed in 90s",
1055 at,
1056 )],
1057 )
1058 .unwrap()
1059 .to_json()
1060 }
1061
1062 #[tokio::test]
1063 async fn an_accepted_artifact_is_published_gated_and_advances_the_tier() {
1064 // The seam: an artifact Sando did not build reaches the same published,
1065 // gated, tier-advanced end state a Sando-built one does. No worktree
1066 // exists anywhere in this test, which is the point — everything from
1067 // `finalize_local_release` onward stopped caring where the bytes came
1068 // from.
1069 let (pool, cfg, topo, _art, run_id, _version, tmp) = stage_fixture(vec![]).await;
1070 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1071
1072 let staged = cfg.release_root.join("staging").join("intake-1");
1073 tokio::fs::create_dir_all(&staged).await.unwrap();
1074 tokio::fs::write(staged.join("makenotwork"), b"bytes built elsewhere")
1075 .await
1076 .unwrap();
1077 let record = record_for(&staged, "1.2.3", "linux/aarch64").await;
1078
1079 // Two calls now, on purpose: the route answers its caller on the first
1080 // and spawns the second. Acceptance is what the producer waits for.
1081 let published = accept_intake(&pool, &cfg, &staged, &record, run_id)
1082 .await
1083 .expect("the bytes are believed");
1084 gate_intake(
1085 pool.clone(),
1086 cfg.clone(),
1087 topo,
1088 published,
1089 crate::events::channel(),
1090 run_id,
1091 deploy_lock,
1092 )
1093 .await
1094 .expect("a green intake settles the run");
1095
1096 // Published content-addressed, and the staging dir is gone: renamed,
1097 // not copied.
1098 let (digest, staged_path, platform): (Option<String>, Option<String>, Option<String>) =
1099 sqlx::query_as(
1100 "SELECT bundle_digest, staged_path, platform FROM build_runs WHERE id = ?",
1101 )
1102 .bind(run_id.0)
1103 .fetch_one(&pool)
1104 .await
1105 .unwrap();
1106 let digest = digest.expect("bundle_digest recorded");
1107 let staged_path = staged_path.expect("staged_path recorded");
1108 assert_eq!(digest.len(), 64);
1109 assert!(
1110 !staged.exists(),
1111 "staging was renamed into the release root"
1112 );
1113 assert_eq!(
1114 std::path::Path::new(&staged_path),
1115 tmp.path()
1116 .join("release-root")
1117 .join("releases")
1118 .join(&digest[..16]),
1119 );
1120
1121 // The platform came off the record's provenance and is on the row. This
1122 // is what makes two bundles of one version tellable apart later.
1123 assert_eq!(platform.as_deref(), Some("linux/aarch64"));
1124
1125 // Green gates advanced the host tier, exactly as a build would have.
1126 let (result, _summary) = run_result(&pool, run_id).await;
1127 assert_eq!(result, "passed");
1128 let (current, _prev) = tier_versions(&pool, "host").await;
1129 assert_eq!(current.as_deref(), Some("1.2.3"));
1130 }
1131
1132 #[tokio::test]
1133 async fn an_intake_whose_bytes_drifted_never_reaches_the_gates() {
1134 // Identity is decided before anything else happens to the bundle, so a
1135 // record vouching for one set of bytes arriving with another fails the
1136 // run rather than gating and shipping.
1137 let (pool, cfg, topo, _art, run_id, _version, _tmp) = stage_fixture(vec![]).await;
1138 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1139
1140 let staged = cfg.release_root.join("staging").join("intake-1");
1141 tokio::fs::create_dir_all(&staged).await.unwrap();
1142 tokio::fs::write(staged.join("makenotwork"), b"bytes built elsewhere")
1143 .await
1144 .unwrap();
1145 let record = record_for(&staged, "1.2.3", "linux/aarch64").await;
1146 tokio::fs::write(staged.join("makenotwork"), b"other bytes entirely")
1147 .await
1148 .unwrap();
1149
1150 // Refused by ACCEPTANCE, not by gating — which is what lets the route
1151 // answer the producer with the refusal instead of a `202`-shaped lie.
1152 let _ = (&topo, &deploy_lock);
1153 let err = accept_intake(&pool, &cfg, &staged, &record, run_id)
1154 .await
1155 .expect_err("a drifted bundle is refused");
1156 assert!(err.to_string().contains("makenotwork"), "{err}");
1157
1158 // Nothing advanced, and the bytes were left where they were.
1159 let (current, _prev) = tier_versions(&pool, "host").await;
1160 assert_eq!(current, None);
1161 assert!(staged.exists(), "a refused intake leaves the bytes alone");
1162 }
1163
1164 #[tokio::test]
1165 async fn a_gate_that_reads_source_refuses_against_an_accepted_artifact() {
1166 // The boundary showing up at runtime. `code_smoke` is artifact-scoped —
1167 // it compiles frontends and boots the binary against a scratch DB — and
1168 // an accepted artifact has no checkout for it to read. It has to say so
1169 // rather than pass on having run nothing, which is what an unwrapped
1170 // `worktree.join(..)` against an empty path would have done.
1171 let (pool, cfg, _topo, _art, run_id, version, _tmp) = stage_fixture(vec![]).await;
1172 let ctx = crate::gates::GateCtx {
1173 pool,
1174 cfg,
1175 tier: crate::domain::TierId::new("host"),
1176 version,
1177 worktree: None,
1178 bundle: Some(PathBuf::from("/r/abc")),
1179 events: crate::events::channel(),
1180 nodes: Vec::new(),
1181 build_id: Some(run_id.0),
1182 public_url: None,
1183 aux_dirs: std::collections::HashMap::default(),
1184 };
1185 let outcome = ctx
1186 .worktree_for(crate::domain::GateKind::CodeSmoke)
1187 .expect_err("no worktree means no source-reading gate");
1188 assert!(!outcome.is_passed());
1189 let crate::outcome::GateStatus::Failed { failure } = &outcome.status else {
1190 panic!("expected a failure, got {:?}", outcome.status)
1191 };
1192 assert!(
1193 matches!(failure, crate::outcome::GateFailure::NeedsSource { .. }),
1194 "{failure:?}"
1195 );
1196 assert!(
1197 failure.summary().contains("built elsewhere"),
1198 "{}",
1199 failure.summary()
1200 );
1201 }
1202
1203 #[tokio::test]
1204 async fn migrations_come_from_the_bundle_before_the_worktree() {
1205 // What the gate proves has to be what ships. Migrations staged into the
1206 // bundle are inside its digest; the same files sitting in a checkout are
1207 // not, and a checkout can be edited between the dry run and the deploy.
1208 // So when both hold a copy, the bundle wins.
1209 let (pool, cfg, _topo, _art, run_id, version, tmp) = stage_fixture(vec![]).await;
1210 let bundle = tmp.path().join("bundle");
1211 let worktree = tmp.path().join("wt");
1212 for root in [&bundle, &worktree] {
1213 tokio::fs::create_dir_all(root.join("server/migrations"))
1214 .await
1215 .unwrap();
1216 }
1217 let ctx = crate::gates::GateCtx {
1218 pool,
1219 cfg,
1220 tier: crate::domain::TierId::new("host"),
1221 version,
1222 worktree: Some(worktree.clone()),
1223 bundle: Some(bundle.clone()),
1224 events: crate::events::channel(),
1225 nodes: Vec::new(),
1226 build_id: Some(run_id.0),
1227 public_url: None,
1228 aux_dirs: std::collections::HashMap::default(),
1229 };
1230 assert_eq!(
1231 ctx.migrations_dir(std::path::Path::new("server/migrations")),
1232 Some(bundle.join("server/migrations")),
1233 );
1234
1235 // The worktree is the fallback for a build whose config has not opted
1236 // into bundling them yet, which is every MNW build before this lands.
1237 let ctx = crate::gates::GateCtx {
1238 bundle: Some(tmp.path().join("empty-bundle")),
1239 ..ctx
1240 };
1241 assert_eq!(
1242 ctx.migrations_dir(std::path::Path::new("server/migrations")),
1243 Some(worktree.join("server/migrations")),
1244 );
1245
1246 // And neither is not silently green: an accepted artifact whose builder
1247 // did not bundle its migrations has nothing to dry-run, and the gate
1248 // has to be told so rather than restore a dump and report success.
1249 let ctx = crate::gates::GateCtx {
1250 worktree: None,
1251 ..ctx
1252 };
1253 assert_eq!(
1254 ctx.migrations_dir(std::path::Path::new("server/migrations")),
1255 None
1256 );
1257 }
1258
1259 // ---- checkout_aux_repos ----
1260
1261 async fn git_in(dir: &std::path::Path, args: &[&str]) {
1262 let out = tokio::process::Command::new("git")
1263 .args(["-c", "user.email=t@t", "-c", "user.name=t"])
1264 .current_dir(dir)
1265 .args(args)
1266 .output()
1267 .await
1268 .unwrap();
1269 assert!(
1270 out.status.success(),
1271 "git {args:?}: {}",
1272 String::from_utf8_lossy(&out.stderr)
1273 );
1274 }
1275
1276 /// A minimal `Config` whose only field this test path reads is `workdir`.
1277 fn cfg_with_workdir(workdir: PathBuf) -> AppConfig {
1278 AppConfig {
1279 page_smoke_cmd: None,
1280 platform: None,
1281 code_smoke_env: BTreeMap::default(),
1282 id: crate::domain::AppId::default(),
1283 topology_path: PathBuf::from("/tmp/test-sando.toml"),
1284 build_host: Some("test-host".into()),
1285 workdir,
1286 release_root: PathBuf::from("/tmp/rr"),
1287 scratch_db_url: None,
1288 scratch_owner_role: "makenotwork".into(),
1289 boot_smoke_port: 18181,
1290 code_smoke_port: 18182,
1291 bin_names: vec!["makenotwork".into()],
1292 logs_root: PathBuf::from("/tmp/logs"),
1293 release_contents: vec![],
1294 cargo_target_dir: None,
1295 gate_timeout_secs: 2400,
1296 companions: Vec::new(),
1297 test_targets: vec![],
1298 migration_checks: vec![],
1299 frontend_builds: vec![],
1300 backup_max_age_hours: 48,
1301 }
1302 }
1303
1304 fn topo_with_aux(aux_repos: Vec<AuxRepo>) -> Topology {
1305 Topology {
1306 repo: Some(RepoConfig {
1307 bare_path: "/tmp/x.git".into(),
1308 branch: "main".into(),
1309 upstream: None,
1310 }),
1311 backup: vec![BackupConfig {
1312 name: "server".into(),
1313 source: "s".into(),
1314 local_path: "/tmp/d".into(),
1315 }],
1316 tiers: vec![],
1317 aux_repos,
1318 }
1319 }
1320
1321 #[tokio::test]
1322 async fn a_gate_looks_where_the_aux_checkout_actually_landed() {
1323 // The two halves of the aux-repo test_target path: checkout_aux_repos
1324 // writes the tree, and GateCtx::target_dir reads it. Nothing but this
1325 // stops one from being changed without the other, and the failure would
1326 // be a warn-and-skip — a green gate that ran one crate fewer.
1327 let tmp = tempfile::tempdir().unwrap();
1328 let src = tmp.path().join("docengine-src");
1329 tokio::fs::create_dir_all(&src).await.unwrap();
1330 git_in(&src, &["init", "-q", "-b", "main"]).await;
1331 tokio::fs::write(src.join("Cargo.toml"), b"[package]\nname = \"docengine\"\n")
1332 .await
1333 .unwrap();
1334 git_in(&src, &["add", "."]).await;
1335 git_in(&src, &["commit", "-q", "-m", "one"]).await;
1336
1337 let workdir = tmp.path().join("work");
1338 tokio::fs::create_dir_all(&workdir).await.unwrap();
1339 let cfg = cfg_with_workdir(workdir.clone());
1340 let topo = topo_with_aux(vec![AuxRepo {
1341 name: "docengine".into(),
1342 bare_path: tmp
1343 .path()
1344 .join("docengine.git")
1345 .to_string_lossy()
1346 .into_owned(),
1347 upstream: src.to_string_lossy().into_owned(),
1348 branch: "main".into(),
1349 // Nested, as the real one is: it must not be mistaken for a path
1350 // under the per-sha worktree.
1351 checkout_dir: "Libraries/docengine".into(),
1352 }]);
1353 checkout_aux_repos(&cfg, &topo).await.unwrap();
1354
1355 let ctx = crate::gates::GateCtx {
1356 pool: sqlx::SqlitePool::connect_lazy("sqlite::memory:").unwrap(),
1357 cfg: Arc::new(cfg.clone()),
1358 tier: crate::domain::TierId::new("host"),
1359 version: "0.1.0".parse().unwrap(),
1360 worktree: Some(workdir.join("abc123")),
1361 bundle: None,
1362 events: crate::events::channel(),
1363 nodes: Vec::new(),
1364 build_id: None,
1365 public_url: None,
1366 aux_dirs: super::aux_checkout_dirs(&cfg, &topo),
1367 };
1368 let target = crate::config::TestTarget {
1369 dir: PathBuf::new(),
1370 aux_repo: Some("docengine".into()),
1371 features: Vec::new(),
1372 all_features: true,
1373 scratch_db: false,
1374 };
1375 let resolved = ctx.target_dir(&target).expect("aux repo is checked out");
1376 assert!(
1377 resolved.join("Cargo.toml").is_file(),
1378 "gate would skip the aux target as absent; resolved {}",
1379 resolved.display(),
1380 );
1381 assert!(
1382 !resolved.starts_with(ctx.worktree.as_ref().unwrap()),
1383 "an aux checkout is a sibling of the worktree, not under it",
1384 );
1385 }
1386
1387 #[tokio::test]
1388 async fn checkout_aux_repos_places_repo_beside_worktree_and_refreshes_to_branch_head() {
1389 let tmp = tempfile::tempdir().unwrap();
1390
1391 // An "upstream" source repo with a marker file on main.
1392 let src = tmp.path().join("synckit-src");
1393 tokio::fs::create_dir_all(&src).await.unwrap();
1394 git_in(&src, &["init", "-q", "-b", "main"]).await;
1395 tokio::fs::write(src.join("VERSION"), b"v1").await.unwrap();
1396 git_in(&src, &["add", "."]).await;
1397 git_in(&src, &["commit", "-q", "-m", "one"]).await;
1398
1399 let workdir = tmp.path().join("work");
1400 tokio::fs::create_dir_all(&workdir).await.unwrap();
1401 let cfg = cfg_with_workdir(workdir.clone());
1402 let topo = topo_with_aux(vec![AuxRepo {
1403 name: "synckit".into(),
1404 bare_path: tmp
1405 .path()
1406 .join("synckit.git")
1407 .to_string_lossy()
1408 .into_owned(),
1409 upstream: src.to_string_lossy().into_owned(),
1410 branch: "main".into(),
1411 checkout_dir: "synckit".into(),
1412 }]);
1413
1414 // First build: the aux repo lands at workdir/synckit at v1.
1415 checkout_aux_repos(&cfg, &topo).await.unwrap();
1416 let dest = workdir.join("synckit");
1417 assert_eq!(
1418 tokio::fs::read(dest.join("VERSION")).await.unwrap(),
1419 b"v1",
1420 "aux repo checked out beside the worktree",
1421 );
1422
1423 // Upstream advances; a later build refreshes the shared checkout to HEAD.
1424 tokio::fs::write(src.join("VERSION"), b"v2").await.unwrap();
1425 git_in(&src, &["add", "."]).await;
1426 git_in(&src, &["commit", "-q", "-m", "two"]).await;
1427 checkout_aux_repos(&cfg, &topo).await.unwrap();
1428 assert_eq!(
1429 tokio::fs::read(dest.join("VERSION")).await.unwrap(),
1430 b"v2",
1431 "aux checkout refreshed to the new branch HEAD",
1432 );
1433
1434 // The aux bare carries no build-trigger hook.
1435 assert!(
1436 !tmp.path().join("synckit.git/hooks/post-receive").exists(),
1437 "aux bare must be hookless",
1438 );
1439 }
1440
1441 #[tokio::test]
1442 async fn checkout_aux_repos_is_a_noop_without_aux_repos() {
1443 let tmp = tempfile::tempdir().unwrap();
1444 let cfg = cfg_with_workdir(tmp.path().to_path_buf());
1445 checkout_aux_repos(&cfg, &topo_with_aux(vec![]))
1446 .await
1447 .unwrap();
1448 }
1449
1450 #[tokio::test]
1451 async fn checkout_aux_repos_fails_on_an_unresolvable_branch() {
1452 let tmp = tempfile::tempdir().unwrap();
1453 let src = tmp.path().join("src");
1454 tokio::fs::create_dir_all(&src).await.unwrap();
1455 git_in(&src, &["init", "-q", "-b", "main"]).await;
1456 tokio::fs::write(src.join("f"), b"x").await.unwrap();
1457 git_in(&src, &["add", "."]).await;
1458 git_in(&src, &["commit", "-q", "-m", "c"]).await;
1459
1460 let cfg = cfg_with_workdir(tmp.path().join("work"));
1461 let topo = topo_with_aux(vec![AuxRepo {
1462 name: "synckit".into(),
1463 bare_path: tmp.path().join("s.git").to_string_lossy().into_owned(),
1464 upstream: src.to_string_lossy().into_owned(),
1465 branch: "nonexistent".into(),
1466 checkout_dir: "synckit".into(),
1467 }]);
1468 let err = checkout_aux_repos(&cfg, &topo).await.unwrap_err();
1469 assert!(
1470 format!("{err:#}").contains("synckit"),
1471 "error names the aux repo: {err:#}",
1472 );
1473 }
1474
1475 async fn tier_versions(pool: &SqlitePool, tier: &str) -> (Option<String>, Option<String>) {
1476 sqlx::query_as("SELECT current_version, previous_version FROM tier_state WHERE tier = ?")
1477 .bind(tier)
1478 .fetch_one(pool)
1479 .await
1480 .unwrap()
1481 }
1482
1483 async fn run_result(pool: &SqlitePool, run_id: RunId) -> (String, Option<String>) {
1484 sqlx::query_as("SELECT result, failure_summary FROM build_runs WHERE id = ?")
1485 .bind(run_id.0)
1486 .fetch_one(pool)
1487 .await
1488 .unwrap()
1489 }
1490
1491 #[tokio::test]
1492 async fn stage_and_gate_stages_advances_and_flips_the_symlink_when_gates_are_green() {
1493 let (pool, cfg, topo, art, run_id, version, tmp) = stage_fixture(vec![]).await;
1494 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1495
1496 stage_and_gate(
1497 pool.clone(),
1498 cfg.clone(),
1499 topo,
1500 art,
1501 crate::events::channel(),
1502 run_id,
1503 deploy_lock,
1504 )
1505 .await
1506 .expect("green host pipeline returns Ok");
1507
1508 // Tier advanced to the built version (previous was NULL -> stays NULL).
1509 let (current, previous) = tier_versions(&pool, "host").await;
1510 assert_eq!(current.as_deref(), Some(version.to_string().as_str()));
1511 assert_eq!(previous, None);
1512
1513 // Run settled green.
1514 let (result, summary) = run_result(&pool, run_id).await;
1515 assert_eq!(result, "passed");
1516 assert_eq!(summary, None);
1517
1518 // Identity: the build row carries the bundle digest (64 hex) and the
1519 // content-addressed dir it was published to (releases/<digest16>).
1520 let (digest, staged_path): (Option<String>, Option<String>) =
1521 sqlx::query_as("SELECT bundle_digest, staged_path FROM build_runs WHERE id = ?")
1522 .bind(run_id.0)
1523 .fetch_one(&pool)
1524 .await
1525 .unwrap();
1526 let digest = digest.expect("bundle_digest recorded");
1527 let staged_path = staged_path.expect("staged_path recorded");
1528 assert_eq!(digest.len(), 64);
1529 let releases = tmp.path().join("release-root").join("releases");
1530 assert_eq!(
1531 std::path::Path::new(&staged_path),
1532 releases.join(&digest[..16]),
1533 "bundle is published content-addressed at releases/<digest16>"
1534 );
1535
1536 // versions.artifact_path points at the primary binary inside that dir,
1537 // and it exists on disk.
1538 let staged_bin: String =
1539 sqlx::query_scalar("SELECT artifact_path FROM versions WHERE version = ?")
1540 .bind(version.to_string())
1541 .fetch_one(&pool)
1542 .await
1543 .unwrap();
1544 let expected_bin = releases.join(&digest[..16]).join("makenotwork");
1545 assert_eq!(staged_bin, expected_bin.to_string_lossy());
1546 assert!(
1547 expected_bin.exists(),
1548 "staged binary missing at {expected_bin:?}"
1549 );
1550
1551 // The bundle carries its MANIFEST (for node-side verification), and the
1552 // recorded digest recomputes over the published dir (MANIFEST excluded).
1553 assert!(
1554 releases.join(&digest[..16]).join("MANIFEST").exists(),
1555 "MANIFEST written into the bundle"
1556 );
1557 let recomputed = crate::bundle::digest_dir(std::path::Path::new(&staged_path))
1558 .await
1559 .unwrap();
1560 assert_eq!(
1561 digest, recomputed.full,
1562 "recorded digest matches the bundle"
1563 );
1564
1565 // The `current` symlink flipped to the content-addressed release.
1566 let link = tmp.path().join("release-root").join("current");
1567 let target = std::fs::read_link(&link).expect("current is a symlink");
1568 assert_eq!(target, PathBuf::from(format!("releases/{}", &digest[..16])));
1569 }
1570
1571 #[tokio::test]
1572 async fn stage_and_gate_marks_the_run_failed_and_does_not_advance_when_a_gate_is_red() {
1573 // ManualConfirm with no prior confirmation row blocks deterministically.
1574 let (pool, cfg, topo, art, run_id, _version, _tmp) =
1575 stage_fixture(vec![Gate::ManualConfirm]).await;
1576 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1577
1578 // A red gate is a pipeline outcome, not an error: the fn records the
1579 // failure and returns Ok so the spawned task settles the run cleanly.
1580 stage_and_gate(
1581 pool.clone(),
1582 cfg,
1583 topo,
1584 art,
1585 crate::events::channel(),
1586 run_id,
1587 deploy_lock,
1588 )
1589 .await
1590 .expect("a red gate settles the run, it does not error out");
1591
1592 // Tier did NOT advance — still the seeded NULL/NULL.
1593 let (current, previous) = tier_versions(&pool, "host").await;
1594 assert_eq!(current, None);
1595 assert_eq!(previous, None);
1596
1597 // Run settled red with a non-empty summary.
1598 let (result, summary) = run_result(&pool, run_id).await;
1599 assert_eq!(result, "failed");
1600 assert!(
1601 summary.as_deref().is_some_and(|s| !s.is_empty()),
1602 "failed run must carry a summary, got {summary:?}"
1603 );
1604 }
1605
1606 #[test]
1607 fn check_build_host_accepts_matching_host() {
1608 assert!(check_build_host("fw13", "fw13").is_ok());
1609 }
1610
1611 #[test]
1612 fn check_build_host_refuses_mismatched_host() {
1613 // A daemon misdeployed onto prod (e.g. a Hetzner host) must refuse.
1614 let err = check_build_host("alpha-west-1", "fw13")
1615 .unwrap_err()
1616 .to_string();
1617 assert!(err.contains("refusing to build"), "{err}");
1618 assert!(
1619 err.contains("alpha-west-1") && err.contains("fw13"),
1620 "{err}"
1621 );
1622 }
1623
1624 #[test]
1625 fn runtime_hostname_reads_a_nonempty_trimmed_name() {
1626 let h = runtime_hostname().expect("hostname readable on Linux");
1627 assert!(!h.is_empty());
1628 assert_eq!(h, h.trim(), "must be trimmed");
1629 }
1630
1631 #[test]
1632 fn tail_does_not_panic_on_multibyte_boundary() {
1633 // Each '€' is 3 bytes; a byte cap landing mid-codepoint must not panic.
1634 let s = "€".repeat(10); // 30 bytes
1635 for max in 1..=30 {
1636 let out = tail(s.as_bytes(), max);
1637 assert!(out.len() <= max, "max={max} got {} bytes", out.len());
1638 // Result is always valid UTF-8 made only of whole '€'s.
1639 assert!(out.chars().all(|c| c == '€'), "max={max}: {out:?}");
1640 }
1641 }
1642
1643 #[test]
1644 fn tail_returns_whole_input_when_under_cap() {
1645 assert_eq!(tail(b"hello", 100), "hello");
1646 }
1647 }
1648