Skip to main content

max / makenotwork

63.1 KB · 1645 lines History Blame Raw
1 //! Build orchestration: resolve a sha to a worktree, read the server version,
2 //! shell out to `cargo build --release`, record a `versions` row.
3 //!
4 //! Runs as a tokio task spawned from `POST /rebuild`; the HTTP request
5 //! returns the version id immediately and the task drives the rest.
6
7 use crate::config::AppConfig;
8 use crate::deploy;
9 use crate::domain::{GitSha, Platform, RunId, TierId, Version};
10 use crate::gates::{self, GateCtx};
11 use crate::git;
12 use crate::topology::Topology;
13 use anyhow::{Context, Result};
14 use chrono::Utc;
15 use sqlx::SqlitePool;
16 use std::path::{Path, PathBuf};
17 use std::sync::Arc;
18 use tokio::process::Command;
19
20 #[derive(Debug, Clone)]
21 pub struct BuildArtifact {
22 pub version: Version,
23 pub git_sha: GitSha,
24 pub worktree: PathBuf,
25 /// One entry per `cfg.bin_names` in declared order. First is the primary
26 /// (referenced by the systemd unit's ExecStart). Paths are inside the
27 /// worktree's `target/release/`.
28 pub binary_paths: Vec<PathBuf>,
29 /// `(companion name, built binary path)` for each `cfg.companions`, built
30 /// from the same worktree/sha as the server. Staged into the release bundle
31 /// under `companions/<name>/<bin>` and installed by the nodes that opt in.
32 pub companion_paths: Vec<(String, PathBuf)>,
33 }
34
35 /// The live kernel hostname (`/proc/sys/kernel/hostname`, trimmed). Linux-only,
36 /// which Sando is. The build-host guard reads this rather than `$HOSTNAME`
37 /// (not reliably exported) so the check reflects the actual machine.
38 fn runtime_hostname() -> Result<String> {
39 let raw = std::fs::read_to_string("/proc/sys/kernel/hostname")
40 .context("reading /proc/sys/kernel/hostname for the build-host guard")?;
41 Ok(raw.trim().to_string())
42 }
43
44 /// Pure half of the build-host guard: fail unless the live host matches the
45 /// configured build host. Split out from [`enforce_build_host`] so it is unit-
46 /// testable without depending on the test machine's hostname.
47 fn check_build_host(actual: &str, expected: &str) -> Result<()> {
48 anyhow::ensure!(
49 actual == expected,
50 "refusing to build on host {actual}: configured build host is {expected} \
51 (never build on a prod/serving node)",
52 );
53 Ok(())
54 }
55
56 /// Refuse to build unless this daemon is running on the configured build host.
57 fn enforce_build_host(expected: &str) -> Result<()> {
58 check_build_host(&runtime_hostname()?, expected)
59 }
60
61 pub async fn run(
62 pool: SqlitePool,
63 cfg: Arc<AppConfig>,
64 topo: Arc<Topology>,
65 sha: GitSha,
66 events: crate::events::EventTx,
67 run_id: RunId,
68 ) -> Result<BuildArtifact> {
69 // Build-host guard: refuse to compile anywhere but the configured build
70 // host. The build runs locally (`cargo build` in cfg.workdir), so a sandod
71 // misdeployed onto a prod/serving node would otherwise build there — exactly
72 // the "never build on prod" rule. Enforced before any cargo invocation so a
73 // wrong-host daemon fails fast with a clear message rather than compiling.
74 let Some(build_host) = cfg.build_host.as_deref() else {
75 anyhow::bail!(
76 "{} declares no build_host, which makes it intake-only: Sando does not compile \
77 it. Ship it with POST /intake, from a builder that does.",
78 cfg.id
79 );
80 };
81 enforce_build_host(build_host)?;
82
83 let repo = topo.repo.as_ref().with_context(|| {
84 format!(
85 "{} declares no [repo]: it is intake-only and Sando has no source to check out",
86 cfg.id
87 )
88 })?;
89 let worktree = cfg.workdir.join(sha.as_str());
90 let bare = PathBuf::from(&repo.bare_path);
91
92 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Fetching)
93 .await
94 .ok();
95
96 // Pull-based ingestion: if an upstream remote is configured, fetch the
97 // deploy branch so a just-pushed sha is locally resolvable. A fetch
98 // failure is non-fatal — the sha may already be present from a prior
99 // fetch or a direct push; the presence check below is the real gate.
100 if let Some(upstream) = repo.upstream.as_deref()
101 && let Err(e) = git::fetch_upstream(&bare, upstream, &repo.branch).await
102 {
103 tracing::warn!(error = %e, upstream, "upstream fetch failed; proceeding with current bare-repo state");
104 }
105 anyhow::ensure!(
106 git::sha_present(&bare, sha.as_str()).await?,
107 "sha {} not present in bare repo {} after fetch — push the commit to the upstream remote first",
108 sha.as_str(),
109 bare.display(),
110 );
111
112 git::checkout_worktree(&bare, sha.as_str(), &worktree).await?;
113
114 // Check out any auxiliary repos (e.g. synckit) beside the worktree so a
115 // cross-repo path dependency in the server or a companion resolves. Fails the
116 // build if an aux repo can't be assembled — a companion that silently fails to
117 // find its source would fail the compile downstream with a worse message.
118 checkout_aux_repos(&cfg, &topo).await?;
119
120 let server_dir = worktree.join("server");
121 let version = read_pkg_version(&server_dir.join("Cargo.toml"))
122 .await
123 .with_context(|| format!("reading version from {}/Cargo.toml", server_dir.display()))?;
124 crate::runs::set_version(&pool, run_id, &version).await.ok();
125
126 // sqlx compile-time query checking needs a live DB with the current schema.
127 // We point cargo at the scratch DB and prep it (drop public, re-migrate)
128 // before invoking cargo build. The same DB is reset again by
129 // `migration_dry_run` later if it runs as a gate.
130 let mut cargo_cmd = Command::new("cargo");
131 cargo_cmd
132 .arg("build")
133 .arg("--release")
134 .current_dir(&server_dir)
135 .kill_on_drop(true);
136 // Shared build cache across per-sha worktrees: reuse one target dir so an
137 // incremental diff doesn't clean-compile from scratch. Serialized builds
138 // make this contention-free. Unset → cargo's default per-worktree target/.
139 if let Some(target) = cfg.cargo_target_dir.as_deref() {
140 cargo_cmd.env("CARGO_TARGET_DIR", target);
141 }
142 if let Some(scratch_url) = cfg.scratch_db_url.as_deref() {
143 tracing::info!(sha = %sha.as_str(), "preparing scratch DB schema for sqlx compile-time checks");
144 crate::gates::reset_scratch(scratch_url, &cfg.scratch_owner_role)
145 .await
146 .context("scratch DB reset before build")?;
147 crate::gates::run_migrator(scratch_url, &server_dir.join("migrations"))
148 .await
149 .context("applying MNW migrations to scratch DB before build")?;
150 cargo_cmd.env("DATABASE_URL", scratch_url);
151 } else {
152 tracing::warn!("scratch_db_url unset; sqlx will fall back to offline mode and may fail");
153 }
154
155 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Compiling)
156 .await
157 .ok();
158 tracing::info!(sha = %sha, version = %version, dir = %server_dir.display(), "cargo build --release start");
159 crate::events::emit(
160 &events,
161 crate::events::Event::BuildStart {
162 sha: sha.clone(),
163 version: version.clone(),
164 },
165 );
166 let started = std::time::Instant::now();
167 let out = cargo_cmd.output().await.context("spawning cargo build")?;
168 let elapsed_s = started.elapsed().as_secs();
169 if !out.status.success() {
170 tracing::error!(sha = %sha, version = %version, elapsed_s, "cargo build --release failed");
171 crate::events::emit(
172 &events,
173 crate::events::Event::BuildFailed {
174 sha: sha.clone(),
175 version: version.clone(),
176 elapsed_s,
177 },
178 );
179 // Settle the run with the headline compiler diagnostic (not the raw
180 // 4 KB tail) so `GET /runs/{id}` answers "why" without a journald dive.
181 let summary = crate::classify::classify_compile_error(&out.stdout, &out.stderr).summary();
182 if let Err(e) = crate::runs::mark_failed(&pool, run_id, &summary).await {
183 tracing::error!(run_id = %run_id, error = %e, "persisting compile-fail verdict failed; run may show stale 'building' until restart-reconcile");
184 }
185 anyhow::bail!(
186 "cargo build --release failed:\n{}",
187 tail(&out.stderr, 4_000)
188 );
189 }
190 tracing::info!(sha = %sha, version = %version, elapsed_s, "cargo build --release ok");
191 crate::events::emit(
192 &events,
193 crate::events::Event::BuildOk {
194 sha: sha.clone(),
195 version: version.clone(),
196 elapsed_s,
197 },
198 );
199
200 // Binaries land under `<target>/release/`; with a shared target dir that's
201 // not inside the worktree, so resolve it the same way cargo did above.
202 let release_dir = cfg
203 .cargo_target_dir
204 .as_deref()
205 .map_or_else(|| server_dir.join("target/release"), |t| t.join("release"));
206 let mut binary_paths = Vec::with_capacity(cfg.bin_names.len());
207 for name in &cfg.bin_names {
208 let p = release_dir.join(name);
209 anyhow::ensure!(p.exists(), "expected binary at {} after build", p.display());
210 binary_paths.push(p);
211 }
212 // Primary binary path is the one we record in `versions.artifact_path`
213 // (everything downstream — promote, rollback — looks it up by version).
214 let primary = binary_paths[0].clone();
215
216 // Companion crates (e.g. mnw-cli): built from the SAME worktree/sha so a
217 // service that shares the server's internal-API contract cannot drift out of
218 // lockstep (the 2026-07-09 git-hosting outage). A companion build failure
219 // fails the whole pipeline — the server never ships without its companions.
220 let mut companion_paths = Vec::with_capacity(cfg.companions.len());
221 for c in &cfg.companions {
222 let bin = build_companion(&worktree, &cfg, c).await?;
223 companion_paths.push((c.name.clone(), bin));
224 }
225
226 sqlx::query(
227 "INSERT OR IGNORE INTO versions (app, version, git_sha, built_at, artifact_path)
228 VALUES (?, ?, ?, ?, ?)",
229 )
230 .bind(&cfg.id)
231 .bind(&version)
232 .bind(&sha)
233 .bind(Utc::now().to_rfc3339())
234 .bind(primary.to_string_lossy().as_ref())
235 .execute(&pool)
236 .await?;
237
238 Ok(BuildArtifact {
239 version,
240 git_sha: sha,
241 worktree,
242 binary_paths,
243 companion_paths,
244 })
245 }
246
247 /// Fetch and check out every configured auxiliary repo at `cfg.workdir/<checkout_dir>`,
248 /// so a cross-repo path dependency built from the main worktree resolves (wiki
249 /// [[sando-overview]]; the synckit split, task sando-18cdb32f).
250 ///
251 /// Each aux repo is a fixed, shared checkout refreshed to `branch` HEAD — not
252 /// per-sha — because the dependent's relative path resolves to that fixed spot
253 /// regardless of the main sha, and builds serialize. A fetch failure is a warning
254 /// (the branch may already be present from a prior build); an unresolvable branch
255 /// after that is fatal, as is a failed worktree — a half-assembled source tree
256 /// must fail the build here, loudly, not as a downstream compile error.
257 /// Where an aux repo's checkout lands. The single derivation: `checkout_aux_repos`
258 /// creates it here and `GateCtx::aux_dirs` resolves `test_target`s against it, so
259 /// the two cannot drift into looking in different places.
260 pub fn aux_checkout_dir(cfg: &AppConfig, aux: &crate::topology::AuxRepo) -> PathBuf {
261 cfg.workdir.join(&aux.checkout_dir)
262 }
263
264 /// Every aux repo's checkout dir, keyed by name — what `GateCtx::aux_dirs` holds.
265 pub fn aux_checkout_dirs(
266 cfg: &AppConfig,
267 topo: &Topology,
268 ) -> std::collections::HashMap<String, PathBuf> {
269 topo.aux_repos
270 .iter()
271 .map(|a| (a.name.clone(), aux_checkout_dir(cfg, a)))
272 .collect()
273 }
274
275 pub async fn checkout_aux_repos(cfg: &AppConfig, topo: &Topology) -> Result<()> {
276 for aux in &topo.aux_repos {
277 let bare = PathBuf::from(&aux.bare_path);
278 git::ensure_bare_repo_no_hook(&bare)
279 .await
280 .with_context(|| format!("aux repo {}: init bare {}", aux.name, aux.bare_path))?;
281 if let Err(e) = git::fetch_upstream(&bare, &aux.upstream, &aux.branch).await {
282 tracing::warn!(
283 aux = %aux.name, error = %e,
284 "aux repo fetch failed; proceeding with current bare-repo state",
285 );
286 }
287 let sha = git::resolve_ref(&bare, &aux.branch).await.with_context(|| {
288 format!(
289 "aux repo {}: branch {} not resolvable after fetch — is {} reachable with that branch?",
290 aux.name, aux.branch, aux.upstream,
291 )
292 })?;
293 let dest = aux_checkout_dir(cfg, aux);
294 git::checkout_worktree(&bare, &sha, &dest)
295 .await
296 .with_context(|| {
297 format!(
298 "aux repo {}: checking out {} ({}) at {}",
299 aux.name,
300 aux.branch,
301 sha,
302 dest.display()
303 )
304 })?;
305 tracing::info!(
306 aux = %aux.name, branch = %aux.branch, sha = %sha, dest = %dest.display(),
307 "aux repo checked out beside worktree",
308 );
309 }
310 Ok(())
311 }
312
313 /// Build one companion crate from the worktree, returning its release binary
314 /// path. Mirrors the server build's target-dir handling (shared
315 /// `cargo_target_dir` when set, for incremental reuse; else the crate's own
316 /// `target/`). A companion is an API client, not a sqlx crate, so it needs no
317 /// scratch DB. A non-zero exit propagates and fails the pipeline.
318 async fn build_companion(
319 worktree: &Path,
320 cfg: &AppConfig,
321 c: &crate::config::Companion,
322 ) -> Result<PathBuf> {
323 let dir = worktree.join(&c.manifest_dir);
324 anyhow::ensure!(
325 dir.join("Cargo.toml").exists(),
326 "companion {}: no Cargo.toml at {}",
327 c.name,
328 dir.display(),
329 );
330 // Match the server build: no `--locked` (the pipeline builds whatever the
331 // sha pins; a stale lock shouldn't block a deploy the server build allows).
332 let mut cmd = Command::new("cargo");
333 cmd.arg("build")
334 .arg("--release")
335 .current_dir(&dir)
336 .kill_on_drop(true);
337 let release_dir = if let Some(target) = cfg.cargo_target_dir.as_deref() {
338 cmd.env("CARGO_TARGET_DIR", target);
339 target.join("release")
340 } else {
341 dir.join("target/release")
342 };
343 tracing::info!(companion = %c.name, dir = %dir.display(), "cargo build --release (companion) start");
344 let started = std::time::Instant::now();
345 let out = cmd
346 .output()
347 .await
348 .context("spawning cargo build for companion")?;
349 if !out.status.success() {
350 anyhow::bail!(
351 "companion {} build failed:\n{}",
352 c.name,
353 tail(&out.stderr, 4_000),
354 );
355 }
356 let bin = release_dir.join(&c.bin);
357 anyhow::ensure!(
358 bin.exists(),
359 "companion {} produced no binary at {} after build",
360 c.name,
361 bin.display(),
362 );
363 tracing::info!(companion = %c.name, elapsed_s = started.elapsed().as_secs(), "companion build ok");
364 Ok(bin)
365 }
366
367 /// Full host-tier pipeline: build, stage the bundle into the host's
368 /// release_root, run the host tier's configured gates, advance tier_state
369 /// for "host" if all pass. Errors propagate back to the spawned task and
370 /// get logged. (Tier was called "mm" pre-Session-1; renamed to "host"
371 /// since sandod runs on whatever machine ends up being the Sando host.)
372 pub async fn build_and_run_host(
373 pool: SqlitePool,
374 cfg: Arc<AppConfig>,
375 topo: Arc<Topology>,
376 sha: GitSha,
377 events: crate::events::EventTx,
378 run_id: RunId,
379 deploy_lock: Arc<tokio::sync::Mutex<()>>,
380 ) -> Result<()> {
381 let art = run(
382 pool.clone(),
383 cfg.clone(),
384 topo.clone(),
385 sha,
386 events.clone(),
387 run_id,
388 )
389 .await?;
390
391 stage_and_gate(pool, cfg, topo, art, events, run_id, deploy_lock).await
392 }
393
394 /// A bundle assembled on disk and not yet published: the point both the build
395 /// path and the intake path have to reach before anything else can happen to it.
396 ///
397 /// The two paths reach it differently. A build assembles it out of a worktree
398 /// (binaries, `release_contents`, companions); an intake is handed it already
399 /// assembled, with no worktree anywhere. Everything after this point — hashing,
400 /// publishing, recording identity, gating, advancing — is the same work, and
401 /// used to be welded to the assembling half inside `stage_and_gate`.
402 struct StagedBundle {
403 version: Version,
404 /// The staging dir under `release_root/staging/`, pre-publish.
405 staging: PathBuf,
406 /// What the bundle is built to run on, when it is known. A Sando build
407 /// inherits the app's declared platform; an intake takes it from the
408 /// record's provenance.
409 platform: Option<Platform>,
410 }
411
412 /// Post-build half of the host pipeline: stage the artifact into the host's
413 /// release_root, run the host tier's gates, and advance `tier_state` for
414 /// "host" iff all pass. Split from [`build_and_run_host`] at the `run()`
415 /// boundary so the staging/gating/advance logic is reachable in tests from a
416 /// synthetic [`BuildArtifact`] — no real `cargo build --release` required.
417 pub async fn stage_and_gate(
418 pool: SqlitePool,
419 cfg: Arc<AppConfig>,
420 topo: Arc<Topology>,
421 art: BuildArtifact,
422 events: crate::events::EventTx,
423 run_id: RunId,
424 deploy_lock: Arc<tokio::sync::Mutex<()>>,
425 ) -> Result<()> {
426 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Staging)
427 .await
428 .ok();
429 let staged = assemble_from_source(&cfg, &art, run_id).await?;
430 let published = publish(&pool, &cfg, staged, run_id).await?;
431 record_and_gate(
432 pool,
433 cfg,
434 topo,
435 published,
436 events,
437 run_id,
438 deploy_lock,
439 Some(art.worktree),
440 )
441 .await
442 }
443
444 /// Accept an artifact built elsewhere and take it through the same host-tier
445 /// gating and advance a Sando-built one gets.
446 ///
447 /// This is the whole point of the seam. `intake::accept` publishes the bundle
448 /// content-addressed once it has proved the bytes are the ones the record
449 /// vouches for, which lands it at exactly the state [`publish`] leaves a
450 /// Sando-built bundle in — so the two paths join at [`record_and_gate`] and
451 /// nothing downstream knows or cares which one it came from.
452 ///
453 /// `staged` must already sit under `release_root/staging/` (publishing is an
454 /// atomic same-filesystem rename); getting the bytes there is the transport's
455 /// job, not this function's.
456 #[allow(clippy::too_many_arguments)]
457 pub async fn accept_intake(
458 pool: &SqlitePool,
459 cfg: &AppConfig,
460 staged: &Path,
461 record_json: &str,
462 run_id: RunId,
463 ) -> Result<Published> {
464 crate::runs::set_phase(pool, run_id, crate::runs::Phase::Staging)
465 .await
466 .ok();
467
468 let accepted = crate::intake::accept(&cfg.release_root, staged, record_json)
469 .await
470 .map_err(|e| anyhow::anyhow!("{e}"))?;
471
472 let version = Version::parse(&accepted.record.provenance.version).with_context(|| {
473 format!(
474 "artifact record carries version `{}`, which is not semver",
475 accepted.record.provenance.version
476 )
477 })?;
478 let platform = Platform::parse(&accepted.record.provenance.target).with_context(|| {
479 format!(
480 "artifact record carries target `{}`, which is not `os/arch`",
481 accepted.record.provenance.target
482 )
483 })?;
484 let git_sha = GitSha::parse(&accepted.record.provenance.git_sha).with_context(|| {
485 format!(
486 "artifact record carries git_sha `{}`",
487 accepted.record.provenance.git_sha
488 )
489 })?;
490
491 crate::runs::set_version(pool, run_id, &version).await.ok();
492 upsert_version_row(
493 pool,
494 &cfg.id,
495 &version,
496 &git_sha,
497 &accepted.released.join(cfg.primary_bin()),
498 )
499 .await?;
500
501 let published = Published {
502 version,
503 released: accepted.released,
504 digest_full: accepted.record.digest.to_string(),
505 platform: Some(platform),
506 };
507 record_identity(pool, cfg, &published, run_id).await?;
508 Ok(published)
509 }
510
511 /// Gate an artifact that has already been accepted.
512 ///
513 /// Split from [`accept_intake`] so the two can be answered on different clocks.
514 /// Acceptance is fast and is the producer's business — it either believes the
515 /// bytes or names the file that drifted — so the caller waits for it and gets
516 /// the verdict. Gating is Sando's business and can take an hour, so the caller
517 /// does not.
518 ///
519 /// An intake carries no worktree, and the gates that need one refuse rather than
520 /// resolve against nothing. That is the boundary showing up in the type:
521 /// artifact-scoped gates belong to the builder (wiki [[sando-bento-boundary]]),
522 /// so a tier that asks Sando to re-run them against an accepted artifact is
523 /// misconfigured and should be told so.
524 pub async fn gate_intake(
525 pool: SqlitePool,
526 cfg: Arc<AppConfig>,
527 topo: Arc<Topology>,
528 published: Published,
529 events: crate::events::EventTx,
530 run_id: RunId,
531 deploy_lock: Arc<tokio::sync::Mutex<()>>,
532 ) -> Result<()> {
533 record_and_gate(
534 pool,
535 cfg,
536 topo,
537 published,
538 events,
539 run_id,
540 deploy_lock,
541 None,
542 )
543 .await
544 }
545
546 /// Record the `versions` label row for an artifact that arrived rather than was
547 /// built here. The build path writes its own inside [`run`]; this is the same
548 /// row for the path that never ran a compiler.
549 async fn upsert_version_row(
550 pool: &SqlitePool,
551 app: &crate::domain::AppId,
552 version: &Version,
553 git_sha: &GitSha,
554 artifact_path: &Path,
555 ) -> Result<()> {
556 sqlx::query(
557 "INSERT OR IGNORE INTO versions (app, version, git_sha, built_at, artifact_path)
558 VALUES (?, ?, ?, ?, ?)",
559 )
560 .bind(app)
561 .bind(version)
562 .bind(git_sha)
563 .bind(Utc::now().to_rfc3339())
564 .bind(artifact_path.to_string_lossy().as_ref())
565 .execute(pool)
566 .await?;
567 Ok(())
568 }
569
570 /// Assemble a bundle out of a worktree: binaries, `release_contents`, companions.
571 async fn assemble_from_source(
572 cfg: &AppConfig,
573 art: &BuildArtifact,
574 run_id: RunId,
575 ) -> Result<StagedBundle> {
576 // Stage the bundle into `staging/<build_id>/` — a private scratch dir, not
577 // yet a release. It is published content-addressed below, once its digest is
578 // known. This is what makes overwrite unexpressible (wiki
579 // [[release-artifact-identity]]): a build never touches another build's dir.
580 let staging =
581 deploy::stage_local_bundle(&cfg.release_root, run_id.0, &art.binary_paths).await?;
582
583 // Stage every entry from cfg.release_contents into the staged bundle. This is
584 // how non-binary version-coupled content (static assets, docs, error-pages,
585 // ...) makes it into the atomic deploy bundle. Projects opt in via daemon
586 // config — the sando code carries no MNW-specific knowledge.
587 for entry in &cfg.release_contents {
588 stage_entry(&art.worktree, &staging, entry).await?;
589 }
590
591 // Stage companion binaries as `companions/<name>` (the file itself) so they
592 // ride the same atomic bundle rsync to the nodes, and a node can locate its
593 // companion source from the logical name alone — no bin-filename coupling in
594 // the topology. The nodes that opt in install them post-swap (see
595 // deploy::deploy_remote).
596 if !art.companion_paths.is_empty() {
597 let dst_dir = staging.join("companions");
598 tokio::fs::create_dir_all(&dst_dir)
599 .await
600 .with_context(|| format!("create staged companions dir {}", dst_dir.display()))?;
601 for (name, built) in &art.companion_paths {
602 let dst = dst_dir.join(name);
603 tokio::fs::copy(built, &dst).await.with_context(|| {
604 format!(
605 "stage companion {name}: {} -> {}",
606 built.display(),
607 dst.display()
608 )
609 })?;
610 }
611 }
612
613 Ok(StagedBundle {
614 version: art.version.clone(),
615 staging,
616 platform: cfg.platform.clone(),
617 })
618 }
619
620 /// A bundle that has been hashed and published content-addressed. Both paths
621 /// produce one; nothing downstream can tell them apart.
622 ///
623 /// Public because the intake route now hands one from `accept_intake` to
624 /// `gate_intake`: proving the bytes answers the producer, gating them does not,
625 /// so the two run on different clocks and the value passes between them.
626 #[derive(Debug)]
627 pub struct Published {
628 version: Version,
629 released: PathBuf,
630 digest_full: String,
631 platform: Option<Platform>,
632 }
633
634 /// Hash the assembled bundle, write its MANIFEST, and publish it at
635 /// `releases/<digest16>`.
636 ///
637 /// The intake path does not call this: `intake::accept` does the same three
638 /// steps itself, because it has to hash the bytes to verify them and hashing
639 /// them twice would be the one place the two implementations could disagree.
640 async fn publish(
641 pool: &SqlitePool,
642 cfg: &AppConfig,
643 staged: StagedBundle,
644 run_id: RunId,
645 ) -> Result<Published> {
646 // Content identity: hash the fully-staged bundle, write its MANIFEST into the
647 // bundle (for node-side verification), then publish it at `releases/<digest16>`.
648 // The digest is now load-bearing — a hashing failure fails the build rather
649 // than shipping an unidentifiable artifact.
650 let digest = crate::bundle::digest_dir(&staged.staging)
651 .await
652 .context("hashing the staged bundle for content addressing")?;
653 tokio::fs::write(
654 staged.staging.join(crate::bundle::MANIFEST_NAME),
655 digest.manifest.as_bytes(),
656 )
657 .await
658 .context("writing bundle MANIFEST")?;
659 let released =
660 deploy::finalize_local_release(&cfg.release_root, &staged.staging, digest.short()).await?;
661
662 let staged_bin = released.join(cfg.primary_bin());
663 sqlx::query("UPDATE versions SET artifact_path = ? WHERE app = ? AND version = ?")
664 .bind(staged_bin.to_string_lossy().as_ref())
665 .bind(&cfg.id)
666 .bind(&staged.version)
667 .execute(pool)
668 .await?;
669
670 let published = Published {
671 version: staged.version,
672 released,
673 digest_full: digest.full,
674 platform: staged.platform,
675 };
676 record_identity(pool, cfg, &published, run_id).await?;
677 Ok(published)
678 }
679
680 /// Record the identity on the build row: the digest, the content-addressed dir
681 /// the bundle was published to, and what it runs on. This is what promote
682 /// resolves the artifact through, and burn-in/retention key on.
683 async fn record_identity(
684 pool: &SqlitePool,
685 cfg: &AppConfig,
686 published: &Published,
687 run_id: RunId,
688 ) -> Result<()> {
689 let released_path = published.released.to_string_lossy();
690 crate::runs::set_identity(pool, run_id, &published.digest_full, &released_path)
691 .await
692 .ok();
693 // Platform is what lets two bundles of one version be told apart, so a
694 // dropped write here would leave a pom artifact that can be placed nowhere
695 // (a node declaring a platform refuses an artifact that records none). Fail
696 // rather than ship an unplaceable bundle.
697 if let Some(p) = &published.platform {
698 crate::runs::set_platform(pool, run_id, p)
699 .await
700 .with_context(|| format!("recording platform {p} for {}", cfg.id))?;
701 }
702 Ok(())
703 }
704
705 /// The shared tail of both paths: run the host tier's gates against a published
706 /// bundle and advance `tier_state` iff all pass.
707 #[allow(clippy::too_many_arguments)]
708 async fn record_and_gate(
709 pool: SqlitePool,
710 cfg: Arc<AppConfig>,
711 topo: Arc<Topology>,
712 published: Published,
713 events: crate::events::EventTx,
714 run_id: RunId,
715 deploy_lock: Arc<tokio::sync::Mutex<()>>,
716 worktree: Option<PathBuf>,
717 ) -> Result<()> {
718 let host = topo
719 .tiers
720 .iter()
721 .find(|t| t.name.as_str() == "host")
722 .context("topology has no `host` tier")?;
723
724 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Gating)
725 .await
726 .ok();
727 let ctx = GateCtx {
728 pool: pool.clone(),
729 cfg: cfg.clone(),
730 tier: TierId::new("host"),
731 version: published.version.clone(),
732 worktree,
733 // The published bundle. `migration_dry_run` prefers it over the
734 // worktree, so the migrations it proves are the ones inside the digest
735 // rather than ones sitting beside them in a checkout.
736 bundle: Some(published.released.clone()),
737 events: events.clone(),
738 // Host runs build-time gates (cargo_test / migration_dry_run /
739 // boot_smoke) only — `node_health` never appears here, so there are no
740 // nodes to probe.
741 nodes: Vec::new(),
742 // These gates vouch for this build; record its id so promote can resolve
743 // the artifact through the evidence rather than a version string.
744 build_id: Some(run_id.0),
745 // Where checkout_aux_repos put each aux repo, so a test_target naming
746 // one resolves. Shared derivation, so the two cannot disagree.
747 public_url: None,
748 aux_dirs: aux_checkout_dirs(&cfg, &topo),
749 };
750 let failed = gates::run_all(&ctx, &host.gates).await?;
751
752 if failed.is_empty() {
753 // Advance the host tier through the single sealed forward-advance op, under
754 // deploy_lock so this can't interleave with a concurrent `/rollback host`
755 // (the old fetch-then-write here was the one CF3 site outside the lock —
756 // ultra-fuzz Run 2, S1). Held only for the atomic UPDATE, never the gates.
757 {
758 let _deploy_guard = deploy_lock.lock().await;
759 crate::runs::advance_tier(&pool, &cfg.id, "host", &published.version, Some(run_id.0))
760 .await?;
761 }
762 // Terminal verdict: unlike the phase pings above (best-effort), a dropped
763 // pass/fail write leaves the run wedged at `building`. Log it loudly if it
764 // fails — the startup reconcile (main) is the backstop that settles such a
765 // row on the next restart.
766 if let Err(e) = crate::runs::mark_passed(&pool, run_id).await {
767 tracing::error!(run_id = %run_id, error = %e, "persisting host-green verdict failed; run may show stale 'building' until restart-reconcile");
768 }
769 tracing::info!(version = %published.version, "host pipeline green; ready to promote to next tier");
770 } else {
771 // Pull the first red gate's typed summary into the run so the API
772 // answers "which gate, and why" — not just "failed".
773 let summary = crate::runs::first_failed_gate_summary(&pool, &cfg.id, &published.version)
774 .await
775 .unwrap_or_else(|| "host pipeline red".to_string());
776 if let Err(e) = crate::runs::mark_failed(&pool, run_id, &summary).await {
777 tracing::error!(run_id = %run_id, error = %e, "persisting host-red verdict failed; run may show stale 'building' until restart-reconcile");
778 }
779 tracing::warn!(version = %published.version, "host pipeline red; not advancing tier_state");
780 }
781 Ok(())
782 }
783
784 async fn read_pkg_version(cargo_toml: &Path) -> Result<Version> {
785 let raw = tokio::fs::read_to_string(cargo_toml).await?;
786 let parsed: toml::Value = toml::from_str(&raw)?;
787 let v = parsed
788 .get("package")
789 .and_then(|p| p.get("version"))
790 .and_then(|v| v.as_str())
791 .context("package.version not found")?;
792 Version::parse(v).with_context(|| format!("parsing package.version `{v}`"))
793 }
794
795 fn tail(buf: &[u8], max: usize) -> String {
796 let s = String::from_utf8_lossy(buf);
797 if s.len() <= max {
798 return s.into_owned();
799 }
800 // `s.len() - max` can land mid-codepoint; walk forward to the next char
801 // boundary so the slice never panics (returns slightly fewer than `max`
802 // bytes in that case). `floor_char_boundary` is still unstable, so do it by
803 // hand.
804 let mut start = s.len() - max;
805 while start < s.len() && !s.is_char_boundary(start) {
806 start += 1;
807 }
808 s[start..].to_string()
809 }
810
811 /// Copy `worktree/<entry.src>` into `staged/<entry.dst>`. Handles file or
812 /// directory sources transparently. Missing source policy depends on
813 /// `entry.required`:
814 /// - required=true -> error (build fails)
815 /// - required=false -> log warn + skip (e.g. older shas missing a dir)
816 ///
817 /// Uses `cp -a` to preserve modes/symlinks/etc; parent of dst is created if
818 /// needed so entries like `dst = "docs/assumptions.toml"` work without
819 /// extra config.
820 async fn stage_entry(
821 worktree: &Path,
822 staged: &Path,
823 entry: &crate::config::ReleaseEntry,
824 ) -> Result<()> {
825 let src = worktree.join(&entry.src);
826 let dst = staged.join(&entry.dst);
827 if !src.exists() {
828 if entry.required {
829 anyhow::bail!(
830 "required release_contents source missing: {}",
831 src.display()
832 );
833 }
834 tracing::warn!(src = %src.display(), "release_contents source missing (optional); skipping");
835 return Ok(());
836 }
837 if let Some(parent) = dst.parent() {
838 tokio::fs::create_dir_all(parent)
839 .await
840 .with_context(|| format!("create staged parent {}", parent.display()))?;
841 }
842 // Multiple entries with the same dst (e.g. site-docs/public/ +
843 // site-docs/examples/ both landing under docs/) need additive merging.
844 // `cp -a SRC/. DST/` copies SRC's contents into DST without overwriting
845 // the dst dir itself; that's the merge-friendly form when dst is a dir
846 // that may already exist from a prior entry. For non-dir sources or a
847 // missing dst we fall back to the plain `cp -a SRC DST` form.
848 let merge_into_existing_dir = src.is_dir() && dst.is_dir();
849 let mut cmd = Command::new("cp");
850 cmd.arg("-a");
851 if merge_into_existing_dir {
852 let mut src_arg = src.clone().into_os_string();
853 src_arg.push("/.");
854 cmd.arg(src_arg);
855 let mut dst_arg = dst.clone().into_os_string();
856 dst_arg.push("/");
857 cmd.arg(dst_arg);
858 } else {
859 cmd.arg(&src).arg(&dst);
860 }
861 let out = cmd
862 .output()
863 .await
864 .with_context(|| format!("spawning cp for {} -> {}", src.display(), dst.display()))?;
865 anyhow::ensure!(
866 out.status.success(),
867 "stage {} -> {}: {}",
868 src.display(),
869 dst.display(),
870 String::from_utf8_lossy(&out.stderr),
871 );
872 Ok(())
873 }
874
875 #[cfg(test)]
876 mod tests {
877 use super::{
878 BuildArtifact, accept_intake, check_build_host, checkout_aux_repos, gate_intake,
879 runtime_hostname, stage_and_gate, tail,
880 };
881 use crate::config::{AppConfig, TestTarget};
882 use crate::domain::{GitSha, RunId, Version};
883 use crate::topology::{AuxRepo, BackupConfig, CanaryPolicy, Gate, RepoConfig, Tier, Topology};
884 use sqlx::SqlitePool;
885 use sqlx::sqlite::SqlitePoolOptions;
886 use std::path::PathBuf;
887 use std::sync::Arc;
888
889 /// Post-build pipeline fixture: an in-memory store with the `host` tier
890 /// seeded, a synthetic worktree holding a fake primary binary, and a
891 /// build_runs row in flight. Returns everything `stage_and_gate` needs plus
892 /// the tempdir root (drop it to clean up) and the run/version it seeded.
893 ///
894 /// `gates` is the host tier's gate list: `[]` is the green path;
895 /// `[Gate::ManualConfirm]` is a deterministic red — with no prior operator
896 /// confirmation row that gate blocks, and it shells out to nothing.
897 async fn stage_fixture(
898 gates: Vec<Gate>,
899 ) -> (
900 SqlitePool,
901 Arc<AppConfig>,
902 Arc<Topology>,
903 BuildArtifact,
904 RunId,
905 Version,
906 tempfile::TempDir,
907 ) {
908 let tmp = tempfile::tempdir().unwrap();
909 let release_root = tmp.path().join("release-root");
910 let worktree = tmp.path().join("worktree");
911 let bin_dir = worktree.join("target").join("release");
912 tokio::fs::create_dir_all(&bin_dir).await.unwrap();
913 let bin_path = bin_dir.join("makenotwork");
914 tokio::fs::write(&bin_path, b"#!/bin/false\nfake sando artifact\n")
915 .await
916 .unwrap();
917
918 let pool = SqlitePoolOptions::new()
919 .max_connections(1)
920 .connect("sqlite::memory:")
921 .await
922 .unwrap();
923 sqlx::migrate!("./migrations").run(&pool).await.unwrap();
924
925 // gate_runs and tier_state FK into `tiers`; the pipeline only touches host.
926 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('host', 0, 1, 'sequential')")
927 .execute(&pool)
928 .await
929 .unwrap();
930 sqlx::query("INSERT INTO tier_state (tier) VALUES ('host')")
931 .execute(&pool)
932 .await
933 .unwrap();
934
935 let version = Version::parse("1.2.3").unwrap();
936 let git_sha = GitSha::parse("abc1234").unwrap();
937 // gate_runs.version and the `SET artifact_path` UPDATE both need the row.
938 sqlx::query(
939 "INSERT INTO versions (version, git_sha, built_at, artifact_path)
940 VALUES (?, ?, datetime('now'), '')",
941 )
942 .bind(version.to_string())
943 .bind(git_sha.to_string())
944 .execute(&pool)
945 .await
946 .unwrap();
947
948 let run_id = crate::runs::create(
949 &pool,
950 &crate::domain::AppId::default(),
951 &git_sha.to_string(),
952 )
953 .await
954 .unwrap();
955
956 let cfg = AppConfig {
957 page_smoke_cmd: None,
958 platform: None,
959 id: crate::domain::AppId::default(),
960 topology_path: PathBuf::from("/tmp/test-sando.toml"),
961 build_host: Some("test-host".into()),
962 workdir: tmp.path().to_path_buf(),
963 release_root: release_root.clone(),
964 scratch_db_url: None,
965 scratch_owner_role: "makenotwork".into(),
966 boot_smoke_port: 18181,
967 code_smoke_port: 18182,
968 bin_names: vec!["makenotwork".into()],
969 logs_root: tmp.path().join("logs"),
970 release_contents: vec![],
971 cargo_target_dir: None,
972 gate_timeout_secs: 2400,
973 companions: Vec::new(),
974 test_targets: vec![TestTarget {
975 dir: PathBuf::from("server"),
976 aux_repo: None,
977 features: vec!["fast-tests".into()],
978 all_features: false,
979 scratch_db: true,
980 }],
981 migration_checks: vec![],
982 frontend_builds: vec![],
983 backup_max_age_hours: 48,
984 };
985
986 let topo = Topology {
987 repo: Some(RepoConfig {
988 bare_path: "/tmp/test.git".into(),
989 branch: "main".into(),
990 upstream: None,
991 }),
992 backup: vec![BackupConfig {
993 name: "server".into(),
994 source: "file:///tmp/test-backup.sql".into(),
995 local_path: "/tmp/local-backup.sql".into(),
996 }],
997 tiers: vec![Tier {
998 public_url: None,
999 name: "host".into(),
1000 provisioned: true,
1001 gates,
1002 canary: CanaryPolicy::Sequential,
1003 nodes: Vec::new(),
1004 }],
1005 aux_repos: Vec::new(),
1006 };
1007
1008 let art = BuildArtifact {
1009 version: version.clone(),
1010 git_sha,
1011 worktree,
1012 binary_paths: vec![bin_path],
1013 companion_paths: Vec::new(),
1014 };
1015
1016 (
1017 pool,
1018 Arc::new(cfg),
1019 Arc::new(topo),
1020 art,
1021 run_id,
1022 version,
1023 tmp,
1024 )
1025 }
1026
1027 // ---- intake through the seam ----
1028
1029 /// The `ArtifactRecord` Bento would have written for a staged bundle.
1030 async fn record_for(staged: &std::path::Path, version: &str, target: &str) -> String {
1031 use ops_artifact::{ArtifactRecord, GateRecord, Manifest, Provenance, Scope, Verdict};
1032 let computed = crate::bundle::digest_dir(staged).await.unwrap();
1033 let manifest = Manifest::parse(&computed.manifest).unwrap();
1034 let at = chrono::DateTime::<chrono::Utc>::from_timestamp(1_754_000_000, 0).unwrap();
1035 ArtifactRecord::new(
1036 "bento",
1037 manifest,
1038 Provenance {
1039 app: "pom".into(),
1040 version: version.into(),
1041 tag: format!("pom-v{version}"),
1042 git_sha: "a".repeat(40),
1043 target: target.into(),
1044 build_host: "astra".into(),
1045 toolchain: "rustc 1.97.0".into(),
1046 built_at: at,
1047 },
1048 vec![GateRecord::new(
1049 "prebuild",
1050 Scope::Artifact,
1051 Verdict::Passed,
1052 "prebuild passed in 90s",
1053 at,
1054 )],
1055 )
1056 .unwrap()
1057 .to_json()
1058 }
1059
1060 #[tokio::test]
1061 async fn an_accepted_artifact_is_published_gated_and_advances_the_tier() {
1062 // The seam: an artifact Sando did not build reaches the same published,
1063 // gated, tier-advanced end state a Sando-built one does. No worktree
1064 // exists anywhere in this test, which is the point — everything from
1065 // `finalize_local_release` onward stopped caring where the bytes came
1066 // from.
1067 let (pool, cfg, topo, _art, run_id, _version, tmp) = stage_fixture(vec![]).await;
1068 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1069
1070 let staged = cfg.release_root.join("staging").join("intake-1");
1071 tokio::fs::create_dir_all(&staged).await.unwrap();
1072 tokio::fs::write(staged.join("makenotwork"), b"bytes built elsewhere")
1073 .await
1074 .unwrap();
1075 let record = record_for(&staged, "1.2.3", "linux/aarch64").await;
1076
1077 // Two calls now, on purpose: the route answers its caller on the first
1078 // and spawns the second. Acceptance is what the producer waits for.
1079 let published = accept_intake(&pool, &cfg, &staged, &record, run_id)
1080 .await
1081 .expect("the bytes are believed");
1082 gate_intake(
1083 pool.clone(),
1084 cfg.clone(),
1085 topo,
1086 published,
1087 crate::events::channel(),
1088 run_id,
1089 deploy_lock,
1090 )
1091 .await
1092 .expect("a green intake settles the run");
1093
1094 // Published content-addressed, and the staging dir is gone: renamed,
1095 // not copied.
1096 let (digest, staged_path, platform): (Option<String>, Option<String>, Option<String>) =
1097 sqlx::query_as(
1098 "SELECT bundle_digest, staged_path, platform FROM build_runs WHERE id = ?",
1099 )
1100 .bind(run_id.0)
1101 .fetch_one(&pool)
1102 .await
1103 .unwrap();
1104 let digest = digest.expect("bundle_digest recorded");
1105 let staged_path = staged_path.expect("staged_path recorded");
1106 assert_eq!(digest.len(), 64);
1107 assert!(
1108 !staged.exists(),
1109 "staging was renamed into the release root"
1110 );
1111 assert_eq!(
1112 std::path::Path::new(&staged_path),
1113 tmp.path()
1114 .join("release-root")
1115 .join("releases")
1116 .join(&digest[..16]),
1117 );
1118
1119 // The platform came off the record's provenance and is on the row. This
1120 // is what makes two bundles of one version tellable apart later.
1121 assert_eq!(platform.as_deref(), Some("linux/aarch64"));
1122
1123 // Green gates advanced the host tier, exactly as a build would have.
1124 let (result, _summary) = run_result(&pool, run_id).await;
1125 assert_eq!(result, "passed");
1126 let (current, _prev) = tier_versions(&pool, "host").await;
1127 assert_eq!(current.as_deref(), Some("1.2.3"));
1128 }
1129
1130 #[tokio::test]
1131 async fn an_intake_whose_bytes_drifted_never_reaches_the_gates() {
1132 // Identity is decided before anything else happens to the bundle, so a
1133 // record vouching for one set of bytes arriving with another fails the
1134 // run rather than gating and shipping.
1135 let (pool, cfg, topo, _art, run_id, _version, _tmp) = stage_fixture(vec![]).await;
1136 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1137
1138 let staged = cfg.release_root.join("staging").join("intake-1");
1139 tokio::fs::create_dir_all(&staged).await.unwrap();
1140 tokio::fs::write(staged.join("makenotwork"), b"bytes built elsewhere")
1141 .await
1142 .unwrap();
1143 let record = record_for(&staged, "1.2.3", "linux/aarch64").await;
1144 tokio::fs::write(staged.join("makenotwork"), b"other bytes entirely")
1145 .await
1146 .unwrap();
1147
1148 // Refused by ACCEPTANCE, not by gating — which is what lets the route
1149 // answer the producer with the refusal instead of a `202`-shaped lie.
1150 let _ = (&topo, &deploy_lock);
1151 let err = accept_intake(&pool, &cfg, &staged, &record, run_id)
1152 .await
1153 .expect_err("a drifted bundle is refused");
1154 assert!(err.to_string().contains("makenotwork"), "{err}");
1155
1156 // Nothing advanced, and the bytes were left where they were.
1157 let (current, _prev) = tier_versions(&pool, "host").await;
1158 assert_eq!(current, None);
1159 assert!(staged.exists(), "a refused intake leaves the bytes alone");
1160 }
1161
1162 #[tokio::test]
1163 async fn a_gate_that_reads_source_refuses_against_an_accepted_artifact() {
1164 // The boundary showing up at runtime. `code_smoke` is artifact-scoped —
1165 // it compiles frontends and boots the binary against a scratch DB — and
1166 // an accepted artifact has no checkout for it to read. It has to say so
1167 // rather than pass on having run nothing, which is what an unwrapped
1168 // `worktree.join(..)` against an empty path would have done.
1169 let (pool, cfg, _topo, _art, run_id, version, _tmp) = stage_fixture(vec![]).await;
1170 let ctx = crate::gates::GateCtx {
1171 pool,
1172 cfg,
1173 tier: crate::domain::TierId::new("host"),
1174 version,
1175 worktree: None,
1176 bundle: Some(PathBuf::from("/r/abc")),
1177 events: crate::events::channel(),
1178 nodes: Vec::new(),
1179 build_id: Some(run_id.0),
1180 public_url: None,
1181 aux_dirs: std::collections::HashMap::default(),
1182 };
1183 let outcome = ctx
1184 .worktree_for(crate::domain::GateKind::CodeSmoke)
1185 .expect_err("no worktree means no source-reading gate");
1186 assert!(!outcome.is_passed());
1187 let crate::outcome::GateStatus::Failed { failure } = &outcome.status else {
1188 panic!("expected a failure, got {:?}", outcome.status)
1189 };
1190 assert!(
1191 matches!(failure, crate::outcome::GateFailure::NeedsSource { .. }),
1192 "{failure:?}"
1193 );
1194 assert!(
1195 failure.summary().contains("built elsewhere"),
1196 "{}",
1197 failure.summary()
1198 );
1199 }
1200
1201 #[tokio::test]
1202 async fn migrations_come_from_the_bundle_before_the_worktree() {
1203 // What the gate proves has to be what ships. Migrations staged into the
1204 // bundle are inside its digest; the same files sitting in a checkout are
1205 // not, and a checkout can be edited between the dry run and the deploy.
1206 // So when both hold a copy, the bundle wins.
1207 let (pool, cfg, _topo, _art, run_id, version, tmp) = stage_fixture(vec![]).await;
1208 let bundle = tmp.path().join("bundle");
1209 let worktree = tmp.path().join("wt");
1210 for root in [&bundle, &worktree] {
1211 tokio::fs::create_dir_all(root.join("server/migrations"))
1212 .await
1213 .unwrap();
1214 }
1215 let ctx = crate::gates::GateCtx {
1216 pool,
1217 cfg,
1218 tier: crate::domain::TierId::new("host"),
1219 version,
1220 worktree: Some(worktree.clone()),
1221 bundle: Some(bundle.clone()),
1222 events: crate::events::channel(),
1223 nodes: Vec::new(),
1224 build_id: Some(run_id.0),
1225 public_url: None,
1226 aux_dirs: std::collections::HashMap::default(),
1227 };
1228 assert_eq!(
1229 ctx.migrations_dir(std::path::Path::new("server/migrations")),
1230 Some(bundle.join("server/migrations")),
1231 );
1232
1233 // The worktree is the fallback for a build whose config has not opted
1234 // into bundling them yet, which is every MNW build before this lands.
1235 let ctx = crate::gates::GateCtx {
1236 bundle: Some(tmp.path().join("empty-bundle")),
1237 ..ctx
1238 };
1239 assert_eq!(
1240 ctx.migrations_dir(std::path::Path::new("server/migrations")),
1241 Some(worktree.join("server/migrations")),
1242 );
1243
1244 // And neither is not silently green: an accepted artifact whose builder
1245 // did not bundle its migrations has nothing to dry-run, and the gate
1246 // has to be told so rather than restore a dump and report success.
1247 let ctx = crate::gates::GateCtx {
1248 worktree: None,
1249 ..ctx
1250 };
1251 assert_eq!(
1252 ctx.migrations_dir(std::path::Path::new("server/migrations")),
1253 None
1254 );
1255 }
1256
1257 // ---- checkout_aux_repos ----
1258
1259 async fn git_in(dir: &std::path::Path, args: &[&str]) {
1260 let out = tokio::process::Command::new("git")
1261 .args(["-c", "user.email=t@t", "-c", "user.name=t"])
1262 .current_dir(dir)
1263 .args(args)
1264 .output()
1265 .await
1266 .unwrap();
1267 assert!(
1268 out.status.success(),
1269 "git {args:?}: {}",
1270 String::from_utf8_lossy(&out.stderr)
1271 );
1272 }
1273
1274 /// A minimal `Config` whose only field this test path reads is `workdir`.
1275 fn cfg_with_workdir(workdir: PathBuf) -> AppConfig {
1276 AppConfig {
1277 page_smoke_cmd: None,
1278 platform: None,
1279 id: crate::domain::AppId::default(),
1280 topology_path: PathBuf::from("/tmp/test-sando.toml"),
1281 build_host: Some("test-host".into()),
1282 workdir,
1283 release_root: PathBuf::from("/tmp/rr"),
1284 scratch_db_url: None,
1285 scratch_owner_role: "makenotwork".into(),
1286 boot_smoke_port: 18181,
1287 code_smoke_port: 18182,
1288 bin_names: vec!["makenotwork".into()],
1289 logs_root: PathBuf::from("/tmp/logs"),
1290 release_contents: vec![],
1291 cargo_target_dir: None,
1292 gate_timeout_secs: 2400,
1293 companions: Vec::new(),
1294 test_targets: vec![],
1295 migration_checks: vec![],
1296 frontend_builds: vec![],
1297 backup_max_age_hours: 48,
1298 }
1299 }
1300
1301 fn topo_with_aux(aux_repos: Vec<AuxRepo>) -> Topology {
1302 Topology {
1303 repo: Some(RepoConfig {
1304 bare_path: "/tmp/x.git".into(),
1305 branch: "main".into(),
1306 upstream: None,
1307 }),
1308 backup: vec![BackupConfig {
1309 name: "server".into(),
1310 source: "s".into(),
1311 local_path: "/tmp/d".into(),
1312 }],
1313 tiers: vec![],
1314 aux_repos,
1315 }
1316 }
1317
1318 #[tokio::test]
1319 async fn a_gate_looks_where_the_aux_checkout_actually_landed() {
1320 // The two halves of the aux-repo test_target path: checkout_aux_repos
1321 // writes the tree, and GateCtx::target_dir reads it. Nothing but this
1322 // stops one from being changed without the other, and the failure would
1323 // be a warn-and-skip — a green gate that ran one crate fewer.
1324 let tmp = tempfile::tempdir().unwrap();
1325 let src = tmp.path().join("docengine-src");
1326 tokio::fs::create_dir_all(&src).await.unwrap();
1327 git_in(&src, &["init", "-q", "-b", "main"]).await;
1328 tokio::fs::write(src.join("Cargo.toml"), b"[package]\nname = \"docengine\"\n")
1329 .await
1330 .unwrap();
1331 git_in(&src, &["add", "."]).await;
1332 git_in(&src, &["commit", "-q", "-m", "one"]).await;
1333
1334 let workdir = tmp.path().join("work");
1335 tokio::fs::create_dir_all(&workdir).await.unwrap();
1336 let cfg = cfg_with_workdir(workdir.clone());
1337 let topo = topo_with_aux(vec![AuxRepo {
1338 name: "docengine".into(),
1339 bare_path: tmp
1340 .path()
1341 .join("docengine.git")
1342 .to_string_lossy()
1343 .into_owned(),
1344 upstream: src.to_string_lossy().into_owned(),
1345 branch: "main".into(),
1346 // Nested, as the real one is: it must not be mistaken for a path
1347 // under the per-sha worktree.
1348 checkout_dir: "Libraries/docengine".into(),
1349 }]);
1350 checkout_aux_repos(&cfg, &topo).await.unwrap();
1351
1352 let ctx = crate::gates::GateCtx {
1353 pool: sqlx::SqlitePool::connect_lazy("sqlite::memory:").unwrap(),
1354 cfg: Arc::new(cfg.clone()),
1355 tier: crate::domain::TierId::new("host"),
1356 version: "0.1.0".parse().unwrap(),
1357 worktree: Some(workdir.join("abc123")),
1358 bundle: None,
1359 events: crate::events::channel(),
1360 nodes: Vec::new(),
1361 build_id: None,
1362 public_url: None,
1363 aux_dirs: super::aux_checkout_dirs(&cfg, &topo),
1364 };
1365 let target = crate::config::TestTarget {
1366 dir: PathBuf::new(),
1367 aux_repo: Some("docengine".into()),
1368 features: Vec::new(),
1369 all_features: true,
1370 scratch_db: false,
1371 };
1372 let resolved = ctx.target_dir(&target).expect("aux repo is checked out");
1373 assert!(
1374 resolved.join("Cargo.toml").is_file(),
1375 "gate would skip the aux target as absent; resolved {}",
1376 resolved.display(),
1377 );
1378 assert!(
1379 !resolved.starts_with(ctx.worktree.as_ref().unwrap()),
1380 "an aux checkout is a sibling of the worktree, not under it",
1381 );
1382 }
1383
1384 #[tokio::test]
1385 async fn checkout_aux_repos_places_repo_beside_worktree_and_refreshes_to_branch_head() {
1386 let tmp = tempfile::tempdir().unwrap();
1387
1388 // An "upstream" source repo with a marker file on main.
1389 let src = tmp.path().join("synckit-src");
1390 tokio::fs::create_dir_all(&src).await.unwrap();
1391 git_in(&src, &["init", "-q", "-b", "main"]).await;
1392 tokio::fs::write(src.join("VERSION"), b"v1").await.unwrap();
1393 git_in(&src, &["add", "."]).await;
1394 git_in(&src, &["commit", "-q", "-m", "one"]).await;
1395
1396 let workdir = tmp.path().join("work");
1397 tokio::fs::create_dir_all(&workdir).await.unwrap();
1398 let cfg = cfg_with_workdir(workdir.clone());
1399 let topo = topo_with_aux(vec![AuxRepo {
1400 name: "synckit".into(),
1401 bare_path: tmp
1402 .path()
1403 .join("synckit.git")
1404 .to_string_lossy()
1405 .into_owned(),
1406 upstream: src.to_string_lossy().into_owned(),
1407 branch: "main".into(),
1408 checkout_dir: "synckit".into(),
1409 }]);
1410
1411 // First build: the aux repo lands at workdir/synckit at v1.
1412 checkout_aux_repos(&cfg, &topo).await.unwrap();
1413 let dest = workdir.join("synckit");
1414 assert_eq!(
1415 tokio::fs::read(dest.join("VERSION")).await.unwrap(),
1416 b"v1",
1417 "aux repo checked out beside the worktree",
1418 );
1419
1420 // Upstream advances; a later build refreshes the shared checkout to HEAD.
1421 tokio::fs::write(src.join("VERSION"), b"v2").await.unwrap();
1422 git_in(&src, &["add", "."]).await;
1423 git_in(&src, &["commit", "-q", "-m", "two"]).await;
1424 checkout_aux_repos(&cfg, &topo).await.unwrap();
1425 assert_eq!(
1426 tokio::fs::read(dest.join("VERSION")).await.unwrap(),
1427 b"v2",
1428 "aux checkout refreshed to the new branch HEAD",
1429 );
1430
1431 // The aux bare carries no build-trigger hook.
1432 assert!(
1433 !tmp.path().join("synckit.git/hooks/post-receive").exists(),
1434 "aux bare must be hookless",
1435 );
1436 }
1437
1438 #[tokio::test]
1439 async fn checkout_aux_repos_is_a_noop_without_aux_repos() {
1440 let tmp = tempfile::tempdir().unwrap();
1441 let cfg = cfg_with_workdir(tmp.path().to_path_buf());
1442 checkout_aux_repos(&cfg, &topo_with_aux(vec![]))
1443 .await
1444 .unwrap();
1445 }
1446
1447 #[tokio::test]
1448 async fn checkout_aux_repos_fails_on_an_unresolvable_branch() {
1449 let tmp = tempfile::tempdir().unwrap();
1450 let src = tmp.path().join("src");
1451 tokio::fs::create_dir_all(&src).await.unwrap();
1452 git_in(&src, &["init", "-q", "-b", "main"]).await;
1453 tokio::fs::write(src.join("f"), b"x").await.unwrap();
1454 git_in(&src, &["add", "."]).await;
1455 git_in(&src, &["commit", "-q", "-m", "c"]).await;
1456
1457 let cfg = cfg_with_workdir(tmp.path().join("work"));
1458 let topo = topo_with_aux(vec![AuxRepo {
1459 name: "synckit".into(),
1460 bare_path: tmp.path().join("s.git").to_string_lossy().into_owned(),
1461 upstream: src.to_string_lossy().into_owned(),
1462 branch: "nonexistent".into(),
1463 checkout_dir: "synckit".into(),
1464 }]);
1465 let err = checkout_aux_repos(&cfg, &topo).await.unwrap_err();
1466 assert!(
1467 format!("{err:#}").contains("synckit"),
1468 "error names the aux repo: {err:#}",
1469 );
1470 }
1471
1472 async fn tier_versions(pool: &SqlitePool, tier: &str) -> (Option<String>, Option<String>) {
1473 sqlx::query_as("SELECT current_version, previous_version FROM tier_state WHERE tier = ?")
1474 .bind(tier)
1475 .fetch_one(pool)
1476 .await
1477 .unwrap()
1478 }
1479
1480 async fn run_result(pool: &SqlitePool, run_id: RunId) -> (String, Option<String>) {
1481 sqlx::query_as("SELECT result, failure_summary FROM build_runs WHERE id = ?")
1482 .bind(run_id.0)
1483 .fetch_one(pool)
1484 .await
1485 .unwrap()
1486 }
1487
1488 #[tokio::test]
1489 async fn stage_and_gate_stages_advances_and_flips_the_symlink_when_gates_are_green() {
1490 let (pool, cfg, topo, art, run_id, version, tmp) = stage_fixture(vec![]).await;
1491 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1492
1493 stage_and_gate(
1494 pool.clone(),
1495 cfg.clone(),
1496 topo,
1497 art,
1498 crate::events::channel(),
1499 run_id,
1500 deploy_lock,
1501 )
1502 .await
1503 .expect("green host pipeline returns Ok");
1504
1505 // Tier advanced to the built version (previous was NULL -> stays NULL).
1506 let (current, previous) = tier_versions(&pool, "host").await;
1507 assert_eq!(current.as_deref(), Some(version.to_string().as_str()));
1508 assert_eq!(previous, None);
1509
1510 // Run settled green.
1511 let (result, summary) = run_result(&pool, run_id).await;
1512 assert_eq!(result, "passed");
1513 assert_eq!(summary, None);
1514
1515 // Identity: the build row carries the bundle digest (64 hex) and the
1516 // content-addressed dir it was published to (releases/<digest16>).
1517 let (digest, staged_path): (Option<String>, Option<String>) =
1518 sqlx::query_as("SELECT bundle_digest, staged_path FROM build_runs WHERE id = ?")
1519 .bind(run_id.0)
1520 .fetch_one(&pool)
1521 .await
1522 .unwrap();
1523 let digest = digest.expect("bundle_digest recorded");
1524 let staged_path = staged_path.expect("staged_path recorded");
1525 assert_eq!(digest.len(), 64);
1526 let releases = tmp.path().join("release-root").join("releases");
1527 assert_eq!(
1528 std::path::Path::new(&staged_path),
1529 releases.join(&digest[..16]),
1530 "bundle is published content-addressed at releases/<digest16>"
1531 );
1532
1533 // versions.artifact_path points at the primary binary inside that dir,
1534 // and it exists on disk.
1535 let staged_bin: String =
1536 sqlx::query_scalar("SELECT artifact_path FROM versions WHERE version = ?")
1537 .bind(version.to_string())
1538 .fetch_one(&pool)
1539 .await
1540 .unwrap();
1541 let expected_bin = releases.join(&digest[..16]).join("makenotwork");
1542 assert_eq!(staged_bin, expected_bin.to_string_lossy());
1543 assert!(
1544 expected_bin.exists(),
1545 "staged binary missing at {expected_bin:?}"
1546 );
1547
1548 // The bundle carries its MANIFEST (for node-side verification), and the
1549 // recorded digest recomputes over the published dir (MANIFEST excluded).
1550 assert!(
1551 releases.join(&digest[..16]).join("MANIFEST").exists(),
1552 "MANIFEST written into the bundle"
1553 );
1554 let recomputed = crate::bundle::digest_dir(std::path::Path::new(&staged_path))
1555 .await
1556 .unwrap();
1557 assert_eq!(
1558 digest, recomputed.full,
1559 "recorded digest matches the bundle"
1560 );
1561
1562 // The `current` symlink flipped to the content-addressed release.
1563 let link = tmp.path().join("release-root").join("current");
1564 let target = std::fs::read_link(&link).expect("current is a symlink");
1565 assert_eq!(target, PathBuf::from(format!("releases/{}", &digest[..16])));
1566 }
1567
1568 #[tokio::test]
1569 async fn stage_and_gate_marks_the_run_failed_and_does_not_advance_when_a_gate_is_red() {
1570 // ManualConfirm with no prior confirmation row blocks deterministically.
1571 let (pool, cfg, topo, art, run_id, _version, _tmp) =
1572 stage_fixture(vec![Gate::ManualConfirm]).await;
1573 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1574
1575 // A red gate is a pipeline outcome, not an error: the fn records the
1576 // failure and returns Ok so the spawned task settles the run cleanly.
1577 stage_and_gate(
1578 pool.clone(),
1579 cfg,
1580 topo,
1581 art,
1582 crate::events::channel(),
1583 run_id,
1584 deploy_lock,
1585 )
1586 .await
1587 .expect("a red gate settles the run, it does not error out");
1588
1589 // Tier did NOT advance — still the seeded NULL/NULL.
1590 let (current, previous) = tier_versions(&pool, "host").await;
1591 assert_eq!(current, None);
1592 assert_eq!(previous, None);
1593
1594 // Run settled red with a non-empty summary.
1595 let (result, summary) = run_result(&pool, run_id).await;
1596 assert_eq!(result, "failed");
1597 assert!(
1598 summary.as_deref().is_some_and(|s| !s.is_empty()),
1599 "failed run must carry a summary, got {summary:?}"
1600 );
1601 }
1602
1603 #[test]
1604 fn check_build_host_accepts_matching_host() {
1605 assert!(check_build_host("fw13", "fw13").is_ok());
1606 }
1607
1608 #[test]
1609 fn check_build_host_refuses_mismatched_host() {
1610 // A daemon misdeployed onto prod (e.g. a Hetzner host) must refuse.
1611 let err = check_build_host("alpha-west-1", "fw13")
1612 .unwrap_err()
1613 .to_string();
1614 assert!(err.contains("refusing to build"), "{err}");
1615 assert!(
1616 err.contains("alpha-west-1") && err.contains("fw13"),
1617 "{err}"
1618 );
1619 }
1620
1621 #[test]
1622 fn runtime_hostname_reads_a_nonempty_trimmed_name() {
1623 let h = runtime_hostname().expect("hostname readable on Linux");
1624 assert!(!h.is_empty());
1625 assert_eq!(h, h.trim(), "must be trimmed");
1626 }
1627
1628 #[test]
1629 fn tail_does_not_panic_on_multibyte_boundary() {
1630 // Each '€' is 3 bytes; a byte cap landing mid-codepoint must not panic.
1631 let s = "€".repeat(10); // 30 bytes
1632 for max in 1..=30 {
1633 let out = tail(s.as_bytes(), max);
1634 assert!(out.len() <= max, "max={max} got {} bytes", out.len());
1635 // Result is always valid UTF-8 made only of whole '€'s.
1636 assert!(out.chars().all(|c| c == '€'), "max={max}: {out:?}");
1637 }
1638 }
1639
1640 #[test]
1641 fn tail_returns_whole_input_when_under_cap() {
1642 assert_eq!(tail(b"hello", 100), "hello");
1643 }
1644 }
1645