Skip to main content

max / makenotwork

62.9 KB · 1638 lines History Blame Raw
1 //! Build orchestration: resolve a sha to a worktree, read the server version,
2 //! shell out to `cargo build --release`, record a `versions` row.
3 //!
4 //! Runs as a tokio task spawned from `POST /rebuild`; the HTTP request
5 //! returns the version id immediately and the task drives the rest.
6
7 use crate::config::AppConfig;
8 use crate::deploy;
9 use crate::domain::{GitSha, Platform, RunId, TierId, Version};
10 use crate::gates::{self, GateCtx};
11 use crate::git;
12 use crate::topology::Topology;
13 use anyhow::{Context, Result};
14 use chrono::Utc;
15 use sqlx::SqlitePool;
16 use std::path::{Path, PathBuf};
17 use std::sync::Arc;
18 use tokio::process::Command;
19
20 #[derive(Debug, Clone)]
21 pub struct BuildArtifact {
22 pub version: Version,
23 pub git_sha: GitSha,
24 pub worktree: PathBuf,
25 /// One entry per `cfg.bin_names` in declared order. First is the primary
26 /// (referenced by the systemd unit's ExecStart). Paths are inside the
27 /// worktree's `target/release/`.
28 pub binary_paths: Vec<PathBuf>,
29 /// `(companion name, built binary path)` for each `cfg.companions`, built
30 /// from the same worktree/sha as the server. Staged into the release bundle
31 /// under `companions/<name>/<bin>` and installed by the nodes that opt in.
32 pub companion_paths: Vec<(String, PathBuf)>,
33 }
34
35 /// The live kernel hostname (`/proc/sys/kernel/hostname`, trimmed). Linux-only,
36 /// which Sando is. The build-host guard reads this rather than `$HOSTNAME`
37 /// (not reliably exported) so the check reflects the actual machine.
38 fn runtime_hostname() -> Result<String> {
39 let raw = std::fs::read_to_string("/proc/sys/kernel/hostname")
40 .context("reading /proc/sys/kernel/hostname for the build-host guard")?;
41 Ok(raw.trim().to_string())
42 }
43
44 /// Pure half of the build-host guard: fail unless the live host matches the
45 /// configured build host. Split out from [`enforce_build_host`] so it is unit-
46 /// testable without depending on the test machine's hostname.
47 fn check_build_host(actual: &str, expected: &str) -> Result<()> {
48 anyhow::ensure!(
49 actual == expected,
50 "refusing to build on host {actual}: configured build host is {expected} \
51 (never build on a prod/serving node)",
52 );
53 Ok(())
54 }
55
56 /// Refuse to build unless this daemon is running on the configured build host.
57 fn enforce_build_host(expected: &str) -> Result<()> {
58 check_build_host(&runtime_hostname()?, expected)
59 }
60
61 pub async fn run(
62 pool: SqlitePool,
63 cfg: Arc<AppConfig>,
64 topo: Arc<Topology>,
65 sha: GitSha,
66 events: crate::events::EventTx,
67 run_id: RunId,
68 ) -> Result<BuildArtifact> {
69 // Build-host guard: refuse to compile anywhere but the configured build
70 // host. The build runs locally (`cargo build` in cfg.workdir), so a sandod
71 // misdeployed onto a prod/serving node would otherwise build there — exactly
72 // the "never build on prod" rule. Enforced before any cargo invocation so a
73 // wrong-host daemon fails fast with a clear message rather than compiling.
74 let Some(build_host) = cfg.build_host.as_deref() else {
75 anyhow::bail!(
76 "{} declares no build_host, which makes it intake-only: Sando does not compile \
77 it. Ship it with POST /intake, from a builder that does.",
78 cfg.id
79 );
80 };
81 enforce_build_host(build_host)?;
82
83 let repo = topo.repo.as_ref().with_context(|| {
84 format!(
85 "{} declares no [repo]: it is intake-only and Sando has no source to check out",
86 cfg.id
87 )
88 })?;
89 let worktree = cfg.workdir.join(sha.as_str());
90 let bare = PathBuf::from(&repo.bare_path);
91
92 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Fetching)
93 .await
94 .ok();
95
96 // Pull-based ingestion: if an upstream remote is configured, fetch the
97 // deploy branch so a just-pushed sha is locally resolvable. A fetch
98 // failure is non-fatal — the sha may already be present from a prior
99 // fetch or a direct push; the presence check below is the real gate.
100 if let Some(upstream) = repo.upstream.as_deref()
101 && let Err(e) = git::fetch_upstream(&bare, upstream, &repo.branch).await
102 {
103 tracing::warn!(error = %e, upstream, "upstream fetch failed; proceeding with current bare-repo state");
104 }
105 anyhow::ensure!(
106 git::sha_present(&bare, sha.as_str()).await?,
107 "sha {} not present in bare repo {} after fetch — push the commit to the upstream remote first",
108 sha.as_str(),
109 bare.display(),
110 );
111
112 git::checkout_worktree(&bare, sha.as_str(), &worktree).await?;
113
114 // Check out any auxiliary repos (e.g. synckit) beside the worktree so a
115 // cross-repo path dependency in the server or a companion resolves. Fails the
116 // build if an aux repo can't be assembled — a companion that silently fails to
117 // find its source would fail the compile downstream with a worse message.
118 checkout_aux_repos(&cfg, &topo).await?;
119
120 let server_dir = worktree.join("server");
121 let version = read_pkg_version(&server_dir.join("Cargo.toml"))
122 .await
123 .with_context(|| format!("reading version from {}/Cargo.toml", server_dir.display()))?;
124 crate::runs::set_version(&pool, run_id, &version).await.ok();
125
126 // sqlx compile-time query checking needs a live DB with the current schema.
127 // We point cargo at the scratch DB and prep it (drop public, re-migrate)
128 // before invoking cargo build. The same DB is reset again by
129 // `migration_dry_run` later if it runs as a gate.
130 let mut cargo_cmd = Command::new("cargo");
131 cargo_cmd
132 .arg("build")
133 .arg("--release")
134 .current_dir(&server_dir)
135 .kill_on_drop(true);
136 // Shared build cache across per-sha worktrees: reuse one target dir so an
137 // incremental diff doesn't clean-compile from scratch. Serialized builds
138 // make this contention-free. Unset → cargo's default per-worktree target/.
139 if let Some(target) = cfg.cargo_target_dir.as_deref() {
140 cargo_cmd.env("CARGO_TARGET_DIR", target);
141 }
142 if let Some(scratch_url) = cfg.scratch_db_url.as_deref() {
143 tracing::info!(sha = %sha.as_str(), "preparing scratch DB schema for sqlx compile-time checks");
144 crate::gates::reset_scratch(scratch_url, &cfg.scratch_owner_role)
145 .await
146 .context("scratch DB reset before build")?;
147 crate::gates::run_migrator(scratch_url, &server_dir.join("migrations"))
148 .await
149 .context("applying MNW migrations to scratch DB before build")?;
150 cargo_cmd.env("DATABASE_URL", scratch_url);
151 } else {
152 tracing::warn!("scratch_db_url unset; sqlx will fall back to offline mode and may fail");
153 }
154
155 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Compiling)
156 .await
157 .ok();
158 tracing::info!(sha = %sha, version = %version, dir = %server_dir.display(), "cargo build --release start");
159 crate::events::emit(
160 &events,
161 crate::events::Event::BuildStart {
162 sha: sha.clone(),
163 version: version.clone(),
164 },
165 );
166 let started = std::time::Instant::now();
167 let out = cargo_cmd.output().await.context("spawning cargo build")?;
168 let elapsed_s = started.elapsed().as_secs();
169 if !out.status.success() {
170 tracing::error!(sha = %sha, version = %version, elapsed_s, "cargo build --release failed");
171 crate::events::emit(
172 &events,
173 crate::events::Event::BuildFailed {
174 sha: sha.clone(),
175 version: version.clone(),
176 elapsed_s,
177 },
178 );
179 // Settle the run with the headline compiler diagnostic (not the raw
180 // 4 KB tail) so `GET /runs/{id}` answers "why" without a journald dive.
181 let summary = crate::classify::classify_compile_error(&out.stdout, &out.stderr).summary();
182 if let Err(e) = crate::runs::mark_failed(&pool, run_id, &summary).await {
183 tracing::error!(run_id = %run_id, error = %e, "persisting compile-fail verdict failed; run may show stale 'building' until restart-reconcile");
184 }
185 anyhow::bail!(
186 "cargo build --release failed:\n{}",
187 tail(&out.stderr, 4_000)
188 );
189 }
190 tracing::info!(sha = %sha, version = %version, elapsed_s, "cargo build --release ok");
191 crate::events::emit(
192 &events,
193 crate::events::Event::BuildOk {
194 sha: sha.clone(),
195 version: version.clone(),
196 elapsed_s,
197 },
198 );
199
200 // Binaries land under `<target>/release/`; with a shared target dir that's
201 // not inside the worktree, so resolve it the same way cargo did above.
202 let release_dir = cfg
203 .cargo_target_dir
204 .as_deref()
205 .map_or_else(|| server_dir.join("target/release"), |t| t.join("release"));
206 let mut binary_paths = Vec::with_capacity(cfg.bin_names.len());
207 for name in &cfg.bin_names {
208 let p = release_dir.join(name);
209 anyhow::ensure!(p.exists(), "expected binary at {} after build", p.display());
210 binary_paths.push(p);
211 }
212 // Primary binary path is the one we record in `versions.artifact_path`
213 // (everything downstream — promote, rollback — looks it up by version).
214 let primary = binary_paths[0].clone();
215
216 // Companion crates (e.g. mnw-cli): built from the SAME worktree/sha so a
217 // service that shares the server's internal-API contract cannot drift out of
218 // lockstep (the 2026-07-09 git-hosting outage). A companion build failure
219 // fails the whole pipeline — the server never ships without its companions.
220 let mut companion_paths = Vec::with_capacity(cfg.companions.len());
221 for c in &cfg.companions {
222 let bin = build_companion(&worktree, &cfg, c).await?;
223 companion_paths.push((c.name.clone(), bin));
224 }
225
226 sqlx::query(
227 "INSERT OR IGNORE INTO versions (app, version, git_sha, built_at, artifact_path)
228 VALUES (?, ?, ?, ?, ?)",
229 )
230 .bind(&cfg.id)
231 .bind(&version)
232 .bind(&sha)
233 .bind(Utc::now().to_rfc3339())
234 .bind(primary.to_string_lossy().as_ref())
235 .execute(&pool)
236 .await?;
237
238 Ok(BuildArtifact {
239 version,
240 git_sha: sha,
241 worktree,
242 binary_paths,
243 companion_paths,
244 })
245 }
246
247 /// Fetch and check out every configured auxiliary repo at `cfg.workdir/<checkout_dir>`,
248 /// so a cross-repo path dependency built from the main worktree resolves (wiki
249 /// [[sando-overview]]; the synckit split, task sando-18cdb32f).
250 ///
251 /// Each aux repo is a fixed, shared checkout refreshed to `branch` HEAD — not
252 /// per-sha — because the dependent's relative path resolves to that fixed spot
253 /// regardless of the main sha, and builds serialize. A fetch failure is a warning
254 /// (the branch may already be present from a prior build); an unresolvable branch
255 /// after that is fatal, as is a failed worktree — a half-assembled source tree
256 /// must fail the build here, loudly, not as a downstream compile error.
257 /// Where an aux repo's checkout lands. The single derivation: `checkout_aux_repos`
258 /// creates it here and `GateCtx::aux_dirs` resolves `test_target`s against it, so
259 /// the two cannot drift into looking in different places.
260 pub fn aux_checkout_dir(cfg: &AppConfig, aux: &crate::topology::AuxRepo) -> PathBuf {
261 cfg.workdir.join(&aux.checkout_dir)
262 }
263
264 /// Every aux repo's checkout dir, keyed by name — what `GateCtx::aux_dirs` holds.
265 pub fn aux_checkout_dirs(
266 cfg: &AppConfig,
267 topo: &Topology,
268 ) -> std::collections::HashMap<String, PathBuf> {
269 topo.aux_repos
270 .iter()
271 .map(|a| (a.name.clone(), aux_checkout_dir(cfg, a)))
272 .collect()
273 }
274
275 pub async fn checkout_aux_repos(cfg: &AppConfig, topo: &Topology) -> Result<()> {
276 for aux in &topo.aux_repos {
277 let bare = PathBuf::from(&aux.bare_path);
278 git::ensure_bare_repo_no_hook(&bare)
279 .await
280 .with_context(|| format!("aux repo {}: init bare {}", aux.name, aux.bare_path))?;
281 if let Err(e) = git::fetch_upstream(&bare, &aux.upstream, &aux.branch).await {
282 tracing::warn!(
283 aux = %aux.name, error = %e,
284 "aux repo fetch failed; proceeding with current bare-repo state",
285 );
286 }
287 let sha = git::resolve_ref(&bare, &aux.branch).await.with_context(|| {
288 format!(
289 "aux repo {}: branch {} not resolvable after fetch — is {} reachable with that branch?",
290 aux.name, aux.branch, aux.upstream,
291 )
292 })?;
293 let dest = aux_checkout_dir(cfg, aux);
294 git::checkout_worktree(&bare, &sha, &dest)
295 .await
296 .with_context(|| {
297 format!(
298 "aux repo {}: checking out {} ({}) at {}",
299 aux.name,
300 aux.branch,
301 sha,
302 dest.display()
303 )
304 })?;
305 tracing::info!(
306 aux = %aux.name, branch = %aux.branch, sha = %sha, dest = %dest.display(),
307 "aux repo checked out beside worktree",
308 );
309 }
310 Ok(())
311 }
312
313 /// Build one companion crate from the worktree, returning its release binary
314 /// path. Mirrors the server build's target-dir handling (shared
315 /// `cargo_target_dir` when set, for incremental reuse; else the crate's own
316 /// `target/`). A companion is an API client, not a sqlx crate, so it needs no
317 /// scratch DB. A non-zero exit propagates and fails the pipeline.
318 async fn build_companion(
319 worktree: &Path,
320 cfg: &AppConfig,
321 c: &crate::config::Companion,
322 ) -> Result<PathBuf> {
323 let dir = worktree.join(&c.manifest_dir);
324 anyhow::ensure!(
325 dir.join("Cargo.toml").exists(),
326 "companion {}: no Cargo.toml at {}",
327 c.name,
328 dir.display(),
329 );
330 // Match the server build: no `--locked` (the pipeline builds whatever the
331 // sha pins; a stale lock shouldn't block a deploy the server build allows).
332 let mut cmd = Command::new("cargo");
333 cmd.arg("build")
334 .arg("--release")
335 .current_dir(&dir)
336 .kill_on_drop(true);
337 let release_dir = if let Some(target) = cfg.cargo_target_dir.as_deref() {
338 cmd.env("CARGO_TARGET_DIR", target);
339 target.join("release")
340 } else {
341 dir.join("target/release")
342 };
343 tracing::info!(companion = %c.name, dir = %dir.display(), "cargo build --release (companion) start");
344 let started = std::time::Instant::now();
345 let out = cmd
346 .output()
347 .await
348 .context("spawning cargo build for companion")?;
349 if !out.status.success() {
350 anyhow::bail!(
351 "companion {} build failed:\n{}",
352 c.name,
353 tail(&out.stderr, 4_000),
354 );
355 }
356 let bin = release_dir.join(&c.bin);
357 anyhow::ensure!(
358 bin.exists(),
359 "companion {} produced no binary at {} after build",
360 c.name,
361 bin.display(),
362 );
363 tracing::info!(companion = %c.name, elapsed_s = started.elapsed().as_secs(), "companion build ok");
364 Ok(bin)
365 }
366
367 /// Full host-tier pipeline: build, stage the bundle into the host's
368 /// release_root, run the host tier's configured gates, advance tier_state
369 /// for "host" if all pass. Errors propagate back to the spawned task and
370 /// get logged. (Tier was called "mm" pre-Session-1; renamed to "host"
371 /// since sandod runs on whatever machine ends up being the Sando host.)
372 pub async fn build_and_run_host(
373 pool: SqlitePool,
374 cfg: Arc<AppConfig>,
375 topo: Arc<Topology>,
376 sha: GitSha,
377 events: crate::events::EventTx,
378 run_id: RunId,
379 deploy_lock: Arc<tokio::sync::Mutex<()>>,
380 ) -> Result<()> {
381 let art = run(
382 pool.clone(),
383 cfg.clone(),
384 topo.clone(),
385 sha,
386 events.clone(),
387 run_id,
388 )
389 .await?;
390
391 stage_and_gate(pool, cfg, topo, art, events, run_id, deploy_lock).await
392 }
393
394 /// A bundle assembled on disk and not yet published: the point both the build
395 /// path and the intake path have to reach before anything else can happen to it.
396 ///
397 /// The two paths reach it differently. A build assembles it out of a worktree
398 /// (binaries, `release_contents`, companions); an intake is handed it already
399 /// assembled, with no worktree anywhere. Everything after this point — hashing,
400 /// publishing, recording identity, gating, advancing — is the same work, and
401 /// used to be welded to the assembling half inside `stage_and_gate`.
402 struct StagedBundle {
403 version: Version,
404 /// The staging dir under `release_root/staging/`, pre-publish.
405 staging: PathBuf,
406 /// What the bundle is built to run on, when it is known. A Sando build
407 /// inherits the app's declared platform; an intake takes it from the
408 /// record's provenance.
409 platform: Option<Platform>,
410 }
411
412 /// Post-build half of the host pipeline: stage the artifact into the host's
413 /// release_root, run the host tier's gates, and advance `tier_state` for
414 /// "host" iff all pass. Split from [`build_and_run_host`] at the `run()`
415 /// boundary so the staging/gating/advance logic is reachable in tests from a
416 /// synthetic [`BuildArtifact`] — no real `cargo build --release` required.
417 pub async fn stage_and_gate(
418 pool: SqlitePool,
419 cfg: Arc<AppConfig>,
420 topo: Arc<Topology>,
421 art: BuildArtifact,
422 events: crate::events::EventTx,
423 run_id: RunId,
424 deploy_lock: Arc<tokio::sync::Mutex<()>>,
425 ) -> Result<()> {
426 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Staging)
427 .await
428 .ok();
429 let staged = assemble_from_source(&cfg, &art, run_id).await?;
430 let published = publish(&pool, &cfg, staged, run_id).await?;
431 record_and_gate(
432 pool,
433 cfg,
434 topo,
435 published,
436 events,
437 run_id,
438 deploy_lock,
439 Some(art.worktree),
440 )
441 .await
442 }
443
444 /// Accept an artifact built elsewhere and take it through the same host-tier
445 /// gating and advance a Sando-built one gets.
446 ///
447 /// This is the whole point of the seam. `intake::accept` publishes the bundle
448 /// content-addressed once it has proved the bytes are the ones the record
449 /// vouches for, which lands it at exactly the state [`publish`] leaves a
450 /// Sando-built bundle in — so the two paths join at [`record_and_gate`] and
451 /// nothing downstream knows or cares which one it came from.
452 ///
453 /// `staged` must already sit under `release_root/staging/` (publishing is an
454 /// atomic same-filesystem rename); getting the bytes there is the transport's
455 /// job, not this function's.
456 #[allow(clippy::too_many_arguments)]
457 pub async fn accept_intake(
458 pool: &SqlitePool,
459 cfg: &AppConfig,
460 staged: &Path,
461 record_json: &str,
462 run_id: RunId,
463 ) -> Result<Published> {
464 crate::runs::set_phase(pool, run_id, crate::runs::Phase::Staging)
465 .await
466 .ok();
467
468 let accepted = crate::intake::accept(&cfg.release_root, staged, record_json)
469 .await
470 .map_err(|e| anyhow::anyhow!("{e}"))?;
471
472 let version = Version::parse(&accepted.record.provenance.version).with_context(|| {
473 format!(
474 "artifact record carries version `{}`, which is not semver",
475 accepted.record.provenance.version
476 )
477 })?;
478 let platform = Platform::parse(&accepted.record.provenance.target).with_context(|| {
479 format!(
480 "artifact record carries target `{}`, which is not `os/arch`",
481 accepted.record.provenance.target
482 )
483 })?;
484 let git_sha = GitSha::parse(&accepted.record.provenance.git_sha).with_context(|| {
485 format!(
486 "artifact record carries git_sha `{}`",
487 accepted.record.provenance.git_sha
488 )
489 })?;
490
491 crate::runs::set_version(pool, run_id, &version).await.ok();
492 upsert_version_row(
493 pool,
494 &cfg.id,
495 &version,
496 &git_sha,
497 &accepted.released.join(cfg.primary_bin()),
498 )
499 .await?;
500
501 let published = Published {
502 version,
503 released: accepted.released,
504 digest_full: accepted.record.digest.to_string(),
505 platform: Some(platform),
506 };
507 record_identity(pool, cfg, &published, run_id).await?;
508 Ok(published)
509 }
510
511 /// Gate an artifact that has already been accepted.
512 ///
513 /// Split from [`accept_intake`] so the two can be answered on different clocks.
514 /// Acceptance is fast and is the producer's business — it either believes the
515 /// bytes or names the file that drifted — so the caller waits for it and gets
516 /// the verdict. Gating is Sando's business and can take an hour, so the caller
517 /// does not.
518 ///
519 /// An intake carries no worktree, and the gates that need one refuse rather than
520 /// resolve against nothing. That is the boundary showing up in the type:
521 /// artifact-scoped gates belong to the builder (wiki [[sando-bento-boundary]]),
522 /// so a tier that asks Sando to re-run them against an accepted artifact is
523 /// misconfigured and should be told so.
524 pub async fn gate_intake(
525 pool: SqlitePool,
526 cfg: Arc<AppConfig>,
527 topo: Arc<Topology>,
528 published: Published,
529 events: crate::events::EventTx,
530 run_id: RunId,
531 deploy_lock: Arc<tokio::sync::Mutex<()>>,
532 ) -> Result<()> {
533 record_and_gate(
534 pool,
535 cfg,
536 topo,
537 published,
538 events,
539 run_id,
540 deploy_lock,
541 None,
542 )
543 .await
544 }
545
546 /// Record the `versions` label row for an artifact that arrived rather than was
547 /// built here. The build path writes its own inside [`run`]; this is the same
548 /// row for the path that never ran a compiler.
549 async fn upsert_version_row(
550 pool: &SqlitePool,
551 app: &crate::domain::AppId,
552 version: &Version,
553 git_sha: &GitSha,
554 artifact_path: &Path,
555 ) -> Result<()> {
556 sqlx::query(
557 "INSERT OR IGNORE INTO versions (app, version, git_sha, built_at, artifact_path)
558 VALUES (?, ?, ?, ?, ?)",
559 )
560 .bind(app)
561 .bind(version)
562 .bind(git_sha)
563 .bind(Utc::now().to_rfc3339())
564 .bind(artifact_path.to_string_lossy().as_ref())
565 .execute(pool)
566 .await?;
567 Ok(())
568 }
569
570 /// Assemble a bundle out of a worktree: binaries, `release_contents`, companions.
571 async fn assemble_from_source(
572 cfg: &AppConfig,
573 art: &BuildArtifact,
574 run_id: RunId,
575 ) -> Result<StagedBundle> {
576 // Stage the bundle into `staging/<build_id>/` — a private scratch dir, not
577 // yet a release. It is published content-addressed below, once its digest is
578 // known. This is what makes overwrite unexpressible (wiki
579 // [[release-artifact-identity]]): a build never touches another build's dir.
580 let staging =
581 deploy::stage_local_bundle(&cfg.release_root, run_id.0, &art.binary_paths).await?;
582
583 // Stage every entry from cfg.release_contents into the staged bundle. This is
584 // how non-binary version-coupled content (static assets, docs, error-pages,
585 // ...) makes it into the atomic deploy bundle. Projects opt in via daemon
586 // config — the sando code carries no MNW-specific knowledge.
587 for entry in &cfg.release_contents {
588 stage_entry(&art.worktree, &staging, entry).await?;
589 }
590
591 // Stage companion binaries as `companions/<name>` (the file itself) so they
592 // ride the same atomic bundle rsync to the nodes, and a node can locate its
593 // companion source from the logical name alone — no bin-filename coupling in
594 // the topology. The nodes that opt in install them post-swap (see
595 // deploy::deploy_remote).
596 if !art.companion_paths.is_empty() {
597 let dst_dir = staging.join("companions");
598 tokio::fs::create_dir_all(&dst_dir)
599 .await
600 .with_context(|| format!("create staged companions dir {}", dst_dir.display()))?;
601 for (name, built) in &art.companion_paths {
602 let dst = dst_dir.join(name);
603 tokio::fs::copy(built, &dst).await.with_context(|| {
604 format!(
605 "stage companion {name}: {} -> {}",
606 built.display(),
607 dst.display()
608 )
609 })?;
610 }
611 }
612
613 Ok(StagedBundle {
614 version: art.version.clone(),
615 staging,
616 platform: cfg.platform.clone(),
617 })
618 }
619
620 /// A bundle that has been hashed and published content-addressed. Both paths
621 /// produce one; nothing downstream can tell them apart.
622 ///
623 /// Public because the intake route now hands one from `accept_intake` to
624 /// `gate_intake`: proving the bytes answers the producer, gating them does not,
625 /// so the two run on different clocks and the value passes between them.
626 #[derive(Debug)]
627 pub struct Published {
628 version: Version,
629 released: PathBuf,
630 digest_full: String,
631 platform: Option<Platform>,
632 }
633
634 /// Hash the assembled bundle, write its MANIFEST, and publish it at
635 /// `releases/<digest16>`.
636 ///
637 /// The intake path does not call this: `intake::accept` does the same three
638 /// steps itself, because it has to hash the bytes to verify them and hashing
639 /// them twice would be the one place the two implementations could disagree.
640 async fn publish(
641 pool: &SqlitePool,
642 cfg: &AppConfig,
643 staged: StagedBundle,
644 run_id: RunId,
645 ) -> Result<Published> {
646 // Content identity: hash the fully-staged bundle, write its MANIFEST into the
647 // bundle (for node-side verification), then publish it at `releases/<digest16>`.
648 // The digest is now load-bearing — a hashing failure fails the build rather
649 // than shipping an unidentifiable artifact.
650 let digest = crate::bundle::digest_dir(&staged.staging)
651 .await
652 .context("hashing the staged bundle for content addressing")?;
653 tokio::fs::write(
654 staged.staging.join(crate::bundle::MANIFEST_NAME),
655 digest.manifest.as_bytes(),
656 )
657 .await
658 .context("writing bundle MANIFEST")?;
659 let released =
660 deploy::finalize_local_release(&cfg.release_root, &staged.staging, digest.short()).await?;
661
662 let staged_bin = released.join(cfg.primary_bin());
663 sqlx::query("UPDATE versions SET artifact_path = ? WHERE app = ? AND version = ?")
664 .bind(staged_bin.to_string_lossy().as_ref())
665 .bind(&cfg.id)
666 .bind(&staged.version)
667 .execute(pool)
668 .await?;
669
670 let published = Published {
671 version: staged.version,
672 released,
673 digest_full: digest.full,
674 platform: staged.platform,
675 };
676 record_identity(pool, cfg, &published, run_id).await?;
677 Ok(published)
678 }
679
680 /// Record the identity on the build row: the digest, the content-addressed dir
681 /// the bundle was published to, and what it runs on. This is what promote
682 /// resolves the artifact through, and burn-in/retention key on.
683 async fn record_identity(
684 pool: &SqlitePool,
685 cfg: &AppConfig,
686 published: &Published,
687 run_id: RunId,
688 ) -> Result<()> {
689 let released_path = published.released.to_string_lossy();
690 crate::runs::set_identity(pool, run_id, &published.digest_full, &released_path)
691 .await
692 .ok();
693 // Platform is what lets two bundles of one version be told apart, so a
694 // dropped write here would leave a pom artifact that can be placed nowhere
695 // (a node declaring a platform refuses an artifact that records none). Fail
696 // rather than ship an unplaceable bundle.
697 if let Some(p) = &published.platform {
698 crate::runs::set_platform(pool, run_id, p)
699 .await
700 .with_context(|| format!("recording platform {p} for {}", cfg.id))?;
701 }
702 Ok(())
703 }
704
705 /// The shared tail of both paths: run the host tier's gates against a published
706 /// bundle and advance `tier_state` iff all pass.
707 #[allow(clippy::too_many_arguments)]
708 async fn record_and_gate(
709 pool: SqlitePool,
710 cfg: Arc<AppConfig>,
711 topo: Arc<Topology>,
712 published: Published,
713 events: crate::events::EventTx,
714 run_id: RunId,
715 deploy_lock: Arc<tokio::sync::Mutex<()>>,
716 worktree: Option<PathBuf>,
717 ) -> Result<()> {
718 let host = topo
719 .tiers
720 .iter()
721 .find(|t| t.name.as_str() == "host")
722 .context("topology has no `host` tier")?;
723
724 crate::runs::set_phase(&pool, run_id, crate::runs::Phase::Gating)
725 .await
726 .ok();
727 let ctx = GateCtx {
728 pool: pool.clone(),
729 cfg: cfg.clone(),
730 tier: TierId::new("host"),
731 version: published.version.clone(),
732 worktree,
733 // The published bundle. `migration_dry_run` prefers it over the
734 // worktree, so the migrations it proves are the ones inside the digest
735 // rather than ones sitting beside them in a checkout.
736 bundle: Some(published.released.clone()),
737 events: events.clone(),
738 // Host runs build-time gates (cargo_test / migration_dry_run /
739 // boot_smoke) only — `node_health` never appears here, so there are no
740 // nodes to probe.
741 nodes: Vec::new(),
742 // These gates vouch for this build; record its id so promote can resolve
743 // the artifact through the evidence rather than a version string.
744 build_id: Some(run_id.0),
745 // Where checkout_aux_repos put each aux repo, so a test_target naming
746 // one resolves. Shared derivation, so the two cannot disagree.
747 aux_dirs: aux_checkout_dirs(&cfg, &topo),
748 };
749 let failed = gates::run_all(&ctx, &host.gates).await?;
750
751 if failed.is_empty() {
752 // Advance the host tier through the single sealed forward-advance op, under
753 // deploy_lock so this can't interleave with a concurrent `/rollback host`
754 // (the old fetch-then-write here was the one CF3 site outside the lock —
755 // ultra-fuzz Run 2, S1). Held only for the atomic UPDATE, never the gates.
756 {
757 let _deploy_guard = deploy_lock.lock().await;
758 crate::runs::advance_tier(&pool, &cfg.id, "host", &published.version, Some(run_id.0))
759 .await?;
760 }
761 // Terminal verdict: unlike the phase pings above (best-effort), a dropped
762 // pass/fail write leaves the run wedged at `building`. Log it loudly if it
763 // fails — the startup reconcile (main) is the backstop that settles such a
764 // row on the next restart.
765 if let Err(e) = crate::runs::mark_passed(&pool, run_id).await {
766 tracing::error!(run_id = %run_id, error = %e, "persisting host-green verdict failed; run may show stale 'building' until restart-reconcile");
767 }
768 tracing::info!(version = %published.version, "host pipeline green; ready to promote to next tier");
769 } else {
770 // Pull the first red gate's typed summary into the run so the API
771 // answers "which gate, and why" — not just "failed".
772 let summary = crate::runs::first_failed_gate_summary(&pool, &cfg.id, &published.version)
773 .await
774 .unwrap_or_else(|| "host pipeline red".to_string());
775 if let Err(e) = crate::runs::mark_failed(&pool, run_id, &summary).await {
776 tracing::error!(run_id = %run_id, error = %e, "persisting host-red verdict failed; run may show stale 'building' until restart-reconcile");
777 }
778 tracing::warn!(version = %published.version, "host pipeline red; not advancing tier_state");
779 }
780 Ok(())
781 }
782
783 async fn read_pkg_version(cargo_toml: &Path) -> Result<Version> {
784 let raw = tokio::fs::read_to_string(cargo_toml).await?;
785 let parsed: toml::Value = toml::from_str(&raw)?;
786 let v = parsed
787 .get("package")
788 .and_then(|p| p.get("version"))
789 .and_then(|v| v.as_str())
790 .context("package.version not found")?;
791 Version::parse(v).with_context(|| format!("parsing package.version `{v}`"))
792 }
793
794 fn tail(buf: &[u8], max: usize) -> String {
795 let s = String::from_utf8_lossy(buf);
796 if s.len() <= max {
797 return s.into_owned();
798 }
799 // `s.len() - max` can land mid-codepoint; walk forward to the next char
800 // boundary so the slice never panics (returns slightly fewer than `max`
801 // bytes in that case). `floor_char_boundary` is still unstable, so do it by
802 // hand.
803 let mut start = s.len() - max;
804 while start < s.len() && !s.is_char_boundary(start) {
805 start += 1;
806 }
807 s[start..].to_string()
808 }
809
810 /// Copy `worktree/<entry.src>` into `staged/<entry.dst>`. Handles file or
811 /// directory sources transparently. Missing source policy depends on
812 /// `entry.required`:
813 /// - required=true -> error (build fails)
814 /// - required=false -> log warn + skip (e.g. older shas missing a dir)
815 ///
816 /// Uses `cp -a` to preserve modes/symlinks/etc; parent of dst is created if
817 /// needed so entries like `dst = "docs/assumptions.toml"` work without
818 /// extra config.
819 async fn stage_entry(
820 worktree: &Path,
821 staged: &Path,
822 entry: &crate::config::ReleaseEntry,
823 ) -> Result<()> {
824 let src = worktree.join(&entry.src);
825 let dst = staged.join(&entry.dst);
826 if !src.exists() {
827 if entry.required {
828 anyhow::bail!(
829 "required release_contents source missing: {}",
830 src.display()
831 );
832 }
833 tracing::warn!(src = %src.display(), "release_contents source missing (optional); skipping");
834 return Ok(());
835 }
836 if let Some(parent) = dst.parent() {
837 tokio::fs::create_dir_all(parent)
838 .await
839 .with_context(|| format!("create staged parent {}", parent.display()))?;
840 }
841 // Multiple entries with the same dst (e.g. site-docs/public/ +
842 // site-docs/examples/ both landing under docs/) need additive merging.
843 // `cp -a SRC/. DST/` copies SRC's contents into DST without overwriting
844 // the dst dir itself; that's the merge-friendly form when dst is a dir
845 // that may already exist from a prior entry. For non-dir sources or a
846 // missing dst we fall back to the plain `cp -a SRC DST` form.
847 let merge_into_existing_dir = src.is_dir() && dst.is_dir();
848 let mut cmd = Command::new("cp");
849 cmd.arg("-a");
850 if merge_into_existing_dir {
851 let mut src_arg = src.clone().into_os_string();
852 src_arg.push("/.");
853 cmd.arg(src_arg);
854 let mut dst_arg = dst.clone().into_os_string();
855 dst_arg.push("/");
856 cmd.arg(dst_arg);
857 } else {
858 cmd.arg(&src).arg(&dst);
859 }
860 let out = cmd
861 .output()
862 .await
863 .with_context(|| format!("spawning cp for {} -> {}", src.display(), dst.display()))?;
864 anyhow::ensure!(
865 out.status.success(),
866 "stage {} -> {}: {}",
867 src.display(),
868 dst.display(),
869 String::from_utf8_lossy(&out.stderr),
870 );
871 Ok(())
872 }
873
874 #[cfg(test)]
875 mod tests {
876 use super::{
877 BuildArtifact, accept_intake, check_build_host, checkout_aux_repos, gate_intake,
878 runtime_hostname, stage_and_gate, tail,
879 };
880 use crate::config::{AppConfig, TestTarget};
881 use crate::domain::{GitSha, RunId, Version};
882 use crate::topology::{AuxRepo, BackupConfig, CanaryPolicy, Gate, RepoConfig, Tier, Topology};
883 use sqlx::SqlitePool;
884 use sqlx::sqlite::SqlitePoolOptions;
885 use std::path::PathBuf;
886 use std::sync::Arc;
887
888 /// Post-build pipeline fixture: an in-memory store with the `host` tier
889 /// seeded, a synthetic worktree holding a fake primary binary, and a
890 /// build_runs row in flight. Returns everything `stage_and_gate` needs plus
891 /// the tempdir root (drop it to clean up) and the run/version it seeded.
892 ///
893 /// `gates` is the host tier's gate list: `[]` is the green path;
894 /// `[Gate::ManualConfirm]` is a deterministic red — with no prior operator
895 /// confirmation row that gate blocks, and it shells out to nothing.
896 async fn stage_fixture(
897 gates: Vec<Gate>,
898 ) -> (
899 SqlitePool,
900 Arc<AppConfig>,
901 Arc<Topology>,
902 BuildArtifact,
903 RunId,
904 Version,
905 tempfile::TempDir,
906 ) {
907 let tmp = tempfile::tempdir().unwrap();
908 let release_root = tmp.path().join("release-root");
909 let worktree = tmp.path().join("worktree");
910 let bin_dir = worktree.join("target").join("release");
911 tokio::fs::create_dir_all(&bin_dir).await.unwrap();
912 let bin_path = bin_dir.join("makenotwork");
913 tokio::fs::write(&bin_path, b"#!/bin/false\nfake sando artifact\n")
914 .await
915 .unwrap();
916
917 let pool = SqlitePoolOptions::new()
918 .max_connections(1)
919 .connect("sqlite::memory:")
920 .await
921 .unwrap();
922 sqlx::migrate!("./migrations").run(&pool).await.unwrap();
923
924 // gate_runs and tier_state FK into `tiers`; the pipeline only touches host.
925 sqlx::query("INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('host', 0, 1, 'sequential')")
926 .execute(&pool)
927 .await
928 .unwrap();
929 sqlx::query("INSERT INTO tier_state (tier) VALUES ('host')")
930 .execute(&pool)
931 .await
932 .unwrap();
933
934 let version = Version::parse("1.2.3").unwrap();
935 let git_sha = GitSha::parse("abc1234").unwrap();
936 // gate_runs.version and the `SET artifact_path` UPDATE both need the row.
937 sqlx::query(
938 "INSERT INTO versions (version, git_sha, built_at, artifact_path)
939 VALUES (?, ?, datetime('now'), '')",
940 )
941 .bind(version.to_string())
942 .bind(git_sha.to_string())
943 .execute(&pool)
944 .await
945 .unwrap();
946
947 let run_id = crate::runs::create(
948 &pool,
949 &crate::domain::AppId::default(),
950 &git_sha.to_string(),
951 )
952 .await
953 .unwrap();
954
955 let cfg = AppConfig {
956 platform: None,
957 id: crate::domain::AppId::default(),
958 topology_path: PathBuf::from("/tmp/test-sando.toml"),
959 build_host: Some("test-host".into()),
960 workdir: tmp.path().to_path_buf(),
961 release_root: release_root.clone(),
962 scratch_db_url: None,
963 scratch_owner_role: "makenotwork".into(),
964 boot_smoke_port: 18181,
965 code_smoke_port: 18182,
966 bin_names: vec!["makenotwork".into()],
967 logs_root: tmp.path().join("logs"),
968 release_contents: vec![],
969 cargo_target_dir: None,
970 gate_timeout_secs: 2400,
971 companions: Vec::new(),
972 test_targets: vec![TestTarget {
973 dir: PathBuf::from("server"),
974 aux_repo: None,
975 features: vec!["fast-tests".into()],
976 all_features: false,
977 scratch_db: true,
978 }],
979 migration_checks: vec![],
980 frontend_builds: vec![],
981 backup_max_age_hours: 48,
982 };
983
984 let topo = Topology {
985 repo: Some(RepoConfig {
986 bare_path: "/tmp/test.git".into(),
987 branch: "main".into(),
988 upstream: None,
989 }),
990 backup: vec![BackupConfig {
991 name: "server".into(),
992 source: "file:///tmp/test-backup.sql".into(),
993 local_path: "/tmp/local-backup.sql".into(),
994 }],
995 tiers: vec![Tier {
996 name: "host".into(),
997 provisioned: true,
998 gates,
999 canary: CanaryPolicy::Sequential,
1000 nodes: Vec::new(),
1001 }],
1002 aux_repos: Vec::new(),
1003 };
1004
1005 let art = BuildArtifact {
1006 version: version.clone(),
1007 git_sha,
1008 worktree,
1009 binary_paths: vec![bin_path],
1010 companion_paths: Vec::new(),
1011 };
1012
1013 (
1014 pool,
1015 Arc::new(cfg),
1016 Arc::new(topo),
1017 art,
1018 run_id,
1019 version,
1020 tmp,
1021 )
1022 }
1023
1024 // ---- intake through the seam ----
1025
1026 /// The `ArtifactRecord` Bento would have written for a staged bundle.
1027 async fn record_for(staged: &std::path::Path, version: &str, target: &str) -> String {
1028 use ops_artifact::{ArtifactRecord, GateRecord, Manifest, Provenance, Scope, Verdict};
1029 let computed = crate::bundle::digest_dir(staged).await.unwrap();
1030 let manifest = Manifest::parse(&computed.manifest).unwrap();
1031 let at = chrono::DateTime::<chrono::Utc>::from_timestamp(1_754_000_000, 0).unwrap();
1032 ArtifactRecord::new(
1033 "bento",
1034 manifest,
1035 Provenance {
1036 app: "pom".into(),
1037 version: version.into(),
1038 tag: format!("pom-v{version}"),
1039 git_sha: "a".repeat(40),
1040 target: target.into(),
1041 build_host: "astra".into(),
1042 toolchain: "rustc 1.97.0".into(),
1043 built_at: at,
1044 },
1045 vec![GateRecord::new(
1046 "prebuild",
1047 Scope::Artifact,
1048 Verdict::Passed,
1049 "prebuild passed in 90s",
1050 at,
1051 )],
1052 )
1053 .unwrap()
1054 .to_json()
1055 }
1056
1057 #[tokio::test]
1058 async fn an_accepted_artifact_is_published_gated_and_advances_the_tier() {
1059 // The seam: an artifact Sando did not build reaches the same published,
1060 // gated, tier-advanced end state a Sando-built one does. No worktree
1061 // exists anywhere in this test, which is the point — everything from
1062 // `finalize_local_release` onward stopped caring where the bytes came
1063 // from.
1064 let (pool, cfg, topo, _art, run_id, _version, tmp) = stage_fixture(vec![]).await;
1065 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1066
1067 let staged = cfg.release_root.join("staging").join("intake-1");
1068 tokio::fs::create_dir_all(&staged).await.unwrap();
1069 tokio::fs::write(staged.join("makenotwork"), b"bytes built elsewhere")
1070 .await
1071 .unwrap();
1072 let record = record_for(&staged, "1.2.3", "linux/aarch64").await;
1073
1074 // Two calls now, on purpose: the route answers its caller on the first
1075 // and spawns the second. Acceptance is what the producer waits for.
1076 let published = accept_intake(&pool, &cfg, &staged, &record, run_id)
1077 .await
1078 .expect("the bytes are believed");
1079 gate_intake(
1080 pool.clone(),
1081 cfg.clone(),
1082 topo,
1083 published,
1084 crate::events::channel(),
1085 run_id,
1086 deploy_lock,
1087 )
1088 .await
1089 .expect("a green intake settles the run");
1090
1091 // Published content-addressed, and the staging dir is gone: renamed,
1092 // not copied.
1093 let (digest, staged_path, platform): (Option<String>, Option<String>, Option<String>) =
1094 sqlx::query_as(
1095 "SELECT bundle_digest, staged_path, platform FROM build_runs WHERE id = ?",
1096 )
1097 .bind(run_id.0)
1098 .fetch_one(&pool)
1099 .await
1100 .unwrap();
1101 let digest = digest.expect("bundle_digest recorded");
1102 let staged_path = staged_path.expect("staged_path recorded");
1103 assert_eq!(digest.len(), 64);
1104 assert!(
1105 !staged.exists(),
1106 "staging was renamed into the release root"
1107 );
1108 assert_eq!(
1109 std::path::Path::new(&staged_path),
1110 tmp.path()
1111 .join("release-root")
1112 .join("releases")
1113 .join(&digest[..16]),
1114 );
1115
1116 // The platform came off the record's provenance and is on the row. This
1117 // is what makes two bundles of one version tellable apart later.
1118 assert_eq!(platform.as_deref(), Some("linux/aarch64"));
1119
1120 // Green gates advanced the host tier, exactly as a build would have.
1121 let (result, _summary) = run_result(&pool, run_id).await;
1122 assert_eq!(result, "passed");
1123 let (current, _prev) = tier_versions(&pool, "host").await;
1124 assert_eq!(current.as_deref(), Some("1.2.3"));
1125 }
1126
1127 #[tokio::test]
1128 async fn an_intake_whose_bytes_drifted_never_reaches_the_gates() {
1129 // Identity is decided before anything else happens to the bundle, so a
1130 // record vouching for one set of bytes arriving with another fails the
1131 // run rather than gating and shipping.
1132 let (pool, cfg, topo, _art, run_id, _version, _tmp) = stage_fixture(vec![]).await;
1133 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1134
1135 let staged = cfg.release_root.join("staging").join("intake-1");
1136 tokio::fs::create_dir_all(&staged).await.unwrap();
1137 tokio::fs::write(staged.join("makenotwork"), b"bytes built elsewhere")
1138 .await
1139 .unwrap();
1140 let record = record_for(&staged, "1.2.3", "linux/aarch64").await;
1141 tokio::fs::write(staged.join("makenotwork"), b"other bytes entirely")
1142 .await
1143 .unwrap();
1144
1145 // Refused by ACCEPTANCE, not by gating — which is what lets the route
1146 // answer the producer with the refusal instead of a `202`-shaped lie.
1147 let _ = (&topo, &deploy_lock);
1148 let err = accept_intake(&pool, &cfg, &staged, &record, run_id)
1149 .await
1150 .expect_err("a drifted bundle is refused");
1151 assert!(err.to_string().contains("makenotwork"), "{err}");
1152
1153 // Nothing advanced, and the bytes were left where they were.
1154 let (current, _prev) = tier_versions(&pool, "host").await;
1155 assert_eq!(current, None);
1156 assert!(staged.exists(), "a refused intake leaves the bytes alone");
1157 }
1158
1159 #[tokio::test]
1160 async fn a_gate_that_reads_source_refuses_against_an_accepted_artifact() {
1161 // The boundary showing up at runtime. `code_smoke` is artifact-scoped —
1162 // it compiles frontends and boots the binary against a scratch DB — and
1163 // an accepted artifact has no checkout for it to read. It has to say so
1164 // rather than pass on having run nothing, which is what an unwrapped
1165 // `worktree.join(..)` against an empty path would have done.
1166 let (pool, cfg, _topo, _art, run_id, version, _tmp) = stage_fixture(vec![]).await;
1167 let ctx = crate::gates::GateCtx {
1168 pool,
1169 cfg,
1170 tier: crate::domain::TierId::new("host"),
1171 version,
1172 worktree: None,
1173 bundle: Some(PathBuf::from("/r/abc")),
1174 events: crate::events::channel(),
1175 nodes: Vec::new(),
1176 build_id: Some(run_id.0),
1177 aux_dirs: std::collections::HashMap::default(),
1178 };
1179 let outcome = ctx
1180 .worktree_for(crate::domain::GateKind::CodeSmoke)
1181 .expect_err("no worktree means no source-reading gate");
1182 assert!(!outcome.is_passed());
1183 let crate::outcome::GateStatus::Failed { failure } = &outcome.status else {
1184 panic!("expected a failure, got {:?}", outcome.status)
1185 };
1186 assert!(
1187 matches!(failure, crate::outcome::GateFailure::NeedsSource { .. }),
1188 "{failure:?}"
1189 );
1190 assert!(
1191 failure.summary().contains("built elsewhere"),
1192 "{}",
1193 failure.summary()
1194 );
1195 }
1196
1197 #[tokio::test]
1198 async fn migrations_come_from_the_bundle_before_the_worktree() {
1199 // What the gate proves has to be what ships. Migrations staged into the
1200 // bundle are inside its digest; the same files sitting in a checkout are
1201 // not, and a checkout can be edited between the dry run and the deploy.
1202 // So when both hold a copy, the bundle wins.
1203 let (pool, cfg, _topo, _art, run_id, version, tmp) = stage_fixture(vec![]).await;
1204 let bundle = tmp.path().join("bundle");
1205 let worktree = tmp.path().join("wt");
1206 for root in [&bundle, &worktree] {
1207 tokio::fs::create_dir_all(root.join("server/migrations"))
1208 .await
1209 .unwrap();
1210 }
1211 let ctx = crate::gates::GateCtx {
1212 pool,
1213 cfg,
1214 tier: crate::domain::TierId::new("host"),
1215 version,
1216 worktree: Some(worktree.clone()),
1217 bundle: Some(bundle.clone()),
1218 events: crate::events::channel(),
1219 nodes: Vec::new(),
1220 build_id: Some(run_id.0),
1221 aux_dirs: std::collections::HashMap::default(),
1222 };
1223 assert_eq!(
1224 ctx.migrations_dir(std::path::Path::new("server/migrations")),
1225 Some(bundle.join("server/migrations")),
1226 );
1227
1228 // The worktree is the fallback for a build whose config has not opted
1229 // into bundling them yet, which is every MNW build before this lands.
1230 let ctx = crate::gates::GateCtx {
1231 bundle: Some(tmp.path().join("empty-bundle")),
1232 ..ctx
1233 };
1234 assert_eq!(
1235 ctx.migrations_dir(std::path::Path::new("server/migrations")),
1236 Some(worktree.join("server/migrations")),
1237 );
1238
1239 // And neither is not silently green: an accepted artifact whose builder
1240 // did not bundle its migrations has nothing to dry-run, and the gate
1241 // has to be told so rather than restore a dump and report success.
1242 let ctx = crate::gates::GateCtx {
1243 worktree: None,
1244 ..ctx
1245 };
1246 assert_eq!(
1247 ctx.migrations_dir(std::path::Path::new("server/migrations")),
1248 None
1249 );
1250 }
1251
1252 // ---- checkout_aux_repos ----
1253
1254 async fn git_in(dir: &std::path::Path, args: &[&str]) {
1255 let out = tokio::process::Command::new("git")
1256 .args(["-c", "user.email=t@t", "-c", "user.name=t"])
1257 .current_dir(dir)
1258 .args(args)
1259 .output()
1260 .await
1261 .unwrap();
1262 assert!(
1263 out.status.success(),
1264 "git {args:?}: {}",
1265 String::from_utf8_lossy(&out.stderr)
1266 );
1267 }
1268
1269 /// A minimal `Config` whose only field this test path reads is `workdir`.
1270 fn cfg_with_workdir(workdir: PathBuf) -> AppConfig {
1271 AppConfig {
1272 platform: None,
1273 id: crate::domain::AppId::default(),
1274 topology_path: PathBuf::from("/tmp/test-sando.toml"),
1275 build_host: Some("test-host".into()),
1276 workdir,
1277 release_root: PathBuf::from("/tmp/rr"),
1278 scratch_db_url: None,
1279 scratch_owner_role: "makenotwork".into(),
1280 boot_smoke_port: 18181,
1281 code_smoke_port: 18182,
1282 bin_names: vec!["makenotwork".into()],
1283 logs_root: PathBuf::from("/tmp/logs"),
1284 release_contents: vec![],
1285 cargo_target_dir: None,
1286 gate_timeout_secs: 2400,
1287 companions: Vec::new(),
1288 test_targets: vec![],
1289 migration_checks: vec![],
1290 frontend_builds: vec![],
1291 backup_max_age_hours: 48,
1292 }
1293 }
1294
1295 fn topo_with_aux(aux_repos: Vec<AuxRepo>) -> Topology {
1296 Topology {
1297 repo: Some(RepoConfig {
1298 bare_path: "/tmp/x.git".into(),
1299 branch: "main".into(),
1300 upstream: None,
1301 }),
1302 backup: vec![BackupConfig {
1303 name: "server".into(),
1304 source: "s".into(),
1305 local_path: "/tmp/d".into(),
1306 }],
1307 tiers: vec![],
1308 aux_repos,
1309 }
1310 }
1311
1312 #[tokio::test]
1313 async fn a_gate_looks_where_the_aux_checkout_actually_landed() {
1314 // The two halves of the aux-repo test_target path: checkout_aux_repos
1315 // writes the tree, and GateCtx::target_dir reads it. Nothing but this
1316 // stops one from being changed without the other, and the failure would
1317 // be a warn-and-skip — a green gate that ran one crate fewer.
1318 let tmp = tempfile::tempdir().unwrap();
1319 let src = tmp.path().join("docengine-src");
1320 tokio::fs::create_dir_all(&src).await.unwrap();
1321 git_in(&src, &["init", "-q", "-b", "main"]).await;
1322 tokio::fs::write(src.join("Cargo.toml"), b"[package]\nname = \"docengine\"\n")
1323 .await
1324 .unwrap();
1325 git_in(&src, &["add", "."]).await;
1326 git_in(&src, &["commit", "-q", "-m", "one"]).await;
1327
1328 let workdir = tmp.path().join("work");
1329 tokio::fs::create_dir_all(&workdir).await.unwrap();
1330 let cfg = cfg_with_workdir(workdir.clone());
1331 let topo = topo_with_aux(vec![AuxRepo {
1332 name: "docengine".into(),
1333 bare_path: tmp
1334 .path()
1335 .join("docengine.git")
1336 .to_string_lossy()
1337 .into_owned(),
1338 upstream: src.to_string_lossy().into_owned(),
1339 branch: "main".into(),
1340 // Nested, as the real one is: it must not be mistaken for a path
1341 // under the per-sha worktree.
1342 checkout_dir: "Libraries/docengine".into(),
1343 }]);
1344 checkout_aux_repos(&cfg, &topo).await.unwrap();
1345
1346 let ctx = crate::gates::GateCtx {
1347 pool: sqlx::SqlitePool::connect_lazy("sqlite::memory:").unwrap(),
1348 cfg: Arc::new(cfg.clone()),
1349 tier: crate::domain::TierId::new("host"),
1350 version: "0.1.0".parse().unwrap(),
1351 worktree: Some(workdir.join("abc123")),
1352 bundle: None,
1353 events: crate::events::channel(),
1354 nodes: Vec::new(),
1355 build_id: None,
1356 aux_dirs: super::aux_checkout_dirs(&cfg, &topo),
1357 };
1358 let target = crate::config::TestTarget {
1359 dir: PathBuf::new(),
1360 aux_repo: Some("docengine".into()),
1361 features: Vec::new(),
1362 all_features: true,
1363 scratch_db: false,
1364 };
1365 let resolved = ctx.target_dir(&target).expect("aux repo is checked out");
1366 assert!(
1367 resolved.join("Cargo.toml").is_file(),
1368 "gate would skip the aux target as absent; resolved {}",
1369 resolved.display(),
1370 );
1371 assert!(
1372 !resolved.starts_with(ctx.worktree.as_ref().unwrap()),
1373 "an aux checkout is a sibling of the worktree, not under it",
1374 );
1375 }
1376
1377 #[tokio::test]
1378 async fn checkout_aux_repos_places_repo_beside_worktree_and_refreshes_to_branch_head() {
1379 let tmp = tempfile::tempdir().unwrap();
1380
1381 // An "upstream" source repo with a marker file on main.
1382 let src = tmp.path().join("synckit-src");
1383 tokio::fs::create_dir_all(&src).await.unwrap();
1384 git_in(&src, &["init", "-q", "-b", "main"]).await;
1385 tokio::fs::write(src.join("VERSION"), b"v1").await.unwrap();
1386 git_in(&src, &["add", "."]).await;
1387 git_in(&src, &["commit", "-q", "-m", "one"]).await;
1388
1389 let workdir = tmp.path().join("work");
1390 tokio::fs::create_dir_all(&workdir).await.unwrap();
1391 let cfg = cfg_with_workdir(workdir.clone());
1392 let topo = topo_with_aux(vec![AuxRepo {
1393 name: "synckit".into(),
1394 bare_path: tmp
1395 .path()
1396 .join("synckit.git")
1397 .to_string_lossy()
1398 .into_owned(),
1399 upstream: src.to_string_lossy().into_owned(),
1400 branch: "main".into(),
1401 checkout_dir: "synckit".into(),
1402 }]);
1403
1404 // First build: the aux repo lands at workdir/synckit at v1.
1405 checkout_aux_repos(&cfg, &topo).await.unwrap();
1406 let dest = workdir.join("synckit");
1407 assert_eq!(
1408 tokio::fs::read(dest.join("VERSION")).await.unwrap(),
1409 b"v1",
1410 "aux repo checked out beside the worktree",
1411 );
1412
1413 // Upstream advances; a later build refreshes the shared checkout to HEAD.
1414 tokio::fs::write(src.join("VERSION"), b"v2").await.unwrap();
1415 git_in(&src, &["add", "."]).await;
1416 git_in(&src, &["commit", "-q", "-m", "two"]).await;
1417 checkout_aux_repos(&cfg, &topo).await.unwrap();
1418 assert_eq!(
1419 tokio::fs::read(dest.join("VERSION")).await.unwrap(),
1420 b"v2",
1421 "aux checkout refreshed to the new branch HEAD",
1422 );
1423
1424 // The aux bare carries no build-trigger hook.
1425 assert!(
1426 !tmp.path().join("synckit.git/hooks/post-receive").exists(),
1427 "aux bare must be hookless",
1428 );
1429 }
1430
1431 #[tokio::test]
1432 async fn checkout_aux_repos_is_a_noop_without_aux_repos() {
1433 let tmp = tempfile::tempdir().unwrap();
1434 let cfg = cfg_with_workdir(tmp.path().to_path_buf());
1435 checkout_aux_repos(&cfg, &topo_with_aux(vec![]))
1436 .await
1437 .unwrap();
1438 }
1439
1440 #[tokio::test]
1441 async fn checkout_aux_repos_fails_on_an_unresolvable_branch() {
1442 let tmp = tempfile::tempdir().unwrap();
1443 let src = tmp.path().join("src");
1444 tokio::fs::create_dir_all(&src).await.unwrap();
1445 git_in(&src, &["init", "-q", "-b", "main"]).await;
1446 tokio::fs::write(src.join("f"), b"x").await.unwrap();
1447 git_in(&src, &["add", "."]).await;
1448 git_in(&src, &["commit", "-q", "-m", "c"]).await;
1449
1450 let cfg = cfg_with_workdir(tmp.path().join("work"));
1451 let topo = topo_with_aux(vec![AuxRepo {
1452 name: "synckit".into(),
1453 bare_path: tmp.path().join("s.git").to_string_lossy().into_owned(),
1454 upstream: src.to_string_lossy().into_owned(),
1455 branch: "nonexistent".into(),
1456 checkout_dir: "synckit".into(),
1457 }]);
1458 let err = checkout_aux_repos(&cfg, &topo).await.unwrap_err();
1459 assert!(
1460 format!("{err:#}").contains("synckit"),
1461 "error names the aux repo: {err:#}",
1462 );
1463 }
1464
1465 async fn tier_versions(pool: &SqlitePool, tier: &str) -> (Option<String>, Option<String>) {
1466 sqlx::query_as("SELECT current_version, previous_version FROM tier_state WHERE tier = ?")
1467 .bind(tier)
1468 .fetch_one(pool)
1469 .await
1470 .unwrap()
1471 }
1472
1473 async fn run_result(pool: &SqlitePool, run_id: RunId) -> (String, Option<String>) {
1474 sqlx::query_as("SELECT result, failure_summary FROM build_runs WHERE id = ?")
1475 .bind(run_id.0)
1476 .fetch_one(pool)
1477 .await
1478 .unwrap()
1479 }
1480
1481 #[tokio::test]
1482 async fn stage_and_gate_stages_advances_and_flips_the_symlink_when_gates_are_green() {
1483 let (pool, cfg, topo, art, run_id, version, tmp) = stage_fixture(vec![]).await;
1484 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1485
1486 stage_and_gate(
1487 pool.clone(),
1488 cfg.clone(),
1489 topo,
1490 art,
1491 crate::events::channel(),
1492 run_id,
1493 deploy_lock,
1494 )
1495 .await
1496 .expect("green host pipeline returns Ok");
1497
1498 // Tier advanced to the built version (previous was NULL -> stays NULL).
1499 let (current, previous) = tier_versions(&pool, "host").await;
1500 assert_eq!(current.as_deref(), Some(version.to_string().as_str()));
1501 assert_eq!(previous, None);
1502
1503 // Run settled green.
1504 let (result, summary) = run_result(&pool, run_id).await;
1505 assert_eq!(result, "passed");
1506 assert_eq!(summary, None);
1507
1508 // Identity: the build row carries the bundle digest (64 hex) and the
1509 // content-addressed dir it was published to (releases/<digest16>).
1510 let (digest, staged_path): (Option<String>, Option<String>) =
1511 sqlx::query_as("SELECT bundle_digest, staged_path FROM build_runs WHERE id = ?")
1512 .bind(run_id.0)
1513 .fetch_one(&pool)
1514 .await
1515 .unwrap();
1516 let digest = digest.expect("bundle_digest recorded");
1517 let staged_path = staged_path.expect("staged_path recorded");
1518 assert_eq!(digest.len(), 64);
1519 let releases = tmp.path().join("release-root").join("releases");
1520 assert_eq!(
1521 std::path::Path::new(&staged_path),
1522 releases.join(&digest[..16]),
1523 "bundle is published content-addressed at releases/<digest16>"
1524 );
1525
1526 // versions.artifact_path points at the primary binary inside that dir,
1527 // and it exists on disk.
1528 let staged_bin: String =
1529 sqlx::query_scalar("SELECT artifact_path FROM versions WHERE version = ?")
1530 .bind(version.to_string())
1531 .fetch_one(&pool)
1532 .await
1533 .unwrap();
1534 let expected_bin = releases.join(&digest[..16]).join("makenotwork");
1535 assert_eq!(staged_bin, expected_bin.to_string_lossy());
1536 assert!(
1537 expected_bin.exists(),
1538 "staged binary missing at {expected_bin:?}"
1539 );
1540
1541 // The bundle carries its MANIFEST (for node-side verification), and the
1542 // recorded digest recomputes over the published dir (MANIFEST excluded).
1543 assert!(
1544 releases.join(&digest[..16]).join("MANIFEST").exists(),
1545 "MANIFEST written into the bundle"
1546 );
1547 let recomputed = crate::bundle::digest_dir(std::path::Path::new(&staged_path))
1548 .await
1549 .unwrap();
1550 assert_eq!(
1551 digest, recomputed.full,
1552 "recorded digest matches the bundle"
1553 );
1554
1555 // The `current` symlink flipped to the content-addressed release.
1556 let link = tmp.path().join("release-root").join("current");
1557 let target = std::fs::read_link(&link).expect("current is a symlink");
1558 assert_eq!(target, PathBuf::from(format!("releases/{}", &digest[..16])));
1559 }
1560
1561 #[tokio::test]
1562 async fn stage_and_gate_marks_the_run_failed_and_does_not_advance_when_a_gate_is_red() {
1563 // ManualConfirm with no prior confirmation row blocks deterministically.
1564 let (pool, cfg, topo, art, run_id, _version, _tmp) =
1565 stage_fixture(vec![Gate::ManualConfirm]).await;
1566 let deploy_lock = Arc::new(tokio::sync::Mutex::new(()));
1567
1568 // A red gate is a pipeline outcome, not an error: the fn records the
1569 // failure and returns Ok so the spawned task settles the run cleanly.
1570 stage_and_gate(
1571 pool.clone(),
1572 cfg,
1573 topo,
1574 art,
1575 crate::events::channel(),
1576 run_id,
1577 deploy_lock,
1578 )
1579 .await
1580 .expect("a red gate settles the run, it does not error out");
1581
1582 // Tier did NOT advance — still the seeded NULL/NULL.
1583 let (current, previous) = tier_versions(&pool, "host").await;
1584 assert_eq!(current, None);
1585 assert_eq!(previous, None);
1586
1587 // Run settled red with a non-empty summary.
1588 let (result, summary) = run_result(&pool, run_id).await;
1589 assert_eq!(result, "failed");
1590 assert!(
1591 summary.as_deref().is_some_and(|s| !s.is_empty()),
1592 "failed run must carry a summary, got {summary:?}"
1593 );
1594 }
1595
1596 #[test]
1597 fn check_build_host_accepts_matching_host() {
1598 assert!(check_build_host("fw13", "fw13").is_ok());
1599 }
1600
1601 #[test]
1602 fn check_build_host_refuses_mismatched_host() {
1603 // A daemon misdeployed onto prod (e.g. a Hetzner host) must refuse.
1604 let err = check_build_host("alpha-west-1", "fw13")
1605 .unwrap_err()
1606 .to_string();
1607 assert!(err.contains("refusing to build"), "{err}");
1608 assert!(
1609 err.contains("alpha-west-1") && err.contains("fw13"),
1610 "{err}"
1611 );
1612 }
1613
1614 #[test]
1615 fn runtime_hostname_reads_a_nonempty_trimmed_name() {
1616 let h = runtime_hostname().expect("hostname readable on Linux");
1617 assert!(!h.is_empty());
1618 assert_eq!(h, h.trim(), "must be trimmed");
1619 }
1620
1621 #[test]
1622 fn tail_does_not_panic_on_multibyte_boundary() {
1623 // Each '€' is 3 bytes; a byte cap landing mid-codepoint must not panic.
1624 let s = "€".repeat(10); // 30 bytes
1625 for max in 1..=30 {
1626 let out = tail(s.as_bytes(), max);
1627 assert!(out.len() <= max, "max={max} got {} bytes", out.len());
1628 // Result is always valid UTF-8 made only of whole '€'s.
1629 assert!(out.chars().all(|c| c == '€'), "max={max}: {out:?}");
1630 }
1631 }
1632
1633 #[test]
1634 fn tail_returns_whole_input_when_under_cap() {
1635 assert_eq!(tail(b"hello", 100), "hello");
1636 }
1637 }
1638