//! Build runner, dispatches and executes OTA builds via SSH to remote hosts. //! //! The scheduler calls `dispatch_pending_build()` each tick. If no build is //! running and one is pending, it spawns a `tokio::spawn` task that SSHes to //! the appropriate build host, clones, builds, signs, and uploads artifacts. use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; use axum::extract::FromRef; use crate::AppState; use crate::config::Config; use crate::constants::{BUILD_MAX_LOG_BYTES, BUILD_TIMEOUT_SECS}; use crate::db::{self, BuildStatus, DbBuild, DbBuildConfig}; use crate::scanning::ScanPipeline; use crate::storage::StorageBackend; use crate::wam_client::WamClient; /// The slice of [`AppState`] the OTA build runner needs: the DB pool it drives /// every query through, the config for build-host lookup, the optional scanner /// it enqueues uploaded artifacts onto, the SyncKit blob bucket it uploads to, /// and the optional WAM client for build-failure tickets. /// /// Field names mirror [`AppState`] (`ctx.db`, `ctx.config`, `ctx.scanner`, /// `ctx.wam`) so the runner body reads like the pre-decomposition code. This is a /// projection view (like the handler slices in [`crate`]); [`dispatch_pending_build`] /// takes it instead of the whole `AppState`, so the runner's dependencies are /// stated, not ambient. `synckit_s3` is flattened out of `AppStorage` here since /// the runner touches only that one bucket. #[derive(Clone)] pub struct BuildCtx { pub db: sqlx::PgPool, pub config: Config, pub scanner: Option>, pub synckit_s3: Option>, pub wam: Option, } impl FromRef for BuildCtx { fn from_ref(s: &AppState) -> Self { Self { db: s.db.clone(), config: s.config.clone(), scanner: s.scanner.clone(), synckit_s3: s.storage.synckit_s3.clone(), wam: s.wam.clone(), } } } /// Wall-clock cap for an artifact SCP download. Only the build ssh was /// timeout-wrapped; a build host that stalled mid-transfer wedged the single /// build slot forever. Artifacts can be large, so this is /// generous; `kill_on_drop` reaps the scp when the timed-out future is dropped. const SCP_TRANSFER_TIMEOUT_SECS: u64 = 600; /// Wall-clock cap for best-effort remote cleanup (`rm -rf` of the build dir). /// Short: cleanup should be near-instant, and a stalled cleanup must not hold /// the build slot. const SSH_CLEANUP_TIMEOUT_SECS: u64 = 60; /// Best-effort remote build-dir cleanup with a bounded wall-clock. `kill_on_drop` /// in `run_ssh_command` reaps the ssh process if the timeout fires. async fn cleanup_remote_dir(host: &str, build_dir: &str) { let cmd = format!("rm -rf {}", shell_escape(build_dir)); // Genuinely nothing to do on failure: the build must not be held up over a // leftover directory, and a retry on a host that just timed out would hold // the slot longer. It still gets named, because the host is now carrying a // build dir nobody will remove and that is an operator's problem. match tokio::time::timeout( Duration::from_secs(SSH_CLEANUP_TIMEOUT_SECS), Box::pin(run_ssh_command(host, &cmd)), ) .await { Ok(Ok(_)) => {} Ok(Err(e)) => { tracing::warn!(host, build_dir, error = %e, "remote build dir cleanup failed"); } Err(_) => { tracing::warn!(host, build_dir, "remote build dir cleanup timed out"); } } } /// Post-receive hook script template. /// `__HMAC__` is replaced with a per-repo HMAC signature so the global token /// is never stored on disk. The server verifies via `HMAC(token, owner:repo)`. /// /// Both curl calls run backgrounded so they never block the git push, but /// their stdout+stderr is appended to `hooks/post-receive.log` next to this /// script. A non-zero curl exit also writes a "FAILED" line with the exit /// code, so a build that never triggers is diagnosable from the repo rather /// than from "why didn't anything happen." The log is append-only and grows /// unbounded; truncate or rotate via the host's logrotate. const POST_RECEIVE_HOOK_TEMPLATE: &str = r#"#!/bin/bash REPO_PATH="$(cd "$(dirname "$0")/.." && pwd)" LOG="$REPO_PATH/hooks/post-receive.log" REPO_NAME="$(basename "$REPO_PATH" .git)" OWNER="$(basename "$(dirname "$REPO_PATH")")" while read oldrev newrev refname; do case "$refname" in refs/tags/v[0-9]*) TAG="${refname#refs/tags/}" ( exec >>"$LOG" 2>&1 echo "[$(date -u +%FT%TZ)] tag-push $OWNER/$REPO_NAME tag=$TAG" curl -sf -X POST \ -H "Authorization: Bearer __HMAC__" \ -H "Content-Type: application/json" \ -d "{\"repo_owner\": \"$OWNER\", \"repo_name\": \"$REPO_NAME\", \"tag\": \"$TAG\"}" \ "http://localhost:3000/api/internal/builds/trigger" \ || echo "[$(date -u +%FT%TZ)] FAILED builds/trigger exit=$?" ) & ;; refs/heads/*) BRANCH="${refname#refs/heads/}" ( exec >>"$LOG" 2>&1 echo "[$(date -u +%FT%TZ)] branch-push $OWNER/$REPO_NAME branch=$BRANCH" curl -sf -X POST \ -H "Authorization: Bearer __HMAC__" \ -H "Content-Type: application/json" \ -d "{\"repo_owner\": \"$OWNER\", \"repo_name\": \"$REPO_NAME\", \"ref_name\": \"$BRANCH\", \"before\": \"$oldrev\", \"after\": \"$newrev\"}" \ "http://localhost:3000/api/internal/issues/process-push" \ || echo "[$(date -u +%FT%TZ)] FAILED issues/process-push exit=$?" ) & ;; refs/notes/*) # A note that arrived by push. Backgrounded like the builds and # issues arms: the index is a projection of what the push already # landed, so there is nothing to tell the pusher and nothing they # would do about it. The inbox arm below is the one that answers. ( exec >>"$LOG" 2>&1 echo "[$(date -u +%FT%TZ)] notes-index $OWNER/$REPO_NAME ref=$refname" curl -sf -X POST \ -H "Authorization: Bearer __HMAC__" \ -H "Content-Type: application/json" \ -d "{\"repo_owner\": \"$OWNER\", \"repo_name\": \"$REPO_NAME\", \"ref_name\": \"$refname\"}" \ "http://localhost:3000/api/internal/notes/reindex" \ || echo "[$(date -u +%FT%TZ)] FAILED notes/reindex exit=$?" ) & ;; refs/mnw/notes-inbox/*) # The one arm that is NOT backgrounded. Builds and issue links are # things the pusher learns about later; a notes merge is the answer # to the push itself, and post-receive stdout is the only channel # back to them. The timeouts bound what that costs: a push cannot # hang on a server that is not answering. echo "[$(date -u +%FT%TZ)] notes-push $OWNER/$REPO_NAME ref=$refname" >>"$LOG" RESULT="$(curl -sf --connect-timeout 5 --max-time 30 -X POST \ -H "Authorization: Bearer __HMAC__" \ -H "Content-Type: application/json" \ -d "{\"repo_owner\": \"$OWNER\", \"repo_name\": \"$REPO_NAME\", \"ref_name\": \"$refname\"}" \ "http://localhost:3000/api/internal/notes/merge-inbox" 2>>"$LOG")" if [ -n "$RESULT" ]; then echo "notes: merged into refs/notes/${refname#refs/mnw/notes-inbox/}" echo "[$(date -u +%FT%TZ)] notes-merged $RESULT" >>"$LOG" else # The notes are in the inbox ref either way, so nothing is lost; # the next push to the namespace merges them. Say so rather than # letting the push look like it did nothing. echo "notes: received, merge deferred (the server did not answer)" echo "[$(date -u +%FT%TZ)] FAILED notes/merge-inbox" >>"$LOG" fi ;; esac done "#; /// The `update` hook: refuse a push to a namespace MNW owns. /// /// `update` rather than `pre-receive` because it runs once per ref and rejects /// only that one. A push carrying a branch and a stray `refs/notes/mnw/*` should /// land the branch and refuse the note, not fail whole. /// /// This is the enforcement half of a policy the in-process write paths already /// hold (`validate_note_namespace`): the browser, the JSON API and the notes /// inbox all refuse the prefix. A direct push reached none of them, which left /// the one door with no lock on it. The server writes these refs through gix ref /// transactions, which run no hooks, so the door being locked from outside does /// not lock us out. /// /// Static, so it carries no HMAC and needs no per-repo generation. The `mnw` /// literal is `validation::RESERVED_NOTE_NAMESPACE`; a test pins the two /// together, since bash cannot read the constant. pub const UPDATE_HOOK: &str = r#"#!/bin/bash case "$1" in refs/notes/mnw|refs/notes/mnw/*) echo "refs/notes/mnw/* is written by makenot.work and cannot be pushed." echo "Annotate under a namespace of your own instead:" echo " git push origin refs/notes/:refs/mnw/notes-inbox/" exit 1 ;; esac exit 0 "#; /// Compute a per-repo HMAC so the global token never touches disk. pub fn repo_hmac(token: &str, owner: &str, repo: &str) -> String { use hmac::{Hmac, KeyInit, Mac}; use sha2::Sha256; let mut mac = Hmac::::new_from_slice(token.as_bytes()).expect("HMAC accepts any key length"); mac.update(format!("{owner}:{repo}").as_bytes()); hex::encode(mac.finalize().into_bytes()) } /// Generate the post-receive hook script with a per-repo HMAC signature. pub fn post_receive_hook(token: &str, owner: &str, repo: &str) -> String { let hmac = repo_hmac(token, owner, repo); POST_RECEIVE_HOOK_TEMPLATE.replace("__HMAC__", &hmac) } /// Map (os, arch) to a Rust target triple. pub fn rust_target(os: &str, arch: &str) -> Option<&'static str> { match (os, arch) { ("linux", "x86_64") => Some("x86_64-unknown-linux-gnu"), ("linux", "aarch64") => Some("aarch64-unknown-linux-gnu"), ("darwin", "x86_64") => Some("x86_64-apple-darwin"), ("darwin", "aarch64") => Some("aarch64-apple-darwin"), _ => None, } } /// Get the SSH build host for a target OS from config. fn build_host_for_target<'a>(config: &'a crate::config::Config, os: &str) -> Option<&'a str> { match os { "linux" => config.build.host_linux.as_deref(), "darwin" => config.build.host_darwin.as_deref(), _ => None, } } /// Check for a pending build and spawn it if no build is currently running. /// /// Called from the scheduler loop. Non-blocking, spawns the build task and returns. #[tracing::instrument(skip_all, name = "build_runner::dispatch")] pub async fn dispatch_pending_build(ctx: &BuildCtx) { // Recover from stale running builds (e.g. server crashed mid-build) match db::builds::fail_stale_running_builds(&ctx.db, BUILD_TIMEOUT_SECS as i64).await { Ok(n) if n > 0 => { tracing::warn!(count = n, "marked stale running builds as failed"); } Err(e) => { tracing::error!(error = ?e, "failed to check stale builds"); } _ => {} } let build = match db::builds::claim_pending_build(&ctx.db).await { Ok(Some(b)) => b, Ok(None) => return, Err(e) => { tracing::error!(error = ?e, "failed to claim pending build"); return; } }; let config = match db::builds::get_build_config_by_app(&ctx.db, build.app_id).await { Ok(Some(c)) => c, Ok(None) => { tracing::error!(build_id = %build.id, "build config not found for pending build"); if let Err(e) = db::builds::update_build_status( &ctx.db, build.id, BuildStatus::Failed, Some("Build config not found"), ) .await { tracing::error!(build_id = %build.id, error = ?e, "failed to mark build as failed (config not found)"); } return; } Err(e) => { tracing::error!(error = ?e, "failed to get build config"); return; } }; let ctx = ctx.clone(); tokio::spawn(async move { run_build(&ctx, &build, &config).await; // After the run rather than inside it, and from a re-read row rather // than the one above: `run_build` has several exits and the note has to // say what the build finally was, whichever one it took. A path added // later is covered without anybody remembering to cover it. annotate_build(&ctx, build.id, &config).await; }); } /// Mirror a finished build's outcome into `refs/notes/mnw/builds`. /// /// Every failure here is a warning and nothing more. The build result is in /// Postgres and served from there; the note is a copy of it that the creator /// keeps when they leave. async fn annotate_build(ctx: &BuildCtx, build_id: db::BuildId, config: &DbBuildConfig) { let build = match db::builds::get_build(&ctx.db, build_id).await { Ok(Some(b)) => b, Ok(None) => return, Err(e) => { tracing::warn!(build_id = %build_id, error = ?e, "could not re-read build to annotate it"); return; } }; let Some((owner, repo_name, repo_id)) = repo_for_build(ctx, config.repo_id).await else { return; }; // The tag is what triggered the build; the note goes on the commit it names, // which is the page somebody actually opens. Peeling also covers an // annotated tag, where the ref points at a tag object rather than a commit. let Some(target) = crate::routes::git::notes_server::resolve_tag_commit( &ctx.config, &owner, &repo_name, &build.tag, ) .await else { tracing::warn!( build_id = %build_id, tag = %build.tag, "build tag does not resolve to a commit; no note written" ); return; }; crate::routes::git::notes_server::note_build( &crate::routes::git::notes_server::OwnedRepo { db: &ctx.db, config: &ctx.config, id: repo_id, owner: &owner, name: &repo_name, }, target, &build, &config.targets, ) .await; } /// Resolve a build config's repo to the `(owner, name, id)` the notes writer /// needs. async fn repo_for_build( ctx: &BuildCtx, repo_id: db::GitRepoId, ) -> Option<(String, String, db::GitRepoId)> { let repo = match db::git_repos::get_repo_by_id(&ctx.db, repo_id).await { Ok(Some(r)) => r, Ok(None) => return None, Err(e) => { tracing::warn!(error = ?e, "could not load the repo behind a build"); return None; } }; let owner = match db::users::get_user_by_id(&ctx.db, repo.user_id).await { Ok(Some(u)) => u, Ok(None) => return None, Err(e) => { tracing::warn!(error = ?e, "could not load the owner of a build's repo"); return None; } }; Some((owner.username.to_string(), repo.name.clone(), repo.id)) } fn build_failure_message(succeeded: usize, failed: usize, first_error: Option<&str>) -> String { if succeeded == 0 { first_error .unwrap_or("no targets produced artifacts") .to_string() } else { let total = succeeded + failed; format!("partial build failure ({succeeded}/{total} targets succeeded)") } } /// A successfully built target: `(target_os, arch, s3_key, signature)`. type TargetArtifact = (String, String, String, String); /// A failed target: `(target_str, error)`. type TargetError = (String, String); /// One host group's results: its artifacts and its per-target failures. type GroupOutput = (Vec, Vec); /// Execute a full build: iterate targets, SSH to hosts, build, upload artifacts. #[tracing::instrument(skip_all, name = "build_runner::run_build", fields(build_id = %build.id, version = %build.version))] async fn run_build(ctx: &BuildCtx, build: &DbBuild, config: &DbBuildConfig) { let mut artifact_keys: Vec<(String, String, String, String)> = Vec::new(); // (target_os, arch, s3_key, signature) let mut failed_count: usize = 0; let mut first_error: Option = None; // Shared across the per-host tasks, so a log line dropped inside any target // still reaches the final status message. let log_drops = Arc::new(AtomicUsize::new(0)); // Resolve each target to its build host up front. Synchronous failures (bad // target format, no host configured) are tallied here; resolvable targets are // grouped by host so independent hosts (e.g. linux vs darwin) build // concurrently while same-host targets stay serial, a multi-target release no // longer serializes end to end at up to 30 min/target (Perf-S2, Run 9). let mut groups: Vec<(String, Vec<(String, String)>)> = Vec::new(); // host -> [(os, arch)] for target_str in &config.targets { let Some((target_os, arch)): Option<(&str, &str)> = target_str.split_once('/') else { let msg = format!("invalid target format: {target_str}\n"); append_log(ctx, build.id, &msg, &log_drops).await; failed_count += 1; if first_error.is_none() { first_error = Some(format!("invalid target format: {target_str}")); } continue; }; let Some(host) = build_host_for_target(&ctx.config, target_os) else { let msg = format!("no build host for {target_os}, skipping {target_str}\n"); tracing::warn!("{}", msg.trim()); append_log(ctx, build.id, &msg, &log_drops).await; failed_count += 1; if first_error.is_none() { first_error = Some(format!("no build host for {target_os}")); } continue; }; let entry = (target_os.to_string(), arch.to_string()); match groups.iter_mut().find(|(h, _)| h == host) { Some((_, targets)) => targets.push(entry), None => groups.push((host.to_string(), vec![entry])), } } // One task per host; targets within a host run serially. Results are gathered // by group index so the merged artifact order stays deterministic regardless // of which host finishes first. let group_count = groups.len(); let mut set: tokio::task::JoinSet<(usize, GroupOutput)> = tokio::task::JoinSet::new(); for (idx, (host, targets)) in groups.into_iter().enumerate() { let ctx = ctx.clone(); let build = build.clone(); let config = config.clone(); let log_drops = Arc::clone(&log_drops); set.spawn(async move { let mut oks: Vec = Vec::new(); let mut errs: Vec = Vec::new(); for (target_os, arch) in &targets { match Box::pin(execute_target( &ctx, &build, &config, &host, target_os, arch, &log_drops, )) .await { Ok((s3_key, signature)) => { oks.push((target_os.clone(), arch.clone(), s3_key, signature)); } Err(e) => errs.push((format!("{target_os}/{arch}"), e)), } } (idx, (oks, errs)) }); } let mut gathered: Vec> = (0..group_count).map(|_| None).collect(); while let Some(res) = set.join_next().await { match res { Ok((idx, out)) => gathered[idx] = Some(out), Err(e) => { // A host group's task panicked; the build can't be considered // complete, so fail it rather than silently dropping its targets. tracing::error!(error = ?e, "build host group task panicked"); failed_count += 1; if first_error.is_none() { first_error = Some("a build host group task panicked".to_string()); } } } } for (oks, errs) in gathered.into_iter().flatten() { artifact_keys.extend(oks); for (target_str, e) in errs { let msg = format!("target {target_str} failed: {e}\n"); tracing::error!("{}", msg.trim()); append_log(ctx, build.id, &msg, &log_drops).await; failed_count += 1; if first_error.is_none() { first_error = Some(e); } } } if artifact_keys.is_empty() || failed_count > 0 { let mut err_msg = build_failure_message(artifact_keys.len(), failed_count, first_error.as_deref()); if let Some(note) = incomplete_log_note(&log_drops) { err_msg.push_str(¬e); } if let Err(e) = db::builds::update_build_status(&ctx.db, build.id, BuildStatus::Failed, Some(&err_msg)) .await { tracing::error!(build_id = %build.id, error = ?e, "failed to mark build as failed"); } if let Some(ref wam) = ctx.wam { let title = format!("Build failed: {} v{}", build.tag, build.version); wam.create_ticket( &title, Some(&err_msg), "high", "build-failed", Some(&build.id.to_string()), ) .await; } return; } // Every artifact is signed independently and served with its own signature. // An unsigned artifact can never be installed (Tauri refuses an unsigned // update), so if any successful target lacks a signature, fail the build // loudly rather than publishing a release with a dead platform. if let Some((target_os, arch, _, _)) = artifact_keys.iter().find(|(_, _, _, sig)| sig.is_empty()) { let msg = format!( "build produced an unsigned artifact ({target_os}/{arch}); refusing to publish" ); if let Err(e) = db::builds::update_build_status(&ctx.db, build.id, BuildStatus::Failed, Some(&msg)) .await { tracing::error!(build_id = %build.id, error = ?e, "failed to mark build failed (missing signature)"); } return; } // Create OTA release (only for fully successful builds) let release = match db::ota::create_release( &ctx.db, build.app_id, &build.version, &format!("Automated build from tag {}", build.tag), ) .await { Ok(r) => r, Err(e) => { let msg = format!("failed to create OTA release: {e}"); if let Err(e) = db::builds::update_build_status(&ctx.db, build.id, BuildStatus::Failed, Some(&msg)) .await { tracing::error!(build_id = %build.id, error = ?e, "failed to mark build as failed (release creation)"); } return; } }; // The app owner is the responsible identity for the artifact scans. let owner_id = db::synckit::get_sync_app_by_id(&ctx.db, build.app_id) .await .ok() .flatten() .map(|app| app.creator_id); // Record artifacts and enqueue each for malware scanning. The artifact stays // `pending` (not served) until the scan clears it, same gate as the item // channel. for (target_os, arch, s3_key, signature) in &artifact_keys { // Get file size from S3 via HEAD request (best-effort, use 0 if unavailable) let file_size = if let Some(s3) = ctx.synckit_s3.as_ref() { s3.object_size(s3_key).await.ok().flatten().unwrap_or(0) } else { 0 }; match db::ota::create_artifact( &ctx.db, release.id, target_os, arch, s3_key, file_size, signature, ) .await { Ok(artifact) => { if let Some(owner_id) = owner_id && let Err(e) = crate::routes::ota::enqueue_ota_artifact_scan( &ctx.db, ctx.scanner.as_ref(), artifact.id, s3_key, owner_id, file_size, ) .await { tracing::error!(artifact_id = %artifact.id, error = ?e, "failed to enqueue OTA artifact scan"); } } Err(e) => tracing::error!(error = ?e, "failed to record artifact"), } } // Link build to release if let Err(e) = db::builds::set_build_release(&ctx.db, build.id, release.id).await { tracing::error!(build_id = %build.id, release_id = %release.id, error = ?e, "failed to link build to release"); } // All targets succeeded (partial failures return early above). The build is // still a success if log lines were lost, but the row says so rather than // presenting a short log as the whole story. let note = incomplete_log_note(&log_drops); if let Err(e) = db::builds::update_build_status(&ctx.db, build.id, BuildStatus::Succeeded, note.as_deref()) .await { tracing::error!(build_id = %build.id, error = ?e, "failed to mark build as succeeded"); } tracing::info!( build_id = %build.id, version = %build.version, artifacts = artifact_keys.len(), "build succeeded" ); } /// Execute a single target: SSH to host, clone, build, upload artifact. async fn execute_target( ctx: &BuildCtx, build: &DbBuild, config: &DbBuildConfig, host: &str, target_os: &str, arch: &str, log_drops: &AtomicUsize, ) -> std::result::Result<(String, String), String> { let target = format!("{target_os}/{arch}"); let rust_triple = rust_target(target_os, arch).ok_or_else(|| format!("unsupported target: {target}"))?; // Look up repo for clone URL let repo = db::git_repos::get_repo_by_id(&ctx.db, config.repo_id) .await .map_err(|e| format!("failed to look up repo: {e}"))? .ok_or("repo not found")?; let repo_owner = db::users::get_user_by_id(&ctx.db, repo.user_id) .await .map_err(|e| format!("failed to look up repo owner: {e}"))? .ok_or("repo owner not found")?; let git_root = ctx .config .build .git_repos_path .as_deref() .ok_or("git_repos_path not configured")?; let clone_path = format!("{git_root}/{}/{}.git", repo_owner.username, repo.name); let build_dir = format!("/tmp/mnw-build-{}", build.id); // Template substitution for build_command and artifact_path let build_cmd = config .build_command .replace("{target}", rust_triple) .replace("{version}", &build.version); let artifact_path = config .artifact_path .replace("{target}", rust_triple) .replace("{version}", &build.version); // Parse build_command into a structured, fully-escaped remote command and // validate artifact_path before interpolation into the shell script. let remote_cmd = RemoteCommand::parse(&build_cmd).map_err(|e| format!("invalid build command: {e}"))?; validate_artifact_path(&artifact_path).map_err(|e| format!("invalid artifact path: {e}"))?; // Build the SSH command sequence. Every interpolated value is shell-escaped: // the git/cd/test scaffolding args via `shell_escape`, and the operator build // command via `RemoteCommand::render` (each token escaped, env applied via // `env`). No operator- or record-derived byte reaches the shell unescaped. let remote_script = format!( "set -e && \ git clone --depth 1 --branch {tag} {clone_path} {build_dir} && \ cd {build_dir} && \ {build_cmd} && \ test -f {artifact_path}", tag = shell_escape(&build.tag), clone_path = shell_escape(&clone_path), build_dir = shell_escape(&build_dir), build_cmd = remote_cmd.render(), artifact_path = shell_escape(&artifact_path), ); let log_msg = format!("[{target}] building on {host}...\n"); append_log(ctx, build.id, &log_msg, log_drops).await; // Execute via SSH with timeout let ssh_result = tokio::time::timeout( Duration::from_secs(BUILD_TIMEOUT_SECS), Box::pin(run_ssh_command(host, &remote_script)), ) .await; let output = match ssh_result { Ok(Ok(output)) => output, Ok(Err(e)) => { // Cleanup remote build dir (best-effort) Box::pin(cleanup_remote_dir(host, &build_dir)).await; return Err(format!("SSH command failed: {e}")); } Err(_) => { Box::pin(cleanup_remote_dir(host, &build_dir)).await; return Err("build timed out".to_string()); } }; append_log( ctx, build.id, &format!("[{target}] {}\n", output.trim()), log_drops, ) .await; // SCP artifact back and upload to S3 let s3_key = crate::storage::S3Client::generate_ota_artifact_key( build.app_id, &build.version, target_os, arch, ); // Copy artifact from remote to local temp let local_tmp = format!("/tmp/mnw-artifact-{}-{target_os}-{arch}", build.id); let scp_remote_path = format!( "{}/{}", build_dir.trim_end_matches('/'), artifact_path.trim_start_matches('/') ); let scp_result = run_scp_download(host, &scp_remote_path, &local_tmp).await; // Best-effort: try to download the .sig file (Tauri builds produce one) let local_sig_tmp = format!("{local_tmp}.sig"); let scp_sig_result = run_scp_download(host, &format!("{scp_remote_path}.sig"), &local_sig_tmp).await; Box::pin(cleanup_remote_dir(host, &build_dir)).await; if let Err(e) = scp_result { // The main artifact failed, but the .sig sidecar may already be on disk // from its own scp above. Clean it up before bailing so a retry loop // doesn't accumulate orphaned .sig temp files (the main temp is removed // unconditionally further down, but on this early return it was never // created). remove_temp_file(&local_sig_tmp).await; return Err(format!("SCP download failed: {e}")); } // Read signature from .sig file if it was downloaded let signature = if scp_sig_result.is_ok() { let sig = tokio::fs::read_to_string(&local_sig_tmp) .await .unwrap_or_default(); remove_temp_file(&local_sig_tmp).await; sig } else { String::new() }; // Upload to S3 via multipart streaming from disk, the previous // implementation `tokio::fs::read` → `Vec` → `upload_object` pinned // the entire artifact (up to ~100 MB per build) in RAM during upload. // `upload_multipart` reads the file in chunks and lets the S3 SDK do // parallel part uploads, keeping memory bounded regardless of artifact // size. let synckit_s3 = ctx .synckit_s3 .as_ref() .ok_or("SyncKit storage not configured")?; let upload_result = synckit_s3 .upload_multipart( &s3_key, "application/octet-stream", std::path::Path::new(&local_tmp), ) .await .map_err(|e| format!("S3 multipart upload failed: {e}")); // Always remove the local temp file, even if the upload failed, leaving // it on disk fills the build runner's tmp directory across retries. remove_temp_file(&local_tmp).await; upload_result?; if signature.is_empty() { append_log( ctx, build.id, &format!("[{target}] uploaded to {s3_key}\n"), log_drops, ) .await; } else { append_log( ctx, build.id, &format!("[{target}] uploaded to {s3_key} (signed)\n"), log_drops, ) .await; } Ok((s3_key.into_string(), signature)) } /// Path to a known_hosts file for build SSH connections. /// When present, StrictHostKeyChecking=yes is used (pinned keys). /// When absent, StrictHostKeyChecking=accept-new (trust on first use). const BUILD_SSH_KNOWN_HOSTS: &str = "/etc/mnw/known_hosts"; /// SSH/SCP host-key verification options. Pins host keys /// (`StrictHostKeyChecking=yes`) when the known_hosts file is present; otherwise /// falls back to trust-on-first-use AND logs a warning, so an unprovisioned /// known_hosts is visible in the logs rather than silently accepting any host /// key. Returned owned so callers can push them straight into an argv vector. /// Shared by [`run_ssh_command`] and [`run_scp_download`] so the two can't drift. fn ssh_host_key_args() -> Vec { if std::path::Path::new(BUILD_SSH_KNOWN_HOSTS).exists() { vec![ "-o".into(), "StrictHostKeyChecking=yes".into(), "-o".into(), format!("UserKnownHostsFile={BUILD_SSH_KNOWN_HOSTS}"), ] } else { tracing::warn!( known_hosts = BUILD_SSH_KNOWN_HOSTS, "build SSH known_hosts file absent; falling back to trust-on-first-use (accept-new). \ Provision {BUILD_SSH_KNOWN_HOSTS} to pin build-host keys", ); vec!["-o".into(), "StrictHostKeyChecking=accept-new".into()] } } /// Run a command on a remote host via SSH. async fn run_ssh_command(host: &str, command: &str) -> std::result::Result { let mut args: Vec = vec![ "-o".into(), "ConnectTimeout=10".into(), "-o".into(), "BatchMode=yes".into(), ]; args.extend(ssh_host_key_args()); args.push(host.to_string()); args.push(command.to_string()); let mut child = tokio::process::Command::new("ssh") .args(&args) .stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) // Kill the ssh process if this future is dropped (e.g. the 30-min build // timeout fires): otherwise the dropped future leaves ssh, and the // remote build it drives, running orphaned (ultra-fuzz Run 11 Perf). .kill_on_drop(true) .spawn() .map_err(|e| format!("failed to spawn ssh: {e}"))?; let stdout_pipe = child.stdout.take().expect("stdout piped"); let stderr_pipe = child.stderr.take().expect("stderr piped"); // Stream both pipes with a per-stream cap instead of Command::output(), which // would buffer all of a chatty 30-min build's output in RAM before the log // cap is ever applied. read_capped keeps draining past the cap (so the child // never blocks on a full pipe) but only retains the first BUILD_MAX_LOG_BYTES // (ultra-fuzz Run 10 Perf S2). Reading both concurrently with wait() avoids // the single-pipe-fills-and-deadlocks hazard. let (stdout_buf, stderr_buf, status) = tokio::join!( read_capped(stdout_pipe, BUILD_MAX_LOG_BYTES), read_capped(stderr_pipe, BUILD_MAX_LOG_BYTES), child.wait(), ); let status = status.map_err(|e| format!("ssh wait failed: {e}"))?; if status.success() { Ok(stdout_buf) } else { Err(format!( "exit code {}: {}", status.code().unwrap_or(-1), stderr_buf.trim() )) } } /// Drain an async reader to completion but retain only the first `cap` bytes. /// Draining past the cap keeps the child process from blocking on a full pipe /// buffer; retaining only `cap` bounds memory regardless of output volume. async fn read_capped(mut reader: R, cap: usize) -> String where R: tokio::io::AsyncRead + Unpin, { use tokio::io::AsyncReadExt; let mut kept = Vec::new(); let mut chunk = [0u8; 8192]; loop { match reader.read(&mut chunk).await { Ok(0) | Err(_) => break, Ok(n) => { if kept.len() < cap { let take = n.min(cap - kept.len()); kept.extend_from_slice(&chunk[..take]); } } } } String::from_utf8_lossy(&kept).into_owned() } /// Download a file from a remote host via SCP. async fn run_scp_download( host: &str, remote_path: &str, local_path: &str, ) -> std::result::Result<(), String> { let remote = format!("{host}:{remote_path}"); let mut args: Vec = vec![ "-o".into(), "ConnectTimeout=10".into(), "-o".into(), "BatchMode=yes".into(), ]; args.extend(ssh_host_key_args()); args.push(remote); args.push(local_path.to_string()); let scp = tokio::process::Command::new("scp") .args(&args) // Kill scp if this future is dropped (build timeout, or the transfer // timeout below) rather than leaving an orphaned transfer running // (ultra-fuzz Run 11 Perf). .kill_on_drop(true) .output(); // Bound the whole transfer: a build host that stalls mid-stream would // otherwise hang the build task and pin the single build slot forever // (Run 15 Resilience). ConnectTimeout only covers connection setup. let output = match tokio::time::timeout(Duration::from_secs(SCP_TRANSFER_TIMEOUT_SECS), scp).await { Ok(r) => r.map_err(|e| format!("failed to spawn scp: {e}"))?, Err(_) => return Err("scp transfer timed out".to_string()), }; if output.status.success() { Ok(()) } else { let stderr = String::from_utf8_lossy(&output.stderr); Err(format!( "exit code {}: {}", output.status.code().unwrap_or(-1), stderr.trim() )) } } /// Append to build log, respecting the max log size. /// /// Probes `octet_length(log)` instead of fetching the whole row (the log /// column tops out at 5 MiB and is read on every line append). /// Remove a build temp file, naming it if it could not be removed. /// /// The caller is either bailing out or done with the file, so there is nothing /// to propagate to; what matters is that a temp file left behind is visible, /// since these accumulate across retries and fill the runner's tmp directory. /// A missing file is the expected case on the error paths and is not a failure. async fn remove_temp_file(path: &str) { match tokio::fs::remove_file(path).await { Ok(()) => {} Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} Err(e) => tracing::warn!(path, error = %e, "failed to remove build temp file"), } } /// A one-line note for the build row when log lines were lost, or `None` when /// the log is whole. fn incomplete_log_note(drops: &AtomicUsize) -> Option { match drops.load(Ordering::Relaxed) { 0 => None, n => Some(format!( " (build log incomplete: {n} line(s) could not be stored)" )), } } /// Append a build-log line, counting the line against `drops` if it could not /// be stored. /// /// Nothing useful can be done at the call site (the line is already produced /// and the build is mid-flight), but a build whose log silently lost lines must /// not finish looking clean: `run_build` reads the counter and says so on the /// build row. That is the difference between "the build printed nothing here" /// and "we failed to write down what it printed". async fn append_log(ctx: &BuildCtx, build_id: db::BuildId, line: &str, drops: &AtomicUsize) { if let Err(e) = append_log_bounded(ctx, build_id, line).await { drops.fetch_add(1, Ordering::Relaxed); tracing::error!(build_id = %build_id, error = ?e, "build log append failed; build log is incomplete"); } } async fn append_log_bounded( ctx: &BuildCtx, build_id: db::BuildId, line: &str, ) -> crate::error::Result<()> { const TRUNCATED: &str = "[log truncated]\n"; if let Some((current_len, already_truncated)) = db::builds::get_build_log_size(&ctx.db, build_id, TRUNCATED).await? && (current_len as usize) + line.len() > BUILD_MAX_LOG_BYTES { if !already_truncated { tracing::warn!(build_id = %build_id, "Build log exceeded {} bytes, truncating", BUILD_MAX_LOG_BYTES); db::builds::append_build_log(&ctx.db, build_id, TRUNCATED).await?; } return Ok(()); } let sanitized = strip_ansi_escapes(line); db::builds::append_build_log(&ctx.db, build_id, &sanitized).await } /// Strip ANSI escape sequences (e.g. color codes) from build output before /// storing it in the database. fn strip_ansi_escapes(s: &str) -> String { let mut result = String::with_capacity(s.len()); let mut chars = s.chars(); while let Some(c) = chars.next() { if c == '\x1b' { // Consume the next char; if it's '[' we have a CSI sequence // and we skip parameter/intermediate bytes up to the final byte. // Otherwise (OSC / other sequences) just drop the two-char escape. if let Some(next) = chars.next() && next == '[' { // CSI sequence: skip until we hit a letter (0x40..=0x7E). for tail in chars.by_ref() { if tail.is_ascii_alphabetic() { break; } } } } else { result.push(c); } } result } /// A build command parsed into a shell-injection-proof structured form. /// /// The operator-configured `build_command` is a single string (e.g. /// `RUSTFLAGS=--cfg cargo build --release`). Rather than interpolate it raw into /// the remote `sh -c` script, where its safety rested entirely on a /// metacharacter denylist, one added allowed character away from reopening /// injection, it is tokenised into leading `NAME=VALUE` environment /// assignments followed by a program and its arguments. `render` emits every /// element individually shell-escaped, applying assignments via `env`, so no /// operator byte can break out of its shell word. Shell injection is /// structurally impossible here, not denylist-gated; the per-token charset /// check below is defense-in-depth, no longer the sole guard. struct RemoteCommand { /// Leading `NAME=VALUE` assignments, applied via `env` before the program. assignments: Vec, /// The program to execute. program: String, /// The program's arguments. args: Vec, } impl RemoteCommand { /// Parse a (template-substituted) build command string into its structured /// form. Tokenised on ASCII whitespace; leading `NAME=VALUE` tokens become /// env assignments, the first remaining token is the program, the rest are /// arguments. Each token is charset-validated as defense-in-depth. fn parse(cmd: &str) -> std::result::Result { if cmd.len() > 1024 { return Err("build command too long (max 1024 chars)".to_string()); } let tokens: Vec<&str> = cmd.split_whitespace().collect(); if tokens.is_empty() { return Err("build command is empty".to_string()); } for tok in &tokens { validate_command_token(tok)?; } let mut assignments = Vec::new(); let mut rest = tokens.as_slice(); while let Some((first, tail)) = rest.split_first() { if is_env_assignment(first) { assignments.push((*first).to_string()); rest = tail; } else { break; } } let (program, args) = rest.split_first().ok_or_else(|| { "build command has environment assignments but no program".to_string() })?; Ok(Self { assignments, program: (*program).to_string(), args: args.iter().map(|s| (*s).to_string()).collect(), }) } /// Render as a single shell command line with every element escaped. Safe to /// interpolate into a larger `sh -c` script: no element can inject. fn render(&self) -> String { let mut parts = Vec::with_capacity(self.assignments.len() + self.args.len() + 2); if !self.assignments.is_empty() { parts.push("env".to_string()); parts.extend(self.assignments.iter().map(|a| shell_escape(a))); } parts.push(shell_escape(&self.program)); parts.extend(self.args.iter().map(|a| shell_escape(a))); parts.join(" ") } } /// True if a token is a `NAME=VALUE` environment assignment (a valid shell /// identifier before the first `=`). fn is_env_assignment(tok: &str) -> bool { match tok.split_once('=') { Some((name, _)) => { !name.is_empty() && name.chars().enumerate().all(|(i, c)| { c == '_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit()) }) } None => false, } } /// Validate a single build-command token's charset. Rejects shell /// metacharacters and control characters as defense-in-depth; the rendered /// command escapes every token regardless, so this is not the sole guard. fn validate_command_token(tok: &str) -> std::result::Result<(), String> { for (i, c) in tok.chars().enumerate() { match c { 'a'..='z' | 'A'..='Z' | '0'..='9' => {} '-' | '_' | '.' | '/' | '=' | ':' | ',' | '+' | '{' | '}' | '@' => {} _ => { return Err(format!( "character '{}' at position {} in token '{}' is not allowed", c.escape_default(), i, tok.escape_default(), )); } } } Ok(()) } /// Validate a build command for shell safety at config-write time. Validation is /// exactly "parses into a [`RemoteCommand`]", the same parser the executor uses ///, so a stored command that validates here can never fail to render safely. pub fn validate_build_command(cmd: &str) -> std::result::Result<(), String> { RemoteCommand::parse(cmd).map(|_| ()) } /// Validate an artifact path for shell and path safety. /// /// Must be a relative path with no shell metacharacters or path traversal. pub fn validate_artifact_path(path: &str) -> std::result::Result<(), String> { if path.is_empty() { return Err("artifact path is empty".to_string()); } if path.len() > 512 { return Err("artifact path too long (max 512 chars)".to_string()); } if path.starts_with('/') { return Err("artifact path must be relative".to_string()); } if path.contains("..") { return Err("artifact path must not contain '..'".to_string()); } for (i, c) in path.chars().enumerate() { match c { 'a'..='z' | 'A'..='Z' | '0'..='9' => {} '-' | '_' | '.' | '/' | '{' | '}' | '+' => {} ';' | '&' | '|' | '$' | '`' | '(' | ')' | '<' | '>' | '!' | '\\' | '\'' | '"' | ' ' | '\n' | '\r' | '\0' => { return Err(format!( "character '{}' at position {} is not allowed in artifact path", c.escape_default(), i )); } _ => { return Err(format!( "unexpected character '{}' at position {} is not allowed in artifact path", c.escape_default(), i )); } } } Ok(()) } /// Escape a string for safe use in a shell command. fn shell_escape(s: &str) -> String { format!("'{}'", s.replace('\'', "'\\''")) } #[cfg(test)] mod tests { use super::*; #[test] fn the_update_hook_guards_the_namespace_validation_reserves() { // Bash cannot read a Rust constant, so the literal in the hook is a // copy. Renaming the reserved prefix without editing the hook would // leave the new one pushable and the old one locked, which is the // failure this pins: two doors, one policy. let reserved = crate::validation::RESERVED_NOTE_NAMESPACE; assert!( UPDATE_HOOK.contains(&format!("refs/notes/{reserved}|refs/notes/{reserved}/*")), "the update hook does not guard refs/notes/{reserved}/*:\n{UPDATE_HOOK}" ); // The bare prefix and the subtree are separate patterns in a glob, and // matching only the subtree would leave `refs/notes/mnw` itself open. assert!(UPDATE_HOOK.contains("exit 1"), "{UPDATE_HOOK}"); } #[tokio::test] async fn read_capped_truncates_to_cap() { // 10k bytes through a 4k cap retains exactly 4k (the rest is drained and // discarded so the child never blocks on a full pipe). let data = vec![b'x'; 10_000]; let out = read_capped(&data[..], 4096).await; assert_eq!(out.len(), 4096); } #[tokio::test] async fn read_capped_returns_all_when_under_cap() { let out = read_capped(&b"hello world"[..], 4096).await; assert_eq!(out, "hello world"); } #[test] fn build_failure_message_partial() { assert_eq!( build_failure_message(1, 2, Some("boom")), "partial build failure (1/3 targets succeeded)" ); assert_eq!( build_failure_message(2, 1, Some("boom")), "partial build failure (2/3 targets succeeded)" ); } #[test] fn build_failure_message_total_failure_uses_first_error() { assert_eq!(build_failure_message(0, 3, Some("ssh down")), "ssh down"); assert_eq!( build_failure_message(0, 0, None), "no targets produced artifacts" ); } #[test] fn rust_target_mapping() { assert_eq!( rust_target("linux", "x86_64"), Some("x86_64-unknown-linux-gnu") ); assert_eq!( rust_target("linux", "aarch64"), Some("aarch64-unknown-linux-gnu") ); assert_eq!(rust_target("darwin", "x86_64"), Some("x86_64-apple-darwin")); assert_eq!( rust_target("darwin", "aarch64"), Some("aarch64-apple-darwin") ); assert_eq!(rust_target("windows", "x86_64"), None); } #[test] fn hook_template_contains_hmac_not_raw_token() { let hook = post_receive_hook("secret-token-123", "alice", "myrepo"); let expected_hmac = repo_hmac("secret-token-123", "alice", "myrepo"); assert!( hook.contains(&expected_hmac), "hook should contain per-repo HMAC" ); assert!( !hook.contains("secret-token-123"), "hook must not contain raw token" ); assert!(!hook.contains("__HMAC__"), "placeholder should be replaced"); assert!(hook.contains("/api/internal/builds/trigger")); } /// The two notes arms answer different refs and must not be confused for /// each other: an inbox push is merged and answered synchronously, a notes /// push is only indexed. A `case` pattern that caught both would either /// merge a ref that is already the namespace or leave a push unindexed. #[test] fn the_hook_indexes_a_notes_push_and_merges_an_inbox_push() { let hook = post_receive_hook("secret-token-123", "alice", "myrepo"); assert!(hook.contains("/api/internal/notes/reindex")); assert!(hook.contains("/api/internal/notes/merge-inbox")); assert!(hook.contains("refs/notes/*)")); assert!(hook.contains("refs/mnw/notes-inbox/*)")); // The inbox lives under refs/mnw/, so nothing an inbox push does can // fall into the indexing arm. `notes_inbox` pins that prefix itself. assert!(!"refs/mnw/notes-inbox/commits".starts_with("refs/notes/")); } /// Fixed vector, duplicated in mnw-cli's `repo_hmac` test. mnw-cli installs /// hooks for repos it auto-creates over SSH, and this endpoint verifies /// them; if either side's derivation moves, both tests have to move /// together or those pushes stop triggering builds. #[test] fn repo_hmac_matches_mnw_cli_vector() { assert_eq!( repo_hmac("test-token", "max", "repo"), "198b4a4f542c0c27c4ca333030dfe09fe670aa44bae34d7a586481bb13fd9eb0" ); } #[test] fn repo_hmac_differs_per_repo() { let h1 = repo_hmac("token", "alice", "repo-a"); let h2 = repo_hmac("token", "alice", "repo-b"); assert_ne!(h1, h2, "different repos should produce different HMACs"); } #[test] fn shell_escape_basic() { assert_eq!(shell_escape("hello"), "'hello'"); assert_eq!(shell_escape("it's"), "'it'\\''s'"); } #[test] fn validate_build_command_accepts_safe_commands() { assert!( validate_build_command("cargo build --release --target x86_64-unknown-linux-gnu") .is_ok() ); assert!(validate_build_command("make -j4").is_ok()); assert!(validate_build_command("RUSTFLAGS=--cfg tokio_unstable cargo build").is_ok()); } #[test] fn validate_build_command_rejects_injection() { assert!(validate_build_command("cargo build; curl evil.com").is_err()); assert!(validate_build_command("cargo build && rm -rf /").is_err()); assert!(validate_build_command("cargo build | tee log").is_err()); assert!(validate_build_command("$(whoami)").is_err()); assert!(validate_build_command("`whoami`").is_err()); assert!(validate_build_command("cargo build > /dev/null").is_err()); assert!(validate_build_command("").is_err()); assert!( validate_build_command(" ").is_err(), "whitespace-only has no program" ); assert!( validate_build_command("FOO=bar").is_err(), "assignment with no program" ); } #[test] fn remote_command_parse_separates_env_program_args() { let c = RemoteCommand::parse("cargo build --release").unwrap(); assert!(c.assignments.is_empty()); assert_eq!(c.program, "cargo"); assert_eq!(c.args, vec!["build", "--release"]); let c = RemoteCommand::parse("RUSTFLAGS=--cfg CARGO_INCREMENTAL=0 cargo build").unwrap(); assert_eq!( c.assignments, vec!["RUSTFLAGS=--cfg", "CARGO_INCREMENTAL=0"] ); assert_eq!(c.program, "cargo"); assert_eq!(c.args, vec!["build"]); } #[test] fn remote_command_render_escapes_every_token() { // Plain command: each token individually single-quoted. let c = RemoteCommand::parse("cargo build --release").unwrap(); assert_eq!(c.render(), "'cargo' 'build' '--release'"); // Env prefix: applied via `env`, each element escaped. let c = RemoteCommand::parse("RUSTFLAGS=--cfg cargo build").unwrap(); assert_eq!(c.render(), "env 'RUSTFLAGS=--cfg' 'cargo' 'build'"); } #[test] fn is_env_assignment_recognizes_valid_identifiers_only() { assert!(is_env_assignment("FOO=bar")); assert!(is_env_assignment("_X1=y")); assert!(is_env_assignment("A=")); // empty value is a valid assignment assert!( !is_env_assignment("1FOO=bar"), "identifier can't start with a digit" ); assert!(!is_env_assignment("cargo"), "no '='"); assert!(!is_env_assignment("--target=x"), "not a shell identifier"); } #[test] fn render_defuses_would_be_injection_even_if_charset_bypassed() { // Construct a RemoteCommand directly with a hostile arg (bypassing the // token charset check) to prove render() is the real guard: the shell // sees a single quoted word, not a command separator. let c = RemoteCommand { assignments: vec![], program: "cargo".to_string(), args: vec!["build; rm -rf /".to_string()], }; assert_eq!(c.render(), "'cargo' 'build; rm -rf /'"); } #[test] fn validate_artifact_path_accepts_safe_paths() { assert!(validate_artifact_path("target/x86_64-unknown-linux-gnu/release/myapp").is_ok()); assert!(validate_artifact_path("dist/app-v0.1.0.tar.gz").is_ok()); } #[test] fn validate_artifact_path_rejects_unsafe() { assert!(validate_artifact_path("/etc/passwd").is_err()); assert!(validate_artifact_path("../../../etc/passwd").is_err()); assert!(validate_artifact_path("path with spaces").is_err()); assert!(validate_artifact_path("$(whoami)").is_err()); assert!(validate_artifact_path("").is_err()); } }