Skip to main content

max / makenotwork

sando: test deploy_remote choreography and multi-node canary promote Add a FakeExec test double for the executor trait so the remote deploy path becomes testable without a reachable host. deploy.rs: cover deploy_remote's step choreography (mkdir, rsync, arch guard, config-drift check, swap+restart, companion install, gc) including ordering, fail-closed-before-swap on rsync/arch-guard failure, and the config-check opt-out. routes: a FleetFake records deploy ops across the fleet, covering sequential-canary multi-node promote (order + advance) and a mid-canary node failure that rolls the touched nodes back to the prior version and leaves tier_state unadvanced. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-23 02:19 UTC
Commit: 8e945597018e28f06320cee9380fa8c3b3d445db
Parent: 22e1584
2 files changed, +531 insertions, -1 deletion
M sando/daemon/src/deploy.rs +238
@@ -427,7 +427,10 @@ async fn gc_remote_releases(executor: &dyn Executor, release_root: &str) -> Resu
427 427 #[cfg(test)]
428 428 mod tests {
429 429 use super::*;
430 + use crate::topology::NodeCompanion;
430 431 use ops_exec::{CapabilitySet, LocalExec, SshExec};
432 + use std::os::unix::process::ExitStatusExt;
433 + use std::sync::{Arc, Mutex as StdMutex};
431 434 use std::time::SystemTime;
432 435
433 436 /// A LocalExec granted the default node capabilities (deploy + restart).
@@ -1000,4 +1003,239 @@ mod tests {
1000 1003 "expected capability denial"
1001 1004 );
1002 1005 }
1006 +
1007 + // ---- FakeExec: the deploy_remote choreography without a real host ----
1008 + //
1009 + // deploy_node's local fast-path is covered above with a real LocalExec, but
1010 + // the remote path (rsync + arch guard + config-drift + swap + companions +
1011 + // gc) short-circuits on `ssh_target != "local"` and so never ran under test
1012 + // without a reachable node. FakeExec records every executor call in order
1013 + // and can be told to fail one shell step (matched by substring) or the rsync
1014 + // push, so the ordering and the fail-closed-before-swap contract are
1015 + // assertable in-process.
1016 +
1017 + struct FakeExec {
1018 + caps: CapabilitySet,
1019 + calls: Arc<StdMutex<Vec<String>>>,
1020 + /// The first `run_streaming` whose script contains this substring exits
1021 + /// non-zero (a failed shell step), e.g. the arch guard.
1022 + fail_run_matching: Option<String>,
1023 + /// `push_dir` (the rsync) returns an error.
1024 + fail_push_dir: bool,
1025 + }
1026 +
1027 + impl FakeExec {
1028 + fn new() -> Self {
1029 + Self {
1030 + caps: CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
1031 + calls: Arc::new(StdMutex::new(Vec::new())),
1032 + fail_run_matching: None,
1033 + fail_push_dir: false,
1034 + }
1035 + }
1036 + fn log(&self) -> Vec<String> {
1037 + self.calls.lock().unwrap().clone()
1038 + }
1039 + }
1040 +
1041 + #[async_trait]
1042 + impl Executor for FakeExec {
1043 + async fn run_streaming(&self, step: &Step, _sink: &mut dyn LogSink) -> Result<RunOutput> {
1044 + // Every deploy step is a `Step::shell`, so the script is argv's tail.
1045 + let script = step.argv.last().cloned().unwrap_or_default();
1046 + self.calls.lock().unwrap().push(format!("run:{script}"));
1047 + let fail = self
1048 + .fail_run_matching
1049 + .as_deref()
1050 + .is_some_and(|m| script.contains(m));
1051 + Ok(RunOutput {
1052 + status: std::process::ExitStatus::from_raw(if fail { 1 << 8 } else { 0 }),
1053 + stdout: Vec::new(),
1054 + stderr: if fail {
1055 + b"fake step failure".to_vec()
1056 + } else {
1057 + Vec::new()
1058 + },
1059 + })
1060 + }
1061 + async fn pull_file(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1062 + self.calls.lock().unwrap().push("pull_file".into());
1063 + Ok(())
1064 + }
1065 + async fn pull_dir(&self, _remote: &Path, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1066 + self.calls.lock().unwrap().push("pull_dir".into());
1067 + Ok(())
1068 + }
1069 + async fn pull_glob(&self, _glob: &str, _local: &Path, _opts: &SyncOpts) -> Result<()> {
1070 + self.calls.lock().unwrap().push("pull_glob".into());
1071 + Ok(())
1072 + }
1073 + async fn push_dir(&self, _local: &Path, remote: &Path, _opts: &SyncOpts) -> Result<()> {
1074 + self.calls
1075 + .lock()
1076 + .unwrap()
1077 + .push(format!("push_dir:{}", remote.display()));
1078 + if self.fail_push_dir {
1079 + anyhow::bail!("fake rsync failure");
1080 + }
1081 + Ok(())
1082 + }
1083 + fn capabilities(&self) -> &CapabilitySet {
1084 + &self.caps
1085 + }
1086 + }
1087 +
1088 + fn remote_node(config_check: bool, companions: Vec<NodeCompanion>) -> Node {
1089 + Node {
1090 + name: "web-a".into(),
1091 + ssh_target: "deploy@web-a".into(),
1092 + release_root: "/opt/mnw".into(),
1093 + service_name: "makenotwork.service".into(),
1094 + health_url: None,
1095 + config_check_env_file: config_check.then(|| "/etc/mnw/node.env".to_string()),
1096 + actuate: crate::topology::default_actuate(),
1097 + observe: crate::topology::default_observe(),
1098 + companions,
1099 + }
1100 + }
1101 +
1102 + fn companion() -> NodeCompanion {
1103 + NodeCompanion {
1104 + name: "mnw-cli".into(),
1105 + install_path: "/opt/mnw-cli/mnw-cli".into(),
1106 + service_name: "mnw-cli.service".into(),
1107 + }
1108 + }
1109 +
1110 + /// Index of the first recorded call whose text contains `needle` (panics if
1111 + /// absent — the assertion message names what was missing).
1112 + fn pos(log: &[String], needle: &str) -> usize {
1113 + log.iter()
1114 + .position(|c| c.contains(needle))
1115 + .unwrap_or_else(|| panic!("no call matched {needle:?} in {log:#?}"))
1116 + }
1117 +
1118 + #[tokio::test]
1119 + async fn deploy_remote_runs_the_full_choreography_in_order() {
1120 + // A node opted into the config-drift check and carrying one companion:
1121 + // mkdir -> rsync -> arch guard -> config check -> swap+restart ->
1122 + // companion install -> gc, in that order.
1123 + let tmp = tempfile::tempdir().unwrap();
1124 + let staged = tmp.path().join("releases").join("0.9.0");
1125 + tokio::fs::create_dir_all(&staged).await.unwrap();
1126 +
1127 + let node = remote_node(true, vec![companion()]);
1128 + let exec = FakeExec::new();
1129 + let out = deploy_node(&exec, &node, "0.9.0", &staged, "makenotwork")
1130 + .await
1131 + .expect("deploy_remote should succeed against the fake");
1132 + assert_eq!(out, PathBuf::from("/opt/mnw/releases/0.9.0"));
1133 +
1134 + let log = exec.log();
1135 + let mkdir = pos(&log, "mkdir -p");
1136 + let rsync = pos(&log, "push_dir:/opt/mnw/releases/0.9.0");
1137 + let arch = pos(&log, "e_machine");
1138 + let cfg = pos(&log, "MNW_CHECK_CONFIG=1");
1139 + let swap = pos(&log, "reload-or-restart");
1140 + let comp = pos(&log, "install-companion.sh");
1141 + let gc = pos(&log, "ls -1t");
1142 + assert!(
1143 + mkdir < rsync && rsync < arch && arch < cfg && cfg < swap && swap < comp && comp < gc,
1144 + "deploy steps out of order: {log:#?}"
1145 + );
1146 + }
1147 +
1148 + #[tokio::test]
1149 + async fn deploy_remote_aborts_before_swap_when_rsync_fails() {
1150 + // The rsync failing must fail the deploy BEFORE the symlink swap — the
1151 + // "current symlink left intact" contract. Assert the swap never ran.
1152 + let tmp = tempfile::tempdir().unwrap();
1153 + let staged = tmp.path().join("releases").join("0.9.0");
1154 + tokio::fs::create_dir_all(&staged).await.unwrap();
1155 +
1156 + let node = remote_node(false, Vec::new());
1157 + let mut exec = FakeExec::new();
1158 + exec.fail_push_dir = true;
1159 + let err = deploy_node(&exec, &node, "0.9.0", &staged, "makenotwork")
1160 + .await
1161 + .expect_err("rsync failure must fail the deploy");
1162 + assert!(
1163 + format!("{err:#}").contains("rsync"),
1164 + "error should attribute the rsync: {err:#}"
1165 + );
1166 + let log = exec.log();
1167 + assert!(
1168 + !log.iter().any(|c| c.contains("reload-or-restart")),
1169 + "swap must not run after a failed rsync: {log:#?}"
1170 + );
1171 + }
1172 +
1173 + #[tokio::test]
1174 + async fn deploy_remote_aborts_before_swap_when_arch_guard_fails() {
1175 + // A wrong-arch binary must fail closed before the swap. The fake fails
1176 + // the arch-guard shell step; the swap must not follow.
1177 + let tmp = tempfile::tempdir().unwrap();
1178 + let staged = tmp.path().join("releases").join("0.9.0");
1179 + tokio::fs::create_dir_all(&staged).await.unwrap();
1180 +
1181 + let node = remote_node(false, Vec::new());
1182 + let mut exec = FakeExec::new();
1183 + exec.fail_run_matching = Some("e_machine".into());
1184 + let err = deploy_node(&exec, &node, "0.9.0", &staged, "makenotwork")
1185 + .await
1186 + .expect_err("arch mismatch must fail the deploy");
1187 + assert!(
1188 + format!("{err:#}").contains("architecture"),
1189 + "error should mention the arch check: {err:#}"
1190 + );
1191 + let log = exec.log();
1192 + assert!(
1193 + !log.iter().any(|c| c.contains("reload-or-restart")),
1194 + "swap must not run after a failed arch guard: {log:#?}"
1195 + );
1196 + }
1197 +
1198 + #[tokio::test]
1199 + async fn deploy_remote_skips_config_check_when_node_opts_out() {
1200 + // No config_check_env_file => the pre-swap config check is skipped, but
1201 + // the rest of the choreography (including the swap) still runs.
1202 + let tmp = tempfile::tempdir().unwrap();
1203 + let staged = tmp.path().join("releases").join("0.9.0");
1204 + tokio::fs::create_dir_all(&staged).await.unwrap();
1205 +
1206 + let node = remote_node(false, Vec::new());
1207 + let exec = FakeExec::new();
1208 + deploy_node(&exec, &node, "0.9.0", &staged, "makenotwork")
1209 + .await
1210 + .unwrap();
1211 + let log = exec.log();
1212 + assert!(
1213 + !log.iter().any(|c| c.contains("MNW_CHECK_CONFIG=1")),
1214 + "config check must be skipped when the node opts out: {log:#?}"
1215 + );
1216 + assert!(
1217 + log.iter().any(|c| c.contains("reload-or-restart")),
1218 + "the swap must still run: {log:#?}"
1219 + );
1220 + }
1221 +
1222 + #[tokio::test]
1223 + async fn deploy_remote_installs_companion_after_the_swap() {
1224 + // Companions are After= the server: their install must land after the
1225 + // symlink swap + service restart, never before.
1226 + let tmp = tempfile::tempdir().unwrap();
1227 + let staged = tmp.path().join("releases").join("0.9.0");
1228 + tokio::fs::create_dir_all(&staged).await.unwrap();
1229 +
1230 + let node = remote_node(false, vec![companion()]);
1231 + let exec = FakeExec::new();
1232 + deploy_node(&exec, &node, "0.9.0", &staged, "makenotwork")
1233 + .await
1234 + .unwrap();
1235 + let log = exec.log();
1236 + assert!(
1237 + pos(&log, "reload-or-restart") < pos(&log, "install-companion.sh"),
1238 + "companion install must follow the swap: {log:#?}"
1239 + );
1240 + }
1003 1241 }
M sando/daemon/src/routes/mod.rs +293 -1
@@ -837,13 +837,16 @@ mod tests {
837 837 use super::*;
838 838 use crate::config::Config;
839 839 use crate::topology::{BackupConfig, CanaryPolicy, Gate, Node, RepoConfig, Tier, Topology};
840 + use async_trait::async_trait;
840 841 use axum::body::Body;
841 842 use axum::http::{Request, StatusCode};
842 843 use http_body_util::BodyExt;
844 + use ops_exec::{CapabilitySet, Executor, LogSink, RunOutput, Step, SyncOpts};
843 845 use sqlx::SqlitePool;
844 846 use sqlx::sqlite::SqlitePoolOptions;
847 + use std::os::unix::process::ExitStatusExt;
845 848 use std::path::PathBuf;
846 - use std::sync::Arc;
849 + use std::sync::{Arc, Mutex as StdMutex};
847 850 use tower::ServiceExt;
848 851
849 852 async fn fresh_pool() -> SqlitePool {
@@ -1662,6 +1665,295 @@ mod tests {
1662 1665 );
1663 1666 }
1664 1667
1668 + // ---- FleetFake: a multi-node promote across recorded fake executors ----
1669 + //
1670 + // The route-level promote tests until now ran a single local node against a
1671 + // real LocalExec, so the sequential-canary fan-out, the cross-node deploy
1672 + // ordering, and the mid-canary rollback of already-flipped nodes had no
1673 + // coverage. FleetFake records every deploy op (tagged by node) into one
1674 + // shared log so ordering is visible across the fleet, and can fail any op
1675 + // whose shell script or rsync target contains a marker — used to fail a
1676 + // node's forward deploy of the new version while its rollback to the prior
1677 + // version (a different `releases/<v>` path) still succeeds.
1678 +
1679 + struct FleetFake {
1680 + tag: String,
1681 + caps: CapabilitySet,
1682 + log: Arc<StdMutex<Vec<String>>>,
1683 + fail_if_contains: Option<String>,
1684 + }
1685 +
1686 + impl FleetFake {
1687 + fn fails(&self, text: &str) -> bool {
1688 + self.fail_if_contains
1689 + .as_deref()
1690 + .is_some_and(|m| text.contains(m))
1691 + }
1692 + }
1693 +
1694 + #[async_trait]
1695 + impl Executor for FleetFake {
1696 + async fn run_streaming(
1697 + &self,
1698 + step: &Step,
1699 + _sink: &mut dyn LogSink,
1700 + ) -> anyhow::Result<RunOutput> {
1701 + let script = step.argv.last().cloned().unwrap_or_default();
1702 + self.log
1703 + .lock()
1704 + .unwrap()
1705 + .push(format!("{}:run:{script}", self.tag));
1706 + Ok(RunOutput {
1707 + status: std::process::ExitStatus::from_raw(if self.fails(&script) {
1708 + 1 << 8
1709 + } else {
1710 + 0
1711 + }),
1712 + stdout: Vec::new(),
1713 + stderr: Vec::new(),
1714 + })
1715 + }
1716 + async fn pull_file(
1717 + &self,
1718 + _r: &std::path::Path,
1719 + _l: &std::path::Path,
1720 + _o: &SyncOpts,
1721 + ) -> anyhow::Result<()> {
1722 + Ok(())
1723 + }
1724 + async fn pull_dir(
1725 + &self,
1726 + _r: &std::path::Path,
1727 + _l: &std::path::Path,
1728 + _o: &SyncOpts,
1729 + ) -> anyhow::Result<()> {
1730 + Ok(())
1731 + }
1732 + async fn pull_glob(
1733 + &self,
1734 + _g: &str,
1735 + _l: &std::path::Path,
1736 + _o: &SyncOpts,
1737 + ) -> anyhow::Result<()> {
1738 + Ok(())
1739 + }
1740 + async fn push_dir(
1741 + &self,
1742 + _local: &std::path::Path,
1743 + remote: &std::path::Path,
1744 + _o: &SyncOpts,
1745 + ) -> anyhow::Result<()> {
1746 + let dst = remote.display().to_string();
1747 + self.log
1748 + .lock()
1749 + .unwrap()
1750 + .push(format!("{}:push:{dst}", self.tag));
1751 + if self.fails(&dst) {
1752 + anyhow::bail!("fake rsync failure on {}", self.tag);
1753 + }
1754 + Ok(())
1755 + }
1756 + fn capabilities(&self) -> &CapabilitySet {
1757 + &self.caps
1758 + }
1759 + }
1760 +
1761 + /// Rebuild tier "a" with `names` as remote fake nodes sharing one op log,
1762 + /// seed the version/tier_state prerequisites for a promote of 3.0.0 up from
1763 + /// `host` (tier "a" starts on 2.0.0 with 1.0.0 behind it), and optionally
1764 + /// make `fail_node` fail any op containing `fail_marker`. Returns the state
1765 + /// and the shared log.
1766 + async fn fleet_fixture(
1767 + names: &[&str],
1768 + fail_node: Option<&str>,
1769 + fail_marker: &str,
1770 + ) -> (AppState, Arc<StdMutex<Vec<String>>>) {
1771 + use crate::topology::{default_actuate, default_observe};
1772 + let mut state = test_state().await;
1773 + let log = Arc::new(StdMutex::new(Vec::<String>::new()));
1774 +
1775 + let nodes: Vec<Node> = names
1776 + .iter()
1777 + .map(|name| Node {
1778 + name: (*name).into(),
1779 + ssh_target: format!("deploy@{name}"),
1780 + release_root: format!("/tmp/fleet/{name}"),
1781 + service_name: "makenotwork.service".into(),
1782 + health_url: None,
1783 + config_check_env_file: None,
1784 + actuate: default_actuate(),
1785 + observe: default_observe(),
1786 + companions: Vec::new(),
1787 + })
1788 + .collect();
1789 +
1790 + let mut topo = (*state.topo).clone();
1791 + topo.tiers[1].nodes = nodes.clone();
1792 + // Drop the tier's post-deploy gate: the deploy fan-out is the subject
1793 + // here, and node_health would need its own probe wiring.
1794 + topo.tiers[1].gates = vec![];
1795 + state.topo = Arc::new(topo);
1796 +
1797 + let execs: crate::state::ExecutorMap = nodes
1798 + .iter()
1799 + .map(|n| {
1800 + let nm = n.name.to_string();
1801 + let fail = fail_node == Some(nm.as_str());
1802 + let exec: Arc<dyn Executor> = Arc::new(FleetFake {
1803 + tag: nm,
1804 + caps: CapabilitySet::from_tokens(["deploy", "restart"], ["health"]),
1805 + log: log.clone(),
1806 + fail_if_contains: fail.then(|| fail_marker.to_string()),
1807 + });
1808 + (n.name.clone(), exec)
1809 + })
1810 + .collect();
1811 + state.executors = Arc::new(execs);
1812 +
1813 + for n in &nodes {
1814 + // deploys.node FKs into `nodes`.
1815 + sqlx::query(
1816 + "INSERT INTO nodes (name, tier, ssh_target, release_root) VALUES (?, 'a', ?, ?)",
1817 + )
1818 + .bind(&n.name)
1819 + .bind(&n.ssh_target)
1820 + .bind(&n.release_root)
1821 + .execute(&state.pool)
1822 + .await
1823 + .unwrap();
1824 + }
1825 + for v in ["1.0.0", "2.0.0", "3.0.0"] {
1826 + sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES (?, 'sha', datetime('now'), '/tmp/staged/makenotwork')")
1827 + .bind(v).execute(&state.pool).await.unwrap();
1828 + }
1829 + sqlx::query("UPDATE tier_state SET current_version = '3.0.0' WHERE tier = 'host'")
1830 + .execute(&state.pool)
1831 + .await
1832 + .unwrap();
1833 + sqlx::query(
1834 + "UPDATE tier_state SET current_version = '2.0.0', previous_version = '1.0.0' WHERE tier = 'a'",
1835 + )
1836 + .execute(&state.pool)
1837 + .await
1838 + .unwrap();
1839 + (state, log)
1840 + }
1841 +
1842 + /// Index of the first logged op belonging to `node` (panics if the node was
1843 + /// never touched — the message names it).
1844 + fn first_touch(log: &[String], node: &str) -> usize {
1845 + let prefix = format!("{node}:");
1846 + log.iter()
1847 + .position(|e| e.starts_with(&prefix))
1848 + .unwrap_or_else(|| panic!("node {node:?} was never deployed to; log: {log:#?}"))
1849 + }
1850 +
1851 + #[tokio::test]
1852 + async fn promote_deploys_every_node_in_tier_order_and_advances() {
1853 + let (state, log) = fleet_fixture(&["a1", "a2", "a3"], None, "").await;
1854 + let pool = state.pool.clone();
1855 +
1856 + let Json(body) = promote_inner(state, "a".into(), PromoteBody::default())
1857 + .await
1858 + .expect("all nodes deploy, so the promote succeeds");
1859 + assert_eq!(
1860 + body["nodes_deployed"],
1861 + serde_json::json!(["a1", "a2", "a3"]),
1862 + "the response names every node the promote reached",
1863 + );
1864 +
1865 + let (cur, prev) = tier_versions(&pool, "a").await;
1866 + assert_eq!(cur.as_deref(), Some("3.0.0"));
1867 + assert_eq!(prev.as_deref(), Some("2.0.0"));
1868 +
1869 + // Sequential canary: a1 is fully touched before a2, a2 before a3.
1870 + let log = log.lock().unwrap().clone();
1871 + assert!(
1872 + first_touch(&log, "a1") < first_touch(&log, "a2")
1873 + && first_touch(&log, "a2") < first_touch(&log, "a3"),
1874 + "nodes must deploy in tier order: {log:#?}",
1875 + );
1876 +
1877 + // Every node has a green deploy row for the promoted version.
1878 + let ok: i64 = sqlx::query_scalar(
1879 + "SELECT COUNT(*) FROM deploys WHERE version = '3.0.0' AND outcome = 'ok'",
1880 + )
1881 + .fetch_one(&pool)
1882 + .await
1883 + .unwrap();
1884 + assert_eq!(ok, 3, "one ok deploy row per node");
1885 + }
1886 +
1887 + #[tokio::test]
1888 + async fn a_mid_canary_deploy_failure_rolls_touched_nodes_back_and_does_not_advance() {
1889 + // a2 fails its forward deploy of 3.0.0; a1 was already flipped, a3 is
1890 + // never reached. The touched nodes (a1, a2) roll back to 2.0.0 — their
1891 + // rollback ops target `releases/2.0.0`, which the marker does not match —
1892 + // and tier_state must NOT advance.
1893 + let (state, log) = fleet_fixture(&["a1", "a2", "a3"], Some("a2"), "releases/3.0.0").await;
1894 + let pool = state.pool.clone();
1895 +
1896 + let err = promote_inner(state, "a".into(), PromoteBody::default())
1897 + .await
1898 + .expect_err("a mid-canary deploy failure must fail the promote");
1899 + assert!(
1900 + matches!(err, crate::error::Error::Other(_)),
1901 + "a deploy failure propagates as Other, got: {err:?}",
1902 + );
1903 +
1904 + // tier_state untouched: the failure returns before advance_tier.
1905 + let (cur, prev) = tier_versions(&pool, "a").await;
1906 + assert_eq!(
1907 + cur.as_deref(),
1908 + Some("2.0.0"),
1909 + "a failed rollout must not advance"
1910 + );
1911 + assert_eq!(prev.as_deref(), Some("1.0.0"));
1912 +
1913 + let log = log.lock().unwrap().clone();
1914 + // a3 sits after the failed a2 in the sequence and is never touched.
1915 + assert!(
1916 + !log.iter().any(|e| e.starts_with("a3:")),
1917 + "nodes after the failure must not be deployed to: {log:#?}",
1918 + );
1919 + // Both touched nodes were rolled back to the prior version.
1920 + for n in ["a1", "a2"] {
1921 + assert!(
1922 + log.iter()
1923 + .any(|e| e.starts_with(&format!("{n}:")) && e.contains("releases/2.0.0")),
1924 + "touched node {n} must be restored to 2.0.0: {log:#?}",
1925 + );
1926 + }
1927 +
1928 + // The forward attempt is on the record: a1 ok, a2 failed.
1929 + let a1: String = sqlx::query_scalar(
1930 + "SELECT outcome FROM deploys WHERE version = '3.0.0' AND node = 'a1'",
1931 + )
1932 + .fetch_one(&pool)
1933 + .await
1934 + .unwrap();
1935 + assert_eq!(a1, "ok");
1936 + let a2: String = sqlx::query_scalar(
1937 + "SELECT outcome FROM deploys WHERE version = '3.0.0' AND node = 'a2'",
1938 + )
1939 + .fetch_one(&pool)
1940 + .await
1941 + .unwrap();
1942 + assert_eq!(a2, "failed");
1943 +
1944 + // Both touched nodes restored => the tier is consistent on 2.0.0, so the
1945 + // partial flag is cleared, not set.
1946 + let reason: Option<String> =
1947 + sqlx::query_scalar("SELECT partial_reason FROM tier_state WHERE tier = 'a'")
1948 + .fetch_one(&pool)
1949 + .await
1950 + .unwrap();
1951 + assert_eq!(
1952 + reason, None,
1953 + "a fully-restored canary leaves the tier consistent, not partial",
1954 + );
1955 + }
1956 +
1665 1957 /// Build a one-node tier "a" on a tempdir release root, pre-seeded as if a
1666 1958 /// promote had flipped it to `current` with `prev` still staged on disk.
1667 1959 /// Returns the state (topology rewired to the tempdir node) and the tempdir,