Skip to main content

max / makenotwork

Kill the sando mutation survivors in platform and promote resolution The 11 survivors on astra's 2026-08-08 run were four functions nothing asserted against: Platform's charset and its two accessors, the backup rule keyed on migration_dry_run, and the per-node bundle resolution under a two-architecture promote and rollback. Ten are now killed by direct tests, each verified by hand-applying the recorded mutation and watching the new test fail. The eleventh, the += in the Placement arm of rollback_deployed_nodes, is an equivalent mutant: bundles_for_nodes can only hand that caller a matching pairing, so the arm is unreachable. It carries a comment saying so. Survivor counts were not re-measured; mutation runs stay off fw13.
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-08-23 22:29 UTC
Signed with PGP, not checked
Commit: 20deb52b06e38b8e926a1d9d7641c4c9c09916f0
Parent: 32a08ab
4 files changed, +285 insertions, -0 deletions
M sando/daemon/src/domain.rs +36
@@ -739,4 +739,40 @@
739 739 let id = GateRunId(42);
740 740 assert_eq!(serde_json::to_string(&id).unwrap(), "42");
741 741 }
742 +
743 + #[test]
744 + fn platform_components_take_the_punctuation_target_names_use() {
745 + // Real target names carry `_`, `-` and `.` inside a component, and the
746 + // charset here is the only thing that lets them through. A narrower
747 + // predicate rejects `unknown-linux-gnu` or `armv7.hf` while still
748 + // parsing `linux/aarch64`, which is why the shape test above cannot
749 + // see the difference.
750 + for good in [
751 + "linux_gnu/x86_64",
752 + "unknown-linux/aarch64",
753 + "linux/armv7.hf",
754 + "linux/x86_64",
755 + ] {
756 + assert!(Platform::parse(good).is_ok(), "{good:?} should parse");
757 + }
758 + for bad in [
759 + "linux+gnu/x86_64",
760 + "linux/x86 64",
761 + "linux/x86:64",
762 + "li nux/x",
763 + ] {
764 + assert!(Platform::parse(bad).is_err(), "{bad:?} should not parse");
765 + }
766 + }
767 +
768 + #[test]
769 + fn platform_halves_read_back_lowercased() {
770 + // `os()` and `arch()` are what a caller compares and what the SQL
771 + // lookup binds, so they have to be the normalized halves rather than
772 + // the spelling the config used.
773 + let p = Platform::parse("Linux/AArch64").unwrap();
774 + assert_eq!(p.os(), "linux");
775 + assert_eq!(p.arch(), "aarch64");
776 + assert_eq!(p.to_string(), "linux/aarch64");
777 + }
742 778 }
M sando/daemon/src/topology.rs +48
@@ -1083,6 +1083,54 @@
1083 1083 Ok(topo)
1084 1084 }
1085 1085
1086 + /// A topology with no `[backup]` at all, so the dump rules are exercised
1087 + /// by what its gates ask for rather than by what it declares.
1088 + fn topo_without_backup(gates: &str) -> Result<Topology> {
1089 + let raw = format!(
1090 + r#"
1091 + backup = []
1092 +
1093 + [repo]
1094 + bare_path = "/tmp/repo.git"
1095 + branch = "main"
1096 +
1097 + [[tier]]
1098 + name = "b"
1099 + provisioned = true
1100 + gates = [{gates}]
1101 + [[tier.node]]
1102 + name = "prod-1"
1103 + ssh_target = "prod-1"
1104 + release_root = "/srv/mnw"
1105 + "#
1106 + );
1107 + let topo: Topology = toml::from_str(&raw)?;
1108 + topo.validate_for_test()?;
1109 + Ok(topo)
1110 + }
1111 +
1112 + #[test]
1113 + fn a_product_that_never_dry_runs_migrations_owes_no_dump() {
1114 + // pom is this product: no postgres schema, so no tier configures
1115 + // migration_dry_run and demanding a prod dump would be demanding a
1116 + // fixture for a gate that never runs.
1117 + let topo = topo_without_backup(r#"{ kind = "node_health" }"#)
1118 + .expect("a topology with no migration gate loads without a [backup]");
1119 + assert!(topo.backup.is_empty());
1120 + }
1121 +
1122 + #[test]
1123 + fn a_migration_dry_run_with_no_backup_is_rejected_at_load() {
1124 + // The gate restores a dump into the scratch database. With nothing
1125 + // declared it would have nothing to restore, and the discovery would
1126 + // come mid-promote.
1127 + let err =
1128 + topo_without_backup(r#"{ kind = "node_health" }, { kind = "migration_dry_run" }"#)
1129 + .expect_err("a dry-run gate with no dump declared must not load")
1130 + .to_string();
1131 + assert!(err.contains("declares no [backup]"), "{err}");
1132 + }
1133 +
1086 1134 #[test]
1087 1135 fn a_single_backup_table_still_parses_as_one_named_server() {
1088 1136 // Back-compat is the point: every deployed sando.toml uses the single
M sando/daemon/src/routes/mod.rs +194
@@ -3951,4 +3951,198 @@
3951 3951 sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES (?, 'sha', datetime('now'), '/tmp/x') ON CONFLICT DO NOTHING")
3952 3952 .bind(version).execute(pool).await.unwrap();
3953 3953 }
3954 +
3955 + // ---- per-node bundle resolution (wiki release-artifact-identity) ----
3956 +
3957 + /// A settled build of `version` recorded as `platform`'s half of it.
3958 + async fn seed_platform_build(
3959 + pool: &SqlitePool,
3960 + sha: &str,
3961 + version: &str,
3962 + platform: &str,
3963 + staged_path: &str,
3964 + ) -> i64 {
3965 + sqlx::query_scalar(
3966 + "INSERT INTO build_runs (sha, version, phase, result, started_at, bundle_digest, staged_path, platform)
3967 + VALUES (?, ?, 'done', 'passed', datetime('now'), ?, ?, ?) RETURNING id",
3968 + )
3969 + .bind(sha)
3970 + .bind(version)
3971 + .bind(format!("{sha}-digest"))
3972 + .bind(staged_path)
3973 + .bind(platform)
3974 + .fetch_one(pool)
3975 + .await
3976 + .unwrap()
3977 + }
3978 +
3979 + fn node_on(name: &str, platform: Option<&str>) -> Node {
3980 + Node {
3981 + platform: platform.map(plat),
3982 + name: name.into(),
3983 + ssh_target: "local".into(),
3984 + release_root: format!("/tmp/{name}"),
3985 + service_name: "makenotwork.service".into(),
3986 + health_url: None,
3987 + config_check_env_file: None,
3988 + actuate: crate::topology::default_actuate(),
3989 + observe: crate::topology::default_observe(),
3990 + companions: Vec::new(),
3991 + }
3992 + }
3993 +
3994 + /// One pom version is two bundles with two digests. The node states which
3995 + /// architecture it can run, and the sibling bundle is resolved out of
3996 + /// `build_runs` rather than the caller's own half being shipped everywhere.
3997 + #[tokio::test]
3998 + async fn a_node_on_the_other_architecture_gets_its_own_bundle() {
3999 + let state = test_state().await;
4000 + seed_version(&state.pool, "5.0.0").await;
4001 + let arm = seed_platform_build(
4002 + &state.pool,
4003 + "sha-arm",
4004 + "5.0.0",
4005 + "linux/aarch64",
4006 + "/rel/aaaaaaaaaaaaaaaa",
4007 + )
4008 + .await;
4009 + let x86 = seed_platform_build(
4010 + &state.pool,
4011 + "sha-x86",
4012 + "5.0.0",
4013 + "linux/x86_64",
4014 + "/rel/bbbbbbbbbbbbbbbb",
4015 + )
4016 + .await;
4017 +
4018 + let n_arm = node_on("astra", Some("linux/aarch64"));
4019 + let n_x86 = node_on("hetzner", Some("linux/x86_64"));
4020 + let bundles = super::promotion::bundles_for_nodes(
4021 + &state,
4022 + "5.0.0",
4023 + &[&n_arm, &n_x86],
4024 + std::path::Path::new("/rel/aaaaaaaaaaaaaaaa"),
4025 + Some(&plat("linux/aarch64")),
4026 + Some(arm),
4027 + )
4028 + .await
4029 + .expect("both architectures have a green bundle at this version");
4030 +
4031 + assert_eq!(bundles.len(), 2);
4032 + assert_eq!(
4033 + bundles[0].1,
4034 + std::path::PathBuf::from("/rel/aaaaaaaaaaaaaaaa")
4035 + );
4036 + assert_eq!(bundles[0].2, Some(plat("linux/aarch64")));
4037 + assert_eq!(bundles[0].3, Some(arm));
4038 + // The one the caller never held: resolved by platform, and it carries
4039 + // the sibling's own build id so its own gate evidence is what gets
4040 + // checked.
4041 + assert_eq!(
4042 + bundles[1].1,
4043 + std::path::PathBuf::from("/rel/bbbbbbbbbbbbbbbb")
4044 + );
4045 + assert_eq!(bundles[1].2, Some(plat("linux/x86_64")));
4046 + assert_eq!(bundles[1].3, Some(x86));
4047 + }
4048 +
4049 + /// A version with no green bundle for the node's architecture fails the
4050 + /// whole resolution, before any node is touched.
4051 + #[tokio::test]
4052 + async fn a_missing_architecture_half_refuses_the_promote_rather_than_defaulting() {
4053 + let state = test_state().await;
4054 + seed_version(&state.pool, "5.0.0").await;
4055 + seed_platform_build(
4056 + &state.pool,
4057 + "sha-arm",
4058 + "5.0.0",
4059 + "linux/aarch64",
4060 + "/rel/aaaaaaaaaaaaaaaa",
4061 + )
4062 + .await;
4063 + let n_x86 = node_on("hetzner", Some("linux/x86_64"));
4064 + let err = super::promotion::bundles_for_nodes(
4065 + &state,
4066 + "5.0.0",
4067 + &[&n_x86],
4068 + std::path::Path::new("/rel/aaaaaaaaaaaaaaaa"),
4069 + Some(&plat("linux/aarch64")),
4070 + None,
4071 + )
4072 + .await
4073 + .expect_err("the x86_64 half was never built, so there is nothing to ship");
4074 + assert!(
4075 + format!("{err:?}").contains("no green linux/x86_64 bundle"),
4076 + "{err:?}"
4077 + );
4078 + }
4079 +
4080 + /// When the node and the caller name the same platform, the caller's bundle
4081 + /// is the answer and no lookup happens. A single-platform product whose
4082 + /// nodes have started stating a platform still has no `build_runs.platform`
4083 + /// row to find, so a lookup here would refuse a promote that is fine.
4084 + #[tokio::test]
4085 + async fn a_node_that_agrees_with_the_caller_takes_the_callers_bundle() {
4086 + let state = test_state().await;
4087 + seed_version(&state.pool, "5.0.0").await;
4088 + let node = node_on("hetzner", Some("linux/x86_64"));
4089 + let bundles = super::promotion::bundles_for_nodes(
4090 + &state,
4091 + "5.0.0",
4092 + &[&node],
4093 + std::path::Path::new("/rel/legacy"),
4094 + Some(&plat("linux/x86_64")),
4095 + None,
4096 + )
4097 + .await
4098 + .expect("the caller already holds the bundle this node wants");
4099 + assert_eq!(bundles[0].1, std::path::PathBuf::from("/rel/legacy"));
4100 + assert_eq!(bundles[0].2, Some(plat("linux/x86_64")));
4101 + assert_eq!(bundles[0].3, None);
4102 + }
4103 +
4104 + /// The previous version is two bundles too. If the one this node runs
4105 + /// cannot be resolved, nothing is attempted anywhere and every touched node
4106 + /// is indeterminate — which is the truth, not a default.
4107 + #[tokio::test]
4108 + async fn a_rollback_that_cannot_resolve_a_bundle_reports_every_node_indeterminate() {
4109 + let state = test_state().await;
4110 + sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('1.0.0','sha',datetime('now'),'/tmp/staged/releases/1.0.0/makenotwork')")
4111 + .execute(&state.pool).await.unwrap();
4112 + // No build_runs row for either node's architecture at 1.0.0.
4113 + let n1 = node_on("n1", Some("linux/aarch64"));
4114 + let n2 = node_on("n2", Some("linux/aarch64"));
4115 + let report = rollback_deployed_nodes(&state, &tid("a"), &[&n1, &n2], "1.0.0").await;
4116 + assert_eq!(report.restored, 0);
4117 + assert_eq!(
4118 + report.indeterminate, 2,
4119 + "one per node the rollback never reached: {report:?}"
4120 + );
4121 + assert!(!report.is_consistent());
4122 + }
4123 +
4124 + /// A rollback that fails at the symlink swap on one node of several leaves
4125 + /// that node unknown and the rest restored. The count has to be per node:
4126 + /// a tier reported wholesale indeterminate sends an operator to inspect
4127 + /// boxes that are fine, and a tier reported wholesale restored hides the
4128 + /// one that is not.
4129 + #[tokio::test]
4130 + async fn a_rollback_failing_on_one_node_counts_only_that_node_indeterminate() {
4131 + // The marker matches the swap-and-restart script, which is the only op
4132 + // annotated AtOrAfterSwap; a1's rollback therefore lands in the
4133 + // indeterminate arm rather than the already-on-previous one.
4134 + let (state, _log) = fleet_fixture(&["a1", "a2"], Some("a1"), "reload-or-restart").await;
4135 + let nodes: Vec<Node> = state.topo.tiers[1].nodes.clone();
4136 + let refs: Vec<&Node> = nodes.iter().collect();
4137 +
4138 + let report = rollback_deployed_nodes(&state, &tid("a"), &refs, "1.0.0").await;
4139 + assert_eq!(report.restored, 1, "a2 came back: {report:?}");
4140 + assert_eq!(
4141 + report.indeterminate, 1,
4142 + "only the node whose swap failed is unknown: {report:?}"
4143 + );
4144 + assert_eq!(report.already_on_previous, 0);
4145 + assert_eq!(report.touched(), 2);
4146 + assert!(!report.is_consistent());
4147 + }
3954 4148 }
M sando/daemon/src/routes/promotion.rs +7
@@ -872,6 +872,13 @@
872 872 .get(&node.name)
873 873 .cloned()
874 874 .unwrap_or_else(|| crate::state::build_executor(node));
875 + // Unreachable from here, and deliberately kept: `bundles_for_nodes`
876 + // hands a stating node its own platform and a silent node the silent
877 + // fallback (a rollback passes no fallback platform), so the pairing
878 + // always matches. It stays because `Placement` is the only way to get a
879 + // deployable bundle and the arm is what makes that hold if either side
880 + // ever starts resolving differently. A mutation survivor here is that
881 + // dead arm, not a coverage gap.
875 882 let placement =
876 883 match crate::deploy::Placement::check(node, node_bundle, node_bundle_platform.as_ref())
877 884 {