Skip to main content

max / makenotwork

11.8 KB · 280 lines History Blame Raw
1 use anyhow::Result;
2 use sando_daemon::{config, db, events, git, reconcile, routes, runs, state, sync, topology};
3 use std::net::SocketAddr;
4 use std::path::Path;
5 use std::sync::Arc;
6
7 /// CF2 fail-closed startup guard: refuse to start when the deploy mutators would
8 /// be exposed unauthenticated (no token) on a non-loopback bind. A loopback bind
9 /// without a token is allowed (local-only); any bind with a token is allowed.
10 fn refuse_unauthenticated_bind(addr: &SocketAddr, has_token: bool) -> bool {
11 !has_token && !addr.ip().is_loopback()
12 }
13
14 fn main() -> Result<()> {
15 // `--check-config`: prove THIS binary can load and parse its live daemon
16 // config + topology, then exit — no runtime, no DB, no network, no bind.
17 // This is the exact parse a stale self-update build once failed: a sandod
18 // missing the `node_health` Gate variant could not parse the node_health
19 // topology and crash-looped on every restart (postmortem 2026-07-09, #6).
20 // The self-update runs this against the live config *before* swapping the
21 // new binary in, so a binary that can't understand its config never gets
22 // installed. Pure by design: a green here means "this build understands its
23 // config", isolated from any environmental (DB/host) failure.
24 if std::env::args().skip(1).any(|a| a == "--check-config") {
25 return check_config();
26 }
27 run()
28 }
29
30 /// Load + validate the live config chain the daemon would boot against, without
31 /// starting anything. Mirrors the config/topology steps of `run` (below) so a
32 /// pass here guarantees `run` gets past them. Errors carry anyhow's context
33 /// chain and exit non-zero via `main`'s `Result`.
34 fn check_config() -> Result<()> {
35 let (_daemon, apps) = config::DaemonConfig::load()?;
36 // Every product, not just the first. A daemon that can parse one config and
37 // not another still crash-loops, and the whole point of this check is that a
38 // binary which cannot understand what it is about to run never gets
39 // installed.
40 for (id, cfg) in &apps {
41 let topo = validate_loaded(cfg)?;
42 println!(
43 "sandod --check-config: OK — app `{id}`, {} tier(s), topology {}",
44 topo.tiers.len(),
45 cfg.topology_path.display()
46 );
47 }
48 Ok(())
49 }
50
51 /// The parse + invariant work `check_config` does once the daemon config is in
52 /// hand. Split out so it can be exercised against a fixture config without
53 /// touching process env (`AppConfig::load` reads `SANDO_CONFIG`/CWD).
54 fn validate_loaded(cfg: &config::AppConfig) -> Result<topology::Topology> {
55 cfg.validate()?;
56 let topo = topology::Topology::load(&cfg.topology_path)?;
57 if let Some(h) = cfg.build_host.as_deref() {
58 topo.ensure_build_host_not_serving(h)?;
59 }
60 topo.ensure_migration_checks_have_backups(&cfg.migration_checks)?;
61 topo.ensure_node_companions_are_built(&cfg.companions)?;
62 topo.ensure_test_target_aux_repos_exist(&cfg.test_targets)?;
63 Ok(topo)
64 }
65
66 #[tokio::main]
67 async fn run() -> Result<()> {
68 tracing_subscriber::fmt()
69 // stdout is block-buffered under systemd (no TTY) so events never
70 // reach journald until the buffer fills or the process exits. stderr
71 // is line-buffered, which is what we want for a long-running service.
72 .with_writer(std::io::stderr)
73 .with_env_filter(
74 tracing_subscriber::EnvFilter::try_from_default_env()
75 // Modules live under the library crate `sando_daemon` (since
76 // the step-5 lib/bin split). `sandod` is kept for any
77 // top-level events that originate in main.rs itself.
78 .unwrap_or_else(|_| "sando_daemon=info,sandod=info,tower_http=info".into()),
79 )
80 .init();
81
82 let (daemon, app_cfgs) = config::DaemonConfig::load()?;
83 // Bring every product up before binding: its topology parsed, its
84 // cross-file invariants checked, its directories made, its bare repo
85 // present. A product that cannot start is a startup failure, not a surprise
86 // at its first /rebuild.
87 let mut apps: state::AppMap = std::collections::BTreeMap::new();
88 for (id, cfg) in app_cfgs {
89 let topo = Arc::new(topology::Topology::load(&cfg.topology_path)?);
90 if let Some(h) = cfg.build_host.as_deref() {
91 topo.ensure_build_host_not_serving(h)?;
92 }
93 topo.ensure_migration_checks_have_backups(&cfg.migration_checks)?;
94 topo.ensure_node_companions_are_built(&cfg.companions)?;
95 topo.ensure_test_target_aux_repos_exist(&cfg.test_targets)?;
96 tokio::fs::create_dir_all(&cfg.workdir).await?;
97 tokio::fs::create_dir_all(&cfg.release_root).await?;
98 // An intake-only product has no bare repo on this host to create.
99 if let Some(repo) = topo.repo.as_ref() {
100 git::ensure_bare_repo(Path::new(&repo.bare_path)).await?;
101 }
102 let executors = Arc::new(state::build_executors(&topo));
103 tracing::info!(
104 app = %id,
105 tiers = topo.tiers.len(),
106 bare = topo.repo.as_ref().map_or("(intake-only)", |r| r.bare_path.as_str()),
107 "app loaded"
108 );
109 apps.insert(
110 id,
111 Arc::new(state::App {
112 cfg,
113 topo,
114 executors,
115 }),
116 );
117 }
118 let apps = Arc::new(apps);
119 // The product a request that names none is about. First in declared order,
120 // which for a legacy config is the only one there is.
121 let default_app = apps
122 .keys()
123 .next()
124 .cloned()
125 .expect("DaemonConfig::load refuses an empty app set");
126 let default = apps[&default_app].clone();
127 let (cfg, topo) = (default.cfg.clone(), default.topo.clone());
128 let pool = db::connect(&daemon.db_path).await?;
129 db::migrate(&pool).await?;
130 // Reconcile any build_runs left 'building' by a previous daemon that died
131 // mid-build, so a restart can't leave a run wedged 'building' forever.
132 match runs::recover_orphaned_running(&pool).await {
133 Ok(0) => {}
134 Ok(n) => tracing::warn!(
135 reconciled = n,
136 "settled orphaned 'building' run(s) from a prior daemon"
137 ),
138 Err(e) => {
139 tracing::error!(error = %e, "failed to reconcile orphaned 'building' runs at startup");
140 }
141 }
142 for (id, app) in apps.iter() {
143 sync::sync(&pool, id, &app.topo).await?;
144 tracing::debug!(app = %id, "topology synced");
145 }
146
147 // Reconcile deploys against tier_state (after sync, so every tier_state row
148 // exists): catch a promote that landed on the nodes but died before advancing
149 // tier_state, so a stale-green tier on the wrong version is flagged partial
150 // instead of misleading /state and a later /rollback (audit
151 // sando-h5-crash-mid-promote).
152 match reconcile::recover_unrecorded_deploys(&pool).await {
153 Ok(0) => {}
154 Ok(n) => tracing::error!(
155 flagged = n,
156 "flagged tier(s) with an unrecorded deploy from a prior daemon; see /state partial_reason"
157 ),
158 Err(e) => {
159 tracing::error!(error = %e, "failed to reconcile deploys against tier_state at startup");
160 }
161 }
162
163 // Fail closed on the scratch cluster's privileges before any gate can hit
164 // them, rather than letting the first migration_dry_run discover it as an
165 // opaque "permission denied". Unset scratch_db_url is already a per-gate
166 // Blocked outcome, so there is nothing to assert.
167 if let Some(scratch_url) = cfg.scratch_db_url.as_deref() {
168 sando_daemon::gates::preflight_scratch_privileges(scratch_url).await?;
169 }
170
171 let addr: SocketAddr = daemon.listen.parse()?;
172
173 // Deploy-API auth (CF2). Token comes from SANDO_API_TOKEN (systemd
174 // EnvironmentFile). Refuse to expose the mutators unauthenticated on a
175 // non-loopback bind — that exact posture (tailnet bind, no auth) was the
176 // finding.
177 let api_token: Option<Arc<str>> = std::env::var("SANDO_API_TOKEN")
178 .ok()
179 .map(|s| s.trim().to_string())
180 .filter(|s| !s.is_empty())
181 .map(|s| Arc::from(s.as_str()));
182 if refuse_unauthenticated_bind(&addr, api_token.is_some()) {
183 anyhow::bail!(
184 "SANDO_API_TOKEN is unset but listen={addr} is not loopback; refusing to expose deploy \
185 controls unauthenticated. Set SANDO_API_TOKEN (via EnvironmentFile) or bind 127.0.0.1."
186 );
187 }
188 match &api_token {
189 Some(_) => tracing::info!("deploy endpoints require a bearer token"),
190 None => {
191 tracing::warn!(%addr, "SANDO_API_TOKEN unset; deploy endpoints are UNAUTHENTICATED (loopback bind)");
192 }
193 }
194
195 let executors = default.executors.clone();
196 let app_state = state::AppState {
197 pool,
198 apps,
199 default_app,
200 topo,
201 cfg,
202 active_build: Arc::new(tokio::sync::Mutex::new(None)),
203 deploy_lock: Arc::new(tokio::sync::Mutex::new(())),
204 events: events::channel(),
205 executors,
206 api_token,
207 };
208 let app = routes::router_for_apps(app_state);
209 tracing::info!(%addr, "sando daemon listening");
210 let listener = tokio::net::TcpListener::bind(addr).await?;
211 // `into_make_service_with_connect_info` surfaces the peer `SocketAddr` in
212 // request extensions so the bearer middleware can log who issued a mutation.
213 axum::serve(
214 listener,
215 app.into_make_service_with_connect_info::<SocketAddr>(),
216 )
217 .await?;
218 Ok(())
219 }
220
221 #[cfg(test)]
222 mod tests {
223 use super::{refuse_unauthenticated_bind, validate_loaded};
224 use std::net::SocketAddr;
225
226 fn addr(s: &str) -> SocketAddr {
227 s.parse().unwrap()
228 }
229
230 #[test]
231 fn check_config_accepts_the_shipped_topology() {
232 // The `--check-config` path a self-update runs before installing a new
233 // binary must pass against the topology this build ships with — the
234 // regression guard for the 2026-07-09 brick, where a binary that could
235 // not parse the live node_health config was installed and crash-looped.
236 // Both halves of the pair are the shipped ones. This used to hand-roll a
237 // minimal daemon config, which meant the production daemon config was
238 // never validated by anything and the cross-file checks had nothing real
239 // to check: the fixture declared no companions while the shipped
240 // topology installs two, so `ensure_node_companions_are_built` would
241 // have passed on a fixture and failed on the box.
242 //
243 // Only `topology_path` is overridden, because the shipped value is the
244 // installed location (/etc/sando/sando.toml) and the repo copy is what
245 // this build ships.
246 let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
247 let raw = std::fs::read_to_string(root.join("../deploy/sando-daemon.toml.example"))
248 .expect("reading the shipped production daemon config");
249 let mut cfg: super::config::AppConfig =
250 toml::from_str(&raw).expect("shipped daemon config must parse");
251 cfg.topology_path = root.join("../sando.toml");
252 validate_loaded(&cfg).expect("shipped config must pass --check-config");
253 }
254
255 #[test]
256 fn non_loopback_without_token_is_refused() {
257 // The exact CF2 posture: tailnet bind, no token.
258 assert!(refuse_unauthenticated_bind(
259 &addr("100.103.89.95:7766"),
260 false
261 ));
262 assert!(refuse_unauthenticated_bind(&addr("0.0.0.0:7766"), false));
263 }
264
265 #[test]
266 fn loopback_without_token_is_allowed() {
267 assert!(!refuse_unauthenticated_bind(&addr("127.0.0.1:7766"), false));
268 assert!(!refuse_unauthenticated_bind(&addr("[::1]:7766"), false));
269 }
270
271 #[test]
272 fn any_bind_with_token_is_allowed() {
273 assert!(!refuse_unauthenticated_bind(
274 &addr("100.103.89.95:7766"),
275 true
276 ));
277 assert!(!refuse_unauthenticated_bind(&addr("127.0.0.1:7766"), true));
278 }
279 }
280