Skip to main content

max / makenotwork

13.3 KB · 311 lines History Blame Raw
1 use anyhow::Result;
2 use sando_daemon::{
3 config, db, events, git, reconcile, retention, routes, runs, state, sync, topology,
4 };
5 use std::net::SocketAddr;
6 use std::path::Path;
7 use std::sync::Arc;
8
9 /// CF2 fail-closed startup guard: refuse to start when the deploy mutators would
10 /// be exposed unauthenticated (no token) on a non-loopback bind. A loopback bind
11 /// without a token is allowed (local-only); any bind with a token is allowed.
12 fn refuse_unauthenticated_bind(addr: &SocketAddr, has_token: bool) -> bool {
13 !has_token && !addr.ip().is_loopback()
14 }
15
16 fn main() -> Result<()> {
17 // `--check-config`: prove THIS binary can load and parse its live daemon
18 // config + topology, then exit — no runtime, no DB, no network, no bind.
19 // This is the exact parse a stale self-update build once failed: a sandod
20 // missing the `node_health` Gate variant could not parse the node_health
21 // topology and crash-looped on every restart (postmortem 2026-07-09, #6).
22 // The self-update runs this against the live config *before* swapping the
23 // new binary in, so a binary that can't understand its config never gets
24 // installed. Pure by design: a green here means "this build understands its
25 // config", isolated from any environmental (DB/host) failure.
26 if std::env::args().skip(1).any(|a| a == "--check-config") {
27 return check_config();
28 }
29 run()
30 }
31
32 /// Load + validate the live config chain the daemon would boot against, without
33 /// starting anything. Mirrors the config/topology steps of `run` (below) so a
34 /// pass here guarantees `run` gets past them. Errors carry anyhow's context
35 /// chain and exit non-zero via `main`'s `Result`.
36 fn check_config() -> Result<()> {
37 let (_daemon, apps) = config::DaemonConfig::load()?;
38 // Every product, not just the first. A daemon that can parse one config and
39 // not another still crash-loops, and the whole point of this check is that a
40 // binary which cannot understand what it is about to run never gets
41 // installed.
42 for (id, cfg) in &apps {
43 let topo = validate_loaded(cfg)?;
44 println!(
45 "sandod --check-config: OK — app `{id}`, {} tier(s), topology {}",
46 topo.tiers.len(),
47 cfg.topology_path.display()
48 );
49 }
50 Ok(())
51 }
52
53 /// The parse + invariant work `check_config` does once the daemon config is in
54 /// hand. Split out so it can be exercised against a fixture config without
55 /// touching process env (`AppConfig::load` reads `SANDO_CONFIG`/CWD).
56 fn validate_loaded(cfg: &config::AppConfig) -> Result<topology::Topology> {
57 cfg.validate()?;
58 let topo = topology::Topology::load(&cfg.topology_path)?;
59 if let Some(h) = cfg.build_host.as_deref() {
60 topo.ensure_build_host_not_serving(h)?;
61 }
62 topo.ensure_migration_checks_have_backups(&cfg.migration_checks)?;
63 topo.ensure_node_companions_are_built(&cfg.companions)?;
64 topo.ensure_test_target_aux_repos_exist(&cfg.test_targets)?;
65 Ok(topo)
66 }
67
68 #[tokio::main]
69 async fn run() -> Result<()> {
70 tracing_subscriber::fmt()
71 // stdout is block-buffered under systemd (no TTY) so events never
72 // reach journald until the buffer fills or the process exits. stderr
73 // is line-buffered, which is what we want for a long-running service.
74 .with_writer(std::io::stderr)
75 .with_env_filter(
76 tracing_subscriber::EnvFilter::try_from_default_env()
77 // Modules live under the library crate `sando_daemon` (since
78 // the step-5 lib/bin split). `sandod` is kept for any
79 // top-level events that originate in main.rs itself.
80 .unwrap_or_else(|_| "sando_daemon=info,sandod=info,tower_http=info".into()),
81 )
82 .init();
83
84 let (daemon, app_cfgs) = config::DaemonConfig::load()?;
85 // Bring every product up before binding: its topology parsed, its
86 // cross-file invariants checked, its directories made, its bare repo
87 // present. A product that cannot start is a startup failure, not a surprise
88 // at its first /rebuild.
89 let mut apps: state::AppMap = std::collections::BTreeMap::new();
90 for (id, cfg) in app_cfgs {
91 let topo = Arc::new(topology::Topology::load(&cfg.topology_path)?);
92 if let Some(h) = cfg.build_host.as_deref() {
93 topo.ensure_build_host_not_serving(h)?;
94 }
95 topo.ensure_migration_checks_have_backups(&cfg.migration_checks)?;
96 topo.ensure_node_companions_are_built(&cfg.companions)?;
97 topo.ensure_test_target_aux_repos_exist(&cfg.test_targets)?;
98 tokio::fs::create_dir_all(&cfg.workdir).await?;
99 tokio::fs::create_dir_all(&cfg.release_root).await?;
100 // An intake-only product has no bare repo on this host to create.
101 if let Some(repo) = topo.repo.as_ref() {
102 git::ensure_bare_repo(Path::new(&repo.bare_path)).await?;
103 }
104 let executors = Arc::new(state::build_executors(&topo));
105 tracing::info!(
106 app = %id,
107 tiers = topo.tiers.len(),
108 bare = topo.repo.as_ref().map_or("(intake-only)", |r| r.bare_path.as_str()),
109 "app loaded"
110 );
111 apps.insert(
112 id,
113 Arc::new(state::App {
114 cfg,
115 topo,
116 executors,
117 }),
118 );
119 }
120 let apps = Arc::new(apps);
121 // The product a request that names none is about. First in declared order,
122 // which for a legacy config is the only one there is.
123 let default_app = apps
124 .keys()
125 .next()
126 .cloned()
127 .expect("DaemonConfig::load refuses an empty app set");
128 let default = apps[&default_app].clone();
129 let (cfg, topo) = (default.cfg.clone(), default.topo.clone());
130 let pool = db::connect(&daemon.db_path).await?;
131 db::migrate(&pool).await?;
132 // Reconcile any build_runs left 'building' by a previous daemon that died
133 // mid-build, so a restart can't leave a run wedged 'building' forever.
134 match runs::recover_orphaned_running(&pool).await {
135 Ok(0) => {}
136 Ok(n) => tracing::warn!(
137 reconciled = n,
138 "settled orphaned 'building' run(s) from a prior daemon"
139 ),
140 Err(e) => {
141 tracing::error!(error = %e, "failed to reconcile orphaned 'building' runs at startup");
142 }
143 }
144 for (id, app) in apps.iter() {
145 sync::sync(&pool, id, &app.topo).await?;
146 tracing::debug!(app = %id, "topology synced");
147 }
148
149 // Reconcile deploys against tier_state (after sync, so every tier_state row
150 // exists): catch a promote that landed on the nodes but died before advancing
151 // tier_state, so a stale-green tier on the wrong version is flagged partial
152 // instead of misleading /state and a later /rollback (audit
153 // sando-h5-crash-mid-promote).
154 match reconcile::recover_unrecorded_deploys(&pool).await {
155 Ok(0) => {}
156 Ok(n) => tracing::error!(
157 flagged = n,
158 "flagged tier(s) with an unrecorded deploy from a prior daemon; see /state partial_reason"
159 ),
160 Err(e) => {
161 tracing::error!(error = %e, "failed to reconcile deploys against tier_state at startup");
162 }
163 }
164
165 // Say at startup whether every artifact the deployed state names is still on
166 // disk. `/state` answers the same question on every poll; this is so the
167 // journal carries it too, because the two occurrences of the retention bug
168 // were both discovered by an rsync failing mid-promote rather than by
169 // anything that had looked. Never fatal: a stranded artifact is a lost
170 // rollback, not a reason to refuse to start — refusing would take the
171 // controller down as well.
172 for (id, app) in apps.iter() {
173 match retention::tier_refs(&pool, id, app.cfg.primary_bin()).await {
174 Ok(refs) => {
175 let gone = retention::missing(&refs).await;
176 if gone.is_empty() {
177 tracing::info!(app = %id, referenced = refs.len(), "every referenced release artifact is present");
178 } else {
179 tracing::error!(
180 app = %id,
181 missing = gone.len(),
182 detail = %retention::describe(&gone),
183 "referenced release artifact(s) missing from the store; promote and rollback \
184 through them will fail at the rsync. See /state missing_artifact"
185 );
186 }
187 }
188 Err(e) => {
189 tracing::error!(app = %id, error = %e, "could not check referenced release artifacts");
190 }
191 }
192 }
193
194 // Fail closed on the scratch cluster's privileges before any gate can hit
195 // them, rather than letting the first migration_dry_run discover it as an
196 // opaque "permission denied". Unset scratch_db_url is already a per-gate
197 // Blocked outcome, so there is nothing to assert.
198 if let Some(scratch_url) = cfg.scratch_db_url.as_deref() {
199 sando_daemon::gates::preflight_scratch_privileges(scratch_url).await?;
200 }
201
202 let addr: SocketAddr = daemon.listen.parse()?;
203
204 // Deploy-API auth (CF2). Token comes from SANDO_API_TOKEN (systemd
205 // EnvironmentFile). Refuse to expose the mutators unauthenticated on a
206 // non-loopback bind — that exact posture (tailnet bind, no auth) was the
207 // finding.
208 let api_token: Option<Arc<str>> = std::env::var("SANDO_API_TOKEN")
209 .ok()
210 .map(|s| s.trim().to_string())
211 .filter(|s| !s.is_empty())
212 .map(|s| Arc::from(s.as_str()));
213 if refuse_unauthenticated_bind(&addr, api_token.is_some()) {
214 anyhow::bail!(
215 "SANDO_API_TOKEN is unset but listen={addr} is not loopback; refusing to expose deploy \
216 controls unauthenticated. Set SANDO_API_TOKEN (via EnvironmentFile) or bind 127.0.0.1."
217 );
218 }
219 match &api_token {
220 Some(_) => tracing::info!("deploy endpoints require a bearer token"),
221 None => {
222 tracing::warn!(%addr, "SANDO_API_TOKEN unset; deploy endpoints are UNAUTHENTICATED (loopback bind)");
223 }
224 }
225
226 let executors = default.executors.clone();
227 let app_state = state::AppState {
228 pool,
229 apps,
230 default_app,
231 topo,
232 cfg,
233 active_build: Arc::new(tokio::sync::Mutex::new(None)),
234 deploy_lock: Arc::new(tokio::sync::Mutex::new(())),
235 events: events::channel(),
236 executors,
237 api_token,
238 };
239 let app = routes::router_for_apps(app_state);
240 tracing::info!(%addr, "sando daemon listening");
241 let listener = tokio::net::TcpListener::bind(addr).await?;
242 // `into_make_service_with_connect_info` surfaces the peer `SocketAddr` in
243 // request extensions so the bearer middleware can log who issued a mutation.
244 axum::serve(
245 listener,
246 app.into_make_service_with_connect_info::<SocketAddr>(),
247 )
248 .await?;
249 Ok(())
250 }
251
252 #[cfg(test)]
253 mod tests {
254 use super::{refuse_unauthenticated_bind, validate_loaded};
255 use std::net::SocketAddr;
256
257 fn addr(s: &str) -> SocketAddr {
258 s.parse().unwrap()
259 }
260
261 #[test]
262 fn check_config_accepts_the_shipped_topology() {
263 // The `--check-config` path a self-update runs before installing a new
264 // binary must pass against the topology this build ships with — the
265 // regression guard for the 2026-07-09 brick, where a binary that could
266 // not parse the live node_health config was installed and crash-looped.
267 // Both halves of the pair are the shipped ones. This used to hand-roll a
268 // minimal daemon config, which meant the production daemon config was
269 // never validated by anything and the cross-file checks had nothing real
270 // to check: the fixture declared no companions while the shipped
271 // topology installs two, so `ensure_node_companions_are_built` would
272 // have passed on a fixture and failed on the box.
273 //
274 // Only `topology_path` is overridden, because the shipped value is the
275 // installed location (/etc/sando/sando.toml) and the repo copy is what
276 // this build ships.
277 let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
278 let raw = std::fs::read_to_string(root.join("../deploy/sando-daemon.toml.example"))
279 .expect("reading the shipped production daemon config");
280 let mut cfg: super::config::AppConfig =
281 toml::from_str(&raw).expect("shipped daemon config must parse");
282 cfg.topology_path = root.join("../sando.toml");
283 validate_loaded(&cfg).expect("shipped config must pass --check-config");
284 }
285
286 #[test]
287 fn non_loopback_without_token_is_refused() {
288 // The exact CF2 posture: tailnet bind, no token.
289 assert!(refuse_unauthenticated_bind(
290 &addr("100.103.89.95:7766"),
291 false
292 ));
293 assert!(refuse_unauthenticated_bind(&addr("0.0.0.0:7766"), false));
294 }
295
296 #[test]
297 fn loopback_without_token_is_allowed() {
298 assert!(!refuse_unauthenticated_bind(&addr("127.0.0.1:7766"), false));
299 assert!(!refuse_unauthenticated_bind(&addr("[::1]:7766"), false));
300 }
301
302 #[test]
303 fn any_bind_with_token_is_allowed() {
304 assert!(!refuse_unauthenticated_bind(
305 &addr("100.103.89.95:7766"),
306 true
307 ));
308 assert!(!refuse_unauthenticated_bind(&addr("127.0.0.1:7766"), true));
309 }
310 }
311