Skip to main content

max / makenotwork

21.6 KB · 524 lines History Blame Raw
1 //! Handing a finished artifact to Sando.
2 //!
3 //! Design + rationale: maintainer wiki.
4 //! <!-- wiki: sando-bento-boundary -->
5 //!
6 //! The boundary is "Bento builds and packages, Sando decides whether a thing
7 //! advances a stage". Sando has an intake that proves an incoming bundle against
8 //! its record three ways and publishes it content-addressed — but `POST /intake`
9 //! takes a bundle already sitting under Sando's staging directory, and until now
10 //! nothing put one there. This is the half that moves the bytes.
11 //!
12 //! Two motions, in this order, per target:
13 //!
14 //! 1. rsync the target's collect directory into
15 //! `<staging_root>/<app>-<version>-<target>/` on the Sando host.
16 //! 2. POST that path plus the artifact record to sandod, which verifies and
17 //! publishes it, or refuses it and names the file that drifted.
18 //!
19 //! **The record does not travel with the bytes.** It names the digest of the
20 //! bundle, and the digest covers every file in the bundle, so a record copied in
21 //! among the artifacts would change the digest it names — Sando would recompute,
22 //! disagree, and refuse. It goes in the request body instead. That is the same
23 //! "evidence rides alongside, never inside" rule the contract crate is built on,
24 //! showing up here as an rsync exclude.
25 //!
26 //! **The staging directory is mirrored, not merged.** A retry after a partial
27 //! transfer would otherwise leave a file from the earlier attempt behind, and an
28 //! extra file is a manifest mismatch: the honest bundle would be refused for
29 //! carrying a leftover. `--delete` makes the directory hold exactly this attempt.
30 //!
31 //! Failure fails the target run. A build whose artifact never reached the deploy
32 //! controller has not finished the job the handoff exists to do, and the same
33 //! argument the archive deposit makes applies with more force: a silently
34 //! skipped handoff means the release nobody watched is the one Sando never heard
35 //! about.
36
37 use crate::config::{Config, Handoff};
38 use crate::domain::{AppId, Target, Version};
39 use ops_exec::{CapabilitySet, Executor, LocalExec, SshExec, SyncOpts};
40 use std::path::{Path, PathBuf};
41 use std::sync::Arc;
42
43 /// Where one target's bundle is staged on the Sando host.
44 ///
45 /// Flat and fully qualified rather than nested `<app>/<version>/<target>/`,
46 /// because Sando's intake resolves this path against its own staging root and
47 /// publishes by renaming the directory out of it. One component means one
48 /// rename, and the name still says which build it is when an operator finds it
49 /// left behind after a refused intake.
50 pub fn staged_dir(staging_root: &Path, app: &AppId, version: &Version, target: Target) -> PathBuf {
51 staging_root.join(format!(
52 "{}-{}-{}",
53 app.as_str(),
54 version,
55 crate::archive::target_slug(target)
56 ))
57 }
58
59 /// The transport that writes into Sando's staging directory.
60 ///
61 /// Granted nothing, like the archive's: it only ever calls `push_dir`, so no
62 /// capability in `ops_exec` is exercisable through this handle and the Sando
63 /// host cannot become a build host by holding one. It is not in the topology
64 /// either, so no recipe can name it.
65 fn transport(handoff: &Handoff) -> Arc<dyn Executor> {
66 const NONE: [&str; 0] = [];
67 let caps = CapabilitySet::from_tokens(NONE, NONE);
68 if handoff.host == "local" || handoff.host.is_empty() {
69 return Arc::new(LocalExec::new(caps));
70 }
71 Arc::new(SshExec::new(handoff.host.clone(), caps))
72 }
73
74 /// `POST` target for this handoff: the unprefixed mount is Sando's default
75 /// product, and a named one is nested under `/apps/<id>`.
76 fn intake_url(handoff: &Handoff) -> String {
77 let base = handoff.url.trim_end_matches('/');
78 match &handoff.sando_app {
79 Some(id) => format!("{base}/apps/{id}/intake"),
80 None => format!("{base}/intake"),
81 }
82 }
83
84 /// Read the bearer token for this handoff out of the environment.
85 ///
86 /// From the environment and not from `secrets_root`, which is where this used
87 /// to read: a bearer token is not a signing key. The recipe secrets under the
88 /// private layer are files because they are files — a keystore, a notary
89 /// credential, things a tool opens by path — while this is one opaque string
90 /// held for the life of the process, and writing it to disk to read it back
91 /// bought nothing but a plaintext credential in a tree that is now under git.
92 /// Both daemons already take their own tokens this way (`SANDO_API_TOKEN`,
93 /// `BENTO_API_TOKEN`), so this is the existing mechanism rather than a third.
94 ///
95 /// Never logged, and the value is trimmed: an `EnvironmentFile` line pasted
96 /// with a trailing space would otherwise put it inside the header value, which
97 /// fails as a 401 with nothing to see in it.
98 fn token(handoff: &Handoff) -> anyhow::Result<Option<String>> {
99 let Some(name) = &handoff.token_env else {
100 return Ok(None);
101 };
102 token_from(name, std::env::var(name).ok())
103 }
104
105 /// The value half of `token`, split out so the rules can be tested without
106 /// touching the process environment. `set_var` is global and unsynchronized, and
107 /// a test that sets one affects every other test in the binary (see
108 /// `engine::git::tests::expand_tilde_handles_home`).
109 fn token_from(name: &str, raw: Option<String>) -> anyhow::Result<Option<String>> {
110 // Unset and empty are one case on purpose. `EnvironmentFile` turns a line
111 // whose value was never filled in into an empty variable rather than no
112 // variable, so treating empty as "no header" would make a half-finished
113 // bootstrap look like a deliberately unauthenticated handoff.
114 let trimmed = raw.unwrap_or_default().trim().to_string();
115 anyhow::ensure!(
116 !trimmed.is_empty(),
117 "the sando bearer token is missing: `{name}` is unset or empty. Set it in \
118 bentod's EnvironmentFile (~/.config/bento/bento.env) to match \
119 SANDO_API_TOKEN in /etc/sando/sando.env on the sando host."
120 );
121 Ok(Some(trimmed))
122 }
123
124 /// Send one target's finished bundle to Sando and ask it to take the artifact
125 /// in. A no-op for an app with no handoff configured.
126 ///
127 /// `local_dir` is the target's collect directory and `record_path` the artifact
128 /// record beside it — written by `artifact_record::emit` after the recipe, which
129 /// is why this runs there and not inside `collect`.
130 pub async fn send(
131 cfg: &Config,
132 local_dir: &Path,
133 record_path: &Path,
134 app: &AppId,
135 version: &Version,
136 target: Target,
137 ) -> anyhow::Result<()> {
138 let Some(handoff) = cfg.handoff.get(app.as_str()) else {
139 return Ok(());
140 };
141
142 // Read the record first. It is what makes the transfer worth doing, and a
143 // missing or unreadable one means shipping bytes Sando could only refuse.
144 let record = tokio::fs::read_to_string(record_path).await.map_err(|e| {
145 anyhow::anyhow!(
146 "reading the artifact record at {}: {e}",
147 record_path.display()
148 )
149 })?;
150
151 let dest = staged_dir(&handoff.staging_root, app, version, target);
152 transport(handoff)
153 .push_dir(
154 local_dir,
155 &dest,
156 &SyncOpts {
157 // Exactly this attempt's bytes: see the module header.
158 delete: true,
159 exclude: vec![crate::artifact_record::RECORD_FILE.to_string()],
160 ..SyncOpts::archive_deposit()
161 },
162 )
163 .await
164 .map_err(|e| {
165 anyhow::anyhow!(
166 "staging {} at {}:{}: {e}",
167 local_dir.display(),
168 handoff.host,
169 dest.display()
170 )
171 })?;
172
173 let url = intake_url(handoff);
174 let mut req = crate::tls::builder()
175 .build()?
176 .post(&url)
177 .json(&serde_json::json!({
178 "staged": dest.to_string_lossy(),
179 "record": record,
180 }));
181 if let Some(t) = token(handoff)? {
182 req = req.bearer_auth(t);
183 }
184 let resp = req
185 .send()
186 .await
187 .map_err(|e| anyhow::anyhow!("posting the intake to {url}: {e}"))?;
188
189 let status = resp.status();
190 let body = resp.text().await.unwrap_or_default();
191 anyhow::ensure!(
192 status.is_success(),
193 // Sando's refusals are the useful half — it names the file that drifted
194 // — so the body is carried into the error rather than reduced to a code.
195 "sando refused the intake at {url}: {status} {body}"
196 );
197
198 tracing::info!(
199 %app, %target, %version, host = %handoff.host, dest = %dest.display(),
200 "handed the artifact to sando"
201 );
202 Ok(())
203 }
204
205 #[cfg(test)]
206 mod tests {
207 use super::*;
208
209 fn app() -> AppId {
210 AppId::new("pom")
211 }
212 fn version() -> Version {
213 "0.3.1".parse().unwrap()
214 }
215 fn handoff(root: &Path) -> Handoff {
216 Handoff {
217 host: "local".into(),
218 staging_root: root.join("staging"),
219 url: "http://127.0.0.1:1".into(),
220 sando_app: None,
221 token_env: None,
222 }
223 }
224
225 /// One component, carrying every dimension that distinguishes a build. Two
226 /// architectures of one version are two bundles, and staging them at one
227 /// path would have the second overwrite the first.
228 #[test]
229 fn a_staged_dir_names_app_version_and_target_in_one_component() {
230 let dir = staged_dir(
231 Path::new("/srv/sando/staging"),
232 &app(),
233 &version(),
234 "linux/aarch64".parse().unwrap(),
235 );
236 assert_eq!(dir, Path::new("/srv/sando/staging/pom-0.3.1-linux-aarch64"));
237 assert_eq!(dir.parent(), Some(Path::new("/srv/sando/staging")));
238
239 let other = staged_dir(
240 Path::new("/srv/sando/staging"),
241 &app(),
242 &version(),
243 "linux/x86_64".parse().unwrap(),
244 );
245 assert_ne!(dir, other);
246 }
247
248 /// The default product is the unprefixed mount, because that is what Sando's
249 /// router does with a product it was not asked to nest.
250 #[test]
251 fn the_intake_url_nests_only_a_named_product() {
252 let mut h = handoff(Path::new("/tmp"));
253 h.url = "http://100.103.89.95:7766".into();
254 assert_eq!(intake_url(&h), "http://100.103.89.95:7766/intake");
255
256 h.sando_app = Some("pom".into());
257 assert_eq!(intake_url(&h), "http://100.103.89.95:7766/apps/pom/intake");
258
259 // A trailing slash is an operator's habit, not a second path segment.
260 h.url = "http://100.103.89.95:7766/".into();
261 assert_eq!(intake_url(&h), "http://100.103.89.95:7766/apps/pom/intake");
262 }
263
264 /// An app nobody configured a handoff for is untouched — the ordinary case
265 /// for every app Bento ships that Sando does not deploy.
266 #[tokio::test]
267 async fn an_app_with_no_handoff_sends_nothing() {
268 let tmp = tempfile::tempdir().unwrap();
269 let cfg = Config::for_tests(tmp.path());
270 assert!(cfg.handoff.is_empty());
271 send(
272 &cfg,
273 tmp.path(),
274 &tmp.path().join("record.json"),
275 &app(),
276 &version(),
277 "linux/x86_64".parse().unwrap(),
278 )
279 .await
280 .expect("a no-op cannot fail");
281 }
282
283 /// The staging transfer, up to the POST (which fails against a dead port).
284 /// What it proves is the shape of what lands: the artifacts arrive, and the
285 /// record does NOT — a record inside the bundle would change the digest it
286 /// names and Sando would refuse the honest bytes.
287 #[tokio::test]
288 async fn the_bundle_is_staged_without_its_record() {
289 let tmp = tempfile::tempdir().unwrap();
290 let mut cfg = Config::for_tests(tmp.path());
291 cfg.handoff.insert("pom".into(), handoff(tmp.path()));
292
293 let collected = tmp.path().join("collected");
294 std::fs::create_dir_all(&collected).unwrap();
295 std::fs::write(collected.join("pom"), b"binary").unwrap();
296 let record_path = collected.join(crate::artifact_record::RECORD_FILE);
297 std::fs::write(&record_path, b"{\"producer\":\"bento\"}").unwrap();
298
299 let target: Target = "linux/x86_64".parse().unwrap();
300 // The POST cannot succeed here; the transfer before it still ran.
301 let _ = send(&cfg, &collected, &record_path, &app(), &version(), target).await;
302
303 let staged = staged_dir(&tmp.path().join("staging"), &app(), &version(), target);
304 assert_eq!(std::fs::read(staged.join("pom")).unwrap(), b"binary");
305 assert!(
306 !staged.join(crate::artifact_record::RECORD_FILE).exists(),
307 "the record must not travel inside the bundle it describes"
308 );
309 }
310
311 /// A leftover from an earlier attempt is pruned. Without `--delete` it would
312 /// survive as an extra file, and an extra file is a manifest mismatch — the
313 /// retry of an honest build would be refused for carrying it.
314 #[tokio::test]
315 async fn a_retry_leaves_nothing_from_the_previous_attempt() {
316 let tmp = tempfile::tempdir().unwrap();
317 let mut cfg = Config::for_tests(tmp.path());
318 cfg.handoff.insert("pom".into(), handoff(tmp.path()));
319
320 let collected = tmp.path().join("collected");
321 std::fs::create_dir_all(&collected).unwrap();
322 std::fs::write(collected.join("pom"), b"binary").unwrap();
323 let record_path = collected.join(crate::artifact_record::RECORD_FILE);
324 std::fs::write(&record_path, b"{}").unwrap();
325
326 let target: Target = "linux/x86_64".parse().unwrap();
327 let staged = staged_dir(&tmp.path().join("staging"), &app(), &version(), target);
328 std::fs::create_dir_all(&staged).unwrap();
329 std::fs::write(staged.join("stale-from-last-time"), b"junk").unwrap();
330
331 let _ = send(&cfg, &collected, &record_path, &app(), &version(), target).await;
332
333 assert!(staged.join("pom").exists());
334 assert!(
335 !staged.join("stale-from-last-time").exists(),
336 "the staging dir must hold exactly this attempt"
337 );
338 }
339
340 /// A missing record is refused before any bytes move. Sando could only
341 /// refuse the bundle anyway, and staging first would leave a directory
342 /// behind for an intake that was never going to be requested.
343 #[tokio::test]
344 async fn a_missing_record_fails_before_anything_is_staged() {
345 let tmp = tempfile::tempdir().unwrap();
346 let mut cfg = Config::for_tests(tmp.path());
347 cfg.handoff.insert("pom".into(), handoff(tmp.path()));
348
349 let collected = tmp.path().join("collected");
350 std::fs::create_dir_all(&collected).unwrap();
351 std::fs::write(collected.join("pom"), b"binary").unwrap();
352
353 let target: Target = "linux/x86_64".parse().unwrap();
354 let err = send(
355 &cfg,
356 &collected,
357 &collected.join("record.json"),
358 &app(),
359 &version(),
360 target,
361 )
362 .await
363 .expect_err("no record, no handoff");
364 assert!(format!("{err:#}").contains("artifact record"), "{err:#}");
365
366 let staged = staged_dir(&tmp.path().join("staging"), &app(), &version(), target);
367 assert!(!staged.exists(), "nothing should have been staged");
368 }
369
370 /// An unset or empty variable is a bootstrap that half-ran, and both are
371 /// the same mistake: `EnvironmentFile` turns `BENTO_SANDO_TOKEN=` into an
372 /// empty variable, not a missing one. Sending the header anyway would fail
373 /// at sandod as a plain 401, which reads as a wrong token rather than a
374 /// missing one, and the error names the two files that have to agree.
375 #[test]
376 fn an_unset_or_empty_token_variable_is_an_error_not_an_empty_header() {
377 for raw in [None, Some(String::new()), Some("\n".into())] {
378 let err = token_from("BENTO_SANDO_TOKEN", raw)
379 .expect_err("nothing in the variable is not a token");
380 let msg = format!("{err:#}");
381 assert!(msg.contains("BENTO_SANDO_TOKEN"), "{msg}");
382 assert!(msg.contains("SANDO_API_TOKEN"), "{msg}");
383 }
384 }
385
386 /// The value is trimmed. An `EnvironmentFile` line pasted with a trailing
387 /// space puts that space inside the header value, which fails as a 401 with
388 /// nothing visible in it to explain why.
389 #[test]
390 fn a_token_is_read_trimmed() {
391 assert_eq!(
392 token_from("BENTO_SANDO_TOKEN", Some(" s3cr3t \n".into()))
393 .unwrap()
394 .as_deref(),
395 Some("s3cr3t")
396 );
397 }
398
399 /// No variable named means no header, not an empty one — the shape a
400 /// single-product sandod on loopback wants.
401 #[test]
402 fn no_token_env_means_no_header() {
403 let tmp = tempfile::tempdir().unwrap();
404 assert_eq!(token(&handoff(tmp.path())).unwrap(), None);
405 }
406
407 /// The whole motion against a sandod-shaped listener: the bytes are staged,
408 /// and the request names the path they were staged at and carries the record
409 /// as its own field. Those two together are the contract — Sando resolves
410 /// `staged` on its own disk and proves it against `record`, so a handoff that
411 /// staged one path and reported another would be refused for corruption when
412 /// nothing had corrupted.
413 #[tokio::test]
414 async fn the_post_names_the_path_the_bytes_were_staged_at() {
415 use axum::{Json, Router, routing::post};
416 use std::sync::{Arc, Mutex};
417
418 let seen: Arc<Mutex<Option<serde_json::Value>>> = Arc::new(Mutex::new(None));
419 let auth: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
420 let (s, a) = (seen.clone(), auth.clone());
421 let app_router = Router::new().route(
422 "/apps/pom/intake",
423 post(
424 move |headers: axum::http::HeaderMap, Json(b): Json<serde_json::Value>| {
425 let (s, a) = (s.clone(), a.clone());
426 async move {
427 *s.lock().unwrap() = Some(b);
428 *a.lock().unwrap() = headers
429 .get("authorization")
430 .and_then(|v| v.to_str().ok())
431 .map(str::to_owned);
432 Json(serde_json::json!({ "accepted": true, "run_id": 1 }))
433 }
434 },
435 ),
436 );
437 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
438 let addr = listener.local_addr().unwrap();
439 tokio::spawn(async move { axum::serve(listener, app_router).await.unwrap() });
440
441 let tmp = tempfile::tempdir().unwrap();
442 let mut cfg = Config::for_tests(tmp.path());
443 // The one place this binary sets a variable. Distinct from the HOME
444 // lesson in `engine::git::tests`: the name is this test's own, so no other
445 // test reads it, and it is never removed — the value it holds is the
446 // whole point of the assertion at the bottom.
447 unsafe { std::env::set_var("BENTO_HANDOFF_WIRE_TEST_TOKEN", "s3cr3t\n") };
448 let mut h = handoff(tmp.path());
449 h.url = format!("http://{addr}");
450 h.sando_app = Some("pom".into());
451 h.token_env = Some("BENTO_HANDOFF_WIRE_TEST_TOKEN".into());
452 cfg.handoff.insert("pom".into(), h);
453
454 let collected = tmp.path().join("collected");
455 std::fs::create_dir_all(&collected).unwrap();
456 std::fs::write(collected.join("pom"), b"binary").unwrap();
457 let record_path = collected.join(crate::artifact_record::RECORD_FILE);
458 std::fs::write(&record_path, b"{\"producer\":\"bento\"}").unwrap();
459
460 let target: Target = "linux/x86_64".parse().unwrap();
461 send(&cfg, &collected, &record_path, &app(), &version(), target)
462 .await
463 .expect("the intake was accepted");
464
465 let body = seen.lock().unwrap().clone().expect("sandod was called");
466 let staged = staged_dir(&tmp.path().join("staging"), &app(), &version(), target);
467 assert_eq!(body["staged"], staged.to_string_lossy().as_ref());
468 assert_eq!(body["record"], "{\"producer\":\"bento\"}");
469 assert!(
470 Path::new(body["staged"].as_str().unwrap())
471 .join("pom")
472 .exists()
473 );
474 assert_eq!(auth.lock().unwrap().as_deref(), Some("Bearer s3cr3t"));
475 }
476
477 /// A refusal is a failed handoff, and it carries Sando's own words. Its
478 /// errors name the file that drifted, which is the whole return on a
479 /// per-file manifest; reducing them to a status code would throw that away
480 /// at the one moment somebody needs it.
481 #[tokio::test]
482 async fn a_refusal_fails_and_carries_what_sando_said() {
483 use axum::{Router, http::StatusCode, routing::post};
484
485 let app_router = Router::new().route(
486 "/intake",
487 post(|| async {
488 (
489 StatusCode::BAD_REQUEST,
490 "the bundle is not what its record describes; `pom` differs",
491 )
492 }),
493 );
494 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
495 let addr = listener.local_addr().unwrap();
496 tokio::spawn(async move { axum::serve(listener, app_router).await.unwrap() });
497
498 let tmp = tempfile::tempdir().unwrap();
499 let mut cfg = Config::for_tests(tmp.path());
500 let mut h = handoff(tmp.path());
501 h.url = format!("http://{addr}");
502 cfg.handoff.insert("pom".into(), h);
503
504 let collected = tmp.path().join("collected");
505 std::fs::create_dir_all(&collected).unwrap();
506 std::fs::write(collected.join("pom"), b"binary").unwrap();
507 let record_path = collected.join(crate::artifact_record::RECORD_FILE);
508 std::fs::write(&record_path, b"{}").unwrap();
509
510 let err = send(
511 &cfg,
512 &collected,
513 &record_path,
514 &app(),
515 &version(),
516 "linux/x86_64".parse().unwrap(),
517 )
518 .await
519 .expect_err("a refused intake is a failed handoff");
520 let msg = format!("{err:#}");
521 assert!(msg.contains("`pom` differs"), "{msg}");
522 }
523 }
524