Skip to main content

max / makenotwork

15.4 KB · 390 lines History Blame Raw
1 //! Accepting an artifact Sando did not build.
2 //!
3 //! Design: wiki [[sando-bento-boundary]], [[release-artifact-identity]].
4 //!
5 //! Everything downstream of `build.rs` assumes Sando compiled the thing it is
6 //! about to ship. That assumption is what the boundary removes: Bento builds,
7 //! Sando decides whether the result advances. Deciding requires being handed
8 //! bytes, and being handed bytes requires a reason to believe they are the ones
9 //! the evidence is about.
10 //!
11 //! That belief is not taken on trust here. An accepted artifact has been checked
12 //! three ways, each catching something the others cannot:
13 //!
14 //! 1. The record is internally consistent: its claimed digest is the digest of
15 //! the manifest it carries. Catches a hand-edited or truncated document.
16 //! 2. The bytes on disk hash to that manifest, file for file. Catches a bundle
17 //! that drifted in transit, or one swapped for another, and names the file.
18 //! 3. The digest of what arrived equals the digest the record claims. Implied by
19 //! (2), and checked anyway because it is the value everything downstream
20 //! keys on, and a mismatch here means the two computations disagree.
21 //!
22 //! Intake is about identity, not policy. A record whose builder gates failed is
23 //! accepted and says so: whether that artifact may advance is the gating layer's
24 //! call, and it cannot make that call about an artifact it was never handed.
25
26 use crate::bundle;
27 use ops_artifact::ArtifactRecord;
28 use std::path::{Path, PathBuf};
29
30 /// An artifact that arrived with credible paperwork, published into the local
31 /// release root.
32 #[derive(Debug)]
33 pub struct AcceptedArtifact {
34 /// The builder's document, verified against the bytes.
35 pub record: ArtifactRecord,
36 /// The content-addressed directory the bundle now lives at.
37 pub released: PathBuf,
38 }
39
40 impl AcceptedArtifact {
41 /// True iff every gate the builder ran passed. Not a decision to advance:
42 /// Sando's own gates have not run yet, and they are the ones about this
43 /// artifact in an environment.
44 pub fn builder_gates_passed(&self) -> bool {
45 self.record.all_gates_passed()
46 }
47 }
48
49 /// Why an artifact was turned away.
50 ///
51 /// Typed rather than a string because the operator response differs per case
52 /// and only one of these is a transfer problem. A record that does not parse is
53 /// a producer bug; a manifest mismatch is corruption or substitution; a failure
54 /// to publish is a local disk problem on this machine.
55 #[derive(Debug)]
56 pub enum IntakeError {
57 /// The record did not parse, or failed its own validation.
58 BadRecord(ops_artifact::ContractError),
59 /// The bundle could not be hashed (unreadable, or not a directory).
60 Unreadable(String),
61 /// The bytes are not the ones the record describes. `first_difference` names
62 /// a file the two disagree about, when one can be identified.
63 ManifestMismatch {
64 first_difference: Option<String>,
65 claimed_files: usize,
66 found_files: usize,
67 },
68 /// The recomputed digest is not the one claimed.
69 DigestMismatch { claimed: String, computed: String },
70 /// Publishing the verified bundle into the release root failed.
71 Publish(String),
72 }
73
74 impl std::fmt::Display for IntakeError {
75 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
76 match self {
77 IntakeError::BadRecord(e) => write!(f, "artifact record is not usable: {e}"),
78 IntakeError::Unreadable(d) => write!(f, "could not read the incoming bundle: {d}"),
79 IntakeError::ManifestMismatch {
80 first_difference: Some(p),
81 ..
82 } => write!(
83 f,
84 "the bundle is not what its record describes; `{p}` differs. \
85 Refusing an artifact whose evidence is about other bytes"
86 ),
87 IntakeError::ManifestMismatch {
88 first_difference: None,
89 claimed_files,
90 found_files,
91 } => write!(
92 f,
93 "the bundle is not what its record describes: the record lists {claimed_files} \
94 file(s), the bundle holds {found_files}"
95 ),
96 IntakeError::DigestMismatch { claimed, computed } => write!(
97 f,
98 "bundle digest {computed} does not match the claimed {claimed}"
99 ),
100 IntakeError::Publish(d) => write!(f, "publishing the accepted bundle failed: {d}"),
101 }
102 }
103 }
104
105 impl std::error::Error for IntakeError {}
106
107 /// Verify an incoming bundle against its record and publish it
108 /// content-addressed into `release_root`.
109 ///
110 /// `staged` must already be a directory under `release_root/staging/`, because
111 /// publishing is an atomic same-filesystem rename and a cross-device staging dir
112 /// would silently become a copy. The caller owns getting the bytes there; this
113 /// owns deciding whether they may stay.
114 ///
115 /// `pinned` is passed straight through to publishing, which gc's the store: it
116 /// names the release dirs the deployed state still points at
117 /// ([`crate::retention::pinned_dirs`]). Passing an empty set is what a caller
118 /// with no deployed state to protect does, not a shortcut.
119 pub async fn accept(
120 release_root: &Path,
121 staged: &Path,
122 record_json: &str,
123 pinned: &crate::retention::PinnedReleases,
124 ) -> Result<AcceptedArtifact, IntakeError> {
125 let record = ArtifactRecord::parse(record_json).map_err(IntakeError::BadRecord)?;
126
127 // Hash what actually arrived. Sando's own bundle walker, deliberately: the
128 // artifact will later be verified on the node by the same code, so intake
129 // proving the bundle with a different implementation than the deploy uses
130 // would leave a gap exactly where the two disagree.
131 let computed = bundle::digest_dir(staged)
132 .await
133 .map_err(|e| IntakeError::Unreadable(format!("{e:#}")))?;
134
135 let claimed_manifest = record.manifest.to_text();
136 if computed.manifest != claimed_manifest {
137 return Err(manifest_mismatch(&claimed_manifest, &computed.manifest));
138 }
139 if computed.full != record.digest.as_str() {
140 return Err(IntakeError::DigestMismatch {
141 claimed: record.digest.to_string(),
142 computed: computed.full,
143 });
144 }
145
146 // Write the MANIFEST into the bundle so the node can verify per file after
147 // the transfer. Excluded from its own hash, so this does not change what was
148 // just proved.
149 tokio::fs::write(
150 staged.join(bundle::MANIFEST_NAME),
151 computed.manifest.as_bytes(),
152 )
153 .await
154 .map_err(|e| IntakeError::Publish(format!("writing MANIFEST: {e}")))?;
155
156 let released =
157 crate::deploy::finalize_local_release(release_root, staged, computed.short(), pinned)
158 .await
159 .map_err(|e| IntakeError::Publish(format!("{e:#}")))?;
160
161 tracing::info!(
162 app = %record.provenance.app,
163 version = %record.provenance.version,
164 target = %record.provenance.target,
165 producer = %record.producer,
166 digest = %record.digest.short(),
167 "accepted an artifact built elsewhere"
168 );
169 Ok(AcceptedArtifact { record, released })
170 }
171
172 /// Name a file the two manifests disagree about, for the error.
173 ///
174 /// A digest mismatch alone says only "not the same"; the useful half is which
175 /// file drifted, which is the whole reason the manifest is per-file rather than
176 /// one hash over a tarball.
177 fn manifest_mismatch(claimed: &str, computed: &str) -> IntakeError {
178 let paths = |text: &str| -> Vec<(String, String)> {
179 text.lines()
180 .filter_map(|l| l.split_once(" "))
181 .map(|(sha, path)| (path.to_string(), sha.to_string()))
182 .collect()
183 };
184 let (a, b) = (paths(claimed), paths(computed));
185 let first_difference = a
186 .iter()
187 .find(|(path, sha)| !b.iter().any(|(p, s)| p == path && s == sha))
188 .or_else(|| b.iter().find(|(path, _)| !a.iter().any(|(p, _)| p == path)))
189 .map(|(path, _)| path.clone());
190 IntakeError::ManifestMismatch {
191 first_difference,
192 claimed_files: a.len(),
193 found_files: b.len(),
194 }
195 }
196
197 #[cfg(test)]
198 mod tests {
199 /// No deployed state in a unit test, so nothing is pinned and gc is free to
200 /// apply the count alone. Retention is exercised in `deploy`'s own tests.
201 fn no_pins() -> crate::retention::PinnedReleases {
202 crate::retention::PinnedReleases::none()
203 }
204
205 use super::*;
206 use chrono::{DateTime, Utc};
207 use ops_artifact::{GateRecord, Manifest, Provenance, Scope, Verdict};
208
209 fn at() -> DateTime<Utc> {
210 DateTime::<Utc>::from_timestamp(1_754_000_000, 0).unwrap()
211 }
212
213 async fn write(root: &Path, rel: &str, bytes: &[u8]) {
214 let p = root.join(rel);
215 tokio::fs::create_dir_all(p.parent().unwrap())
216 .await
217 .unwrap();
218 tokio::fs::write(&p, bytes).await.unwrap();
219 }
220
221 /// A staging dir under `release_root`, holding a small bundle.
222 async fn staged_bundle(release_root: &Path) -> PathBuf {
223 let staged = release_root.join("staging").join("1");
224 write(&staged, "pom", b"binary bytes").await;
225 write(&staged, "static/app.css", b"body{}").await;
226 staged
227 }
228
229 fn provenance() -> Provenance {
230 Provenance {
231 app: "pom".into(),
232 version: "0.4.1".into(),
233 tag: "pom-v0.4.1".into(),
234 git_sha: "a".repeat(40),
235 target: "linux/aarch64".into(),
236 build_host: "astra".into(),
237 toolchain: "rustc 1.97.0".into(),
238 built_at: at(),
239 }
240 }
241
242 /// The record Bento would have written for `staged`.
243 async fn record_for(staged: &Path, gates: Vec<GateRecord>) -> String {
244 let computed = bundle::digest_dir(staged).await.unwrap();
245 let manifest = Manifest::parse(&computed.manifest).unwrap();
246 ArtifactRecord::new("bento", manifest, provenance(), gates)
247 .unwrap()
248 .to_json()
249 }
250
251 fn passing_gate() -> GateRecord {
252 GateRecord::new(
253 "prebuild",
254 Scope::Artifact,
255 Verdict::Passed,
256 "prebuild passed in 90s",
257 at(),
258 )
259 }
260
261 /// The load-bearing agreement: Sando's bundle walker and the contract's
262 /// manifest must produce the same text, byte for byte. They are separate
263 /// implementations on either side of the boundary, and a divergence would
264 /// make every honest artifact look corrupt.
265 #[tokio::test]
266 async fn sandos_walker_and_the_contract_manifest_agree_exactly() {
267 let dir = tempfile::tempdir().unwrap();
268 let staged = staged_bundle(dir.path()).await;
269 let computed = bundle::digest_dir(&staged).await.unwrap();
270 let parsed = Manifest::parse(&computed.manifest).unwrap();
271 assert_eq!(parsed.to_text(), computed.manifest);
272 assert_eq!(parsed.digest().as_str(), computed.full);
273 }
274
275 #[tokio::test]
276 async fn a_matching_bundle_is_accepted_and_published_content_addressed() {
277 let dir = tempfile::tempdir().unwrap();
278 let staged = staged_bundle(dir.path()).await;
279 let json = record_for(&staged, vec![passing_gate()]).await;
280
281 let accepted = accept(dir.path(), &staged, &json, &no_pins())
282 .await
283 .unwrap();
284 assert!(accepted.builder_gates_passed());
285 assert!(
286 accepted.released.ends_with(accepted.record.digest.short()),
287 "published at {}, expected a dir named for the digest",
288 accepted.released.display()
289 );
290 // The MANIFEST rides along for node-side verification.
291 assert!(accepted.released.join(bundle::MANIFEST_NAME).exists());
292 // Staging is gone: the bundle was renamed, not copied.
293 assert!(!staged.exists());
294 }
295
296 #[tokio::test]
297 async fn a_bundle_that_drifted_is_refused_and_names_the_file() {
298 // The failure this whole boundary exists to make impossible: evidence
299 // that vouches for one set of bytes arriving with another.
300 let dir = tempfile::tempdir().unwrap();
301 let staged = staged_bundle(dir.path()).await;
302 let json = record_for(&staged, vec![passing_gate()]).await;
303 write(&staged, "static/app.css", b"tampered").await;
304
305 let err = accept(dir.path(), &staged, &json, &no_pins())
306 .await
307 .unwrap_err();
308 match err {
309 IntakeError::ManifestMismatch {
310 first_difference: Some(p),
311 ..
312 } => assert_eq!(p, "static/app.css"),
313 other => panic!("expected a manifest mismatch naming the file, got {other}"),
314 }
315 // Nothing was published.
316 assert!(!dir.path().join("releases").exists());
317 }
318
319 #[tokio::test]
320 async fn an_extra_file_in_the_bundle_is_refused() {
321 // Same bytes for everything the record lists, plus something it does not.
322 // The digest would differ, but the useful error names the intruder.
323 let dir = tempfile::tempdir().unwrap();
324 let staged = staged_bundle(dir.path()).await;
325 let json = record_for(&staged, vec![passing_gate()]).await;
326 write(&staged, "companions/unexpected", b"who put this here").await;
327
328 let err = accept(dir.path(), &staged, &json, &no_pins())
329 .await
330 .unwrap_err();
331 match err {
332 IntakeError::ManifestMismatch {
333 first_difference: Some(p),
334 ..
335 } => assert_eq!(p, "companions/unexpected"),
336 other => panic!("expected the extra file to be named, got {other}"),
337 }
338 }
339
340 #[tokio::test]
341 async fn a_record_that_does_not_parse_is_refused_before_anything_is_hashed() {
342 let dir = tempfile::tempdir().unwrap();
343 let staged = staged_bundle(dir.path()).await;
344 let err = accept(dir.path(), &staged, "{\"not\": \"a record\"}", &no_pins())
345 .await
346 .unwrap_err();
347 assert!(matches!(err, IntakeError::BadRecord(_)), "{err}");
348 assert!(staged.exists(), "a refused intake leaves the bytes alone");
349 }
350
351 #[tokio::test]
352 async fn a_record_whose_digest_was_edited_is_refused() {
353 // Editing the digest to match tampered bytes does not help: the record
354 // validates its own digest against its own manifest first.
355 let dir = tempfile::tempdir().unwrap();
356 let staged = staged_bundle(dir.path()).await;
357 let json = record_for(&staged, vec![passing_gate()]).await;
358 let forged = json.replace(&record_digest(&json), &"f".repeat(64));
359 let err = accept(dir.path(), &staged, &forged, &no_pins())
360 .await
361 .unwrap_err();
362 assert!(matches!(err, IntakeError::BadRecord(_)), "{err}");
363 }
364
365 #[tokio::test]
366 async fn a_failed_builder_gate_is_accepted_and_reported_not_hidden() {
367 // Intake decides identity, not whether the thing may ship. Refusing here
368 // would leave the evidence of a failed build nowhere Sando can see it.
369 let dir = tempfile::tempdir().unwrap();
370 let staged = staged_bundle(dir.path()).await;
371 let failed = GateRecord::new(
372 "prebuild",
373 Scope::Artifact,
374 Verdict::Failed,
375 "12 test(s) failed",
376 at(),
377 );
378 let json = record_for(&staged, vec![failed]).await;
379 let accepted = accept(dir.path(), &staged, &json, &no_pins())
380 .await
381 .unwrap();
382 assert!(!accepted.builder_gates_passed());
383 }
384
385 fn record_digest(json: &str) -> String {
386 let v: serde_json::Value = serde_json::from_str(json).unwrap();
387 v["digest"].as_str().unwrap().to_string()
388 }
389 }
390