| 492 |
492 |
|
for face in &base.faces {
|
| 493 |
493 |
|
let url = face.url.as_deref().unwrap_or_default();
|
| 494 |
494 |
|
let path = cache.join(face.cache_name(&base.id, &base.version));
|
| 495 |
|
- |
let data = cached(url, &path, offline)?;
|
|
495 |
+ |
let data = cached(url, &path, offline, &face.sha256)?;
|
| 496 |
496 |
|
verify(&data, &face.sha256).map_err(|found| Error::ArchiveChecksum {
|
| 497 |
497 |
|
path,
|
| 498 |
498 |
|
url: url.to_owned(),
|
| 510 |
510 |
|
fn load_from_archive(base: &Base, cache: &Path, offline: bool) -> Result<Vec<BaseFace>, Error> {
|
| 511 |
511 |
|
let url = base.url.as_deref().unwrap_or_default();
|
| 512 |
512 |
|
let archive = cache.join(format!("{}-{}.zip", base.id, base.version));
|
| 513 |
|
- |
let bytes = cached(url, &archive, offline)?;
|
|
513 |
+ |
let bytes = cached(
|
|
514 |
+ |
url,
|
|
515 |
+ |
&archive,
|
|
516 |
+ |
offline,
|
|
517 |
+ |
base.sha256.as_deref().unwrap_or_default(),
|
|
518 |
+ |
)?;
|
| 514 |
519 |
|
verify(&bytes, base.sha256.as_deref().unwrap_or_default()).map_err(|found| {
|
| 515 |
520 |
|
Error::ArchiveChecksum {
|
| 516 |
521 |
|
path: archive.clone(),
|
| 541 |
546 |
|
Ok(faces)
|
| 542 |
547 |
|
}
|
| 543 |
548 |
|
|
|
549 |
+ |
/// The environment variable naming a mirror of the pinned files.
|
|
550 |
+ |
///
|
|
551 |
+ |
/// A base URL. Every pinned file is addressed under it by the sha256 the pin
|
|
552 |
+ |
/// already carries, so `QUASI_TYPE_MIRROR=https://example.invalid/bases` looks
|
|
553 |
+ |
/// for `https://example.invalid/bases/<sha256>`.
|
|
554 |
+ |
pub const MIRROR_ENV: &str = "QUASI_TYPE_MIRROR";
|
|
555 |
+ |
|
|
556 |
+ |
/// Where a mirror would hold the file whose pinned digest is `sha256`.
|
|
557 |
+ |
///
|
|
558 |
+ |
/// Content-addressed on purpose, and it is what keeps this from being a new
|
|
559 |
+ |
/// trust assumption: the digest is the one the pin already carries and the
|
|
560 |
+ |
/// bytes are verified against it either way, so a mirror can serve the pinned
|
|
561 |
+ |
/// file or nothing. It cannot serve a different one. That is also why no
|
|
562 |
+ |
/// signature or TLS pinning is wanted here — the integrity guarantee was never
|
|
563 |
+ |
/// the transport.
|
|
564 |
+ |
fn mirrored(sha256: &str) -> Option<String> {
|
|
565 |
+ |
mirror_url(&std::env::var(MIRROR_ENV).ok()?, sha256)
|
|
566 |
+ |
}
|
|
567 |
+ |
|
|
568 |
+ |
/// [`mirrored`] against an explicit base, so the shaping is testable.
|
|
569 |
+ |
///
|
|
570 |
+ |
/// Split out because `set_var` is unsafe in a threaded test binary and a test
|
|
571 |
+ |
/// that sets the mirror under the other tests is a flake waiting for a slow
|
|
572 |
+ |
/// machine.
|
|
573 |
+ |
fn mirror_url(base: &str, sha256: &str) -> Option<String> {
|
|
574 |
+ |
let base = base.trim().trim_end_matches('/');
|
|
575 |
+ |
(!base.is_empty()).then(|| format!("{base}/{sha256}"))
|
|
576 |
+ |
}
|
|
577 |
+ |
|
| 544 |
578 |
|
/// Read a pinned file from the cache, fetching it first if it is not there.
|
| 545 |
|
- |
fn cached(url: &str, path: &Path, offline: bool) -> Result<Vec<u8>, Error> {
|
|
579 |
+ |
///
|
|
580 |
+ |
/// **The mirror is tried first and upstream is the fallback**, which is the
|
|
581 |
+ |
/// order the outage argues for: the pins name raw.githubusercontent.com, which
|
|
582 |
+ |
/// rate-limits by IP, and one clean image build asks it four times. A mirror
|
|
583 |
+ |
/// that is consulted only after a failure would still be paying for the
|
|
584 |
+ |
/// upstream round trip on every build that works.
|
|
585 |
+ |
///
|
|
586 |
+ |
/// Upstream stays reachable, and that is deliberate rather than a leftover: a
|
|
587 |
+ |
/// machine with no access to our infrastructure still builds, so the mirror
|
|
588 |
+ |
/// adds a source rather than moving the project onto one.
|
|
589 |
+ |
///
|
|
590 |
+ |
/// `expect` is the pinned digest, and it is checked *here* for a mirrored file
|
|
591 |
+ |
/// rather than only by the caller. A mirror serving the wrong bytes has to fall
|
|
592 |
+ |
/// through to upstream, not fail the build: without that, an out-of-date mirror
|
|
593 |
+ |
/// would be a worse outcome than no mirror at all. The caller verifies again on
|
|
594 |
+ |
/// the way out, which is unchanged — these fail differently, the same way the
|
|
595 |
+ |
/// archive's two checks do.
|
|
596 |
+ |
fn cached(url: &str, path: &Path, offline: bool, expect: &str) -> Result<Vec<u8>, Error> {
|
| 546 |
597 |
|
if !path.exists() {
|
| 547 |
598 |
|
if offline {
|
| 548 |
599 |
|
return Err(Error::Offline {
|
| 550 |
601 |
|
url: url.to_owned(),
|
| 551 |
602 |
|
});
|
| 552 |
603 |
|
}
|
| 553 |
|
- |
fetch(url, path)?;
|
|
604 |
+ |
let mirror = mirrored(expect).filter(|mirror| {
|
|
605 |
+ |
fetch(mirror, path).is_ok()
|
|
606 |
+ |
&& std::fs::read(path).is_ok_and(|bytes| verify(&bytes, expect).is_ok())
|
|
607 |
+ |
});
|
|
608 |
+ |
if mirror.is_none() {
|
|
609 |
+ |
let _ = std::fs::remove_file(path);
|
|
610 |
+ |
fetch(url, path)?;
|
|
611 |
+ |
}
|
| 554 |
612 |
|
}
|
| 555 |
613 |
|
std::fs::read(path).map_err(|e| Error::Io(path.to_path_buf(), e))
|
| 556 |
614 |
|
}
|
| 560 |
618 |
|
pub fn license_text(base: &Base, cache: &Path, offline: bool) -> Result<Vec<u8>, Error> {
|
| 561 |
619 |
|
if let Some(url) = &base.license_url {
|
| 562 |
620 |
|
let path = cache.join(format!("{}-{}-LICENSE.txt", base.id, base.version));
|
| 563 |
|
- |
let data = cached(url, &path, offline)?;
|
|
621 |
+ |
let data = cached(
|
|
622 |
+ |
url,
|
|
623 |
+ |
&path,
|
|
624 |
+ |
offline,
|
|
625 |
+ |
base.license_sha256.as_deref().unwrap_or_default(),
|
|
626 |
+ |
)?;
|
| 564 |
627 |
|
if let Some(expected) = &base.license_sha256 {
|
| 565 |
628 |
|
verify(&data, expected).map_err(|found| Error::FaceChecksum {
|
| 566 |
629 |
|
path: url.clone(),
|
| 627 |
690 |
|
/// are the other half of what a shared network does to a build.
|
| 628 |
691 |
|
///
|
| 629 |
692 |
|
/// This bounds the damage rather than fixing the cause. The cause is that four
|
| 630 |
|
- |
/// fetches of the same two files leave the machine, and the real answer is a
|
| 631 |
|
- |
/// mirror on infrastructure we own — the same conversation as mirroring the
|
| 632 |
|
- |
/// Fedora base images (GO alloy `ebf30337`, and the fragility itself is alloy
|
| 633 |
|
- |
/// `3d41d15a`).
|
|
693 |
+ |
/// fetches of the same two files leave the machine, and the answer is a mirror
|
|
694 |
+ |
/// on infrastructure we own — the same conversation as mirroring the Fedora
|
|
695 |
+ |
/// base images (GO alloy `ebf30337`, and the fragility itself is alloy
|
|
696 |
+ |
/// `3d41d15a`). [`MIRROR_ENV`] is this end of that: point it at a host we run
|
|
697 |
+ |
/// and the pinned files come from there, with upstream still the fallback.
|
| 634 |
698 |
|
fn fetch(url: &str, dest: &Path) -> Result<(), Error> {
|
| 635 |
699 |
|
if let Some(parent) = dest.parent() {
|
| 636 |
700 |
|
std::fs::create_dir_all(parent).map_err(|e| Error::Io(parent.to_path_buf(), e))?;
|
| 668 |
732 |
|
pub fn cache_dir(root: &Path) -> PathBuf {
|
| 669 |
733 |
|
root.join("bases").join("cache")
|
| 670 |
734 |
|
}
|
|
735 |
+ |
|
|
736 |
+ |
#[cfg(test)]
|
|
737 |
+ |
mod tests {
|
|
738 |
+ |
use super::*;
|
|
739 |
+ |
|
|
740 |
+ |
// The mirror is addressed by the digest the pin already carries, so a
|
|
741 |
+ |
// mirror can serve the pinned file or nothing at all.
|
|
742 |
+ |
#[test]
|
|
743 |
+ |
fn a_mirrored_file_is_addressed_by_its_pinned_digest() {
|
|
744 |
+ |
assert_eq!(
|
|
745 |
+ |
mirror_url("https://example.invalid/bases", "abc123").as_deref(),
|
|
746 |
+ |
Some("https://example.invalid/bases/abc123"),
|
|
747 |
+ |
);
|
|
748 |
+ |
}
|
|
749 |
+ |
|
|
750 |
+ |
// A trailing slash in the variable is the obvious way to write it and must
|
|
751 |
+ |
// not produce a double slash, which some hosts serve and some 404.
|
|
752 |
+ |
#[test]
|
|
753 |
+ |
fn a_trailing_slash_on_the_base_is_absorbed() {
|
|
754 |
+ |
assert_eq!(
|
|
755 |
+ |
mirror_url("https://example.invalid/bases/ ", "abc123").as_deref(),
|
|
756 |
+ |
Some("https://example.invalid/bases/abc123"),
|
|
757 |
+ |
);
|
|
758 |
+ |
}
|
|
759 |
+ |
|
|
760 |
+ |
// Set-but-empty means no mirror, not an empty base URL: an exported
|
|
761 |
+ |
// variable someone cleared reads that way, and building a URL out of it
|
|
762 |
+ |
// would send every fetch at `/abc123` on nothing.
|
|
763 |
+ |
#[test]
|
|
764 |
+ |
fn an_empty_base_is_no_mirror() {
|
|
765 |
+ |
assert!(mirror_url("", "abc123").is_none());
|
|
766 |
+ |
assert!(mirror_url(" ", "abc123").is_none());
|
|
767 |
+ |
}
|
|
768 |
+ |
}
|