| 587 |
587 |
|
/// machine with no access to our infrastructure still builds, so the mirror
|
| 588 |
588 |
|
/// adds a source rather than moving the project onto one.
|
| 589 |
589 |
|
///
|
|
590 |
+ |
/// The mirror attempt is quiet and gives up quickly ([`Attempt::Mirror`]),
|
|
591 |
+ |
/// because it is spent before a request that is going to be made anyway when it
|
|
592 |
+ |
/// fails. Retrying a 404 or printing curl's error would make an absent mirror
|
|
593 |
+ |
/// cost more than no mirror, which is the one thing this must not do.
|
|
594 |
+ |
///
|
| 590 |
595 |
|
/// `expect` is the pinned digest, and it is checked *here* for a mirrored file
|
| 591 |
596 |
|
/// rather than only by the caller. A mirror serving the wrong bytes has to fall
|
| 592 |
597 |
|
/// through to upstream, not fail the build: without that, an out-of-date mirror
|
| 602 |
607 |
|
});
|
| 603 |
608 |
|
}
|
| 604 |
609 |
|
let mirror = mirrored(expect).filter(|mirror| {
|
| 605 |
|
- |
fetch(mirror, path).is_ok()
|
|
610 |
+ |
fetch_with(mirror, path, Attempt::Mirror).is_ok()
|
| 606 |
611 |
|
&& std::fs::read(path).is_ok_and(|bytes| verify(&bytes, expect).is_ok())
|
| 607 |
612 |
|
});
|
| 608 |
613 |
|
if mirror.is_none() {
|
| 696 |
701 |
|
/// `3d41d15a`). [`MIRROR_ENV`] is this end of that: point it at a host we run
|
| 697 |
702 |
|
/// and the pinned files come from there, with upstream still the fallback.
|
| 698 |
703 |
|
fn fetch(url: &str, dest: &Path) -> Result<(), Error> {
|
|
704 |
+ |
fetch_with(url, dest, Attempt::Upstream)
|
|
705 |
+ |
}
|
|
706 |
+ |
|
|
707 |
+ |
/// Which of the two fetches this is, because they want opposite curl flags.
|
|
708 |
+ |
///
|
|
709 |
+ |
/// Split out after the first build against a mirror that was not there yet: the
|
|
710 |
+ |
/// mirror attempt inherited [`Attempt::Upstream`]'s flags and both of them were
|
|
711 |
+ |
/// wrong for it. `--retry-all-errors` retries a 404, so four files the mirror
|
|
712 |
+ |
/// did not have cost five retries and four delays each before the build reached
|
|
713 |
+ |
/// the url that would answer; and `--show-error` printed `curl: (22) ... 404`
|
|
714 |
+ |
/// four times, which reads as a build failing rather than as a fallback working.
|
|
715 |
+ |
#[derive(Clone, Copy)]
|
|
716 |
+ |
enum Attempt {
|
|
717 |
+ |
/// The pinned url. Retry hard: this is the one that has to work, and the
|
|
718 |
+ |
/// failure it is retrying through is somebody else's rate limiter.
|
|
719 |
+ |
Upstream,
|
|
720 |
+ |
/// A mirror, which is allowed not to have the file. Ask once, say nothing,
|
|
721 |
+ |
/// and give up quickly, because every second here is spent before the
|
|
722 |
+ |
/// request that was always going to be made anyway.
|
|
723 |
+ |
Mirror,
|
|
724 |
+ |
}
|
|
725 |
+ |
|
|
726 |
+ |
fn fetch_with(url: &str, dest: &Path, attempt: Attempt) -> Result<(), Error> {
|
| 699 |
727 |
|
if let Some(parent) = dest.parent() {
|
| 700 |
728 |
|
std::fs::create_dir_all(parent).map_err(|e| Error::Io(parent.to_path_buf(), e))?;
|
| 701 |
729 |
|
}
|
| 702 |
730 |
|
let partial = dest.with_extension("part");
|
| 703 |
|
- |
let status = Command::new("curl")
|
| 704 |
|
- |
.args([
|
| 705 |
|
- |
"--fail",
|
| 706 |
|
- |
"--location",
|
| 707 |
|
- |
"--silent",
|
| 708 |
|
- |
"--show-error",
|
| 709 |
|
- |
"--retry",
|
| 710 |
|
- |
"5",
|
| 711 |
|
- |
"--retry-delay",
|
| 712 |
|
- |
"2",
|
| 713 |
|
- |
"--retry-all-errors",
|
| 714 |
|
- |
"--output",
|
| 715 |
|
- |
])
|
|
731 |
+ |
let mut curl = Command::new("curl");
|
|
732 |
+ |
curl.args(["--fail", "--location", "--silent"]);
|
|
733 |
+ |
match attempt {
|
|
734 |
+ |
Attempt::Upstream => {
|
|
735 |
+ |
curl.args([
|
|
736 |
+ |
"--show-error",
|
|
737 |
+ |
"--retry",
|
|
738 |
+ |
"5",
|
|
739 |
+ |
"--retry-delay",
|
|
740 |
+ |
"2",
|
|
741 |
+ |
"--retry-all-errors",
|
|
742 |
+ |
]);
|
|
743 |
+ |
}
|
|
744 |
+ |
Attempt::Mirror => {
|
|
745 |
+ |
curl.args(["--connect-timeout", "10", "--max-time", "120"]);
|
|
746 |
+ |
}
|
|
747 |
+ |
}
|
|
748 |
+ |
let status = curl
|
|
749 |
+ |
.arg("--output")
|
| 716 |
750 |
|
.arg(&partial)
|
| 717 |
751 |
|
.arg(url)
|
| 718 |
752 |
|
.status()
|