Skip to main content

max / quasi-type

37.1 KB · 1041 lines History Blame Raw
1 //! Fetching a pinned base, and measuring the parameters a refit needs.
2 //!
3 //! Nothing here draws. It answers one question about a base face: what are its
4 //! stroke weights, its cell, and the band its own symbols are fitted into.
5
6 use std::collections::BTreeMap;
7 use std::io::Read;
8 use std::path::{Path, PathBuf};
9 use std::process::Command;
10
11 use read_fonts::tables::cmap::{Cmap, CmapSubtable};
12 use read_fonts::types::GlyphId;
13 use read_fonts::{FontRef, TableProvider};
14 use sha2::{Digest, Sha256};
15
16 use crate::Error;
17 use crate::pins::Base;
18
19 /// A base face, verified against its pin and read into memory.
20 pub struct BaseFace {
21 pub style: String,
22 pub bytes: Vec<u8>,
23 }
24
25 /// Everything a drawing recipe is allowed to depend on, measured off the base.
26 ///
27 /// Every field is read from the face rather than configured. That is what makes
28 /// the set a pipeline instead of seven one-offs: point the recipes at a
29 /// different base and the marks refit to its weight and cell.
30 #[derive(Debug, Clone, Copy)]
31 pub struct BaseParams {
32 pub upem: u16,
33 /// The cell width. Marks centre on `advance / 2`.
34 pub advance: u16,
35 pub cap_height: i16,
36 pub x_height: i16,
37 /// `|`'s bbox width: the base's vertical stroke weight.
38 pub stem: i16,
39 /// `-`'s bbox height: the base's horizontal stroke weight.
40 pub stroke: i16,
41 /// `+`'s bbox: the band the base fits its own symbols into.
42 pub band_x0: i16,
43 pub band_x1: i16,
44 pub band_y0: i16,
45 pub band_y1: i16,
46 /// The cell's top, from `hhea`. Positive.
47 ///
48 /// The band is where the base fits its *symbols*; this is the box a
49 /// terminal gives a character. Cell primitives — box drawing, block
50 /// elements — are sized off this and never off the band, because their
51 /// whole job is to meet the cell above and the cell beside them exactly.
52 /// A vertical bar drawn to the band's height would leave a gap at every row
53 /// boundary.
54 ///
55 /// `hhea` rather than OS/2, because `hhea` is what a terminal lays lines
56 /// out with, and it is what `shop`'s shaper takes the baseline from.
57 pub ascent: i16,
58 /// The cell's bottom, from `hhea`. Negative, as the table stores it.
59 pub descent: i16,
60 }
61
62 impl BaseParams {
63 pub fn band_width(&self) -> f64 {
64 f64::from(self.band_x1 - self.band_x0)
65 }
66
67 pub fn band_height(&self) -> f64 {
68 f64::from(self.band_y1 - self.band_y0)
69 }
70
71 pub fn band_center_y(&self) -> f64 {
72 f64::from(self.band_y0 + self.band_y1) / 2.0
73 }
74
75 /// Horizontal centre of the cell, which is not the band's centre in a face
76 /// whose symbols are asymmetric.
77 pub fn center_x(&self) -> f64 {
78 f64::from(self.advance) / 2.0
79 }
80
81 pub fn x_height_center_y(&self) -> f64 {
82 f64::from(self.x_height) / 2.0
83 }
84
85 /// The cell's height: ascender to descender.
86 pub fn cell_height(&self) -> f64 {
87 f64::from(self.ascent - self.descent)
88 }
89
90 /// Where a horizontal rule sits: the middle of the cell, not of the band.
91 ///
92 /// A rule on the band's centre would be near the x-height, so a table
93 /// border would run through the middle of the text beside it rather than
94 /// between the lines.
95 pub fn cell_center_y(&self) -> f64 {
96 f64::from(self.ascent + self.descent) / 2.0
97 }
98 }
99
100 /// The reference glyphs a base must have before it can be refitted against.
101 /// A base missing one cannot be measured, and guessing is worse than refusing.
102 const REFERENCES: [(char, &str); 3] = [
103 ('|', "the vertical stroke weight"),
104 ('-', "the horizontal stroke weight"),
105 ('+', "the symbol band"),
106 ];
107
108 pub fn measure(bytes: &[u8]) -> Result<BaseParams, Error> {
109 let font = FontRef::new(bytes).map_err(|e| Error::Font(format!("base is unreadable: {e}")))?;
110 let head = font.head().map_err(table_err("head"))?;
111 let hhea = font.hhea().map_err(table_err("hhea"))?;
112 let os2 = font.os2().map_err(table_err("OS/2"))?;
113 let hmtx = font.hmtx().map_err(table_err("hmtx"))?;
114 let cmap = font.cmap().map_err(table_err("cmap"))?;
115
116 for (ch, what) in REFERENCES {
117 if lookup(&cmap, ch).is_none() {
118 return Err(Error::UnmeasurableBase {
119 missing: ch,
120 what: what.to_owned(),
121 });
122 }
123 }
124
125 let bar = bbox(&font, lookup(&cmap, '|').unwrap())?;
126 let hyphen = bbox(&font, lookup(&cmap, '-').unwrap())?;
127 let plus = bbox(&font, lookup(&cmap, '+').unwrap())?;
128
129 // A monospace face gives the same advance for every glyph; taking it off a
130 // reference glyph rather than off hhea keeps that assumption checkable.
131 let advance = hmtx
132 .advance(lookup(&cmap, '+').unwrap())
133 .ok_or_else(|| Error::Font("base has no advance for `+`".into()))?;
134
135 let cap_height = os2
136 .s_cap_height()
137 .unwrap_or(bbox_or_zero(&font, &cmap, 'H').3);
138 let x_height = os2.sx_height().unwrap_or(bbox_or_zero(&font, &cmap, 'x').3);
139
140 Ok(BaseParams {
141 upem: head.units_per_em(),
142 advance,
143 cap_height,
144 x_height,
145 stem: bar.2 - bar.0,
146 stroke: hyphen.3 - hyphen.1,
147 band_x0: plus.0,
148 band_x1: plus.2,
149 band_y0: plus.1,
150 band_y1: plus.3,
151 ascent: hhea.ascender().to_i16(),
152 descent: hhea.descender().to_i16(),
153 })
154 }
155
156 /// A base's variation axis, and the style its default instance carries.
157 ///
158 /// One axis only, deliberately. A face with two would need a grid of masters
159 /// and a rule for what happens at the corners, and no base in front of us has
160 /// one; refusing says so instead of interpolating a guess.
161 #[derive(Debug, Clone)]
162 pub struct Variation {
163 pub tag: String,
164 pub min: f32,
165 pub default: f32,
166 pub max: f32,
167 /// The subfamily name of the instance sitting at the axis default.
168 ///
169 /// Read off the face rather than assumed. Atkinson Mono's default is `wght`
170 /// 200 and this reads `ExtraLight`, which is the trap the whole variable
171 /// path is written around.
172 pub default_style: String,
173 }
174
175 /// One location a mark is drawn at.
176 ///
177 /// `peak` is a normalized coordinate and is only ever -1, 0 or 1. `avar` is
178 /// required to map those three to themselves, so a master at an axis end needs
179 /// no `avar` arithmetic and the deltas mean the same thing to every rasteriser.
180 #[derive(Debug, Clone, Copy)]
181 pub struct Master {
182 /// User coordinate, e.g. `wght` 800.
183 pub user: f32,
184 pub peak: f32,
185 }
186
187 impl Variation {
188 /// The default location, which is what the glyph itself is drawn at.
189 pub fn default_master(&self) -> Master {
190 Master {
191 user: self.default,
192 peak: 0.0,
193 }
194 }
195
196 /// The locations that get `gvar` deltas: each end of the axis that is not
197 /// already the default. Atkinson Mono defaults to its own minimum, so it
198 /// has one; a face defaulting to the middle of its range has two.
199 pub fn delta_masters(&self) -> Vec<Master> {
200 let mut out = Vec::new();
201 if self.min < self.default {
202 out.push(Master {
203 user: self.min,
204 peak: -1.0,
205 });
206 }
207 if self.max > self.default {
208 out.push(Master {
209 user: self.max,
210 peak: 1.0,
211 });
212 }
213 out
214 }
215 }
216
217 /// The base's axis, or `None` when the base is static.
218 pub fn variation(bytes: &[u8]) -> Result<Option<Variation>, Error> {
219 use skrifa::MetadataProvider;
220
221 let font =
222 skrifa::FontRef::new(bytes).map_err(|e| Error::Font(format!("base is unreadable: {e}")))?;
223 let axes = font.axes();
224 match axes.len() {
225 0 => return Ok(None),
226 1 => {}
227 n => {
228 return Err(Error::Font(format!(
229 "the base varies on {n} axes. The pipeline draws a master per axis end, \
230 which describes one axis and says nothing about the corners of two. \
231 Decide the master grid before pinning a base like this."
232 )));
233 }
234 }
235 let axis = axes.get(0).expect("one axis");
236 let default = axis.default_value();
237 let default_style = font
238 .named_instances()
239 .iter()
240 .find(|instance| {
241 instance
242 .user_coords()
243 .next()
244 .is_some_and(|c| (c - default).abs() < f32::EPSILON)
245 })
246 .and_then(|instance| {
247 font.localized_strings(instance.subfamily_name_id())
248 .english_or_first()
249 .map(|s| s.chars().collect::<String>())
250 })
251 .ok_or_else(|| {
252 Error::Font(
253 "the base names no instance at its own axis default, so there is no \
254 truthful style name for the face a cut produces"
255 .into(),
256 )
257 })?;
258 Ok(Some(Variation {
259 tag: axis.tag().to_string(),
260 min: axis.min_value(),
261 default,
262 max: axis.max_value(),
263 default_style,
264 }))
265 }
266
267 /// Measure a variable base at one location on its axis.
268 ///
269 /// The static path reads bounding boxes straight out of `glyf`, which is only
270 /// the default instance. Here the outline is drawn at the location first, so
271 /// `stroke`, `stem` and the band are the base's real measurements at that
272 /// weight rather than at its default one.
273 pub fn measure_at(bytes: &[u8], variation: &Variation, at: Master) -> Result<BaseParams, Error> {
274 use skrifa::MetadataProvider;
275 use skrifa::instance::Size;
276
277 let font =
278 skrifa::FontRef::new(bytes).map_err(|e| Error::Font(format!("base is unreadable: {e}")))?;
279 let tag = skrifa::Tag::new_checked(variation.tag.as_bytes())
280 .map_err(|_| Error::Font(format!("`{}` is not an axis tag", variation.tag)))?;
281 let location = font.axes().location([(tag, at.user)]);
282 let charmap = font.charmap();
283 let outlines = font.outline_glyphs();
284
285 let measured = |ch: char, what: &str| -> Result<Extent, Error> {
286 let gid = charmap.map(ch).ok_or_else(|| Error::UnmeasurableBase {
287 missing: ch,
288 what: what.to_owned(),
289 })?;
290 let glyph = outlines.get(gid).ok_or_else(|| Error::UnmeasurableBase {
291 missing: ch,
292 what: what.to_owned(),
293 })?;
294 let mut pen = Extent::default();
295 glyph
296 .draw(
297 skrifa::outline::DrawSettings::unhinted(Size::unscaled(), &location),
298 &mut pen,
299 )
300 .map_err(|e| Error::Font(format!("could not draw `{ch}` at {}: {e}", at.user)))?;
301 if pen.empty() {
302 return Err(Error::UnmeasurableBase {
303 missing: ch,
304 what: what.to_owned(),
305 });
306 }
307 Ok(pen)
308 };
309
310 let bar = measured('|', "the vertical stroke weight")?;
311 let hyphen = measured('-', "the horizontal stroke weight")?;
312 let plus = measured('+', "the symbol band")?;
313
314 let metrics = font.metrics(Size::unscaled(), &location);
315 let advance = font
316 .glyph_metrics(Size::unscaled(), &location)
317 .advance_width(charmap.map('+').expect("`+` was measured above"))
318 .ok_or_else(|| Error::Font("base has no advance for `+`".into()))?;
319
320 Ok(BaseParams {
321 upem: metrics.units_per_em,
322 advance: advance.round() as u16,
323 cap_height: round_i16(metrics.cap_height.unwrap_or(0.0)),
324 x_height: round_i16(metrics.x_height.unwrap_or(0.0)),
325 stem: bar.width(),
326 stroke: hyphen.height(),
327 band_x0: round_i16(plus.x0),
328 band_x1: round_i16(plus.x1),
329 band_y0: round_i16(plus.y0),
330 band_y1: round_i16(plus.y1),
331 // The cell does not vary with the axis, and must not: a face whose line
332 // height moved with its weight would reflow a terminal on a bold
333 // heading. Taken from the same metrics call as everything else so it
334 // stays one read of the face.
335 ascent: round_i16(metrics.ascent),
336 descent: round_i16(metrics.descent),
337 })
338 }
339
340 /// A drawn outline's extent, accumulated straight off the pen.
341 #[derive(Debug, Clone, Copy)]
342 struct Extent {
343 x0: f32,
344 y0: f32,
345 x1: f32,
346 y1: f32,
347 }
348
349 impl Default for Extent {
350 fn default() -> Self {
351 Self {
352 x0: f32::MAX,
353 y0: f32::MAX,
354 x1: f32::MIN,
355 y1: f32::MIN,
356 }
357 }
358 }
359
360 impl Extent {
361 fn empty(self) -> bool {
362 self.x0 > self.x1 || self.y0 > self.y1
363 }
364
365 fn width(self) -> i16 {
366 round_i16(self.x1 - self.x0)
367 }
368
369 fn height(self) -> i16 {
370 round_i16(self.y1 - self.y0)
371 }
372
373 fn add(&mut self, x: f32, y: f32) {
374 self.x0 = self.x0.min(x);
375 self.y0 = self.y0.min(y);
376 self.x1 = self.x1.max(x);
377 self.y1 = self.y1.max(y);
378 }
379 }
380
381 /// Control points count toward the extent the same way the stored `glyf`
382 /// bounding box counts them, so the two measurement paths agree on a base that
383 /// happens to be readable both ways.
384 impl skrifa::outline::OutlinePen for Extent {
385 fn move_to(&mut self, x: f32, y: f32) {
386 self.add(x, y);
387 }
388
389 fn line_to(&mut self, x: f32, y: f32) {
390 self.add(x, y);
391 }
392
393 fn quad_to(&mut self, cx0: f32, cy0: f32, x: f32, y: f32) {
394 self.add(cx0, cy0);
395 self.add(x, y);
396 }
397
398 fn curve_to(&mut self, cx0: f32, cy0: f32, cx1: f32, cy1: f32, x: f32, y: f32) {
399 self.add(cx0, cy0);
400 self.add(cx1, cy1);
401 self.add(x, y);
402 }
403
404 fn close(&mut self) {}
405 }
406
407 fn round_i16(value: f32) -> i16 {
408 value
409 .round()
410 .clamp(f32::from(i16::MIN), f32::from(i16::MAX)) as i16
411 }
412
413 fn table_err(tag: &'static str) -> impl Fn(read_fonts::ReadError) -> Error {
414 move |e| Error::Font(format!("base has no readable `{tag}` table: {e}"))
415 }
416
417 /// The Unicode subtable a base maps through, preferring full-repertoire.
418 pub fn best_subtable<'a>(cmap: &Cmap<'a>) -> Option<CmapSubtable<'a>> {
419 let mut best: Option<(u8, CmapSubtable<'a>)> = None;
420 for record in cmap.encoding_records() {
421 use read_fonts::tables::cmap::PlatformId::{Unicode, Windows};
422 let rank = match (record.platform_id(), record.encoding_id()) {
423 (Windows, 10) => 4,
424 (Unicode, 4 | 6) => 3,
425 (Windows, 1) => 2,
426 (Unicode, 0..=3) => 1,
427 _ => continue,
428 };
429 let Ok(subtable) = record.subtable(cmap.offset_data()) else {
430 continue;
431 };
432 if best.as_ref().is_none_or(|(r, _)| rank > *r) {
433 best = Some((rank, subtable));
434 }
435 }
436 best.map(|(_, s)| s)
437 }
438
439 fn lookup(cmap: &Cmap<'_>, ch: char) -> Option<GlyphId> {
440 cmap.map_codepoint(ch)
441 }
442
443 /// Every codepoint the base maps, so the rebuilt cmap keeps all of it.
444 pub fn mappings(bytes: &[u8]) -> Result<BTreeMap<u32, GlyphId>, Error> {
445 let font = FontRef::new(bytes).map_err(|e| Error::Font(format!("base is unreadable: {e}")))?;
446 let cmap = font.cmap().map_err(table_err("cmap"))?;
447 let subtable = best_subtable(&cmap)
448 .ok_or_else(|| Error::Font("base has no Unicode cmap subtable".into()))?;
449 let mut out = BTreeMap::new();
450 for (codepoint, gid) in subtable.iter() {
451 if gid.to_u32() != 0 && char::from_u32(codepoint).is_some() {
452 out.insert(codepoint, gid);
453 }
454 }
455 Ok(out)
456 }
457
458 /// `(x_min, y_min, x_max, y_max)` for a glyph, composites included.
459 fn bbox(font: &FontRef<'_>, gid: GlyphId) -> Result<(i16, i16, i16, i16), Error> {
460 let loca = font.loca(None).map_err(table_err("loca"))?;
461 let glyf = font.glyf().map_err(table_err("glyf"))?;
462 let glyph = loca
463 .get_glyf(gid, &glyf)
464 .map_err(|e| Error::Font(format!("unreadable glyph {gid}: {e}")))?
465 .ok_or_else(|| Error::Font(format!("glyph {gid} is empty")))?;
466 Ok(match glyph {
467 read_fonts::tables::glyf::Glyph::Simple(g) => (g.x_min(), g.y_min(), g.x_max(), g.y_max()),
468 read_fonts::tables::glyf::Glyph::Composite(g) => {
469 (g.x_min(), g.y_min(), g.x_max(), g.y_max())
470 }
471 })
472 }
473
474 fn bbox_or_zero(font: &FontRef<'_>, cmap: &Cmap<'_>, ch: char) -> (i16, i16, i16, i16) {
475 lookup(cmap, ch)
476 .and_then(|gid| bbox(font, gid).ok())
477 .unwrap_or((0, 0, 0, 0))
478 }
479
480 /// Fetch the pinned archive if it is not cached, verify it, and read out the
481 /// faces the pin names.
482 ///
483 /// The archive is verified before anything is read out of it, and each face is
484 /// verified again on the way out. Two checks rather than one because they fail
485 /// differently: the first says upstream moved, the second says the pin names a
486 /// path that no longer holds what it did.
487 pub fn load(base: &Base, cache: &Path, offline: bool) -> Result<Vec<BaseFace>, Error> {
488 if base.is_archive() {
489 return load_from_archive(base, cache, offline);
490 }
491 let mut faces = Vec::new();
492 for face in &base.faces {
493 let url = face.url.as_deref().unwrap_or_default();
494 let path = cache.join(face.cache_name(&base.id, &base.version));
495 let data = cached(url, &path, offline, &face.sha256)?;
496 verify(&data, &face.sha256).map_err(|found| Error::ArchiveChecksum {
497 path,
498 url: url.to_owned(),
499 expected: face.sha256.clone(),
500 found,
501 })?;
502 faces.push(BaseFace {
503 style: face.style.clone(),
504 bytes: data,
505 });
506 }
507 Ok(faces)
508 }
509
510 fn load_from_archive(base: &Base, cache: &Path, offline: bool) -> Result<Vec<BaseFace>, Error> {
511 let url = base.url.as_deref().unwrap_or_default();
512 let archive = cache.join(format!("{}-{}.zip", base.id, base.version));
513 let bytes = cached(
514 url,
515 &archive,
516 offline,
517 base.sha256.as_deref().unwrap_or_default(),
518 )?;
519 verify(&bytes, base.sha256.as_deref().unwrap_or_default()).map_err(|found| {
520 Error::ArchiveChecksum {
521 path: archive.clone(),
522 url: url.to_owned(),
523 expected: base.sha256.clone().unwrap_or_default(),
524 found,
525 }
526 })?;
527
528 let cursor = std::io::Cursor::new(&bytes);
529 let mut zip = zip::ZipArchive::new(cursor)
530 .map_err(|e| Error::Archive(format!("{} is not a readable zip: {e}", archive.display())))?;
531
532 let mut faces = Vec::new();
533 for face in &base.faces {
534 let path = face.path.as_deref().unwrap_or_default();
535 let data = read_entry(&mut zip, path)?;
536 verify(&data, &face.sha256).map_err(|found| Error::FaceChecksum {
537 path: path.to_owned(),
538 expected: face.sha256.clone(),
539 found,
540 })?;
541 faces.push(BaseFace {
542 style: face.style.clone(),
543 bytes: data,
544 });
545 }
546 Ok(faces)
547 }
548
549 /// The environment variable naming a mirror of the pinned files.
550 ///
551 /// A base URL. Every pinned file is addressed under it by the sha256 the pin
552 /// already carries, so `QUASI_TYPE_MIRROR=https://example.invalid/bases` looks
553 /// for `https://example.invalid/bases/<sha256>`.
554 pub const MIRROR_ENV: &str = "QUASI_TYPE_MIRROR";
555
556 /// Where a mirror holds the file whose pinned digest is `sha256`.
557 ///
558 /// Content-addressed on purpose, and it is what keeps this from being a new
559 /// trust assumption: the digest is the one the pin already carries and the
560 /// bytes are verified against it either way, so a mirror can serve the pinned
561 /// file or nothing. It cannot serve a different one. That is also why no
562 /// signature or TLS pinning is wanted here: the integrity guarantee was never
563 /// the transport.
564 ///
565 /// Takes the base rather than reading [`MIRROR_ENV`], so the shaping and the
566 /// order [`cached_from`] tries its two sources in are both testable without
567 /// `set_var`, which is unsafe in a threaded test binary and would set the
568 /// mirror under every other test in the run.
569 fn mirror_url(base: &str, sha256: &str) -> Option<String> {
570 let base = base.trim().trim_end_matches('/');
571 (!base.is_empty()).then(|| format!("{base}/{sha256}"))
572 }
573
574 /// Read a pinned file from the cache, fetching it first if it is not there.
575 ///
576 /// **The mirror is tried first and upstream is the fallback**, which is the
577 /// order the outage argues for: the pins name raw.githubusercontent.com, which
578 /// rate-limits by IP, and one clean image build asks it four times. A mirror
579 /// that is consulted only after a failure would still be paying for the
580 /// upstream round trip on every build that works.
581 ///
582 /// Upstream stays reachable, and that is deliberate rather than a leftover: a
583 /// machine with no access to our infrastructure still builds, so the mirror
584 /// adds a source rather than moving the project onto one.
585 ///
586 /// The mirror attempt is quiet and gives up quickly ([`Attempt::Mirror`]),
587 /// because it is spent before a request that is going to be made anyway when it
588 /// fails. Retrying a 404 or printing curl's error would make an absent mirror
589 /// cost more than no mirror, which is the one thing this must not do.
590 ///
591 /// `expect` is the pinned digest, and it is checked *here* for a mirrored file
592 /// rather than only by the caller. A mirror serving the wrong bytes has to fall
593 /// through to upstream, not fail the build: without that, an out-of-date mirror
594 /// would be a worse outcome than no mirror at all. The caller verifies again on
595 /// the way out, which is unchanged — these fail differently, the same way the
596 /// archive's two checks do.
597 fn cached(url: &str, path: &Path, offline: bool, expect: &str) -> Result<Vec<u8>, Error> {
598 let mirror = std::env::var(MIRROR_ENV).ok();
599 cached_from(mirror.as_deref(), url, path, offline, expect)
600 }
601
602 /// [`cached`] against an explicit mirror base, which is where the two sources
603 /// are actually ordered.
604 fn cached_from(
605 mirror: Option<&str>,
606 url: &str,
607 path: &Path,
608 offline: bool,
609 expect: &str,
610 ) -> Result<Vec<u8>, Error> {
611 if !path.exists() {
612 if offline {
613 return Err(Error::Offline {
614 wanted: path.to_path_buf(),
615 url: url.to_owned(),
616 });
617 }
618 let mirror = mirror
619 .and_then(|base| mirror_url(base, expect))
620 .filter(|mirror| {
621 fetch_with(mirror, path, Attempt::Mirror).is_ok()
622 && std::fs::read(path).is_ok_and(|bytes| verify(&bytes, expect).is_ok())
623 });
624 if mirror.is_none() {
625 let _ = std::fs::remove_file(path);
626 fetch(url, path)?;
627 }
628 }
629 std::fs::read(path).map_err(|e| Error::Io(path.to_path_buf(), e))
630 }
631
632 /// The upstream licence text, which travels with every build the OFL requires
633 /// it to.
634 pub fn license_text(base: &Base, cache: &Path, offline: bool) -> Result<Vec<u8>, Error> {
635 if let Some(url) = &base.license_url {
636 let path = cache.join(format!("{}-{}-LICENSE.txt", base.id, base.version));
637 let data = cached(
638 url,
639 &path,
640 offline,
641 base.license_sha256.as_deref().unwrap_or_default(),
642 )?;
643 if let Some(expected) = &base.license_sha256 {
644 verify(&data, expected).map_err(|found| Error::FaceChecksum {
645 path: url.clone(),
646 expected: expected.clone(),
647 found,
648 })?;
649 }
650 return Ok(data);
651 }
652 let archive = cache.join(format!("{}-{}.zip", base.id, base.version));
653 let bytes = std::fs::read(&archive).map_err(|e| Error::Io(archive.clone(), e))?;
654 let cursor = std::io::Cursor::new(&bytes);
655 let mut zip = zip::ZipArchive::new(cursor)
656 .map_err(|e| Error::Archive(format!("{} is not a readable zip: {e}", archive.display())))?;
657 read_entry(&mut zip, base.license_path.as_deref().unwrap_or_default())
658 }
659
660 fn read_entry<R: std::io::Read + std::io::Seek>(
661 zip: &mut zip::ZipArchive<R>,
662 path: &str,
663 ) -> Result<Vec<u8>, Error> {
664 let mut entry = zip
665 .by_name(path)
666 .map_err(|_| Error::Archive(format!("the pin names `{path}`, which the archive lacks")))?;
667 let mut data = Vec::new();
668 entry
669 .read_to_end(&mut data)
670 .map_err(|e| Error::Archive(format!("`{path}` is unreadable: {e}")))?;
671 Ok(data)
672 }
673
674 fn verify(bytes: &[u8], expected: &str) -> Result<(), String> {
675 let found = hex(&Sha256::digest(bytes));
676 if found == expected {
677 Ok(())
678 } else {
679 Err(found)
680 }
681 }
682
683 pub fn hex(bytes: &[u8]) -> String {
684 use std::fmt::Write;
685 bytes.iter().fold(String::new(), |mut out, b| {
686 let _ = write!(out, "{b:02x}");
687 out
688 })
689 }
690
691 /// Fetching shells out to curl rather than linking an HTTP client.
692 ///
693 /// This is a build-time tool that downloads one pinned URL. A TLS stack would
694 /// be the largest thing in the dependency tree and would drag in the crypto
695 /// provider question for no gain: the integrity guarantee here is the sha256
696 /// below, not the transport.
697 ///
698 /// **It retries, and that is not defensive coding.** The pins point at
699 /// raw.githubusercontent.com, which rate-limits by IP, and a machine that
700 /// builds Alloy asks it four times in one build: the image cuts two slots, and
701 /// shop's build script cuts a third face into its own cache.
702 ///
703 /// This bounds the damage rather than fixing the cause. The cause is that four
704 /// fetches of the same two files leave the machine, and the answer is a mirror
705 /// on infrastructure we own — the same conversation as mirroring the Fedora
706 /// base images (GO alloy `ebf30337`, and the fragility itself is alloy
707 /// `3d41d15a`). [`MIRROR_ENV`] is this end of that: point it at a host we run
708 /// and the pinned files come from there, with upstream still the fallback.
709 fn fetch(url: &str, dest: &Path) -> Result<(), Error> {
710 fetch_with(url, dest, Attempt::Upstream)
711 }
712
713 /// Which of the two fetches this is, because they want opposite curl flags.
714 ///
715 /// Split out after the first build against a mirror that was not there yet: the
716 /// mirror attempt inherited [`Attempt::Upstream`]'s flags and both of them were
717 /// wrong for it. `--retry-all-errors` retries a 404, so four files the mirror
718 /// did not have cost five retries and four delays each before the build reached
719 /// the url that would answer; and `--show-error` printed `curl: (22) ... 404`
720 /// four times, which reads as a build failing rather than as a fallback working.
721 #[derive(Clone, Copy)]
722 enum Attempt {
723 /// The pinned url. Retry hard: this is the one that has to work, and the
724 /// failure it is retrying through is somebody else's rate limiter.
725 Upstream,
726 /// A mirror, which is allowed not to have the file. Ask once, say nothing,
727 /// and give up quickly, because every second here is spent before the
728 /// request that was always going to be made anyway.
729 Mirror,
730 }
731
732 fn fetch_with(url: &str, dest: &Path, attempt: Attempt) -> Result<(), Error> {
733 if let Some(parent) = dest.parent() {
734 std::fs::create_dir_all(parent).map_err(|e| Error::Io(parent.to_path_buf(), e))?;
735 }
736 let partial = dest.with_extension("part");
737 let mut curl = Command::new("curl");
738 curl.args(["--fail", "--location", "--silent"]);
739 match attempt {
740 Attempt::Upstream => {
741 curl.args([
742 "--show-error",
743 "--retry",
744 "5",
745 "--retry-delay",
746 "2",
747 "--retry-all-errors",
748 ]);
749 }
750 Attempt::Mirror => {
751 curl.args(["--connect-timeout", "10", "--max-time", "120"]);
752 }
753 }
754 let status = curl
755 .arg("--output")
756 .arg(&partial)
757 .arg(url)
758 .status()
759 .map_err(|e| Error::Fetch(format!("could not run curl: {e}")))?;
760 if !status.success() {
761 let _ = std::fs::remove_file(&partial);
762 return Err(Error::Fetch(format!(
763 "curl failed on {url} ({status}), after retrying. A 429 here is this \
764 machine's IP being rate-limited by the host the base is pinned at, and \
765 the fix is a mirror rather than another retry."
766 )));
767 }
768 std::fs::rename(&partial, dest).map_err(|e| Error::Io(dest.to_path_buf(), e))?;
769 Ok(())
770 }
771
772 pub fn cache_dir(root: &Path) -> PathBuf {
773 root.join("bases").join("cache")
774 }
775
776 #[cfg(test)]
777 mod tests {
778 use super::*;
779
780 use std::collections::HashMap;
781 use std::io::{BufRead, BufReader, Write};
782 use std::net::{TcpListener, TcpStream};
783 use std::sync::{Arc, Mutex};
784
785 /// An HTTP server that answers a fixed table of paths and records every
786 /// path it was asked for.
787 ///
788 /// Real sockets and the real `curl` invocation, because what is under test
789 /// is which host the bytes came from, and a stubbed fetch would be a test
790 /// of the stub. The table is exact: a path the test did not name gets a
791 /// 404, so a mirror URL shaped wrongly reaches the same fallback a mirror
792 /// that lacks the file does.
793 struct Stub {
794 base: String,
795 asked: Arc<Mutex<Vec<String>>>,
796 }
797
798 impl Stub {
799 fn new(routes: &[(&str, &[u8])]) -> Self {
800 let listener = TcpListener::bind("127.0.0.1:0").expect("binding a stub server");
801 let port = listener.local_addr().expect("stub address").port();
802 let asked: Arc<Mutex<Vec<String>>> = Arc::new(Mutex::new(Vec::new()));
803 let table: HashMap<String, Vec<u8>> = routes
804 .iter()
805 .map(|(path, body)| ((*path).to_owned(), body.to_vec()))
806 .collect();
807 let log = Arc::clone(&asked);
808 std::thread::spawn(move || {
809 for stream in listener.incoming() {
810 let Ok(stream) = stream else { continue };
811 Self::answer(stream, &table, &log);
812 }
813 });
814 Self {
815 base: format!("http://127.0.0.1:{port}/bases"),
816 asked,
817 }
818 }
819
820 fn answer(
821 mut stream: TcpStream,
822 table: &HashMap<String, Vec<u8>>,
823 log: &Mutex<Vec<String>>,
824 ) {
825 let mut request = String::new();
826 let mut reader = BufReader::new(stream.try_clone().expect("cloning the socket"));
827 loop {
828 let mut line = String::new();
829 if reader.read_line(&mut line).unwrap_or(0) == 0 || line.trim().is_empty() {
830 break;
831 }
832 if request.is_empty() {
833 request = line;
834 }
835 }
836 let path = request.split_whitespace().nth(1).unwrap_or("").to_owned();
837 log.lock().expect("the request log").push(path.clone());
838 let response = match table.get(&path) {
839 Some(body) => {
840 let mut head = format!(
841 "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
842 body.len(),
843 )
844 .into_bytes();
845 head.extend_from_slice(body);
846 head
847 }
848 None => b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
849 .to_vec(),
850 };
851 let _ = stream.write_all(&response);
852 let _ = stream.flush();
853 }
854
855 /// The URL a pin would carry, for the server standing in for upstream.
856 fn url(&self, path: &str) -> String {
857 format!("{}{path}", self.base.trim_end_matches("/bases"))
858 }
859
860 fn asked(&self) -> Vec<String> {
861 self.asked.lock().expect("the request log").clone()
862 }
863 }
864
865 /// A cache directory that removes itself, so a failing assertion leaves no
866 /// tree behind.
867 struct Cache(PathBuf);
868
869 impl Cache {
870 fn new(label: &str) -> Self {
871 let dir = std::env::temp_dir()
872 .join(format!("quasi-type-mirror-{label}-{}", std::process::id()));
873 let _ = std::fs::remove_dir_all(&dir);
874 std::fs::create_dir_all(&dir).expect("cache dir");
875 Self(dir)
876 }
877
878 fn file(&self) -> PathBuf {
879 self.0.join("atkinson-mono-2.001-LICENSE.txt")
880 }
881 }
882
883 impl Drop for Cache {
884 fn drop(&mut self) {
885 let _ = std::fs::remove_dir_all(&self.0);
886 }
887 }
888
889 // The mirror is addressed by the digest the pin already carries, so a
890 // mirror can serve the pinned file or nothing at all.
891 #[test]
892 fn a_mirrored_file_is_addressed_by_its_pinned_digest() {
893 assert_eq!(
894 mirror_url("https://example.invalid/bases", "abc123").as_deref(),
895 Some("https://example.invalid/bases/abc123"),
896 );
897 }
898
899 // A trailing slash in the variable is the obvious way to write it and must
900 // not produce a double slash, which some hosts serve and some 404.
901 #[test]
902 fn a_trailing_slash_on_the_base_is_absorbed() {
903 assert_eq!(
904 mirror_url("https://example.invalid/bases/ ", "abc123").as_deref(),
905 Some("https://example.invalid/bases/abc123"),
906 );
907 }
908
909 // Set-but-empty means no mirror, not an empty base URL: an exported
910 // variable someone cleared reads that way, and building a URL out of it
911 // would send every fetch at `/abc123` on nothing.
912 #[test]
913 fn an_empty_base_is_no_mirror() {
914 assert!(mirror_url("", "abc123").is_none());
915 assert!(mirror_url(" ", "abc123").is_none());
916 }
917
918 // The mirror is the first source and upstream is not asked at all when it
919 // answers. Ordering is the whole point of the mirror: upstream rate-limits
920 // by IP, and a mirror consulted only after a failure would still pay for
921 // the upstream round trip on every build that works. Reverse the two and
922 // the upstream server here records a request.
923 #[test]
924 fn a_mirrored_file_is_taken_from_the_mirror_and_upstream_is_not_asked() {
925 let bytes = b"the pinned licence text";
926 let digest = hex(&Sha256::digest(bytes));
927 let mirror = Stub::new(&[(&format!("/bases/{digest}"), bytes)]);
928 let upstream = Stub::new(&[("/mono/OFL.txt", bytes)]);
929 let cache = Cache::new("hit");
930
931 let got = cached_from(
932 Some(&format!("{}/", mirror.base)),
933 &upstream.url("/mono/OFL.txt"),
934 &cache.file(),
935 false,
936 &digest,
937 )
938 .expect("the mirrored file");
939
940 assert_eq!(got, bytes, "the bytes are not the ones the mirror served");
941 assert_eq!(
942 mirror.asked(),
943 vec![format!("/bases/{digest}")],
944 "the mirror was asked for something other than the pinned digest",
945 );
946 assert!(
947 upstream.asked().is_empty(),
948 "upstream was asked for a file the mirror had: {:?}",
949 upstream.asked(),
950 );
951 assert_eq!(
952 std::fs::read(cache.file()).expect("the cached file"),
953 bytes,
954 "the mirrored bytes did not land in the cache",
955 );
956 }
957
958 // A mirror serving the wrong bytes falls through to upstream rather than
959 // failing the build, or an out-of-date mirror would be worse than no
960 // mirror. The digest is checked here and again by the caller.
961 #[test]
962 fn a_mirror_serving_the_wrong_bytes_falls_through_to_upstream() {
963 let bytes = b"the pinned licence text";
964 let digest = hex(&Sha256::digest(bytes));
965 let mirror = Stub::new(&[(&format!("/bases/{digest}"), b"an older licence")]);
966 let upstream = Stub::new(&[("/mono/OFL.txt", bytes)]);
967 let cache = Cache::new("wrong-bytes");
968
969 let got = cached_from(
970 Some(&mirror.base),
971 &upstream.url("/mono/OFL.txt"),
972 &cache.file(),
973 false,
974 &digest,
975 )
976 .expect("the upstream file");
977
978 assert_eq!(got, bytes, "the wrong bytes were kept");
979 assert_eq!(
980 upstream.asked(),
981 vec!["/mono/OFL.txt".to_owned()],
982 "the fallback did not reach upstream",
983 );
984 assert_eq!(
985 std::fs::read(cache.file()).expect("the cached file"),
986 bytes,
987 "the mirror's bytes were left in the cache for the caller to verify",
988 );
989 }
990
991 // A mirror that does not hold the file is a slower build and not a broken
992 // one, which is what lets the variable default to a host that may be down.
993 #[test]
994 fn a_mirror_without_the_file_falls_through_to_upstream() {
995 let bytes = b"the pinned licence text";
996 let digest = hex(&Sha256::digest(bytes));
997 let mirror = Stub::new(&[]);
998 let upstream = Stub::new(&[("/mono/OFL.txt", bytes)]);
999 let cache = Cache::new("miss");
1000
1001 let got = cached_from(
1002 Some(&mirror.base),
1003 &upstream.url("/mono/OFL.txt"),
1004 &cache.file(),
1005 false,
1006 &digest,
1007 )
1008 .expect("the upstream file");
1009
1010 assert_eq!(got, bytes);
1011 assert_eq!(
1012 mirror.asked(),
1013 vec![format!("/bases/{digest}")],
1014 "the mirror was not asked first",
1015 );
1016 assert_eq!(upstream.asked(), vec!["/mono/OFL.txt".to_owned()]);
1017 }
1018
1019 // No mirror named is no request, which is what `QUASI_TYPE_MIRROR=` in a
1020 // build is asking for.
1021 #[test]
1022 fn no_mirror_named_asks_only_upstream() {
1023 let bytes = b"the pinned licence text";
1024 let digest = hex(&Sha256::digest(bytes));
1025 let upstream = Stub::new(&[("/mono/OFL.txt", bytes)]);
1026 let cache = Cache::new("no-mirror");
1027
1028 let got = cached_from(
1029 None,
1030 &upstream.url("/mono/OFL.txt"),
1031 &cache.file(),
1032 false,
1033 &digest,
1034 )
1035 .expect("the upstream file");
1036
1037 assert_eq!(got, bytes);
1038 assert_eq!(upstream.asked(), vec!["/mono/OFL.txt".to_owned()]);
1039 }
1040 }
1041