Skip to main content

max / makenotwork

Ship the base mirror as tarballs, not a registry tree Reverts the registry machinery added earlier today. A static registry tree does work — measured, podman 4.9.3 pulls from plain files by tag and by digest over three GET shapes — but it was built without pricing the simpler option ebf30337 asked for, and the simpler option wins. No installed machine ever pulls the base. Alloy publishes no images, so the only consumer is a build script we write, and the registry protocol buys being pullable by anything from anywhere. Against that it costs: /v2/ at the host root dictating the bucket's top-level layout, colons and extensionless digest names in the keys, a load-bearing manifest Content-Type whose failure reads as malformed JSON, and a front rule to answer a path a bucket cannot hold. So the mirror is one .tar per image per arch, fetched by URL and checked against a digest the alloy repo records. What that removes: - generate_artifact_key goes back to a plain extension allowlist, with tar added. No colon in the segment alphabet, no extensionless exemptions. Registry-shaped keys are now asserted to be refused. - MUTABLE_INDEX_NAMES is one entry again. repomd.xml is the only index the store holds; an archive is immutable under its own name. - mnw-cli loses --kind and the manifest media-type reader. The repodata warning is inferred from the tree holding packages rather than asked for on the command line. The one-store consolidation, the uploader, the placement policy and the hotfix half are untouched. Wiki mnw-package-hosting is rewritten as one current statement rather than a ruling plus corrections.
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session
https://claude.ai/code/session_01J1yAFMokhGYCs3UT8UaCZr
Author: Max Johnson <me@maxj.phd> · 2026-09-07 21:33 UTC
Signed with PGP, not checked
Commit: 3bb3f2796d2806670172d4594c15c43f7c07cb94
Parent: b4a8310
9 files changed, +190 insertions, -459 deletions
M mnw-cli/Cargo.lock +20 -20
@@ -1947,7 +1947,7 @@
1947 1947
1948 1948 [[package]]
1949 1949 name = "mnw-cli"
1950 - version = "0.1.5"
1950 + version = "0.1.6"
1951 1951 dependencies = [
1952 1952 "anyhow",
1953 1953 "bytes",
@@ -4777,6 +4777,22 @@
4777 4777 source = "registry+https://github.com/rust-lang/crates.io-index"
4778 4778 checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
4779 4779
4780 + [[patch.unused]]
4781 + name = "docengine"
4782 + version = "0.7.0"
4783 +
4784 + [[patch.unused]]
4785 + name = "makeover-build"
4786 + version = "0.67.0"
4787 +
4788 + [[patch.unused]]
4789 + name = "makeover-webview"
4790 + version = "0.74.0"
4791 +
4792 + [[patch.unused]]
4793 + name = "quasi-type"
4794 + version = "0.1.3"
4795 +
4780 4796 [[patch.unused]]
4781 4797 name = "kberg"
4782 4798 version = "0.1.0"
@@ -4794,20 +4810,8 @@
4794 4810 version = "0.4.1"
4795 4811
4796 4812 [[patch.unused]]
4797 - name = "quasi-type"
4798 - version = "0.1.3"
4799 -
4800 - [[patch.unused]]
4801 - name = "docengine"
4802 - version = "0.7.0"
4803 -
4804 - [[patch.unused]]
4805 - name = "makeover-build"
4806 - version = "0.67.0"
4807 -
4808 - [[patch.unused]]
4809 - name = "makeover-webview"
4810 - version = "0.74.0"
4813 + name = "synckit-config"
4814 + version = "0.2.0"
4811 4815
4812 4816 [[patch.unused]]
4813 4817 name = "quasi-axum"
@@ -4819,7 +4823,7 @@
4819 4823
4820 4824 [[patch.unused]]
4821 4825 name = "quasi-declare"
4822 - version = "0.1.11"
4826 + version = "0.1.12"
4823 4827
4824 4828 [[patch.unused]]
4825 4829 name = "quasi-http"
@@ -4848,7 +4852,3 @@
4848 4852 [[patch.unused]]
4849 4853 name = "quasi-webview"
4850 4854 version = "0.103.0"
4851 -
4852 - [[patch.unused]]
4853 - name = "synckit-config"
4854 - version = "0.2.0"
M mnw-cli/Cargo.toml +1 -1
@@ -1,6 +1,6 @@
1 1 [package]
2 2 name = "mnw-cli"
3 - version = "0.1.6"
3 + version = "0.1.7"
4 4 edition = "2024"
5 5 license = "LicenseRef-PolyForm-Noncommercial-1.0.0"
6 6
M server/Cargo.lock +10 -10
@@ -5270,7 +5270,7 @@
5270 5270
5271 5271 [[package]]
5272 5272 name = "makenotwork"
5273 - version = "0.16.6"
5273 + version = "0.16.7"
5274 5274 dependencies = [
5275 5275 "anyhow",
5276 5276 "apple-codesign",
@@ -6653,7 +6653,7 @@
6653 6653
6654 6654 [[package]]
6655 6655 name = "quasi-declare"
6656 - version = "0.1.11"
6656 + version = "0.1.12"
6657 6657 dependencies = [
6658 6658 "proc-macro2",
6659 6659 "quote",
@@ -10767,14 +10767,6 @@
10767 10767 "pkg-config",
10768 10768 ]
10769 10769
10770 - [[patch.unused]]
10771 - name = "synckit-client"
10772 - version = "0.10.0"
10773 -
10774 - [[patch.unused]]
10775 - name = "synckit-config"
10776 - version = "0.2.0"
10777 -
10778 10770 [[patch.unused]]
10779 10771 name = "kberg"
10780 10772 version = "0.1.0"
@@ -10787,6 +10779,14 @@
10787 10779 name = "painhours"
10788 10780 version = "0.1.0"
10789 10781
10782 + [[patch.unused]]
10783 + name = "synckit-client"
10784 + version = "0.10.0"
10785 +
10786 + [[patch.unused]]
10787 + name = "synckit-config"
10788 + version = "0.2.0"
10789 +
10790 10790 [[patch.unused]]
10791 10791 name = "quasi-immediate"
10792 10792 version = "0.103.0"
M server/Cargo.toml +1 -1
@@ -1,6 +1,6 @@
1 1 [package]
2 2 name = "makenotwork"
3 - version = "0.16.6"
3 + version = "0.16.7"
4 4 edition = "2024"
5 5 license = "LicenseRef-PolyForm-Noncommercial-1.0.0"
6 6 # Server binary: never published to a registry. Marks the crate private so
M server/openapi.json +1 -1
@@ -6,7 +6,7 @@
6 6 "license": {
7 7 "name": "PolyForm Noncommercial 1.0.0"
8 8 },
9 - "version": "0.16.6"
9 + "version": "0.16.7"
10 10 },
11 11 "paths": {
12 12 "/api/git/{owner}/{repo}/notes": {
M mnw-cli/src/artifact.rs +57 -235
@@ -18,19 +18,17 @@
18 18 //!
19 19 //! ## Upload order, and why it is not an implementation detail
20 20 //!
21 - //! Every format the store carries has one mutable entry point naming objects
22 - //! that are addressed by content. An RPM repository has `repodata/repomd.xml`
23 - //! over hashed metadata and packages; an OCI layout has `index.json` and
24 - //! `oci-layout` over digest-named blobs. Either way the entry point goes up
25 - //! last, so a publish that dies partway leaves objects nothing points at —
26 - //! invisible to a client, and overwritten by the next run — rather than an
27 - //! index promising objects that are not there yet.
21 + //! `repodata/repomd.xml` is an RPM repository's index: a client reads it, then
22 + //! fetches the metadata and packages it names. So packages go up first, then the
23 + //! hashed metadata, then `repomd.xml` and its signature last. A publish that
24 + //! dies partway therefore leaves objects nothing points at — invisible to `dnf`,
25 + //! and overwritten by the next run — rather than an index promising packages
26 + //! that are not there yet.
27 + //!
28 + //! Nothing else the store holds has an index. The base image mirror is archives
29 + //! pinned by digest, so its objects are independent and the ordering costs it
30 + //! nothing.
28 31 //!
29 - //! `--kind` picks which of those two shapes the tree is. It defaults to `rpm`
30 - //! and its only job is deciding what counts as the index; a wrong answer is a
31 - //! publish ordered as if the tree were something else, which is why it is an
32 - //! explicit flag rather than a guess at the directory's contents.
33 -
34 32 use std::collections::BTreeMap;
35 33 use std::path::{Path, PathBuf};
36 34
@@ -60,17 +58,16 @@
60 58
61 59 fn print_usage() {
62 60 eprintln!(
63 - "Usage: mnw-cli artifact publish --dir DIR [--kind KIND] [--prefix PATH] [flags]\n\
61 + "Usage: mnw-cli artifact publish --dir DIR [--prefix PATH] [flags]\n\
64 62 \n\
65 63 Uploads a directory tree to the makenot.work artifact store. Content\n\
66 64 goes up first and the index last, so an interrupted publish never\n\
67 65 leaves an index pointing at objects that are not there.\n\
68 66 \n\
69 67 Required:\n\
70 - \x20 --dir Tree root (an RPM repository, or an OCI layout)\n\
68 + \x20 --dir Tree root (an RPM repository, or a directory of archives)\n\
71 69 \n\
72 70 Optional:\n\
73 - \x20 --kind rpm (default) or registry. Decides what the index is.\n\
74 71 \x20 --prefix Path prefix inside the bucket (e.g. alloy/f43/x86_64)\n\
75 72 \x20 --dry-run List what would be uploaded, in order, and stop\n\
76 73 \n\
@@ -85,7 +82,6 @@
85 82
86 83 struct PublishArgs {
87 84 dir: PathBuf,
88 - kind: Kind,
89 85 prefix: String,
90 86 dry_run: bool,
91 87 api_key: String,
@@ -99,7 +95,6 @@
99 95 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
100 96 f.debug_struct("PublishArgs")
101 97 .field("dir", &self.dir)
102 - .field("kind", &self.kind)
103 98 .field("prefix", &self.prefix)
104 99 .field("dry_run", &self.dry_run)
105 100 .field("api_key", &"<redacted>")
@@ -111,7 +106,6 @@
111 106
112 107 fn parse_args(flags: &[String]) -> Result<PublishArgs> {
113 108 let mut dir = None;
114 - let mut kind = Kind::Rpm;
115 109 let mut prefix = String::new();
116 110 let mut dry_run = false;
117 111 let mut api_key = std::env::var("MNW_OTA_API_KEY").ok();
@@ -127,7 +121,6 @@
127 121 };
128 122 match flag.as_str() {
129 123 "--dir" => dir = Some(PathBuf::from(take("--dir")?)),
130 - "--kind" => kind = Kind::parse(&take("--kind")?)?,
131 124 "--prefix" => prefix = take("--prefix")?,
132 125 "--dry-run" => dry_run = true,
133 126 "--api-key" => api_key = Some(take("--api-key")?),
@@ -150,7 +143,6 @@
150 143
151 144 Ok(PublishArgs {
152 145 dir: dir.ok_or_else(|| missing("--dir"))?,
153 - kind,
154 146 prefix,
155 147 dry_run,
156 148 // Credentials are only needed for a real publish; a dry run should work
@@ -167,11 +159,6 @@
167 159 path: PathBuf,
168 160 object_path: String,
169 161 size: i64,
170 - /// Set only where the server's extension-keyed derivation cannot reach the
171 - /// right answer: a manifest has no extension and must not be served as a
172 - /// byte stream. `None` everywhere else, so there is one derivation rather
173 - /// than two disagreeing ones.
174 - content_type: Option<String>,
175 162 }
176 163
177 164 /// Which stage of the publish an object belongs to. The derived `Ord` is the
@@ -189,71 +176,14 @@
189 176 Index,
190 177 }
191 178
192 - /// Which layout the tree under `--dir` is. Asked so that [`stage_of`] knows
193 - /// what the index is, and so that a manifest is uploaded under the content type
194 - /// its own bytes declare.
195 - #[derive(Debug, Clone, Copy, PartialEq, Eq)]
196 - enum Kind {
197 - /// A `createrepo_c` repository: `repomd.xml` over `repodata/` over packages.
198 - Rpm,
199 - /// A registry tree: `v2/<name>/manifests/<ref>` over
200 - /// `v2/<name>/blobs/<algorithm>:<digest>`.
201 - ///
202 - /// **Not an OCI image layout**, and the difference is the whole reason this
203 - /// variant is named for the registry. A layout on disk is `index.json` and
204 - /// `oci-layout` over `blobs/sha256/<hex>`, and there is no transport that
205 - /// pulls one over HTTP: `oci:` wants a local path. Measured against podman
206 - /// 4.9.3, a pull asks for exactly three shapes, all GET, no auth and no
207 - /// write verbs, which is why a static store can answer them:
208 - ///
209 - /// ```text
210 - /// GET /v2/
211 - /// GET /v2/<name>/manifests/<tag or sha256:hex>
212 - /// GET /v2/<name>/blobs/sha256:<hex>
213 - /// ```
214 - ///
215 - /// So the mirror publishes the registry tree, not the layout it was built
216 - /// from. See wiki `mnw-package-hosting`.
217 - Registry,
218 - }
219 -
220 - impl Kind {
221 - fn parse(value: &str) -> Result<Self> {
222 - match value {
223 - "rpm" => Ok(Self::Rpm),
224 - "registry" => Ok(Self::Registry),
225 - other => bail!("unknown --kind {other}; expected rpm or registry"),
226 - }
227 - }
228 - }
229 -
230 - fn stage_of(kind: Kind, object_path: &str) -> Stage {
179 + fn stage_of(object_path: &str) -> Stage {
231 180 let filename = object_path.rsplit('/').next().unwrap_or(object_path);
232 - match kind {
233 - Kind::Rpm => {
234 - if filename.starts_with("repomd.xml") {
235 - Stage::Index
236 - } else if object_path.contains("/repodata/") || object_path.starts_with("repodata/") {
237 - Stage::Metadata
238 - } else {
239 - Stage::Package
240 - }
241 - }
242 - // A registry tree has no middle tier. Blobs are named by their own
243 - // digest, config and layers alike, so nothing under `blobs/` refers to
244 - // anything else by a name that could go stale.
245 - //
246 - // Manifests are the index, including the ones addressed by digest: a
247 - // multi-arch pull reads the index manifest and then fetches a per-arch
248 - // manifest from the same directory, so publishing any of them before
249 - // their blobs is what leaves a client chasing a blob that is not there.
250 - Kind::Registry => {
251 - if object_path.contains("/manifests/") {
252 - Stage::Index
253 - } else {
254 - Stage::Package
255 - }
256 - }
181 + if filename.starts_with("repomd.xml") {
182 + Stage::Index
183 + } else if object_path.contains("/repodata/") || object_path.starts_with("repodata/") {
184 + Stage::Metadata
185 + } else {
186 + Stage::Package
257 187 }
258 188 }
259 189
@@ -262,9 +192,9 @@
262 192 /// Sorted by (stage, path): stage gives the ordering the publish depends on,
263 193 /// and the path tiebreak makes a run's output reproducible so two dry runs of
264 194 /// the same tree are diffable.
265 - fn collect(kind: Kind, root: &Path, prefix: &str) -> Result<Vec<Upload>> {
195 + fn collect(root: &Path, prefix: &str) -> Result<Vec<Upload>> {
266 196 let mut found = BTreeMap::new();
267 - walk(kind, root, root, prefix, &mut found)?;
197 + walk(root, root, prefix, &mut found)?;
268 198 if found.is_empty() {
269 199 bail!("no files under {}", root.display());
270 200 }
@@ -272,7 +202,6 @@
272 202 }
273 203
274 204 fn walk(
275 - kind: Kind,
276 205 root: &Path,
277 206 dir: &Path,
278 207 prefix: &str,
@@ -288,7 +217,7 @@
288 217 .with_context(|| format!("stat {}", path.display()))?;
289 218
290 219 if meta.is_dir() {
291 - walk(kind, root, &path, prefix, found)?;
220 + walk(root, &path, prefix, found)?;
292 221 continue;
293 222 }
294 223 // Symlinks and anything else that is not a plain file: a repository is
@@ -326,16 +255,9 @@
326 255 );
327 256 }
328 257
329 - let content_type = if kind == Kind::Registry && object_path.contains("/manifests/") {
330 - manifest_content_type(&path)
331 - } else {
332 - None
333 - };
334 -
335 258 found.insert(
336 - (stage_of(kind, &object_path), object_path.clone()),
259 + (stage_of(&object_path), object_path.clone()),
337 260 Upload {
338 - content_type,
339 261 path,
340 262 object_path,
341 263 size,
@@ -349,32 +271,6 @@
349 271 struct PresignRequest<'a> {
350 272 path: &'a str,
351 273 size: i64,
352 - #[serde(skip_serializing_if = "Option::is_none")]
353 - content_type: Option<String>,
354 - }
355 -
356 - /// The content type a manifest must be served under, read out of its own bytes.
357 - ///
358 - /// MEASURED, podman 4.9.3, and it is not a nicety. A manifest served as
359 - /// `application/octet-stream` fails the pull with `unsupported schema version
360 - /// 2`, which reads as a malformed manifest and sends you to inspect the JSON.
361 - /// The JSON is fine; the header is what was wrong. The same pull succeeds with
362 - /// the media type set and no `Docker-Content-Digest` header at all, so this is
363 - /// the one transport detail that matters.
364 - ///
365 - /// The value comes from the manifest's own `mediaType` rather than from a guess
366 - /// keyed on the path, because that field is what the registry spec says a
367 - /// client will be handed and an index and an image manifest differ in it.
368 - /// Unreadable or absent, this returns `None` and the server derives a type as
369 - /// usual, which fails loudly at pull time rather than silently here.
370 - fn manifest_content_type(path: &Path) -> Option<String> {
371 - let bytes = std::fs::read(path).ok()?;
372 - let value: serde_json::Value = serde_json::from_slice(&bytes).ok()?;
373 - value
374 - .get("mediaType")?
375 - .as_str()
376 - .filter(|media| media.starts_with("application/"))
377 - .map(str::to_string)
378 274 }
379 275
380 276 #[derive(serde::Deserialize)]
@@ -391,28 +287,23 @@
391 287 if !args.dir.is_dir() {
392 288 bail!("--dir {} is not a directory", args.dir.display());
393 289 }
394 - // Shape check, per kind, and a warning rather than a refusal: publishing a
395 - // tree that is not yet a repository is a mistake worth naming, but the
396 - // store holds bytes at paths and nothing here should decide it cannot.
397 - match args.kind {
398 - Kind::Rpm if !args.dir.join("repodata").is_dir() => {
399 - eprintln!(
400 - "warning: {} has no repodata/ directory. dnf will not see a repository here \
401 - until createrepo_c has run.",
402 - args.dir.display()
403 - );
404 - }
405 - Kind::Registry if !args.dir.join("v2").is_dir() => {
406 - eprintln!(
407 - "warning: {} has no v2/ directory. A registry pull begins at /v2/, so a \
408 - client will find nothing here.",
409 - args.dir.display()
410 - );
411 - }
412 - _ => {}
413 - }
290 + let uploads = collect(&args.dir, &args.prefix)?;
414 291
415 - let uploads = collect(args.kind, &args.dir, &args.prefix)?;
292 + // A warning rather than a refusal, and inferred from the tree rather than
293 + // asked for on the command line: packages with no metadata beside them is
294 + // the one publish that looks fine and serves nothing, because `dnf` reads
295 + // `repomd.xml` and would find none. A tree of archives is not a repository
296 + // and is not missing anything, so it says nothing.
297 + let has_packages = uploads
298 + .iter()
299 + .any(|u| u.object_path.ends_with(".rpm") && !u.object_path.contains("/repodata/"));
300 + if has_packages && !args.dir.join("repodata").is_dir() {
301 + eprintln!(
302 + "warning: {} holds packages but no repodata/ directory. dnf will not see a \
303 + repository here until createrepo_c has run.",
304 + args.dir.display()
305 + );
306 + }
416 307 let total_bytes: i64 = uploads.iter().map(|u| u.size).sum();
417 308
418 309 println!(
@@ -478,7 +369,6 @@
478 369 .json(&PresignRequest {
479 370 path: &upload.object_path,
480 371 size: upload.size,
481 - content_type: upload.content_type.clone(),
482 372 })
483 373 .send()
484 374 .await
@@ -545,35 +435,22 @@
545 435 #[test]
546 436 fn index_uploads_after_metadata_and_packages() {
547 437 assert!(
548 - stage_of(Kind::Rpm, "alloy/f43/x86_64/alloy-1.2.3.rpm")
549 - < stage_of(Kind::Rpm, "repodata/abc-primary.xml.zst")
550 - );
551 - assert!(
552 - stage_of(Kind::Rpm, "repodata/abc-primary.xml.zst")
553 - < stage_of(Kind::Rpm, "repodata/repomd.xml")
438 + stage_of("alloy/f43/x86_64/alloy-1.2.3.rpm") < stage_of("repodata/abc-primary.xml.zst")
554 439 );
440 + assert!(stage_of("repodata/abc-primary.xml.zst") < stage_of("repodata/repomd.xml"));
555 441 }
556 442
557 443 #[test]
558 444 fn repomd_signature_and_key_are_index_stage() {
559 - assert_eq!(
560 - stage_of(Kind::Rpm, "a/repodata/repomd.xml.asc"),
561 - Stage::Index
562 - );
563 - assert_eq!(
564 - stage_of(Kind::Rpm, "a/repodata/repomd.xml.key"),
565 - Stage::Index
566 - );
445 + assert_eq!(stage_of("a/repodata/repomd.xml.asc"), Stage::Index);
446 + assert_eq!(stage_of("a/repodata/repomd.xml.key"), Stage::Index);
567 447 }
568 448
569 449 #[test]
570 450 fn a_package_inside_a_directory_named_repodata_is_still_metadata_stage() {
571 451 // Deliberate: anything under repodata/ is the index's business, and
572 452 // ordering it with the packages would put it ahead of nothing useful.
573 - assert_eq!(
574 - stage_of(Kind::Rpm, "alloy/repodata/whatever.rpm"),
575 - Stage::Metadata
576 - );
453 + assert_eq!(stage_of("alloy/repodata/whatever.rpm"), Stage::Metadata);
577 454 }
578 455
579 456 #[test]
@@ -587,7 +464,7 @@
587 464 std::fs::write(root.join("b.rpm"), b"pkgb").unwrap();
588 465 std::fs::write(root.join("a.rpm"), b"pkga").unwrap();
589 466
590 - let got: Vec<String> = collect(Kind::Rpm, root, "alloy/f43")
467 + let got: Vec<String> = collect(root, "alloy/f43")
591 468 .unwrap()
592 469 .into_iter()
593 470 .map(|u| u.object_path)
@@ -605,29 +482,17 @@
605 482 );
606 483 }
607 484
608 - /// Writes the tree podman actually requests, measured against 4.9.3:
609 - /// `/v2/<name>/manifests/<ref>` over `/v2/<name>/blobs/sha256:<hex>`.
610 - fn registry_tree(root: &Path) {
611 - std::fs::create_dir_all(root.join("v2/fedora-bootc/manifests")).unwrap();
612 - std::fs::create_dir_all(root.join("v2/fedora-bootc/blobs")).unwrap();
613 - std::fs::write(
614 - root.join("v2/fedora-bootc/blobs/sha256:beef"),
615 - b"a layer's bytes",
616 - )
617 - .unwrap();
618 - std::fs::write(
619 - root.join("v2/fedora-bootc/manifests/43"),
620 - br#"{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json"}"#,
621 - )
622 - .unwrap();
623 - }
624 -
485 + /// A tree of archives has no index, so every object is a package stage and
486 + /// the ordering costs it nothing. This is the mirror's shape: tarballs
487 + /// pinned by digest, not a registry.
625 488 #[test]
626 - fn a_registry_tree_puts_blobs_before_manifests() {
489 + fn a_tree_of_archives_publishes_in_path_order() {
627 490 let dir = tempfile::tempdir().unwrap();
628 - registry_tree(dir.path());
491 + let root = dir.path();
492 + std::fs::write(root.join("fedora-bootc-43-arm64.tar"), b"arm").unwrap();
493 + std::fs::write(root.join("fedora-bootc-43-amd64.tar"), b"amd").unwrap();
629 494
630 - let got: Vec<String> = collect(Kind::Registry, dir.path(), "alloy/base")
495 + let got: Vec<String> = collect(root, "base")
631 496 .unwrap()
632 497 .into_iter()
633 498 .map(|u| u.object_path)
@@ -636,60 +501,17 @@
636 501 assert_eq!(
637 502 got,
638 503 vec![
639 - "alloy/base/v2/fedora-bootc/blobs/sha256:beef",
640 - "alloy/base/v2/fedora-bootc/manifests/43",
504 + "base/fedora-bootc-43-amd64.tar",
505 + "base/fedora-bootc-43-arm64.tar",
641 506 ]
642 507 );
643 508 }
644 509
645 - #[test]
646 - fn a_manifest_is_published_under_the_media_type_it_declares() {
647 - // The measured failure this prevents: served as octet-stream, podman
648 - // 4.9.3 rejects the pull with "unsupported schema version 2".
649 - let dir = tempfile::tempdir().unwrap();
650 - registry_tree(dir.path());
651 -
652 - let uploads = collect(Kind::Registry, dir.path(), "").unwrap();
653 - let manifest = uploads
654 - .iter()
655 - .find(|u| u.object_path.contains("/manifests/"))
656 - .expect("the tree has a manifest");
657 - assert_eq!(
658 - manifest.content_type.as_deref(),
659 - Some("application/vnd.oci.image.index.v1+json")
660 - );
661 -
662 - let blob = uploads
663 - .iter()
664 - .find(|u| u.object_path.contains("/blobs/"))
665 - .expect("the tree has a blob");
666 - assert_eq!(
667 - blob.content_type, None,
668 - "a blob has no extension either, and octet-stream is the right answer for it"
669 - );
670 - }
671 -
672 - #[test]
673 - fn kind_decides_what_the_index_is() {
674 - // The same path is content under one kind and the index under the
675 - // other, which is the whole reason --kind is not inferred.
676 - assert_eq!(
677 - stage_of(Kind::Registry, "a/v2/x/manifests/43"),
678 - Stage::Index
679 - );
680 - assert_eq!(stage_of(Kind::Rpm, "a/v2/x/manifests/43"), Stage::Package);
681 - assert_eq!(stage_of(Kind::Rpm, "a/repodata/repomd.xml"), Stage::Index);
682 - assert_eq!(
683 - stage_of(Kind::Registry, "a/repodata/repomd.xml"),
684 - Stage::Package
685 - );
686 - }
687 -
688 510 #[test]
689 511 fn collect_refuses_an_empty_file() {
690 512 let dir = tempfile::tempdir().unwrap();
691 513 std::fs::write(dir.path().join("empty.rpm"), b"").unwrap();
692 - let err = collect(Kind::Rpm, dir.path(), "").unwrap_err().to_string();
514 + let err = collect(dir.path(), "").unwrap_err().to_string();
693 515 assert!(err.contains("zero-byte"), "{err}");
694 516 }
695 517
@@ -697,7 +519,7 @@
697 519 fn an_empty_prefix_publishes_at_the_bucket_root() {
698 520 let dir = tempfile::tempdir().unwrap();
699 521 std::fs::write(dir.path().join("a.rpm"), b"pkg").unwrap();
700 - let got = collect(Kind::Rpm, dir.path(), "").unwrap();
522 + let got = collect(dir.path(), "").unwrap();
701 523 assert_eq!(got[0].object_path, "a.rpm");
702 524 }
703 525 }
M server/src/routes/artifacts.rs +24 -24
@@ -8,6 +8,14 @@
8 8 //! they are one store under three prefixes, which is what this endpoint writes
9 9 //! to. See wiki `mnw-package-hosting`.
10 10 //!
11 + //! Every object is a file with an extension, and that is a design constraint
12 + //! rather than a coincidence. The mirror ships **archives over S3**, not a
13 + //! registry: a registry would have put `/v2/` at the host root, extensionless
14 + //! digest names in the keys and a colon in the key alphabet, and would have
15 + //! made a manifest's `Content-Type` load-bearing. A tarball fetched by URL and
16 + //! checked against a digest needs none of that, and the only consumer is a
17 + //! build script we write.
18 + //!
11 19 //! One route: an admin asks for a presigned `PutObject` against the artifact
12 20 //! bucket, then PUTs the bytes straight to object storage. The server never
13 21 //! sees the object, and the publisher never holds an S3 credential — the
@@ -82,25 +90,19 @@
82 90
83 91 /// Content type for a stored object, by extension.
84 92 ///
85 - /// Clients key off the object's own contents rather than transport metadata:
86 - /// `dnf` reads `repomd.xml`, and an OCI client reads the descriptor that named
87 - /// the blob. So this is about not serving a package as something a browser
88 - /// would render, not about correctness of the fetch. Anything unrecognised is a
89 - /// byte stream, which is the safe answer and is what every OCI blob under
90 - /// `blobs/sha256/` gets, since a digest has no extension.
93 + /// `dnf` keys off `repomd.xml`'s own contents rather than transport metadata,
94 + /// and an archive is verified against a digest the puller already holds, so
95 + /// this is about not serving a package as something a browser would render
96 + /// rather than about correctness of the fetch. Anything unrecognised is a byte
97 + /// stream, which is the safe answer.
91 98 fn content_type_for(key: &str) -> &'static str {
92 - // `oci-layout` carries JSON under a name with no extension, so it is
93 - // matched whole before the extension split below sees it.
94 - if key.rsplit('/').next().unwrap_or(key) == "oci-layout" {
95 - return "application/json";
96 - }
97 99 match key
98 100 .rsplit_once('.')
99 101 .map(|(_, ext)| ext.to_ascii_lowercase())
100 102 {
101 103 Some(ext) => match ext.as_str() {
102 104 "rpm" => "application/x-rpm",
103 - "json" => "application/json",
105 + "tar" => "application/x-tar",
104 106 "xml" => "application/xml",
105 107 "yaml" => "application/yaml",
106 108 "gz" => "application/gzip",
@@ -116,19 +118,17 @@
116 118
117 119 /// The filenames that are rewritten in place rather than content-addressed.
118 120 ///
119 - /// Every format this store carries has the same two-tier shape: a mutable entry
120 - /// point at a fixed name, and everything it references named by content. So the
121 - /// rule is one list rather than one policy per prefix.
121 + /// One entry, and the store is shaped so it stays that way. `repomd.xml` is an
122 + /// RPM repository's index, matched by prefix so its `.asc` and `.key` siblings
123 + /// come with it. Everything else the store holds is immutable under its own
124 + /// name: a package carries name-version-release-arch, and a mirrored base image
125 + /// is an archive pinned by the digest of its bytes.
122 126 ///
123 - /// - `repomd.xml` is an RPM repository's index, with its `.asc` and `.key`
124 - /// siblings, which is why this matches by prefix.
125 - /// - `index.json` and `oci-layout` are an OCI image layout's entry point. The
126 - /// manifests and blobs it names live under `blobs/` addressed by digest.
127 - ///
128 - /// A format whose entry point is not here would be cached at the edge for a
129 - /// year and its publishes would stop arriving, so adding a prefix to the store
130 - /// means checking its entry point against this list.
131 - const MUTABLE_INDEX_NAMES: [&str; 3] = ["repomd.xml", "index.json", "oci-layout"];
127 + /// A mutable file that is not here would be cached at the edge for a year and
128 + /// its publishes would stop arriving, so adding a prefix to the store means
129 + /// asking whether it has an entry point and adding it if so. Prefer a prefix
130 + /// that has none.
131 + const MUTABLE_INDEX_NAMES: [&str; 1] = ["repomd.xml"];
132 132
133 133 /// Cache-Control to stamp on the object, by role in the repository.
134 134 ///
M server/src/storage/key.rs +56 -110
@@ -73,68 +73,14 @@
73 73 ///
74 74 /// An RPM repository wants the package itself, the `createrepo_c` metadata under
75 75 /// `repodata/` (XML, in whatever compression the generator chose, or the sqlite
76 - /// variants), and the detached signature and public key beside `repomd.xml`. An
77 - /// OCI layout wants `.json` for `index.json`. Anything else is a publish
78 - /// mistake: nothing in a client's fetch path asks for it, so serving it is pure
79 - /// surface.
76 + /// variants), and the detached signature and public key beside `repomd.xml`. The
77 + /// base image mirror wants `.tar`, because a mirrored image is an archive and
78 + /// not a registry. Anything else is a publish mistake: nothing in a client's
79 + /// fetch path asks for it, so serving it is pure surface.
80 80 const ARTIFACT_EXTENSIONS: &[&str] = &[
81 - "rpm", "xml", "zst", "gz", "xz", "bz2", "sqlite", "asc", "key", "sig", "yaml", "json",
81 + "rpm", "xml", "zst", "gz", "xz", "bz2", "sqlite", "asc", "key", "sig", "yaml", "tar",
82 82 ];
83 83
84 - /// The extensionless names an OCI image layout requires at its root.
85 - const OCI_BARE_NAMES: &[&str] = &["oci-layout"];
86 -
87 - /// Whether a segment is a digest reference: `<algorithm>:<hex>`.
88 - ///
89 - /// The only reason `:` is allowed in a key at all. A registry pull addresses
90 - /// blobs and pinned manifests this way — `blobs/sha256:<hex>` — so refusing the
91 - /// colon would refuse the base-image mirror, and allowing it everywhere would
92 - /// widen the key alphabet for one shape's benefit. Measured against podman
93 - /// 4.9.3: those are the literal paths it requests.
94 - fn is_digest_reference(segment: &str) -> bool {
95 - let Some((algorithm, hex)) = segment.split_once(':') else {
96 - return false;
97 - };
98 - !algorithm.is_empty()
99 - && algorithm
100 - .chars()
101 - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit())
102 - && hex.len() >= 32
103 - && hex.chars().all(|c| c.is_ascii_hexdigit())
104 - }
105 -
106 - /// Whether a path names an object that carries no extension and is still
107 - /// legitimate.
108 - ///
109 - /// Three shapes, and every one of them is content a container client fetches by
110 - /// a name it was given rather than by a name a person typed:
111 - ///
112 - /// - `.../blobs/<algorithm>/<digest>` — an OCI image layout's blob.
113 - /// - `.../blobs/<algorithm>:<digest>` — the same blob as a registry serves it.
114 - /// - `.../manifests/<reference>` — a manifest, by tag or by digest. A tag is an
115 - /// arbitrary string, so this one is admitted by its parent segment rather
116 - /// than by its own shape.
117 - ///
118 - /// A blob's name is the digest of its own contents, which is a stronger
119 - /// guarantee than an extension allowlist gives rather than a weaker one. A
120 - /// stray extensionless file anywhere else in the tree still fails.
121 - fn is_registry_content(segments: &[&str]) -> bool {
122 - match segments {
123 - [.., blobs, algorithm, digest]
124 - if *blobs == "blobs"
125 - && !algorithm.is_empty()
126 - && algorithm.chars().all(|c| c.is_ascii_alphanumeric())
127 - && digest.len() >= 32
128 - && digest.chars().all(|c| c.is_ascii_hexdigit()) =>
129 - {
130 - true
131 - }
132 - [.., blobs, reference] if *blobs == "blobs" => is_digest_reference(reference),
133 - [.., manifests, _] if *manifests == "manifests" => true,
134 - _ => false,
135 - }
136 - }
137 -
138 84 impl S3Client {
139 85 /// Generate a consistent S3 key for an object
140 86 /// Format: {user_id}/{item_id}/{file_type}/{filename}
@@ -175,8 +121,9 @@
175 121 ))
176 122 }
177 123
178 - /// Key for an object in the artifact store, from the relative
179 - /// path the publisher names (e.g. `alloy/f43/x86_64/repodata/repomd.xml`).
124 + /// Key for an object in the artifact store, from the relative path the
125 + /// publisher names (e.g. `hotfix/f43/x86_64/repodata/repomd.xml`, or
126 + /// `base/fedora-bootc-43-amd64.tar`).
180 127 ///
181 128 /// The odd one out among the generators, and deliberately so: every other
182 129 /// key layout here is derived from ids we hold, but a yum repository *is* a
@@ -184,14 +131,19 @@
184 131 /// `repomd.xml`. The server cannot invent it without reimplementing
185 132 /// createrepo, so the caller supplies it. That makes this the one generator
186 133 /// whose whole job is refusing bad input, and it returns `Result` for that
187 - /// reason. Which layout the repo actually uses is
188 - /// [`86cb87b9`](https://makenot.work)'s business, not this function's, hence
189 - /// no structure is imposed beyond a segment count.
134 + /// reason. Which layout a prefix actually uses is its own business, not this
135 + /// function's, hence no structure is imposed beyond a segment count.
136 + ///
137 + /// Every object here is a file with an extension, because the store serves
138 + /// archives and repository files rather than a registry. That is what keeps
139 + /// this check as tight as it is: a registry would have forced extensionless
140 + /// digest names and a colon in the alphabet, and the mirror ships tarballs
141 + /// instead. See wiki `mnw-package-hosting`.
190 142 ///
191 143 /// Refused: absolute paths, empty segments (so `//` and a trailing `/`),
192 144 /// `.` and `..` in any position, a segment starting `.` or `-`, anything
193 145 /// outside `[A-Za-z0-9._+~-]`, and a final segment whose extension is not
194 - /// one a yum repository serves. Together those make traversal
146 + /// one the store serves. Together those make traversal
195 147 /// unrepresentable rather than merely unlikely, and keep a presigned PUT
196 148 /// from writing an object the Caddy block would then serve as something it
197 149 /// is not.
@@ -232,11 +184,9 @@
232 184 if !segment
233 185 .chars()
234 186 .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '+' | '~' | '-'))
235 - && !is_digest_reference(segment)
236 187 {
237 188 return Err(bad(
238 - "a path segment may hold only letters, digits, and `.` `_` `+` `~` `-`, \
239 - or be a digest reference like `sha256:<hex>`",
189 + "a path segment may hold only letters, digits, and `.` `_` `+` `~` `-`",
240 190 ));
241 191 }
242 192 }
@@ -244,25 +194,15 @@
244 194 // Unwrap: `split` on a non-empty string always yields at least one
245 195 // segment, and every segment was proven non-empty above.
246 196 let filename = segments.last().copied().unwrap_or_default();
247 -
248 - // Two shapes carry no extension and are still legitimate, so they are
249 - // checked before the allowlist rather than exempted from it.
250 - if !is_registry_content(&segments) && !OCI_BARE_NAMES.contains(&filename) {
251 - let ext = filename
252 - .rsplit_once('.')
253 - .map(|(_, ext)| ext.to_ascii_lowercase())
254 - .ok_or_else(|| {
255 - bad(
256 - "the final path segment needs a file extension, unless it is a blob, \
257 - a manifest, or the layout's own oci-layout file",
258 - )
259 - })?;
260 - if !ARTIFACT_EXTENSIONS.contains(&ext.as_str()) {
261 - return Err(bad(&format!(
262 - "`.{ext}` is not served from the artifact store. Allowed: {}",
263 - ARTIFACT_EXTENSIONS.join(", ")
264 - )));
265 - }
197 + let ext = filename
198 + .rsplit_once('.')
199 + .map(|(_, ext)| ext.to_ascii_lowercase())
200 + .ok_or_else(|| bad("the final path segment needs a file extension"))?;
201 + if !ARTIFACT_EXTENSIONS.contains(&ext.as_str()) {
202 + return Err(bad(&format!(
203 + "`.{ext}` is not served from the artifact store. Allowed: {}",
204 + ARTIFACT_EXTENSIONS.join(", ")
205 + )));
266 206 }
267 207
268 208 Ok(S3Key(path.to_string()))
@@ -486,37 +426,43 @@
486 426 mod artifact_key_tests {
487 427 use super::*;
488 428
489 - /// The literal paths podman 4.9.3 requests during a pull, taken from a
490 - /// static tree it pulled from successfully. Every one has to be a key the
491 - /// store will sign, or the mirror cannot hold what a client asks for.
492 429 #[test]
493 - fn the_paths_a_pull_requests_are_signable() {
494 - let hex = "2fae60d22615a2a211b043a79296367cd8dd4e085f7e03e225083145013ba652";
430 + fn the_mirror_and_the_hotfix_repo_are_both_signable() {
495 431 for path in [
496 - format!("alloy/base/v2/fedora-bootc/manifests/sha256:{hex}"),
497 - format!("alloy/base/v2/fedora-bootc/blobs/sha256:{hex}"),
498 - "alloy/base/v2/fedora-bootc/manifests/43".to_string(),
499 - // The layout form, for a tree that is copied rather than served.
500 - format!("alloy/base/blobs/sha256/{hex}"),
501 - "alloy/base/oci-layout".to_string(),
432 + "base/fedora-bootc-43-amd64.tar",
433 + "base/fedora-bootc-43-arm64.tar",
434 + "hotfix/f43/x86_64/alloy-1.0.0-1.fc43.x86_64.rpm",
435 + "hotfix/f43/x86_64/repodata/abc-primary.xml.zst",
436 + "hotfix/f43/x86_64/repodata/repomd.xml.asc",
502 437 ] {
503 438 assert!(
504 - S3Client::generate_artifact_key(&path).is_ok(),
505 - "a pull asks for {path} and the store refused to sign it"
439 + S3Client::generate_artifact_key(path).is_ok(),
440 + "{path} is store content and was refused"
441 + );
442 + }
443 + }
444 +
445 + /// The store holds archives and repository files, both of which carry an
446 + /// extension. Nothing here needs the registry alphabet: no colons, and no
447 + /// extensionless digest names.
448 + #[test]
449 + fn registry_shaped_keys_are_refused() {
450 + for path in [
451 + "v2/alloy/base/manifests/43",
452 + "v2/alloy/base/blobs/sha256:3f786850e387550fdab836ed7e6dc881de23001b",
453 + "base/blobs/sha256/3f786850e387550fdab836ed7e6dc881de23001b",
454 + "base/oci-layout",
455 + ] {
456 + assert!(
457 + S3Client::generate_artifact_key(path).is_err(),
458 + "{path} is registry shape and the store does not serve one"
506 459 );
507 460 }
508 461 }
509 462
510 463 #[test]
511 - fn the_colon_is_admitted_for_digests_and_nothing_else() {
512 - assert!(S3Client::generate_artifact_key("a/blobs/sha256:beef").is_err());
513 - assert!(S3Client::generate_artifact_key("a/v2/we:ird/x.rpm").is_err());
514 - assert!(S3Client::generate_artifact_key("a/v2/n/manifests/tag:with:colons").is_err());
515 - }
516 -
517 - #[test]
518 - fn an_extensionless_file_outside_the_registry_shapes_is_still_refused() {
519 - assert!(S3Client::generate_artifact_key("alloy/hotfix/payload").is_err());
520 - assert!(S3Client::generate_artifact_key("alloy/hotfix/blobs/sha256/nothex").is_err());
464 + fn an_extension_the_store_does_not_serve_is_refused() {
465 + assert!(S3Client::generate_artifact_key("base/payload.sh").is_err());
466 + assert!(S3Client::generate_artifact_key("base/index.html").is_err());
521 467 }
522 468 }
M server/tests/workflows/artifact_publish.rs +20 -57
@@ -107,23 +107,11 @@
107 107 path: &str,
108 108 size: i64,
109 109 ) -> crate::harness::client::TestResponse {
110 - presign_typed(h, path, size, None).await
111 - }
112 -
113 - /// The same mint with an explicit content type, which is how a manifest is
114 - /// published: its path carries no extension the server could key on.
115 - async fn presign_typed(
116 - h: &mut TestHarness,
117 - path: &str,
118 - size: i64,
119 - content_type: Option<&str>,
120 - ) -> crate::harness::client::TestResponse {
121 - let mut body = json!({ "path": path, "size": size });
122 - if let Some(content_type) = content_type {
123 - body["content_type"] = json!(content_type);
124 - }
125 110 h.client
126 - .post_json("/api/v1/admin/artifacts/uploads", &body.to_string())
111 + .post_json(
112 + "/api/v1/admin/artifacts/uploads",
113 + &json!({ "path": path, "size": size }).to_string(),
114 + )
127 115 .await
128 116 }
129 117
@@ -263,14 +251,10 @@
263 251 "alloy/hotfix/f43/x86_64/alloy-1.0.0-1.fc43.x86_64.rpm",
264 252 "alloy/hotfix/f43/x86_64/repodata/abc123-primary.xml.zst",
265 253 "alloy/hotfix/f43/x86_64/repodata/repomd.xml.asc",
266 - // The mirror's half of the store, in the shape podman actually
267 - // requests. A blob and a manifest both lack an extension, so they are
268 - // admitted by the shape of their path rather than by the allowlist,
269 - // and a blob's segment carries the colon nothing else may.
270 - "alloy/base/v2/fedora-bootc/manifests/43",
271 - "alloy/base/v2/fedora-bootc/manifests/sha256:3f786850e387550fdab836ed7e6dc881de23001b",
272 - "alloy/base/v2/fedora-bootc/blobs/sha256:3f786850e387550fdab836ed7e6dc881de23001b",
273 - "alloy/base/oci-layout",
254 + // The mirror's half of the store: archives, pinned by digest, with an
255 + // extension like everything else here.
256 + "base/fedora-bootc-43-amd64.tar",
257 + "base/fedora-bootc-43-arm64.tar",
274 258 ] {
275 259 let resp = presign(&mut h, path, 512).await;
276 260 assert_eq!(
@@ -283,14 +267,13 @@
283 267 for path in [
284 268 "alloy/f43/index.html",
285 269 "alloy/f43/payload.sh",
286 - // Extensionless, but not under blobs/<algorithm>/, so the OCI exemption
287 - // does not reach it.
288 270 "alloy/f43/noextension",
289 - // Under blobs/, and still not a digest.
290 - "alloy/base/blobs/sha256/not-hex-at-all",
291 - // A colon outside a digest reference stays refused, so admitting the
292 - // mirror did not widen the key alphabet generally.
293 - "alloy/base/v2/weird:name/thing.rpm",
271 + // Registry shape. The mirror ships archives, so none of this is store
272 + // content and the key alphabet stays narrow: no colons, no
273 + // extensionless digest names.
274 + "v2/alloy/base/manifests/43",
275 + "v2/alloy/base/blobs/sha256:3f786850e387550fdab836ed7e6dc881de23001b",
276 + "base/oci-layout",
294 277 ] {
295 278 let resp = presign(&mut h, path, 512).await;
296 279 assert_eq!(
@@ -313,39 +296,19 @@
313 296 .json();
314 297 assert_eq!(meta.content_type, "application/zstd");
315 298
316 - // The mirror's entry point, which has JSON in a name with no extension.
317 - let layout: PresignResponse = presign(&mut h, "alloy/base/oci-layout", 512).await.json();
318 - assert_eq!(layout.content_type, "application/json");
319 -
320 - // A manifest cannot be derived from its path, so the publisher sends the
321 - // media type its bytes declare and the server signs it verbatim. Serving a
322 - // manifest as a byte stream fails the pull; measured on podman 4.9.3.
323 - let manifest: PresignResponse = presign_typed(
324 - &mut h,
325 - "alloy/base/v2/fedora-bootc/manifests/43",
326 - 512,
327 - Some("application/vnd.oci.image.index.v1+json"),
328 - )
329 - .await
330 - .json();
331 - assert_eq!(
332 - manifest.content_type,
333 - "application/vnd.oci.image.index.v1+json"
334 - );
299 + let archive: PresignResponse = presign(&mut h, "base/fedora-bootc-43-amd64.tar", 512)
300 + .await
301 + .json();
302 + assert_eq!(archive.content_type, "application/x-tar");
335 303 }
336 304
337 305 #[tokio::test]
338 306 async fn an_index_is_revalidated_and_content_is_cached_forever() {
339 307 // The whole reason cache-control is not one value: an index that an edge
340 - // holds is a published fix that never arrives. Both formats' entry points
341 - // are checked, because the store carries both.
308 + // holds is a published fix that never arrives.
342 309 let mut h = harness_as_admin(None).await;
343 310
344 - for path in [
345 - "alloy/hotfix/f43/repodata/repomd.xml",
346 - "alloy/base/index.json",
347 - "alloy/base/oci-layout",
348 - ] {
311 + for path in ["alloy/hotfix/f43/repodata/repomd.xml"] {
349 312 let resp = presign(&mut h, path, 512).await;
350 313 assert_eq!(resp.status, 201, "{path}: {}", resp.text);
351 314 }