Skip to main content

max / makenotwork

18.5 KB · 526 lines History Blame Raw
1 //! `mnw-cli artifact publish` — push a directory tree to the makenot.work
2 //! artifact store.
3 //!
4 //! One uploader for everything the store holds: the Alloy hotfix RPM repository
5 //! (alloy `86cb87b9`), the mirror of the Fedora base images (`ebf30337`), and
6 //! the locked snapshot of the layered RPM set (`7ee5a694`). They were ruled
7 //! separately and are one store under three prefixes, so they get one command
8 //! rather than three. See wiki `mnw-package-hosting`.
9 //!
10 //! Every PUT goes straight to object storage through a URL the server signs, so
11 //! this command holds no S3 credential — the reason the shape was chosen over
12 //! putting a write key on a laptop (alloy `32423bdd`, ruled (c)).
13 //!
14 //! Nothing about a prefix's layout is decided here. The `--prefix` is handed
15 //! through verbatim and the local tree under `--dir` is mirrored beneath it, so
16 //! whatever layout the hosting task settles on is expressible without changing
17 //! this file.
18 //!
19 //! ## Upload order, and why it is not an implementation detail
20 //!
21 //! `repodata/repomd.xml` is an RPM repository's index: a client reads it, then
22 //! fetches the metadata and packages it names. So packages go up first, then the
23 //! hashed metadata, then `repomd.xml` and its signature last. A publish that
24 //! dies partway therefore leaves objects nothing points at — invisible to `dnf`,
25 //! and overwritten by the next run — rather than an index promising packages
26 //! that are not there yet.
27 //!
28 //! Nothing else the store holds has an index. The base image mirror is archives
29 //! pinned by digest, so its objects are independent and the ordering costs it
30 //! nothing.
31 //!
32 use std::collections::BTreeMap;
33 use std::path::{Path, PathBuf};
34
35 use anyhow::{Context, Result, bail};
36 use synckit_client::{SyncKitClient, SyncKitConfig};
37
38 use crate::ota::authenticate_oauth;
39
40 const DEFAULT_SERVER: &str = "https://makenot.work";
41
42 /// Entry point for the `artifact` subcommand. `rest` is everything after
43 /// `artifact`.
44 pub(crate) async fn run(rest: &[String]) -> Result<()> {
45 match rest.first().map(String::as_str) {
46 Some("publish") => publish(&rest[1..]).await,
47 Some("-h" | "--help") | None => {
48 print_usage();
49 Ok(())
50 }
51 Some(other) => {
52 eprintln!("Unknown artifact subcommand: {other}\n");
53 print_usage();
54 std::process::exit(2);
55 }
56 }
57 }
58
59 fn print_usage() {
60 eprintln!(
61 "Usage: mnw-cli artifact publish --dir DIR [--prefix PATH] [flags]\n\
62 \n\
63 Uploads a directory tree to the makenot.work artifact store. Content\n\
64 goes up first and the index last, so an interrupted publish never\n\
65 leaves an index pointing at objects that are not there.\n\
66 \n\
67 Required:\n\
68 \x20 --dir Tree root (an RPM repository, or a directory of archives)\n\
69 \n\
70 Optional:\n\
71 \x20 --prefix Path prefix inside the bucket (e.g. alloy/f43/x86_64)\n\
72 \x20 --dry-run List what would be uploaded, in order, and stop\n\
73 \n\
74 \x20 --api-key SyncKit app API key (env MNW_OTA_API_KEY)\n\
75 \x20 --key SyncKit SDK key (env MNW_OTA_KEY)\n\
76 \x20 --server Server URL (env MNW_OTA_SERVER, default {DEFAULT_SERVER})\n\
77 \n\
78 Auth is MNW OAuth: a browser opens on this machine. The account must be\n\
79 the server's configured admin; anything else gets a 404."
80 );
81 }
82
83 struct PublishArgs {
84 dir: PathBuf,
85 prefix: String,
86 dry_run: bool,
87 api_key: String,
88 key: String,
89 server: String,
90 }
91
92 // Manual Debug that redacts the credentials so they never reach logs or a
93 // failing-test backtrace. Same discipline as `ota::PublishArgs`.
94 impl std::fmt::Debug for PublishArgs {
95 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
96 f.debug_struct("PublishArgs")
97 .field("dir", &self.dir)
98 .field("prefix", &self.prefix)
99 .field("dry_run", &self.dry_run)
100 .field("api_key", &"<redacted>")
101 .field("key", &"<redacted>")
102 .field("server", &self.server)
103 .finish()
104 }
105 }
106
107 fn parse_args(flags: &[String]) -> Result<PublishArgs> {
108 let mut dir = None;
109 let mut prefix = String::new();
110 let mut dry_run = false;
111 let mut api_key = std::env::var("MNW_OTA_API_KEY").ok();
112 let mut key = std::env::var("MNW_OTA_KEY").ok();
113 let mut server = std::env::var("MNW_OTA_SERVER").unwrap_or_else(|_| DEFAULT_SERVER.to_string());
114
115 let mut it = flags.iter();
116 while let Some(flag) = it.next() {
117 let mut take = |name: &str| -> Result<String> {
118 it.next()
119 .cloned()
120 .with_context(|| format!("{name} requires a value"))
121 };
122 match flag.as_str() {
123 "--dir" => dir = Some(PathBuf::from(take("--dir")?)),
124 "--prefix" => prefix = take("--prefix")?,
125 "--dry-run" => dry_run = true,
126 "--api-key" => api_key = Some(take("--api-key")?),
127 "--key" => key = Some(take("--key")?),
128 "--server" => server = take("--server")?,
129 "-h" | "--help" => {
130 print_usage();
131 std::process::exit(0);
132 }
133 other => bail!("Unknown flag: {other}"),
134 }
135 }
136
137 let missing = |name: &str| anyhow::anyhow!("missing required {name}");
138 // Trim the separators rather than rejecting them: `--prefix alloy/f43/` is
139 // what anyone would type, and the server's key validator refuses an empty
140 // segment, so passing it through untouched would fail on a typo that costs
141 // nothing to accept.
142 let prefix = prefix.trim_matches('/').to_string();
143
144 Ok(PublishArgs {
145 dir: dir.ok_or_else(|| missing("--dir"))?,
146 prefix,
147 dry_run,
148 // Credentials are only needed for a real publish; a dry run should work
149 // on a machine that has none.
150 api_key: api_key.unwrap_or_default(),
151 key: key.unwrap_or_default(),
152 server,
153 })
154 }
155
156 /// One object to upload: where it is locally, and where it goes in the bucket.
157 #[derive(Debug)]
158 struct Upload {
159 path: PathBuf,
160 object_path: String,
161 size: i64,
162 }
163
164 /// Which stage of the publish an object belongs to. The derived `Ord` is the
165 /// upload order, which is the point: sorting by this key is what makes the
166 /// index land last.
167 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
168 enum Stage {
169 /// Packages. Nothing references them until the metadata does.
170 Package,
171 /// Hashed metadata under `repodata/`. Content-addressed names, so a new one
172 /// never overwrites the one the live `repomd.xml` still points at.
173 Metadata,
174 /// `repomd.xml` and its detached signature / key. Rewritten in place, and
175 /// the moment they land the new metadata is live.
176 Index,
177 }
178
179 fn stage_of(object_path: &str) -> Stage {
180 let filename = object_path.rsplit('/').next().unwrap_or(object_path);
181 if filename.starts_with("repomd.xml") {
182 Stage::Index
183 } else if object_path.contains("/repodata/") || object_path.starts_with("repodata/") {
184 Stage::Metadata
185 } else {
186 Stage::Package
187 }
188 }
189
190 /// Walk `root` and pair every file with the object path it publishes to.
191 ///
192 /// Sorted by (stage, path): stage gives the ordering the publish depends on,
193 /// and the path tiebreak makes a run's output reproducible so two dry runs of
194 /// the same tree are diffable.
195 fn collect(root: &Path, prefix: &str) -> Result<Vec<Upload>> {
196 let mut found = BTreeMap::new();
197 walk(root, root, prefix, &mut found)?;
198 if found.is_empty() {
199 bail!("no files under {}", root.display());
200 }
201 Ok(found.into_values().collect())
202 }
203
204 fn walk(
205 root: &Path,
206 dir: &Path,
207 prefix: &str,
208 found: &mut BTreeMap<(Stage, String), Upload>,
209 ) -> Result<()> {
210 let entries =
211 std::fs::read_dir(dir).with_context(|| format!("reading directory {}", dir.display()))?;
212 for entry in entries {
213 let entry = entry.with_context(|| format!("reading directory {}", dir.display()))?;
214 let path = entry.path();
215 let meta = entry
216 .metadata()
217 .with_context(|| format!("stat {}", path.display()))?;
218
219 if meta.is_dir() {
220 walk(root, &path, prefix, found)?;
221 continue;
222 }
223 // Symlinks and anything else that is not a plain file: a repository is
224 // bytes at paths, and following a link out of the tree is exactly the
225 // publish nobody meant to make.
226 if !meta.is_file() {
227 bail!(
228 "{} is not a regular file. A repository directory holds only files and \
229 directories.",
230 path.display()
231 );
232 }
233
234 let relative = path
235 .strip_prefix(root)
236 .expect("walk only descends into root")
237 .to_str()
238 .with_context(|| format!("{} is not valid UTF-8", path.display()))?
239 .replace('\\', "/");
240
241 let object_path = if prefix.is_empty() {
242 relative
243 } else {
244 format!("{prefix}/{relative}")
245 };
246
247 let size: i64 = meta
248 .len()
249 .try_into()
250 .with_context(|| format!("{} is too large to publish", path.display()))?;
251 if size == 0 {
252 bail!(
253 "{} is empty; refusing to publish a zero-byte object",
254 path.display()
255 );
256 }
257
258 found.insert(
259 (stage_of(&object_path), object_path.clone()),
260 Upload {
261 path,
262 object_path,
263 size,
264 },
265 );
266 }
267 Ok(())
268 }
269
270 #[derive(serde::Serialize)]
271 struct PresignRequest<'a> {
272 path: &'a str,
273 size: i64,
274 }
275
276 #[derive(serde::Deserialize)]
277 struct PresignResponse {
278 upload_url: String,
279 object_key: String,
280 public_url: Option<String>,
281 content_type: String,
282 }
283
284 async fn publish(flags: &[String]) -> Result<()> {
285 let args = parse_args(flags)?;
286
287 if !args.dir.is_dir() {
288 bail!("--dir {} is not a directory", args.dir.display());
289 }
290 let uploads = collect(&args.dir, &args.prefix)?;
291
292 // A warning rather than a refusal, and inferred from the tree rather than
293 // asked for on the command line: packages with no metadata beside them is
294 // the one publish that looks fine and serves nothing, because `dnf` reads
295 // `repomd.xml` and would find none. A tree of archives is not a repository
296 // and is not missing anything, so it says nothing.
297 let has_packages = uploads
298 .iter()
299 .any(|u| u.object_path.ends_with(".rpm") && !u.object_path.contains("/repodata/"));
300 if has_packages && !args.dir.join("repodata").is_dir() {
301 eprintln!(
302 "warning: {} holds packages but no repodata/ directory. dnf will not see a \
303 repository here until createrepo_c has run.",
304 args.dir.display()
305 );
306 }
307 let total_bytes: i64 = uploads.iter().map(|u| u.size).sum();
308
309 println!(
310 "Publishing {} objects ({total_bytes} bytes) from {} to {}",
311 uploads.len(),
312 args.dir.display(),
313 args.server
314 );
315
316 if args.dry_run {
317 for upload in &uploads {
318 println!(" {} ({} bytes)", upload.object_path, upload.size);
319 }
320 println!("\nDry run: nothing was uploaded.");
321 return Ok(());
322 }
323
324 if args.api_key.is_empty() {
325 bail!("missing required --api-key / MNW_OTA_API_KEY");
326 }
327 if args.key.is_empty() {
328 bail!("missing required --key / MNW_OTA_KEY");
329 }
330
331 let client = SyncKitClient::new(SyncKitConfig {
332 server_url: args.server.clone(),
333 api_key: args.api_key.clone(),
334 });
335 authenticate_oauth(&client, &args.key).await?;
336 let token = client
337 .session_info()
338 .context("authenticated but no session token was returned")?
339 .token;
340 println!(" authenticated");
341
342 let http = reqwest::Client::new();
343 let base = args.server.trim_end_matches('/');
344 let mut last_public_url = None;
345
346 for upload in &uploads {
347 print!(" {} ... ", upload.object_path);
348 // Read per object rather than up front: a repository can be many
349 // gigabytes, and only one object is in flight at a time.
350 let bytes = tokio::fs::read(&upload.path)
351 .await
352 .with_context(|| format!("reading {}", upload.path.display()))?;
353 // The server signed `size` into Content-Length. A file that changed
354 // between the walk and the read would fail at S3 with an opaque
355 // signature error, so catch it here where the message can say why.
356 if bytes.len() as i64 != upload.size {
357 bail!(
358 "{} changed size while publishing ({} bytes at scan, {} at read)",
359 upload.path.display(),
360 upload.size,
361 bytes.len()
362 );
363 }
364
365 let presign: PresignResponse = {
366 let response = http
367 .post(format!("{base}/api/v1/admin/artifacts/uploads"))
368 .bearer_auth(&*token)
369 .json(&PresignRequest {
370 path: &upload.object_path,
371 size: upload.size,
372 })
373 .send()
374 .await
375 .with_context(|| format!("requesting a presign for {}", upload.object_path))?;
376 let status = response.status();
377 if status == reqwest::StatusCode::NOT_FOUND {
378 bail!(
379 "the server answered 404 for the publish endpoint. Either this build of the \
380 server predates it, or this account is not the configured admin."
381 );
382 }
383 if !status.is_success() {
384 let body = response.text().await.unwrap_or_default();
385 bail!(
386 "presign for {} failed ({status}): {body}",
387 upload.object_path
388 );
389 }
390 response
391 .json()
392 .await
393 .with_context(|| format!("decoding the presign for {}", upload.object_path))?
394 };
395
396 let put = http
397 .put(&presign.upload_url)
398 // Must match what the server signed, byte for byte, or S3 rejects
399 // the signature.
400 .header("content-type", &presign.content_type)
401 .body(bytes)
402 .send()
403 .await
404 .with_context(|| format!("uploading {}", upload.object_path))?;
405 if !put.status().is_success() {
406 let status = put.status();
407 let body = put.text().await.unwrap_or_default();
408 bail!("upload of {} failed ({status}): {body}", upload.object_path);
409 }
410
411 println!("ok ({})", presign.object_key);
412 if presign.public_url.is_some() {
413 last_public_url = presign.public_url;
414 }
415 }
416
417 println!("\nPublished {} objects.", uploads.len());
418 if let Some(url) = last_public_url {
419 // The index is the last object uploaded, so this is repomd.xml's URL —
420 // the one worth checking by hand.
421 println!("Index: {url}");
422 } else {
423 println!(
424 "The server has no ARTIFACT_BASE_URL configured, so it cannot say where these are \
425 served from."
426 );
427 }
428 Ok(())
429 }
430
431 #[cfg(test)]
432 mod tests {
433 use super::*;
434
435 #[test]
436 fn index_uploads_after_metadata_and_packages() {
437 assert!(
438 stage_of("alloy/f43/x86_64/alloy-1.2.3.rpm") < stage_of("repodata/abc-primary.xml.zst")
439 );
440 assert!(stage_of("repodata/abc-primary.xml.zst") < stage_of("repodata/repomd.xml"));
441 }
442
443 #[test]
444 fn repomd_signature_and_key_are_index_stage() {
445 assert_eq!(stage_of("a/repodata/repomd.xml.asc"), Stage::Index);
446 assert_eq!(stage_of("a/repodata/repomd.xml.key"), Stage::Index);
447 }
448
449 #[test]
450 fn a_package_inside_a_directory_named_repodata_is_still_metadata_stage() {
451 // Deliberate: anything under repodata/ is the index's business, and
452 // ordering it with the packages would put it ahead of nothing useful.
453 assert_eq!(stage_of("alloy/repodata/whatever.rpm"), Stage::Metadata);
454 }
455
456 #[test]
457 fn collect_orders_by_stage_then_path() {
458 let dir = tempfile::tempdir().unwrap();
459 let root = dir.path();
460 std::fs::create_dir(root.join("repodata")).unwrap();
461 std::fs::write(root.join("repodata/repomd.xml"), b"index").unwrap();
462 std::fs::write(root.join("repodata/2-primary.xml.zst"), b"meta2").unwrap();
463 std::fs::write(root.join("repodata/1-primary.xml.zst"), b"meta1").unwrap();
464 std::fs::write(root.join("b.rpm"), b"pkgb").unwrap();
465 std::fs::write(root.join("a.rpm"), b"pkga").unwrap();
466
467 let got: Vec<String> = collect(root, "alloy/f43")
468 .unwrap()
469 .into_iter()
470 .map(|u| u.object_path)
471 .collect();
472
473 assert_eq!(
474 got,
475 vec![
476 "alloy/f43/a.rpm",
477 "alloy/f43/b.rpm",
478 "alloy/f43/repodata/1-primary.xml.zst",
479 "alloy/f43/repodata/2-primary.xml.zst",
480 "alloy/f43/repodata/repomd.xml",
481 ]
482 );
483 }
484
485 /// A tree of archives has no index, so every object is a package stage and
486 /// the ordering costs it nothing. This is the mirror's shape: tarballs
487 /// pinned by digest, not a registry.
488 #[test]
489 fn a_tree_of_archives_publishes_in_path_order() {
490 let dir = tempfile::tempdir().unwrap();
491 let root = dir.path();
492 std::fs::write(root.join("fedora-bootc-43-arm64.tar"), b"arm").unwrap();
493 std::fs::write(root.join("fedora-bootc-43-amd64.tar"), b"amd").unwrap();
494
495 let got: Vec<String> = collect(root, "base")
496 .unwrap()
497 .into_iter()
498 .map(|u| u.object_path)
499 .collect();
500
501 assert_eq!(
502 got,
503 vec![
504 "base/fedora-bootc-43-amd64.tar",
505 "base/fedora-bootc-43-arm64.tar",
506 ]
507 );
508 }
509
510 #[test]
511 fn collect_refuses_an_empty_file() {
512 let dir = tempfile::tempdir().unwrap();
513 std::fs::write(dir.path().join("empty.rpm"), b"").unwrap();
514 let err = collect(dir.path(), "").unwrap_err().to_string();
515 assert!(err.contains("zero-byte"), "{err}");
516 }
517
518 #[test]
519 fn an_empty_prefix_publishes_at_the_bucket_root() {
520 let dir = tempfile::tempdir().unwrap();
521 std::fs::write(dir.path().join("a.rpm"), b"pkg").unwrap();
522 let got = collect(dir.path(), "").unwrap();
523 assert_eq!(got[0].object_path, "a.rpm");
524 }
525 }
526