//! `mnw-cli artifact publish` — push a directory tree to the makenot.work //! artifact store. //! //! One uploader for everything the store holds: the Alloy hotfix RPM repository //! (alloy `86cb87b9`), the mirror of the Fedora base images (`ebf30337`), and //! the locked snapshot of the layered RPM set (`7ee5a694`). They were ruled //! separately and are one store under three prefixes, so they get one command //! rather than three. See wiki `mnw-package-hosting`. //! //! Every PUT goes straight to object storage through a URL the server signs, so //! this command holds no S3 credential — the reason the shape was chosen over //! putting a write key on a laptop (alloy `32423bdd`, ruled (c)). //! //! Nothing about a prefix's layout is decided here. The `--prefix` is handed //! through verbatim and the local tree under `--dir` is mirrored beneath it, so //! whatever layout the hosting task settles on is expressible without changing //! this file. //! //! ## Upload order, and why it is not an implementation detail //! //! `repodata/repomd.xml` is an RPM repository's index: a client reads it, then //! fetches the metadata and packages it names. So packages go up first, then the //! hashed metadata, then `repomd.xml` and its signature last. A publish that //! dies partway therefore leaves objects nothing points at — invisible to `dnf`, //! and overwritten by the next run — rather than an index promising packages //! that are not there yet. //! //! Nothing else the store holds has an index. The base image mirror is archives //! pinned by digest, so its objects are independent and the ordering costs it //! nothing. //! use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use synckit_client::{SyncKitClient, SyncKitConfig}; use crate::ota::authenticate_oauth; const DEFAULT_SERVER: &str = "https://makenot.work"; /// Entry point for the `artifact` subcommand. `rest` is everything after /// `artifact`. pub(crate) async fn run(rest: &[String]) -> Result<()> { match rest.first().map(String::as_str) { Some("publish") => publish(&rest[1..]).await, Some("-h" | "--help") | None => { print_usage(); Ok(()) } Some(other) => { eprintln!("Unknown artifact subcommand: {other}\n"); print_usage(); std::process::exit(2); } } } fn print_usage() { eprintln!( "Usage: mnw-cli artifact publish --dir DIR [--prefix PATH] [flags]\n\ \n\ Uploads a directory tree to the makenot.work artifact store. Content\n\ goes up first and the index last, so an interrupted publish never\n\ leaves an index pointing at objects that are not there.\n\ \n\ Required:\n\ \x20 --dir Tree root (an RPM repository, or a directory of archives)\n\ \n\ Optional:\n\ \x20 --prefix Path prefix inside the bucket (e.g. alloy/f43/x86_64)\n\ \x20 --dry-run List what would be uploaded, in order, and stop\n\ \n\ \x20 --api-key SyncKit app API key (env MNW_OTA_API_KEY)\n\ \x20 --key SyncKit SDK key (env MNW_OTA_KEY)\n\ \x20 --server Server URL (env MNW_OTA_SERVER, default {DEFAULT_SERVER})\n\ \n\ Auth is MNW OAuth: a browser opens on this machine. The account must be\n\ the server's configured admin; anything else gets a 404." ); } struct PublishArgs { dir: PathBuf, prefix: String, dry_run: bool, api_key: String, key: String, server: String, } // Manual Debug that redacts the credentials so they never reach logs or a // failing-test backtrace. Same discipline as `ota::PublishArgs`. impl std::fmt::Debug for PublishArgs { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("PublishArgs") .field("dir", &self.dir) .field("prefix", &self.prefix) .field("dry_run", &self.dry_run) .field("api_key", &"") .field("key", &"") .field("server", &self.server) .finish() } } fn parse_args(flags: &[String]) -> Result { let mut dir = None; let mut prefix = String::new(); let mut dry_run = false; let mut api_key = std::env::var("MNW_OTA_API_KEY").ok(); let mut key = std::env::var("MNW_OTA_KEY").ok(); let mut server = std::env::var("MNW_OTA_SERVER").unwrap_or_else(|_| DEFAULT_SERVER.to_string()); let mut it = flags.iter(); while let Some(flag) = it.next() { let mut take = |name: &str| -> Result { it.next() .cloned() .with_context(|| format!("{name} requires a value")) }; match flag.as_str() { "--dir" => dir = Some(PathBuf::from(take("--dir")?)), "--prefix" => prefix = take("--prefix")?, "--dry-run" => dry_run = true, "--api-key" => api_key = Some(take("--api-key")?), "--key" => key = Some(take("--key")?), "--server" => server = take("--server")?, "-h" | "--help" => { print_usage(); std::process::exit(0); } other => bail!("Unknown flag: {other}"), } } let missing = |name: &str| anyhow::anyhow!("missing required {name}"); // Trim the separators rather than rejecting them: `--prefix alloy/f43/` is // what anyone would type, and the server's key validator refuses an empty // segment, so passing it through untouched would fail on a typo that costs // nothing to accept. let prefix = prefix.trim_matches('/').to_string(); Ok(PublishArgs { dir: dir.ok_or_else(|| missing("--dir"))?, prefix, dry_run, // Credentials are only needed for a real publish; a dry run should work // on a machine that has none. api_key: api_key.unwrap_or_default(), key: key.unwrap_or_default(), server, }) } /// One object to upload: where it is locally, and where it goes in the bucket. #[derive(Debug)] struct Upload { path: PathBuf, object_path: String, size: i64, } /// Which stage of the publish an object belongs to. The derived `Ord` is the /// upload order, which is the point: sorting by this key is what makes the /// index land last. #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] enum Stage { /// Packages. Nothing references them until the metadata does. Package, /// Hashed metadata under `repodata/`. Content-addressed names, so a new one /// never overwrites the one the live `repomd.xml` still points at. Metadata, /// `repomd.xml` and its detached signature / key. Rewritten in place, and /// the moment they land the new metadata is live. Index, } fn stage_of(object_path: &str) -> Stage { let filename = object_path.rsplit('/').next().unwrap_or(object_path); if filename.starts_with("repomd.xml") { Stage::Index } else if object_path.contains("/repodata/") || object_path.starts_with("repodata/") { Stage::Metadata } else { Stage::Package } } /// Walk `root` and pair every file with the object path it publishes to. /// /// Sorted by (stage, path): stage gives the ordering the publish depends on, /// and the path tiebreak makes a run's output reproducible so two dry runs of /// the same tree are diffable. fn collect(root: &Path, prefix: &str) -> Result> { let mut found = BTreeMap::new(); walk(root, root, prefix, &mut found)?; if found.is_empty() { bail!("no files under {}", root.display()); } Ok(found.into_values().collect()) } fn walk( root: &Path, dir: &Path, prefix: &str, found: &mut BTreeMap<(Stage, String), Upload>, ) -> Result<()> { let entries = std::fs::read_dir(dir).with_context(|| format!("reading directory {}", dir.display()))?; for entry in entries { let entry = entry.with_context(|| format!("reading directory {}", dir.display()))?; let path = entry.path(); let meta = entry .metadata() .with_context(|| format!("stat {}", path.display()))?; if meta.is_dir() { walk(root, &path, prefix, found)?; continue; } // Symlinks and anything else that is not a plain file: a repository is // bytes at paths, and following a link out of the tree is exactly the // publish nobody meant to make. if !meta.is_file() { bail!( "{} is not a regular file. A repository directory holds only files and \ directories.", path.display() ); } let relative = path .strip_prefix(root) .expect("walk only descends into root") .to_str() .with_context(|| format!("{} is not valid UTF-8", path.display()))? .replace('\\', "/"); let object_path = if prefix.is_empty() { relative } else { format!("{prefix}/{relative}") }; let size: i64 = meta .len() .try_into() .with_context(|| format!("{} is too large to publish", path.display()))?; if size == 0 { bail!( "{} is empty; refusing to publish a zero-byte object", path.display() ); } found.insert( (stage_of(&object_path), object_path.clone()), Upload { path, object_path, size, }, ); } Ok(()) } #[derive(serde::Serialize)] struct PresignRequest<'a> { path: &'a str, size: i64, } #[derive(serde::Deserialize)] struct PresignResponse { upload_url: String, object_key: String, public_url: Option, content_type: String, } async fn publish(flags: &[String]) -> Result<()> { let args = parse_args(flags)?; if !args.dir.is_dir() { bail!("--dir {} is not a directory", args.dir.display()); } let uploads = collect(&args.dir, &args.prefix)?; // A warning rather than a refusal, and inferred from the tree rather than // asked for on the command line: packages with no metadata beside them is // the one publish that looks fine and serves nothing, because `dnf` reads // `repomd.xml` and would find none. A tree of archives is not a repository // and is not missing anything, so it says nothing. let has_packages = uploads .iter() .any(|u| u.object_path.ends_with(".rpm") && !u.object_path.contains("/repodata/")); if has_packages && !args.dir.join("repodata").is_dir() { eprintln!( "warning: {} holds packages but no repodata/ directory. dnf will not see a \ repository here until createrepo_c has run.", args.dir.display() ); } let total_bytes: i64 = uploads.iter().map(|u| u.size).sum(); println!( "Publishing {} objects ({total_bytes} bytes) from {} to {}", uploads.len(), args.dir.display(), args.server ); if args.dry_run { for upload in &uploads { println!(" {} ({} bytes)", upload.object_path, upload.size); } println!("\nDry run: nothing was uploaded."); return Ok(()); } if args.api_key.is_empty() { bail!("missing required --api-key / MNW_OTA_API_KEY"); } if args.key.is_empty() { bail!("missing required --key / MNW_OTA_KEY"); } let client = SyncKitClient::new(SyncKitConfig { server_url: args.server.clone(), api_key: args.api_key.clone(), }); authenticate_oauth(&client, &args.key).await?; let token = client .session_info() .context("authenticated but no session token was returned")? .token; println!(" authenticated"); let http = reqwest::Client::new(); let base = args.server.trim_end_matches('/'); let mut last_public_url = None; for upload in &uploads { print!(" {} ... ", upload.object_path); // Read per object rather than up front: a repository can be many // gigabytes, and only one object is in flight at a time. let bytes = tokio::fs::read(&upload.path) .await .with_context(|| format!("reading {}", upload.path.display()))?; // The server signed `size` into Content-Length. A file that changed // between the walk and the read would fail at S3 with an opaque // signature error, so catch it here where the message can say why. if bytes.len() as i64 != upload.size { bail!( "{} changed size while publishing ({} bytes at scan, {} at read)", upload.path.display(), upload.size, bytes.len() ); } let presign: PresignResponse = { let response = http .post(format!("{base}/api/v1/admin/artifacts/uploads")) .bearer_auth(&*token) .json(&PresignRequest { path: &upload.object_path, size: upload.size, }) .send() .await .with_context(|| format!("requesting a presign for {}", upload.object_path))?; let status = response.status(); if status == reqwest::StatusCode::NOT_FOUND { bail!( "the server answered 404 for the publish endpoint. Either this build of the \ server predates it, or this account is not the configured admin." ); } if !status.is_success() { let body = response.text().await.unwrap_or_default(); bail!( "presign for {} failed ({status}): {body}", upload.object_path ); } response .json() .await .with_context(|| format!("decoding the presign for {}", upload.object_path))? }; let put = http .put(&presign.upload_url) // Must match what the server signed, byte for byte, or S3 rejects // the signature. .header("content-type", &presign.content_type) .body(bytes) .send() .await .with_context(|| format!("uploading {}", upload.object_path))?; if !put.status().is_success() { let status = put.status(); let body = put.text().await.unwrap_or_default(); bail!("upload of {} failed ({status}): {body}", upload.object_path); } println!("ok ({})", presign.object_key); if presign.public_url.is_some() { last_public_url = presign.public_url; } } println!("\nPublished {} objects.", uploads.len()); if let Some(url) = last_public_url { // The index is the last object uploaded, so this is repomd.xml's URL — // the one worth checking by hand. println!("Index: {url}"); } else { println!( "The server has no ARTIFACT_BASE_URL configured, so it cannot say where these are \ served from." ); } Ok(()) } #[cfg(test)] mod tests { use super::*; #[test] fn index_uploads_after_metadata_and_packages() { assert!( stage_of("alloy/f43/x86_64/alloy-1.2.3.rpm") < stage_of("repodata/abc-primary.xml.zst") ); assert!(stage_of("repodata/abc-primary.xml.zst") < stage_of("repodata/repomd.xml")); } #[test] fn repomd_signature_and_key_are_index_stage() { assert_eq!(stage_of("a/repodata/repomd.xml.asc"), Stage::Index); assert_eq!(stage_of("a/repodata/repomd.xml.key"), Stage::Index); } #[test] fn a_package_inside_a_directory_named_repodata_is_still_metadata_stage() { // Deliberate: anything under repodata/ is the index's business, and // ordering it with the packages would put it ahead of nothing useful. assert_eq!(stage_of("alloy/repodata/whatever.rpm"), Stage::Metadata); } #[test] fn collect_orders_by_stage_then_path() { let dir = tempfile::tempdir().unwrap(); let root = dir.path(); std::fs::create_dir(root.join("repodata")).unwrap(); std::fs::write(root.join("repodata/repomd.xml"), b"index").unwrap(); std::fs::write(root.join("repodata/2-primary.xml.zst"), b"meta2").unwrap(); std::fs::write(root.join("repodata/1-primary.xml.zst"), b"meta1").unwrap(); std::fs::write(root.join("b.rpm"), b"pkgb").unwrap(); std::fs::write(root.join("a.rpm"), b"pkga").unwrap(); let got: Vec = collect(root, "alloy/f43") .unwrap() .into_iter() .map(|u| u.object_path) .collect(); assert_eq!( got, vec![ "alloy/f43/a.rpm", "alloy/f43/b.rpm", "alloy/f43/repodata/1-primary.xml.zst", "alloy/f43/repodata/2-primary.xml.zst", "alloy/f43/repodata/repomd.xml", ] ); } /// A tree of archives has no index, so every object is a package stage and /// the ordering costs it nothing. This is the mirror's shape: tarballs /// pinned by digest, not a registry. #[test] fn a_tree_of_archives_publishes_in_path_order() { let dir = tempfile::tempdir().unwrap(); let root = dir.path(); std::fs::write(root.join("fedora-bootc-43-arm64.tar"), b"arm").unwrap(); std::fs::write(root.join("fedora-bootc-43-amd64.tar"), b"amd").unwrap(); let got: Vec = collect(root, "base") .unwrap() .into_iter() .map(|u| u.object_path) .collect(); assert_eq!( got, vec![ "base/fedora-bootc-43-amd64.tar", "base/fedora-bootc-43-arm64.tar", ] ); } #[test] fn collect_refuses_an_empty_file() { let dir = tempfile::tempdir().unwrap(); std::fs::write(dir.path().join("empty.rpm"), b"").unwrap(); let err = collect(dir.path(), "").unwrap_err().to_string(); assert!(err.contains("zero-byte"), "{err}"); } #[test] fn an_empty_prefix_publishes_at_the_bucket_root() { let dir = tempfile::tempdir().unwrap(); std::fs::write(dir.path().join("a.rpm"), b"pkg").unwrap(); let got = collect(dir.path(), "").unwrap(); assert_eq!(got[0].object_path, "a.rpm"); } }