//! `mnw-cli rpm publish` — push an Alloy hotfix RPM repository to makenot.work. //! //! Takes the directory `createrepo_c` wrote and uploads it object by object. //! Every PUT goes straight to object storage through a URL the server signs, so //! this command holds no S3 credential — the reason the shape was chosen over //! putting a write key on a laptop (alloy `32423bdd`, ruled (c)). //! //! Nothing about the repo's layout is decided here. The `--prefix` is handed //! through verbatim and the local tree under `--dir` is mirrored beneath it, so //! whatever layout the hosting task settles on is expressible without changing //! this file. //! //! ## Upload order, and why it is not an implementation detail //! //! `repodata/repomd.xml` is the index: a client reads it, then fetches the //! metadata and packages it names. So this uploads packages first, then the //! hashed metadata, then `repomd.xml` and its signature last. A publish that //! dies partway therefore leaves objects nothing points at — invisible to //! `dnf`, and overwritten by the next run — rather than an index promising //! packages that are not there yet. use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use synckit_client::{SyncKitClient, SyncKitConfig}; use crate::ota::authenticate_oauth; const DEFAULT_SERVER: &str = "https://makenot.work"; /// Entry point for the `rpm` subcommand. `rest` is everything after `rpm`. pub(crate) async fn run(rest: &[String]) -> Result<()> { match rest.first().map(String::as_str) { Some("publish") => publish(&rest[1..]).await, Some("-h" | "--help") | None => { print_usage(); Ok(()) } Some(other) => { eprintln!("Unknown rpm subcommand: {other}\n"); print_usage(); std::process::exit(2); } } } fn print_usage() { eprintln!( "Usage: mnw-cli rpm publish --dir DIR [--prefix PATH] [flags]\n\ \n\ Uploads a createrepo_c repository to the makenot.work RPM bucket.\n\ Packages go up first and repomd.xml last, so an interrupted publish\n\ never leaves an index pointing at objects that are not there.\n\ \n\ Required:\n\ \x20 --dir Repository root (the directory holding repodata/)\n\ \n\ Optional:\n\ \x20 --prefix Path prefix inside the bucket (e.g. alloy/f43/x86_64)\n\ \x20 --dry-run List what would be uploaded, in order, and stop\n\ \n\ \x20 --api-key SyncKit app API key (env MNW_OTA_API_KEY)\n\ \x20 --key SyncKit SDK key (env MNW_OTA_KEY)\n\ \x20 --server Server URL (env MNW_OTA_SERVER, default {DEFAULT_SERVER})\n\ \n\ Auth is MNW OAuth: a browser opens on this machine. The account must be\n\ the server's configured admin; anything else gets a 404." ); } struct PublishArgs { dir: PathBuf, prefix: String, dry_run: bool, api_key: String, key: String, server: String, } // Manual Debug that redacts the credentials so they never reach logs or a // failing-test backtrace. Same discipline as `ota::PublishArgs`. impl std::fmt::Debug for PublishArgs { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("PublishArgs") .field("dir", &self.dir) .field("prefix", &self.prefix) .field("dry_run", &self.dry_run) .field("api_key", &"") .field("key", &"") .field("server", &self.server) .finish() } } fn parse_args(flags: &[String]) -> Result { let mut dir = None; let mut prefix = String::new(); let mut dry_run = false; let mut api_key = std::env::var("MNW_OTA_API_KEY").ok(); let mut key = std::env::var("MNW_OTA_KEY").ok(); let mut server = std::env::var("MNW_OTA_SERVER").unwrap_or_else(|_| DEFAULT_SERVER.to_string()); let mut it = flags.iter(); while let Some(flag) = it.next() { let mut take = |name: &str| -> Result { it.next() .cloned() .with_context(|| format!("{name} requires a value")) }; match flag.as_str() { "--dir" => dir = Some(PathBuf::from(take("--dir")?)), "--prefix" => prefix = take("--prefix")?, "--dry-run" => dry_run = true, "--api-key" => api_key = Some(take("--api-key")?), "--key" => key = Some(take("--key")?), "--server" => server = take("--server")?, "-h" | "--help" => { print_usage(); std::process::exit(0); } other => bail!("Unknown flag: {other}"), } } let missing = |name: &str| anyhow::anyhow!("missing required {name}"); // Trim the separators rather than rejecting them: `--prefix alloy/f43/` is // what anyone would type, and the server's key validator refuses an empty // segment, so passing it through untouched would fail on a typo that costs // nothing to accept. let prefix = prefix.trim_matches('/').to_string(); Ok(PublishArgs { dir: dir.ok_or_else(|| missing("--dir"))?, prefix, dry_run, // Credentials are only needed for a real publish; a dry run should work // on a machine that has none. api_key: api_key.unwrap_or_default(), key: key.unwrap_or_default(), server, }) } /// One object to upload: where it is locally, and where it goes in the bucket. #[derive(Debug)] struct Upload { path: PathBuf, object_path: String, size: i64, } /// Which stage of the publish an object belongs to. The derived `Ord` is the /// upload order, which is the point: sorting by this key is what makes the /// index land last. #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] enum Stage { /// Packages. Nothing references them until the metadata does. Package, /// Hashed metadata under `repodata/`. Content-addressed names, so a new one /// never overwrites the one the live `repomd.xml` still points at. Metadata, /// `repomd.xml` and its detached signature / key. Rewritten in place, and /// the moment they land the new metadata is live. Index, } fn stage_of(object_path: &str) -> Stage { let filename = object_path.rsplit('/').next().unwrap_or(object_path); if filename.starts_with("repomd.xml") { Stage::Index } else if object_path.contains("/repodata/") || object_path.starts_with("repodata/") { Stage::Metadata } else { Stage::Package } } /// Walk `root` and pair every file with the object path it publishes to. /// /// Sorted by (stage, path): stage gives the ordering the publish depends on, /// and the path tiebreak makes a run's output reproducible so two dry runs of /// the same tree are diffable. fn collect(root: &Path, prefix: &str) -> Result> { let mut found = BTreeMap::new(); walk(root, root, prefix, &mut found)?; if found.is_empty() { bail!("no files under {}", root.display()); } Ok(found.into_values().collect()) } fn walk( root: &Path, dir: &Path, prefix: &str, found: &mut BTreeMap<(Stage, String), Upload>, ) -> Result<()> { let entries = std::fs::read_dir(dir).with_context(|| format!("reading directory {}", dir.display()))?; for entry in entries { let entry = entry.with_context(|| format!("reading directory {}", dir.display()))?; let path = entry.path(); let meta = entry .metadata() .with_context(|| format!("stat {}", path.display()))?; if meta.is_dir() { walk(root, &path, prefix, found)?; continue; } // Symlinks and anything else that is not a plain file: a repository is // bytes at paths, and following a link out of the tree is exactly the // publish nobody meant to make. if !meta.is_file() { bail!( "{} is not a regular file. A repository directory holds only files and \ directories.", path.display() ); } let relative = path .strip_prefix(root) .expect("walk only descends into root") .to_str() .with_context(|| format!("{} is not valid UTF-8", path.display()))? .replace('\\', "/"); let object_path = if prefix.is_empty() { relative } else { format!("{prefix}/{relative}") }; let size: i64 = meta .len() .try_into() .with_context(|| format!("{} is too large to publish", path.display()))?; if size == 0 { bail!( "{} is empty; refusing to publish a zero-byte object", path.display() ); } found.insert( (stage_of(&object_path), object_path.clone()), Upload { path, object_path, size, }, ); } Ok(()) } #[derive(serde::Serialize)] struct PresignRequest<'a> { path: &'a str, size: i64, } #[derive(serde::Deserialize)] struct PresignResponse { upload_url: String, object_key: String, public_url: Option, content_type: String, } async fn publish(flags: &[String]) -> Result<()> { let args = parse_args(flags)?; if !args.dir.is_dir() { bail!("--dir {} is not a directory", args.dir.display()); } if !args.dir.join("repodata").is_dir() { eprintln!( "warning: {} has no repodata/ directory. dnf will not see a repository here \ until createrepo_c has run.", args.dir.display() ); } let uploads = collect(&args.dir, &args.prefix)?; let total_bytes: i64 = uploads.iter().map(|u| u.size).sum(); println!( "Publishing {} objects ({total_bytes} bytes) from {} to {}", uploads.len(), args.dir.display(), args.server ); if args.dry_run { for upload in &uploads { println!(" {} ({} bytes)", upload.object_path, upload.size); } println!("\nDry run: nothing was uploaded."); return Ok(()); } if args.api_key.is_empty() { bail!("missing required --api-key / MNW_OTA_API_KEY"); } if args.key.is_empty() { bail!("missing required --key / MNW_OTA_KEY"); } let client = SyncKitClient::new(SyncKitConfig { server_url: args.server.clone(), api_key: args.api_key.clone(), }); authenticate_oauth(&client, &args.key).await?; let token = client .session_info() .context("authenticated but no session token was returned")? .token; println!(" authenticated"); let http = reqwest::Client::new(); let base = args.server.trim_end_matches('/'); let mut last_public_url = None; for upload in &uploads { print!(" {} ... ", upload.object_path); // Read per object rather than up front: a repository can be many // gigabytes, and only one object is in flight at a time. let bytes = tokio::fs::read(&upload.path) .await .with_context(|| format!("reading {}", upload.path.display()))?; // The server signed `size` into Content-Length. A file that changed // between the walk and the read would fail at S3 with an opaque // signature error, so catch it here where the message can say why. if bytes.len() as i64 != upload.size { bail!( "{} changed size while publishing ({} bytes at scan, {} at read)", upload.path.display(), upload.size, bytes.len() ); } let presign: PresignResponse = { let response = http .post(format!("{base}/api/v1/admin/rpm/uploads")) .bearer_auth(&*token) .json(&PresignRequest { path: &upload.object_path, size: upload.size, }) .send() .await .with_context(|| format!("requesting a presign for {}", upload.object_path))?; let status = response.status(); if status == reqwest::StatusCode::NOT_FOUND { bail!( "the server answered 404 for the publish endpoint. Either this build of the \ server predates it, or this account is not the configured admin." ); } if !status.is_success() { let body = response.text().await.unwrap_or_default(); bail!( "presign for {} failed ({status}): {body}", upload.object_path ); } response .json() .await .with_context(|| format!("decoding the presign for {}", upload.object_path))? }; let put = http .put(&presign.upload_url) // Must match what the server signed, byte for byte, or S3 rejects // the signature. .header("content-type", &presign.content_type) .body(bytes) .send() .await .with_context(|| format!("uploading {}", upload.object_path))?; if !put.status().is_success() { let status = put.status(); let body = put.text().await.unwrap_or_default(); bail!("upload of {} failed ({status}): {body}", upload.object_path); } println!("ok ({})", presign.object_key); if presign.public_url.is_some() { last_public_url = presign.public_url; } } println!("\nPublished {} objects.", uploads.len()); if let Some(url) = last_public_url { // The index is the last object uploaded, so this is repomd.xml's URL — // the one worth checking by hand. println!("Index: {url}"); } else { println!( "The server has no RPM_BASE_URL configured, so it cannot say where these are \ served from." ); } Ok(()) } #[cfg(test)] mod tests { use super::*; #[test] fn index_uploads_after_metadata_and_packages() { assert!( stage_of("alloy/f43/x86_64/alloy-1.2.3.rpm") < stage_of("repodata/abc-primary.xml.zst") ); assert!(stage_of("repodata/abc-primary.xml.zst") < stage_of("repodata/repomd.xml")); } #[test] fn repomd_signature_and_key_are_index_stage() { assert_eq!(stage_of("a/repodata/repomd.xml.asc"), Stage::Index); assert_eq!(stage_of("a/repodata/repomd.xml.key"), Stage::Index); } #[test] fn a_package_inside_a_directory_named_repodata_is_still_metadata_stage() { // Deliberate: anything under repodata/ is the index's business, and // ordering it with the packages would put it ahead of nothing useful. assert_eq!(stage_of("alloy/repodata/whatever.rpm"), Stage::Metadata); } #[test] fn collect_orders_by_stage_then_path() { let dir = tempfile::tempdir().unwrap(); let root = dir.path(); std::fs::create_dir(root.join("repodata")).unwrap(); std::fs::write(root.join("repodata/repomd.xml"), b"index").unwrap(); std::fs::write(root.join("repodata/2-primary.xml.zst"), b"meta2").unwrap(); std::fs::write(root.join("repodata/1-primary.xml.zst"), b"meta1").unwrap(); std::fs::write(root.join("b.rpm"), b"pkgb").unwrap(); std::fs::write(root.join("a.rpm"), b"pkga").unwrap(); let got: Vec = collect(root, "alloy/f43") .unwrap() .into_iter() .map(|u| u.object_path) .collect(); assert_eq!( got, vec![ "alloy/f43/a.rpm", "alloy/f43/b.rpm", "alloy/f43/repodata/1-primary.xml.zst", "alloy/f43/repodata/2-primary.xml.zst", "alloy/f43/repodata/repomd.xml", ] ); } #[test] fn collect_refuses_an_empty_file() { let dir = tempfile::tempdir().unwrap(); std::fs::write(dir.path().join("empty.rpm"), b"").unwrap(); let err = collect(dir.path(), "").unwrap_err().to_string(); assert!(err.contains("zero-byte"), "{err}"); } #[test] fn an_empty_prefix_publishes_at_the_bucket_root() { let dir = tempfile::tempdir().unwrap(); std::fs::write(dir.path().join("a.rpm"), b"pkg").unwrap(); let got = collect(dir.path(), "").unwrap(); assert_eq!(got[0].object_path, "a.rpm"); } }