Skip to main content

max / makenotwork

16.6 KB · 482 lines History Blame Raw
1 //! `mnw-cli rpm publish` — push an Alloy hotfix RPM repository to makenot.work.
2 //!
3 //! Takes the directory `createrepo_c` wrote and uploads it object by object.
4 //! Every PUT goes straight to object storage through a URL the server signs, so
5 //! this command holds no S3 credential — the reason the shape was chosen over
6 //! putting a write key on a laptop (alloy `32423bdd`, ruled (c)).
7 //!
8 //! Nothing about the repo's layout is decided here. The `--prefix` is handed
9 //! through verbatim and the local tree under `--dir` is mirrored beneath it, so
10 //! whatever layout the hosting task settles on is expressible without changing
11 //! this file.
12 //!
13 //! ## Upload order, and why it is not an implementation detail
14 //!
15 //! `repodata/repomd.xml` is the index: a client reads it, then fetches the
16 //! metadata and packages it names. So this uploads packages first, then the
17 //! hashed metadata, then `repomd.xml` and its signature last. A publish that
18 //! dies partway therefore leaves objects nothing points at — invisible to
19 //! `dnf`, and overwritten by the next run — rather than an index promising
20 //! packages that are not there yet.
21
22 use std::collections::BTreeMap;
23 use std::path::{Path, PathBuf};
24
25 use anyhow::{Context, Result, bail};
26 use synckit_client::{SyncKitClient, SyncKitConfig};
27
28 use crate::ota::authenticate_oauth;
29
30 const DEFAULT_SERVER: &str = "https://makenot.work";
31
32 /// Entry point for the `rpm` subcommand. `rest` is everything after `rpm`.
33 pub(crate) async fn run(rest: &[String]) -> Result<()> {
34 match rest.first().map(String::as_str) {
35 Some("publish") => publish(&rest[1..]).await,
36 Some("-h" | "--help") | None => {
37 print_usage();
38 Ok(())
39 }
40 Some(other) => {
41 eprintln!("Unknown rpm subcommand: {other}\n");
42 print_usage();
43 std::process::exit(2);
44 }
45 }
46 }
47
48 fn print_usage() {
49 eprintln!(
50 "Usage: mnw-cli rpm publish --dir DIR [--prefix PATH] [flags]\n\
51 \n\
52 Uploads a createrepo_c repository to the makenot.work RPM bucket.\n\
53 Packages go up first and repomd.xml last, so an interrupted publish\n\
54 never leaves an index pointing at objects that are not there.\n\
55 \n\
56 Required:\n\
57 \x20 --dir Repository root (the directory holding repodata/)\n\
58 \n\
59 Optional:\n\
60 \x20 --prefix Path prefix inside the bucket (e.g. alloy/f43/x86_64)\n\
61 \x20 --dry-run List what would be uploaded, in order, and stop\n\
62 \n\
63 \x20 --api-key SyncKit app API key (env MNW_OTA_API_KEY)\n\
64 \x20 --key SyncKit SDK key (env MNW_OTA_KEY)\n\
65 \x20 --server Server URL (env MNW_OTA_SERVER, default {DEFAULT_SERVER})\n\
66 \n\
67 Auth is MNW OAuth: a browser opens on this machine. The account must be\n\
68 the server's configured admin; anything else gets a 404."
69 );
70 }
71
72 struct PublishArgs {
73 dir: PathBuf,
74 prefix: String,
75 dry_run: bool,
76 api_key: String,
77 key: String,
78 server: String,
79 }
80
81 // Manual Debug that redacts the credentials so they never reach logs or a
82 // failing-test backtrace. Same discipline as `ota::PublishArgs`.
83 impl std::fmt::Debug for PublishArgs {
84 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
85 f.debug_struct("PublishArgs")
86 .field("dir", &self.dir)
87 .field("prefix", &self.prefix)
88 .field("dry_run", &self.dry_run)
89 .field("api_key", &"<redacted>")
90 .field("key", &"<redacted>")
91 .field("server", &self.server)
92 .finish()
93 }
94 }
95
96 fn parse_args(flags: &[String]) -> Result<PublishArgs> {
97 let mut dir = None;
98 let mut prefix = String::new();
99 let mut dry_run = false;
100 let mut api_key = std::env::var("MNW_OTA_API_KEY").ok();
101 let mut key = std::env::var("MNW_OTA_KEY").ok();
102 let mut server = std::env::var("MNW_OTA_SERVER").unwrap_or_else(|_| DEFAULT_SERVER.to_string());
103
104 let mut it = flags.iter();
105 while let Some(flag) = it.next() {
106 let mut take = |name: &str| -> Result<String> {
107 it.next()
108 .cloned()
109 .with_context(|| format!("{name} requires a value"))
110 };
111 match flag.as_str() {
112 "--dir" => dir = Some(PathBuf::from(take("--dir")?)),
113 "--prefix" => prefix = take("--prefix")?,
114 "--dry-run" => dry_run = true,
115 "--api-key" => api_key = Some(take("--api-key")?),
116 "--key" => key = Some(take("--key")?),
117 "--server" => server = take("--server")?,
118 "-h" | "--help" => {
119 print_usage();
120 std::process::exit(0);
121 }
122 other => bail!("Unknown flag: {other}"),
123 }
124 }
125
126 let missing = |name: &str| anyhow::anyhow!("missing required {name}");
127 // Trim the separators rather than rejecting them: `--prefix alloy/f43/` is
128 // what anyone would type, and the server's key validator refuses an empty
129 // segment, so passing it through untouched would fail on a typo that costs
130 // nothing to accept.
131 let prefix = prefix.trim_matches('/').to_string();
132
133 Ok(PublishArgs {
134 dir: dir.ok_or_else(|| missing("--dir"))?,
135 prefix,
136 dry_run,
137 // Credentials are only needed for a real publish; a dry run should work
138 // on a machine that has none.
139 api_key: api_key.unwrap_or_default(),
140 key: key.unwrap_or_default(),
141 server,
142 })
143 }
144
145 /// One object to upload: where it is locally, and where it goes in the bucket.
146 #[derive(Debug)]
147 struct Upload {
148 path: PathBuf,
149 object_path: String,
150 size: i64,
151 }
152
153 /// Which stage of the publish an object belongs to. The derived `Ord` is the
154 /// upload order, which is the point: sorting by this key is what makes the
155 /// index land last.
156 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
157 enum Stage {
158 /// Packages. Nothing references them until the metadata does.
159 Package,
160 /// Hashed metadata under `repodata/`. Content-addressed names, so a new one
161 /// never overwrites the one the live `repomd.xml` still points at.
162 Metadata,
163 /// `repomd.xml` and its detached signature / key. Rewritten in place, and
164 /// the moment they land the new metadata is live.
165 Index,
166 }
167
168 fn stage_of(object_path: &str) -> Stage {
169 let filename = object_path.rsplit('/').next().unwrap_or(object_path);
170 if filename.starts_with("repomd.xml") {
171 Stage::Index
172 } else if object_path.contains("/repodata/") || object_path.starts_with("repodata/") {
173 Stage::Metadata
174 } else {
175 Stage::Package
176 }
177 }
178
179 /// Walk `root` and pair every file with the object path it publishes to.
180 ///
181 /// Sorted by (stage, path): stage gives the ordering the publish depends on,
182 /// and the path tiebreak makes a run's output reproducible so two dry runs of
183 /// the same tree are diffable.
184 fn collect(root: &Path, prefix: &str) -> Result<Vec<Upload>> {
185 let mut found = BTreeMap::new();
186 walk(root, root, prefix, &mut found)?;
187 if found.is_empty() {
188 bail!("no files under {}", root.display());
189 }
190 Ok(found.into_values().collect())
191 }
192
193 fn walk(
194 root: &Path,
195 dir: &Path,
196 prefix: &str,
197 found: &mut BTreeMap<(Stage, String), Upload>,
198 ) -> Result<()> {
199 let entries =
200 std::fs::read_dir(dir).with_context(|| format!("reading directory {}", dir.display()))?;
201 for entry in entries {
202 let entry = entry.with_context(|| format!("reading directory {}", dir.display()))?;
203 let path = entry.path();
204 let meta = entry
205 .metadata()
206 .with_context(|| format!("stat {}", path.display()))?;
207
208 if meta.is_dir() {
209 walk(root, &path, prefix, found)?;
210 continue;
211 }
212 // Symlinks and anything else that is not a plain file: a repository is
213 // bytes at paths, and following a link out of the tree is exactly the
214 // publish nobody meant to make.
215 if !meta.is_file() {
216 bail!(
217 "{} is not a regular file. A repository directory holds only files and \
218 directories.",
219 path.display()
220 );
221 }
222
223 let relative = path
224 .strip_prefix(root)
225 .expect("walk only descends into root")
226 .to_str()
227 .with_context(|| format!("{} is not valid UTF-8", path.display()))?
228 .replace('\\', "/");
229
230 let object_path = if prefix.is_empty() {
231 relative
232 } else {
233 format!("{prefix}/{relative}")
234 };
235
236 let size: i64 = meta
237 .len()
238 .try_into()
239 .with_context(|| format!("{} is too large to publish", path.display()))?;
240 if size == 0 {
241 bail!(
242 "{} is empty; refusing to publish a zero-byte object",
243 path.display()
244 );
245 }
246
247 found.insert(
248 (stage_of(&object_path), object_path.clone()),
249 Upload {
250 path,
251 object_path,
252 size,
253 },
254 );
255 }
256 Ok(())
257 }
258
259 #[derive(serde::Serialize)]
260 struct PresignRequest<'a> {
261 path: &'a str,
262 size: i64,
263 }
264
265 #[derive(serde::Deserialize)]
266 struct PresignResponse {
267 upload_url: String,
268 object_key: String,
269 public_url: Option<String>,
270 content_type: String,
271 }
272
273 async fn publish(flags: &[String]) -> Result<()> {
274 let args = parse_args(flags)?;
275
276 if !args.dir.is_dir() {
277 bail!("--dir {} is not a directory", args.dir.display());
278 }
279 if !args.dir.join("repodata").is_dir() {
280 eprintln!(
281 "warning: {} has no repodata/ directory. dnf will not see a repository here \
282 until createrepo_c has run.",
283 args.dir.display()
284 );
285 }
286
287 let uploads = collect(&args.dir, &args.prefix)?;
288 let total_bytes: i64 = uploads.iter().map(|u| u.size).sum();
289
290 println!(
291 "Publishing {} objects ({total_bytes} bytes) from {} to {}",
292 uploads.len(),
293 args.dir.display(),
294 args.server
295 );
296
297 if args.dry_run {
298 for upload in &uploads {
299 println!(" {} ({} bytes)", upload.object_path, upload.size);
300 }
301 println!("\nDry run: nothing was uploaded.");
302 return Ok(());
303 }
304
305 if args.api_key.is_empty() {
306 bail!("missing required --api-key / MNW_OTA_API_KEY");
307 }
308 if args.key.is_empty() {
309 bail!("missing required --key / MNW_OTA_KEY");
310 }
311
312 let client = SyncKitClient::new(SyncKitConfig {
313 server_url: args.server.clone(),
314 api_key: args.api_key.clone(),
315 });
316 authenticate_oauth(&client, &args.key).await?;
317 let token = client
318 .session_info()
319 .context("authenticated but no session token was returned")?
320 .token;
321 println!(" authenticated");
322
323 let http = reqwest::Client::new();
324 let base = args.server.trim_end_matches('/');
325 let mut last_public_url = None;
326
327 for upload in &uploads {
328 print!(" {} ... ", upload.object_path);
329 // Read per object rather than up front: a repository can be many
330 // gigabytes, and only one object is in flight at a time.
331 let bytes = tokio::fs::read(&upload.path)
332 .await
333 .with_context(|| format!("reading {}", upload.path.display()))?;
334 // The server signed `size` into Content-Length. A file that changed
335 // between the walk and the read would fail at S3 with an opaque
336 // signature error, so catch it here where the message can say why.
337 if bytes.len() as i64 != upload.size {
338 bail!(
339 "{} changed size while publishing ({} bytes at scan, {} at read)",
340 upload.path.display(),
341 upload.size,
342 bytes.len()
343 );
344 }
345
346 let presign: PresignResponse = {
347 let response = http
348 .post(format!("{base}/api/v1/admin/rpm/uploads"))
349 .bearer_auth(&*token)
350 .json(&PresignRequest {
351 path: &upload.object_path,
352 size: upload.size,
353 })
354 .send()
355 .await
356 .with_context(|| format!("requesting a presign for {}", upload.object_path))?;
357 let status = response.status();
358 if status == reqwest::StatusCode::NOT_FOUND {
359 bail!(
360 "the server answered 404 for the publish endpoint. Either this build of the \
361 server predates it, or this account is not the configured admin."
362 );
363 }
364 if !status.is_success() {
365 let body = response.text().await.unwrap_or_default();
366 bail!(
367 "presign for {} failed ({status}): {body}",
368 upload.object_path
369 );
370 }
371 response
372 .json()
373 .await
374 .with_context(|| format!("decoding the presign for {}", upload.object_path))?
375 };
376
377 let put = http
378 .put(&presign.upload_url)
379 // Must match what the server signed, byte for byte, or S3 rejects
380 // the signature.
381 .header("content-type", &presign.content_type)
382 .body(bytes)
383 .send()
384 .await
385 .with_context(|| format!("uploading {}", upload.object_path))?;
386 if !put.status().is_success() {
387 let status = put.status();
388 let body = put.text().await.unwrap_or_default();
389 bail!("upload of {} failed ({status}): {body}", upload.object_path);
390 }
391
392 println!("ok ({})", presign.object_key);
393 if presign.public_url.is_some() {
394 last_public_url = presign.public_url;
395 }
396 }
397
398 println!("\nPublished {} objects.", uploads.len());
399 if let Some(url) = last_public_url {
400 // The index is the last object uploaded, so this is repomd.xml's URL —
401 // the one worth checking by hand.
402 println!("Index: {url}");
403 } else {
404 println!(
405 "The server has no RPM_BASE_URL configured, so it cannot say where these are \
406 served from."
407 );
408 }
409 Ok(())
410 }
411
412 #[cfg(test)]
413 mod tests {
414 use super::*;
415
416 #[test]
417 fn index_uploads_after_metadata_and_packages() {
418 assert!(
419 stage_of("alloy/f43/x86_64/alloy-1.2.3.rpm") < stage_of("repodata/abc-primary.xml.zst")
420 );
421 assert!(stage_of("repodata/abc-primary.xml.zst") < stage_of("repodata/repomd.xml"));
422 }
423
424 #[test]
425 fn repomd_signature_and_key_are_index_stage() {
426 assert_eq!(stage_of("a/repodata/repomd.xml.asc"), Stage::Index);
427 assert_eq!(stage_of("a/repodata/repomd.xml.key"), Stage::Index);
428 }
429
430 #[test]
431 fn a_package_inside_a_directory_named_repodata_is_still_metadata_stage() {
432 // Deliberate: anything under repodata/ is the index's business, and
433 // ordering it with the packages would put it ahead of nothing useful.
434 assert_eq!(stage_of("alloy/repodata/whatever.rpm"), Stage::Metadata);
435 }
436
437 #[test]
438 fn collect_orders_by_stage_then_path() {
439 let dir = tempfile::tempdir().unwrap();
440 let root = dir.path();
441 std::fs::create_dir(root.join("repodata")).unwrap();
442 std::fs::write(root.join("repodata/repomd.xml"), b"index").unwrap();
443 std::fs::write(root.join("repodata/2-primary.xml.zst"), b"meta2").unwrap();
444 std::fs::write(root.join("repodata/1-primary.xml.zst"), b"meta1").unwrap();
445 std::fs::write(root.join("b.rpm"), b"pkgb").unwrap();
446 std::fs::write(root.join("a.rpm"), b"pkga").unwrap();
447
448 let got: Vec<String> = collect(root, "alloy/f43")
449 .unwrap()
450 .into_iter()
451 .map(|u| u.object_path)
452 .collect();
453
454 assert_eq!(
455 got,
456 vec![
457 "alloy/f43/a.rpm",
458 "alloy/f43/b.rpm",
459 "alloy/f43/repodata/1-primary.xml.zst",
460 "alloy/f43/repodata/2-primary.xml.zst",
461 "alloy/f43/repodata/repomd.xml",
462 ]
463 );
464 }
465
466 #[test]
467 fn collect_refuses_an_empty_file() {
468 let dir = tempfile::tempdir().unwrap();
469 std::fs::write(dir.path().join("empty.rpm"), b"").unwrap();
470 let err = collect(dir.path(), "").unwrap_err().to_string();
471 assert!(err.contains("zero-byte"), "{err}");
472 }
473
474 #[test]
475 fn an_empty_prefix_publishes_at_the_bucket_root() {
476 let dir = tempfile::tempdir().unwrap();
477 std::fs::write(dir.path().join("a.rpm"), b"pkg").unwrap();
478 let got = collect(dir.path(), "").unwrap();
479 assert_eq!(got[0].object_path, "a.rpm");
480 }
481 }
482