Skip to main content

max / makenotwork

docengine: add link graph, broken-link detection, and backlinks Build an internal-link adjacency graph in DocLoader during the load it already does, then expose broken_links() (an internal [..](x.md) link resolving to a slug no page serves) and backlinks() ("what links here"), both mirroring the existing collisions() report contract: warn in logs, expose an accessor, leave hard-fail to the caller. Link classification is factored into one classify_link shared by rewrite_links and the graph collector, so graph edges are exactly the links that get rewritten and served. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-23 15:37 UTC
Commit: 6d74cacd4da1e7ea18269d0cc1ac159dbc4fa079
Parent: 91e82ec
1 file changed, +326 insertions, -42 deletions
M shared/docengine/src/doc_loader.rs +326 -42
@@ -101,12 +101,38 @@ pub struct SlugCollision {
101 101 pub winning_section: String,
102 102 }
103 103
104 + /// An internal doc link whose target slug matches no loaded page.
105 + ///
106 + /// `[text](missing.md)` rewrites to `{link_prefix}/missing` and serves a live
107 + /// link to a 404 with nothing reporting it. A broken link is an edge in the
108 + /// link graph whose target is absent from the page store — the same notion of
109 + /// "present" the router uses ([`DocLoader::get`]), so detection is faithful to
110 + /// what is actually served.
111 + #[derive(Clone, Debug, PartialEq, Eq)]
112 + pub struct BrokenLink {
113 + /// Slug of the page containing the link.
114 + pub source_slug: String,
115 + /// Target slug the link resolves to, which no page serves.
116 + pub target_slug: String,
117 + }
118 +
104 119 /// In-memory store of rendered documentation pages, built once at startup.
105 120 #[derive(Clone, Debug)]
106 121 pub struct DocLoader {
107 122 pages: HashMap<String, DocPage>,
108 123 index: Vec<DocIndexEntry>,
109 124 collisions: Vec<SlugCollision>,
125 + /// Adjacency list of the internal-link graph: source slug -> the target
126 + /// slugs it links to, in first-seen order, deduplicated per source. Only
127 + /// internal cross-doc `.md` links appear; external, `mailto:`, absolute,
128 + /// and stripped-unpublished links are excluded, exactly as [`rewrite_links`]
129 + /// classifies them.
130 + links: HashMap<String, Vec<String>>,
131 + /// Reverse of [`links`](Self::links): target slug -> the source slugs that
132 + /// link to it ("what links here"), each source once, in docs-index order.
133 + /// A key may be a slug no page serves (a broken target still has backlinks).
134 + backlinks: HashMap<String, Vec<String>>,
135 + broken: Vec<BrokenLink>,
110 136 }
111 137
112 138 impl DocLoader {
@@ -117,6 +143,7 @@ impl DocLoader {
117 143 let mut pages = HashMap::new();
118 144 let mut index = Vec::new();
119 145 let mut collisions = Vec::new();
146 + let mut links: HashMap<String, Vec<String>> = HashMap::new();
120 147
121 148 for (dir_name, section_display) in &config.sections {
122 149 let section_path = base_path.join(dir_name);
@@ -168,6 +195,11 @@ impl DocLoader {
168 195 };
169 196
170 197 let title = crate::text::extract_title(&raw_md).unwrap_or_else(|| slug.clone());
198 + // Collect the internal-link graph from the same pre-processed
199 + // markdown `rewrite_links` sees, so graph edges are exactly the
200 + // links actually served.
201 + let link_targets =
202 + collect_link_targets(&raw_md, config.unpublished_pattern.as_deref());
171 203 let rewritten_md = rewrite_links(
172 204 &raw_md,
173 205 &config.link_prefix,
@@ -198,6 +230,11 @@ impl DocLoader {
198 230 section: page.section.clone(),
199 231 });
200 232
233 + // Key the graph by the served slug, last-writer-wins in step
234 + // with the page store above, so the graph describes the page a
235 + // caller actually reaches under this slug.
236 + links.insert(page.slug.clone(), link_targets);
237 +
201 238 let slug_key = page.slug.clone();
202 239 if let Some(displaced) = pages.insert(slug_key, page) {
203 240 // Last writer wins, as it always has. Surfacing the clash
@@ -221,10 +258,50 @@ impl DocLoader {
221 258 }
222 259 }
223 260
261 + // Resolve broken links once the full page set is known: a forward link
262 + // to a page loaded later in the corpus is not broken. Walk in index
263 + // order (not the nondeterministic map order) so the report is stable.
264 + let mut broken = Vec::new();
265 + let mut backlinks: HashMap<String, Vec<String>> = HashMap::new();
266 + let mut seen = std::collections::HashSet::new();
267 + for entry in &index {
268 + // A slug collision indexes two pages under one slug; the graph holds
269 + // only the winner's edges. Process each slug once so the loser's
270 + // duplicate index entry doesn't double-report or double-link.
271 + if !seen.insert(entry.slug.as_str()) {
272 + continue;
273 + }
274 + let Some(targets) = links.get(&entry.slug) else {
275 + continue;
276 + };
277 + for target in targets {
278 + // Reverse edge: index-ordered, one source per target (targets are
279 + // already deduped per source, so no duplicate source appears).
280 + backlinks
281 + .entry(target.clone())
282 + .or_default()
283 + .push(entry.slug.clone());
284 + if !pages.contains_key(target) {
285 + tracing::warn!(
286 + source = %entry.slug,
287 + target = %target,
288 + "docs broken link: internal link resolves to a slug no page serves"
289 + );
290 + broken.push(BrokenLink {
291 + source_slug: entry.slug.clone(),
292 + target_slug: target.clone(),
293 + });
294 + }
295 + }
296 + }
297 +
224 298 DocLoader {
225 299 pages,
226 300 index,
227 301 collisions,
302 + links,
303 + backlinks,
304 + broken,
228 305 }
229 306 }
230 307
@@ -247,6 +324,37 @@ impl DocLoader {
247 324 &self.collisions
248 325 }
249 326
327 + /// Target slugs the page `slug` links to, in first-seen order.
328 + ///
329 + /// Only internal cross-doc `.md` links are recorded; a target here need not
330 + /// resolve to a live page (see [`broken_links`](Self::broken_links)). Empty
331 + /// for a page with no internal links, or for an unknown slug.
332 + pub fn links(&self, slug: &str) -> &[String] {
333 + self.links.get(slug).map_or(&[], Vec::as_slice)
334 + }
335 +
336 + /// Source slugs that link to the page `slug` ("what links here"), in
337 + /// docs-index order, each source once.
338 + ///
339 + /// Reads straight off the reverse link graph — the wiki-style backlinks a
340 + /// page template can render. Empty for a page nothing links to, or an
341 + /// unknown slug.
342 + pub fn backlinks(&self, slug: &str) -> &[String] {
343 + self.backlinks.get(slug).map_or(&[], Vec::as_slice)
344 + }
345 +
346 + /// Internal links whose target slug matches no loaded page, in docs-index
347 + /// order.
348 + ///
349 + /// Empty for a healthy corpus. A non-empty result means a page serves a link
350 + /// to a 404; each is also logged at `warn`. As with
351 + /// [`collisions`](Self::collisions), whether that warns or fails the boot is
352 + /// the caller's call — a build step or startup check can fail loudly on a
353 + /// non-empty slice.
354 + pub fn broken_links(&self) -> &[BrokenLink] {
355 + &self.broken
356 + }
357 +
250 358 /// Build a search index with HTML stripped to plain text.
251 359 pub fn search_index(&self) -> Vec<DocSearchEntry> {
252 360 self.index
@@ -342,58 +450,100 @@ fn strip_html_tags(html: &str) -> String {
342 450 .replace("&#39;", "'")
343 451 }
344 452
453 + /// How a markdown link URL is treated by the doc pipeline. Shared by
454 + /// [`rewrite_links`] (which rewrites) and [`collect_link_targets`] (which graphs
455 + /// the edges) so the two share one notion of what an internal link is.
456 + enum LinkKind<'a> {
457 + /// Absolute URL, `mailto:`, internal route, or non-`.md` link — left
458 + /// untouched by rewriting and not a graph edge.
459 + Passthrough,
460 + /// Matches the unpublished pattern — link stripped, text kept; not an edge.
461 + Unpublished,
462 + /// Internal cross-doc `.md` link resolving to `slug`, with an optional
463 + /// `#anchor`. This is the one case that is both rewritten and a graph edge.
464 + Internal {
465 + slug: &'a str,
466 + anchor: Option<&'a str>,
467 + },
468 + }
469 +
470 + /// Classify a link URL. Pure over `url`; the sole source of truth for "is this
471 + /// an internal doc link, and to which slug".
472 + fn classify_link<'a>(url: &'a str, unpublished_pattern: Option<&str>) -> LinkKind<'a> {
473 + // Preserve absolute URLs, mailto, and internal routes.
474 + if url.starts_with("http://")
475 + || url.starts_with("https://")
476 + || url.starts_with("mailto:")
477 + || url.starts_with('/')
478 + {
479 + return LinkKind::Passthrough;
480 + }
481 +
482 + // Unpublished docs: strip link, keep text.
483 + if let Some(pattern) = unpublished_pattern
484 + && url.contains(pattern)
485 + {
486 + return LinkKind::Unpublished;
487 + }
488 +
489 + // Only internal links containing .md resolve to a doc slug.
490 + if !url.contains(".md") {
491 + return LinkKind::Passthrough;
492 + }
493 +
494 + // Split off any #anchor.
495 + let (path_part, anchor) = match url.split_once('#') {
496 + Some((p, a)) => (p, Some(a)),
497 + None => (url, None),
498 + };
499 +
500 + // Extract slug from filename: ../support/faq.md -> faq
501 + let slug = path_part
502 + .rsplit('/')
503 + .next()
504 + .unwrap_or(path_part)
505 + .trim_end_matches(".md");
506 +
507 + LinkKind::Internal { slug, anchor }
508 + }
509 +
345 510 /// Rewrite relative `.md` links to the configured prefix.
346 511 fn rewrite_links(markdown: &str, link_prefix: &str, unpublished_pattern: Option<&str>) -> String {
347 512 LINK_RE
348 513 .replace_all(markdown, |caps: &regex_lite::Captures| {
349 514 let text = &caps[1];
350 - let url = &caps[2];
351 -
352 - // Preserve absolute URLs, mailto, and internal routes.
353 - if url.starts_with("http://")
354 - || url.starts_with("https://")
355 - || url.starts_with("mailto:")
356 - || url.starts_with('/')
357 - {
358 - return caps[0].to_string();
359 - }
360 -
361 - // Unpublished docs: strip link, keep text.
362 - if let Some(pattern) = unpublished_pattern
363 - && url.contains(pattern)
364 - {
365 - return text.to_string();
366 - }
367 -
368 - // Only rewrite links containing .md
369 - if !url.contains(".md") {
370 - return caps[0].to_string();
371 - }
372 -
373 - // Split off any #anchor.
374 - let (path_part, anchor): (&str, Option<&str>) = match url.split_once('#') {
375 - Some((p, a)) => (p, Some(a)),
376 - None => (url, None),
377 - };
378 -
379 - // Extract slug from filename: ../support/faq.md -> faq
380 - let filename = path_part
381 - .rsplit('/')
382 - .next()
383 - .unwrap_or(path_part)
384 - .trim_end_matches(".md");
385 -
386 - let mut new_url = format!("{link_prefix}/{filename}");
387 - if let Some(anchor) = anchor {
388 - new_url.push('#');
389 - new_url.push_str(anchor);
515 + match classify_link(&caps[2], unpublished_pattern) {
516 + LinkKind::Passthrough => caps[0].to_string(),
517 + LinkKind::Unpublished => text.to_string(),
518 + LinkKind::Internal { slug, anchor } => {
519 + let mut new_url = format!("{link_prefix}/{slug}");
520 + if let Some(anchor) = anchor {
521 + new_url.push('#');
522 + new_url.push_str(anchor);
523 + }
524 + format!("[{text}]({new_url})")
525 + }
390 526 }
391 -
392 - format!("[{text}]({new_url})")
393 527 })
394 528 .to_string()
395 529 }
396 530
531 + /// Collect the internal-link targets of a page, deduplicated in first-seen
532 + /// order. Reads the same pre-processed markdown and the same [`classify_link`]
533 + /// rules as [`rewrite_links`], so every recorded edge corresponds to a link that
534 + /// is actually rewritten and served.
535 + fn collect_link_targets(markdown: &str, unpublished_pattern: Option<&str>) -> Vec<String> {
536 + let mut targets: Vec<String> = Vec::new();
537 + for caps in LINK_RE.captures_iter(markdown) {
538 + if let LinkKind::Internal { slug, .. } = classify_link(&caps[2], unpublished_pattern)
539 + && !targets.iter().any(|t| t == slug)
540 + {
541 + targets.push(slug.to_string());
542 + }
543 + }
544 + targets
545 + }
546 +
397 547 #[cfg(test)]
398 548 mod tests {
399 549 use super::*;
@@ -795,6 +945,140 @@ mod tests {
795 945 assert!(e.body_text.contains("inline code"));
796 946 }
797 947
948 + // ── link graph + broken-link detection ──
949 +
950 + #[test]
951 + fn collect_link_targets_dedups_internal_md_links_only() {
952 + let md = "See [a](./a.md), [b](../x/b.md#sec), [a again](a.md), \
953 + [ext](https://e.com/z.md), [img](./p.png), [mail](mailto:x@y.md).";
954 + let targets = collect_link_targets(md, Some("unpublished/"));
955 + // Only internal .md links, deduped, anchors dropped; external/mailto/png
956 + // excluded exactly as rewrite_links classifies them.
957 + assert_eq!(targets, vec!["a".to_string(), "b".to_string()]);
958 + }
959 +
960 + #[test]
961 + fn collect_link_targets_excludes_unpublished() {
962 + let md = "[hidden](../unpublished/legal/x.md) and [ok](./ok.md).";
963 + let targets = collect_link_targets(md, Some("unpublished/"));
964 + assert_eq!(targets, vec!["ok".to_string()]);
965 + }
966 +
967 + #[test]
968 + fn links_graph_records_outbound_edges() {
969 + let tmp = tempfile::tempdir().unwrap();
970 + let base = tmp.path();
971 + write(
972 + base,
973 + "guide/main.md",
974 + "# Main\n\nSee [FAQ](../support/faq.md).",
975 + );
976 + write(base, "support/faq.md", "# FAQ\n\nbody");
977 +
978 + let loader = DocLoader::load(base, &config_for(base));
979 + assert_eq!(loader.links("main"), ["faq".to_string()]);
980 + assert!(loader.links("faq").is_empty());
981 + assert!(loader.links("does-not-exist").is_empty());
982 + }
983 +
984 + #[test]
985 + fn broken_link_to_missing_page_is_reported() {
986 + let tmp = tempfile::tempdir().unwrap();
987 + let base = tmp.path();
988 + write(
989 + base,
990 + "guide/main.md",
991 + "# Main\n\nSee [gone](./ghost.md) and [ok](./ok.md).",
992 + );
993 + write(base, "guide/ok.md", "# OK\n\nbody");
994 +
995 + let loader = DocLoader::load(base, &config_for(base));
996 + assert_eq!(
997 + loader.broken_links(),
998 + [BrokenLink {
999 + source_slug: "main".to_string(),
1000 + target_slug: "ghost".to_string(),
1001 + }]
1002 + );
1003 + }
1004 +
1005 + #[test]
1006 + fn forward_link_to_later_page_is_not_broken() {
1007 + // `main` (guide) links a page defined later in `support`. Broken-link
1008 + // resolution must run after the whole corpus is loaded.
1009 + let tmp = tempfile::tempdir().unwrap();
1010 + let base = tmp.path();
1011 + write(base, "guide/main.md", "# Main\n\nSee [FAQ](faq.md).");
1012 + write(base, "support/faq.md", "# FAQ\n\nbody");
1013 +
1014 + let loader = DocLoader::load(base, &config_for(base));
1015 + assert!(
1016 + loader.broken_links().is_empty(),
1017 + "forward reference wrongly flagged: {:?}",
1018 + loader.broken_links()
1019 + );
1020 + }
1021 +
1022 + #[test]
1023 + fn healthy_corpus_has_no_broken_links() {
1024 + let tmp = tempfile::tempdir().unwrap();
1025 + let base = tmp.path();
1026 + write(base, "guide/a.md", "# A\n\n[to b](./b.md)");
1027 + write(base, "guide/b.md", "# B\n\n[to a](./a.md)");
1028 +
1029 + let loader = DocLoader::load(base, &config_for(base));
1030 + assert!(loader.broken_links().is_empty());
1031 + }
1032 +
1033 + #[test]
1034 + fn unpublished_link_is_never_broken() {
1035 + // Stripped to plain text, so it is not an edge and cannot be broken.
1036 + let tmp = tempfile::tempdir().unwrap();
1037 + let base = tmp.path();
1038 + write(
1039 + base,
1040 + "guide/main.md",
1041 + "# Main\n\nSee [hidden](../unpublished/x.md).",
1042 + );
1043 +
1044 + let loader = DocLoader::load(base, &config_for(base));
1045 + assert!(loader.broken_links().is_empty());
1046 + assert!(loader.links("main").is_empty());
1047 + }
1048 +
1049 + #[test]
1050 + fn backlinks_reverse_the_link_graph_in_index_order() {
1051 + let tmp = tempfile::tempdir().unwrap();
1052 + let base = tmp.path();
1053 + // Two guide pages both link the support hub; index order is
1054 + // guide/aaa, guide/zzz, support/hub.
1055 + write(base, "guide/aaa.md", "# A\n\n[hub](../support/hub.md)");
1056 + write(base, "guide/zzz.md", "# Z\n\n[hub](../support/hub.md)");
1057 + write(base, "support/hub.md", "# Hub\n\nno outbound links");
1058 +
1059 + let loader = DocLoader::load(base, &config_for(base));
1060 + // "what links here" for hub: both sources, once each, in index order.
1061 + assert_eq!(
1062 + loader.backlinks("hub"),
1063 + ["aaa".to_string(), "zzz".to_string()]
1064 + );
1065 + // A page nothing links to, and an unknown slug, are both empty.
1066 + assert!(loader.backlinks("aaa").is_empty());
1067 + assert!(loader.backlinks("nope").is_empty());
1068 + }
1069 +
1070 + #[test]
1071 + fn backlinks_exist_even_for_a_broken_target() {
1072 + // "what links here" is answerable for a slug no page serves — that is
1073 + // precisely the set of pages whose link to it is broken.
1074 + let tmp = tempfile::tempdir().unwrap();
1075 + let base = tmp.path();
1076 + write(base, "guide/a.md", "# A\n\n[ghost](./ghost.md)");
1077 +
1078 + let loader = DocLoader::load(base, &config_for(base));
1079 + assert_eq!(loader.backlinks("ghost"), ["a".to_string()]);
1080 + }
1081 +
798 1082 // ── resolve_ui_examples ──
799 1083
800 1084 #[test]