Skip to main content

max / makenotwork

5.6 KB · 180 lines History Blame Raw
1 //! `/robots.txt` and `/sitemap.xml` for crawler discovery.
2
3 use std::fmt::Write as _;
4 use std::sync::{Mutex, OnceLock};
5 use std::time::{Duration, Instant};
6
7 use axum::{
8 extract::State,
9 http::header,
10 response::{IntoResponse, Response},
11 };
12 use chrono::{DateTime, Utc};
13 use sqlx::PgPool;
14
15 use crate::{config::Config, error::Result};
16
17 /// In-memory cache for the rendered sitemap XML. Crawlers hit this rarely
18 /// and the same response is fine for ~10 min; without the cache an attacker
19 /// hammering `/sitemap.xml` could pin the DB pool on two unbounded
20 /// `fetch_all` queries per request.
21 const SITEMAP_CACHE_TTL: Duration = Duration::from_mins(10);
22 static SITEMAP_CACHE: OnceLock<Mutex<Option<(Instant, String)>>> = OnceLock::new();
23
24 /// `/robots.txt`. Permits indexing the public surface; blocks the dashboard,
25 /// API, admin tooling, and authentication paths where indexed URLs would
26 /// be noise or actively harmful (login-page leakage, stale checkout URLs).
27 pub(super) async fn robots_txt(State(config): State<Config>) -> impl IntoResponse {
28 let host = &config.host_url;
29 let body = format!(
30 "User-agent: *\n\
31 Disallow: /dashboard\n\
32 Disallow: /admin\n\
33 Disallow: /api/\n\
34 Disallow: /auth/\n\
35 Disallow: /login\n\
36 Disallow: /logout\n\
37 Disallow: /join\n\
38 Disallow: /stripe/\n\
39 Disallow: /oauth/\n\
40 Disallow: /checkout/\n\
41 Disallow: /cart\n\
42 Disallow: /library\n\
43 Disallow: /buy/\n\
44 Disallow: /purchase/\n\
45 Disallow: /download/\n\
46 Disallow: /claim\n\
47 \n\
48 Sitemap: {host}/sitemap.xml\n"
49 );
50 ([(header::CONTENT_TYPE, "text/plain; charset=utf-8")], body)
51 }
52
53 /// `/sitemap.xml`. Includes the public top-level pages plus active creator
54 /// profiles and their public items. Capped to keep the response bounded and
55 /// cached for SITEMAP_CACHE_TTL to absorb crawler-or-attacker hammering
56 /// without firing two large `fetch_all` queries per request.
57 pub(super) async fn sitemap_xml(
58 State(db): State<PgPool>,
59 State(config): State<Config>,
60 ) -> Result<Response> {
61 let cache = SITEMAP_CACHE.get_or_init(|| Mutex::new(None));
62 if let Ok(guard) = cache.lock()
63 && let Some((generated_at, cached_xml)) = guard.as_ref()
64 && generated_at.elapsed() < SITEMAP_CACHE_TTL
65 {
66 return Ok((
67 [(header::CONTENT_TYPE, "application/xml; charset=utf-8")],
68 cached_xml.clone(),
69 )
70 .into_response());
71 }
72
73 let host = config.host_url.trim_end_matches('/').to_string();
74
75 // Active creator usernames (anyone with at least one public, listed,
76 // non-deleted item under a public project).
77 let creator_rows: Vec<(String,)> = sqlx::query_as(
78 r"
79 SELECT DISTINCT u.username
80 FROM users u
81 JOIN projects p ON p.user_id = u.id
82 JOIN items i ON i.project_id = p.id
83 WHERE p.is_public = true
84 AND i.is_public = true
85 AND i.listed = true
86 AND i.deleted_at IS NULL
87 ORDER BY u.username
88 LIMIT 5000
89 ",
90 )
91 .fetch_all(&db)
92 .await?;
93
94 // Public items (id + updated_at for <lastmod>).
95 let item_rows: Vec<(uuid::Uuid, DateTime<Utc>)> = sqlx::query_as(
96 r"
97 SELECT i.id, i.updated_at
98 FROM items i
99 JOIN projects p ON i.project_id = p.id
100 WHERE p.is_public = true
101 AND i.is_public = true
102 AND i.listed = true
103 AND i.deleted_at IS NULL
104 ORDER BY i.updated_at DESC
105 LIMIT 20000
106 ",
107 )
108 .fetch_all(&db)
109 .await?;
110
111 let mut xml = String::with_capacity(64 * 1024);
112 xml.push_str(r#"<?xml version="1.0" encoding="UTF-8"?>"#);
113 xml.push('\n');
114 xml.push_str(r#"<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">"#);
115 xml.push('\n');
116
117 // Top-level public pages
118 for path in [
119 "/",
120 "/discover",
121 "/creators",
122 "/pricing",
123 "/fan-plus",
124 "/policy",
125 "/changelog",
126 "/docs",
127 "/use-cases",
128 "/team",
129 ] {
130 writeln!(xml, " <url><loc>{host}{path}</loc></url>").unwrap();
131 }
132
133 for (username,) in &creator_rows {
134 writeln!(
135 xml,
136 " <url><loc>{host}/u/{}</loc></url>",
137 xml_escape(username),
138 )
139 .unwrap();
140 }
141
142 for (id, updated_at) in &item_rows {
143 writeln!(
144 xml,
145 " <url><loc>{host}/i/{id}</loc><lastmod>{}</lastmod></url>",
146 updated_at.format("%Y-%m-%d"),
147 )
148 .unwrap();
149 }
150
151 xml.push_str("</urlset>\n");
152
153 // Cache the rendered XML for the next SITEMAP_CACHE_TTL window. Multiple
154 // concurrent requests can each pass the stale check above, run the
155 // queries in parallel, then overwrite each other here, that's a
156 // self-correcting thundering-herd worth at most a few extra queries
157 // every 10 min, not worth a dedicated single-flight primitive.
158 if let Ok(mut guard) = cache.lock() {
159 *guard = Some((Instant::now(), xml.clone()));
160 }
161
162 Ok((
163 [(header::CONTENT_TYPE, "application/xml; charset=utf-8")],
164 xml,
165 )
166 .into_response())
167 }
168
169 /// Minimal XML escape for the handful of characters that matter inside
170 /// `<loc>` text. Usernames are already alphanumeric+underscore at validation
171 /// time, so this is belt-and-braces, but the encoder must exist if any
172 /// future content gets piped through.
173 fn xml_escape(s: &str) -> String {
174 s.replace('&', "&amp;")
175 .replace('<', "&lt;")
176 .replace('>', "&gt;")
177 .replace('"', "&quot;")
178 .replace('\'', "&apos;")
179 }
180