Skip to main content

max / makenotwork

6.5 KB · 201 lines History Blame Raw
1 //! `/robots.txt` and `/sitemap.xml` for crawler discovery.
2
3 use std::fmt::Write as _;
4 use std::sync::{Mutex, OnceLock};
5 use std::time::{Duration, Instant};
6
7 use axum::{
8 extract::State,
9 http::header,
10 response::{IntoResponse, Response},
11 };
12 use chrono::{DateTime, Utc};
13 use sqlx::PgPool;
14
15 use crate::{config::Config, error::Result};
16
17 /// In-memory cache for the rendered sitemap XML. Crawlers hit this rarely
18 /// and the same response is fine for ~10 min; without the cache an attacker
19 /// hammering `/sitemap.xml` could pin the DB pool on two unbounded
20 /// `fetch_all` queries per request.
21 const SITEMAP_CACHE_TTL: Duration = Duration::from_mins(10);
22 static SITEMAP_CACHE: OnceLock<Mutex<Option<(Instant, String)>>> = OnceLock::new();
23
24 /// `/robots.txt`. Permits indexing the public surface; blocks the dashboard,
25 /// API, admin tooling, and authentication paths where indexed URLs would
26 /// be noise or actively harmful (login-page leakage, stale checkout URLs).
27 pub(super) async fn robots_txt(State(config): State<Config>) -> impl IntoResponse {
28 let host = &config.host_url;
29 let body = format!(
30 "User-agent: *\n\
31 Disallow: /dashboard\n\
32 Disallow: /admin\n\
33 Disallow: /api/\n\
34 Disallow: /auth/\n\
35 Disallow: /login\n\
36 Disallow: /logout\n\
37 Disallow: /join\n\
38 Disallow: /stripe/\n\
39 Disallow: /oauth/\n\
40 Disallow: /checkout/\n\
41 Disallow: /cart\n\
42 Disallow: /library\n\
43 Disallow: /buy/\n\
44 Disallow: /purchase/\n\
45 Disallow: /download/\n\
46 Disallow: /claim\n\
47 \n\
48 Sitemap: {host}/sitemap.xml\n"
49 );
50 ([(header::CONTENT_TYPE, "text/plain; charset=utf-8")], body)
51 }
52
53 /// `/sitemap.xml`. Includes the public top-level pages plus active creator
54 /// profiles and their public items. Capped to keep the response bounded and
55 /// cached for SITEMAP_CACHE_TTL to absorb crawler-or-attacker hammering
56 /// without firing two large `fetch_all` queries per request.
57 pub(super) async fn sitemap_xml(
58 State(db): State<PgPool>,
59 State(config): State<Config>,
60 State(docs): State<std::sync::Arc<docengine::DocLoader>>,
61 ) -> Result<Response> {
62 let cache = SITEMAP_CACHE.get_or_init(|| Mutex::new(None));
63 if let Ok(guard) = cache.lock()
64 && let Some((generated_at, cached_xml)) = guard.as_ref()
65 && generated_at.elapsed() < SITEMAP_CACHE_TTL
66 {
67 return Ok((
68 [(header::CONTENT_TYPE, "application/xml; charset=utf-8")],
69 cached_xml.clone(),
70 )
71 .into_response());
72 }
73
74 let host = config.host_url.trim_end_matches('/').to_string();
75
76 // Active creator usernames (anyone with at least one public, listed,
77 // non-deleted item under a public project).
78 let creator_rows: Vec<(String,)> = sqlx::query_as(
79 r"
80 SELECT DISTINCT u.username
81 FROM users u
82 JOIN projects p ON p.user_id = u.id
83 JOIN items i ON i.project_id = p.id
84 WHERE p.is_public = true
85 AND i.is_public = true
86 AND i.listed = true
87 AND i.deleted_at IS NULL
88 ORDER BY u.username
89 LIMIT 5000
90 ",
91 )
92 .fetch_all(&db)
93 .await?;
94
95 // Public items (id + updated_at for <lastmod>).
96 let item_rows: Vec<(uuid::Uuid, DateTime<Utc>)> = sqlx::query_as(
97 r"
98 SELECT i.id, i.updated_at
99 FROM items i
100 JOIN projects p ON i.project_id = p.id
101 WHERE p.is_public = true
102 AND i.is_public = true
103 AND i.listed = true
104 AND i.deleted_at IS NULL
105 ORDER BY i.updated_at DESC
106 LIMIT 20000
107 ",
108 )
109 .fetch_all(&db)
110 .await?;
111
112 let mut xml = String::with_capacity(64 * 1024);
113 xml.push_str(r#"<?xml version="1.0" encoding="UTF-8"?>"#);
114 xml.push('\n');
115 xml.push_str(r#"<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">"#);
116 xml.push('\n');
117
118 // Top-level public pages. `/changelog` is an alias for the changelog
119 // project's blog and 404s until that project is published, so listing it
120 // unconditionally hands crawlers a dead URL. Same invariant the footer
121 // already honours in `base.html`; see `crate::changelog`. The sitemap is
122 // cached for `SITEMAP_CACHE_TTL`, so the entry appears up to that long
123 // after the project goes public.
124 for path in [
125 "/",
126 "/discover",
127 "/creators",
128 "/pricing",
129 "/fan-plus",
130 "/policy",
131 "/changelog",
132 "/docs",
133 "/use-cases",
134 "/team",
135 ] {
136 if path == "/changelog" && !crate::changelog::is_published() {
137 continue;
138 }
139 writeln!(xml, " <url><loc>{host}{path}</loc></url>").unwrap();
140 }
141
142 // Every published doc page. These are the largest body of static public
143 // content on the site and the set is fixed at load time, so there is no
144 // cap to apply and no query behind it.
145 for entry in docs.index() {
146 writeln!(
147 xml,
148 " <url><loc>{host}/docs/{}</loc></url>",
149 xml_escape(&entry.slug),
150 )
151 .unwrap();
152 }
153
154 for (username,) in &creator_rows {
155 writeln!(
156 xml,
157 " <url><loc>{host}/u/{}</loc></url>",
158 xml_escape(username),
159 )
160 .unwrap();
161 }
162
163 for (id, updated_at) in &item_rows {
164 writeln!(
165 xml,
166 " <url><loc>{host}/i/{id}</loc><lastmod>{}</lastmod></url>",
167 updated_at.format("%Y-%m-%d"),
168 )
169 .unwrap();
170 }
171
172 xml.push_str("</urlset>\n");
173
174 // Cache the rendered XML for the next SITEMAP_CACHE_TTL window. Multiple
175 // concurrent requests can each pass the stale check above, run the
176 // queries in parallel, then overwrite each other here, that's a
177 // self-correcting thundering-herd worth at most a few extra queries
178 // every 10 min, not worth a dedicated single-flight primitive.
179 if let Ok(mut guard) = cache.lock() {
180 *guard = Some((Instant::now(), xml.clone()));
181 }
182
183 Ok((
184 [(header::CONTENT_TYPE, "application/xml; charset=utf-8")],
185 xml,
186 )
187 .into_response())
188 }
189
190 /// Minimal XML escape for the handful of characters that matter inside
191 /// `<loc>` text. Usernames are already alphanumeric+underscore at validation
192 /// time, so this is belt-and-braces, but the encoder must exist if any
193 /// future content gets piped through.
194 fn xml_escape(s: &str) -> String {
195 s.replace('&', "&amp;")
196 .replace('<', "&lt;")
197 .replace('>', "&gt;")
198 .replace('"', "&quot;")
199 .replace('\'', "&apos;")
200 }
201