Skip to main content

max / makenotwork

Search discover with Postgres full-text, not trigram Add tsvector columns and indexes for items and projects, a word-match scoring function, and the query path that reads them. Drop the posts body trigram index in multithreaded, which no longer has a reader.
Author: Max Johnson <me@maxj.phd> · 2026-07-30 20:01 UTC
Signed with PGP, not checked
Commit: 95f7250349a8ce86e17fc85f901dfa1039598fc7
Parent: d2582c1
20 files changed, +1092 insertions, -42 deletions
M server/static/style.css +77
@@ -11474,3 +11474,80 @@
11474 11474 whose class didn't already set an aspect-ratio / fixed size. */
11475 11475 .library-locked-cover { aspect-ratio: 1 / 1; object-fit: cover; }
11476 11476 .proj-image-preview img { width: 120px; height: 120px; object-fit: cover; }
11477 +
11478 + /* ---- Discover search: how well a result matched -------------------------
11479 + Two surfaces, doing two different jobs (db/discover.rs, migration 179).
11480 +
11481 + .results-tier-break is the single heading, and it sits exactly where the
11482 + claim it makes turns true: above it every row holds at least one word the
11483 + searcher typed, below it none of them do. .row-match-note is the per-row
11484 + half, "Matched 3 of 5 words", because coverage varies row to row and one
11485 + page can hold a 4-of-5 beside a 1-of-5.
11486 +
11487 + Both are deliberately quiet. They explain a result the reader is already
11488 + looking at; competing with the title would make search feel like it is
11489 + apologising. Muted colour and small type, no accent, no border of their own
11490 + beyond the rule that separates the tiers. */
11491 + .results-tier-break {
11492 + display: flex;
11493 + align-items: baseline;
11494 + gap: var(--space-2);
11495 + flex-wrap: wrap;
11496 + padding: var(--space-3) 0.75rem var(--space-2);
11497 + border-top: 1px solid var(--border);
11498 + background: var(--surface-page);
11499 + }
11500 +
11501 + /* .results-table sets border-top: none and lets .table-row draw the rules, so
11502 + the break supplies its own top border. Mid-list that sits against the
11503 + preceding row's border-bottom and reads as one heavier 2px rule, which is
11504 + wanted here: it is a section change, not another row. At the top of an
11505 + all-fuzzy page there is no preceding row, and the same border supplies the
11506 + table's missing top edge. */
11507 + .results-tier-label {
11508 + font-family: var(--font-heading);
11509 + font-weight: bold;
11510 + font-size: 0.8rem;
11511 + text-transform: uppercase;
11512 + letter-spacing: 0.03em;
11513 + color: var(--content);
11514 + }
11515 +
11516 + .results-tier-hint {
11517 + font-size: 0.75rem;
11518 + color: var(--content-muted);
11519 + }
11520 +
11521 + /* The grid is `repeat(auto-fill, minmax(280px, 1fr))`, so without this the
11522 + heading takes one card's slot and the tiers interleave visually. */
11523 + .results-container.results-grid .results-tier-break {
11524 + grid-column: 1 / -1;
11525 + padding-left: 0;
11526 + padding-right: 0;
11527 + background: none;
11528 + }
11529 +
11530 + /* In list view this rides inside .row-info, under the creator line, not as
11531 + another child of .table-row-link: that is a fixed five-column grid and a
11532 + sixth child would invent a column and shift every row that carried one. */
11533 + .row-match-note {
11534 + font-family: var(--font-mono);
11535 + font-size: 0.7rem;
11536 + color: var(--content-muted);
11537 + white-space: nowrap;
11538 + overflow: hidden;
11539 + text-overflow: ellipsis;
11540 + }
11541 +
11542 + /* Grid cards have room to breathe; give the note its own line under the meta. */
11543 + .grid-card .row-match-note {
11544 + display: block;
11545 + margin-top: var(--space-1);
11546 + }
11547 +
11548 + /* The narrow list layouts drop columns rather than wrap, and .row-info is the
11549 + column that survives. Keeping the note there would crowd the title on a
11550 + phone, where the tier heading above still carries the important half. */
11551 + @media (max-width: 700px) {
11552 + .table-row .row-match-note { display: none; }
11553 + }
M server/templates/base.html +3 -1
@@ -32,7 +32,9 @@
32 32 <a href="/docs">Docs</a>
33 33 <a href="/policy">Legal</a>
34 34 <a href="/docs/credits">Credits</a>
35 - <a href="/changelog">Changelog</a>
35 + {# Linked only while a published changelog project exists; the route
36 + 404s otherwise. See crate::changelog. #}
37 + {% if crate::changelog::is_published() %}<a href="/changelog">Changelog</a>{% endif %}
36 38 <a href="mailto:info@makenot.work">Contact</a>
37 39 <a href="/health" title="Service status and uptime">Status</a>
38 40 <a href="#" data-prevent data-action="showWhatsNewModal">What's new</a>
M server/src/db/discover.rs +246 -26
@@ -50,24 +50,55 @@
50 50 }
51 51 }
52 52
53 - // Shared SQL fragments for fuzzy search (trigram + ILIKE fallback).
53 + // Shared SQL fragments for search (lexical tsvector + trigram/ILIKE fallback).
54 54 // The ILIKE escapes \, %, _ in the search term to prevent LIKE metacharacter interpretation.
55 55
56 + /// Candidate membership: any word the searcher typed, none they excluded.
57 + ///
58 + /// A macro rather than a `const` so `concat!` can splice it into the clauses at
59 + /// compile time; written once, it cannot drift between the predicate that
60 + /// decides a row matches and the expression that counts how well.
61 + ///
62 + /// This is deliberately not `websearch_to_tsquery`, which ANDs every term. That
63 + /// put a cliff at 100% coverage: a row holding four of five typed words was
64 + /// excluded from the word tier alongside rows holding none, and the UI then
65 + /// told both of them they had matched on spelling only. The rule is now the one
66 + /// stated at the top of migration 179 -- exclusions filter, everything else
67 + /// ranks -- and the tier is a count rather than a yes/no.
68 + ///
69 + /// The function is IMMUTABLE and takes a constant config, so repeating it in
70 + /// the same statement does not repeat the work.
71 + macro_rules! any_term_query {
72 + () => {
73 + "mnw_any_term_query($1)"
74 + };
75 + }
76 +
56 77 // The description terms are written as `COALESCE(i.description, '')` to match
57 78 // `idx_items_desc_trgm`, which is built on that expression rather than on the
58 79 // bare column. Written any other way Postgres will not use the index, and this
59 80 // clause runs on the busiest public page.
60 - const ITEM_SEARCH_CLAUSE: &str = r" AND (
61 - i.title % $1
81 + const ITEM_SEARCH_CLAUSE: &str = concat!(
82 + r" AND (
83 + i.search_tsv @@ ",
84 + any_term_query!(),
85 + r"
86 + OR i.title % $1
62 87 OR i.title ILIKE '%' || replace(replace(replace($1, '\', '\\'), '%', '\%'), '_', '\_') || '%'
63 88 OR COALESCE(i.description, '') % $1
64 89 OR COALESCE(i.description, '') ILIKE '%' || replace(replace(replace($1, '\', '\\'), '%', '\%'), '_', '\_') || '%'
65 - )";
90 + )"
91 + );
66 92
67 - const PROJECT_SEARCH_CLAUSE: &str = r" AND (
68 - p.title % $1
93 + const PROJECT_SEARCH_CLAUSE: &str = concat!(
94 + r" AND (
95 + p.search_tsv @@ ",
96 + any_term_query!(),
97 + r"
98 + OR p.title % $1
69 99 OR p.title ILIKE '%' || replace(replace(replace($1, '\', '\\'), '%', '\%'), '_', '\_') || '%'
70 - )";
100 + )"
101 + );
71 102
72 103 // ILIKE-only variants for short queries (1-2 chars) where trigram similarity is unreliable.
73 104
@@ -80,6 +111,106 @@
80 111 p.title ILIKE '%' || replace(replace(replace($1, '\', '\\'), '%', '\%'), '_', '\_') || '%'
81 112 )";
82 113
114 + // Tiered relevance. Full rationale in migration 179; the short version:
115 + //
116 + // One cut, placed at the truth condition of the heading it drives.
117 + //
118 + // tier 2 -- holds at least one word the searcher typed. Renders in the main
119 + // block; a row with some but not all of them prints its own count.
120 + // tier 1 -- holds none of them, and is here on character overlap alone. Sits
121 + // under "Near matches. Matched on spelling, not on words", which is
122 + // true of every row in the tier by construction.
123 + //
124 + // The earlier version cut at *all* words present, so four of five ranked with
125 + // zero of five and the heading lied about everything between. Coverage is
126 + // continuous; the tier is the one place it is honest to make it binary, because
127 + // that is where the sentence flips from false to true.
128 + //
129 + // The two tiers rank by different measures and never compare across the
130 + // boundary. `ts_rank` above (frequency and length normalization over words that
131 + // are actually there), `mnw_word_match_score` below (per-word trigram, so a
132 + // misspelling costs one word instead of poisoning the comparison). They do not
133 + // share a scale, which is fine precisely because the tier is decided first, by
134 + // a count rather than by either score.
135 + //
136 + // See `templates/partials/discover_results.html` for the rendering.
137 +
138 + /// Holds at least one word the searcher typed. Ranked by `ts_rank`.
139 + pub const MATCH_TIER_LEXICAL: i16 = 2;
140 +
141 + /// Holds none of the typed words; character overlap only. Ranked by per-word
142 + /// trigram scoring (`mnw_word_match_score`, migration 179).
143 + ///
144 + /// Only the long-query path assigns tiers. A 1-2 character search is
145 + /// substring-only by construction, so every row would land here and the
146 + /// distinction would say nothing; those rows get `NULL` instead and the UI
147 + /// draws no boundary.
148 + pub const MATCH_TIER_FUZZY: i16 = 1;
149 +
150 + /// Tier 2 when the row holds any typed word, tier 1 when it holds none.
151 + /// `smallint` to match [`DbDiscoverItemRow::match_tier`]'s `Option<i16>`.
152 + ///
153 + /// The literals here and the constants above are checked against each other by
154 + /// `match_tier_literals_match_constants`.
155 + macro_rules! match_tier_expr {
156 + ($tsv:expr) => {
157 + concat!(
158 + "CASE WHEN mnw_exact_terms(",
159 + $tsv,
160 + ", $1) > 0 THEN 2 ELSE 1 END::smallint as match_tier"
161 + )
162 + };
163 + }
164 +
165 + /// How many of the typed words this row holds, and how many were typed.
166 + ///
167 + /// Projected so the template can print "Matched 3 of 5 words" on a row that has
168 + /// some but not all. The denominator is constant per query, but it is stemmed
169 + /// in Postgres, so it comes back per row rather than being recomputed in Rust
170 + /// off the raw string and disagreeing with the numerator.
171 + macro_rules! match_terms_expr {
172 + ($tsv:expr) => {
173 + concat!(
174 + "mnw_exact_terms(",
175 + $tsv,
176 + ", $1)::int as matched_terms,
177 + cardinality(mnw_query_terms($1))::int as query_terms"
178 + )
179 + };
180 + }
181 +
182 + /// Rank within a tier: `ts_rank` for lexical rows, per-word trigram scoring for
183 + /// fuzzy ones.
184 + ///
185 + /// Both arms read their weighting off the `setweight` A/B split in the
186 + /// generated `search_tsv` column (migration 176) rather than carrying a
187 + /// constant: `ts_rank` honours lexeme weights natively, and
188 + /// `mnw_word_match_score` (migration 179) applies the same halving to
189 + /// description lexemes that used to be written into this expression.
190 + ///
191 + /// The fuzzy arm scores each query word separately and averages, instead of
192 + /// comparing the whole query string against the whole title. Whole-phrase
193 + /// `similarity` is worst on precisely the queries that reach this tier: a long
194 + /// query against a short title scores low however well the words line up,
195 + /// because the query's surplus trigrams count against it. Per-word, a
196 + /// misspelling costs one word.
197 + macro_rules! match_score_expr {
198 + ($tsv:expr) => {
199 + concat!(
200 + "CASE WHEN mnw_exact_terms(",
201 + $tsv,
202 + ", $1) > 0 THEN ts_rank(",
203 + $tsv,
204 + ", ",
205 + any_term_query!(),
206 + ")
207 + ELSE mnw_word_match_score(",
208 + $tsv,
209 + ", $1) END::real as match_score"
210 + )
211 + };
212 + }
213 +
83 214 /// Maximum allowed search term length. Queries longer than this are truncated.
84 215 const MAX_SEARCH_LEN: usize = 200;
85 216
@@ -256,15 +387,17 @@
256 387
257 388 /// Search/browse public items with optional text search, item_type, tag, and price filters.
258 389 ///
259 - /// Uses PostgreSQL `pg_trgm` for fuzzy text matching. The search strategy:
260 - /// - `%` (trigram similarity operator) catches misspellings and near-matches
261 - /// - `ILIKE` fallback catches exact substrings that trigrams might miss for short queries
262 - /// - `similarity(description) * 0.5` weights description matches lower than title matches
263 - /// so that a title hit always ranks above a description-only hit
390 + /// Search is tiered (see the `match_tier` notes above the tier/score
391 + /// expressions):
392 + /// - `search_tsv @@ websearch_to_tsquery` is the lexical tier: stemmed,
393 + /// word-order independent, honours quoted phrases and `-exclusions`
394 + /// - `%` (trigram similarity) and `ILIKE` form the fuzzy tier: misspellings,
395 + /// partial words, and substrings a stemmer will not reach
264 396 ///
265 397 /// When a search term is present but no explicit sort is requested, results
266 - /// are ordered by relevance (`match_score DESC`). Otherwise, the caller can
267 - /// choose `most_sold`, `price_asc`, `price_desc`, or the default `newest`.
398 + /// are ordered by relevance (`match_tier DESC, match_score DESC`) and the UI
399 + /// draws the tier boundary. Otherwise, the caller can choose `most_sold`,
400 + /// `price_asc`, `price_desc`, or the default `newest`.
268 401 #[tracing::instrument(skip_all)]
269 402 pub async fn discover_items(
270 403 pool: &PgPool,
@@ -282,7 +415,7 @@
282 415 // Use LEFT JOIN with pre-aggregated transaction counts to avoid N+1 subquery per row
283 416 // LEFT JOIN item_tags/tags for primary tag display
284 417 let mut query = if has_search && !short_query {
285 - String::from(
418 + format!(
286 419 r"
287 420 SELECT
288 421 i.id,
@@ -298,10 +431,9 @@
298 431 i.pwyw_enabled,
299 432 i.pwyw_min_cents,
300 433 i.ai_tier,
301 - GREATEST(
302 - similarity(i.title, $1),
303 - similarity(COALESCE(i.description, ''), $1) * 0.5
304 - )::real as match_score
434 + {tier},
435 + {terms},
436 + {score}
305 437 FROM items i
306 438 JOIN projects p ON i.project_id = p.id
307 439 JOIN users u ON p.user_id = u.id
@@ -309,6 +441,9 @@
309 441 LEFT JOIN tags pt ON pt.id = pit.tag_id
310 442 WHERE i.is_public = true AND i.listed = true AND p.is_public = true AND i.scan_status = 'clean' AND u.is_sandbox = FALSE AND i.deleted_at IS NULL
311 443 ",
444 + tier = match_tier_expr!("i.search_tsv"),
445 + terms = match_terms_expr!("i.search_tsv"),
446 + score = match_score_expr!("i.search_tsv"),
312 447 )
313 448 } else if has_search {
314 449 // Short query: constant match_score, skip trigram similarity computation
@@ -328,6 +463,9 @@
328 463 i.pwyw_enabled,
329 464 i.pwyw_min_cents,
330 465 i.ai_tier,
466 + NULL::smallint as match_tier,
467 + NULL::int as matched_terms,
468 + NULL::int as query_terms,
331 469 1.0::real as match_score
332 470 FROM items i
333 471 JOIN projects p ON i.project_id = p.id
@@ -354,6 +492,9 @@
354 492 i.pwyw_enabled,
355 493 i.pwyw_min_cents,
356 494 i.ai_tier,
495 + NULL::smallint as match_tier,
496 + NULL::int as matched_terms,
497 + NULL::int as query_terms,
357 498 NULL::real as match_score
358 499 FROM items i
359 500 JOIN projects p ON i.project_id = p.id
@@ -371,7 +512,10 @@
371 512 let order = if has_search
372 513 && (filters.sort_by.is_none() || filters.sort_by == Some(DiscoverSort::Newest))
373 514 {
374 - "match_score DESC NULLS LAST, i.created_at DESC"
515 + // Tier first: a lexical match outranks every fuzzy one, whatever their
516 + // scores say. The two scores are not on one scale, so this is the only
517 + // ordering that means anything.
518 + "match_tier DESC NULLS LAST, match_score DESC NULLS LAST, i.created_at DESC"
375 519 } else {
376 520 match filters.sort_by {
377 521 Some(DiscoverSort::MostSold) => "sales_count DESC, i.created_at DESC",
@@ -443,7 +587,7 @@
443 587 let short_query = search_term.as_deref().is_some_and(is_short_query);
444 588
445 589 let mut query = if has_search && !short_query {
446 - String::from(
590 + format!(
447 591 r"
448 592 SELECT
449 593 p.slug,
@@ -453,10 +597,9 @@
453 597 p.created_at,
454 598 u.username,
455 599 p.item_count::bigint as item_count,
456 - GREATEST(
457 - similarity(p.title, $1),
458 - similarity(COALESCE(p.description, ''), $1) * 0.5
459 - )::real as match_score,
600 + {tier},
601 + {terms},
602 + {score},
460 603 pc.name as category_name,
461 604 pc.slug as category_slug
462 605 FROM projects p
@@ -464,6 +607,9 @@
464 607 LEFT JOIN project_categories pc ON pc.id = p.category_id
465 608 WHERE p.is_public = true AND u.is_sandbox = FALSE
466 609 ",
610 + tier = match_tier_expr!("p.search_tsv"),
611 + terms = match_terms_expr!("p.search_tsv"),
612 + score = match_score_expr!("p.search_tsv"),
467 613 )
468 614 } else if has_search {
469 615 // Short query: constant match_score, skip trigram similarity computation
@@ -477,6 +623,9 @@
477 623 p.created_at,
478 624 u.username,
479 625 p.item_count::bigint as item_count,
626 + NULL::smallint as match_tier,
627 + NULL::int as matched_terms,
628 + NULL::int as query_terms,
480 629 1.0::real as match_score,
481 630 pc.name as category_name,
482 631 pc.slug as category_slug
@@ -497,6 +646,9 @@
497 646 p.created_at,
498 647 u.username,
499 648 p.item_count::bigint as item_count,
649 + NULL::smallint as match_tier,
650 + NULL::int as matched_terms,
651 + NULL::int as query_terms,
500 652 NULL::real as match_score,
501 653 pc.name as category_name,
502 654 pc.slug as category_slug
@@ -528,7 +680,7 @@
528 680 // by a trigger), so the query has no aggregate to group (Run 11 Perf SER-3).
529 681
530 682 let order = if has_search && (sort_by.is_none() || sort_by == Some(DiscoverSort::Newest)) {
531 - "match_score DESC NULLS LAST, p.created_at DESC"
683 + "match_tier DESC NULLS LAST, match_score DESC NULLS LAST, p.created_at DESC"
532 684 } else {
533 685 match sort_by {
534 686 Some(DiscoverSort::MostSold) => "item_count DESC, p.created_at DESC",
@@ -996,6 +1148,74 @@
996 1148 let mut q = String::new();
997 1149 append_item_discover_filters(&mut q, &filters, true, false);
998 1150 assert!(q.contains("i.title % $1"));
1151 + // The word branch has to be OR'd in alongside the fuzzy one, or a
1152 + // stemmed-only match ("loops" for "looping") never reaches the result
1153 + // set and the tier it would have been sorted into stays empty.
1154 + assert!(q.contains("i.search_tsv @@ mnw_any_term_query($1)"));
1155 + // Membership must be ANY typed word, never all of them. websearch_to_tsquery
1156 + // ANDs its terms, which is the cliff this replaced: four of five words
1157 + // present ranked with none of them, under a heading that said so falsely.
1158 + assert!(!q.contains("websearch_to_tsquery"));
1159 + }
1160 +
1161 + /// The tier values live twice: as SQL literals inside `match_tier_expr!`,
1162 + /// and as the constants Rust compares against. Nothing links them, so a
1163 + /// change to one silently inverts the ordering rather than failing.
1164 + #[test]
1165 + fn match_tier_literals_match_constants() {
1166 + let expr = match_tier_expr!("i.search_tsv");
1167 + assert!(
1168 + expr.contains(&format!(
1169 + "THEN {MATCH_TIER_LEXICAL} ELSE {MATCH_TIER_FUZZY} END"
1170 + )),
1171 + "tier literals in the SQL drifted from the constants: {expr}"
1172 + );
1173 + // The cut is "holds any typed word", not "holds all of them". Anything
1174 + // stricter puts partial matches under a heading that denies they matched
1175 + // words at all.
1176 + assert!(
1177 + expr.contains("mnw_exact_terms(i.search_tsv, $1) > 0"),
1178 + "tier is no longer a coverage cut: {expr}"
1179 + );
1180 + assert!(
1181 + MATCH_TIER_LEXICAL > MATCH_TIER_FUZZY,
1182 + "ORDER BY match_tier DESC puts lexical first only if it is the larger value"
1183 + );
1184 + }
1185 +
1186 + /// The fuzzy arm calls a function defined in a migration, so a rename on
1187 + /// either side only fails at runtime, against the busiest public page.
1188 + /// Pin the name here and the pairing is at least stated in one place.
1189 + /// The label's denominator and the tier's numerator must come from one
1190 + /// place. Counting query words in Rust off the raw string would disagree
1191 + /// with Postgres the moment stemming or stopword removal applied, and the
1192 + /// row would read "Matched 3 of 7 words" against a 5-word tsquery.
1193 + #[test]
1194 + fn label_counts_come_from_the_same_stemmer_as_the_tier() {
1195 + let terms = match_terms_expr!("i.search_tsv");
1196 + assert!(terms.contains("mnw_exact_terms(i.search_tsv, $1)"));
1197 + assert!(terms.contains("cardinality(mnw_query_terms($1))"));
1198 + assert!(match_tier_expr!("i.search_tsv").contains("mnw_exact_terms"));
1199 + }
1200 +
1201 + #[test]
1202 + fn fuzzy_arm_calls_the_word_match_function() {
1203 + let expr = match_score_expr!("i.search_tsv");
1204 + assert!(
1205 + expr.contains("mnw_word_match_score(i.search_tsv, $1)"),
1206 + "fuzzy ranking no longer calls mnw_word_match_score (migrations/179): {expr}"
1207 + );
1208 + // Whole-phrase similarity is what 179 replaced. If it comes back here,
1209 + // the long-query case it fails on is back too.
1210 + assert!(!expr.contains("similarity("));
1211 + }
1212 +
1213 + /// A tier is only meaningful when both kinds of match can occur. The short
1214 + /// path is ILIKE-only, so it must not claim one.
1215 + #[test]
1216 + fn short_query_rows_are_untiered() {
1217 + assert!(!ITEM_SEARCH_CLAUSE_SHORT.contains("search_tsv"));
1218 + assert!(!PROJECT_SEARCH_CLAUSE_SHORT.contains("search_tsv"));
999 1219 }
1000 1220
1001 1221 #[test]
M server/src/db/follows.rs +3
@@ -166,6 +166,9 @@
166 166 pt.name as primary_tag_name,
167 167 i.pwyw_enabled,
168 168 i.pwyw_min_cents,
169 + NULL::smallint as match_tier,
170 + NULL::int as matched_terms,
171 + NULL::int as query_terms,
169 172 NULL::real as match_score,
170 173 i.ai_tier
171 174 FROM items i
M server/src/templates/partials.rs +10
M server/src/types/conversions.rs +57
M server/src/types/discover.rs +19
M server/src/types/mod.rs +3 -1
M server/templates/pages/discover.html +1 -1
M server/templates/partials/discover_results.html +71 -7
M server/tests/workflows/blog.rs +71
M server/tests/workflows/discover.rs +90 -1
M server/src/db/models/discover.rs +28 -2
M server/src/templates/public/mod.rs +10
M server/src/routes/pages/public/discover.rs +79 -3
A multithreaded/migrations/036_drop_posts_body_trgm_index.sql +36
A server/migrations/176_items_search_tsv.sql +83
A server/migrations/177_items_search_tsv_index.sql +19
A server/migrations/178_projects_search_tsv_index.sql +8
A server/migrations/179_word_match_score.sql +178