Skip to main content

max / makenotwork

13.5 KB · 388 lines History Blame Raw
1 //! The Postgres index over `refs/notes/*`.
2 //!
3 //! <!-- wiki: mnw-server-git-notes -->
4 //!
5 //! Every row here is a projection of a git object. The repository is truth and
6 //! this table is rebuildable from it, so nothing in this module may be the only
7 //! copy of anything: see migration 197 and the wiki note's load-bearing rule.
8 //! The consequence for this file is that every write is a whole-namespace
9 //! statement of fact rather than an edit, and every read is allowed to be wrong
10 //! in the direction of "ask the repository instead".
11 //!
12 //! Two rules the callers depend on:
13 //!
14 //! - **A namespace's rows and its index-state row move together, in one
15 //! transaction.** The state row says which notes-ref tip the rows were built
16 //! from, and the next reindex diffs against it. A state row ahead of the rows
17 //! it describes would make the next diff skip changes it never applied, and
18 //! nothing later would notice.
19 //! - **Visibility is not stored and is not checked here.** These functions take
20 //! a repo id and answer about that repository. Whether the person asking may
21 //! see it is the caller's question, decided the same way it is for the
22 //! repository's other pages.
23
24 use std::collections::HashMap;
25
26 use chrono::{DateTime, Utc};
27 use sqlx::PgPool;
28
29 use super::GitRepoId;
30 use crate::error::Result;
31
32 /// One note as the reindex hands it over.
33 ///
34 /// `target_summary` and `target_time` are the annotated commit's, resolved once
35 /// at index time so the feed does not read a commit header per row. They are
36 /// empty and `None` for a note on a blob or a tree.
37 #[derive(Debug, Clone)]
38 pub struct NoteUpsert {
39 pub target_oid: String,
40 pub blob_oid: String,
41 pub content: String,
42 pub target_is_commit: bool,
43 pub target_summary: String,
44 pub target_time: Option<DateTime<Utc>>,
45 }
46
47 /// One indexed note, as the read paths want it.
48 #[derive(Debug, Clone, sqlx::FromRow)]
49 pub struct IndexedNote {
50 pub namespace: String,
51 pub target_oid: String,
52 pub blob_oid: String,
53 pub content: String,
54 pub target_is_commit: bool,
55 pub target_summary: String,
56 pub target_time: Option<DateTime<Utc>>,
57 pub updated_at: DateTime<Utc>,
58 pub updated_by: String,
59 }
60
61 /// The tip a namespace was last indexed from, or `None` when it has never been
62 /// indexed.
63 ///
64 /// The distinction is load-bearing on both sides. The reindex uses it as the old
65 /// side of the diff, so an absent row means "walk the whole namespace". The read
66 /// paths use it to tell a cold index from a repository with no notes, which an
67 /// empty result cannot distinguish.
68 #[tracing::instrument(skip_all)]
69 pub async fn indexed_tip(
70 pool: &PgPool,
71 repo_id: GitRepoId,
72 namespace: &str,
73 ) -> Result<Option<String>> {
74 let tip = sqlx::query_scalar::<_, String>(
75 "SELECT indexed_tip FROM git_notes_index_state WHERE repo_id = $1 AND namespace = $2",
76 )
77 .bind(repo_id)
78 .bind(namespace)
79 .fetch_optional(pool)
80 .await?;
81 Ok(tip)
82 }
83
84 /// Every namespace this repository has an index for, whether or not it still has
85 /// notes in it.
86 #[tracing::instrument(skip_all)]
87 pub async fn indexed_namespaces(pool: &PgPool, repo_id: GitRepoId) -> Result<Vec<String>> {
88 let names = sqlx::query_scalar::<_, String>(
89 "SELECT namespace FROM git_notes_index_state WHERE repo_id = $1 ORDER BY namespace",
90 )
91 .bind(repo_id)
92 .fetch_all(pool)
93 .await?;
94 Ok(names)
95 }
96
97 /// Whether anything in this repository has been indexed.
98 ///
99 /// The read paths ask this before trusting an empty result: a repository with no
100 /// state rows has a cold index, not an unannotated history, and the answer to a
101 /// cold index is to walk the repository rather than to report nothing.
102 #[tracing::instrument(skip_all)]
103 pub async fn is_indexed(pool: &PgPool, repo_id: GitRepoId) -> Result<bool> {
104 let found = sqlx::query_scalar::<_, bool>(
105 "SELECT EXISTS (SELECT 1 FROM git_notes_index_state WHERE repo_id = $1)",
106 )
107 .bind(repo_id)
108 .fetch_one(pool)
109 .await?;
110 Ok(found)
111 }
112
113 /// One namespace's diff, and the tip it was computed against.
114 ///
115 /// A struct rather than eight arguments because every field has to describe the
116 /// same reindex: a `tip` that did not produce these `upserts` is the one way to
117 /// corrupt the index silently, and keeping them in one value makes that hard to
118 /// do by accident.
119 pub struct NamespaceUpdate<'a> {
120 pub namespace: &'a str,
121 /// Notes added or changed since the previously indexed tip.
122 pub upserts: &'a [NoteUpsert],
123 /// Target ids whose note is gone, hex.
124 pub removals: &'a [String],
125 /// The notes-ref tip this diff brings the index up to, or `None` for a
126 /// batch that is part of a larger diff. A batch that claimed the tip before
127 /// the rest of the diff was written would make the next reindex skip the
128 /// remainder; leaving the old tip in place makes it recompute and redo work
129 /// that is idempotent anyway.
130 pub tip: Option<&'a str>,
131 /// When the annotations were written, and by whom: the notes commit's own
132 /// committer, not the person who triggered the reindex.
133 pub updated_at: DateTime<Utc>,
134 pub updated_by: &'a str,
135 }
136
137 /// Apply one namespace's diff and record the tip it was computed against.
138 ///
139 /// The upserts and removals are what changed between the previously indexed tip
140 /// and `update.tip`; everything else in the namespace is left alone, which is
141 /// what makes a push cost the size of its diff. All of it commits together with
142 /// the state row, so a failure halfway leaves the namespace exactly as it was
143 /// and the next reindex recomputes the same diff.
144 #[tracing::instrument(skip_all)]
145 pub async fn apply_changes(
146 pool: &PgPool,
147 repo_id: GitRepoId,
148 update: &NamespaceUpdate<'_>,
149 ) -> Result<()> {
150 let NamespaceUpdate {
151 namespace,
152 upserts,
153 removals,
154 tip,
155 updated_at,
156 updated_by,
157 } = *update;
158
159 let mut tx = pool.begin().await?;
160
161 if !removals.is_empty() {
162 sqlx::query(
163 "DELETE FROM git_notes
164 WHERE repo_id = $1 AND namespace = $2 AND target_oid = ANY($3)",
165 )
166 .bind(repo_id)
167 .bind(namespace)
168 .bind(removals)
169 .execute(&mut *tx)
170 .await?;
171 }
172
173 if !upserts.is_empty() {
174 // One statement for the batch. A push that annotates a thousand commits
175 // is one round trip rather than a thousand, and the arrays keep the
176 // parameter count fixed no matter how large the batch is.
177 let targets: Vec<&str> = upserts.iter().map(|n| n.target_oid.as_str()).collect();
178 let blobs: Vec<&str> = upserts.iter().map(|n| n.blob_oid.as_str()).collect();
179 let contents: Vec<&str> = upserts.iter().map(|n| n.content.as_str()).collect();
180 let is_commit: Vec<bool> = upserts.iter().map(|n| n.target_is_commit).collect();
181 let summaries: Vec<&str> = upserts.iter().map(|n| n.target_summary.as_str()).collect();
182 let times: Vec<Option<DateTime<Utc>>> = upserts.iter().map(|n| n.target_time).collect();
183
184 sqlx::query(
185 "INSERT INTO git_notes (
186 repo_id, namespace, target_oid, blob_oid, content,
187 target_is_commit, target_summary, target_time, updated_at, updated_by
188 )
189 SELECT $1, $2, t.target, t.blob, t.content, t.is_commit, t.summary, t.time, $9, $10
190 FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bool[], $7::text[], $8::timestamptz[])
191 AS t(target, blob, content, is_commit, summary, time)
192 ON CONFLICT (repo_id, namespace, target_oid) DO UPDATE SET
193 blob_oid = EXCLUDED.blob_oid,
194 content = EXCLUDED.content,
195 target_is_commit = EXCLUDED.target_is_commit,
196 target_summary = EXCLUDED.target_summary,
197 target_time = EXCLUDED.target_time,
198 updated_at = EXCLUDED.updated_at,
199 updated_by = EXCLUDED.updated_by",
200 )
201 .bind(repo_id)
202 .bind(namespace)
203 .bind(&targets)
204 .bind(&blobs)
205 .bind(&contents)
206 .bind(&is_commit)
207 .bind(&summaries)
208 .bind(&times)
209 .bind(updated_at)
210 .bind(updated_by)
211 .execute(&mut *tx)
212 .await?;
213 }
214
215 if let Some(tip) = tip {
216 sqlx::query(
217 "INSERT INTO git_notes_index_state (repo_id, namespace, indexed_tip, indexed_at)
218 VALUES ($1, $2, $3, NOW())
219 ON CONFLICT (repo_id, namespace) DO UPDATE SET
220 indexed_tip = EXCLUDED.indexed_tip,
221 indexed_at = EXCLUDED.indexed_at",
222 )
223 .bind(repo_id)
224 .bind(namespace)
225 .bind(tip)
226 .execute(&mut *tx)
227 .await?;
228 }
229
230 tx.commit().await?;
231 Ok(())
232 }
233
234 /// Forget a namespace: its notes and the fact that it was ever indexed.
235 ///
236 /// What a deleted `refs/notes/<ns>` produces. The state row goes too, so the
237 /// namespace reads as cold rather than as empty, and a ref that comes back is
238 /// walked in full.
239 #[tracing::instrument(skip_all)]
240 pub async fn forget_namespace(pool: &PgPool, repo_id: GitRepoId, namespace: &str) -> Result<()> {
241 let mut tx = pool.begin().await?;
242 sqlx::query("DELETE FROM git_notes WHERE repo_id = $1 AND namespace = $2")
243 .bind(repo_id)
244 .bind(namespace)
245 .execute(&mut *tx)
246 .await?;
247 sqlx::query("DELETE FROM git_notes_index_state WHERE repo_id = $1 AND namespace = $2")
248 .bind(repo_id)
249 .bind(namespace)
250 .execute(&mut *tx)
251 .await?;
252 tx.commit().await?;
253 Ok(())
254 }
255
256 /// How many notes are indexed per namespace, for the Notes tab's counts.
257 #[tracing::instrument(skip_all)]
258 pub async fn counts_by_namespace(pool: &PgPool, repo_id: GitRepoId) -> Result<Vec<(String, i64)>> {
259 let rows = sqlx::query_as::<_, (String, i64)>(
260 "SELECT namespace, COUNT(*) FROM git_notes WHERE repo_id = $1
261 GROUP BY namespace ORDER BY namespace",
262 )
263 .bind(repo_id)
264 .fetch_all(pool)
265 .await?;
266 Ok(rows)
267 }
268
269 /// How many namespaces annotate each of `targets`.
270 ///
271 /// The log-page path: one query for a page of commits, keyed by the hex the
272 /// caller passed in. Targets with no note are absent from the map rather than
273 /// present with a zero.
274 #[tracing::instrument(skip_all)]
275 pub async fn annotation_counts(
276 pool: &PgPool,
277 repo_id: GitRepoId,
278 targets: &[String],
279 ) -> Result<HashMap<String, i64>> {
280 if targets.is_empty() {
281 return Ok(HashMap::new());
282 }
283 let rows = sqlx::query_as::<_, (String, i64)>(
284 "SELECT target_oid, COUNT(*) FROM git_notes
285 WHERE repo_id = $1 AND target_oid = ANY($2)
286 GROUP BY target_oid",
287 )
288 .bind(repo_id)
289 .bind(targets)
290 .fetch_all(pool)
291 .await?;
292 Ok(rows.into_iter().collect())
293 }
294
295 /// The annotation feed: newest annotation first.
296 ///
297 /// Ordered by when the note was written rather than by the annotated commit's
298 /// own date, because a feed answers "what has been said lately" and annotating a
299 /// five-year-old commit is news. `commits_only` is the "annotated commits"
300 /// filter; `namespace` of `None` reads across all of them.
301 #[tracing::instrument(skip_all)]
302 pub async fn feed(
303 pool: &PgPool,
304 repo_id: GitRepoId,
305 namespace: Option<&str>,
306 commits_only: bool,
307 limit: i64,
308 offset: i64,
309 ) -> Result<Vec<IndexedNote>> {
310 let rows = sqlx::query_as::<_, IndexedNote>(
311 "SELECT namespace, target_oid, blob_oid, content, target_is_commit,
312 target_summary, target_time, updated_at, updated_by
313 FROM git_notes
314 WHERE repo_id = $1
315 AND ($2::text IS NULL OR namespace = $2)
316 AND (NOT $3::bool OR target_is_commit)
317 ORDER BY updated_at DESC, target_oid
318 LIMIT $4 OFFSET $5",
319 )
320 .bind(repo_id)
321 .bind(namespace)
322 .bind(commits_only)
323 .bind(limit)
324 .bind(offset)
325 .fetch_all(pool)
326 .await?;
327 Ok(rows)
328 }
329
330 /// Notes matching the feed's filters, for the page count beside the rows.
331 #[tracing::instrument(skip_all)]
332 pub async fn count_notes(
333 pool: &PgPool,
334 repo_id: GitRepoId,
335 namespace: Option<&str>,
336 commits_only: bool,
337 ) -> Result<i64> {
338 let count = sqlx::query_scalar::<_, i64>(
339 "SELECT COUNT(*) FROM git_notes
340 WHERE repo_id = $1
341 AND ($2::text IS NULL OR namespace = $2)
342 AND (NOT $3::bool OR target_is_commit)",
343 )
344 .bind(repo_id)
345 .bind(namespace)
346 .bind(commits_only)
347 .fetch_one(pool)
348 .await?;
349 Ok(count)
350 }
351
352 /// Full-text search within one repository's notes.
353 ///
354 /// `websearch_to_tsquery` rather than `plainto_tsquery`: it accepts quoted
355 /// phrases and `or`/`-` the way a person types them into a search box, and it
356 /// never raises on malformed input, so a stray quote is a query that finds
357 /// little rather than a 500.
358 #[tracing::instrument(skip_all)]
359 pub async fn search(
360 pool: &PgPool,
361 repo_id: GitRepoId,
362 query: &str,
363 namespace: Option<&str>,
364 commits_only: bool,
365 limit: i64,
366 ) -> Result<Vec<IndexedNote>> {
367 let rows = sqlx::query_as::<_, IndexedNote>(
368 "SELECT namespace, target_oid, blob_oid, content, target_is_commit,
369 target_summary, target_time, updated_at, updated_by
370 FROM git_notes
371 WHERE repo_id = $1
372 AND ($3::text IS NULL OR namespace = $3)
373 AND (NOT $4::bool OR target_is_commit)
374 AND search_tsv @@ websearch_to_tsquery('english', $2)
375 ORDER BY ts_rank(search_tsv, websearch_to_tsquery('english', $2)) DESC,
376 updated_at DESC
377 LIMIT $5",
378 )
379 .bind(repo_id)
380 .bind(query)
381 .bind(namespace)
382 .bind(commits_only)
383 .bind(limit)
384 .fetch_all(pool)
385 .await?;
386 Ok(rows)
387 }
388