Skip to main content

max / makenotwork

20.5 KB · 567 lines History Blame Raw
1 //! The Postgres index over `refs/notes/*`.
2 //!
3 //! <!-- wiki: mnw-server-git-notes -->
4 //!
5 //! Every row here is a projection of a git object. The repository is truth and
6 //! this table is rebuildable from it, so nothing in this module may be the only
7 //! copy of anything: see migration 197 and the wiki note's load-bearing rule.
8 //! The consequence for this file is that every write is a whole-namespace
9 //! statement of fact rather than an edit, and every read is allowed to be wrong
10 //! in the direction of "ask the repository instead".
11 //!
12 //! Two rules the callers depend on:
13 //!
14 //! - **A namespace's rows and its index-state row move together, in one
15 //! transaction.** The state row says which notes-ref tip the rows were built
16 //! from, and the next reindex diffs against it. A state row ahead of the rows
17 //! it describes would make the next diff skip changes it never applied, and
18 //! nothing later would notice.
19 //! - **Visibility is not stored and is not checked here.** These functions take
20 //! a repo id and answer about that repository. Whether the person asking may
21 //! see it is the caller's question, decided the same way it is for the
22 //! repository's other pages. The personal-annotation functions are the same
23 //! rule in a sharper form: they scope by repository ownership, so the
24 //! `user_id` they are handed must be the signed-in account's, and passing any
25 //! other shows one person another person's private writing.
26
27 use std::collections::HashMap;
28
29 use chrono::{DateTime, Utc};
30 use sqlx::PgPool;
31
32 use super::{GitRepoId, UserId};
33 use crate::error::Result;
34
35 /// One note as the reindex hands it over.
36 ///
37 /// `target_summary` and `target_time` are the annotated commit's, resolved once
38 /// at index time so the feed does not read a commit header per row. They are
39 /// empty and `None` for a note on a blob or a tree.
40 #[derive(Debug, Clone)]
41 pub struct NoteUpsert {
42 pub target_oid: String,
43 pub blob_oid: String,
44 pub content: String,
45 pub target_is_commit: bool,
46 pub target_summary: String,
47 pub target_time: Option<DateTime<Utc>>,
48 }
49
50 /// One indexed note, as the read paths want it.
51 #[derive(Debug, Clone, sqlx::FromRow)]
52 pub struct IndexedNote {
53 pub namespace: String,
54 pub target_oid: String,
55 pub blob_oid: String,
56 pub content: String,
57 pub target_is_commit: bool,
58 pub target_summary: String,
59 pub target_time: Option<DateTime<Utc>>,
60 pub updated_at: DateTime<Utc>,
61 pub updated_by: String,
62 }
63
64 /// One personal annotation: a note in the viewer's own annotation repository
65 /// against an object that lives in somebody else's.
66 ///
67 /// `repo_id` and `repo_name` are the annotation repository's, not the annotated
68 /// commit's. Which repository serves the target is not knowable from a row here
69 /// and is not meant to be: a hash is global, and the question of who still
70 /// carries that object is asked of the repositories the viewer can see, at the
71 /// moment the page renders.
72 #[derive(Debug, Clone, sqlx::FromRow)]
73 pub struct PersonalAnnotation {
74 pub repo_id: GitRepoId,
75 pub repo_name: String,
76 pub namespace: String,
77 pub target_oid: String,
78 pub blob_oid: String,
79 pub content: String,
80 pub updated_at: DateTime<Utc>,
81 pub updated_by: String,
82 }
83
84 /// The tip a namespace was last indexed from, or `None` when it has never been
85 /// indexed.
86 ///
87 /// The distinction is load-bearing on both sides. The reindex uses it as the old
88 /// side of the diff, so an absent row means "walk the whole namespace". The read
89 /// paths use it to tell a cold index from a repository with no notes, which an
90 /// empty result cannot distinguish.
91 #[tracing::instrument(skip_all)]
92 pub async fn indexed_tip(
93 pool: &PgPool,
94 repo_id: GitRepoId,
95 namespace: &str,
96 ) -> Result<Option<String>> {
97 let tip = sqlx::query_scalar::<_, String>(
98 "SELECT indexed_tip FROM git_notes_index_state WHERE repo_id = $1 AND namespace = $2",
99 )
100 .bind(repo_id)
101 .bind(namespace)
102 .fetch_optional(pool)
103 .await?;
104 Ok(tip)
105 }
106
107 /// Every namespace this repository has an index for, whether or not it still has
108 /// notes in it.
109 #[tracing::instrument(skip_all)]
110 pub async fn indexed_namespaces(pool: &PgPool, repo_id: GitRepoId) -> Result<Vec<String>> {
111 let names = sqlx::query_scalar::<_, String>(
112 "SELECT namespace FROM git_notes_index_state WHERE repo_id = $1 ORDER BY namespace",
113 )
114 .bind(repo_id)
115 .fetch_all(pool)
116 .await?;
117 Ok(names)
118 }
119
120 /// Whether anything in this repository has been indexed.
121 ///
122 /// The read paths ask this before trusting an empty result: a repository with no
123 /// state rows has a cold index, not an unannotated history, and the answer to a
124 /// cold index is to walk the repository rather than to report nothing.
125 #[tracing::instrument(skip_all)]
126 pub async fn is_indexed(pool: &PgPool, repo_id: GitRepoId) -> Result<bool> {
127 let found = sqlx::query_scalar::<_, bool>(
128 "SELECT EXISTS (SELECT 1 FROM git_notes_index_state WHERE repo_id = $1)",
129 )
130 .bind(repo_id)
131 .fetch_one(pool)
132 .await?;
133 Ok(found)
134 }
135
136 /// One namespace's diff, and the tip it was computed against.
137 ///
138 /// A struct rather than eight arguments because every field has to describe the
139 /// same reindex: a `tip` that did not produce these `upserts` is the one way to
140 /// corrupt the index silently, and keeping them in one value makes that hard to
141 /// do by accident.
142 pub struct NamespaceUpdate<'a> {
143 pub namespace: &'a str,
144 /// Notes added or changed since the previously indexed tip.
145 pub upserts: &'a [NoteUpsert],
146 /// Target ids whose note is gone, hex.
147 pub removals: &'a [String],
148 /// The notes-ref tip this diff brings the index up to, or `None` for a
149 /// batch that is part of a larger diff. A batch that claimed the tip before
150 /// the rest of the diff was written would make the next reindex skip the
151 /// remainder; leaving the old tip in place makes it recompute and redo work
152 /// that is idempotent anyway.
153 pub tip: Option<&'a str>,
154 /// When the annotations were written, and by whom: the notes commit's own
155 /// committer, not the person who triggered the reindex.
156 pub updated_at: DateTime<Utc>,
157 pub updated_by: &'a str,
158 }
159
160 /// Apply one namespace's diff and record the tip it was computed against.
161 ///
162 /// The upserts and removals are what changed between the previously indexed tip
163 /// and `update.tip`; everything else in the namespace is left alone, which is
164 /// what makes a push cost the size of its diff. All of it commits together with
165 /// the state row, so a failure halfway leaves the namespace exactly as it was
166 /// and the next reindex recomputes the same diff.
167 #[tracing::instrument(skip_all)]
168 pub async fn apply_changes(
169 pool: &PgPool,
170 repo_id: GitRepoId,
171 update: &NamespaceUpdate<'_>,
172 ) -> Result<()> {
173 let NamespaceUpdate {
174 namespace,
175 upserts,
176 removals,
177 tip,
178 updated_at,
179 updated_by,
180 } = *update;
181
182 let mut tx = pool.begin().await?;
183
184 if !removals.is_empty() {
185 sqlx::query(
186 "DELETE FROM git_notes
187 WHERE repo_id = $1 AND namespace = $2 AND target_oid = ANY($3)",
188 )
189 .bind(repo_id)
190 .bind(namespace)
191 .bind(removals)
192 .execute(&mut *tx)
193 .await?;
194 }
195
196 if !upserts.is_empty() {
197 // One statement for the batch. A push that annotates a thousand commits
198 // is one round trip rather than a thousand, and the arrays keep the
199 // parameter count fixed no matter how large the batch is.
200 let targets: Vec<&str> = upserts.iter().map(|n| n.target_oid.as_str()).collect();
201 let blobs: Vec<&str> = upserts.iter().map(|n| n.blob_oid.as_str()).collect();
202 let contents: Vec<&str> = upserts.iter().map(|n| n.content.as_str()).collect();
203 let is_commit: Vec<bool> = upserts.iter().map(|n| n.target_is_commit).collect();
204 let summaries: Vec<&str> = upserts.iter().map(|n| n.target_summary.as_str()).collect();
205 let times: Vec<Option<DateTime<Utc>>> = upserts.iter().map(|n| n.target_time).collect();
206
207 sqlx::query(
208 "INSERT INTO git_notes (
209 repo_id, namespace, target_oid, blob_oid, content,
210 target_is_commit, target_summary, target_time, updated_at, updated_by
211 )
212 SELECT $1, $2, t.target, t.blob, t.content, t.is_commit, t.summary, t.time, $9, $10
213 FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bool[], $7::text[], $8::timestamptz[])
214 AS t(target, blob, content, is_commit, summary, time)
215 ON CONFLICT (repo_id, namespace, target_oid) DO UPDATE SET
216 blob_oid = EXCLUDED.blob_oid,
217 content = EXCLUDED.content,
218 target_is_commit = EXCLUDED.target_is_commit,
219 target_summary = EXCLUDED.target_summary,
220 target_time = EXCLUDED.target_time,
221 updated_at = EXCLUDED.updated_at,
222 updated_by = EXCLUDED.updated_by",
223 )
224 .bind(repo_id)
225 .bind(namespace)
226 .bind(&targets)
227 .bind(&blobs)
228 .bind(&contents)
229 .bind(&is_commit)
230 .bind(&summaries)
231 .bind(&times)
232 .bind(updated_at)
233 .bind(updated_by)
234 .execute(&mut *tx)
235 .await?;
236 }
237
238 if let Some(tip) = tip {
239 sqlx::query(
240 "INSERT INTO git_notes_index_state (repo_id, namespace, indexed_tip, indexed_at)
241 VALUES ($1, $2, $3, NOW())
242 ON CONFLICT (repo_id, namespace) DO UPDATE SET
243 indexed_tip = EXCLUDED.indexed_tip,
244 indexed_at = EXCLUDED.indexed_at",
245 )
246 .bind(repo_id)
247 .bind(namespace)
248 .bind(tip)
249 .execute(&mut *tx)
250 .await?;
251 }
252
253 tx.commit().await?;
254 Ok(())
255 }
256
257 /// Forget a namespace: its notes and the fact that it was ever indexed.
258 ///
259 /// What a deleted `refs/notes/<ns>` produces. The state row goes too, so the
260 /// namespace reads as cold rather than as empty, and a ref that comes back is
261 /// walked in full.
262 #[tracing::instrument(skip_all)]
263 pub async fn forget_namespace(pool: &PgPool, repo_id: GitRepoId, namespace: &str) -> Result<()> {
264 let mut tx = pool.begin().await?;
265 sqlx::query("DELETE FROM git_notes WHERE repo_id = $1 AND namespace = $2")
266 .bind(repo_id)
267 .bind(namespace)
268 .execute(&mut *tx)
269 .await?;
270 sqlx::query("DELETE FROM git_notes_index_state WHERE repo_id = $1 AND namespace = $2")
271 .bind(repo_id)
272 .bind(namespace)
273 .execute(&mut *tx)
274 .await?;
275 tx.commit().await?;
276 Ok(())
277 }
278
279 /// How many notes are indexed per namespace, for the Notes tab's counts.
280 #[tracing::instrument(skip_all)]
281 pub async fn counts_by_namespace(pool: &PgPool, repo_id: GitRepoId) -> Result<Vec<(String, i64)>> {
282 let rows = sqlx::query_as::<_, (String, i64)>(
283 "SELECT namespace, COUNT(*) FROM git_notes WHERE repo_id = $1
284 GROUP BY namespace ORDER BY namespace",
285 )
286 .bind(repo_id)
287 .fetch_all(pool)
288 .await?;
289 Ok(rows)
290 }
291
292 /// How many namespaces annotate each of `targets`.
293 ///
294 /// The log-page path: one query for a page of commits, keyed by the hex the
295 /// caller passed in. Targets with no note are absent from the map rather than
296 /// present with a zero.
297 #[tracing::instrument(skip_all)]
298 pub async fn annotation_counts(
299 pool: &PgPool,
300 repo_id: GitRepoId,
301 targets: &[String],
302 ) -> Result<HashMap<String, i64>> {
303 if targets.is_empty() {
304 return Ok(HashMap::new());
305 }
306 let rows = sqlx::query_as::<_, (String, i64)>(
307 "SELECT target_oid, COUNT(*) FROM git_notes
308 WHERE repo_id = $1 AND target_oid = ANY($2)
309 GROUP BY target_oid",
310 )
311 .bind(repo_id)
312 .bind(targets)
313 .fetch_all(pool)
314 .await?;
315 Ok(rows.into_iter().collect())
316 }
317
318 /// The annotation feed: newest annotation first.
319 ///
320 /// Ordered by when the note was written rather than by the annotated commit's
321 /// own date, because a feed answers "what has been said lately" and annotating a
322 /// five-year-old commit is news. `commits_only` is the "annotated commits"
323 /// filter; `namespace` of `None` reads across all of them.
324 #[tracing::instrument(skip_all)]
325 pub async fn feed(
326 pool: &PgPool,
327 repo_id: GitRepoId,
328 namespace: Option<&str>,
329 commits_only: bool,
330 limit: i64,
331 offset: i64,
332 ) -> Result<Vec<IndexedNote>> {
333 let rows = sqlx::query_as::<_, IndexedNote>(
334 "SELECT namespace, target_oid, blob_oid, content, target_is_commit,
335 target_summary, target_time, updated_at, updated_by
336 FROM git_notes
337 WHERE repo_id = $1
338 AND ($2::text IS NULL OR namespace = $2)
339 AND (NOT $3::bool OR target_is_commit)
340 ORDER BY updated_at DESC, target_oid
341 LIMIT $4 OFFSET $5",
342 )
343 .bind(repo_id)
344 .bind(namespace)
345 .bind(commits_only)
346 .bind(limit)
347 .bind(offset)
348 .fetch_all(pool)
349 .await?;
350 Ok(rows)
351 }
352
353 /// Notes matching the feed's filters, for the page count beside the rows.
354 #[tracing::instrument(skip_all)]
355 pub async fn count_notes(
356 pool: &PgPool,
357 repo_id: GitRepoId,
358 namespace: Option<&str>,
359 commits_only: bool,
360 ) -> Result<i64> {
361 let count = sqlx::query_scalar::<_, i64>(
362 "SELECT COUNT(*) FROM git_notes
363 WHERE repo_id = $1
364 AND ($2::text IS NULL OR namespace = $2)
365 AND (NOT $3::bool OR target_is_commit)",
366 )
367 .bind(repo_id)
368 .bind(namespace)
369 .bind(commits_only)
370 .fetch_one(pool)
371 .await?;
372 Ok(count)
373 }
374
375 /// Full-text search within one repository's notes.
376 ///
377 /// `websearch_to_tsquery` rather than `plainto_tsquery`: it accepts quoted
378 /// phrases and `or`/`-` the way a person types them into a search box, and it
379 /// never raises on malformed input, so a stray quote is a query that finds
380 /// little rather than a 500.
381 #[tracing::instrument(skip_all)]
382 pub async fn search(
383 pool: &PgPool,
384 repo_id: GitRepoId,
385 query: &str,
386 namespace: Option<&str>,
387 commits_only: bool,
388 limit: i64,
389 ) -> Result<Vec<IndexedNote>> {
390 let rows = sqlx::query_as::<_, IndexedNote>(
391 "SELECT namespace, target_oid, blob_oid, content, target_is_commit,
392 target_summary, target_time, updated_at, updated_by
393 FROM git_notes
394 WHERE repo_id = $1
395 AND ($3::text IS NULL OR namespace = $3)
396 AND (NOT $4::bool OR target_is_commit)
397 AND search_tsv @@ websearch_to_tsquery('english', $2)
398 ORDER BY ts_rank(search_tsv, websearch_to_tsquery('english', $2)) DESC,
399 updated_at DESC
400 LIMIT $5",
401 )
402 .bind(repo_id)
403 .bind(query)
404 .bind(namespace)
405 .bind(commits_only)
406 .bind(limit)
407 .fetch_all(pool)
408 .await?;
409 Ok(rows)
410 }
411
412 // --- Personal annotations ---
413 //
414 // The other direction from the rest of this file. Everything above answers
415 // "what does this repository carry"; a commit page has to ask "does the person
416 // reading have anything against this hash", which is a question about an
417 // account rather than about a repository.
418 //
419 // It is a two-table question and stays one. No user id is denormalized onto
420 // `git_notes`: `git_repos.user_id` already records who owns the repository a
421 // note lives in, a copy would need a backfill the day a repository changes
422 // hands, and an account owns one annotation repository, so the `git_repos` half
423 // of the join is a single row reached through `idx_git_repos_one_annotation_repo`.
424 // The `git_notes` half is `idx_git_notes_target (repo_id, target_oid)`, which
425 // migration 197 already ships.
426
427 /// Every annotation `user_id` has written against `target_oid`, across their
428 /// annotation repositories.
429 ///
430 /// The commit page's query. The rows are a projection: the annotation repository
431 /// holds the note and this table can be dropped and rebuilt from it, so a miss
432 /// here means "the index has not caught up", never "there is no such note".
433 ///
434 /// `user_id` is the session's. This module checks nothing about who is asking.
435 ///
436 /// An annotation whose target no repository the viewer can see still serves is
437 /// returned like any other. Nothing collects those: a commit can go away with
438 /// the repository that carried it and the sentence somebody wrote about it is
439 /// still theirs.
440 #[tracing::instrument(skip_all)]
441 pub async fn annotations_by_user_for_target(
442 pool: &PgPool,
443 user_id: UserId,
444 target_oid: &str,
445 ) -> Result<Vec<PersonalAnnotation>> {
446 let rows = sqlx::query_as::<_, PersonalAnnotation>(
447 "SELECT r.id AS repo_id, r.name AS repo_name, n.namespace, n.target_oid,
448 n.blob_oid, n.content, n.updated_at, n.updated_by
449 FROM git_notes n
450 JOIN git_repos r ON r.id = n.repo_id
451 WHERE r.user_id = $1 AND r.kind = 'annotations' AND n.target_oid = $2
452 ORDER BY r.name, n.namespace",
453 )
454 .bind(user_id)
455 .bind(target_oid)
456 .fetch_all(pool)
457 .await?;
458 Ok(rows)
459 }
460
461 /// How many annotations `user_id` has against each of `targets`.
462 ///
463 /// The log page's badge: one query for a page of commits, keyed by the hex the
464 /// caller passed in. Targets with no annotation are absent from the map rather
465 /// than present with a zero.
466 ///
467 /// The same projection and the same trust: the counts are the index's, and
468 /// `user_id` has to be the signed-in account's because nothing here checks it.
469 #[tracing::instrument(skip_all)]
470 pub async fn annotation_counts_for_user(
471 pool: &PgPool,
472 user_id: UserId,
473 targets: &[String],
474 ) -> Result<HashMap<String, i64>> {
475 if targets.is_empty() {
476 return Ok(HashMap::new());
477 }
478 let rows = sqlx::query_as::<_, (String, i64)>(
479 "SELECT n.target_oid, COUNT(*)
480 FROM git_notes n
481 JOIN git_repos r ON r.id = n.repo_id
482 WHERE r.user_id = $1 AND r.kind = 'annotations' AND n.target_oid = ANY($2)
483 GROUP BY n.target_oid",
484 )
485 .bind(user_id)
486 .bind(targets)
487 .fetch_all(pool)
488 .await?;
489 Ok(rows.into_iter().collect())
490 }
491
492 /// Everything `user_id` has ever annotated, newest first.
493 ///
494 /// Ordered by when the annotation was written rather than by the annotated
495 /// commit's own date, for the reason the repository feed is: annotating a
496 /// five-year-old commit is news. A full rebuild restamps `updated_at` from the
497 /// notes ref's committer time, so a rebuilt timeline is the repository's, not
498 /// the original keystrokes'.
499 ///
500 /// A projection, `user_id` is the session's, and orphans are included: an
501 /// annotation whose target nothing serves any more is a row like any other.
502 #[tracing::instrument(skip_all)]
503 pub async fn user_annotations(
504 pool: &PgPool,
505 user_id: UserId,
506 limit: i64,
507 offset: i64,
508 ) -> Result<Vec<PersonalAnnotation>> {
509 let rows = sqlx::query_as::<_, PersonalAnnotation>(
510 "SELECT r.id AS repo_id, r.name AS repo_name, n.namespace, n.target_oid,
511 n.blob_oid, n.content, n.updated_at, n.updated_by
512 FROM git_notes n
513 JOIN git_repos r ON r.id = n.repo_id
514 WHERE r.user_id = $1 AND r.kind = 'annotations'
515 ORDER BY n.updated_at DESC, n.target_oid
516 LIMIT $2 OFFSET $3",
517 )
518 .bind(user_id)
519 .bind(limit)
520 .bind(offset)
521 .fetch_all(pool)
522 .await?;
523 Ok(rows)
524 }
525
526 /// How many annotations `user_id` has, for the page count beside the rows.
527 ///
528 /// The index's count, which is the same projection the list is. `user_id` is
529 /// the session's, and orphans are counted because they are shown.
530 #[tracing::instrument(skip_all)]
531 pub async fn count_user_annotations(pool: &PgPool, user_id: UserId) -> Result<i64> {
532 let count = sqlx::query_scalar::<_, i64>(
533 "SELECT COUNT(*)
534 FROM git_notes n
535 JOIN git_repos r ON r.id = n.repo_id
536 WHERE r.user_id = $1 AND r.kind = 'annotations'",
537 )
538 .bind(user_id)
539 .fetch_one(pool)
540 .await?;
541 Ok(count)
542 }
543
544 /// Whether this account's annotations have ever been indexed.
545 ///
546 /// `is_indexed`'s twin, and needed for the same reason: an empty result cannot
547 /// tell a cold index from an account that has annotated nothing. A caller that
548 /// skips this reports "no annotations" to somebody whose annotations predate
549 /// the index, and the answer to a cold index is to walk the annotation
550 /// repository instead.
551 ///
552 /// `user_id` is the session's; this module checks nothing about who is asking.
553 #[tracing::instrument(skip_all)]
554 pub async fn user_annotation_index_is_warm(pool: &PgPool, user_id: UserId) -> Result<bool> {
555 let found = sqlx::query_scalar::<_, bool>(
556 "SELECT EXISTS (
557 SELECT 1 FROM git_notes_index_state s
558 JOIN git_repos r ON r.id = s.repo_id
559 WHERE r.user_id = $1 AND r.kind = 'annotations'
560 )",
561 )
562 .bind(user_id)
563 .fetch_one(pool)
564 .await?;
565 Ok(found)
566 }
567