Skip to main content

max / makenotwork

server: the git_notes index table and the diff that feeds it P4 of the notes program, first half: somewhere to put an index of refs/notes/*, and the walk that works out what changed. The repo stays truth. git_notes and git_notes_index_state are both rebuildable from the repositories on disk, and nothing may be stored in them alone. What they buy is the reads a tree cannot answer cheaply: search, an annotated-commits filter, a feed ordered by when annotations were written, and per-commit counts without flattening a tree. The state table is the part the plan did not call for. It records the notes-ref tip each namespace was indexed from, which does two things. It gives the diff its old side without being told: the post-receive hook knows the previous tip, but the inbox merge and the browser write both move refs/notes/* in-process and fire no hook, so reading it from the table makes all three call sites the same call. And it distinguishes a cold index from a namespace with no notes, which an empty result cannot, so a read path can fall back to walking the repository rather than reporting a repository as unannotated. diff_notes prunes on subtree id: a push annotating one commit in a namespace with ten thousand notes reads the fanout levels on the path to that note and nothing else. A note whose path moved because git rebalanced the tree resolves to a set rather than a removal, in either direction, which a test pins along with the pruning itself.
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-08-09 00:58 UTC
Signed with PGP, not checked
Commit: 30e029974289fc496de8024b158a2876f5291037
Parent: 0262a7f
5 files changed, +940 insertions, -1 deletion
M server/src/db/mod.rs +1
@@ -28,6 +28,7 @@
28 28 pub(crate) mod follows;
29 29 pub mod gallery_images;
30 30 pub mod git_access_tokens;
31 + pub mod git_notes; // pub so mnw-admin can rebuild the index from the repositories
31 32 pub mod git_repos;
32 33 pub(crate) mod health;
33 34 mod id_types;
M server/src/git/notes/mod.rs +203 -1
@@ -274,7 +274,19 @@
274 274 root_tree: Oid,
275 275 visit: &mut impl FnMut(Oid, Oid),
276 276 ) -> Result<(), NotesError> {
277 - let mut level: Vec<(Vec<u8>, Oid)> = vec![(Vec::new(), root_tree)];
277 + flatten_under(engine, root_tree, &[], visit)
278 + }
279 +
280 + /// [`flatten`] over a subtree, where `prefix` is the path spelled by the
281 + /// directories above it. The diff uses this to enumerate the notes under a
282 + /// fanout directory that a change removed outright.
283 + fn flatten_under<E: NoteObjects>(
284 + engine: &E,
285 + root_tree: Oid,
286 + prefix: &[u8],
287 + visit: &mut impl FnMut(Oid, Oid),
288 + ) -> Result<(), NotesError> {
289 + let mut level: Vec<(Vec<u8>, Oid)> = vec![(prefix.to_vec(), root_tree)];
278 290
279 291 for _ in 0..MAX_FANOUT_DEPTH {
280 292 if level.is_empty() {
@@ -306,6 +318,196 @@
306 318 Ok(())
307 319 }
308 320
321 + // ── Diffing ──
322 +
323 + /// What changed about one note between two states of a namespace.
324 + #[derive(Debug, Clone, Copy, PartialEq, Eq)]
325 + pub enum NoteChange {
326 + /// The target now carries this note blob. Covers both a note that did not
327 + /// exist and one whose content changed; the two are the same statement to
328 + /// anybody maintaining a projection of the tree.
329 + Set { target: Oid, blob: Oid },
330 + /// The target no longer carries a note.
331 + Removed { target: Oid },
332 + }
333 +
334 + impl NoteChange {
335 + pub fn target(self) -> Oid {
336 + match self {
337 + Self::Set { target, .. } | Self::Removed { target } => target,
338 + }
339 + }
340 + }
341 +
342 + /// What changed between two states of a notes namespace, sorted by target.
343 + ///
344 + /// Either tip may be `None`, meaning the namespace did not exist: `None` to
345 + /// `Some` is every note added, `Some` to `None` is every note removed. Equal
346 + /// tips produce nothing.
347 + ///
348 + /// Subtrees whose ids match are skipped whole, which is the entire point of
349 + /// diffing rather than flattening both sides. A push that annotates one commit
350 + /// in a repository with ten thousand notes reads the fanout levels on the path
351 + /// to that one note and nothing else, because every sibling directory still
352 + /// hashes to what it hashed to before.
353 + ///
354 + /// A target that both sides disagree about in more than one way resolves to
355 + /// `Set`: git rebalances the fanout while writing, so a note moving from
356 + /// `ab/cdef…` to the flat `abcdef…` name shows up as a removal of the old path
357 + /// and a set of the new one, and reporting only the removal would drop a note
358 + /// that is still there.
359 + pub fn diff_notes<E: NoteObjects>(
360 + engine: &E,
361 + old_tip: Option<Oid>,
362 + new_tip: Option<Oid>,
363 + ) -> Result<Vec<NoteChange>, NotesError> {
364 + if old_tip == new_tip {
365 + return Ok(Vec::new());
366 + }
367 + let root = |tip: Option<Oid>| -> Result<Option<Oid>, NotesError> {
368 + match tip {
369 + Some(tip) => Ok(Some(engine.read_commit(tip)?.tree)),
370 + None => Ok(None),
371 + }
372 + };
373 +
374 + let mut changed: HashMap<Oid, Option<Oid>> = HashMap::new();
375 + diff_trees(engine, root(old_tip)?, root(new_tip)?, &[], 0, &mut changed)?;
376 +
377 + let mut out: Vec<NoteChange> = changed
378 + .into_iter()
379 + .map(|(target, blob)| match blob {
380 + Some(blob) => NoteChange::Set { target, blob },
381 + None => NoteChange::Removed { target },
382 + })
383 + .collect();
384 + out.sort_unstable_by_key(|change| change.target());
385 + Ok(out)
386 + }
387 +
388 + /// Record a note as gone, unless something already said it is there.
389 + ///
390 + /// `or_insert` rather than `insert` is what makes the diff order-independent: a
391 + /// note reached both as a removal (its old path vanished) and as a set (it lives
392 + /// at a new path) has to end up set, and the two are found in whichever order
393 + /// the trees happen to be walked.
394 + fn note_removed(out: &mut HashMap<Oid, Option<Oid>>, path: &[u8]) {
395 + if let Ok(target) = Oid::from_hex(path) {
396 + out.entry(target).or_insert(None);
397 + }
398 + }
399 +
400 + /// Everything under a fanout directory that is going away.
401 + fn notes_under<E: NoteObjects>(
402 + engine: &E,
403 + tree: Oid,
404 + prefix: &[u8],
405 + out: &mut HashMap<Oid, Option<Oid>>,
406 + ) -> Result<(), NotesError> {
407 + flatten_under(engine, tree, prefix, &mut |target, _| {
408 + out.entry(target).or_insert(None);
409 + })
410 + }
411 +
412 + /// Diff one level, descending only into subtrees whose ids differ.
413 + fn diff_trees<E: NoteObjects>(
414 + engine: &E,
415 + old: Option<Oid>,
416 + new: Option<Oid>,
417 + prefix: &[u8],
418 + depth: usize,
419 + out: &mut HashMap<Oid, Option<Oid>>,
420 + ) -> Result<(), NotesError> {
421 + // Identical subtrees hold identical notes. This is the pruning.
422 + if old == new {
423 + return Ok(());
424 + }
425 + // Deeper than any legitimate fanout. The reader stops here too, so a note
426 + // below it is one nothing can find and reporting it would put a row in the
427 + // index that no page could ever link to.
428 + if depth >= MAX_FANOUT_DEPTH {
429 + return Ok(());
430 + }
431 +
432 + let entries = |tree: Option<Oid>| -> Result<Vec<NewEntry>, NotesError> {
433 + match tree {
434 + Some(tree) => read_entries(engine, tree),
435 + None => Ok(Vec::new()),
436 + }
437 + };
438 + let old_entries = entries(old)?;
439 + let new_entries = entries(new)?;
440 + let named = |list: &[NewEntry], name: &[u8]| list.iter().find(|e| e.name == name).cloned();
441 +
442 + for entry in &new_entries {
443 + let mut path = prefix.to_vec();
444 + path.extend_from_slice(&entry.name);
445 + let before = named(&old_entries, &entry.name);
446 +
447 + if entry.kind == EntryKind::Tree {
448 + // Only descend paths that could still spell an id, the same rule
449 + // the flatten walk applies. A notes tree may carry unrelated
450 + // directories and they are not the index's business.
451 + if path.len() >= 64 || !path.iter().all(u8::is_ascii_hexdigit) {
452 + continue;
453 + }
454 + match &before {
455 + Some(old_entry) if old_entry.kind == EntryKind::Tree => {
456 + diff_trees(
457 + engine,
458 + Some(old_entry.oid),
459 + Some(entry.oid),
460 + &path,
461 + depth + 1,
462 + out,
463 + )?;
464 + }
465 + other => {
466 + // A leaf became a directory: the leaf's own path stops being
467 + // a note, and everything the directory holds is new.
468 + if other.as_ref().is_some_and(|e| e.kind.is_blob()) {
469 + note_removed(out, &path);
470 + }
471 + diff_trees(engine, None, Some(entry.oid), &path, depth + 1, out)?;
472 + }
473 + }
474 + } else if entry.kind.is_blob() {
475 + let Ok(target) = Oid::from_hex(&path) else {
476 + continue;
477 + };
478 + match &before {
479 + Some(old_entry) if old_entry.kind.is_blob() && old_entry.oid == entry.oid => {}
480 + Some(old_entry) if old_entry.kind == EntryKind::Tree => {
481 + // A directory became a leaf of the same name, which a hand
482 + // -built tree can do even though a splice does not.
483 + notes_under(engine, old_entry.oid, &path, out)?;
484 + out.insert(target, Some(entry.oid));
485 + }
486 + _ => {
487 + out.insert(target, Some(entry.oid));
488 + }
489 + }
490 + }
491 + }
492 +
493 + for entry in &old_entries {
494 + if new_entries.iter().any(|e| e.name == entry.name) {
495 + continue;
496 + }
497 + let mut path = prefix.to_vec();
498 + path.extend_from_slice(&entry.name);
499 +
500 + if entry.kind == EntryKind::Tree {
501 + if path.len() < 64 && path.iter().all(u8::is_ascii_hexdigit) {
502 + notes_under(engine, entry.oid, &path, out)?;
503 + }
504 + } else if entry.kind.is_blob() {
505 + note_removed(out, &path);
506 + }
507 + }
508 + Ok(())
509 + }
510 +
309 511 // ── Splicing ──
310 512
311 513 /// What splicing one level of a notes tree produced.
M server/src/git/notes/tests.rs +242
@@ -1600,3 +1600,245 @@
1600 1600 .collect();
1601 1601 assert_eq!(*cached, direct);
1602 1602 }
1603 +
1604 + // ── Diffing ──
1605 +
1606 + /// A [`GixEngine`] that counts the trees it reads.
1607 + ///
1608 + /// The whole justification for diffing rather than flattening both sides is
1609 + /// that identical subtrees are skipped, and nothing about the returned changes
1610 + /// shows whether that happened. Counting reads is what pins it.
1611 + struct Counting<'repo> {
1612 + inner: GixEngine<'repo>,
1613 + trees: std::cell::Cell<usize>,
1614 + }
1615 +
1616 + impl<'repo> Counting<'repo> {
1617 + fn new(repo: &'repo gix::Repository) -> Self {
1618 + Self {
1619 + inner: GixEngine::new(repo),
1620 + trees: std::cell::Cell::new(0),
1621 + }
1622 + }
1623 + }
1624 +
1625 + impl NoteObjects for Counting<'_> {
1626 + fn resolve_ref(&self, full_name: &str) -> Result<Option<Oid>, NotesError> {
1627 + self.inner.resolve_ref(full_name)
1628 + }
1629 + fn list_refs(&self, prefix: &str, visit: &mut dyn FnMut(&str, Oid)) -> Result<(), NotesError> {
1630 + self.inner.list_refs(prefix, visit)
1631 + }
1632 + fn read_commit(&self, oid: Oid) -> Result<CommitMeta, NotesError> {
1633 + self.inner.read_commit(oid)
1634 + }
1635 + fn read_tree_with(
1636 + &self,
1637 + oid: Oid,
1638 + visit: &mut dyn FnMut(TreeEntry<'_>) -> Walk,
1639 + ) -> Result<(), NotesError> {
1640 + self.trees.set(self.trees.get() + 1);
1641 + self.inner.read_tree_with(oid, visit)
1642 + }
1643 + fn read_blob_into(&self, oid: Oid, out: &mut Vec<u8>) -> Result<(), NotesError> {
1644 + self.inner.read_blob_into(oid, out)
1645 + }
1646 + fn merge_base(&self, one: Oid, two: Oid) -> Result<Option<Oid>, NotesError> {
1647 + self.inner.merge_base(one, two)
1648 + }
1649 + }
1650 +
1651 + /// Annotate `target` in the default namespace and return the new tip.
1652 + fn annotate(engine: &GixEngine<'_>, target: &str, text: &str) -> Oid {
1653 + match write_note(
1654 + engine,
1655 + DEFAULT_NAMESPACE,
1656 + oid(target),
1657 + Some(text.as_bytes()),
1658 + &signature("Fixture"),
1659 + )
1660 + .unwrap()
1661 + {
1662 + Written::Committed { tip, .. } => tip,
1663 + Written::Unchanged => panic!("the fixture wrote nothing"),
1664 + }
1665 + }
1666 +
1667 + /// Remove `target`'s note and return the new tip.
1668 + fn unannotate(engine: &GixEngine<'_>, target: &str) -> Oid {
1669 + match write_note(
1670 + engine,
1671 + DEFAULT_NAMESPACE,
1672 + oid(target),
1673 + None,
1674 + &signature("Fixture"),
1675 + )
1676 + .unwrap()
1677 + {
1678 + Written::Committed { tip, .. } => tip,
1679 + Written::Unchanged => panic!("the fixture removed nothing"),
1680 + }
1681 + }
1682 +
1683 + /// A fourth target, sharing no prefix with the three the fixtures use, so a
1684 + /// note on it lands flat at the root rather than splitting a directory.
1685 + const T4: &str = "40404040404040404040404040404040404040ff";
1686 +
1687 + #[test]
1688 + fn a_diff_from_nothing_is_every_note() {
1689 + let (_tmp, repo) = mixed_fanout_repo();
1690 + let engine = GixEngine::new(&repo);
1691 + let ns = namespace(&engine, DEFAULT_NAMESPACE);
1692 +
1693 + let changes = diff_notes(&engine, None, Some(ns.tip)).unwrap();
1694 + let targets: Vec<Oid> = changes.iter().map(|c| c.target()).collect();
1695 + let mut expected = vec![oid(T1), oid(T2), oid(T3)];
1696 + expected.sort_unstable();
1697 + assert_eq!(targets, expected, "a cold index reads the whole namespace");
1698 +
1699 + // And it names the right blob for each, across all three fanout shapes.
1700 + let flattened: HashMap<Oid, Oid> = annotated_targets(&engine, ns.tip)
1701 + .unwrap()
1702 + .into_iter()
1703 + .collect();
1704 + for change in &changes {
1705 + match change {
1706 + NoteChange::Set { target, blob } => assert_eq!(Some(blob), flattened.get(target)),
1707 + NoteChange::Removed { .. } => panic!("nothing was removed: {change:?}"),
1708 + }
1709 + }
1710 + }
1711 +
1712 + #[test]
1713 + fn a_tip_that_did_not_move_diffs_to_nothing() {
1714 + let (_tmp, repo) = mixed_fanout_repo();
1715 + let engine = GixEngine::new(&repo);
1716 + let ns = namespace(&engine, DEFAULT_NAMESPACE);
1717 +
1718 + assert!(
1719 + diff_notes(&engine, Some(ns.tip), Some(ns.tip))
1720 + .unwrap()
1721 + .is_empty()
1722 + );
1723 + assert!(diff_notes(&engine, None, None).unwrap().is_empty());
1724 + }
1725 +
1726 + #[test]
1727 + fn one_new_note_is_one_change_and_reads_less_than_a_walk() {
1728 + let (_tmp, repo) = mixed_fanout_repo();
1729 + let engine = GixEngine::new(&repo);
1730 + let before = namespace(&engine, DEFAULT_NAMESPACE).tip;
1731 + let after = annotate(&engine, T4, "the fourth");
1732 +
1733 + let counting = Counting::new(&repo);
1734 + let changes = diff_notes(&counting, Some(before), Some(after)).unwrap();
1735 + let read_by_diff = counting.trees.get();
1736 +
1737 + assert_eq!(changes.len(), 1);
1738 + assert_eq!(changes[0].target(), oid(T4));
1739 +
1740 + // The same engine, asked for the new state from cold: that reads every
1741 + // fanout directory in the namespace. The diff reads the two roots and stops,
1742 + // because `aa/` and `cc/` hash to what they hashed to before.
1743 + let counting = Counting::new(&repo);
1744 + diff_notes(&counting, None, Some(after)).unwrap();
1745 + assert!(
1746 + read_by_diff < counting.trees.get(),
1747 + "diff read {read_by_diff} trees, a full walk read {}",
1748 + counting.trees.get()
1749 + );
1750 + }
1751 +
1752 + #[test]
1753 + fn editing_a_note_reports_the_new_blob() {
1754 + let (_tmp, repo) = mixed_fanout_repo();
1755 + let engine = GixEngine::new(&repo);
1756 + let before = namespace(&engine, DEFAULT_NAMESPACE).tip;
1757 + // T2 lives under `aa/`, so this exercises a change inside a fanout level.
1758 + let after = annotate(&engine, T2, "rewritten");
1759 +
1760 + let changes = diff_notes(&engine, Some(before), Some(after)).unwrap();
1761 + let [NoteChange::Set { target, blob }] = changes.as_slice() else {
1762 + panic!("expected exactly one set: {changes:?}");
1763 + };
1764 + assert_eq!(*target, oid(T2));
1765 +
1766 + let mut content = Vec::new();
1767 + engine.read_blob_into(*blob, &mut content).unwrap();
1768 + assert_eq!(content, b"rewritten");
1769 + }
1770 +
1771 + #[test]
1772 + fn removing_a_note_reports_it_removed_and_says_nothing_about_the_others() {
1773 + let (_tmp, repo) = mixed_fanout_repo();
1774 + let engine = GixEngine::new(&repo);
1775 + let before = namespace(&engine, DEFAULT_NAMESPACE).tip;
1776 + // T3 sits two levels down, so removing it collapses `cc/dd/` on the way out.
1777 + let after = unannotate(&engine, T3);
1778 +
1779 + assert_eq!(
1780 + diff_notes(&engine, Some(before), Some(after)).unwrap(),
1781 + vec![NoteChange::Removed { target: oid(T3) }]
1782 + );
1783 + }
1784 +
1785 + #[test]
1786 + fn dropping_the_namespace_removes_every_note() {
1787 + let (_tmp, repo) = mixed_fanout_repo();
1788 + let engine = GixEngine::new(&repo);
1789 + let ns = namespace(&engine, DEFAULT_NAMESPACE);
1790 +
1791 + let changes = diff_notes(&engine, Some(ns.tip), None).unwrap();
1792 + assert_eq!(changes.len(), 3);
1793 + assert!(
1794 + changes
1795 + .iter()
1796 + .all(|c| matches!(c, NoteChange::Removed { .. })),
1797 + "{changes:?}"
1798 + );
1799 + }
1800 +
1801 + #[test]
1802 + fn a_note_that_only_changed_fanout_shape_is_set_rather_than_removed() {
1803 + // Both directions of the rebalance git does for itself: flat to `aa/…`, and
1804 + // back. Either one moves a note's path without changing the note, and a diff
1805 + // that reported only the vanished path would delete a row for a note that is
1806 + // still there.
1807 + let (_tmp, repo) = init_bare();
1808 + let note = blob(&repo, "unchanged");
1809 +
1810 + let flat = tree(&repo, &[(T1, note, GixEntryKind::Blob)]);
1811 + let inner = tree(&repo, &[(&T1[2..], note, GixEntryKind::Blob)]);
1812 + let split = tree(&repo, &[(&T1[..2], inner, GixEntryKind::Tree)]);
1813 +
1814 + let first = commit_notes(
1815 + &repo,
1816 + "refs/notes/commits",
1817 + "Fixture",
1818 + "flat",
1819 + flat,
1820 + Vec::new(),
1821 + );
1822 + let second = commit_notes(
1823 + &repo,
1824 + "refs/notes/commits",
1825 + "Fixture",
1826 + "split",
1827 + split,
1828 + vec![first],
1829 + );
1830 + let engine = GixEngine::new(&repo);
1831 +
1832 + let expected = vec![NoteChange::Set {
1833 + target: oid(T1),
1834 + blob: ours(note),
1835 + }];
1836 + assert_eq!(
1837 + diff_notes(&engine, Some(ours(first)), Some(ours(second))).unwrap(),
1838 + expected
1839 + );
1840 + assert_eq!(
1841 + diff_notes(&engine, Some(ours(second)), Some(ours(first))).unwrap(),
1842 + expected
1843 + );
1844 + }
A server/migrations/197_git_notes.sql +113
@@ -1,0 +1,113 @@
1 + -- The Postgres index over refs/notes/*.
2 + --
3 + -- The repo is truth. Every row here is a projection of an object that exists in
4 + -- a bare repository on disk, and the whole table can be dropped and rebuilt from
5 + -- those repositories (`mnw-admin reindex-notes`). Nothing may be stored here and
6 + -- nowhere else: a creator who leaves takes their annotations because the
7 + -- annotations were never ours. Wiki: mnw-server-git-notes, "the load-bearing
8 + -- rule".
9 + --
10 + -- What the index buys is the reads the tree cannot answer cheaply: full-text
11 + -- search, an "annotated commits" filter, a feed ordered by when annotations were
12 + -- written, and per-commit counts on a log page without flattening a tree. The
13 + -- tip-keyed cache in src/git/notes covers a warm repository; it does not cover
14 + -- searching one, and it cannot order anything by time without reading a commit
15 + -- header per note.
16 + --
17 + -- VISIBILITY IS NOT STORED. A note in a repository inherits the repository's
18 + -- visibility because it lives there; a row in a table inherits nothing. Every
19 + -- read path joins to git_repos and re-checks, and no column here may be allowed
20 + -- to substitute for that check.
21 +
22 + CREATE TABLE IF NOT EXISTS git_notes (
23 + repo_id UUID NOT NULL REFERENCES git_repos(id) ON DELETE CASCADE,
24 +
25 + -- Namespace as a person says it: `commits`, `mnw/builds`. Not the full ref.
26 + namespace TEXT NOT NULL,
27 +
28 + -- The annotated object, and the blob holding the note. Hex, so 40 characters
29 + -- today and 64 in a SHA-256 repository; the notes layer is hash-agnostic and
30 + -- this column has no business being narrower than it is.
31 + target_oid TEXT NOT NULL,
32 + blob_oid TEXT NOT NULL,
33 +
34 + -- The note, decoded lossily. Notes are conventionally UTF-8 and nothing
35 + -- enforces it, so a note holding arbitrary bytes indexes as replacement
36 + -- characters here while the repository keeps the bytes it was given. Search
37 + -- over a mangled copy is the right trade; storing bytea would make the
38 + -- tsvector impossible and buy a fidelity nobody reads this table for.
39 + content TEXT NOT NULL,
40 +
41 + -- What the annotated object turned out to be, resolved once at index time.
42 + -- Notes annotate blobs and trees too, and the "annotated commits" filter is
43 + -- this column: without it the filter would mean re-opening the repository to
44 + -- ask what each target is.
45 + target_is_commit BOOLEAN NOT NULL DEFAULT FALSE,
46 + -- Subject line and commit time of the target, empty and NULL when it is not
47 + -- a commit. Denormalized so the feed is one query rather than one commit
48 + -- header read per row, which is what bounds the repo-backed feed today.
49 + target_summary TEXT NOT NULL DEFAULT '',
50 + target_time TIMESTAMPTZ,
51 +
52 + -- When the annotation was written, and by whom, taken from the notes commit
53 + -- that carried it. A full rebuild has no per-note history to consult without
54 + -- walking the notes ref once per note, so it stamps every row with the tip's
55 + -- own commit time: rebuilding compresses the feed's timeline, and never
56 + -- reorders it against the repository, because the ref's history is gone
57 + -- either way once the rows are rewritten.
58 + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
59 + updated_by TEXT NOT NULL DEFAULT '',
60 +
61 + -- Search. `left(...)` because to_tsvector rejects input past 1MB and a
62 + -- pushed note has no size limit (the browser write path caps at 50k chars;
63 + -- a push does not go through it). Truncating the vector loses the tail of a
64 + -- pathological note from search results, which beats failing its insert and
65 + -- leaving the namespace unindexed.
66 + --
67 + -- No setweight: there is one field, so weighting it would rank nothing
68 + -- against nothing. to_tsvector's config is named explicitly because the bare
69 + -- form reads default_text_search_config and is not IMMUTABLE, which a
70 + -- generated column requires.
71 + search_tsv tsvector GENERATED ALWAYS AS (
72 + to_tsvector('english', left(content, 200000))
73 + ) STORED,
74 +
75 + PRIMARY KEY (repo_id, namespace, target_oid)
76 + );
77 +
78 + CREATE INDEX IF NOT EXISTS idx_git_notes_search ON git_notes USING GIN (search_tsv);
79 +
80 + -- The feed and the atom feed: newest annotation first, within one repository.
81 + CREATE INDEX IF NOT EXISTS idx_git_notes_feed ON git_notes (repo_id, updated_at DESC);
82 +
83 + -- The log page asking "which of these commits carry a note", across namespaces.
84 + -- The primary key leads with repo_id and namespace, so it cannot serve this.
85 + CREATE INDEX IF NOT EXISTS idx_git_notes_target ON git_notes (repo_id, target_oid);
86 +
87 + -- What the index has already seen, per namespace.
88 + --
89 + -- Two jobs, and the second is the reason this is a table rather than a column.
90 + --
91 + -- 1. The reindex diffs the tip it last indexed against the tip the ref holds
92 + -- now, so a push walks what changed rather than the whole namespace. The
93 + -- post-receive hook knows the previous tip, but the other two writers do not:
94 + -- the inbox merge and the browser write both move refs/notes/* in-process and
95 + -- fire no hook. Reading the previous tip from here rather than being told it
96 + -- makes all three call sites the same call, and makes a reindex that ran
97 + -- twice a no-op instead of a rewalk.
98 + --
99 + -- 2. It distinguishes "this namespace has no notes" from "this namespace has
100 + -- never been indexed". A read path cannot tell those apart from an empty
101 + -- result, and answering a cold index with "no notes" would make a repository
102 + -- look unannotated until somebody pushed to it. Absent row means cold, and
103 + -- the read falls back to walking the repository.
104 + CREATE TABLE IF NOT EXISTS git_notes_index_state (
105 + repo_id UUID NOT NULL REFERENCES git_repos(id) ON DELETE CASCADE,
106 + namespace TEXT NOT NULL,
107 + -- The notes ref tip the rows in git_notes were built from. Hex, never NULL:
108 + -- a namespace whose ref is gone has its state row deleted along with its
109 + -- notes, so a row here always names a commit.
110 + indexed_tip TEXT NOT NULL,
111 + indexed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
112 + PRIMARY KEY (repo_id, namespace)
113 + );
A server/src/db/git_notes.rs +381
@@ -1,0 +1,381 @@
1 + //! The Postgres index over `refs/notes/*`.
2 + //!
3 + //! <!-- wiki: mnw-server-git-notes -->
4 + //!
5 + //! Every row here is a projection of a git object. The repository is truth and
6 + //! this table is rebuildable from it, so nothing in this module may be the only
7 + //! copy of anything: see migration 197 and the wiki note's load-bearing rule.
8 + //! The consequence for this file is that every write is a whole-namespace
9 + //! statement of fact rather than an edit, and every read is allowed to be wrong
10 + //! in the direction of "ask the repository instead".
11 + //!
12 + //! Two rules the callers depend on:
13 + //!
14 + //! - **A namespace's rows and its index-state row move together, in one
15 + //! transaction.** The state row says which notes-ref tip the rows were built
16 + //! from, and the next reindex diffs against it. A state row ahead of the rows
17 + //! it describes would make the next diff skip changes it never applied, and
18 + //! nothing later would notice.
19 + //! - **Visibility is not stored and is not checked here.** These functions take
20 + //! a repo id and answer about that repository. Whether the person asking may
21 + //! see it is the caller's question, decided the same way it is for the
22 + //! repository's other pages.
23 +
24 + use std::collections::HashMap;
25 +
26 + use chrono::{DateTime, Utc};
27 + use sqlx::PgPool;
28 +
29 + use super::GitRepoId;
30 + use crate::error::Result;
31 +
32 + /// One note as the reindex hands it over.
33 + ///
34 + /// `target_summary` and `target_time` are the annotated commit's, resolved once
35 + /// at index time so the feed does not read a commit header per row. They are
36 + /// empty and `None` for a note on a blob or a tree.
37 + #[derive(Debug, Clone)]
38 + pub struct NoteUpsert {
39 + pub target_oid: String,
40 + pub blob_oid: String,
41 + pub content: String,
42 + pub target_is_commit: bool,
43 + pub target_summary: String,
44 + pub target_time: Option<DateTime<Utc>>,
45 + }
46 +
47 + /// One indexed note, as the read paths want it.
48 + #[derive(Debug, Clone, sqlx::FromRow)]
49 + pub struct IndexedNote {
50 + pub namespace: String,
51 + pub target_oid: String,
52 + pub blob_oid: String,
53 + pub content: String,
54 + pub target_is_commit: bool,
55 + pub target_summary: String,
56 + pub target_time: Option<DateTime<Utc>>,
57 + pub updated_at: DateTime<Utc>,
58 + pub updated_by: String,
59 + }
60 +
61 + /// The tip a namespace was last indexed from, or `None` when it has never been
62 + /// indexed.
63 + ///
64 + /// The distinction is load-bearing on both sides. The reindex uses it as the old
65 + /// side of the diff, so an absent row means "walk the whole namespace". The read
66 + /// paths use it to tell a cold index from a repository with no notes, which an
67 + /// empty result cannot distinguish.
68 + #[tracing::instrument(skip_all)]
69 + pub async fn indexed_tip(
70 + pool: &PgPool,
71 + repo_id: GitRepoId,
72 + namespace: &str,
73 + ) -> Result<Option<String>> {
74 + let tip = sqlx::query_scalar::<_, String>(
75 + "SELECT indexed_tip FROM git_notes_index_state WHERE repo_id = $1 AND namespace = $2",
76 + )
77 + .bind(repo_id)
78 + .bind(namespace)
79 + .fetch_optional(pool)
80 + .await?;
81 + Ok(tip)
82 + }
83 +
84 + /// Every namespace this repository has an index for, whether or not it still has
85 + /// notes in it.
86 + #[tracing::instrument(skip_all)]
87 + pub async fn indexed_namespaces(pool: &PgPool, repo_id: GitRepoId) -> Result<Vec<String>> {
88 + let names = sqlx::query_scalar::<_, String>(
89 + "SELECT namespace FROM git_notes_index_state WHERE repo_id = $1 ORDER BY namespace",
90 + )
91 + .bind(repo_id)
92 + .fetch_all(pool)
93 + .await?;
94 + Ok(names)
95 + }
96 +
97 + /// Whether anything in this repository has been indexed.
98 + ///
99 + /// The read paths ask this before trusting an empty result: a repository with no
100 + /// state rows has a cold index, not an unannotated history, and the answer to a
101 + /// cold index is to walk the repository rather than to report nothing.
102 + #[tracing::instrument(skip_all)]
103 + pub async fn is_indexed(pool: &PgPool, repo_id: GitRepoId) -> Result<bool> {
104 + let found = sqlx::query_scalar::<_, bool>(
105 + "SELECT EXISTS (SELECT 1 FROM git_notes_index_state WHERE repo_id = $1)",
106 + )
107 + .bind(repo_id)
108 + .fetch_one(pool)
109 + .await?;
110 + Ok(found)
111 + }
112 +
113 + /// One namespace's diff, and the tip it was computed against.
114 + ///
115 + /// A struct rather than eight arguments because every field has to describe the
116 + /// same reindex: a `tip` that did not produce these `upserts` is the one way to
117 + /// corrupt the index silently, and keeping them in one value makes that hard to
118 + /// do by accident.
119 + pub struct NamespaceUpdate<'a> {
120 + pub namespace: &'a str,
121 + /// Notes added or changed since the previously indexed tip.
122 + pub upserts: &'a [NoteUpsert],
123 + /// Target ids whose note is gone, hex.
124 + pub removals: &'a [String],
125 + /// The notes-ref tip this diff brings the index up to.
126 + pub tip: &'a str,
127 + /// When the annotations were written, and by whom: the notes commit's own
128 + /// committer, not the person who triggered the reindex.
129 + pub updated_at: DateTime<Utc>,
130 + pub updated_by: &'a str,
131 + }
132 +
133 + /// Apply one namespace's diff and record the tip it was computed against.
134 + ///
135 + /// The upserts and removals are what changed between the previously indexed tip
136 + /// and `update.tip`; everything else in the namespace is left alone, which is
137 + /// what makes a push cost the size of its diff. All of it commits together with
138 + /// the state row, so a failure halfway leaves the namespace exactly as it was
139 + /// and the next reindex recomputes the same diff.
140 + #[tracing::instrument(skip_all)]
141 + pub async fn apply_changes(
142 + pool: &PgPool,
143 + repo_id: GitRepoId,
144 + update: &NamespaceUpdate<'_>,
145 + ) -> Result<()> {
146 + let NamespaceUpdate {
147 + namespace,
148 + upserts,
149 + removals,
150 + tip,
151 + updated_at,
152 + updated_by,
153 + } = *update;
154 +
155 + let mut tx = pool.begin().await?;
156 +
157 + if !removals.is_empty() {
158 + sqlx::query(
159 + "DELETE FROM git_notes
160 + WHERE repo_id = $1 AND namespace = $2 AND target_oid = ANY($3)",
161 + )
162 + .bind(repo_id)
163 + .bind(namespace)
164 + .bind(removals)
165 + .execute(&mut *tx)
166 + .await?;
167 + }
168 +
169 + if !upserts.is_empty() {
170 + // One statement for the batch. A push that annotates a thousand commits
171 + // is one round trip rather than a thousand, and the arrays keep the
172 + // parameter count fixed no matter how large the batch is.
173 + let targets: Vec<&str> = upserts.iter().map(|n| n.target_oid.as_str()).collect();
174 + let blobs: Vec<&str> = upserts.iter().map(|n| n.blob_oid.as_str()).collect();
175 + let contents: Vec<&str> = upserts.iter().map(|n| n.content.as_str()).collect();
176 + let is_commit: Vec<bool> = upserts.iter().map(|n| n.target_is_commit).collect();
177 + let summaries: Vec<&str> = upserts.iter().map(|n| n.target_summary.as_str()).collect();
178 + let times: Vec<Option<DateTime<Utc>>> = upserts.iter().map(|n| n.target_time).collect();
179 +
180 + sqlx::query(
181 + "INSERT INTO git_notes (
182 + repo_id, namespace, target_oid, blob_oid, content,
183 + target_is_commit, target_summary, target_time, updated_at, updated_by
184 + )
185 + SELECT $1, $2, t.target, t.blob, t.content, t.is_commit, t.summary, t.time, $9, $10
186 + FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bool[], $7::text[], $8::timestamptz[])
187 + AS t(target, blob, content, is_commit, summary, time)
188 + ON CONFLICT (repo_id, namespace, target_oid) DO UPDATE SET
189 + blob_oid = EXCLUDED.blob_oid,
190 + content = EXCLUDED.content,
191 + target_is_commit = EXCLUDED.target_is_commit,
192 + target_summary = EXCLUDED.target_summary,
193 + target_time = EXCLUDED.target_time,
194 + updated_at = EXCLUDED.updated_at,
195 + updated_by = EXCLUDED.updated_by",
196 + )
197 + .bind(repo_id)
198 + .bind(namespace)
199 + .bind(&targets)
200 + .bind(&blobs)
201 + .bind(&contents)
202 + .bind(&is_commit)
203 + .bind(&summaries)
204 + .bind(&times)
205 + .bind(updated_at)
206 + .bind(updated_by)
207 + .execute(&mut *tx)
208 + .await?;
209 + }
210 +
211 + sqlx::query(
212 + "INSERT INTO git_notes_index_state (repo_id, namespace, indexed_tip, indexed_at)
213 + VALUES ($1, $2, $3, NOW())
214 + ON CONFLICT (repo_id, namespace) DO UPDATE SET
215 + indexed_tip = EXCLUDED.indexed_tip,
216 + indexed_at = EXCLUDED.indexed_at",
217 + )
218 + .bind(repo_id)
219 + .bind(namespace)
220 + .bind(tip)
221 + .execute(&mut *tx)
222 + .await?;
223 +
224 + tx.commit().await?;
225 + Ok(())
226 + }
227 +
228 + /// Forget a namespace: its notes and the fact that it was ever indexed.
229 + ///
230 + /// What a deleted `refs/notes/<ns>` produces. The state row goes too, so the
231 + /// namespace reads as cold rather than as empty, and a ref that comes back is
232 + /// walked in full.
233 + #[tracing::instrument(skip_all)]
234 + pub async fn forget_namespace(pool: &PgPool, repo_id: GitRepoId, namespace: &str) -> Result<()> {
235 + let mut tx = pool.begin().await?;
236 + sqlx::query("DELETE FROM git_notes WHERE repo_id = $1 AND namespace = $2")
237 + .bind(repo_id)
238 + .bind(namespace)
239 + .execute(&mut *tx)
240 + .await?;
241 + sqlx::query("DELETE FROM git_notes_index_state WHERE repo_id = $1 AND namespace = $2")
242 + .bind(repo_id)
243 + .bind(namespace)
244 + .execute(&mut *tx)
245 + .await?;
246 + tx.commit().await?;
247 + Ok(())
248 + }
249 +
250 + /// How many notes are indexed per namespace, for the Notes tab's counts.
251 + #[tracing::instrument(skip_all)]
252 + pub async fn counts_by_namespace(pool: &PgPool, repo_id: GitRepoId) -> Result<Vec<(String, i64)>> {
253 + let rows = sqlx::query_as::<_, (String, i64)>(
254 + "SELECT namespace, COUNT(*) FROM git_notes WHERE repo_id = $1
255 + GROUP BY namespace ORDER BY namespace",
256 + )
257 + .bind(repo_id)
258 + .fetch_all(pool)
259 + .await?;
260 + Ok(rows)
261 + }
262 +
263 + /// How many namespaces annotate each of `targets`.
264 + ///
265 + /// The log-page path: one query for a page of commits, keyed by the hex the
266 + /// caller passed in. Targets with no note are absent from the map rather than
267 + /// present with a zero.
268 + #[tracing::instrument(skip_all)]
269 + pub async fn annotation_counts(
270 + pool: &PgPool,
271 + repo_id: GitRepoId,
272 + targets: &[String],
273 + ) -> Result<HashMap<String, i64>> {
274 + if targets.is_empty() {
275 + return Ok(HashMap::new());
276 + }
277 + let rows = sqlx::query_as::<_, (String, i64)>(
278 + "SELECT target_oid, COUNT(*) FROM git_notes
279 + WHERE repo_id = $1 AND target_oid = ANY($2)
280 + GROUP BY target_oid",
281 + )
282 + .bind(repo_id)
283 + .bind(targets)
284 + .fetch_all(pool)
285 + .await?;
286 + Ok(rows.into_iter().collect())
287 + }
288 +
289 + /// The annotation feed: newest annotation first.
290 + ///
291 + /// Ordered by when the note was written rather than by the annotated commit's
292 + /// own date, because a feed answers "what has been said lately" and annotating a
293 + /// five-year-old commit is news. `commits_only` is the "annotated commits"
294 + /// filter; `namespace` of `None` reads across all of them.
295 + #[tracing::instrument(skip_all)]
296 + pub async fn feed(
297 + pool: &PgPool,
298 + repo_id: GitRepoId,
299 + namespace: Option<&str>,
300 + commits_only: bool,
301 + limit: i64,
302 + offset: i64,
303 + ) -> Result<Vec<IndexedNote>> {
304 + let rows = sqlx::query_as::<_, IndexedNote>(
305 + "SELECT namespace, target_oid, blob_oid, content, target_is_commit,
306 + target_summary, target_time, updated_at, updated_by
307 + FROM git_notes
308 + WHERE repo_id = $1
309 + AND ($2::text IS NULL OR namespace = $2)
310 + AND (NOT $3::bool OR target_is_commit)
311 + ORDER BY updated_at DESC, target_oid
312 + LIMIT $4 OFFSET $5",
313 + )
314 + .bind(repo_id)
315 + .bind(namespace)
316 + .bind(commits_only)
317 + .bind(limit)
318 + .bind(offset)
319 + .fetch_all(pool)
320 + .await?;
321 + Ok(rows)
322 + }
323 +
324 + /// Notes matching the feed's filters, for the page count beside the rows.
325 + #[tracing::instrument(skip_all)]
326 + pub async fn count_notes(
327 + pool: &PgPool,
328 + repo_id: GitRepoId,
329 + namespace: Option<&str>,
330 + commits_only: bool,
331 + ) -> Result<i64> {
332 + let count = sqlx::query_scalar::<_, i64>(
333 + "SELECT COUNT(*) FROM git_notes
334 + WHERE repo_id = $1
335 + AND ($2::text IS NULL OR namespace = $2)
336 + AND (NOT $3::bool OR target_is_commit)",
337 + )
338 + .bind(repo_id)
339 + .bind(namespace)
340 + .bind(commits_only)
341 + .fetch_one(pool)
342 + .await?;
343 + Ok(count)
344 + }
345 +
346 + /// Full-text search within one repository's notes.
347 + ///
348 + /// `websearch_to_tsquery` rather than `plainto_tsquery`: it accepts quoted
349 + /// phrases and `or`/`-` the way a person types them into a search box, and it
350 + /// never raises on malformed input, so a stray quote is a query that finds
351 + /// little rather than a 500.
352 + #[tracing::instrument(skip_all)]
353 + pub async fn search(
354 + pool: &PgPool,
355 + repo_id: GitRepoId,
356 + query: &str,
357 + namespace: Option<&str>,
358 + commits_only: bool,
359 + limit: i64,
360 + ) -> Result<Vec<IndexedNote>> {
361 + let rows = sqlx::query_as::<_, IndexedNote>(
362 + "SELECT namespace, target_oid, blob_oid, content, target_is_commit,
363 + target_summary, target_time, updated_at, updated_by
364 + FROM git_notes
365 + WHERE repo_id = $1
366 + AND ($3::text IS NULL OR namespace = $3)
367 + AND (NOT $4::bool OR target_is_commit)
368 + AND search_tsv @@ websearch_to_tsquery('english', $2)
369 + ORDER BY ts_rank(search_tsv, websearch_to_tsquery('english', $2)) DESC,
370 + updated_at DESC
371 + LIMIT $5",
372 + )
373 + .bind(repo_id)
374 + .bind(query)
375 + .bind(namespace)
376 + .bind(commits_only)
377 + .bind(limit)
378 + .fetch_all(pool)
379 + .await?;
380 + Ok(rows)
381 + }