Skip to main content

max / makenotwork

tagtree: test the oracle's own helpers, and record what no test can reach oracle.rs asserted things about the crate and nothing asserted anything about oracle.rs. Measured on astra, 149 mutants scoped to the file: 36 missed, 111 caught, 2 timeouts. That reproduces the 36 recorded on infra a98a96ea from a different run, so the number is not an artifact of scoping. Sixteen tests in a new `#[cfg(test)] mod tests` kill ten of the thirty-six. split_query and split_rename parse the fuzz script's operands and had no test of any kind; both are covered including the two cases that separate them from their plausible neighbours, since rsplit_once and split_once agree on every input with one delimiter. model_valid is the independent reimplementation validate_with is checked against, so its two bounds are asserted from both sides: `>` and `>=` disagree only at equality, and a tag one character under the limit passes under either. Eight go in .cargo/mutants.toml. Seven are the `replace check_X with ()` entry points, unkillable for the reason s3-storage's oracle wrappers are: an oracle only speaks when the code under test is wrong, so killing one would mean shipping a broken tagtree for a test to catch. The eighth is the `||` in model_valid's leading/trailing separator guard, which is equivalent rather than untested -- a tag with a separator at either end also has an empty segment, and the empty-segment check three lines down rejects it whichever way the guard went. The list-diff verification says exactly those eight and nothing that was being caught: 149 to 141. The remaining eighteen are filed as infra cb62491e rather than excluded, because the bar is impossibility and these are cost. Twelve are deleted match arms and operators inside check_index_script_bounded, where the mutation makes the script driver dispatch less while every assertion that still runs continues to hold -- an oracle quietly checking less than it claims, which is the failure mode this file exists to catch and the one place it was blind to itself. The two timeouts are `+=` on loop counters, so they do not terminate rather than answering wrongly; they want a bounded loop in the test, not an exclusion.
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session
https://claude.ai/code/session_01MptwXZ8k65v19rFmdGAyki
Author: Max Johnson <me@maxj.phd> · 2026-08-31 16:01 UTC
Signed with PGP, not checked
Commit: b7442862ba5589a0ccea47fea95256e8ca493220
Parent: ddd53ad
2 files changed, +188 insertions, -0 deletions
M shared/tagtree/.cargo/mutants.toml +36
@@ -79,4 +79,40 @@
79 79 # well, and Levenshtein distance is symmetric: same row length, same
80 80 # `b.len() - a.len()` of zero for the short-circuit, same distance.
81 81 "src/lib.rs:561:29: replace > with >= in edit_distance",
82 +
83 + # --- The oracle's own entry points ------------------------------------
84 + #
85 + # Each of these is a `check_*` function whose entire body is assertions
86 + # about the crate. An oracle only speaks when the code under test is wrong,
87 + # so replacing one with `()` is invisible to every test that PASSES: killing
88 + # it would mean shipping a broken tagtree for a test to catch, which is not
89 + # a test that can exist alongside a correct crate. Same class as
90 + # s3-storage's `oracle::check_plan`/`check_auto`.
91 + #
92 + # The evidence they work is the other half of the same run rather than a
93 + # mutant: 111 of oracle.rs's 149 mutants were caught, and they were caught
94 + # AT these assertions.
95 + #
96 + # This covers only the seven entry points. The operators and match arms
97 + # INSIDE the oracle bodies are a different question -- a test could assert
98 + # that the script driver dispatches every opcode -- and those stay visible
99 + # survivors under infra `cb62491e`.
100 + "src/oracle.rs:151:5: replace check_tag_text with \\(\\)",
101 + "src/oracle.rs:204:5: replace check_index_script with \\(\\)",
102 + "src/oracle.rs:213:5: replace check_index_script_bounded with \\(\\)",
103 + "src/oracle.rs:347:5: replace check_tag with \\(\\)",
104 + "src/oracle.rs:509:5: replace check_escaping with \\(\\)",
105 + "src/oracle.rs:625:5: replace check_pair with \\(\\)",
106 + "src/oracle.rs:784:5: replace check_index_state with \\(\\)",
107 +
108 + # `tag.starts_with(SEPARATOR) || tag.ends_with(SEPARATOR)` in `model_valid`.
109 + # `&&` is equivalent for the accept decision, which is all this function
110 + # returns. A tag with a separator at either end necessarily has an empty
111 + # first or last segment, and the `segs.iter().any(|s| s.is_empty())` check
112 + # three lines below returns false for it regardless of which way this guard
113 + # went. So no input distinguishes them: under `||` the tag is rejected here,
114 + # under `&&` it is rejected there, and `model_valid` answers false either
115 + # way. `model_valid_rejects_leading_and_trailing_separators` pins the
116 + # behaviour; it cannot pin the operator, and says so.
117 + "src/oracle.rs:578:35: replace \\|\\| with && in model_valid",
82 118 ]
M shared/tagtree/src/oracle.rs +152
@@ -922,3 +922,155 @@
922 922 }
923 923 prev[b.len()]
924 924 }
925 +
926 + #[cfg(test)]
927 + mod tests {
928 + //! Unit tests for the oracle's own helpers.
929 + //!
930 + //! The oracle asserts things about the crate; nothing asserted anything about
931 + //! the oracle. `split_query` and `split_rename` parse the fuzz script's
932 + //! operands, and `model_valid` is the independent reimplementation that
933 + //! `validate_with` is checked against, so a wrong answer here does not fail a
934 + //! fuzz run: it makes one check less than it claims. Deleting this module
935 + //! would return all eleven mutants these kill.
936 + //!
937 + //! Inputs avoid 0, 1, "" and "xyzzy" wherever an operator or a constant is
938 + //! under test, because those are the values the mutants themselves return and
939 + //! a test that picks one cannot disagree with the mutant. See
940 + //! `_private/docs/meta/test_style.md`, "Tests that cannot disagree".
941 +
942 + use super::{model_valid, split_query, split_rename};
943 + use crate::TagConfig;
944 +
945 + // ── split_query: `input limit`, splitting on the LAST space ──
946 +
947 + #[test]
948 + fn split_query_reads_the_limit_after_the_last_space() {
949 + // 5 rather than 0 or 1: the four mutants return ("", 0), ("", 1),
950 + // ("xyzzy", 0) and ("xyzzy", 1), so both halves have to differ from all
951 + // of those for one assertion to kill all four.
952 + assert_eq!(split_query("rock 5"), ("rock", 5));
953 + }
954 +
955 + #[test]
956 + fn split_query_defaults_to_ten_with_no_space() {
957 + assert_eq!(split_query("rock"), ("rock", 10));
958 + }
959 +
960 + #[test]
961 + fn split_query_keeps_the_whole_string_when_the_limit_does_not_parse() {
962 + // The Err arm returns `rest`, not `input`, so the trailing word stays
963 + // part of the query rather than being eaten as a bad limit.
964 + assert_eq!(split_query("rock jazz"), ("rock jazz", 10));
965 + }
966 +
967 + #[test]
968 + fn split_query_clamps_the_limit_to_512() {
969 + assert_eq!(split_query("rock 9999"), ("rock", 512));
970 + }
971 +
972 + #[test]
973 + fn split_query_splits_on_the_last_space_not_the_first() {
974 + // Distinguishes rsplit_once from split_once, which agree on every
975 + // single-space input.
976 + assert_eq!(split_query("deep rock 7"), ("deep rock", 7));
977 + }
978 +
979 + // ── split_rename: `old>new`, splitting on the FIRST `>` ──
980 +
981 + #[test]
982 + fn split_rename_splits_on_the_arrow() {
983 + assert_eq!(split_rename("rock>metal"), ("rock", "metal"));
984 + }
985 +
986 + #[test]
987 + fn split_rename_strips_one_leading_space() {
988 + assert_eq!(split_rename(" rock>metal"), ("rock", "metal"));
989 + }
990 +
991 + #[test]
992 + fn split_rename_with_no_arrow_returns_the_input_twice() {
993 + // The unwrap_or arm. Both halves equal and non-empty, so it also fails
994 + // under every one of the four constant-tuple mutants.
995 + assert_eq!(split_rename("rock"), ("rock", "rock"));
996 + }
997 +
998 + #[test]
999 + fn split_rename_splits_on_the_first_arrow_not_the_last() {
1000 + // split_once vs rsplit_once agree on any single-arrow input.
1001 + assert_eq!(split_rename("a>b>c"), ("a", "b>c"));
1002 + }
1003 +
1004 + // ── model_valid: the boundaries, asserted from both sides ──
1005 +
1006 + fn cfg(max_depth: usize, max_length: usize) -> TagConfig {
1007 + TagConfig {
1008 + max_depth,
1009 + max_length,
1010 + semantic_depth: 0,
1011 + }
1012 + }
1013 +
1014 + #[test]
1015 + fn model_valid_accepts_a_tag_of_exactly_max_length() {
1016 + // `count > max_length` vs `>=`: the two disagree only at equality, so a
1017 + // tag one character shorter would pass under either.
1018 + assert!(
1019 + model_valid("abcde", &cfg(3, 5)),
1020 + "five characters is at the limit, not over it"
1021 + );
1022 + }
1023 +
1024 + #[test]
1025 + fn model_valid_rejects_a_tag_one_character_over_max_length() {
1026 + assert!(!model_valid("abcdef", &cfg(3, 5)));
1027 + }
1028 +
1029 + #[test]
1030 + fn model_valid_accepts_a_tag_at_exactly_max_depth() {
1031 + // `segs.len() > max_depth` vs `>=`, same boundary argument.
1032 + assert!(
1033 + model_valid("a.b.c", &cfg(3, 60)),
1034 + "three segments is at the depth limit, not over it"
1035 + );
1036 + }
1037 +
1038 + #[test]
1039 + fn model_valid_rejects_a_tag_one_segment_over_max_depth() {
1040 + assert!(!model_valid("a.b.c.d", &cfg(3, 60)));
1041 + }
1042 +
1043 + #[test]
1044 + fn model_valid_rejects_leading_and_trailing_separators() {
1045 + // Kept for the behaviour rather than for a mutant: the `||` at the
1046 + // leading/trailing guard is an equivalent mutant, because a tag with a
1047 + // separator at either end also has an empty segment and the empty-segment
1048 + // check below returns false for it either way. Recorded in
1049 + // .cargo/mutants.toml.
1050 + assert!(!model_valid(".rock", &cfg(3, 60)));
1051 + assert!(!model_valid("rock.", &cfg(3, 60)));
1052 + }
1053 +
1054 + #[test]
1055 + fn model_valid_rejects_the_empty_tag_and_uppercase() {
1056 + assert!(!model_valid("", &cfg(3, 60)));
1057 + assert!(
1058 + !model_valid("Rock", &cfg(3, 60)),
1059 + "uppercase is outside the allowed set"
1060 + );
1061 + }
1062 +
1063 + #[test]
1064 + fn model_valid_enforces_the_semantic_depth_floor() {
1065 + let c = TagConfig {
1066 + max_depth: 5,
1067 + max_length: 60,
1068 + semantic_depth: 2,
1069 + };
1070 + assert!(
1071 + !model_valid("genre.rock", &c),
1072 + "two segments is one short of the floor"
1073 + );
1074 + assert!(model_valid("genre.rock.metal", &c));
1075 + }
1076 + }