Skip to main content

max / makenotwork

Find markers with the parser's own idea of a quoted string The marker scanner and the expression parser disagreed about where a marker ends. Markers were located with the regex \{\{(.*?)\}\}, which knows nothing about quoting, while split_commas and parse_arg do. A }} inside a string argument ended the marker early: {{ price.basic | money("}}") }} The parser was handed the fragment ` price.basic | money("` and reported an unclosed paren -- true about the fragment, misleading about the input -- and the tail after the false ending was copied through as literal text, so substituting a creator template could emit live template syntax. Found by the soak tier on its second session, GoingsOn problem subst-substitute:4def8fda6141. parser::markers replaces the regex and lives beside the grammar it has to agree with. Two rules carried over deliberately: quotes are tracked exactly as split_commas tracks them, and a marker still cannot span a line -- the regex could not, because . does not match \n, and that bound is worth keeping rather than an accident to fix. Two consequences worth stating. regex-lite was the crate's only dependency and is now unused, so subst has none. And the fuzz target runs 2.7x faster, 6,793 to 18,461 exec/s, which is the same work without a regex engine. The fixed-point oracle that found this is narrowed in the same pass. It does not hold unconditionally even of correct code: a template may ask for text that looks like a marker and gets it back literally, because substitution is one pass and does not rescan its own output. Asserting otherwise would demand the engine escape its output, which fights the reason code spans exist. It is now asserted where it does hold, on input carrying no quote, and the contract it rests on is written on substitute where it had never been written at all.
Author: Max Johnson <me@maxj.phd> · 2026-08-11 23:58 UTC
Signed with PGP, not checked
Commit: db46ba6780ee1c482480e8693f25728cf2436092
Parent: 8ed15ec
6 files changed, +326 insertions, -35 deletions
M shared/subst/Cargo.lock +48 -9
@@ -2,15 +2,54 @@
2 2 # It is not intended for manual editing.
3 3 version = 4
4 4
5 - [[package]]
6 - name = "regex-lite"
7 - version = "0.1.9"
8 - source = "registry+https://github.com/rust-lang/crates.io-index"
9 - checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
10 -
11 5 [[package]]
12 6 name = "subst"
13 7 version = "0.1.0"
14 - dependencies = [
15 - "regex-lite",
16 - ]
8 +
9 + [[patch.unused]]
10 + name = "synckit-client"
11 + version = "0.8.0"
12 +
13 + [[patch.unused]]
14 + name = "synckit-config"
15 + version = "0.2.0"
16 +
17 + [[patch.unused]]
18 + name = "docengine"
19 + version = "0.5.0"
20 +
21 + [[patch.unused]]
22 + name = "kberg"
23 + version = "0.1.0"
24 +
25 + [[patch.unused]]
26 + name = "painhours"
27 + version = "0.1.0"
28 +
29 + [[patch.unused]]
30 + name = "tagtree"
31 + version = "0.4.0"
32 +
33 + [[patch.unused]]
34 + name = "quasi-axum"
35 + version = "0.1.0"
36 +
37 + [[patch.unused]]
38 + name = "quasi-http"
39 + version = "0.1.0"
40 +
41 + [[patch.unused]]
42 + name = "quasi-router"
43 + version = "0.1.0"
44 +
45 + [[patch.unused]]
46 + name = "quasi-store"
47 + version = "0.1.0"
48 +
49 + [[patch.unused]]
50 + name = "quasi-tauri"
51 + version = "0.1.0"
52 +
53 + [[patch.unused]]
54 + name = "quasi-webview"
55 + version = "0.1.0"
M shared/subst/Cargo.toml -1
@@ -5,7 +5,6 @@
5 5 license = "MIT"
6 6
7 7 [dependencies]
8 - regex-lite = "0.1"
9 8
10 9 [dev-dependencies]
11 10
M shared/subst/fuzz/Cargo.lock -9
@@ -107,12 +107,6 @@
107 107 source = "registry+https://github.com/rust-lang/crates.io-index"
108 108 checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
109 109
110 - [[package]]
111 - name = "regex-lite"
112 - version = "0.1.9"
113 - source = "registry+https://github.com/rust-lang/crates.io-index"
114 - checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
115 -
116 110 [[package]]
117 111 name = "shlex"
118 112 version = "2.0.1"
@@ -122,9 +116,6 @@
122 116 [[package]]
123 117 name = "subst"
124 118 version = "0.1.0"
125 - dependencies = [
126 - "regex-lite",
127 - ]
128 119
129 120 [[package]]
130 121 name = "subst-fuzz"
M shared/subst/src/lib.rs +71 -12
@@ -123,11 +123,32 @@
123 123 /// resolved (or every filter that failed). The output is the text with all
124 124 /// resolved markers replaced; unresolved markers are left in place when an
125 125 /// error is returned, so callers can grep for them.
126 + ///
127 + /// # Do not substitute the output again
128 + ///
129 + /// One pass, and the output is not rescanned. A template may therefore ask
130 + /// for text that looks like a marker and get it back verbatim:
131 + ///
132 + /// ```text
133 + /// {{ price | money("{{ other }}") }} -> {{ other }}16.00
134 + /// ```
135 + ///
136 + /// That is the template author's literal string, correctly emitted. It is
137 + /// only a hazard if something feeds the result back in, which would
138 + /// evaluate a marker the first pass deliberately treated as data. **A
139 + /// second pass over untrusted output is a template injection**, and the
140 + /// same is true of any value in the table whose text contains `{{`.
141 + ///
142 + /// Stated here because it had never been stated anywhere, and a caller
143 + /// building a two-stage pipeline would reasonably have assumed the
144 + /// opposite.
126 145 pub fn substitute(&self, text: &str) -> Result<String, SubstError> {
127 - // Matches `{{ … }}` non-greedily — the inner body may contain spaces,
128 - // pipes, parens, and quoted strings (e.g. `{{ x | money("$") }}`).
129 - let re = regex_lite::Regex::new(r"\{\{(.*?)\}\}").expect("static regex");
130 -
146 + // Markers are located by `parser::markers`, which shares the parser's
147 + // own notion of a quoted string. It used to be a non-greedy regex,
148 + // which did not, so a `}}` inside a filter's string argument ended the
149 + // marker early and the tail was copied through as text. See
150 + // `parser::markers` for the case and the reasoning.
151 + //
131 152 // Skip matches inside inline code spans and fenced code blocks so that
132 153 // documentation showing literal `{{ … }}` template syntax (e.g. Tauri
133 154 // updater URL patterns) is preserved verbatim.
@@ -139,26 +160,27 @@
139 160 let mut out = String::with_capacity(text.len());
140 161 let mut last = 0;
141 162
142 - for m in re.find_iter(text) {
143 - out.push_str(&text[last..m.start()]);
144 - last = m.end();
163 + for m in parser::markers(text) {
164 + out.push_str(&text[last..m.start]);
165 + last = m.end;
145 166
146 - if in_code(m.start()) {
147 - out.push_str(m.as_str());
167 + let raw = &text[m.start..m.end];
168 + if in_code(m.start) {
169 + out.push_str(raw);
148 170 continue;
149 171 }
150 172
151 - let body = re.captures(m.as_str()).unwrap().get(1).unwrap().as_str();
173 + let body = m.body;
152 174 match self.resolve(body) {
153 175 Ok(Some(v)) => out.push_str(&v.to_string()),
154 176 Ok(None) => {
155 177 // Path not found in the table. Preserve marker; flag for caller.
156 178 unresolved.push(body.trim().to_string());
157 - out.push_str(m.as_str());
179 + out.push_str(raw);
158 180 }
159 181 Err(e) => {
160 182 errors.push(format!("`{body}`: {e}"));
161 - out.push_str(m.as_str());
183 + out.push_str(raw);
162 184 }
163 185 }
164 186 }
@@ -208,6 +230,43 @@
208 230 .with_value("cohort.lock_duration", Value::String("lifetime".into()))
209 231 }
210 232
233 + /// Regression, from the soak tier 2026-08-11
234 + /// (GoingsOn `subst-substitute:4def8fda6141`).
235 + ///
236 + /// The old regex ended the marker at the `}}` inside the string argument,
237 + /// so the parser was handed ` expenses.F_monthly | money("` and reported an
238 + /// unclosed paren, while `") }}` was copied through as literal text.
239 + #[test]
240 + fn a_close_brace_inside_a_string_argument_is_not_the_end_of_the_marker() {
241 + let out = sample()
242 + .substitute(r#"{{ expenses.F_monthly | money("}}") }}"#)
243 + .expect("parses as one marker");
244 + assert_eq!(out, "}}580.00");
245 + }
246 +
247 + /// The other half of the same finding: with the marker now bounded
248 + /// correctly, this is a fixed point.
249 + #[test]
250 + fn a_close_brace_in_a_string_argument_round_trips() {
251 + let s = sample();
252 + let once = s
253 + .substitute(r#"{{ expenses.F_monthly | money("}}") }}"#)
254 + .unwrap();
255 + assert_eq!(s.substitute(&once).unwrap(), once);
256 + }
257 +
258 + /// A template may ask for text that looks like a marker, and gets it back
259 + /// literally. Substitution is one pass and does not rescan its own output,
260 + /// so this is the documented behaviour rather than a defect -- see the note
261 + /// on [`Substituter::substitute`] about not re-substituting output.
262 + #[test]
263 + fn marker_text_inside_a_string_argument_is_emitted_literally() {
264 + let out = sample()
265 + .substitute(r#"{{ expenses.F_monthly | money("{{ x }}") }}"#)
266 + .unwrap();
267 + assert_eq!(out, "{{ x }}580.00");
268 + }
269 +
211 270 #[test]
212 271 fn substitute_replaces_known_keys() {
213 272 let out = sample()
M shared/subst/src/parser.rs +181
@@ -38,6 +38,103 @@
38 38 }
39 39 }
40 40
41 + /// One `{{ … }}` marker located in a document.
42 + #[derive(Debug, PartialEq, Eq)]
43 + pub(crate) struct Marker<'a> {
44 + /// Byte offset of the opening `{{`.
45 + pub start: usize,
46 + /// Byte offset just past the closing `}}`.
47 + pub end: usize,
48 + /// What sits between them, braces excluded.
49 + pub body: &'a str,
50 + }
51 +
52 + /// Find every marker in `text`, respecting quoted strings.
53 + ///
54 + /// This lives beside the grammar above rather than in `lib.rs` because it has
55 + /// to agree with it. It previously did not: markers were found with the regex
56 + /// `\{\{(.*?)\}\}`, which knows nothing about quoting, while [`split_commas`]
57 + /// and [`parse_arg`] do. The two disagreed about where a marker ends, and a
58 + /// `}}` inside a string argument ended it early:
59 + ///
60 + /// ```text
61 + /// {{ price.basic | money("}}") }}
62 + /// ```
63 + ///
64 + /// The regex stopped inside the string literal and handed `parse_expr` the
65 + /// fragment ` price.basic | money("`, which then reported an unclosed paren --
66 + /// a true statement about the fragment and a misleading one about the input.
67 + /// Worse, the tail after the false ending was copied through as literal text,
68 + /// so substituting a creator template could emit live template syntax. Found by
69 + /// the soak tier 2026-08-11, GoingsOn `subst-substitute:4def8fda6141`.
70 + ///
71 + /// Two rules, and both are deliberate:
72 + ///
73 + /// - **Quotes are tracked exactly as [`split_commas`] tracks them**: `"` and
74 + /// `'` both open, the same character closes, and there is no escape
75 + /// sequence. Sharing the rule is the point. Two scanners with their own
76 + /// notion of "quoted" is what the bug was.
77 + /// - **A marker does not span a line.** The regex could not, because `.` does
78 + /// not match `\n`, and that is worth keeping rather than an accident to fix:
79 + /// it bounds how far a stray `{{` can reach. Without it an unclosed marker in
80 + /// one paragraph would swallow prose until a `}}` turned up further down the
81 + /// document.
82 + ///
83 + /// An unterminated marker is not a marker. It is left in the output verbatim,
84 + /// which is what the regex did and what documentation showing template syntax
85 + /// depends on.
86 + pub(crate) fn markers(text: &str) -> Vec<Marker<'_>> {
87 + let bytes = text.as_bytes();
88 + let mut out = Vec::new();
89 + let mut i = 0;
90 +
91 + while i + 1 < bytes.len() {
92 + if !(bytes[i] == b'{' && bytes[i + 1] == b'{') {
93 + i += 1;
94 + continue;
95 + }
96 + let body_start = i + 2;
97 + let mut j = body_start;
98 + let mut in_string: Option<u8> = None;
99 + let mut end = None;
100 +
101 + while j < bytes.len() {
102 + let b = bytes[j];
103 + match in_string {
104 + Some(q) if b == q => in_string = None,
105 + Some(_) => {}
106 + None => match b {
107 + b'"' | b'\'' => in_string = Some(b),
108 + b'\n' => break,
109 + b'}' if j + 1 < bytes.len() && bytes[j + 1] == b'}' => {
110 + end = Some(j);
111 + break;
112 + }
113 + _ => {}
114 + },
115 + }
116 + j += 1;
117 + }
118 +
119 + match end {
120 + Some(close) => {
121 + out.push(Marker {
122 + start: i,
123 + end: close + 2,
124 + // Both ends are ASCII brace boundaries, so this cannot
125 + // split a character.
126 + body: &text[body_start..close],
127 + });
128 + i = close + 2;
129 + }
130 + // No close on this line. Resume scanning after the `{{` rather than
131 + // past it, so `{{{{ x }}` still finds the inner marker.
132 + None => i = body_start,
133 + }
134 + }
135 + out
136 + }
137 +
41 138 /// Parse the body of a `{{ … }}` marker.
42 139 ///
43 140 /// Grammar:
@@ -207,6 +304,90 @@
207 304 mod tests {
208 305 use super::*;
209 306
307 + #[test]
308 + fn marker_ends_after_a_close_brace_inside_a_string_argument() {
309 + // The bug, minimally. The old regex stopped at the `}}` inside the
310 + // string and handed the parser ` price.basic | money("`.
311 + let m = markers(r#"{{ price.basic | money("}}") }}"#);
312 + assert_eq!(m.len(), 1);
313 + assert_eq!(m[0].body, r#" price.basic | money("}}") "#);
314 + assert_eq!(m[0].start, 0);
315 + }
316 +
317 + #[test]
318 + fn single_quotes_hide_a_close_brace_too() {
319 + let m = markers("{{ x | money('}}') }}");
320 + assert_eq!(m.len(), 1);
321 + assert_eq!(m[0].body, " x | money('}}') ");
322 + }
323 +
324 + /// The crash input from the soak tier, verbatim. One marker, not two.
325 + #[test]
326 + fn nested_marker_text_inside_a_string_is_one_marker() {
327 + let text = r#"{{ price.basic | money("{{ price.basic | money("$") }}$") }}"#;
328 + let m = markers(text);
329 + assert_eq!(m.len(), 1);
330 + assert_eq!(m[0].start, 0);
331 + assert_eq!(m[0].end, text.len());
332 + }
333 +
334 + /// The regex could not span a newline, because `.` does not match one. That
335 + /// bound is kept deliberately: without it a stray `{{` swallows prose until
336 + /// a `}}` turns up further down the document.
337 + #[test]
338 + fn a_marker_does_not_span_a_line() {
339 + assert!(markers("{{ foo\nbar }}").is_empty());
340 + // And the scanner recovers on the next line rather than giving up.
341 + let m = markers("{{ unclosed\n{{ ok }}");
342 + assert_eq!(m.len(), 1);
343 + assert_eq!(m[0].body, " ok ");
344 + }
345 +
346 + #[test]
347 + fn an_unterminated_marker_is_not_a_marker() {
348 + assert!(markers("{{ ").is_empty());
349 + assert!(markers(r#"{{ x | money("unclosed }}"#).is_empty());
350 + }
351 +
352 + /// Matching the old regex exactly: it opened at the first `{{` and closed
353 + /// at the first `}}`, so the extra braces land inside the body and the
354 + /// expression fails to parse there rather than here. Locating a marker and
355 + /// judging its contents are separate jobs.
356 + #[test]
357 + fn doubled_braces_open_at_the_first_pair() {
358 + let m = markers("{{{{ x }}");
359 + assert_eq!(m.len(), 1);
360 + assert_eq!(m[0].body, "{{ x ");
361 + }
362 +
363 + /// The resume path: a `{{` that never closes must not hide a later marker
364 + /// on the same line. Scanning resumes just after the failed open rather
365 + /// than past it.
366 + #[test]
367 + fn an_unclosed_open_does_not_swallow_a_later_marker() {
368 + let m = markers(r#"{{ a | money(" and then {{ b }}"#);
369 + assert_eq!(m.len(), 1);
370 + assert_eq!(m[0].body, " b ");
371 + }
372 +
373 + #[test]
374 + fn adjacent_markers_are_separate() {
375 + let m = markers("{{ a }}{{ b }}");
376 + assert_eq!(m.len(), 2);
377 + assert_eq!(m[0].body, " a ");
378 + assert_eq!(m[1].body, " b ");
379 + assert_eq!(m[1].start, 7);
380 + }
381 +
382 + #[test]
383 + fn multibyte_text_yields_slicable_spans() {
384 + let text = "ü {{ x }} £";
385 + let m = markers(text);
386 + assert_eq!(m.len(), 1);
387 + // The spans are what `substitute` indexes with.
388 + assert_eq!(&text[m[0].start..m[0].end], "{{ x }}");
389 + }
390 +
210 391 #[test]
211 392 fn parses_bare_path() {
212 393 let e = parse_expr("foo.bar").unwrap();
M shared/subst/fuzz/fuzz_targets/substitute.rs +26 -4
@@ -99,9 +99,31 @@
99 99 assert_eq!(out, text, "text with no marker was modified");
100 100 }
101 101
102 - // Oracle 3. A second pass over the output must be a no-op: `substitute` is
103 - // not recursive by design, so anything it produced must be inert.
104 - if let Ok(twice) = s.substitute(&out) {
105 - assert_eq!(twice, out, "substitution is not a fixed point");
102 + // Oracle 3, narrowed on 2026-08-11 by what it found.
103 + //
104 + // It used to assert an unconditional fixed point, and it caught a real bug
105 + // that way: the marker regex was not quote-aware, so a `}}` inside a string
106 + // argument ended the marker early and the tail was copied through as text
107 + // (GoingsOn `subst-substitute:4def8fda6141`, fixed in `parser::markers`).
108 + //
109 + // But the unconditional form is not true even of correct code. A template
110 + // may legitimately ask for text that looks like a marker:
111 + //
112 + // {{ price | money("{{ other }}") }} -> {{ other }}16.00
113 + //
114 + // That is the author's literal string, correctly emitted, and substitution
115 + // is one pass by design. Asserting otherwise would demand the engine escape
116 + // its own output, which fights the crate's whole purpose -- code spans
117 + // exist so documentation can show template syntax.
118 + //
119 + // So the property is asserted where it does hold: with no quote in the
120 + // input there is no string literal to carry marker text, and anything the
121 + // engine emits must be inert. That still covers the class the original bug
122 + // was in, because a scanner that mis-locates a marker mangles quoteless
123 + // input too.
124 + if !text.contains('"') && !text.contains('\'') {
125 + if let Ok(twice) = s.substitute(&out) {
126 + assert_eq!(twice, out, "substitution is not a fixed point");
127 + }
106 128 }
107 129 });