max / quasi

19.7 KB · 537 lines History Blame Raw

1	#!/usr/bin/env python3
2	"""Count the shape functions the `declare!` form has to cover.
3
4	The population is the denominator of phase 1 of the declaration transition
5	(GoingsOn `65d99281`, wiki `quasi-declare-form`). Five counts of it existed
6	before this script -- 289, 375, 402, 425, 487 -- and none reproduced, because
7	none had written its predicate down. This is the predicate, executable.
8
9	THE PREDICATE. A Rust `fn` item with a body, in one of the three shape
10	directories, outside `tests.rs` and `parity.rs`, outside any `tests/` path
11	segment, outside every inline `#[cfg(test)]` block, whose return type -- after
12	stripping `Vec`, `Option`, `Result`, `Box`, `Arc`, `Rc` and `Cow` -- names a
13	`quasi_router` description type, with `use ... as` aliases resolved and
14	file-local types of the same name excluded.
15
16	Comments, string literals, raw strings and char literals are blanked in place
17	before anything is matched, so a type named in a doc comment is not a
18	construction. That is the defect that produced two of the five old numbers.
19
20	python3 scripts/population.py # the count, per tree
21	python3 scripts/population.py --list # one `path:line name -> type` per line
22	python3 scripts/population.py --json # the same, machine-readable
23	python3 scripts/population.py --selftest # the predicate against known answers
24
25	Run `--selftest` before quoting a number from this. Every case in it is a defect
26	that actually happened: a `#[cfg(test)]` stripper that brace-matched from the
27	next `{` blanked four real functions in goingson, a doc-comment mention of a
28	vocabulary type was counted as a construction, and a tuple return was missed.
29
30	Paths are read relative to `~/Code` by default; pass `--root` for a tree
31	somewhere else.
32	"""
33
34	from __future__ import annotations
35
36	import argparse
37	import json
38	import re
39	import sys
40	from pathlib import Path
41
42	# The three shape directories, relative to the root. A fourth app joining the
43	# transition is one line here and a re-run.
44	SHAPE_DIRS = [
45	"MNW/server/src/quasi",
46	"Apps/goingson/src-tauri/src/quasi",
47	"Apps/audiofiles/crates/audiofiles-browser/src/quasi",
48	]
49
50	# The vocabulary. Every public description type `quasi_router` exports that a
51	# shape function can return, plus the two enums that are returned as leaves.
52	# `Node` alone is a fifth of the population.
53	VOCABULARY = {
54	"Node", "Screen", "Slot", "Run", "Field", "Row", "Cell", "Cells",
55	"Act", "Action", "Choice", "Column", "Tag", "Part", "Figure",
56	"Placed", "Consult", "Meter", "Image", "Lexeme", "Document", "Feed",
57	}
58
59	# Returned by a handful of leaf suppliers that section 6 of `quasi-declare-form`
60	# defers, so they sit outside the ratified population and inside `--wide`.
61	# Adding all five moves the count by 10 on the trees as they stand.
62	DEFERRED_LEAVES = {"Rest", "Candidate", "Accepted", "RegionKind", "Chrome"}
63
64	# Wrappers stripped before the name is looked up. A `-> Result<Vec<Row>, E>` is
65	# a shape function returning rows; the wrapper says how it fails, not what it is.
66	WRAPPERS = {"Vec", "Option", "Result", "Box", "Arc", "Rc", "Cow"}
67
68
69	def blank_noncode(src: str) -> str:
70	"""Replace every comment, string, raw string and char literal with spaces.
71
72	Newlines are kept so line numbers still hold. Nothing is deleted, so every
73	offset in the returned text is the offset in the original.
74	"""
75	out = list(src)
76	i, n = 0, len(src)
77	while i < n:
78	c = src[i]
79	# Raw string, with any hash count: r"..", r#".."#, br##".."##
80	m = re.match(r'(?:b)?r(#*)"', src[i:])
81	if m and (i == 0 or not (src[i - 1].isalnum() or src[i - 1] == "_")):
82	hashes = m.group(1)
83	close = '"' + hashes
84	end = src.find(close, i + m.end())
85	end = n if end < 0 else end + len(close)
86	for j in range(i, end):
87	if out[j] != "\n":
88	out[j] = " "
89	i = end
90	continue
91	if c == "/" and i + 1 < n and src[i + 1] == "/":
92	end = src.find("\n", i)
93	end = n if end < 0 else end
94	for j in range(i, end):
95	out[j] = " "
96	i = end
97	continue
98	if c == "/" and i + 1 < n and src[i + 1] == "*":
99	depth, j = 1, i + 2
100	while j < n and depth:
101	if src.startswith("/*", j):
102	depth += 1
103	j += 2
104	elif src.startswith("*/", j):
105	depth -= 1
106	j += 2
107	else:
108	j += 1
109	for k in range(i, j):
110	if out[k] != "\n":
111	out[k] = " "
112	i = j
113	continue
114	if c == '"':
115	j = i + 1
116	while j < n:
117	if src[j] == "\\":
118	j += 2
119	continue
120	if src[j] == '"':
121	j += 1
122	break
123	j += 1
124	for k in range(i, min(j, n)):
125	if out[k] != "\n":
126	out[k] = " "
127	i = j
128	continue
129	if c == "'":
130	# A char literal, not a lifetime: `'a` is a lifetime, `'a'` is a char.
131	m = re.match(r"'(?:\\.\|[^\\'])'", src[i:])
132	if m:
133	for k in range(i, i + m.end()):
134	out[k] = " "
135	i += m.end()
136	continue
137	i += 1
138	return "".join(out)
139
140
141	def strip_cfg_test(src: str) -> str:
142	"""Blank every `#[cfg(test)]` item that is followed by an inline block.
143
144	Only a block: `#[cfg(test)] mod tests;` names a file that is excluded by
145	path anyway, and blanking to the next `}` there would eat the rest of the
146	file.
147	"""
148	out = list(src)
149	for m in re.finditer(r"#\[cfg$test$\]", src):
150	j = m.end()
151	while j < len(src) and src[j] not in "{;":
152	j += 1
153	if j >= len(src) or src[j] == ";":
154	continue
155	depth, k = 0, j
156	while k < len(src):
157	if src[k] == "{":
158	depth += 1
159	elif src[k] == "}":
160	depth -= 1
161	if depth == 0:
162	k += 1
163	break
164	k += 1
165	for p in range(m.start(), k):
166	if out[p] != "\n":
167	out[p] = " "
168	return "".join(out)
169
170
171	def local_types(src: str) -> set[str]:
172	"""Types the file declares itself, which shadow the vocabulary name."""
173	return set(
174	re.findall(r"^\s(?:pub(?:$[^)]$)?\s+)?(?:struct\|enum\|type\|union)\s+(\w+)",
175	src, re.M)
176	)
177
178
179	def imported(src: str) -> tuple[set[str], dict[str, str]]:
180	"""Names brought in from `quasi_router`, and every `as` alias in the file.
181
182	Returns (names in scope from quasi_router, alias -> real name). Braced and
183	single-item `use` forms both, at any nesting depth: the only thing that
184	matters is whether the path starts at `quasi_router`.
185	"""
186	names: set[str] = set()
187	aliases: dict[str, str] = {}
188	for m in re.finditer(r"use\s+([^;]+);", src):
189	body = m.group(1)
190	if "quasi_router" not in body and "quasi_webview" not in body:
191	continue
192	for leaf in re.findall(r"(\w+)(?:\s+as\s+(\w+))?", body):
193	name, alias = leaf
194	if name in VOCABULARY:
195	names.add(alias or name)
196	if alias:
197	aliases[alias] = name
198	return names, aliases
199
200
201	def split_top(s: str) -> list[str]:
202	"""Split on commas at bracket depth zero."""
203	out, depth, start = [], 0, 0
204	for i, c in enumerate(s):
205	if c in "<([":
206	depth += 1
207	elif c in ">)]":
208	depth -= 1
209	elif c == "," and depth == 0:
210	out.append(s[start:i])
211	start = i + 1
212	out.append(s[start:])
213	return [p for p in (x.strip() for x in out) if p]
214
215
216	def head_type(ret: str) -> str \| None:
217	"""The vocabulary name a return type resolves to, or None.
218
219	Strips the wrappers outside-in, takes the last `::` segment, and stops at
220	the first name that is not a wrapper. `Result<Vec<Node>, RouteError>` is
221	`Node`; `Result<(), E>` is nothing.
222	"""
223	ret = ret.strip()
224	for _ in range(6):
225	ret = ret.strip().lstrip("&").strip()
226	ret = re.sub(r"^'\w+\s+", "", ret)
227	m = re.match(r"([\w:]+)\s<(.)>$", ret, re.S)
228	if not m:
229	break
230	head = m.group(1).split("::")[-1]
231	if head not in WRAPPERS:
232	return head
233	inner = m.group(2)
234	# `Result<T, E>`: the first argument at depth zero is the payload.
235	depth, cut = 0, len(inner)
236	for i, c in enumerate(inner):
237	if c in "<([":
238	depth += 1
239	elif c in ">)]":
240	depth -= 1
241	elif c == "," and depth == 0:
242	cut = i
243	break
244	ret = inner[:cut]
245	ret = ret.strip().lstrip("&").strip()
246	ret = re.sub(r"^'\w+\s+", "", ret)
247	# A tuple return. One site in the trees as they stand (goingson
248	# `data.rs:380 csv_rows`), and `shaped` names one type, so the form
249	# defers it -- but it is a shape function and it counts.
250	if ret.startswith("(") and ret.endswith(")"):
251	for part in split_top(ret[1:-1]):
252	head = head_type(part)
253	if head in VOCABULARY:
254	return head
255	return None
256	name = ret.split("::")[-1].strip()
257	return name or None
258
259
260	def match_angle(src: str, start: int) -> int:
261	"""Index just past the `>` closing the `<` at `start`."""
262	depth, i = 0, start
263	while i < len(src):
264	if src[i] == "<":
265	depth += 1
266	elif src[i] == ">":
267	depth -= 1
268	if depth == 0:
269	return i + 1
270	elif src[i] == ";":
271	return -1
272	i += 1
273	return -1
274
275
276	def match_paren(src: str, start: int) -> int:
277	depth, i = 0, start
278	while i < len(src):
279	if src[i] == "(":
280	depth += 1
281	elif src[i] == ")":
282	depth -= 1
283	if depth == 0:
284	return i + 1
285	i += 1
286	return -1
287
288
289	def match_brace(src: str, start: int) -> int:
290	"""Index just past the `}` closing the `{` at `start`, or -1.
291
292	`src` must already be through `blank_noncode`, so a brace inside a string
293	or a comment cannot unbalance the count.
294	"""
295	depth = 0
296	i = start
297	while i < len(src):
298	if src[i] == "{":
299	depth += 1
300	elif src[i] == "}":
301	depth -= 1
302	if depth == 0:
303	return i + 1
304	i += 1
305	return -1
306
307
308	def shapes_in(path: Path) -> list[dict]:
309	raw = path.read_text(encoding="utf-8", errors="replace")
310	src = strip_cfg_test(blank_noncode(raw))
311	scope, aliases = imported(raw)
312	shadowed = local_types(src)
313	found = []
314	for m in re.finditer(r"\bfn\s+(\w+)\s*(?=[<(])", src):
315	name = m.group(1)
316	i = m.end()
317	if src[i] == "<":
318	i = match_angle(src, i)
319	if i < 0:
320	continue
321	while i < len(src) and src[i].isspace():
322	i += 1
323	if i >= len(src) or src[i] != "(":
324	continue
325	i = match_paren(src, i)
326	if i < 0:
327	continue
328	# Return type runs to the body's `{`, the `where` clause, or a `;`.
329	j = i
330	depth = 0
331	while j < len(src):
332	c = src[j]
333	if c in "<([":
334	depth += 1
335	elif c in ">)]":
336	depth -= 1
337	elif c == "{" and depth <= 0:
338	break
339	elif c == ";" and depth <= 0:
340	j = -1
341	break
342	j += 1
343	if j < 0 or j >= len(src):
344	continue # no body: a trait signature or an extern declaration
345	sig = src[i:j]
346	if re.search(r"\bwhere\b", sig):
347	sig = sig[: sig.index("where")]
348	sig = sig.strip()
349	if not sig.startswith("->"):
350	continue # returns unit
351	ret = sig[2:].strip()
352	head = head_type(ret)
353	if head is None:
354	continue
355	real = aliases.get(head, head)
356	if real not in VOCABULARY:
357	continue
358	if head in shadowed:
359	continue # a file-local type wearing a vocabulary name
360	if head not in scope and real not in scope:
361	# Reached by path (`quasi_router::Node`) rather than by import.
362	if "quasi_router" not in ret and "quasi_webview" not in ret:
363	continue
364	end = match_brace(src, j)
365	start_line = src.count("\n", 0, m.start()) + 1
366	# Body length in lines, signature included. Used to stratify a probe
367	# sample (`scripts/probe-sample.py`); it does not enter the predicate,
368	# so the count is unaffected.
369	lines = (src.count("\n", m.start(), end) + 1) if end > 0 else 1
370	found.append(
371	{"file": str(path), "line": start_line,
372	"fn": name, "returns": " ".join(ret.split()), "lines": lines}
373	)
374	return found
375
376
377	# The cases the implementation has been wrong on, as executable assertions.
378	# Every one is a defect that actually happened: a naive `#[cfg(test)]` stripper
379	# ate four real functions, a doc-comment mention was counted as a construction,
380	# and a tuple return was missed. `--selftest` is how those stay fixed.
381	SELFTEST = [
382	# (source, expected [(fn, head)])
383	# A plain shape.
384	("use quasi_router::Node;\nfn body(cx: &Cx) -> Node { Node::page(\"x\") }",
385	[("body", "Node")]),
386	# Wrappers are stripped outside-in; `Result`'s payload is its first argument.
387	("use quasi_router::{Node, Row};\n"
388	"fn a(c: &C) -> Result<Vec<Node>, E> { todo!() }\n"
389	"fn b(c: &C) -> Option<Row> { todo!() }",
390	[("a", "Node"), ("b", "Row")]),
391	# `Result<(), E>` names no vocabulary type and is not a shape.
392	("use quasi_router::Node;\nfn a(c: &C) -> Result<(), E> { todo!() }", []),
393	# A doc comment naming a vocabulary type is not a construction. This is the
394	# defect behind two of the five irreproducible counts.
395	("use quasi_router::Node;\n/// Returns a Node, eventually.\nfn a(c: &C) -> u32 { 0 }",
396	[]),
397	# Nor is a string literal.
398	("use quasi_router::Node;\nfn a(c: &C) -> u32 { let s = \"fn x() -> Node\"; 0 }",
399	[]),
400	# An inline `#[cfg(test)]` block is stripped ...
401	("use quasi_router::Node;\n"
402	"fn real(c: &C) -> Node { todo!() }\n"
403	"#[cfg(test)]\nmod tests {\n fn fake(c: &C) -> Node { todo!() }\n}\n",
404	[("real", "Node")]),
405	# ... but `#[cfg(test)] mod tests;` has no block, and brace-matching from
406	# the next `{` would blank an arbitrary later region. Four real goingson
407	# functions were lost to exactly this.
408	("use quasi_router::Node;\n"
409	"#[cfg(test)]\nmod tests;\n"
410	"fn survives(c: &C) -> Node { todo!() }\n",
411	[("survives", "Node")]),
412	# A signature with no body is a trait method or an extern, not a shape.
413	("use quasi_router::Node;\ntrait T { fn a(&self) -> Node; }", []),
414	# A returned unit is not a shape.
415	("use quasi_router::Node;\nfn a(c: &C) {}", []),
416	# A generic parameter list before the arguments must not break the parse.
417	("use quasi_router::Node;\nfn a<T: Into<String>>(t: T) -> Node { todo!() }",
418	[("a", "Node")]),
419	# A `where` clause sits between the return type and the body.
420	("use quasi_router::Node;\nfn a<T>(t: T) -> Node where T: Into<String> { todo!() }",
421	[("a", "Node")]),
422	# A tuple return: one site in the trees, and `shaped` names one type, but it
423	# is a shape function and it counts.
424	("use quasi_router::Cells;\n"
425	"fn a(c: &C) -> (Vec<&'static str>, Vec<Cells>) { todo!() }",
426	[("a", "Cells")]),
427	# A file-local type wearing a vocabulary name shadows it.
428	("use quasi_router::Node;\nstruct Row;\nfn a(c: &C) -> Row { Row }", []),
429	# Reached by path rather than by import.
430	("fn a(c: &C) -> quasi_router::Node { todo!() }", [("a", "Node")]),
431	# A vocabulary name that was never imported and is not path-qualified is
432	# somebody else's type.
433	("fn a(c: &C) -> Node { todo!() }", []),
434	# An alias is resolved for MEMBERSHIP and reported AS WRITTEN. Both halves
435	# are deliberate: the shape counts, and `--list` says what the source says,
436	# which is why the record's own return-type histogram lists "Described
437	# (Screen alias) 18" as its own row rather than folding it into Screen.
438	("use quasi_router::Screen as Described;\nfn a(c: &C) -> Described { todo!() }",
439	[("a", "Described")]),
440	# `Response` is quasi's transport, not its description vocabulary. 566
441	# further functions in these directories return one; admitting them roughly
442	# doubles the count.
443	("use quasi_router::Response;\nfn a(c: &C) -> Response { todo!() }", []),
444	]
445
446
447	def selftest() -> int:
448	"""Run the predicate over synthetic sources with known answers."""
449	import tempfile
450
451	failures = 0
452	for n, (src, expected) in enumerate(SELFTEST, 1):
453	with tempfile.TemporaryDirectory() as d:
454	f = Path(d) / "case.rs"
455	f.write_text(src, encoding="utf-8")
456	got = [(h["fn"], head_type(h["returns"]) or "?") for h in shapes_in(f)]
457	# head_type is re-applied for the label only; membership already
458	# filtered. Compare as sets, since order is not the claim.
459	want = {(a, b) for a, b in expected}
460	have = {(a, b) for a, b in got}
461	if have != want:
462	failures += 1
463	print(f"case {n} FAILED")
464	print(f" source: {src!r}")
465	print(f" expected: {sorted(want)}")
466	print(f" got: {sorted(have)}")
467	total = len(SELFTEST)
468	if failures:
469	print(f"\n{failures} of {total} cases failed.")
470	return 1
471	print(f"{total} of {total} cases pass.")
472	return 0
473
474
475	def main() -> int:
476	ap = argparse.ArgumentParser(description=__doc__,
477	formatter_class=argparse.RawDescriptionHelpFormatter)
478	ap.add_argument("--root", default=str(Path.home() / "Code"))
479	ap.add_argument("--list", action="store_true")
480	ap.add_argument("--json", action="store_true")
481	ap.add_argument("--selftest", action="store_true",
482	help="run the predicate over synthetic sources with known answers")
483	ap.add_argument("--wide", action="store_true",
484	help="also count the deferred leaf suppliers (Rest, Candidate, "
485	"Accepted, RegionKind, Chrome)")
486	args = ap.parse_args()
487
488	if args.selftest:
489	return selftest()
490
491	if args.wide:
492	VOCABULARY.update(DEFERRED_LEAVES)
493
494	root = Path(args.root)
495	per_tree: dict[str, list[dict]] = {}
496	missing = []
497	for rel in SHAPE_DIRS:
498	d = root / rel
499	if not d.is_dir():
500	missing.append(rel)
501	continue
502	hits: list[dict] = []
503	for f in sorted(d.rglob("*.rs")):
504	parts = f.relative_to(root).parts
505	if f.name in ("tests.rs", "parity.rs") or "tests" in parts:
506	continue
507	hits.extend(shapes_in(f))
508	for h in hits:
509	h["file"] = str(Path(h["file"]).relative_to(root))
510	per_tree[rel] = hits
511
512	total = sum(len(v) for v in per_tree.values())
513
514	if args.json:
515	json.dump({"total": total,
516	"per_tree": {k: len(v) for k, v in per_tree.items()},
517	"shapes": [h for v in per_tree.values() for h in v]},
518	sys.stdout, indent=1)
519	print()
520	return 0
521
522	if args.list:
523	for rel, hits in per_tree.items():
524	for h in hits:
525	print(f"{h['file']}:{h['line']} {h['fn']} -> {h['returns']}")
526
527	for rel, hits in per_tree.items():
528	print(f"{len(hits):5} {rel}")
529	print(f"{total:5} TOTAL")
530	for rel in missing:
531	print(f" ? {rel} (not present under {root})", file=sys.stderr)
532	return 0
533
534
535	if __name__ == "__main__":
536	raise SystemExit(main())
537