| 1 |
export const meta = { |
| 2 |
name: 'quasi-probe-round', |
| 3 |
description: 'Probe round against the amended declare! form: re-author sampled shapes, adjudicate every new failure class against the four-part bar', |
| 4 |
whenToUse: 'Phase 1 of the quasi declaration transition (GoingsOn 65d99281). Run when the form has been amended and the closure claim needs evidence. Read-only: it edits nothing and commits nothing.', |
| 5 |
phases: [ |
| 6 |
{ title: 'Sample', detail: 'draw the stratified batch list from probe-sample.py' }, |
| 7 |
{ title: 'Probe', detail: 'one agent per batch, each re-authoring its shapes in the amended form' }, |
| 8 |
{ title: 'Residue', detail: 'R4(b) reproduction and the grammar over-admission check' }, |
| 9 |
{ title: 'Adjudicate', detail: 'four-part bar applied to every class not already in the record' }, |
| 10 |
{ title: 'Synthesize', detail: 'verdict, coverage, and the hit-rate comparison against round 1' }, |
| 11 |
], |
| 12 |
} |
| 13 |
|
| 14 |
|
| 15 |
|
| 16 |
|
| 17 |
|
| 18 |
|
| 19 |
|
| 20 |
const ROUND_1 = { probes: 15, functions: 26, clean: 4, coverage: 5.1, classes_per_probe: 1 / 1.4 } |
| 21 |
|
| 22 |
const seed = (args && args.seed) || 2 |
| 23 |
const probes = (args && args.probes) || 15 |
| 24 |
const perProbe = (args && args.perProbe) || 3 |
| 25 |
|
| 26 |
const READING = ` |
| 27 |
Read these before writing a single production. They are the contract: |
| 28 |
|
| 29 |
- \`~/Wiki/quasi-declare-form.md\` -- the grammar (EBNF), the twelve amendments |
| 30 |
each with its measurement, the eleven static rules, three worked examples, |
| 31 |
the deferred table in section 6, and the four-part bar in section 8. This is |
| 32 |
the form you are probing. It is long; read sections 4, 6 and 8 in full. |
| 33 |
- \`~/Wiki/construction-holds-the-invariant.md\` -- the companion. |
| 34 |
- \`~/Code/quasi/crates/quasi-router/src/screen.rs\` -- the constructors a |
| 35 |
declaration has to emit. A production that cannot name a real constructor is |
| 36 |
not a production. |
| 37 |
|
| 38 |
You are READ-ONLY. Do not edit, create or commit any file, in any repo. A probe |
| 39 |
is a re-authoring on paper, checked production by production. Its output is |
| 40 |
evidence, not a patch. |
| 41 |
` |
| 42 |
|
| 43 |
const PROBE_SCHEMA = { |
| 44 |
type: 'object', |
| 45 |
required: ['targets', 'verdict', 'classes'], |
| 46 |
properties: { |
| 47 |
targets: { |
| 48 |
type: 'array', |
| 49 |
items: { |
| 50 |
type: 'object', |
| 51 |
required: ['file', 'fn', 'reauthored'], |
| 52 |
properties: { |
| 53 |
file: { type: 'string' }, |
| 54 |
fn: { type: 'string' }, |
| 55 |
reauthored: { |
| 56 |
type: 'boolean', |
| 57 |
description: 'true only if EVERY construct in the function was expressible in the amended form', |
| 58 |
}, |
| 59 |
declaration: { |
| 60 |
type: 'string', |
| 61 |
description: 'the declaration you wrote, verbatim; empty if it could not be written', |
| 62 |
}, |
| 63 |
}, |
| 64 |
}, |
| 65 |
}, |
| 66 |
verdict: { |
| 67 |
type: 'string', |
| 68 |
enum: ['clean', 'failed'], |
| 69 |
description: 'clean only if every target re-authored with no residue', |
| 70 |
}, |
| 71 |
classes: { |
| 72 |
type: 'array', |
| 73 |
description: 'one entry per distinct construct the form could not say. Empty when clean.', |
| 74 |
items: { |
| 75 |
type: 'object', |
| 76 |
required: ['name', 'construct', 'site', 'in_record', 'remedy_available'], |
| 77 |
properties: { |
| 78 |
name: { type: 'string', description: 'short name for the class' }, |
| 79 |
construct: { type: 'string', description: 'the Rust that could not be said' }, |
| 80 |
site: { type: 'string', description: 'file:line of a real occurrence' }, |
| 81 |
in_record: { |
| 82 |
type: 'boolean', |
| 83 |
description: 'true if this class is already one of the twelve amendments or on section 6 deferred table', |
| 84 |
}, |
| 85 |
record_ref: { type: 'string', description: 'which amendment or deferred row, when in_record' }, |
| 86 |
remedy_available: { |
| 87 |
type: 'boolean', |
| 88 |
description: 'true if outcome A applies -- the deferred table names a remedy that covers it', |
| 89 |
}, |
| 90 |
effect: { |
| 91 |
type: 'string', |
| 92 |
description: 'what a user sees, or the compile error, or the status code that changes', |
| 93 |
}, |
| 94 |
}, |
| 95 |
}, |
| 96 |
}, |
| 97 |
}, |
| 98 |
} |
| 99 |
|
| 100 |
const BAR_SCHEMA = { |
| 101 |
type: 'object', |
| 102 |
required: ['classes'], |
| 103 |
properties: { |
| 104 |
classes: { |
| 105 |
type: 'array', |
| 106 |
items: { |
| 107 |
type: 'object', |
| 108 |
required: ['name', 'site_count', 'count_command', 'outcome', 'reasoning'], |
| 109 |
properties: { |
| 110 |
name: { type: 'string' }, |
| 111 |
site_count: { type: 'integer', description: 'measured across all three shape directories' }, |
| 112 |
count_command: { |
| 113 |
type: 'string', |
| 114 |
description: 'the command that produced site_count, runnable as written, with tests.rs / parity.rs / inline #[cfg(test)] excluded and comments and string literals blanked', |
| 115 |
}, |
| 116 |
alternative: { type: 'string', description: 'the named alternative, priced: signature change + call sites, or N new suppliers' }, |
| 117 |
failing_screen: { type: 'string', description: 'file:line of a screen that currently fails, not a hypothetical' }, |
| 118 |
cannot_be_used_for: { type: 'string', description: 'what the proposed production must not admit' }, |
| 119 |
outcome: { |
| 120 |
type: 'string', |
| 121 |
enum: ['A-remedy', 'B-production', 'C-refuse'], |
| 122 |
description: 'A: the deferred table already covers it. B: it clears all four parts of the bar and the threshold. C: refuse it and rewrite the Rust.', |
| 123 |
}, |
| 124 |
reasoning: { type: 'string' }, |
| 125 |
}, |
| 126 |
}, |
| 127 |
}, |
| 128 |
}, |
| 129 |
} |
| 130 |
|
| 131 |
|
| 132 |
phase('Sample') |
| 133 |
const sample = await agent( |
| 134 |
`Run this, from \`~/Code/quasi\`, and return exactly what it prints: |
| 135 |
|
| 136 |
python3 scripts/probe-sample.py --seed ${seed} --probes ${probes} --per-probe ${perProbe} --json |
| 137 |
|
| 138 |
Run \`python3 scripts/population.py --selftest\` first and report its result in |
| 139 |
\`selftest\`; the draw is worthless if the predicate is broken. Do not edit |
| 140 |
anything. Return the parsed JSON, not a description of it.`, |
| 141 |
{ |
| 142 |
label: `sample seed ${seed}`, |
| 143 |
phase: 'Sample', |
| 144 |
effort: 'low', |
| 145 |
schema: { |
| 146 |
type: 'object', |
| 147 |
required: ['seed', 'population', 'sampled', 'coverage_pct', 'selftest', 'batches'], |
| 148 |
properties: { |
| 149 |
seed: { type: 'integer' }, |
| 150 |
population: { type: 'integer' }, |
| 151 |
sampled: { type: 'integer' }, |
| 152 |
coverage_pct: { type: 'number' }, |
| 153 |
selftest: { type: 'string' }, |
| 154 |
batches: { |
| 155 |
type: 'array', |
| 156 |
items: { |
| 157 |
type: 'object', |
| 158 |
required: ['probe', 'targets'], |
| 159 |
properties: { |
| 160 |
probe: { type: 'integer' }, |
| 161 |
targets: { |
| 162 |
type: 'array', |
| 163 |
items: { |
| 164 |
type: 'object', |
| 165 |
properties: { |
| 166 |
file: { type: 'string' }, |
| 167 |
line: { type: 'integer' }, |
| 168 |
fn: { type: 'string' }, |
| 169 |
returns: { type: 'string' }, |
| 170 |
lines: { type: 'integer' }, |
| 171 |
}, |
| 172 |
}, |
| 173 |
}, |
| 174 |
}, |
| 175 |
}, |
| 176 |
}, |
| 177 |
}, |
| 178 |
}, |
| 179 |
} |
| 180 |
) |
| 181 |
|
| 182 |
if (!sample || !sample.batches || !sample.batches.length) { |
| 183 |
log('Sampling returned nothing. Nothing to probe; check probe-sample.py by hand.') |
| 184 |
return { error: 'no sample', sample } |
| 185 |
} |
| 186 |
|
| 187 |
log(`seed ${sample.seed}: ${sample.sampled} of ${sample.population} shapes (${sample.coverage_pct}%), ${sample.batches.length} probes. Round 1 covered ${ROUND_1.coverage}%.`) |
| 188 |
if (sample.coverage_pct <= ROUND_1.coverage) { |
| 189 |
log(`WARNING: this round covers no more than round 1 did. A clean result at this coverage settles nothing.`) |
| 190 |
} |
| 191 |
|
| 192 |
|
| 193 |
|
| 194 |
|
| 195 |
|
| 196 |
const residuePromise = parallel([ |
| 197 |
() => agent( |
| 198 |
`Reproduce or refute R4(b), the owned-payload double move. |
| 199 |
${READING} |
| 200 |
|
| 201 |
R4(b) is the one residue in the record with NO site count. It was reported as a |
| 202 |
defect and never reproduced, and its failure mode is a compile error in generated |
| 203 |
code with no source line to point at -- the worst thing to discover 400 functions |
| 204 |
into the mass phase. Section 8 names "R4(b) reproduced" as one of the four things |
| 205 |
that reopen the closure claim. |
| 206 |
|
| 207 |
Construct the minimal case from the rule as written in section 4, decide whether |
| 208 |
the double move actually occurs, and if it does, measure its incidence across the |
| 209 |
three shape directories with a command you write down and run. If it does not |
| 210 |
occur, say what in the rule prevents it and what the original report probably saw. |
| 211 |
Read-only.`, |
| 212 |
{ label: 'R4(b)', phase: 'Residue', effort: 'high', schema: { |
| 213 |
type: 'object', |
| 214 |
required: ['reproduced', 'evidence'], |
| 215 |
properties: { |
| 216 |
reproduced: { type: 'boolean' }, |
| 217 |
incidence: { type: 'integer', description: 'measured site count when reproduced, -1 when not' }, |
| 218 |
count_command: { type: 'string' }, |
| 219 |
evidence: { type: 'string' }, |
| 220 |
}, |
| 221 |
} } |
| 222 |
), |
| 223 |
() => agent( |
| 224 |
`Test what the grammar OVER-admits. |
| 225 |
${READING} |
| 226 |
|
| 227 |
Section 7 lists this as untested against any implementation: \`node\`'s body accepts |
| 228 |
any emission so \`act "Delete" { region .. }\` parses; prepositions carry no meaning |
| 229 |
so \`field from H\`, \`field to H\` and \`field by H\` are one declaration; \`arg*\` has |
| 230 |
no arity rule so \`image cover "{title}"\` cannot tell \`src\` from \`alt\`; and 24 |
| 231 |
grammar terminals are live function names in these directories (region, page, |
| 232 |
section, text, act, image, empty, form, list, table, row, column, stats, screen, |
| 233 |
across, include, read, require, unless, of, from, get, delete, leaving). |
| 234 |
|
| 235 |
Every case traced was said to be decidable on one token of lookahead. Check that |
| 236 |
claim properly: work the grammar by hand, find every place two productions share a |
| 237 |
prefix, and say which are decidable on one token, which need more, and which are |
| 238 |
genuinely ambiguous. For the 24 terminals, measure how many are actually called as |
| 239 |
functions inside a declaration body rather than merely defined, since that is what |
| 240 |
decides whether the collision bites. |
| 241 |
|
| 242 |
An ambiguity here is a finding about the form, not about the emitter. Read-only.`, |
| 243 |
{ label: 'over-admission', phase: 'Residue', effort: 'high', schema: { |
| 244 |
type: 'object', |
| 245 |
required: ['ambiguities', 'verdict'], |
| 246 |
properties: { |
| 247 |
verdict: { type: 'string', enum: ['one-token-lookahead-holds', 'needs-more-lookahead', 'genuinely-ambiguous'] }, |
| 248 |
ambiguities: { |
| 249 |
type: 'array', |
| 250 |
items: { |
| 251 |
type: 'object', |
| 252 |
required: ['productions', 'lookahead', 'note'], |
| 253 |
properties: { |
| 254 |
productions: { type: 'string' }, |
| 255 |
lookahead: { type: 'string' }, |
| 256 |
note: { type: 'string' }, |
| 257 |
}, |
| 258 |
}, |
| 259 |
}, |
| 260 |
terminal_collisions: { type: 'integer', description: 'of the 24, how many are called inside a declaration body' }, |
| 261 |
}, |
| 262 |
} } |
| 263 |
), |
| 264 |
]) |
| 265 |
|
| 266 |
|
| 267 |
|
| 268 |
|
| 269 |
|
| 270 |
|
| 271 |
const probed = await pipeline( |
| 272 |
sample.batches, |
| 273 |
|
| 274 |
(b) => agent( |
| 275 |
`You are probe ${b.probe} of a round-2 probe against the amended \`declare!\` form. |
| 276 |
${READING} |
| 277 |
|
| 278 |
YOUR TARGETS -- re-author every one of them: |
| 279 |
|
| 280 |
${b.targets.map((t) => ` ${t.file}:${t.line} fn ${t.fn} -> ${t.returns} (${t.lines} lines)`).join('\n')} |
| 281 |
|
| 282 |
HOW TO PROBE. Read the real function first. Then write the whole thing as a |
| 283 |
declaration in the amended form, production by production, checking each against |
| 284 |
the grammar in section 4 and against the constructor it has to emit in |
| 285 |
\`screen.rs\`. Do not skim and pronounce; a probe that did not write the |
| 286 |
declaration out is not a probe. Put what you wrote in \`declaration\`. |
| 287 |
|
| 288 |
WHAT COUNTS AS A FAILURE. Any construct in the function that the amended form |
| 289 |
cannot say. Before you call it a failure, check two things: |
| 290 |
|
| 291 |
1. Is it already one of the twelve amendments? Those are specified but unprobed, |
| 292 |
so exercising one is the POINT of this round. Set \`in_record: true\` and |
| 293 |
\`record_ref\`, and say in \`effect\` whether the amendment as specified actually |
| 294 |
covers the site or falls short of it. An amendment that does not cover its own |
| 295 |
motivating case is the most valuable thing you can find. |
| 296 |
2. Is it on the deferred table in section 6? Then the named remedy applies -- |
| 297 |
a \`-> impl Display\` supplier, a \`-> Vec<T>\` payload supplier, a named |
| 298 |
predicate, a domain accessor. Set \`remedy_available: true\`. That is outcome |
| 299 |
A, the expected one, and it is not a failure of the form. |
| 300 |
|
| 301 |
A target re-authors (\`reauthored: true\`) when every construct in it is either |
| 302 |
expressible or covered by a stated remedy. \`verdict\` is \`clean\` only when all |
| 303 |
of your targets re-authored. |
| 304 |
|
| 305 |
Be exact and be honest. A false clean is worse here than a false failure: this |
| 306 |
round decides whether 289 more functions get re-authored against this form.`, |
| 307 |
{ label: `probe ${b.probe}`, phase: 'Probe', schema: PROBE_SCHEMA } |
| 308 |
), |
| 309 |
|
| 310 |
(r, b) => { |
| 311 |
if (!r) return null |
| 312 |
const fresh = (r.classes || []).filter((c) => !c.in_record && !c.remedy_available) |
| 313 |
if (!fresh.length) return { probe: b.probe, result: r, bar: null } |
| 314 |
return agent( |
| 315 |
`Apply section 8's four-part bar to each class below. They came out of probe |
| 316 |
${b.probe} and none is in the record, so each one either earns a production, takes |
| 317 |
a remedy, or is refused. |
| 318 |
${READING} |
| 319 |
|
| 320 |
THE CLASSES: |
| 321 |
|
| 322 |
${fresh.map((c) => `- ${c.name}: ${c.construct}\n seen at ${c.site}; effect: ${c.effect || 'unstated'}`).join('\n')} |
| 323 |
|
| 324 |
For each one, all four parts, and no shortcuts: |
| 325 |
|
| 326 |
1. A SITE COUNT for the whole class across all three shape directories, produced |
| 327 |
by a command you write down and actually run, with \`tests.rs\`, \`parity.rs\` and |
| 328 |
inline \`#[cfg(test)]\` excluded and comments and string literals blanked before |
| 329 |
matching. A count off \`Type::method(\` alone is NOT a count -- three of the four |
| 330 |
members this record had to rescue are reached by a chained builder or a field |
| 331 |
assignment. Put the command in \`count_command\`, runnable as written. |
| 332 |
2. THE NAMED ALTERNATIVE, PRICED. Either the signature change and its call-site |
| 333 |
count, or the number of new supplier functions. "Move it upstream" with no |
| 334 |
number is not an alternative. |
| 335 |
3. A FAILING SCREEN, by file and line. It must currently produce something a user |
| 336 |
sees that the declaration cannot produce, or force a compile error, or change a |
| 337 |
status code. Convenience is not a reason. |
| 338 |
4. WHAT THE PRODUCTION CANNOT THEN BE USED FOR. A production admitting an |
| 339 |
expression, a block in argument position, a closure, or a \`Type { .. }\` |
| 340 |
aggregate is refused whatever its site count. |
| 341 |
|
| 342 |
THE THRESHOLD. Under 10 sites earns no production -- outcome A or C. Between 10 |
| 343 |
and 30 it earns one only if part 2's alternative touches more call sites than the |
| 344 |
construct has, or part 3's failure is a status-code or data-loss failure rather |
| 345 |
than a rendering one. Over 30 it earns one. |
| 346 |
|
| 347 |
Default to refusing. The form already refuses \`impl Trait\` at 4 sites and |
| 348 |
\`Repeating\` at 7. You are read-only: measure and rule, change nothing.`, |
| 349 |
{ label: `bar: probe ${b.probe}`, phase: 'Adjudicate', effort: 'high', schema: BAR_SCHEMA } |
| 350 |
).then((bar) => ({ probe: b.probe, result: r, bar })) |
| 351 |
} |
| 352 |
) |
| 353 |
|
| 354 |
const residue = await residuePromise |
| 355 |
|
| 356 |
|
| 357 |
phase('Synthesize') |
| 358 |
const ok = probed.filter(Boolean) |
| 359 |
const results = ok.map((p) => p.result) |
| 360 |
const cleanProbes = results.filter((r) => r.verdict === 'clean').length |
| 361 |
const allClasses = results.flatMap((r) => r.classes || []) |
| 362 |
const newClasses = allClasses.filter((c) => !c.in_record && !c.remedy_available) |
| 363 |
const amendmentHits = allClasses.filter((c) => c.in_record) |
| 364 |
const rulings = ok.flatMap((p) => (p.bar && p.bar.classes) || []) |
| 365 |
|
| 366 |
|
| 367 |
|
| 368 |
const distinctNew = [...new Map(newClasses.map((c) => [c.name.toLowerCase(), c])).values()] |
| 369 |
const productions = rulings.filter((r) => r.outcome === 'B-production') |
| 370 |
const over30 = rulings.filter((r) => r.site_count > 30 && r.outcome !== 'B-production') |
| 371 |
|
| 372 |
if (ok.length < sample.batches.length) { |
| 373 |
log(`${sample.batches.length - ok.length} probe(s) returned nothing and are NOT counted as clean.`) |
| 374 |
} |
| 375 |
|
| 376 |
const rate = distinctNew.length / Math.max(1, ok.length) |
| 377 |
const [r4b, overAdmit] = residue |
| 378 |
|
| 379 |
const closed = |
| 380 |
distinctNew.length === 0 && |
| 381 |
productions.length === 0 && |
| 382 |
over30.length === 0 && |
| 383 |
r4b && r4b.reproduced === false && |
| 384 |
ok.length === sample.batches.length |
| 385 |
|
| 386 |
log(`${cleanProbes}/${ok.length} probes clean. ${distinctNew.length} new classes (round 1: ~${(ROUND_1.classes_per_probe).toFixed(2)}/probe, this round ${rate.toFixed(2)}/probe).`) |
| 387 |
log(closed |
| 388 |
? 'Every reopening condition in section 8 is unmet. The closure record can be filed.' |
| 389 |
: 'At least one reopening condition is met. The form is not closed; amend and run round 3.') |
| 390 |
|
| 391 |
return { |
| 392 |
round: 2, |
| 393 |
seed: sample.seed, |
| 394 |
population: sample.population, |
| 395 |
sampled: sample.sampled, |
| 396 |
coverage_pct: sample.coverage_pct, |
| 397 |
selftest: sample.selftest, |
| 398 |
probes_run: ok.length, |
| 399 |
probes_requested: sample.batches.length, |
| 400 |
probes_clean: cleanProbes, |
| 401 |
round_1: ROUND_1, |
| 402 |
new_class_rate_per_probe: Number(rate.toFixed(2)), |
| 403 |
amendments_exercised: amendmentHits.map((c) => ({ ref: c.record_ref, site: c.site, note: c.effect })), |
| 404 |
new_classes: distinctNew, |
| 405 |
rulings, |
| 406 |
productions_earned: productions, |
| 407 |
unruled_over_30: over30, |
| 408 |
r4b, |
| 409 |
over_admission: overAdmit, |
| 410 |
closed, |
| 411 |
next: closed |
| 412 |
? 'File the closure record as wiki `quasi-description-vocabulary-closed`, then close GoingsOn 65d99281 and start the emitter (6873a26c).' |
| 413 |
: 'Amend the form for each earned production, update wiki `quasi-declare-form`, and run another round. Do not file a closure record.', |
| 414 |
} |
| 415 |
|