RingBRP/tools/check-figures.mjs

/**
 * A figure nothing holds is a figure that drifts, and the guard that would catch it cannot
 * see it.
 *
 * `check-cited` resolves every citation marker in the corpus against the artifact it names.
 * It is a good reader and it has exactly one blind spot, which is structural rather than a
 * bug: it can only resolve the markers that exist. A measured figure written into prose with
 * NO marker beside it is not a failed citation — it is not a citation at all, and nothing
 * looks at it ever again.
 *
 * That is not hypothetical. Desk pass 11 found "a median 24 rounds" in the Pacing Note and
 * "24 if the column joins" in EXPOSURE. The baseline holds 21 at two of the column joining
 * and 25 at three. 24 is neither, it had been wrong through every pass that checked
 * citations, and it survived because it carried no marker to check. The same document
 * printed its four-player wipe rate uncited twice while citing it correctly four times —
 * and that figure is the one that was published at 58.7% until a truncated sweep was found.
 *
 * ⚠ THE SHAPES HAVE A BLIND SPOT AND IT IS STRUCTURAL. R-299 said this guard "learns its
 * vocabulary from the marked figures and applies it to the unmarked" — and a vocabulary
 * learned that way cannot contain the phrasing of the figure nobody marked. The concurrent
 * session proved it against a live defect: a THIRD stale 24, in a table cell, reported OK by
 * this file. Its measurement status lived in the header row two lines above —
 *
 *     | | Median rounds |
 *     |---|---|
 *     | Six hollow men + three of the column | **24** |     <- the baseline says 25
 *
 * — and the scan read one line at a time, so a bare number in a column the document itself
 * calls "Median rounds" was invisible. Fixed below by reading the header: a cell inherits
 * the measurement status of its column, which is a claim the DOCUMENT makes rather than one
 * this file's vocabulary has to anticipate. That is the narrow repair. The general point
 * stands and is worth keeping in view — see `check-unmarked`, which attacks the same problem
 * from the other end by treating bold as the corpus's own mark of a published figure.
 *
 * So this is the asymmetric counter for the other direction, and the fifth in this repo:
 *
 *   check-cited        reads markers          -> is the figure beside this marker right?
 *   check-figures      reads FIGURES          -> does this figure have a marker at all?
 *
 * THE OPT-IN RULE. A document that uses citations has opted into the regime, and every
 * measurement-shaped figure in it must carry a marker. A document with no markers anywhere
 * has not, and its figures are counted and held by a ratchet rather than failing the build —
 * because turning three unguarded scenarios red is how a guard gets switched off on the day
 * it is written. The ratchet still means they cannot get worse, and the moment such a
 * document gains its first citation it joins the strict regime in full.
 *
 *   node tools/check-figures.mjs            check the corpus
 *   node tools/check-figures.mjs --list     print every figure found, marked or not
 *   node tools/check-figures.mjs --update   re-record the ratchet for opted-out documents
 */
import { readFileSync, writeFileSync, existsSync } from "node:fs";
import path from "node:path";
import { playableFiles } from "./check-scenarios.mjs";

const ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, "$1")), "..");
const BASELINE = path.join(ROOT, "tools", "figures-baseline.json");
const argv = process.argv.slice(2);
const LIST = argv.includes("--list");
const UPDATE = argv.includes("--update");

/* A number, with the document's own bolding allowed around it. */
const N = String.raw`\*{0,2}(\d+(?:\.\d+)?)\*{0,2}`;

/* THE SHAPES A HARNESS FIGURE TAKES IN THIS CORPUS.
   Deliberately narrow. An earlier draft matched any number within forty-five characters of
   a measurement word and reported 282 candidates, nearly all of them prose — "down 140
   steps", "an engineer on his rounds", "01:06". A guard that cries wolf 282 times is a
   guard nobody runs. These are the phrasings the documents actually use when quoting the
   simulator, and each one was read off a figure that IS cited somewhere. */
const SHAPES = [
  ["median",      new RegExp(String.raw`median(?:\s+of)?\s+` + N, "gi")],
  ["rounds",      new RegExp(N + String.raw`\s*\*{0,2}\s*rounds\b`, "gi")],
  ["wiped",       new RegExp(N + String.raw`\s*%?\s*\*{0,2}\s*(?:wiped|wipes)\b`, "gi")],
  ["wipes-pct",   new RegExp(String.raw`wipe[sd]?\s+` + N + String.raw`\s*%`, "gi")],
  ["wipe-rate",   new RegExp(String.raw`wipe rate[^.]{0,24}?` + N + String.raw`\s*%`, "gi")],
  ["of-N-hurt",   new RegExp(N + String.raw`\s*\*{0,2}\s*of\s+\d+\s+(?:hurt|down)\b`, "gi")],
  ["deaths",      new RegExp(N + String.raw`\s*\*{0,2}\s*deaths?\b`, "gi")],
  ["pct-of-runs", new RegExp(N + String.raw`\s*%[^.]{0,18}\bruns?\b`, "gi")]
];

/* WHAT IS NOT A MEASUREMENT, each with the case that put it here.

   A DENOMINATOR. "one party in 300 wiped" is a rate written longhand; 300 is the bottom of
   a fraction, not a sample statistic, and THROUGH_TRAIN writes rates that way throughout.

   A THRESHOLD. "Past 15 rounds" is a column header in fight-tail's table — it names the
   bucket the figures below are counted into, and it comes from the tool's own definition
   rather than from a run. Same for "1 in 100 runs past".

   BOTH ARE TESTED AGAINST THE CONTEXT, NOT THE MATCH. The first draft tested the matched
   text alone, which cannot work: the `rounds` shape captures "15 rounds" and the word that
   excuses it — "Past" — is outside the match. It reported a column header as an unheld
   figure, which is the false positive that makes a guard get switched off. */
const NOT_A_MEASUREMENT = [
  [/\b(?:in|of)\s+\*{0,2}\d+(?:\.\d+)?\*{0,2}\s*%?\s*\*{0,2}\s*(?:wiped|wipes)\b/i, "a denominator — 'one party in N wiped' is a rate written longhand"],
  [/\b(?:past|beyond|over|under|at least|fewer than)\s+\*{0,2}\d+(?:\.\d+)?\*{0,2}\s*(?:%|\s*rounds?)?\s*$/i, "a threshold, not a sample — it names a bucket, not a result"]
];

/* FIELDS check-cited REFUSES TO LET ANYBODY CITE, and their values.
   `longest` is a sample maximum: same party, same seeds, same runs, and renaming a config
   moved one from 71 to 90 rounds, because the stream is derived from the id. check-cited
   refuses a citation against it for that reason — but refusing the CITATION while the
   document prints the NUMBER leaves the least stable figure in the suite as the only one
   nothing holds. So a bare figure whose value matches an uncitable field is reported as its
   own kind of problem, with the field named. */
const UNCITEABLE_FIELDS = ["longest"];
const uncitableValues = new Map();
for (const rel of ["tools/pack-tables-baseline.json", "tools/fight-tail-baseline.json"]) {
  const abs = path.join(ROOT, rel);
  if (!existsSync(abs)) continue;
  const walk = (o, trail) => {
    if (o && typeof o === "object") {
      for (const [k, v] of Object.entries(o)) {
        if (UNCITEABLE_FIELDS.includes(k) && typeof v === "number") {
          uncitableValues.set(String(v), `${rel} ${[...trail, k].join(".")}`);
        }
        walk(v, [...trail, k]);
      }
    }
  };
  walk(JSON.parse(readFileSync(abs, "utf8")), []);
}

/* WHAT MAKES A COLUMN A MEASUREMENT, and therefore every number in it a figure.
   Drawn from the baselines' own field names, exactly as the shapes above are. A bare `%` is
   deliberately NOT here: a column of percentages may be skill ratings, and this corpus
   prints plenty of those. */
const MEASURE_HEADER = /\b(median|mean|average|wiped?|wipes|hurt|down|deaths?|swing|rounds?)\b/i;

/** Split a markdown table row into cells, keeping each cell's offset in the line. */
function cells(line) {
  const out = [];
  let at = line.indexOf("|");
  if (at < 0) return out;
  at += 1;
  for (;;) {
    const end = line.indexOf("|", at);
    if (end < 0) break;
    out.push({ text: line.slice(at, end), start: at });
    at = end + 1;
  }
  return out;
}

/** For every line, the header row of the table it is a body row of — or null. */
function tableHeaders(lines) {
  const owner = new Array(lines.length).fill(null);
  for (let i = 1; i < lines.length; i++) {
    if (!/^\s*\|[\s:|-]+\|\s*$/.test(lines[i])) continue;      /* the |---|---| separator */
    if (!/^\s*\|/.test(lines[i - 1])) continue;
    for (let j = i + 1; j < lines.length && /^\s*\|/.test(lines[j]); j++) owner[j] = lines[i - 1];
  }
  return owner;
}

/* A CITATION MARKER IS FULL OF DIGITS AND NONE OF THEM ARE FIGURES. `packs line.hollow6.hurt`
   holds a 6; `fight-tail cut.over15` holds a 15. Scanning raw cell text made the guard
   report every correctly-cited row in the document as unheld — 130 false positives, on the
   tables that are the best-marked thing in the corpus. Comments are BLANKED rather than
   removed so every offset still points at the right character of the real line. */
const maskComments = line => line.replace(/<!--[\s\S]*?-->/g, m => " ".repeat(m.length));

/** Figures in a table row, by the header of their own column rather than by phrasing. */
export function cellFiguresIn(line, header) {
  if (!header) return [];
  const heads = cells(header);
  const masked = maskComments(line);
  const out = [];
  for (const [i, cell] of cells(masked).entries()) {
    if (!MEASURE_HEADER.test(heads[i]?.text ?? "")) continue;
    const re = /(\d+(?:\.\d+)?)/g;
    let m;
    while ((m = re.exec(cell.text))) {
      /* "3.48 of 6" — the 6 is the party size the mean is out of, not a measurement. */
      if (/\bof\s+$/i.test(cell.text.slice(0, m.index))) continue;
      const numEnd = cell.start + m.index + m[1].length;
      out.push({
        kind: `column:${(heads[i].text.replace(/[*|]/g, "").trim() || "?").toLowerCase()}`,
        value: m[1],
        text: cell.text.replace(/\s+/g, " ").trim() || m[1],
        at: numEnd,
        marked: MARKED_AFTER.test(line.slice(numEnd))
      });
    }
  }
  return out;
}

/* The marker check, deliberately looser than check-cited's CITE. That reader is strict on
   purpose; here, ANY citation-shaped comment beside the figure means somebody marked it and
   check-cited owns the question of whether they marked it correctly. Two guards disagreeing
   about what counts as marked would leave a figure that neither of them holds. */
const MARKED_AFTER = /^[ \t]*(?:%|x)?\**[ \t]*(?:—|-|–)?[ \t]*<!--[ \t]*cite[ \t]*:/i;

/** Every measurement-shaped figure on one line, with whether it carries a marker. */
export function figuresIn(line) {
  const out = [];
  const seen = new Set();
  for (const [kind, re] of SHAPES) {
    re.lastIndex = 0;
    let m;
    while ((m = re.exec(line))) {
      const numEnd = line.indexOf(m[1], m.index) + m[1].length;
      /* Context, not the match: the word that excuses a figure sits outside it. */
      const before = line.slice(Math.max(0, m.index - 24), numEnd);
      if (NOT_A_MEASUREMENT.some(([re2]) => re2.test(before) || re2.test(m[0]))) continue;
      /* One figure can match two shapes ("2 rounds" inside "median 2 rounds"); the figure is
         the number, so it is counted once at the position it occupies. */
      if (seen.has(numEnd)) continue;
      seen.add(numEnd);
      out.push({ kind, value: m[1], text: m[0].trim(), marked: MARKED_AFTER.test(line.slice(numEnd)) });
    }
  }
  return out.sort((a, b) => a.text.localeCompare(b.text));
}

const strict = [];
const loose = {};
const rows = [];

for (const [label, abs] of playableFiles()) {
  const src = readFileSync(abs, "utf8");
  /* Fenced blocks are the safe harbour, exactly as they are for check-outcomes: a code
     sample or a recorded harness dump is not the document making a claim. */
  const fenced = new Set();
  let inFence = false;
  const lines = src.split("\n");
  lines.forEach((ln, i) => {
    if (/^\s*```/.test(ln)) { inFence = !inFence; fenced.add(i); return; }
    if (inFence) fenced.add(i);
  });

  const optedIn = /<!--[ \t]*cite[ \t]*:/i.test(src);
  const headers = tableHeaders(lines);
  let unmarked = 0;

  lines.forEach((ln, i) => {
    if (fenced.has(i)) return;
    /* Shape pass and header pass can find the same number — "median 21" inside a cell of a
       column headed Median. Counted once, at the position it occupies. */
    const seen = new Set();
    const found = [...figuresIn(ln), ...cellFiguresIn(ln, headers[i])]
      .filter(f => (f.at === undefined ? true : !seen.has(f.at) && seen.add(f.at)));
    for (const f of found) {
      rows.push({ label, line: i + 1, ...f, optedIn });
      if (f.marked) continue;
      unmarked++;
      if (optedIn) {
        const uncitable = uncitableValues.get(f.value);
        strict.push(uncitable
          ? `  ${label}:${i + 1} — "${f.text}" is bare, and ${f.value} is the value of `
            + `${uncitable}, a field check-cited REFUSES to let anybody cite because a sample `
            + `maximum moves by a third on a re-seed. Refusing the citation while printing the `
            + `number leaves the least stable figure in the suite as the only one nothing holds. `
            + `Do not cite it — stop printing it, and lean on the guarantee instead.`
          : `  ${label}:${i + 1} — "${f.text}" carries no citation marker, in a document `
            + `that cites ${(src.match(/<!--[ \t]*cite[ \t]*:/gi) ?? []).length} other figures. `
            + `Either mark it, or say in words why this number is not a measurement.`);
      }
    }
  });
  if (!optedIn) loose[label] = unmarked;
}

/* ── the ratchet, for documents outside the citation regime ────────────────────────────── */

const base = existsSync(BASELINE) ? JSON.parse(readFileSync(BASELINE, "utf8")) : null;

if (UPDATE) {
  writeFileSync(BASELINE, JSON.stringify({
    note: "Unmarked measurement figures in scenarios that use no citations. May fall, never rise. "
        + "Generated by tools/check-figures.mjs --update.",
    uncited: loose
  }, null, 1) + "\n");
  console.log(`check-figures: recorded ${Object.keys(loose).length} document(s) outside the citation regime`);
  process.exit(0);
}

const problems = [...strict];

if (!base) {
  problems.push(`  no ratchet at tools/figures-baseline.json — run: node tools/check-figures.mjs --update`);
} else {
  for (const [label, n] of Object.entries(loose)) {
    const was = base.uncited?.[label];
    if (was === undefined) {
      problems.push(`  ${label} uses no citations and is not in the ratchet. Re-record with --update, `
        + `or give it its first citation marker and it joins the strict regime.`);
    } else if (n > was) {
      problems.push(`  ${label}: unmarked measurement figures went ${was} -> ${n}. This document cites `
        + `nothing, so the ratchet is all that holds it: the count may fall, never rise.`);
    }
  }
  /* A document that LEAVES the loose set has gained its first citation, which is good news
     and must not be reported as a missing row. A document that vanishes entirely has been
     deleted or reclassified, and that is worth saying out loud. */
  for (const label of Object.keys(base.uncited ?? {})) {
    if (!(label in loose) && !rows.some(r => r.label === label)) {
      problems.push(`  ${label} is in the ratchet but no longer in the corpus. If it was renamed or `
        + `deleted, re-record with --update.`);
    }
  }
}

if (LIST) {
  for (const r of rows) {
    console.log(`  ${r.marked ? "cited " : "BARE  "} ${r.label}:${r.line}  [${r.kind}]  ${r.text}`);
  }
  problems.forEach(p => console.log(p));
  process.exit(0);
}
if (problems.length) {
  console.error("check-figures: FAILED — measured figures are printed with nothing holding them");
  problems.forEach(p => console.error(p));
  console.error(`  check-cited resolves the markers that exist; it cannot see a figure that has none. `
    + `That is how "a median 24 rounds" survived ten passes against a baseline holding 21.`);
  process.exit(1);
}
const marked = rows.filter(r => r.marked).length;
const looseTotal = Object.values(loose).reduce((a, b) => a + b, 0);
console.log(`check-figures: OK — ${rows.length} measurement-shaped figures across ${new Set(rows.map(r => r.label)).size} `
  + `scenarios, ${marked} marked; ${looseTotal} bare in ${Object.keys(loose).length} document(s) that cite nothing, held at the ratchet`);