RingBRP/tools/check-lethality.mjs

/**
 * check-lethality — no creature may quietly become something else.
 *
 * The seven guards before this one check that content is WELL FORMED. None of them
 * checks what it DOES. A creature can validate perfectly, pack perfectly, and have
 * become twice as dangerous as it was last week because somebody adjusted a damage
 * modifier, a hit-point formula or the armour on a service vest. That change is
 * invisible in a diff of the creature, because the creature did not change.
 *
 * This is the guard the creature-forge plan called for, built as a regression test
 * rather than as a set of hand-declared bands. Bands were the plan's design and they
 * are the wrong one here: declaring "keepers: dangerous" on forty-six creatures means
 * inventing forty-six judgements, and the judgement that matters is not "is this
 * dangerous" but "is this the same as it was". A baseline answers that exactly, needs
 * no authoring, and cannot be argued with.
 *
 *   node tools/check-lethality.mjs            compare against the committed baseline
 *   node tools/check-lethality.mjs --update    re-record it (after a deliberate change)
 *   node tools/check-lethality.mjs --verbose   print every creature, not just drift
 *
 * WHY A FIXED PARTY. --spread exists because one number cannot describe an encounter:
 * the Act Three standoff runs 0% to 100% depending which four agents were picked. That
 * is true and it is why this guard does NOT try to describe danger. It measures the
 * same creature against the same four agents with the same seed, so the only thing that
 * can move the number is a change to the rules or to the creature. The party below is
 * deliberately mixed — two armed postings and two trades — so that a change affecting
 * either kind of agent shows up, and it is frozen for reproducibility rather than
 * chosen for realism.
 */
import { readFile, writeFile } from "node:fs/promises";
import { existsSync } from "node:fs";
import path from "node:path";
import { collectSpecs } from "./all-specs.mjs";
import { ROSTER } from "./roster.mjs";
import { measure, seedFor } from "./simulate.mjs";

const ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, "$1")), "..");
const BASELINE = path.join(ROOT, "tools", "lethality-baseline.json");

const argv = process.argv.slice(2);
const UPDATE = argv.includes("--update");
const VERBOSE = argv.includes("--verbose");

/* Frozen. Changing this invalidates every recorded number, so if it ever must change,
   change it in the same commit as a --update and say why in the message. */
const PARTY_KEYS = ["pc_holloway", "pc_okonkwo", "pc_nkemdirim", "pc_ferriby"];
const RUNS = 2000;
const SEED = 11;
/* Recorded alongside the numbers, because changing HOW the seed is derived changes every
   row without changing SEED itself — a baseline taken under the old shared-seed scheme
   would otherwise be compared, silently and wrongly, against this one. */
const SEED_MODE = "per-creature";

/* NO TOLERANCE, deliberately. The reasoning it replaces sounded right — "200 runs of a
   coin flip has a standard error around 3.5 points, so anything under 6 is inside the
   noise" — and it is exactly backwards for this measurement. Sampling error would apply
   if the runs were random; they are not. The guard used to allow six points of wipe
   rate and a third of an agent before it complained, which sounds prudent and was why it
   sat silent through the largest combat change the system has had: ddc4f99 routed every
   ordinary blow through the hit location table, 26 of 46 creatures moved, and not one
   of them moved far enough in a single number to trip the threshold. The baseline went
   four commits describing a game nobody was playing.

   A tolerance is for noise, and there is none here: same party, same seed, same counts,
   and measure() builds its own generator per creature, so two recordings of unchanged
   code are byte-identical. Anything that moves is a real change to the rules or to the
   creature, which is exactly what this file exists to notice. `rounds` is compared too;
   it was recorded and then ignored, so a creature could take a round longer to kill
   forever without a word. */

const party = PARTY_KEYS.map(k => {
  const found = ROSTER.find(r => r.key === k);
  if (!found) {
    console.error(`check-lethality: the frozen party names "${k}", which is not on the roster.`);
    process.exit(1);
  }
  return found;
});

/* WHAT IS MEASURED.
 *
 * Every npc-kind spec in the repository, read through collectSpecs() — the one list that
 * check-creatures, make-portraits and mj-queue already share. This file used to import
 * NPCS from content.mjs directly, which was the same thing until the 100-creature
 * bestiary landed in its own files: after that it measured 47 of 147 and reported OK.
 * A guard that cannot see two thirds of the creatures it guards is not a guard, and it
 * failed silently, which is worse than failing loudly.
 *
 * Scenario casts come in with them. They were never measured before and they should have
 * been: the Act Three standoff this repository publishes wipe rates for IS a scenario
 * cast, and a rules change reaches a ticket inspector exactly as it reaches a barghest.
 * 184 specs at roughly 47ms each is about nine seconds.                               */
const SPECS = (await collectSpecs()).filter(r => r.kind === "npc").map(r => r.spec);

/** Solo, because a creature is the unit under test — counts are an encounter's business. */
const current = {};
for (const spec of SPECS) {
  const r = measure(party, [spec], { runs: RUNS, seed: seedFor(SEED, spec.key) });
  current[spec.key] = {
    wipe: Number((r.wipeRate * 100).toFixed(1)),
    down: Number(r.downMean.toFixed(2)),
    rounds: r.roundsMedian
  };
}

if (UPDATE) {
  await writeFile(BASELINE, JSON.stringify({
    note: "Generated by tools/check-lethality.mjs --update. Do not edit by hand.",
    party: PARTY_KEYS, runs: RUNS, seed: SEED, seedMode: SEED_MODE,
    creatures: current
  }, null, 2) + "\n", "utf8");
  console.log(`check-lethality: baseline recorded — ${Object.keys(current).length} creatures, `
    + `party ${PARTY_KEYS.map(k => k.replace(/^pc_/, "")).join(", ")}, ${RUNS} runs, `
    + `seed ${SEED} ${SEED_MODE}`);
  process.exit(0);
}

if (!existsSync(BASELINE)) {
  console.error("check-lethality: no baseline. Run with --update to record one.");
  process.exit(1);
}

const base = JSON.parse(await readFile(BASELINE, "utf8"));

/* The recorded numbers mean nothing if they were taken under different conditions, and
   a guard comparing incomparable numbers is worse than no guard. */
if (base.runs !== RUNS || base.seed !== SEED
    || base.seedMode !== SEED_MODE
    || base.party.join() !== PARTY_KEYS.join()) {
  console.error("check-lethality: FAILED — the baseline was recorded under different conditions "
    + `(party ${base.party.join(",")}, ${base.runs} runs, seed ${base.seed} `
    + `${base.seedMode ?? "shared"}). Re-record it with --update.`);
  process.exit(1);
}

/* Comparing exactly is only honest if the measurement IS exact. Prove it here rather
   than trust it: one creature, measured twice, must come back identical. If a future
   change reaches for Math.random or a Set iteration order, this says so instead of
   letting the whole guard degrade into noise-chasing. */
{
  const probe = SPECS[0];
  const a = measure(party, [probe], { runs: RUNS, seed: seedFor(SEED, probe.key) });
  const b = measure(party, [probe], { runs: RUNS, seed: seedFor(SEED, probe.key) });
  const shape = r => JSON.stringify([r.wipeRate, r.downMean, r.roundsMedian, r.hurtMean]);
  if (shape(a) !== shape(b)) {
    console.error("check-lethality: FAILED — the simulation is not deterministic, so an exact "
      + `baseline cannot mean anything. ${probe.key} measured twice gave ${shape(a)} and ${shape(b)}.`);
    process.exit(1);
  }
}

const problems = [];
const added = [], removed = [];

for (const [key, now] of Object.entries(current)) {
  const was = base.creatures[key];
  if (!was) { added.push(key); continue; }
  if (now.wipe === was.wipe && now.down === was.down && now.rounds === was.rounds) continue;
  const dWipe = now.wipe - was.wipe, dDown = now.down - was.down;
  const bits = [];
  if (now.wipe !== was.wipe)
    bits.push(`wiped ${was.wipe}% -> ${now.wipe}% (${dWipe >= 0 ? "+" : ""}${dWipe.toFixed(1)})`);
  if (now.down !== was.down)
    bits.push(`down ${was.down} -> ${now.down} (${dDown >= 0 ? "+" : ""}${dDown.toFixed(2)} of 4)`);
  if (now.rounds !== was.rounds)
    bits.push(`rounds ${was.rounds} -> ${now.rounds}`);
  problems.push({ key, dWipe, text: `${key}: ${bits.join(", ")}` });
}
for (const key of Object.keys(base.creatures)) if (!(key in current)) removed.push(key);

if (VERBOSE) {
  for (const [key, now] of Object.entries(current)) {
    console.log(`  ${key.padEnd(22)} ${String(now.wipe).padStart(5)}% wiped  ${now.down.toFixed(2)} down  ${now.rounds} rounds`);
  }
}

if (removed.length) console.log(`  note: ${removed.length} creature(s) gone from content — ${removed.join(", ")}`);

/* A creature with no entry in the baseline is UNMEASURED, and it used to be a note.
   The success line underneath said "every one fighting exactly as recorded" while N of
   them had no record to fight differently from — the claim was false by exactly the
   number in the note above it, and the note is the part a green build does not get read
   for. It is the same shape as the seam defect this guard was on the wrong side of
   earlier: 47 creatures measured, OK printed for 184.

   Recording a new creature is one `--update` in the commit that adds it, which this
   guard's own failure message has always instructed. That is a cheap price for a
   success line that means what it says. */
if (added.length) {
  console.error(`check-lethality: FAILED — ${added.length} creature(s) have no recorded fight, `
    + `so nothing here says how they behave:`);
  for (const k of added) console.error(`  ${k}`);
  console.error(`\n  Run with --update in the same commit that adds them, and say in the message `
    + `what they are. A creature nobody has measured is not a creature this guard covers.`);
  process.exit(1);
}

if (problems.length) {
  console.error(`check-lethality: FAILED — ${problems.length} of ${Object.keys(current).length} `
    + `creature(s) fight differently than recorded`);
  problems.sort((a, b) => Math.abs(b.dWipe) - Math.abs(a.dWipe));
  for (const p of problems) console.error("  " + p.text);
  console.error("\n  If this was deliberate, re-record with --update and say what changed in the commit.");
  process.exit(1);
}

console.log(`check-lethality: OK — ${Object.keys(current).length} creatures, every one fighting `
  + `exactly as recorded against the frozen party`);