diff --git a/README.md b/README.md index 36c17a5..204f659 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ packs or regenerate art on that machine: ```bash npm install # pulls classic-level, used to write the LevelDB packs npm run build # rebuilds packs/ from tools/content.mjs -npm run check # the seven guards; the build refuses to run if they fail +npm run check # the eight guards; the build refuses to run if they fail ``` To update a deployed server: `git pull` and restart Foundry. If the pull touches @@ -157,7 +157,7 @@ icons/ fonts/ art/ generated art packs/ built LevelDB compendia (committed — see Deploying) tools/ content.mjs the catalogue: skills, weapons, armour, gear, vehicles, NPCs - build-packs.mjs builds packs/ — runs the seven guards first and refuses on failure + build-packs.mjs builds packs/ — runs the eight guards first and refuses on failure make-icons.mjs draws all 231 icons rules-text.mjs generates the rules journal FROM rules.mjs mission.mjs the case generator @@ -165,6 +165,7 @@ tools/ check-rules.mjs guard: no rule may be defined twice check-kits.mjs guard: every posting's kit must exist and be usable check-lang.mjs guard: no localisation key may be both a value and a branch + check-lethality.mjs guard: no creature may quietly change what it does to a party docs/REVIEW_LOG.md the design and defect log, R-1 onward ``` @@ -176,13 +177,17 @@ stops being identity-equal to what it aliases, if a constant is re-declared as a literal, or if an exported rule has no spot-check. The packs used to be built under one set of numbers and played under another; this makes that impossible to ship. -The build runs all seven guards before it writes anything: +The build runs all eight guards before it writes anything: ``` -check-rules: OK — 103 rules, 31 files scanned, 3 aliases + 46 constant sets checked, 346 formulas verified +check-rules: OK — 103 rules, 32 files scanned, 3 aliases + 46 constant sets checked, 346 formulas verified check-kits: OK — 25 roles, 10 trades, 218 catalogue items, every kit key resolves, every posting can use what it carries, every loadout distinct check-lang: OK — en.json, 995 keys, no leaf/branch collisions +check-templates: OK — 19 templates compile check-behaviour: OK — 67 behavioural tests +check-scenarios: OK — 9 scenario files, 62 mechanic tags, every skill named resolves against 62 catalogue entries +check-creatures: OK — 101 actor specs across 8 sources, every characteristic, skill, kit key, talent, species and style resolves, no duplicate keys +check-lethality: OK — 46 creatures, none drifted more than 6 points of wipe rate against the frozen party ``` diff --git a/package.json b/package.json index b26ad0f..4e21fb5 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "portraits": "node tools/make-portraits.mjs", "mj": "node tools/mj-queue.mjs", "simulate": "node tools/simulate.mjs", - "check": "node tools/check-rules.mjs && node tools/check-kits.mjs && node tools/check-lang.mjs && node tools/check-templates.mjs && node tools/check-behaviour.mjs && node tools/check-scenarios.mjs && node tools/check-creatures.mjs", + "check": "node tools/check-rules.mjs && node tools/check-kits.mjs && node tools/check-lang.mjs && node tools/check-templates.mjs && node tools/check-behaviour.mjs && node tools/check-scenarios.mjs && node tools/check-creatures.mjs && node tools/check-lethality.mjs", "test": "npm run check", "readme": "node tools/update-readme.mjs" }, diff --git a/tools/build-packs.mjs b/tools/build-packs.mjs index fa2467f..b711003 100644 --- a/tools/build-packs.mjs +++ b/tools/build-packs.mjs @@ -515,6 +515,12 @@ async function writePack(packName, entries) { // species-locked talent on the wrong species — all of which pack and play. execFileSync(process.execPath, [fileURLToPath(new URL("check-creatures.mjs", import.meta.url))], { stdio: "inherit" }); + // And refuse a creature that has quietly become something else. The seven guards + // above check that content is well formed; none of them checks what it DOES, so a + // change to a damage modifier or an armour value could double a creature's lethality + // without touching one line of that creature. + execFileSync(process.execPath, [fileURLToPath(new URL("check-lethality.mjs", import.meta.url))], + { stdio: "inherit" }); } catch { console.error("build aborted: pre-build checks failed"); process.exit(1); } } diff --git a/tools/check-lethality.mjs b/tools/check-lethality.mjs new file mode 100644 index 0000000..4c46900 --- /dev/null +++ b/tools/check-lethality.mjs @@ -0,0 +1,136 @@ +/** + * check-lethality — no creature may quietly become something else. + * + * The seven guards before this one check that content is WELL FORMED. None of them + * checks what it DOES. A creature can validate perfectly, pack perfectly, and have + * become twice as dangerous as it was last week because somebody adjusted a damage + * modifier, a hit-point formula or the armour on a service vest. That change is + * invisible in a diff of the creature, because the creature did not change. + * + * This is the guard the creature-forge plan called for, built as a regression test + * rather than as a set of hand-declared bands. Bands were the plan's design and they + * are the wrong one here: declaring "keepers: dangerous" on forty-six creatures means + * inventing forty-six judgements, and the judgement that matters is not "is this + * dangerous" but "is this the same as it was". A baseline answers that exactly, needs + * no authoring, and cannot be argued with. + * + * node tools/check-lethality.mjs compare against the committed baseline + * node tools/check-lethality.mjs --update re-record it (after a deliberate change) + * node tools/check-lethality.mjs --verbose print every creature, not just drift + * + * WHY A FIXED PARTY. --spread exists because one number cannot describe an encounter: + * the Act Three standoff runs 0% to 100% depending which four agents were picked. That + * is true and it is why this guard does NOT try to describe danger. It measures the + * same creature against the same four agents with the same seed, so the only thing that + * can move the number is a change to the rules or to the creature. The party below is + * deliberately mixed — two armed postings and two trades — so that a change affecting + * either kind of agent shows up, and it is frozen for reproducibility rather than + * chosen for realism. + */ +import { readFile, writeFile } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { NPCS } from "./content.mjs"; +import { ROSTER } from "./roster.mjs"; +import { measure } from "./simulate.mjs"; + +const ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, "$1")), ".."); +const BASELINE = path.join(ROOT, "tools", "lethality-baseline.json"); + +const argv = process.argv.slice(2); +const UPDATE = argv.includes("--update"); +const VERBOSE = argv.includes("--verbose"); + +/* Frozen. Changing this invalidates every recorded number, so if it ever must change, + change it in the same commit as a --update and say why in the message. */ +const PARTY_KEYS = ["pc_holloway", "pc_okonkwo", "pc_nkemdirim", "pc_ferriby"]; +const RUNS = 200; +const SEED = 11; + +/* How far a number may move before it is drift rather than noise. 200 runs of a coin + flip has a standard error around 3.5 points, so anything under 6 is inside the noise + and failing on it would make the guard cry wolf until somebody deleted it. */ +const WIPE_TOLERANCE = 6.0; // percentage points +const DOWN_TOLERANCE = 0.35; // agents, of four + +const party = PARTY_KEYS.map(k => { + const found = ROSTER.find(r => r.key === k); + if (!found) { + console.error(`check-lethality: the frozen party names "${k}", which is not on the roster.`); + process.exit(1); + } + return found; +}); + +/** Solo, because a creature is the unit under test — counts are an encounter's business. */ +const current = {}; +for (const spec of NPCS) { + const r = measure(party, [spec], { runs: RUNS, seed: SEED }); + current[spec.key] = { + wipe: Number((r.wipeRate * 100).toFixed(1)), + down: Number(r.downMean.toFixed(2)), + rounds: r.roundsMedian + }; +} + +if (UPDATE) { + await writeFile(BASELINE, JSON.stringify({ + note: "Generated by tools/check-lethality.mjs --update. Do not edit by hand.", + party: PARTY_KEYS, runs: RUNS, seed: SEED, + creatures: current + }, null, 2) + "\n", "utf8"); + console.log(`check-lethality: baseline recorded — ${Object.keys(current).length} creatures, ` + + `party ${PARTY_KEYS.map(k => k.replace(/^pc_/, "")).join(", ")}, ${RUNS} runs, seed ${SEED}`); + process.exit(0); +} + +if (!existsSync(BASELINE)) { + console.error("check-lethality: no baseline. Run with --update to record one."); + process.exit(1); +} + +const base = JSON.parse(await readFile(BASELINE, "utf8")); + +/* The recorded numbers mean nothing if they were taken under different conditions, and + a guard comparing incomparable numbers is worse than no guard. */ +if (base.runs !== RUNS || base.seed !== SEED + || base.party.join() !== PARTY_KEYS.join()) { + console.error("check-lethality: FAILED — the baseline was recorded under different conditions " + + `(party ${base.party.join(",")}, ${base.runs} runs, seed ${base.seed}). Re-record it with --update.`); + process.exit(1); +} + +const problems = []; +const added = [], removed = []; + +for (const [key, now] of Object.entries(current)) { + const was = base.creatures[key]; + if (!was) { added.push(key); continue; } + const dWipe = now.wipe - was.wipe; + const dDown = now.down - was.down; + if (Math.abs(dWipe) > WIPE_TOLERANCE || Math.abs(dDown) > DOWN_TOLERANCE) { + problems.push(`${key}: wiped ${was.wipe}% -> ${now.wipe}% (${dWipe >= 0 ? "+" : ""}${dWipe.toFixed(1)}), ` + + `down ${was.down} -> ${now.down} (${dDown >= 0 ? "+" : ""}${dDown.toFixed(2)} of 4)`); + } +} +for (const key of Object.keys(base.creatures)) if (!(key in current)) removed.push(key); + +if (VERBOSE) { + for (const [key, now] of Object.entries(current)) { + console.log(` ${key.padEnd(22)} ${String(now.wipe).padStart(5)}% wiped ${now.down.toFixed(2)} down ${now.rounds} rounds`); + } +} + +if (added.length) console.log(` note: ${added.length} new creature(s) not in the baseline — ${added.join(", ")}`); +if (removed.length) console.log(` note: ${removed.length} creature(s) gone from content — ${removed.join(", ")}`); + +if (problems.length) { + console.error("check-lethality: FAILED — a creature does a different amount of damage than recorded"); + for (const p of problems) console.error(" " + p); + console.error("\n If this was deliberate, re-record with --update and say what changed in the commit."); + process.exit(1); +} + +console.log(`check-lethality: OK — ${Object.keys(current).length} creatures, none drifted more than ` + + `${WIPE_TOLERANCE} points of wipe rate against the frozen party` + + `${added.length ? `, ${added.length} new` : ""}`); diff --git a/tools/lethality-baseline.json b/tools/lethality-baseline.json new file mode 100644 index 0000000..b2bf77a --- /dev/null +++ b/tools/lethality-baseline.json @@ -0,0 +1,243 @@ +{ + "note": "Generated by tools/check-lethality.mjs --update. Do not edit by hand.", + "party": [ + "pc_holloway", + "pc_okonkwo", + "pc_nkemdirim", + "pc_ferriby" + ], + "runs": 200, + "seed": 11, + "creatures": { + "quiet_neighbours_npc": { + "wipe": 0, + "down": 0.01, + "rounds": 2 + }, + "keepers": { + "wipe": 0, + "down": 0.04, + "rounds": 2 + }, + "cleaner": { + "wipe": 0, + "down": 0.04, + "rounds": 2 + }, + "cleaner_marksman": { + "wipe": 0, + "down": 0.07, + "rounds": 2 + }, + "revenant": { + "wipe": 0, + "down": 0.14, + "rounds": 3 + }, + "duplicate": { + "wipe": 0, + "down": 0.05, + "rounds": 2 + }, + "barghest": { + "wipe": 0, + "down": 0.48, + "rounds": 4 + }, + "kelpie": { + "wipe": 0, + "down": 0.23, + "rounds": 3 + }, + "redcap": { + "wipe": 1, + "down": 0.34, + "rounds": 3 + }, + "boggart": { + "wipe": 0, + "down": 0, + "rounds": 1 + }, + "churchgrim": { + "wipe": 0, + "down": 0.25, + "rounds": 4 + }, + "hollow_man": { + "wipe": 0, + "down": 0, + "rounds": 2 + }, + "the_tenant": { + "wipe": 0, + "down": 0, + "rounds": 1 + }, + "the_choir": { + "wipe": 0, + "down": 0, + "rounds": 2 + }, + "precedent": { + "wipe": 0, + "down": 0.17, + "rounds": 3 + }, + "carrion_file": { + "wipe": 0, + "down": 0.06, + "rounds": 5 + }, + "the_understudy": { + "wipe": 0, + "down": 0.01, + "rounds": 2 + }, + "surveyor": { + "wipe": 7, + "down": 1.33, + "rounds": 11 + }, + "quarantine_unit": { + "wipe": 85.5, + "down": 3.69, + "rounds": 8 + }, + "cuckoo_array": { + "wipe": 1, + "down": 0.76, + "rounds": 8 + }, + "long_walker": { + "wipe": 68, + "down": 3.33, + "rounds": 15 + }, + "auditor": { + "wipe": 0, + "down": 0.1, + "rounds": 3 + }, + "washer_ford": { + "wipe": 0, + "down": 0, + "rounds": 1 + }, + "lantern_man": { + "wipe": 0, + "down": 0, + "rounds": 2 + }, + "grindylow": { + "wipe": 0, + "down": 0.15, + "rounds": 3 + }, + "knockers": { + "wipe": 0, + "down": 0, + "rounds": 2 + }, + "spriggan": { + "wipe": 0, + "down": 0.3, + "rounds": 4 + }, + "nuckelavee": { + "wipe": 16.5, + "down": 1.68, + "rounds": 7 + }, + "hedley_kow": { + "wipe": 0, + "down": 0, + "rounds": 2 + }, + "apple_man": { + "wipe": 2.5, + "down": 1.1, + "rounds": 12 + }, + "the_committee": { + "wipe": 0, + "down": 0, + "rounds": 2 + }, + "night_registry": { + "wipe": 0, + "down": 0.09, + "rounds": 3 + }, + "the_locum": { + "wipe": 0, + "down": 0.06, + "rounds": 2 + }, + "lost_property": { + "wipe": 1.5, + "down": 0.75, + "rounds": 8 + }, + "the_minutes": { + "wipe": 0, + "down": 0, + "rounds": 1 + }, + "waiting_room": { + "wipe": 0, + "down": 0.17, + "rounds": 6 + }, + "the_predecessor": { + "wipe": 0, + "down": 0.09, + "rounds": 3 + }, + "switchboard": { + "wipe": 0, + "down": 0, + "rounds": 4 + }, + "the_census": { + "wipe": 0, + "down": 0.14, + "rounds": 4 + }, + "the_courier": { + "wipe": 6, + "down": 1.37, + "rounds": 9 + }, + "the_relay": { + "wipe": 0.5, + "down": 0.69, + "rounds": 11 + }, + "the_stanchion": { + "wipe": 26, + "down": 2.25, + "rounds": 19 + }, + "the_yield": { + "wipe": 1, + "down": 0.78, + "rounds": 6 + }, + "the_overwriter": { + "wipe": 0, + "down": 0.17, + "rounds": 6 + }, + "the_arrears": { + "wipe": 7.5, + "down": 1.58, + "rounds": 10 + }, + "the_margin": { + "wipe": 0, + "down": 0.03, + "rounds": 3 + } + } +} diff --git a/tools/update-readme.mjs b/tools/update-readme.mjs index d2a20e0..2c745b0 100644 --- a/tools/update-readme.mjs +++ b/tools/update-readme.mjs @@ -29,8 +29,12 @@ const runGuard = name => { { encoding: "utf8" }).split("\n").find(l => l.startsWith(name.replace(".mjs", ""))); } catch { return null; } }; +/* All of them, in the order `npm run check` runs them. This list was four long while the + suite was seven, so the README advertised a subset and silently stopped mentioning + every guard added after it was written — including the three that catch the most. */ const guardLines = ["check-rules.mjs", "check-kits.mjs", "check-lang.mjs", - "check-behaviour.mjs"].map(runGuard).filter(Boolean); + "check-templates.mjs", "check-behaviour.mjs", "check-scenarios.mjs", + "check-creatures.mjs", "check-lethality.mjs"].map(runGuard).filter(Boolean); const box = `| | | |---|---|