metronome/tests/run.mjs

#!/usr/bin/env node
// Conformance runner for the PM track format.
//   node tests/run.mjs            run all golden vectors against engine.js + app.py
//   node tests/run.mjs -v         also print the expected/actual diff for every failure
//
// For each vector it parses `in` with both implementations, normalizes, and compares to
// `norm`. A mismatch on an impl listed in the vector's `expectFail` is "known" (expected);
// any other mismatch is a regression and fails the run. See docs/track-format.md.
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
import { execFileSync } from "node:child_process";
import * as js from "./adapters/js_adapter.mjs";

const here = dirname(fileURLToPath(import.meta.url));
const verbose = process.argv.includes("-v");
const fixtures = JSON.parse(readFileSync(join(here, "fixtures", "track-format.json"), "utf8"));
const pyAdapter = join(here, "adapters", "py_adapter.py");

// stable, key-sorted JSON so deep-equality is a string compare.
const stable = (o) => JSON.stringify(o, (k, v) =>
  v && typeof v === "object" && !Array.isArray(v)
    ? Object.fromEntries(Object.keys(v).sort().map((kk) => [kk, v[kk]]))
    : v);

// `orns` (per-step flam/drag/roll) defaults to all-zeros: an impl MAY emit it always or omit it.
// Drop all-zero `orns` from both sides before comparing so legacy vectors that omit it still match.
const stripZeroOrns = (o) =>
  o && typeof o === "object" && Array.isArray(o.lanes)
    ? { ...o, lanes: o.lanes.map((L) =>
        L && Array.isArray(L.orns) && L.orns.every((v) => !v) ? (({ orns, ...rest }) => rest)(L) : L) }
    : o;

function runJs(patch) {
  try {
    return { norm: js.normalize(patch), canonical: js.canonical(patch), error: null };
  } catch (e) {
    return { norm: null, canonical: null, error: String(e.message || e) };
  }
}

function runPy(patch) {
  try {
    const out = execFileSync("python3", [pyAdapter, patch], { encoding: "utf8" });
    return { ...JSON.parse(out), error: null };
  } catch (e) {
    const msg = (e.stderr || "").toString().trim().split("\n").pop() || e.message;
    return { norm: null, canonical: null, error: msg };
  }
}

const want = (o) => stable(stripZeroOrns(o)); // compare ignoring all-zero `orns`
let regressions = 0, fixedNowCount = 0, nonIdempotent = 0;
const rows = [];

function jsIdempotent(patch) {
  try { const c1 = js.canonical(patch); return c1 === js.canonical(c1); } catch { return false; }
}

for (const c of fixtures.cases) {
  const expected = want(c.norm);
  const expectFail = new Set(c.expectFail || []);
  const r = { id: c.id, status: c.status };

  for (const [impl, res] of [["js", runJs(c.in)], ["py", runPy(c.in)]]) {
    // serialize(parse(x)) must be stable under re-parsing (no silent drift on round-trip).
    const idem = impl === "js" ? jsIdempotent(c.in) : res.idempotent !== false;
    if (res.error == null && !idem) { nonIdempotent++; console.log(`  ! non-idempotent serialize: ${c.id} [${impl}]`); }
    const ok = res.error == null && want(res.norm) === expected;
    const known = expectFail.has(impl);
    let mark;
    if (ok && !known) mark = "PASS";
    else if (ok && known) { mark = "FIXED"; fixedNowCount++; }       // listed as failing but now passes
    else if (!ok && known) mark = "known";                          // expected divergence/not-built
    else { mark = "FAIL"; regressions++; }                          // unexpected → regression
    r[impl] = mark;
    r[impl + "_res"] = res;
    if (mark === "FAIL" && verbose) {
      console.log(`\n--- ${c.id} [${impl}] expected vs actual ---`);
      console.log("expected:", expected);
      console.log("actual:  ", res.error ? "ERROR " + res.error : want(res.norm));
    }
  }
  rows.push(r);
}

// ---- report ----
const pad = (s, n) => String(s).padEnd(n);
console.log("\n  PM track-format conformance\n");
console.log("  " + pad("case", 26) + pad("status", 13) + pad("engine.js", 11) + "app.py");
console.log("  " + "-".repeat(58));
const glyph = { PASS: "✓ pass", known: "· known", FAIL: "✗ FAIL", FIXED: "★ fixed" };
for (const r of rows) {
  console.log("  " + pad(r.id, 26) + pad(r.status, 13) + pad(glyph[r.js], 11) + glyph[r.py]);
}

const counts = rows.reduce((a, r) => { a[r.js] = (a[r.js] || 0) + 1; a[r.py] = (a[r.py] || 0) + 1; return a; }, {});
console.log("\n  " + Object.entries(counts).map(([k, v]) => `${glyph[k] || k}: ${v}`).join("   "));
if (fixedNowCount) console.log(`\n  ${fixedNowCount} case(s) marked expectFail now PASS — update the fixture (remove them from expectFail).`);
if (nonIdempotent) console.log(`  ${nonIdempotent} non-idempotent serialize(s) above.`);
if (regressions || nonIdempotent) {
  console.log(`\n  ✗ ${regressions} unexpected failure(s), ${nonIdempotent} round-trip issue(s). Run with -v for diffs.\n`);
  process.exit(1);
}
console.log("\n  ✓ no unexpected failures; serialize round-trips are stable.\n");