healer-man/src/game/benchmark/benchmarkReport.test.ts

import { describe, expect, it } from "vitest";
import { EMPTY_ITEM_STATS } from "../itemStats";
import { benchmarkScenarioById } from "./benchmarkScenarios";
import {
  aggregateBenchmarkMatrix,
  benchmarkQuantile,
  renderBenchmarkMatrixMarkdown,
  serializeBenchmarkMatrixCsv,
} from "./benchmarkReport";
import type {
  BenchmarkBuildDefinition,
  BenchmarkMatrixReport,
  BenchmarkRunMetrics,
  BenchmarkRunReport,
} from "./benchmarkTypes";

const scenario = benchmarkScenarioById("damage-single-burst")!;

const build: BenchmarkBuildDefinition = {
  id: "mage-burst",
  source: "wow",
  classId: "mage",
  secondaryClassId: null,
  role: "damage",
  level: 60,
  orientation: 'spell, "burst"',
  talentRanks: {},
  inventory: [],
  equipment: {},
  gearPresetId: "level-60-baseline",
};

const defaultMetrics: BenchmarkRunMetrics = {
  rawDamage: 0,
  effectiveDamage: 0,
  priorityTargetDamage: 0,
  effectiveHealing: 0,
  rawHealing: 0,
  overheal: 0,
  absorbsApplied: 0,
  absorbsConsumed: 0,
  rawDamageTaken: 0,
  effectiveDamageTaken: 0,
  incomingAttacks: 0,
  avoidedAttacks: 0,
  deaths: 0,
  minimumPlayerHealthPercent: 100,
  minimumPartyHealthPercent: 100,
  castsStarted: 0,
  castsCompleted: 0,
  movementInterrupts: 0,
  invalidCasts: 0,
  distanceMoved: 0,
  movementTimeMs: 0,
  stackCompliancePercent: null,
  threatControlPercent: null,
  timeToResourceFailureMs: null,
  timeToFirstDeathMs: null,
  dps: 0,
  hps: 0,
  dtps: 0,
  overhealPercent: 0,
  mitigationPercent: 0,
  avoidancePercent: 0,
};

function run(
  seed: number,
  metrics: Partial<BenchmarkRunMetrics>,
  valid = true,
): BenchmarkRunReport {
  return {
    schemaVersion: 1,
    runId: `mage-burst:${seed}`,
    policyId: "baseline-v1",
    seed,
    build,
    scenario,
    simulatedDurationMs: scenario.durationMs,
    completed: seed !== 2,
    metrics: { ...defaultMetrics, ...metrics },
    resources: [],
    abilities: [],
    eventCounts: {},
    validation: { valid, reasons: valid ? [] : ["fixture invalid"] },
    gearStats: EMPTY_ITEM_STATS,
  };
}

function matrix(): BenchmarkMatrixReport {
  return {
    schemaVersion: 1,
    generatedAt: "2026-08-28T12:00:00.000Z",
    appVersion: "test",
    cohort: {
      level: 60,
      gearPresetId: "level-60-baseline",
      seeds: [1, 2, 3, 4],
      durationScale: 1,
      policyId: "baseline-v1",
      rotationVersion: "playerbots-inspired-v1",
    },
    scenarios: [scenario],
    builds: [build],
    runs: [
      run(1, { dps: 10, priorityTargetDamage: 100 }),
      run(2, { dps: 20, priorityTargetDamage: 200 }),
      run(3, { dps: 30, priorityTargetDamage: 300 }),
      run(4, { dps: 999, priorityTargetDamage: 9_999 }, false),
    ],
  };
}

describe("benchmark reporting", () => {
  it("uses deterministic interpolated quantiles", () => {
    expect(benchmarkQuantile([30, null, 10, 20], 0.1)).toBe(12);
    expect(benchmarkQuantile([30, null, 10, 20], 0.5)).toBe(20);
    expect(benchmarkQuantile([30, null, 10, 20], 0.9)).toBe(28);
    expect(benchmarkQuantile([null], 0.5)).toBeNull();
    expect(() => benchmarkQuantile([1], 1.1)).toThrow(RangeError);
  });

  it("aggregates validated runs and retains null-event counts", () => {
    const summary = aggregateBenchmarkMatrix(matrix());
    const [row] = summary.rows;

    expect(summary).toMatchObject({ runCount: 4, validRunCount: 3, invalidRunCount: 1 });
    expect(row).toMatchObject({
      runCount: 4,
      validRunCount: 3,
      invalidRunCount: 1,
      completedRunCount: 3,
    });
    expect(row.metrics.dps).toEqual({
      count: 3,
      nullCount: 0,
      p10: 12,
      median: 20,
      p90: 28,
    });
    expect(row.metrics.timeToFirstDeathMs).toEqual({
      count: 0,
      nullCount: 3,
      p10: null,
      median: null,
      p90: null,
    });
  });

  it("serializes complete metric rows as newline-clean CSV", () => {
    const csv = serializeBenchmarkMatrixCsv(matrix());

    expect(csv).toContain("scenario_id,scenario_version,scenario_name");
    expect(csv).toContain("policy_id,rotation_version,scenario_id");
    expect(csv).toContain('mage-burst,wow,mage,,damage,60,"spell, ""burst"""');
    expect(csv).toContain(",dps,3,0,12,20,28\n");
    expect(csv.endsWith("\n")).toBe(true);
    expect(csv.endsWith("\n\n")).toBe(false);
    expect(csv).not.toContain("\r");
  });

  it("renders separate role metrics instead of a composite ranking", () => {
    const markdown = renderBenchmarkMatrixMarkdown(matrix());

    expect(markdown).toContain("## Damage");
    expect(markdown).toContain("DPS: 20 [12–28]");
    expect(markdown).toContain("1 invalid run(s) are excluded");
    expect(markdown).toContain("does not calculate a composite score");
    expect(markdown).toContain("rotation: `playerbots-inspired-v1`");
    expect(markdown.endsWith("\n")).toBe(true);
    expect(markdown.endsWith("\n\n")).toBe(false);
  });
});