import type { PartyRole } from "../partyRoles"; import type { BenchmarkMatrixReport, BenchmarkRunMetrics, BenchmarkRunReport, BenchmarkScenarioFamily, } from "./benchmarkTypes"; export const BENCHMARK_METRIC_KEYS = Object.freeze([ "rawDamage", "effectiveDamage", "priorityTargetDamage", "effectiveHealing", "rawHealing", "overheal", "absorbsApplied", "absorbsConsumed", "rawDamageTaken", "effectiveDamageTaken", "incomingAttacks", "avoidedAttacks", "deaths", "minimumPlayerHealthPercent", "minimumPartyHealthPercent", "castsStarted", "castsCompleted", "movementInterrupts", "invalidCasts", "distanceMoved", "movementTimeMs", "stackCompliancePercent", "threatControlPercent", "timeToResourceFailureMs", "timeToFirstDeathMs", "dps", "hps", "dtps", "overhealPercent", "mitigationPercent", "avoidancePercent", ] as const satisfies readonly (keyof BenchmarkRunMetrics)[]); export type BenchmarkMetricKey = (typeof BENCHMARK_METRIC_KEYS)[number]; export interface BenchmarkDistribution { /** Number of validated runs with a finite observation for this metric. */ readonly count: number; /** Validated runs where the metric was null or non-finite. */ readonly nullCount: number; readonly median: number | null; readonly p10: number | null; readonly p90: number | null; } export interface BenchmarkMatrixSummaryRow { readonly scenarioId: string; readonly scenarioVersion: number; readonly scenarioName: string; readonly scenarioFamily: BenchmarkScenarioFamily; readonly buildId: string; readonly source: BenchmarkRunReport["build"]["source"]; readonly classId: BenchmarkRunReport["build"]["classId"]; readonly secondaryClassId: BenchmarkRunReport["build"]["secondaryClassId"]; readonly role: PartyRole; readonly level: number; readonly orientation: string; readonly gearPresetId: string; readonly runCount: number; readonly validRunCount: number; readonly invalidRunCount: number; readonly completedRunCount: number; readonly metrics: Readonly>; } export interface BenchmarkMatrixSummary { readonly schemaVersion: 1; readonly generatedAt: string; readonly appVersion: string; readonly cohort: BenchmarkMatrixReport["cohort"]; readonly runCount: number; readonly validRunCount: number; readonly invalidRunCount: number; readonly rows: readonly BenchmarkMatrixSummaryRow[]; } export const ROLE_BENCHMARK_METRICS = Object.freeze({ damage: Object.freeze([ "dps", "priorityTargetDamage", "deaths", "timeToFirstDeathMs", ]), tank: Object.freeze([ "dtps", "mitigationPercent", "avoidancePercent", "threatControlPercent", "minimumPlayerHealthPercent", "deaths", "timeToFirstDeathMs", "dps", ]), healer: Object.freeze([ "hps", "absorbsConsumed", "overhealPercent", "minimumPartyHealthPercent", "deaths", "timeToResourceFailureMs", ]), } as const satisfies Readonly>); const MECHANIC_METRICS = Object.freeze([ "movementInterrupts", "stackCompliancePercent", ] as const satisfies readonly BenchmarkMetricKey[]); const ROLE_ORDER: Readonly> = Object.freeze({ tank: 0, healer: 1, damage: 2, }); const ROLE_LABELS: Readonly> = Object.freeze({ tank: "Tank", healer: "Healer", damage: "Damage", }); const METRIC_LABELS: Readonly> = Object.freeze({ rawDamage: "Raw damage", effectiveDamage: "Effective damage", priorityTargetDamage: "Priority damage", effectiveHealing: "Effective healing", rawHealing: "Raw healing", overheal: "Overheal", absorbsApplied: "Absorbs applied", absorbsConsumed: "Absorbs consumed", rawDamageTaken: "Raw damage taken", effectiveDamageTaken: "Effective damage taken", incomingAttacks: "Incoming attacks", avoidedAttacks: "Avoided attacks", deaths: "Deaths", minimumPlayerHealthPercent: "Minimum player health", minimumPartyHealthPercent: "Minimum party health", castsStarted: "Casts started", castsCompleted: "Casts completed", movementInterrupts: "Movement interrupts", invalidCasts: "Invalid casts", distanceMoved: "Distance moved", movementTimeMs: "Movement time", stackCompliancePercent: "Stack compliance", threatControlPercent: "Threat control", timeToResourceFailureMs: "Resource failure", timeToFirstDeathMs: "First death", dps: "DPS", hps: "HPS", dtps: "DTPS", overhealPercent: "Overheal", mitigationPercent: "Mitigation", avoidancePercent: "Avoidance", }); type BenchmarkReportInput = BenchmarkMatrixReport | BenchmarkMatrixSummary; interface MutableAggregationGroup { readonly representative: BenchmarkRunReport; readonly runs: BenchmarkRunReport[]; } function finiteValues(values: readonly (number | null)[]): number[] { return values.filter((value): value is number => value !== null && Number.isFinite(value)); } /** * Returns a linearly interpolated quantile after dropping null and non-finite values. */ export function benchmarkQuantile( values: readonly (number | null)[], quantile: number, ): number | null { if (quantile < 0 || quantile > 1 || !Number.isFinite(quantile)) { throw new RangeError(`Benchmark quantile must be between 0 and 1; received ${quantile}.`); } const sorted = finiteValues(values).sort((left, right) => left - right); if (sorted.length === 0) return null; if (sorted.length === 1) return sorted[0]; const rank = (sorted.length - 1) * quantile; const lowerIndex = Math.floor(rank); const upperIndex = Math.ceil(rank); const lower = sorted[lowerIndex]; const upper = sorted[upperIndex]; if (lowerIndex === upperIndex) return lower; return lower + ((upper - lower) * (rank - lowerIndex)); } function summarizeMetric( runs: readonly BenchmarkRunReport[], metric: BenchmarkMetricKey, ): BenchmarkDistribution { const values = runs.map((run) => run.metrics[metric]); const observed = finiteValues(values); return { count: observed.length, nullCount: values.length - observed.length, median: benchmarkQuantile(observed, 0.5), p10: benchmarkQuantile(observed, 0.1), p90: benchmarkQuantile(observed, 0.9), }; } function aggregationKey(run: BenchmarkRunReport): string { return JSON.stringify([ run.scenario.id, run.scenario.version, run.build.id, run.build.source, run.build.classId, run.build.secondaryClassId ?? null, run.build.role, run.build.level, run.build.orientation, run.build.gearPresetId, ]); } function compareSummaryRows( left: BenchmarkMatrixSummaryRow, right: BenchmarkMatrixSummaryRow, ): number { return left.scenarioId.localeCompare(right.scenarioId) || left.scenarioVersion - right.scenarioVersion || ROLE_ORDER[left.role] - ROLE_ORDER[right.role] || left.source.localeCompare(right.source) || left.classId.localeCompare(right.classId) || (left.secondaryClassId ?? "").localeCompare(right.secondaryClassId ?? "") || left.buildId.localeCompare(right.buildId); } export function aggregateBenchmarkMatrix(report: BenchmarkMatrixReport): BenchmarkMatrixSummary { const groups = new Map(); for (const run of report.runs) { const key = aggregationKey(run); const existing = groups.get(key); if (existing) { existing.runs.push(run); } else { groups.set(key, { representative: run, runs: [run] }); } } const rows = [...groups.values()].map(({ representative, runs }) => { const validRuns = runs.filter((run) => run.validation.valid); const metrics = Object.fromEntries(BENCHMARK_METRIC_KEYS.map((metric) => ( [metric, summarizeMetric(validRuns, metric)] ))) as unknown as Readonly>; return { scenarioId: representative.scenario.id, scenarioVersion: representative.scenario.version, scenarioName: representative.scenario.name, scenarioFamily: representative.scenario.family, buildId: representative.build.id, source: representative.build.source, classId: representative.build.classId, secondaryClassId: representative.build.secondaryClassId ?? null, role: representative.build.role, level: representative.build.level, orientation: representative.build.orientation, gearPresetId: representative.build.gearPresetId, runCount: runs.length, validRunCount: validRuns.length, invalidRunCount: runs.length - validRuns.length, completedRunCount: runs.filter((run) => run.completed).length, metrics, } satisfies BenchmarkMatrixSummaryRow; }).sort(compareSummaryRows); const validRunCount = report.runs.filter((run) => run.validation.valid).length; return { schemaVersion: 1, generatedAt: report.generatedAt, appVersion: report.appVersion, cohort: report.cohort, runCount: report.runs.length, validRunCount, invalidRunCount: report.runs.length - validRunCount, rows, }; } function asSummary(input: BenchmarkReportInput): BenchmarkMatrixSummary { return "rows" in input ? input : aggregateBenchmarkMatrix(input); } function csvCell(value: string | number | null): string { if (value === null) return ""; const text = String(value); return /[",\r\n]/u.test(text) ? `"${text.replaceAll('"', '""')}"` : text; } function csvLine(values: readonly (string | number | null)[]): string { return values.map(csvCell).join(","); } /** Produces one CSV row per scenario/build/metric, retaining the complete metric vector. */ export function serializeBenchmarkMatrixCsv(input: BenchmarkReportInput): string { const summary = asSummary(input); const lines = [csvLine([ "schema_version", "generated_at", "app_version", "cohort_level", "cohort_gear_preset_id", "seeds", "duration_scale", "policy_id", "rotation_version", "scenario_id", "scenario_version", "scenario_name", "scenario_family", "build_id", "source", "class_id", "secondary_class_id", "role", "build_level", "orientation", "gear_preset_id", "run_count", "valid_run_count", "invalid_run_count", "completed_run_count", "metric", "observed_count", "null_count", "p10", "median", "p90", ])]; for (const row of summary.rows) { for (const metric of BENCHMARK_METRIC_KEYS) { const distribution = row.metrics[metric]; lines.push(csvLine([ summary.schemaVersion, summary.generatedAt, summary.appVersion, summary.cohort.level, summary.cohort.gearPresetId, summary.cohort.seeds.join(";"), summary.cohort.durationScale, summary.cohort.policyId, summary.cohort.rotationVersion ?? null, row.scenarioId, row.scenarioVersion, row.scenarioName, row.scenarioFamily, row.buildId, row.source, row.classId, row.secondaryClassId ?? null, row.role, row.level, row.orientation, row.gearPresetId, row.runCount, row.validRunCount, row.invalidRunCount, row.completedRunCount, metric, distribution.count, distribution.nullCount, distribution.p10, distribution.median, distribution.p90, ])); } } return `${lines.join("\n")}\n`; } function markdownCell(value: string): string { return value.replaceAll("|", "\\|").replace(/[\r\n]+/gu, " "); } function formatNumber(value: number): string { if (Number.isInteger(value)) return value.toLocaleString("en-US"); return value.toLocaleString("en-US", { maximumFractionDigits: 2 }); } function formatMetricValue(metric: BenchmarkMetricKey, value: number): string { if (metric.endsWith("Percent")) return `${formatNumber(value)}%`; if (metric.endsWith("Ms")) return `${formatNumber(value / 1_000)}s`; return formatNumber(value); } function formatDistribution(metric: BenchmarkMetricKey, value: BenchmarkDistribution): string { if (value.median === null || value.p10 === null || value.p90 === null) { return value.nullCount > 0 ? `not observed (0/${value.nullCount})` : "no valid runs"; } const range = `${formatMetricValue(metric, value.p10)}–${formatMetricValue(metric, value.p90)}`; const observation = value.nullCount > 0 ? `; observed ${value.count}/${value.count + value.nullCount}` : ""; return `${formatMetricValue(metric, value.median)} [${range}]${observation}`; } function roleMetricsFor(row: BenchmarkMatrixSummaryRow): readonly BenchmarkMetricKey[] { if (row.scenarioFamily !== "mechanics") return ROLE_BENCHMARK_METRICS[row.role]; return [...ROLE_BENCHMARK_METRICS[row.role], ...MECHANIC_METRICS]; } function formatRoleVector(row: BenchmarkMatrixSummaryRow): string { return roleMetricsFor(row).map((metric) => ( `${METRIC_LABELS[metric]}: ${formatDistribution(metric, row.metrics[metric])}` )).join("; "); } /** Renders a compact, role-separated view without collapsing metrics into a score. */ export function renderBenchmarkMatrixMarkdown(input: BenchmarkReportInput): string { const summary = asSummary(input); const lines = [ "# Balance benchmark summary", "", `Generated ${summary.generatedAt} with app ${summary.appVersion}. ` + `Cohort: level ${summary.cohort.level}, gear \`${summary.cohort.gearPresetId}\`, ` + `${summary.cohort.seeds.length} seed(s), duration scale ${summary.cohort.durationScale}.`, `Policy: \`${summary.cohort.policyId}\`${summary.cohort.rotationVersion ? `; rotation: \`${summary.cohort.rotationVersion}\`` : ""}.`, "", `${summary.validRunCount}/${summary.runCount} runs were valid; ` + `${summary.invalidRunCount} invalid run(s) are excluded from distributions. ` + "Values are median [p10–p90].", ]; for (const role of ["tank", "healer", "damage"] as const) { const roleRows = summary.rows.filter((row) => row.role === role); if (roleRows.length === 0) continue; lines.push( "", `## ${ROLE_LABELS[role]}`, "", "| Scenario | Build | Runs | Role vector |", "| --- | --- | ---: | --- |", ); for (const row of roleRows) { const scenario = `${row.scenarioName} (v${row.scenarioVersion})`; const secondary = row.secondaryClassId ? ` / ${row.secondaryClassId}` : ""; const build = `${row.buildId} — ${row.source}:${row.classId}${secondary}`; const runs = `${row.validRunCount}/${row.runCount}`; lines.push( `| ${markdownCell(scenario)} | ${markdownCell(build)} | ${runs} | ` + `${markdownCell(formatRoleVector(row))} |`, ); } } if (summary.rows.length === 0) { lines.push("", "No benchmark runs were supplied."); } lines.push( "", "Compare builds only within the same scenario version and cohort. " + "Role metrics remain separate; this report does not calculate a composite score.", ); return `${lines.join("\n")}\n`; }