forked from phenom/healer-man
478 lines
15 KiB
TypeScript
478 lines
15 KiB
TypeScript
import type { PartyRole } from "../partyRoles";
|
||
import type {
|
||
BenchmarkMatrixReport,
|
||
BenchmarkRunMetrics,
|
||
BenchmarkRunReport,
|
||
BenchmarkScenarioFamily,
|
||
} from "./benchmarkTypes";
|
||
|
||
export const BENCHMARK_METRIC_KEYS = Object.freeze([
|
||
"rawDamage",
|
||
"effectiveDamage",
|
||
"priorityTargetDamage",
|
||
"effectiveHealing",
|
||
"rawHealing",
|
||
"overheal",
|
||
"absorbsApplied",
|
||
"absorbsConsumed",
|
||
"rawDamageTaken",
|
||
"effectiveDamageTaken",
|
||
"incomingAttacks",
|
||
"avoidedAttacks",
|
||
"deaths",
|
||
"minimumPlayerHealthPercent",
|
||
"minimumPartyHealthPercent",
|
||
"castsStarted",
|
||
"castsCompleted",
|
||
"movementInterrupts",
|
||
"invalidCasts",
|
||
"distanceMoved",
|
||
"movementTimeMs",
|
||
"stackCompliancePercent",
|
||
"threatControlPercent",
|
||
"timeToResourceFailureMs",
|
||
"timeToFirstDeathMs",
|
||
"dps",
|
||
"hps",
|
||
"dtps",
|
||
"overhealPercent",
|
||
"mitigationPercent",
|
||
"avoidancePercent",
|
||
] as const satisfies readonly (keyof BenchmarkRunMetrics)[]);
|
||
|
||
export type BenchmarkMetricKey = (typeof BENCHMARK_METRIC_KEYS)[number];
|
||
|
||
export interface BenchmarkDistribution {
|
||
/** Number of validated runs with a finite observation for this metric. */
|
||
readonly count: number;
|
||
/** Validated runs where the metric was null or non-finite. */
|
||
readonly nullCount: number;
|
||
readonly median: number | null;
|
||
readonly p10: number | null;
|
||
readonly p90: number | null;
|
||
}
|
||
|
||
export interface BenchmarkMatrixSummaryRow {
|
||
readonly scenarioId: string;
|
||
readonly scenarioVersion: number;
|
||
readonly scenarioName: string;
|
||
readonly scenarioFamily: BenchmarkScenarioFamily;
|
||
readonly buildId: string;
|
||
readonly source: BenchmarkRunReport["build"]["source"];
|
||
readonly classId: BenchmarkRunReport["build"]["classId"];
|
||
readonly secondaryClassId: BenchmarkRunReport["build"]["secondaryClassId"];
|
||
readonly role: PartyRole;
|
||
readonly level: number;
|
||
readonly orientation: string;
|
||
readonly gearPresetId: string;
|
||
readonly runCount: number;
|
||
readonly validRunCount: number;
|
||
readonly invalidRunCount: number;
|
||
readonly completedRunCount: number;
|
||
readonly metrics: Readonly<Record<BenchmarkMetricKey, BenchmarkDistribution>>;
|
||
}
|
||
|
||
export interface BenchmarkMatrixSummary {
|
||
readonly schemaVersion: 1;
|
||
readonly generatedAt: string;
|
||
readonly appVersion: string;
|
||
readonly cohort: BenchmarkMatrixReport["cohort"];
|
||
readonly runCount: number;
|
||
readonly validRunCount: number;
|
||
readonly invalidRunCount: number;
|
||
readonly rows: readonly BenchmarkMatrixSummaryRow[];
|
||
}
|
||
|
||
export const ROLE_BENCHMARK_METRICS = Object.freeze({
|
||
damage: Object.freeze([
|
||
"dps",
|
||
"priorityTargetDamage",
|
||
"deaths",
|
||
"timeToFirstDeathMs",
|
||
]),
|
||
tank: Object.freeze([
|
||
"dtps",
|
||
"mitigationPercent",
|
||
"avoidancePercent",
|
||
"threatControlPercent",
|
||
"minimumPlayerHealthPercent",
|
||
"deaths",
|
||
"timeToFirstDeathMs",
|
||
"dps",
|
||
]),
|
||
healer: Object.freeze([
|
||
"hps",
|
||
"absorbsConsumed",
|
||
"overhealPercent",
|
||
"minimumPartyHealthPercent",
|
||
"deaths",
|
||
"timeToResourceFailureMs",
|
||
]),
|
||
} as const satisfies Readonly<Record<PartyRole, readonly BenchmarkMetricKey[]>>);
|
||
|
||
const MECHANIC_METRICS = Object.freeze([
|
||
"movementInterrupts",
|
||
"stackCompliancePercent",
|
||
] as const satisfies readonly BenchmarkMetricKey[]);
|
||
|
||
const ROLE_ORDER: Readonly<Record<PartyRole, number>> = Object.freeze({
|
||
tank: 0,
|
||
healer: 1,
|
||
damage: 2,
|
||
});
|
||
|
||
const ROLE_LABELS: Readonly<Record<PartyRole, string>> = Object.freeze({
|
||
tank: "Tank",
|
||
healer: "Healer",
|
||
damage: "Damage",
|
||
});
|
||
|
||
const METRIC_LABELS: Readonly<Record<BenchmarkMetricKey, string>> = Object.freeze({
|
||
rawDamage: "Raw damage",
|
||
effectiveDamage: "Effective damage",
|
||
priorityTargetDamage: "Priority damage",
|
||
effectiveHealing: "Effective healing",
|
||
rawHealing: "Raw healing",
|
||
overheal: "Overheal",
|
||
absorbsApplied: "Absorbs applied",
|
||
absorbsConsumed: "Absorbs consumed",
|
||
rawDamageTaken: "Raw damage taken",
|
||
effectiveDamageTaken: "Effective damage taken",
|
||
incomingAttacks: "Incoming attacks",
|
||
avoidedAttacks: "Avoided attacks",
|
||
deaths: "Deaths",
|
||
minimumPlayerHealthPercent: "Minimum player health",
|
||
minimumPartyHealthPercent: "Minimum party health",
|
||
castsStarted: "Casts started",
|
||
castsCompleted: "Casts completed",
|
||
movementInterrupts: "Movement interrupts",
|
||
invalidCasts: "Invalid casts",
|
||
distanceMoved: "Distance moved",
|
||
movementTimeMs: "Movement time",
|
||
stackCompliancePercent: "Stack compliance",
|
||
threatControlPercent: "Threat control",
|
||
timeToResourceFailureMs: "Resource failure",
|
||
timeToFirstDeathMs: "First death",
|
||
dps: "DPS",
|
||
hps: "HPS",
|
||
dtps: "DTPS",
|
||
overhealPercent: "Overheal",
|
||
mitigationPercent: "Mitigation",
|
||
avoidancePercent: "Avoidance",
|
||
});
|
||
|
||
type BenchmarkReportInput = BenchmarkMatrixReport | BenchmarkMatrixSummary;
|
||
|
||
interface MutableAggregationGroup {
|
||
readonly representative: BenchmarkRunReport;
|
||
readonly runs: BenchmarkRunReport[];
|
||
}
|
||
|
||
function finiteValues(values: readonly (number | null)[]): number[] {
|
||
return values.filter((value): value is number => value !== null && Number.isFinite(value));
|
||
}
|
||
|
||
/**
|
||
* Returns a linearly interpolated quantile after dropping null and non-finite values.
|
||
*/
|
||
export function benchmarkQuantile(
|
||
values: readonly (number | null)[],
|
||
quantile: number,
|
||
): number | null {
|
||
if (quantile < 0 || quantile > 1 || !Number.isFinite(quantile)) {
|
||
throw new RangeError(`Benchmark quantile must be between 0 and 1; received ${quantile}.`);
|
||
}
|
||
|
||
const sorted = finiteValues(values).sort((left, right) => left - right);
|
||
if (sorted.length === 0) return null;
|
||
if (sorted.length === 1) return sorted[0];
|
||
|
||
const rank = (sorted.length - 1) * quantile;
|
||
const lowerIndex = Math.floor(rank);
|
||
const upperIndex = Math.ceil(rank);
|
||
const lower = sorted[lowerIndex];
|
||
const upper = sorted[upperIndex];
|
||
if (lowerIndex === upperIndex) return lower;
|
||
return lower + ((upper - lower) * (rank - lowerIndex));
|
||
}
|
||
|
||
function summarizeMetric(
|
||
runs: readonly BenchmarkRunReport[],
|
||
metric: BenchmarkMetricKey,
|
||
): BenchmarkDistribution {
|
||
const values = runs.map((run) => run.metrics[metric]);
|
||
const observed = finiteValues(values);
|
||
return {
|
||
count: observed.length,
|
||
nullCount: values.length - observed.length,
|
||
median: benchmarkQuantile(observed, 0.5),
|
||
p10: benchmarkQuantile(observed, 0.1),
|
||
p90: benchmarkQuantile(observed, 0.9),
|
||
};
|
||
}
|
||
|
||
function aggregationKey(run: BenchmarkRunReport): string {
|
||
return JSON.stringify([
|
||
run.scenario.id,
|
||
run.scenario.version,
|
||
run.build.id,
|
||
run.build.source,
|
||
run.build.classId,
|
||
run.build.secondaryClassId ?? null,
|
||
run.build.role,
|
||
run.build.level,
|
||
run.build.orientation,
|
||
run.build.gearPresetId,
|
||
]);
|
||
}
|
||
|
||
function compareSummaryRows(
|
||
left: BenchmarkMatrixSummaryRow,
|
||
right: BenchmarkMatrixSummaryRow,
|
||
): number {
|
||
return left.scenarioId.localeCompare(right.scenarioId)
|
||
|| left.scenarioVersion - right.scenarioVersion
|
||
|| ROLE_ORDER[left.role] - ROLE_ORDER[right.role]
|
||
|| left.source.localeCompare(right.source)
|
||
|| left.classId.localeCompare(right.classId)
|
||
|| (left.secondaryClassId ?? "").localeCompare(right.secondaryClassId ?? "")
|
||
|| left.buildId.localeCompare(right.buildId);
|
||
}
|
||
|
||
export function aggregateBenchmarkMatrix(report: BenchmarkMatrixReport): BenchmarkMatrixSummary {
|
||
const groups = new Map<string, MutableAggregationGroup>();
|
||
|
||
for (const run of report.runs) {
|
||
const key = aggregationKey(run);
|
||
const existing = groups.get(key);
|
||
if (existing) {
|
||
existing.runs.push(run);
|
||
} else {
|
||
groups.set(key, { representative: run, runs: [run] });
|
||
}
|
||
}
|
||
|
||
const rows = [...groups.values()].map(({ representative, runs }) => {
|
||
const validRuns = runs.filter((run) => run.validation.valid);
|
||
const metrics = Object.fromEntries(BENCHMARK_METRIC_KEYS.map((metric) => (
|
||
[metric, summarizeMetric(validRuns, metric)]
|
||
))) as unknown as Readonly<Record<BenchmarkMetricKey, BenchmarkDistribution>>;
|
||
|
||
return {
|
||
scenarioId: representative.scenario.id,
|
||
scenarioVersion: representative.scenario.version,
|
||
scenarioName: representative.scenario.name,
|
||
scenarioFamily: representative.scenario.family,
|
||
buildId: representative.build.id,
|
||
source: representative.build.source,
|
||
classId: representative.build.classId,
|
||
secondaryClassId: representative.build.secondaryClassId ?? null,
|
||
role: representative.build.role,
|
||
level: representative.build.level,
|
||
orientation: representative.build.orientation,
|
||
gearPresetId: representative.build.gearPresetId,
|
||
runCount: runs.length,
|
||
validRunCount: validRuns.length,
|
||
invalidRunCount: runs.length - validRuns.length,
|
||
completedRunCount: runs.filter((run) => run.completed).length,
|
||
metrics,
|
||
} satisfies BenchmarkMatrixSummaryRow;
|
||
}).sort(compareSummaryRows);
|
||
|
||
const validRunCount = report.runs.filter((run) => run.validation.valid).length;
|
||
return {
|
||
schemaVersion: 1,
|
||
generatedAt: report.generatedAt,
|
||
appVersion: report.appVersion,
|
||
cohort: report.cohort,
|
||
runCount: report.runs.length,
|
||
validRunCount,
|
||
invalidRunCount: report.runs.length - validRunCount,
|
||
rows,
|
||
};
|
||
}
|
||
|
||
function asSummary(input: BenchmarkReportInput): BenchmarkMatrixSummary {
|
||
return "rows" in input ? input : aggregateBenchmarkMatrix(input);
|
||
}
|
||
|
||
function csvCell(value: string | number | null): string {
|
||
if (value === null) return "";
|
||
const text = String(value);
|
||
return /[",\r\n]/u.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
|
||
}
|
||
|
||
function csvLine(values: readonly (string | number | null)[]): string {
|
||
return values.map(csvCell).join(",");
|
||
}
|
||
|
||
/** Produces one CSV row per scenario/build/metric, retaining the complete metric vector. */
|
||
export function serializeBenchmarkMatrixCsv(input: BenchmarkReportInput): string {
|
||
const summary = asSummary(input);
|
||
const lines = [csvLine([
|
||
"schema_version",
|
||
"generated_at",
|
||
"app_version",
|
||
"cohort_level",
|
||
"cohort_gear_preset_id",
|
||
"seeds",
|
||
"duration_scale",
|
||
"policy_id",
|
||
"rotation_version",
|
||
"scenario_id",
|
||
"scenario_version",
|
||
"scenario_name",
|
||
"scenario_family",
|
||
"build_id",
|
||
"source",
|
||
"class_id",
|
||
"secondary_class_id",
|
||
"role",
|
||
"build_level",
|
||
"orientation",
|
||
"gear_preset_id",
|
||
"run_count",
|
||
"valid_run_count",
|
||
"invalid_run_count",
|
||
"completed_run_count",
|
||
"metric",
|
||
"observed_count",
|
||
"null_count",
|
||
"p10",
|
||
"median",
|
||
"p90",
|
||
])];
|
||
|
||
for (const row of summary.rows) {
|
||
for (const metric of BENCHMARK_METRIC_KEYS) {
|
||
const distribution = row.metrics[metric];
|
||
lines.push(csvLine([
|
||
summary.schemaVersion,
|
||
summary.generatedAt,
|
||
summary.appVersion,
|
||
summary.cohort.level,
|
||
summary.cohort.gearPresetId,
|
||
summary.cohort.seeds.join(";"),
|
||
summary.cohort.durationScale,
|
||
summary.cohort.policyId,
|
||
summary.cohort.rotationVersion ?? null,
|
||
row.scenarioId,
|
||
row.scenarioVersion,
|
||
row.scenarioName,
|
||
row.scenarioFamily,
|
||
row.buildId,
|
||
row.source,
|
||
row.classId,
|
||
row.secondaryClassId ?? null,
|
||
row.role,
|
||
row.level,
|
||
row.orientation,
|
||
row.gearPresetId,
|
||
row.runCount,
|
||
row.validRunCount,
|
||
row.invalidRunCount,
|
||
row.completedRunCount,
|
||
metric,
|
||
distribution.count,
|
||
distribution.nullCount,
|
||
distribution.p10,
|
||
distribution.median,
|
||
distribution.p90,
|
||
]));
|
||
}
|
||
}
|
||
|
||
return `${lines.join("\n")}\n`;
|
||
}
|
||
|
||
function markdownCell(value: string): string {
|
||
return value.replaceAll("|", "\\|").replace(/[\r\n]+/gu, " ");
|
||
}
|
||
|
||
function formatNumber(value: number): string {
|
||
if (Number.isInteger(value)) return value.toLocaleString("en-US");
|
||
return value.toLocaleString("en-US", { maximumFractionDigits: 2 });
|
||
}
|
||
|
||
function formatMetricValue(metric: BenchmarkMetricKey, value: number): string {
|
||
if (metric.endsWith("Percent")) return `${formatNumber(value)}%`;
|
||
if (metric.endsWith("Ms")) return `${formatNumber(value / 1_000)}s`;
|
||
return formatNumber(value);
|
||
}
|
||
|
||
function formatDistribution(metric: BenchmarkMetricKey, value: BenchmarkDistribution): string {
|
||
if (value.median === null || value.p10 === null || value.p90 === null) {
|
||
return value.nullCount > 0 ? `not observed (0/${value.nullCount})` : "no valid runs";
|
||
}
|
||
|
||
const range = `${formatMetricValue(metric, value.p10)}–${formatMetricValue(metric, value.p90)}`;
|
||
const observation = value.nullCount > 0
|
||
? `; observed ${value.count}/${value.count + value.nullCount}`
|
||
: "";
|
||
return `${formatMetricValue(metric, value.median)} [${range}]${observation}`;
|
||
}
|
||
|
||
function roleMetricsFor(row: BenchmarkMatrixSummaryRow): readonly BenchmarkMetricKey[] {
|
||
if (row.scenarioFamily !== "mechanics") return ROLE_BENCHMARK_METRICS[row.role];
|
||
return [...ROLE_BENCHMARK_METRICS[row.role], ...MECHANIC_METRICS];
|
||
}
|
||
|
||
function formatRoleVector(row: BenchmarkMatrixSummaryRow): string {
|
||
return roleMetricsFor(row).map((metric) => (
|
||
`${METRIC_LABELS[metric]}: ${formatDistribution(metric, row.metrics[metric])}`
|
||
)).join("; ");
|
||
}
|
||
|
||
/** Renders a compact, role-separated view without collapsing metrics into a score. */
|
||
export function renderBenchmarkMatrixMarkdown(input: BenchmarkReportInput): string {
|
||
const summary = asSummary(input);
|
||
const lines = [
|
||
"# Balance benchmark summary",
|
||
"",
|
||
`Generated ${summary.generatedAt} with app ${summary.appVersion}. `
|
||
+ `Cohort: level ${summary.cohort.level}, gear \`${summary.cohort.gearPresetId}\`, `
|
||
+ `${summary.cohort.seeds.length} seed(s), duration scale ${summary.cohort.durationScale}.`,
|
||
`Policy: \`${summary.cohort.policyId}\`${summary.cohort.rotationVersion
|
||
? `; rotation: \`${summary.cohort.rotationVersion}\``
|
||
: ""}.`,
|
||
"",
|
||
`${summary.validRunCount}/${summary.runCount} runs were valid; `
|
||
+ `${summary.invalidRunCount} invalid run(s) are excluded from distributions. `
|
||
+ "Values are median [p10–p90].",
|
||
];
|
||
|
||
for (const role of ["tank", "healer", "damage"] as const) {
|
||
const roleRows = summary.rows.filter((row) => row.role === role);
|
||
if (roleRows.length === 0) continue;
|
||
|
||
lines.push(
|
||
"",
|
||
`## ${ROLE_LABELS[role]}`,
|
||
"",
|
||
"| Scenario | Build | Runs | Role vector |",
|
||
"| --- | --- | ---: | --- |",
|
||
);
|
||
for (const row of roleRows) {
|
||
const scenario = `${row.scenarioName} (v${row.scenarioVersion})`;
|
||
const secondary = row.secondaryClassId ? ` / ${row.secondaryClassId}` : "";
|
||
const build = `${row.buildId} — ${row.source}:${row.classId}${secondary}`;
|
||
const runs = `${row.validRunCount}/${row.runCount}`;
|
||
lines.push(
|
||
`| ${markdownCell(scenario)} | ${markdownCell(build)} | ${runs} | `
|
||
+ `${markdownCell(formatRoleVector(row))} |`,
|
||
);
|
||
}
|
||
}
|
||
|
||
if (summary.rows.length === 0) {
|
||
lines.push("", "No benchmark runs were supplied.");
|
||
}
|
||
|
||
lines.push(
|
||
"",
|
||
"Compare builds only within the same scenario version and cohort. "
|
||
+ "Role metrics remain separate; this report does not calculate a composite score.",
|
||
);
|
||
return `${lines.join("\n")}\n`;
|
||
}
|