import { randomUUID } from "node:crypto"; import { spawn, spawnSync } from "node:child_process"; import { createReadStream, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync, } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { GameDatabaseError } from "./database.mjs"; const SOURCES = new Set(["wow", "coa", "rom", "all"]); const BOSS_CATEGORIES = new Set(["all", "wow", "rom"]); const SPEC_SELECTIONS = new Set(["random", "all", "specific"]); const JOB_STATUSES = new Set(["queued", "running", "complete", "failed"]); const MAX_BATCH_JOBS = 80; const JOB_METADATA_NAME = "job.json"; const JOB_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const ARTIFACTS = new Map([ ["suite-report.html", "text/html; charset=utf-8"], ["suite-report.json", "application/json; charset=utf-8"], ["suite-report.md", "text/markdown; charset=utf-8"], ["party_configurations.csv", "text/csv; charset=utf-8"], ["class_specs_across_parties.csv", "text/csv; charset=utf-8"], ["class_abilities_across_parties.csv", "text/csv; charset=utf-8"], ["npc_class_specs_across_parties.csv", "text/csv; charset=utf-8"], ["npc_class_abilities_across_parties.csv", "text/csv; charset=utf-8"], ["class_spec_by_boss.csv", "text/csv; charset=utf-8"], ["bosses_across_parties.csv", "text/csv; charset=utf-8"], ["boss_by_party.csv", "text/csv; charset=utf-8"], ["suite_runs.csv", "text/csv; charset=utf-8"], ["suite-config.json", "application/json; charset=utf-8"], ]); function integer(value, name, minimum, maximum) { const parsed = Number(value); if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) { throw new GameDatabaseError(`${name} must be a whole number from ${minimum} through ${maximum}.`, 400, "invalid_balance_lab_job"); } return parsed; } export function normalizeBalanceLabJobSpec(input) { const source = String(input?.source ?? "all").toLowerCase(); const bossCategory = String(input?.bossCategory ?? "all").toLowerCase(); const specSelection = String(input?.specSelection ?? "random").toLowerCase(); if (!SOURCES.has(source)) { throw new GameDatabaseError("Party source must be wow, coa, rom, or all.", 400, "invalid_balance_lab_job"); } if (!BOSS_CATEGORIES.has(bossCategory)) { throw new GameDatabaseError("Boss category must be all, wow, or rom.", 400, "invalid_balance_lab_job"); } if (!SPEC_SELECTIONS.has(specSelection)) { throw new GameDatabaseError("Class/spec selection must be random, all, or specific.", 400, "invalid_balance_lab_job"); } if (input?.includeBetaSpecs !== undefined && typeof input.includeBetaSpecs !== "boolean") { throw new GameDatabaseError("Include Beta Specs must be boolean.", 400, "invalid_balance_lab_job"); } const rawBossKeys = Array.isArray(input?.bossKeys) ? input.bossKeys : []; if (rawBossKeys.length > 500) { throw new GameDatabaseError("Select no more than 500 bosses.", 400, "invalid_balance_lab_job"); } const bossKeys = [...new Set(rawBossKeys.map((value) => String(value).trim()).filter(Boolean))]; if (bossKeys.some((key) => key.length > 240 || !/^[A-Za-z0-9._:/-]+$/.test(key))) { throw new GameDatabaseError("A selected boss key is invalid.", 400, "invalid_balance_lab_job"); } const rawBuildIds = Array.isArray(input?.selectedBuildIds) ? input.selectedBuildIds : []; if (rawBuildIds.length > 200) { throw new GameDatabaseError("Select no more than 200 class/spec builds.", 400, "invalid_balance_lab_job"); } const selectedBuildIds = [...new Set(rawBuildIds.map((value) => String(value).trim()).filter(Boolean))]; if (selectedBuildIds.some((id) => id.length > 240 || !/^[A-Za-z0-9._:/-]+$/.test(id))) { throw new GameDatabaseError("A selected class/spec build ID is invalid.", 400, "invalid_balance_lab_job"); } if (specSelection === "specific" && selectedBuildIds.length === 0) { throw new GameDatabaseError("Select at least one class/spec build.", 400, "invalid_balance_lab_job"); } const spec = Object.freeze({ source, specSelection, selectedBuildIds: Object.freeze(specSelection === "specific" ? selectedBuildIds : []), level: integer(input?.level ?? 60, "Level", 1, 80), partyConfigurations: integer(input?.partyConfigurations ?? 10, "Party configurations", 1, 100), simulationsPerParty: integer(input?.simulationsPerParty ?? 3, "Simulations per party", 1, 100), durationSeconds: integer(input?.durationSeconds ?? 60, "Fight duration", 5, 600), partySeed: integer(input?.partySeed ?? 1, "Party seed", -2_147_483_648, 2_147_483_647), bossCategory, bossKeys: Object.freeze(bossKeys), seatSweep: input?.seatSweep === true, includeBetaSpecs: input?.includeBetaSpecs === true, }); const sourceCount = spec.source === "all" ? 3 : 1; const bossEstimate = Math.max(1, spec.bossKeys.length || (spec.bossCategory === "all" ? 500 : 300)); const partyEstimate = spec.specSelection === "random" ? sourceCount * spec.partyConfigurations : spec.specSelection === "specific" ? Math.max(spec.partyConfigurations, spec.selectedBuildIds.length) : Math.max(spec.partyConfigurations, spec.source === "all" ? 100 : 50); const estimatedRuns = partyEstimate * (spec.seatSweep ? 5 : 1) * spec.simulationsPerParty * bossEstimate; if (estimatedRuns > 250_000) { throw new GameDatabaseError( "That job is too large for one server request. Reduce parties, simulations, or selected bosses.", 400, "balance_lab_job_too_large", ); } return spec; } export function normalizeBalanceLabJobBatch(input) { const rawSpecs = Array.isArray(input?.specs) ? input.specs : []; if (rawSpecs.length < 1 || rawSpecs.length > MAX_BATCH_JOBS) { throw new GameDatabaseError( `A Balance Lab batch must contain 1 through ${MAX_BATCH_JOBS} level jobs.`, 400, "invalid_balance_lab_batch", ); } return Object.freeze(rawSpecs.map((spec) => normalizeBalanceLabJobSpec(spec))); } export function balanceLabCommand(spec, outputDir, options = {}) { const projectRoot = path.resolve(options.projectRoot ?? fileURLToPath(new URL("..", import.meta.url))); const executable = String(options.pythonExecutable ?? process.env.BALANCE_LAB_PYTHON ?? (process.platform === "win32" ? "python" : "python3")); const args = [ path.join(projectRoot, "scripts", "balance_lab.py"), "--source", spec.source, "--level", String(spec.level), "--party-mode", "automatic", "--spec-selection", spec.specSelection, "--party-seed", String(spec.partySeed), "--party-configurations", String(spec.partyConfigurations), "--simulations-per-party", String(spec.simulationsPerParty), "--duration", String(spec.durationSeconds), "--boss-category", spec.bossCategory, "--output-dir", outputDir, ]; for (const buildId of spec.selectedBuildIds) args.push("--spec", buildId); for (const bossKey of spec.bossKeys) args.push("--boss", bossKey); if (spec.seatSweep) args.push("--seat-sweep"); if (spec.includeBetaSpecs) args.push("--include-beta-specs"); if (options.cooldownPolicyPath) args.push("--cooldown-policy", options.cooldownPolicyPath); if (options.readinessPolicyPath) args.push("--companion-readiness-policy", options.readinessPolicyPath); return { executable, args, cwd: projectRoot }; } export function inspectBalanceLabRuntime(executable, run = spawnSync) { const command = String(executable || (process.platform === "win32" ? "python" : "python3")); try { const result = run(command, ["--version"], { encoding: "utf8", timeout: 5_000, windowsHide: true, }); const version = `${result?.stdout ?? ""} ${result?.stderr ?? ""}`.trim(); if (result?.status === 0 && /^Python 3\./i.test(version)) { return { available: true, executable: command, version, error: null }; } return { available: false, executable: command, version: version || null, error: version && /^Python 2\./i.test(version) ? "Balance Lab requires Python 3; this server resolved Python 2." : `The server could not execute ${command} as Python 3.`, }; } catch (error) { return { available: false, executable: command, version: null, error: `The server could not execute ${command}: ${error?.message ?? error}`, }; } } function publicJob(job) { return { id: job.id, status: job.status, createdAt: job.createdAt, startedAt: job.startedAt, completedAt: job.completedAt, requestedBy: job.requestedBy, spec: job.spec, progress: { ...job.progress }, artifacts: [...job.artifacts], error: job.error, logTail: [...job.logTail], summary: job.summary ? { ...job.summary } : null, }; } function reportSummary(outputDir) { const reportPath = path.join(outputDir, "suite-report.json"); if (!existsSync(reportPath)) return null; try { const summary = JSON.parse(readFileSync(reportPath, "utf8"))?.summary; if (!summary || typeof summary !== "object") return null; return { partyConfigurations: Number(summary.partyConfigurations) || 0, bosses: Number(summary.bosses) || 0, runs: Number(summary.runs) || 0, classSpecs: Number(summary.classSpecs) || 0, classAbilities: Number(summary.classAbilities) || 0, npcClassSpecs: Number(summary.npcClassSpecs) || 0, npcClassAbilities: Number(summary.npcClassAbilities) || 0, bossPressureMedianDtps: Number(summary.bossPressureMedianDtps) || 0, }; } catch { return null; } } function expectedPartyConfigurations(spec) { const parties = spec.specSelection === "random" ? spec.partyConfigurations * (spec.source === "all" ? 3 : 1) : spec.partyConfigurations; return parties * (spec.seatSweep ? 5 : 1); } function lineCollector(onLine) { let pending = ""; return { push(chunk) { pending += String(chunk); const lines = pending.split(/\r?\n/); pending = lines.pop() ?? ""; for (const line of lines) if (line.trim()) onLine(line.trim()); }, flush() { if (pending.trim()) onLine(pending.trim()); pending = ""; }, }; } function processFailureMessage(job, code) { if (job.logTail.some((line) => line.includes("ENOSPC: System limit for number of file watchers reached"))) { return "Balance Lab could not start because the server file-watcher limit was exhausted."; } return code === 0 ? "Balance Lab ended without producing its suite report." : `Balance Lab process exited with code ${code ?? "unknown"}.`; } export function createBalanceLabJobService(options = {}) { const projectRoot = path.resolve(options.projectRoot ?? fileURLToPath(new URL("..", import.meta.url))); const dataDir = path.resolve(options.dataDir ?? process.env.DATA_DIR ?? "data/runtime"); const jobsDir = path.join(dataDir, "balance-lab", "jobs"); const configuredConcurrency = Number(options.maxConcurrent ?? process.env.BALANCE_LAB_MAX_CONCURRENT ?? 4); const maxConcurrent = Number.isFinite(configuredConcurrency) && configuredConcurrency >= 1 ? Math.min(16, Math.floor(configuredConcurrency)) : 4; const configuredQueueLimit = Number(options.maxQueued ?? process.env.BALANCE_LAB_MAX_QUEUED ?? 100); const maxQueued = Number.isFinite(configuredQueueLimit) && configuredQueueLimit >= 1 ? Math.min(200, Math.floor(configuredQueueLimit)) : 100; const launch = options.spawnProcess ?? spawn; const cooldownPolicyProvider = options.cooldownPolicyProvider; const readinessPolicyProvider = options.readinessPolicyProvider; const pythonExecutable = String(options.pythonExecutable ?? process.env.BALANCE_LAB_PYTHON ?? (process.platform === "win32" ? "python" : "python3")); const runtimeProbe = options.runtimeProbe ?? (options.spawnProcess ? () => ({ available: true, executable: pythonExecutable, version: "test process", error: null }) : () => inspectBalanceLabRuntime(pythonExecutable)); const jobs = new Map(); const queue = []; const children = new Map(); let running = 0; let closed = false; let inspectedRuntime = null; mkdirSync(jobsDir, { recursive: true }); function currentRuntime() { inspectedRuntime ??= runtimeProbe(); return inspectedRuntime; } function persistJob(job) { const record = publicJob(job); writeFileSync(path.join(job.outputDir, JOB_METADATA_NAME), `${JSON.stringify(record, null, 2)}\n`, "utf8"); } function restoreJobs() { for (const entry of readdirSync(jobsDir, { withFileTypes: true })) { if (!entry.isDirectory() || !JOB_ID_PATTERN.test(entry.name)) continue; const outputDir = path.join(jobsDir, entry.name); const metadataPath = path.join(outputDir, JOB_METADATA_NAME); const reportPath = path.join(outputDir, "suite-report.json"); if (!existsSync(metadataPath) && !existsSync(reportPath)) continue; try { let stored; if (existsSync(metadataPath)) { stored = JSON.parse(readFileSync(metadataPath, "utf8")); } else { const legacyReport = JSON.parse(readFileSync(reportPath, "utf8")); const legacyConfig = JSON.parse(readFileSync(path.join(outputDir, "suite-config.json"), "utf8")); stored = { status: "complete", createdAt: legacyReport.generatedAt, completedAt: legacyReport.generatedAt, requestedBy: "unknown (legacy run)", spec: { source: legacyConfig.source, specSelection: legacyConfig.specSelection ?? "random", selectedBuildIds: legacyConfig.selectedBuildIds ?? [], level: legacyConfig.level, partyConfigurations: legacyConfig.specSelection && legacyConfig.specSelection !== "random" ? legacyConfig.partyConfigurations : legacyConfig.partyConfigurationsPerSource, simulationsPerParty: legacyConfig.simulationsPerParty, durationSeconds: legacyConfig.durationSeconds, partySeed: legacyConfig.partySeed, bossCategory: legacyConfig.bossCategory, bossKeys: legacyConfig.bossKeys ?? [], }, progress: {}, logTail: ["Imported from a Balance Lab report created before run-history persistence."], }; } const spec = normalizeBalanceLabJobSpec(stored.spec); const hasReport = existsSync(reportPath); const storedStatus = JOB_STATUSES.has(stored.status) ? stored.status : "failed"; const interrupted = !hasReport && (storedStatus === "queued" || storedStatus === "running"); const restoredSummary = hasReport ? reportSummary(outputDir) : null; const totalParties = restoredSummary?.partyConfigurations || expectedPartyConfigurations(spec); const createdAt = typeof stored.createdAt === "string" ? stored.createdAt : statSync(outputDir).birthtime.toISOString(); const job = { id: entry.name, status: hasReport ? "complete" : interrupted ? "failed" : storedStatus, createdAt, startedAt: typeof stored.startedAt === "string" ? stored.startedAt : null, completedAt: hasReport || interrupted ? (typeof stored.completedAt === "string" ? stored.completedAt : new Date().toISOString()) : null, requestedBy: typeof stored.requestedBy === "string" ? stored.requestedBy : "unknown", spec, progress: { party: Number(stored.progress?.party) || 0, partiesCompleted: hasReport ? totalParties : Number(stored.progress?.partiesCompleted) || 0, partiesTotal: totalParties, simulationsCompleted: Number(stored.progress?.simulationsCompleted) || 0, simulationsTotal: Number(stored.progress?.simulationsTotal) || 0, }, artifacts: hasReport ? [...ARTIFACTS.keys()].filter((name) => existsSync(path.join(outputDir, name))) : [], error: interrupted ? "The server restarted before this Balance Lab job completed. Run it again to produce a report." : (typeof stored.error === "string" ? stored.error : null), logTail: Array.isArray(stored.logTail) ? stored.logTail.filter((line) => typeof line === "string").slice(-80) : [], summary: restoredSummary, outputDir, }; jobs.set(job.id, job); persistJob(job); } catch (error) { console.warn(`[balance-lab] Could not restore ${entry.name}: ${error?.message ?? error}`); } } } restoreJobs(); function appendLog(job, line) { job.logTail.push(line.slice(0, 1_000)); if (job.logTail.length > 80) job.logTail.splice(0, job.logTail.length - 80); const planned = /^Running\s+[\d,]+\s+simulations\s+\((\d+)\s+parties\s+x/i.exec(line); if (planned) job.progress.partiesTotal = Number(planned[1]); const party = /^Party\s+(\d+)\/(\d+):/i.exec(line); if (party) { job.progress.party = Number(party[1]); job.progress.partiesTotal = Number(party[2]); job.progress.partiesCompleted = Math.max(0, Number(party[1]) - 1); job.progress.simulationsCompleted = 0; job.progress.simulationsTotal = 0; } const simulations = /^\[balance-lab]\s+(\d+)\/(\d+)\s+/i.exec(line); if (simulations) { job.progress.simulationsCompleted = Number(simulations[1]); job.progress.simulationsTotal = Number(simulations[2]); } persistJob(job); } function pump() { if (closed) return; while (running < maxConcurrent && queue.length) { const job = queue.shift(); if (!job || job.status !== "queued") continue; running += 1; job.status = "running"; job.startedAt = new Date().toISOString(); persistJob(job); const command = balanceLabCommand(job.spec, job.outputDir, { projectRoot, pythonExecutable, cooldownPolicyPath: job.cooldownPolicyPath, readinessPolicyPath: job.readinessPolicyPath, }); const child = launch(command.executable, command.args, { cwd: command.cwd, env: { ...process.env, PYTHONUNBUFFERED: "1" }, windowsHide: true, stdio: ["ignore", "pipe", "pipe"], }); children.set(job.id, child); const stdout = lineCollector((line) => appendLog(job, line)); const stderr = lineCollector((line) => appendLog(job, line)); child.stdout?.on("data", (chunk) => stdout.push(chunk)); child.stderr?.on("data", (chunk) => stderr.push(chunk)); let settled = false; const finish = (error, code = null) => { if (settled) return; settled = true; stdout.flush(); stderr.flush(); children.delete(job.id); running = Math.max(0, running - 1); job.completedAt = new Date().toISOString(); if (!error && code === 0 && existsSync(path.join(job.outputDir, "suite-report.json"))) { job.status = "complete"; job.progress.partiesCompleted = job.progress.partiesTotal; job.progress.simulationsCompleted = job.progress.simulationsTotal; job.artifacts = [...ARTIFACTS.keys()].filter((name) => existsSync(path.join(job.outputDir, name))); job.summary = reportSummary(job.outputDir); } else { job.status = "failed"; job.error = error?.message ?? processFailureMessage(job, code); } persistJob(job); pump(); }; child.once("error", (error) => finish(new Error( error?.code === "ENOENT" ? `Could not start ${command.executable}. Install Python 3 on the server or set BALANCE_LAB_PYTHON.` : `Could not start the Balance Lab process: ${error?.message ?? error}`, ))); child.once("close", (code) => finish(null, code)); } } function requireJob(id) { const job = jobs.get(id); if (!job) throw new GameDatabaseError("Balance Lab job was not found.", 404, "balance_lab_job_not_found"); return job; } function requireAvailableRuntime() { if (closed) throw new GameDatabaseError("Balance Lab runner is shutting down.", 503, "balance_lab_unavailable"); const runtime = currentRuntime(); if (!runtime.available) { throw new GameDatabaseError( `${runtime.error} Install Python 3 in the server container and set BALANCE_LAB_PYTHON=python3, then restart the app.`, 503, "balance_lab_runtime_unavailable", ); } } function requireQueueCapacity(count) { const pending = [...jobs.values()].filter((job) => job.status === "queued" || job.status === "running").length; if (pending + count > maxQueued) { throw new GameDatabaseError( `The Balance Lab server queue is full for this request; it has room for ${Math.max(0, maxQueued - pending)} more jobs. Wait for running jobs to finish or submit a smaller level batch.`, 429, "balance_lab_queue_full", ); } } function queueJob(spec, account) { const id = randomUUID(); const outputDir = path.join(jobsDir, id); mkdirSync(outputDir, { recursive: true }); const cooldownPolicyPath = path.join(outputDir, "companion-cooldown-policy.json"); if (cooldownPolicyProvider) { const published = cooldownPolicyProvider(); writeFileSync(cooldownPolicyPath, `${JSON.stringify(published.config ?? published, null, 2)}\n`, "utf8"); } const readinessPolicyPath = path.join(outputDir, "companion-spec-readiness.json"); if (readinessPolicyProvider) { const published = readinessPolicyProvider(); writeFileSync(readinessPolicyPath, `${JSON.stringify(published, null, 2)}\n`, "utf8"); } const totalParties = expectedPartyConfigurations(spec); const job = { id, status: "queued", createdAt: new Date().toISOString(), startedAt: null, completedAt: null, requestedBy: account.username, spec, progress: { party: 0, partiesCompleted: 0, partiesTotal: totalParties, simulationsCompleted: 0, simulationsTotal: 0, }, artifacts: [], error: null, logTail: [], summary: null, outputDir, cooldownPolicyPath: cooldownPolicyProvider ? cooldownPolicyPath : null, readinessPolicyPath: readinessPolicyProvider ? readinessPolicyPath : null, }; jobs.set(id, job); persistJob(job); queue.push(job); return job; } return { start(input, account) { const spec = normalizeBalanceLabJobSpec(input); requireAvailableRuntime(); requireQueueCapacity(1); const job = queueJob(spec, account); pump(); return publicJob(job); }, startMany(input, account) { const specs = normalizeBalanceLabJobBatch(input); requireAvailableRuntime(); requireQueueCapacity(specs.length); const batch = specs.map((spec) => queueJob(spec, account)); pump(); return batch.map(publicJob); }, list() { return [...jobs.values()] .sort((left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt)) .slice(0, 100) .map(publicJob); }, runtime() { return { ...currentRuntime(), maxConcurrent, running, queued: queue.length, maxQueued, }; }, get(id) { return publicJob(requireJob(id)); }, artifact(id, name) { const job = requireJob(id); if (!JOB_STATUSES.has(job.status) || job.status !== "complete") { throw new GameDatabaseError("Balance Lab artifacts are available after the job completes.", 409, "balance_lab_job_incomplete"); } const contentType = ARTIFACTS.get(name); if (!contentType || !job.artifacts.includes(name)) { throw new GameDatabaseError("Balance Lab artifact was not found.", 404, "balance_lab_artifact_not_found"); } const filePath = path.join(job.outputDir, name); const stats = statSync(filePath); return { filePath, contentType, size: stats.size, stream: () => createReadStream(filePath) }; }, close() { closed = true; queue.length = 0; for (const child of children.values()) child.kill(); children.clear(); }, }; }