import { createHash } from "node:crypto"; import { access, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { spawnSync } from "node:child_process"; import path from "node:path"; import process from "node:process"; import sharp from "sharp"; import { fileURLToPath, pathToFileURL } from "node:url"; import { patchNativeAlphaBindings } from "./lib/native-alpha-glb-staging.mjs"; import { applyNativeMaterialSemantics } from "./lib/native-material-semantics.mjs"; const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); const projectRoot = path.resolve(scriptDirectory, "../.."); const triageRoot = path.join(projectRoot, "artifacts/playtest/rom-room-coverage/native-source-triage/gate-waterfall-scan"); const defaultOutputRoot = path.join(projectRoot, "artifacts/playtest/rom-room-coverage/native-alpha-stage"); const resourceRoot = path.resolve(projectRoot, "..", "rom-pvt-server-backups/Runewaker/Resource"); const waterTextureSuffix = "gate_waterfall"; const dungeonPlans = [ { slug: "cave-water-dragon", primaryResource: "model\\dungeon\\waterdragon_cavity\\act_water_dragon_hole.ros", }, { slug: "kulech-bones-nest", primaryResource: "model\\dungeon\\qrich_cadaver_den\\atc_qrich_cadaver_den.ros", }, { slug: "mystic-altar", primaryResource: "model\\dungeon\\mystic_altar\\act_mystic_altar_instance.ros", }, { slug: "raksha-temple", sourceResources: [ "model\\element\\thunderhoof_mesa\\prop\\stone\\act_largewaterfall_03.ros", "model\\element\\thunderhoof_mesa\\prop\\stone\\act_largewaterfall_05.ros", "model\\element\\thunderhoof_mesa\\prop\\stone\\act_precipice_01.ros", "model\\element\\thunderhoof_mesa\\prop\\stone\\act_waterfall_09.ros", ], }, { slug: "sardo-castle", primaryResource: "model\\dungeon\\sardo_bastille\\act_sardo_bastille_in.ros", }, { slug: "tomb-seven-heroes", primaryResource: "model\\dungeon\\tomb_of_seven_heroes\\act_tomb_of_seven_heroes.ros", }, ]; const sha256 = (value) => createHash("sha256").update(value).digest("hex"); const normalizeResource = (value) => String(value ?? "").replaceAll("/", "\\").toLowerCase(); const textureStem = (value) => path.posix.basename(String(value ?? "").replaceAll("\\", "/")).replace(/\.(?:dds|png)$/iu, ""); const textureFilename = (value, extension) => `${textureStem(value)}.${extension}`; async function exists(file) { try { await access(file); return true; } catch { return false; } } async function sha256File(file) { return sha256(await readFile(file)); } function parseArguments(args) { const values = new Map(); const flags = new Set(); for (let index = 0; index < args.length; index += 1) { const argument = args[index]; if (!argument.startsWith("--")) throw new Error(`Unexpected argument: ${argument}`); const equals = argument.indexOf("="); if (equals >= 0) { values.set(argument.slice(0, equals), argument.slice(equals + 1)); } else if (args[index + 1] && !args[index + 1].startsWith("--")) { values.set(argument, args[++index]); } else { flags.add(argument); } } return { values, flags }; } function readGlbJson(buffer, label) { if (buffer.length < 20 || buffer.toString("utf8", 0, 4) !== "glTF" || buffer.readUInt32LE(4) !== 2) { throw new Error(`${label} is not a GLB 2.0 file.`); } for (let offset = 12; offset + 8 <= buffer.length;) { const length = buffer.readUInt32LE(offset); const type = buffer.readUInt32LE(offset + 4); if (offset + 8 + length > buffer.length) throw new Error(`${label} has a truncated chunk.`); if (type === 0x4e4f534a) return JSON.parse(buffer.toString("utf8", offset + 8, offset + 8 + length).trim()); offset += 8 + length; } throw new Error(`${label} has no JSON chunk.`); } function readGlbDocument(buffer, label) { if (buffer.length < 20 || buffer.toString("utf8", 0, 4) !== "glTF" || buffer.readUInt32LE(4) !== 2) { throw new Error(`${label} is not a GLB 2.0 file.`); } let json; let bin; for (let offset = 12; offset + 8 <= buffer.length;) { const length = buffer.readUInt32LE(offset); const type = buffer.readUInt32LE(offset + 4); if (offset + 8 + length > buffer.length) throw new Error(`${label} has a truncated chunk.`); const data = buffer.subarray(offset + 8, offset + 8 + length); if (type === 0x4e4f534a) json = JSON.parse(data.toString("utf8").trim()); if (type === 0x004e4942) bin = data; offset += 8 + length; } if (!json || !bin) throw new Error(`${label} must contain JSON and BIN chunks.`); return { json, bin }; } async function decodeGateWaterfallAlpha(glbFile, dungeonOutputRoot, expectedAliases) { const buffer = await readFile(glbFile); const { json, bin } = readGlbDocument(buffer, path.basename(glbFile)); const images = []; for (const [imageIndex, image] of (json.images ?? []).entries()) { const stem = textureStem(image.name); if (!stem.toLowerCase().includes(waterTextureSuffix) && !expectedAliases.has(stem.toLowerCase())) continue; if (image.mimeType !== "image/ktx2") continue; const view = json.bufferViews?.[image.bufferView]; if (!view) throw new Error(`${image.name} does not reference an embedded KTX2 image.`); const start = view.byteOffset ?? 0; const ktxBytes = bin.subarray(start, start + view.byteLength); const tempRoot = path.join(dungeonOutputRoot, `.alpha-decode-${imageIndex}`); const inputFile = path.join(tempRoot, "source.ktx2"); const outputFile = path.join(tempRoot, "decoded.png"); await mkdir(tempRoot, { recursive: true }); try { await writeFile(inputFile, ktxBytes); const ktxExe = path.join(projectRoot, ".tools/ktx/bin/ktx.exe"); await run(ktxExe, ["extract", "--transcode", "rgba8", inputFile, outputFile], `decode ${image.name} alpha`); const metadata = await sharp(outputFile).metadata(); const alpha = (await sharp(outputFile).stats()).channels[3] ?? null; const alphaRange = alpha ? [Math.round(alpha.min), Math.round(alpha.max)] : null; if (!metadata.hasAlpha || !alphaRange || alphaRange[0] >= 255 || alphaRange[1] <= alphaRange[0]) { throw new Error(`${image.name} lost its varying source alpha during KTX2 compression (observed ${JSON.stringify(alphaRange)}).`); } images.push({ imageIndex, imageName: image.name, ktx2Sha256: sha256(ktxBytes), width: metadata.width, height: metadata.height, hasAlpha: metadata.hasAlpha, decodedAlphaRange: alphaRange, decodedAlphaMean: alpha.mean, }); } finally { await rm(tempRoot, { recursive: true, force: true }); } } if (!images.length) throw new Error("Optimized GLB contains no KTX2 gate-waterfall images to verify."); return images; } function sourceGroupsByMaterial(objText) { const groupsByMaterial = new Map(); let group = ""; for (const line of objText.split(/\r?\n/u)) { if (line.startsWith("g ")) group = line.slice(2).trim(); else if (line.startsWith("usemtl ")) { const material = line.slice(7).trim(); if (!group || !material) continue; const groups = groupsByMaterial.get(material) ?? []; if (!groups.includes(group)) groups.push(group); groupsByMaterial.set(material, groups); } } return groupsByMaterial; } async function loadResourceCacheIndex() { const root = path.join(projectRoot, "runewaker-export-work/resource-cache"); const index = new Map(); for (const entry of await readdir(root, { withFileTypes: true })) { if (!entry.isDirectory()) continue; const cacheRoot = path.join(root, entry.name); const manifestFile = path.join(cacheRoot, "cache-manifest.json"); if (!await exists(manifestFile)) continue; let manifest; try { manifest = JSON.parse(await readFile(manifestFile, "utf8")); } catch { continue; } const resource = normalizeResource(manifest.source?.resource); if (!resource) continue; const matches = index.get(resource) ?? []; matches.push({ cacheRoot, manifest, manifestFile }); index.set(resource, matches); } return index; } async function findReportDirectories(sourceResources) { const wanted = new Set(sourceResources.map(normalizeResource)); const reports = []; for (const entry of await readdir(triageRoot, { withFileTypes: true })) { if (!entry.isDirectory()) continue; const directory = path.join(triageRoot, entry.name); const reportFile = path.join(directory, "export-report.json"); const objFile = path.join(directory, "scene.obj"); if (!await exists(reportFile) || !await exists(objFile)) continue; const reportBytes = await readFile(reportFile); const report = JSON.parse(reportBytes.toString("utf8")); const sourceModel = normalizeResource(report.sourceModel); if (!wanted.has(sourceModel)) continue; reports.push({ directory, reportFile, reportBytes, report, objFile, sourceModel }); } const found = new Set(reports.map(({ sourceModel }) => sourceModel)); const missing = [...wanted].filter((resource) => !found.has(resource)); if (missing.length) throw new Error(`Fresh native scan reports missing: ${missing.join(", ")}`); return reports; } function sourceResourceFile(sourceModel) { return path.join(resourceRoot, ...normalizeResource(sourceModel).split("\\")); } async function resolveTextureAssets({ sourceModel, slug, row, cacheIndex, freshDirectory }) { const ddsName = textureFilename(row.stagedTexture, "dds"); const pngName = textureFilename(row.stagedTexture, "png"); const freshDds = path.join(freshDirectory, "textures", ddsName); if (!await exists(freshDds)) throw new Error(`${sourceModel} ${row.name}: fresh source export lacks ${ddsName}.`); const freshDdsBytes = await readFile(freshDds); const matchingCaches = cacheIndex.get(normalizeResource(sourceModel)) ?? []; const rawTexturePaths = [path.join(projectRoot, `runewaker-export-work/${slug}/raw/textures`, ddsName)]; const preparedPngPaths = [path.join(projectRoot, `runewaker-export-work/${slug}/prepared/textures`, pngName)]; for (const cache of matchingCaches) { if (cache.manifest.source?.sha256 !== await sha256File(sourceResourceFile(sourceModel))) { throw new Error(`${sourceModel}: resource cache source SHA does not match the current ROS.`); } rawTexturePaths.push(path.join(cache.cacheRoot, "raw/textures", ddsName)); preparedPngPaths.push(path.join(cache.cacheRoot, "prepared/textures", pngName)); } const ddsMatches = []; for (const candidate of rawTexturePaths) { if (await exists(candidate) && (await readFile(candidate)).equals(freshDdsBytes)) ddsMatches.push(candidate); } if (!ddsMatches.length) { throw new Error(`${sourceModel} ${row.name}: fresh extracted DDS does not match an exact source/cache DDS.`); } const pngCandidate = preparedPngPaths.find((candidate) => candidate.endsWith(path.join("prepared", "textures", pngName))); const existingPngs = []; for (const candidate of preparedPngPaths) if (await exists(candidate)) existingPngs.push(candidate); if (!existingPngs.length) throw new Error(`${sourceModel} ${row.name}: no prepared source PNG ${pngName} exists.`); const pngBytes = await readFile(pngCandidate && await exists(pngCandidate) ? pngCandidate : existingPngs[0]); const pngPath = pngCandidate && await exists(pngCandidate) ? pngCandidate : existingPngs[0]; const pngStats = await sharp(pngBytes).stats(); const alphaStats = pngStats.channels[3] ?? null; const alphaRange = alphaStats ? [Math.round(alphaStats.min), Math.round(alphaStats.max)] : [255, 255]; const sourceAlphaRole = alphaStats && (alphaStats.min < 254.5 || alphaStats.max < 255) ? "blend" : "opaque"; return { sourceDdsPath: freshDds, sourceDdsSha256: sha256(freshDdsBytes), byteMatchedDdsPaths: ddsMatches, sourcePngPath: pngPath, sourcePngBytes: pngBytes, sourcePngSha256: sha256(pngBytes), sourceAlphaRole, sourceAlphaStats: { range: alphaRange, mean: alphaStats?.mean ?? 255 }, }; } async function makeOverrides(plan, reports, cacheIndex, targetBuffer) { const targetJson = readGlbJson(targetBuffer, plan.slug); const targetNodeNames = new Set((targetJson.nodes ?? []).map((node) => node.name)); const overrides = []; const provenance = []; for (const reportRecord of reports) { const { report, sourceModel, directory } = reportRecord; const sourceFile = sourceResourceFile(report.sourceModel); if (!await exists(sourceFile)) throw new Error(`Native ROS source file is missing: ${sourceFile}`); const sourceResourceSha256 = await sha256File(sourceFile); const cacheMatches = cacheIndex.get(sourceModel) ?? []; for (const cache of cacheMatches) { if (cache.manifest.source?.sha256?.toLowerCase() !== sourceResourceSha256) { throw new Error(`${report.sourceModel}: cached source checksum disagrees with the current ROS.`); } } const groupsByMaterial = sourceGroupsByMaterial(await readFile(reportRecord.objFile, "utf8")); for (const row of report.materials ?? []) { if (normalizeResource(row.sourceTexture).split("\\").at(-1) !== waterTextureSuffix) continue; if (!Array.isArray(row.alphaBlendParameters) || !Array.isArray(row.sourceTextureChannels)) { throw new Error(`${report.sourceModel} ${row.name}: fresh report lacks native alpha/channel provenance.`); } const groupNames = groupsByMaterial.get(row.name) ?? []; if (!groupNames.length) throw new Error(`${report.sourceModel} ${row.name}: no matching native OBJ group.`); const missingNodes = groupNames.filter((name) => !targetNodeNames.has(name)); if (missingNodes.length) { throw new Error(`${plan.slug} target GLB lacks ${report.sourceModel} groups: ${missingNodes.join(", ")}`); } const textureAssets = await resolveTextureAssets({ sourceModel: report.sourceModel, slug: plan.slug, row, cacheIndex, freshDirectory: directory, }); const alias = textureStem(row.stagedTexture); overrides.push({ sourceModel: report.sourceModel, sourceResourceSha256, nativeMaterialRow: row, sourceTexture: row.sourceTexture, groupNames, imageName: alias, sourcePngBytes: textureAssets.sourcePngBytes, sourcePngSha256: textureAssets.sourcePngSha256, sourceTextureSha256: textureAssets.sourceDdsSha256, sourceAlphaRole: textureAssets.sourceAlphaRole, sourceAlphaStats: textureAssets.sourceAlphaStats, }); provenance.push({ sourceModel: report.sourceModel, sourceResourceSha256, nativeReportSha256: sha256(reportRecord.reportBytes), nativeReportFile: path.relative(projectRoot, reportRecord.reportFile).split(path.sep).join("/"), nativeMaterialName: row.name, nativeMaterialIndex: Number(row.name.match(/(\d+)$/)?.[1]), nativeTexture: row.sourceTexture, imageName: textureStem(row.stagedTexture), sourceTextureSha256: textureAssets.sourceDdsSha256, sourceTexturePath: path.relative(projectRoot, textureAssets.sourceDdsPath).split(path.sep).join("/"), sourcePngSha256: textureAssets.sourcePngSha256, sourcePngPath: path.relative(projectRoot, textureAssets.sourcePngPath).split(path.sep).join("/"), sourceAlphaRange: textureAssets.sourceAlphaStats.range, sourceAlphaRole: textureAssets.sourceAlphaRole, cacheMatchedDdsPaths: textureAssets.byteMatchedDdsPaths.map((file) => path.relative(projectRoot, file).split(path.sep).join("/")), groups: groupNames, }); } } if (!overrides.length) throw new Error(`${plan.slug} has no native gate-waterfall rows to stage.`); return { overrides, provenance }; } async function run(command, args, label, cwd = projectRoot) { const result = spawnSync(command, args, { cwd, encoding: "utf8", windowsHide: true, maxBuffer: 8 * 1024 * 1024 }); if (result.status !== 0) { throw new Error(`${label} failed (${result.status ?? result.error?.message}).\n${result.stderr || result.stdout || "No command output."}`); } return { stdout: result.stdout.trim(), stderr: result.stderr.trim() }; } async function stagePlan(plan, outputRoot, cacheIndex, sourceOnly) { const sourceResources = plan.sourceResources ?? [plan.primaryResource]; const reports = await findReportDirectories(sourceResources); const sourceFile = path.join(projectRoot, `runewaker-export-work/${plan.slug}/converted/${plan.slug}-visual.glb`); if (!await exists(sourceFile)) throw new Error(`${plan.slug} converted visual GLB is missing.`); const sourceBuffer = await readFile(sourceFile); const sourceFileSha256 = sha256(sourceBuffer); const { overrides, provenance } = await makeOverrides(plan, reports, cacheIndex, sourceBuffer); const patchResult = patchNativeAlphaBindings(sourceBuffer, overrides); const dungeonOutputRoot = path.join(outputRoot, plan.slug); const sourceDirectory = path.join(dungeonOutputRoot, "source"); await mkdir(sourceDirectory, { recursive: true }); const stagedSource = path.join(sourceDirectory, `${plan.slug}-visual.alpha-fixed.glb`); await writeFile(stagedSource, patchResult.buffer); const semanticReport = await applyNativeMaterialSemantics(stagedSource, stagedSource); const stagedSourceBuffer = await readFile(stagedSource); const stagedSourceSha256 = await sha256File(stagedSource); const report = { schemaVersion: 1, dungeonId: plan.slug, sourceVisual: { file: path.relative(projectRoot, sourceFile).split(path.sep).join("/"), sha256: sourceFileSha256, bytes: sourceBuffer.length, }, stagedSource: { file: path.relative(projectRoot, stagedSource).split(path.sep).join("/"), sha256: stagedSourceSha256, bytes: stagedSourceBuffer.length, }, sourceTextureProvenance: provenance, alphaPatch: patchResult.report, nativeMaterialSemantics: semanticReport, optimized: null, limitations: [ "This is a review-stage material/image semantics patch. No shipping or canonical GLB was edited.", "The patch binds source native material rows to matching OBJ group names and requires byte-identical source PNG images embedded in the current GLB.", "KTX2 alpha is checked only when the optimized/compressed review output is generated.", ], }; if (!sourceOnly) { const optimizedFile = path.join(dungeonOutputRoot, `${plan.slug}-visual.optimized.glb`); const reportFile = path.join(dungeonOutputRoot, "texture-quality-report.json"); const gltfTransformCli = path.join(projectRoot, "node_modules/@gltf-transform/cli/bin/cli.js"); const optimize = await run(process.execPath, [ gltfTransformCli, "optimize", stagedSource, optimizedFile, "--compress", "meshopt", "--meshopt-level", "high", "--instance", "true", "--instance-min", "3", "--simplify", "false", "--texture-compress", "false", ], `optimize ${plan.slug} review GLB`); const compress = await run(process.execPath, [ path.join(scriptDirectory, "compress-environment-textures.mjs"), optimizedFile, "--concurrency", "1", "--jobs", "2", "--report", reportFile, ], `compress ${plan.slug} review GLB textures`); report.optimized = { file: path.relative(projectRoot, optimizedFile).split(path.sep).join("/"), sha256: await sha256File(optimizedFile), bytes: (await readFile(optimizedFile)).length, optimizeOutput: optimize.stdout, compressionOutput: compress.stdout, textureQualityReport: path.relative(projectRoot, reportFile).split(path.sep).join("/"), gateWaterfallKtxAlpha: await decodeGateWaterfallAlpha( optimizedFile, dungeonOutputRoot, new Set(provenance.map((entry) => entry.imageName.toLowerCase())), ), }; } const reportFile = path.join(dungeonOutputRoot, "native-waterfall-alpha-stage-report.json"); await writeFile(reportFile, `${JSON.stringify(report, null, 2)}\n`, "utf8"); return report; } async function main() { const { values, flags } = parseArguments(process.argv.slice(2)); const slugArgument = values.get("--slug") ?? "all"; const outputRoot = path.resolve(projectRoot, values.get("--output-root") ?? defaultOutputRoot); const sourceOnly = flags.has("--source-only"); const verifyOptimized = flags.has("--verify-optimized"); const selectedPlans = slugArgument === "all" ? dungeonPlans : dungeonPlans.filter((plan) => plan.slug === slugArgument); if (!selectedPlans.length) throw new Error(`Unknown dungeon slug: ${slugArgument}`); await mkdir(outputRoot, { recursive: true }); if (verifyOptimized) { for (const plan of selectedPlans) { const dungeonOutputRoot = path.join(outputRoot, plan.slug); const reportFile = path.join(dungeonOutputRoot, "native-waterfall-alpha-stage-report.json"); const report = JSON.parse(await readFile(reportFile, "utf8")); if (!report.optimized?.file) throw new Error(`${plan.slug} has no recorded optimized output.`); const optimizedFile = path.resolve(projectRoot, report.optimized.file); const aliases = new Set(report.sourceTextureProvenance.map((entry) => ( entry.imageName ?? path.posix.basename(entry.sourcePngPath).replace(/\.png$/iu, "") ).toLowerCase())); report.optimized.gateWaterfallKtxAlpha = await decodeGateWaterfallAlpha(optimizedFile, dungeonOutputRoot, aliases); await writeFile(reportFile, `${JSON.stringify(report, null, 2)}\n`, "utf8"); console.log(`${plan.slug}: decoded ${report.optimized.gateWaterfallKtxAlpha.length} KTX2 waterfall images; source alpha survived`); } return; } const cacheIndex = await loadResourceCacheIndex(); const results = []; for (const plan of selectedPlans) { const result = await stagePlan(plan, outputRoot, cacheIndex, sourceOnly); results.push({ dungeonId: plan.slug, reportFile: path.relative(projectRoot, path.join(outputRoot, plan.slug, "native-waterfall-alpha-stage-report.json")).split(path.sep).join("/"), bindingCount: result.alphaPatch.bindingCount, sourceResourceCount: new Set(result.sourceTextureProvenance.map((entry) => entry.sourceModel)).size, stagedSource: result.stagedSource, optimized: result.optimized, }); console.log(`${plan.slug}: ${result.alphaPatch.bindingCount} native alpha bindings staged`); } console.log(JSON.stringify({ outputRoot: path.relative(projectRoot, outputRoot).split(path.sep).join("/"), results }, null, 2)); } if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { await main(); }