Files
healer-man/scripts/runewaker-pipeline/stage-native-waterfall-alpha.mjs
2026-10-06 22:25:34 -04:00

494 lines
22 KiB
JavaScript

import { createHash } from "node:crypto";
import { access, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
import { spawnSync } from "node:child_process";
import path from "node:path";
import process from "node:process";
import sharp from "sharp";
import { fileURLToPath, pathToFileURL } from "node:url";
import { patchNativeAlphaBindings } from "./lib/native-alpha-glb-staging.mjs";
import { applyNativeMaterialSemantics } from "./lib/native-material-semantics.mjs";
const scriptDirectory = path.dirname(fileURLToPath(import.meta.url));
const projectRoot = path.resolve(scriptDirectory, "../..");
const triageRoot = path.join(projectRoot, "artifacts/playtest/rom-room-coverage/native-source-triage/gate-waterfall-scan");
const defaultOutputRoot = path.join(projectRoot, "artifacts/playtest/rom-room-coverage/native-alpha-stage");
const resourceRoot = path.resolve(projectRoot, "..", "rom-pvt-server-backups/Runewaker/Resource");
const waterTextureSuffix = "gate_waterfall";
const dungeonPlans = [
{
slug: "cave-water-dragon",
primaryResource: "model\\dungeon\\waterdragon_cavity\\act_water_dragon_hole.ros",
},
{
slug: "kulech-bones-nest",
primaryResource: "model\\dungeon\\qrich_cadaver_den\\atc_qrich_cadaver_den.ros",
},
{
slug: "mystic-altar",
primaryResource: "model\\dungeon\\mystic_altar\\act_mystic_altar_instance.ros",
},
{
slug: "raksha-temple",
sourceResources: [
"model\\element\\thunderhoof_mesa\\prop\\stone\\act_largewaterfall_03.ros",
"model\\element\\thunderhoof_mesa\\prop\\stone\\act_largewaterfall_05.ros",
"model\\element\\thunderhoof_mesa\\prop\\stone\\act_precipice_01.ros",
"model\\element\\thunderhoof_mesa\\prop\\stone\\act_waterfall_09.ros",
],
},
{
slug: "sardo-castle",
primaryResource: "model\\dungeon\\sardo_bastille\\act_sardo_bastille_in.ros",
},
{
slug: "tomb-seven-heroes",
primaryResource: "model\\dungeon\\tomb_of_seven_heroes\\act_tomb_of_seven_heroes.ros",
},
];
const sha256 = (value) => createHash("sha256").update(value).digest("hex");
const normalizeResource = (value) => String(value ?? "").replaceAll("/", "\\").toLowerCase();
const textureStem = (value) => path.posix.basename(String(value ?? "").replaceAll("\\", "/")).replace(/\.(?:dds|png)$/iu, "");
const textureFilename = (value, extension) => `${textureStem(value)}.${extension}`;
async function exists(file) {
try {
await access(file);
return true;
} catch {
return false;
}
}
async function sha256File(file) {
return sha256(await readFile(file));
}
function parseArguments(args) {
const values = new Map();
const flags = new Set();
for (let index = 0; index < args.length; index += 1) {
const argument = args[index];
if (!argument.startsWith("--")) throw new Error(`Unexpected argument: ${argument}`);
const equals = argument.indexOf("=");
if (equals >= 0) {
values.set(argument.slice(0, equals), argument.slice(equals + 1));
} else if (args[index + 1] && !args[index + 1].startsWith("--")) {
values.set(argument, args[++index]);
} else {
flags.add(argument);
}
}
return { values, flags };
}
function readGlbJson(buffer, label) {
if (buffer.length < 20 || buffer.toString("utf8", 0, 4) !== "glTF" || buffer.readUInt32LE(4) !== 2) {
throw new Error(`${label} is not a GLB 2.0 file.`);
}
for (let offset = 12; offset + 8 <= buffer.length;) {
const length = buffer.readUInt32LE(offset);
const type = buffer.readUInt32LE(offset + 4);
if (offset + 8 + length > buffer.length) throw new Error(`${label} has a truncated chunk.`);
if (type === 0x4e4f534a) return JSON.parse(buffer.toString("utf8", offset + 8, offset + 8 + length).trim());
offset += 8 + length;
}
throw new Error(`${label} has no JSON chunk.`);
}
function readGlbDocument(buffer, label) {
if (buffer.length < 20 || buffer.toString("utf8", 0, 4) !== "glTF" || buffer.readUInt32LE(4) !== 2) {
throw new Error(`${label} is not a GLB 2.0 file.`);
}
let json;
let bin;
for (let offset = 12; offset + 8 <= buffer.length;) {
const length = buffer.readUInt32LE(offset);
const type = buffer.readUInt32LE(offset + 4);
if (offset + 8 + length > buffer.length) throw new Error(`${label} has a truncated chunk.`);
const data = buffer.subarray(offset + 8, offset + 8 + length);
if (type === 0x4e4f534a) json = JSON.parse(data.toString("utf8").trim());
if (type === 0x004e4942) bin = data;
offset += 8 + length;
}
if (!json || !bin) throw new Error(`${label} must contain JSON and BIN chunks.`);
return { json, bin };
}
async function decodeGateWaterfallAlpha(glbFile, dungeonOutputRoot, expectedAliases) {
const buffer = await readFile(glbFile);
const { json, bin } = readGlbDocument(buffer, path.basename(glbFile));
const images = [];
for (const [imageIndex, image] of (json.images ?? []).entries()) {
const stem = textureStem(image.name);
if (!stem.toLowerCase().includes(waterTextureSuffix)
&& !expectedAliases.has(stem.toLowerCase())) continue;
if (image.mimeType !== "image/ktx2") continue;
const view = json.bufferViews?.[image.bufferView];
if (!view) throw new Error(`${image.name} does not reference an embedded KTX2 image.`);
const start = view.byteOffset ?? 0;
const ktxBytes = bin.subarray(start, start + view.byteLength);
const tempRoot = path.join(dungeonOutputRoot, `.alpha-decode-${imageIndex}`);
const inputFile = path.join(tempRoot, "source.ktx2");
const outputFile = path.join(tempRoot, "decoded.png");
await mkdir(tempRoot, { recursive: true });
try {
await writeFile(inputFile, ktxBytes);
const ktxExe = path.join(projectRoot, ".tools/ktx/bin/ktx.exe");
await run(ktxExe, ["extract", "--transcode", "rgba8", inputFile, outputFile], `decode ${image.name} alpha`);
const metadata = await sharp(outputFile).metadata();
const alpha = (await sharp(outputFile).stats()).channels[3] ?? null;
const alphaRange = alpha ? [Math.round(alpha.min), Math.round(alpha.max)] : null;
if (!metadata.hasAlpha || !alphaRange || alphaRange[0] >= 255 || alphaRange[1] <= alphaRange[0]) {
throw new Error(`${image.name} lost its varying source alpha during KTX2 compression (observed ${JSON.stringify(alphaRange)}).`);
}
images.push({
imageIndex,
imageName: image.name,
ktx2Sha256: sha256(ktxBytes),
width: metadata.width,
height: metadata.height,
hasAlpha: metadata.hasAlpha,
decodedAlphaRange: alphaRange,
decodedAlphaMean: alpha.mean,
});
} finally {
await rm(tempRoot, { recursive: true, force: true });
}
}
if (!images.length) throw new Error("Optimized GLB contains no KTX2 gate-waterfall images to verify.");
return images;
}
function sourceGroupsByMaterial(objText) {
const groupsByMaterial = new Map();
let group = "";
for (const line of objText.split(/\r?\n/u)) {
if (line.startsWith("g ")) group = line.slice(2).trim();
else if (line.startsWith("usemtl ")) {
const material = line.slice(7).trim();
if (!group || !material) continue;
const groups = groupsByMaterial.get(material) ?? [];
if (!groups.includes(group)) groups.push(group);
groupsByMaterial.set(material, groups);
}
}
return groupsByMaterial;
}
async function loadResourceCacheIndex() {
const root = path.join(projectRoot, "runewaker-export-work/resource-cache");
const index = new Map();
for (const entry of await readdir(root, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const cacheRoot = path.join(root, entry.name);
const manifestFile = path.join(cacheRoot, "cache-manifest.json");
if (!await exists(manifestFile)) continue;
let manifest;
try {
manifest = JSON.parse(await readFile(manifestFile, "utf8"));
} catch {
continue;
}
const resource = normalizeResource(manifest.source?.resource);
if (!resource) continue;
const matches = index.get(resource) ?? [];
matches.push({ cacheRoot, manifest, manifestFile });
index.set(resource, matches);
}
return index;
}
async function findReportDirectories(sourceResources) {
const wanted = new Set(sourceResources.map(normalizeResource));
const reports = [];
for (const entry of await readdir(triageRoot, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const directory = path.join(triageRoot, entry.name);
const reportFile = path.join(directory, "export-report.json");
const objFile = path.join(directory, "scene.obj");
if (!await exists(reportFile) || !await exists(objFile)) continue;
const reportBytes = await readFile(reportFile);
const report = JSON.parse(reportBytes.toString("utf8"));
const sourceModel = normalizeResource(report.sourceModel);
if (!wanted.has(sourceModel)) continue;
reports.push({ directory, reportFile, reportBytes, report, objFile, sourceModel });
}
const found = new Set(reports.map(({ sourceModel }) => sourceModel));
const missing = [...wanted].filter((resource) => !found.has(resource));
if (missing.length) throw new Error(`Fresh native scan reports missing: ${missing.join(", ")}`);
return reports;
}
function sourceResourceFile(sourceModel) {
return path.join(resourceRoot, ...normalizeResource(sourceModel).split("\\"));
}
async function resolveTextureAssets({ sourceModel, slug, row, cacheIndex, freshDirectory }) {
const ddsName = textureFilename(row.stagedTexture, "dds");
const pngName = textureFilename(row.stagedTexture, "png");
const freshDds = path.join(freshDirectory, "textures", ddsName);
if (!await exists(freshDds)) throw new Error(`${sourceModel} ${row.name}: fresh source export lacks ${ddsName}.`);
const freshDdsBytes = await readFile(freshDds);
const matchingCaches = cacheIndex.get(normalizeResource(sourceModel)) ?? [];
const rawTexturePaths = [path.join(projectRoot, `runewaker-export-work/${slug}/raw/textures`, ddsName)];
const preparedPngPaths = [path.join(projectRoot, `runewaker-export-work/${slug}/prepared/textures`, pngName)];
for (const cache of matchingCaches) {
if (cache.manifest.source?.sha256 !== await sha256File(sourceResourceFile(sourceModel))) {
throw new Error(`${sourceModel}: resource cache source SHA does not match the current ROS.`);
}
rawTexturePaths.push(path.join(cache.cacheRoot, "raw/textures", ddsName));
preparedPngPaths.push(path.join(cache.cacheRoot, "prepared/textures", pngName));
}
const ddsMatches = [];
for (const candidate of rawTexturePaths) {
if (await exists(candidate) && (await readFile(candidate)).equals(freshDdsBytes)) ddsMatches.push(candidate);
}
if (!ddsMatches.length) {
throw new Error(`${sourceModel} ${row.name}: fresh extracted DDS does not match an exact source/cache DDS.`);
}
const pngCandidate = preparedPngPaths.find((candidate) => candidate.endsWith(path.join("prepared", "textures", pngName)));
const existingPngs = [];
for (const candidate of preparedPngPaths) if (await exists(candidate)) existingPngs.push(candidate);
if (!existingPngs.length) throw new Error(`${sourceModel} ${row.name}: no prepared source PNG ${pngName} exists.`);
const pngBytes = await readFile(pngCandidate && await exists(pngCandidate) ? pngCandidate : existingPngs[0]);
const pngPath = pngCandidate && await exists(pngCandidate) ? pngCandidate : existingPngs[0];
const pngStats = await sharp(pngBytes).stats();
const alphaStats = pngStats.channels[3] ?? null;
const alphaRange = alphaStats ? [Math.round(alphaStats.min), Math.round(alphaStats.max)] : [255, 255];
const sourceAlphaRole = alphaStats && (alphaStats.min < 254.5 || alphaStats.max < 255) ? "blend" : "opaque";
return {
sourceDdsPath: freshDds,
sourceDdsSha256: sha256(freshDdsBytes),
byteMatchedDdsPaths: ddsMatches,
sourcePngPath: pngPath,
sourcePngBytes: pngBytes,
sourcePngSha256: sha256(pngBytes),
sourceAlphaRole,
sourceAlphaStats: { range: alphaRange, mean: alphaStats?.mean ?? 255 },
};
}
async function makeOverrides(plan, reports, cacheIndex, targetBuffer) {
const targetJson = readGlbJson(targetBuffer, plan.slug);
const targetNodeNames = new Set((targetJson.nodes ?? []).map((node) => node.name));
const overrides = [];
const provenance = [];
for (const reportRecord of reports) {
const { report, sourceModel, directory } = reportRecord;
const sourceFile = sourceResourceFile(report.sourceModel);
if (!await exists(sourceFile)) throw new Error(`Native ROS source file is missing: ${sourceFile}`);
const sourceResourceSha256 = await sha256File(sourceFile);
const cacheMatches = cacheIndex.get(sourceModel) ?? [];
for (const cache of cacheMatches) {
if (cache.manifest.source?.sha256?.toLowerCase() !== sourceResourceSha256) {
throw new Error(`${report.sourceModel}: cached source checksum disagrees with the current ROS.`);
}
}
const groupsByMaterial = sourceGroupsByMaterial(await readFile(reportRecord.objFile, "utf8"));
for (const row of report.materials ?? []) {
if (normalizeResource(row.sourceTexture).split("\\").at(-1) !== waterTextureSuffix) continue;
if (!Array.isArray(row.alphaBlendParameters) || !Array.isArray(row.sourceTextureChannels)) {
throw new Error(`${report.sourceModel} ${row.name}: fresh report lacks native alpha/channel provenance.`);
}
const groupNames = groupsByMaterial.get(row.name) ?? [];
if (!groupNames.length) throw new Error(`${report.sourceModel} ${row.name}: no matching native OBJ group.`);
const missingNodes = groupNames.filter((name) => !targetNodeNames.has(name));
if (missingNodes.length) {
throw new Error(`${plan.slug} target GLB lacks ${report.sourceModel} groups: ${missingNodes.join(", ")}`);
}
const textureAssets = await resolveTextureAssets({
sourceModel: report.sourceModel,
slug: plan.slug,
row,
cacheIndex,
freshDirectory: directory,
});
const alias = textureStem(row.stagedTexture);
overrides.push({
sourceModel: report.sourceModel,
sourceResourceSha256,
nativeMaterialRow: row,
sourceTexture: row.sourceTexture,
groupNames,
imageName: alias,
sourcePngBytes: textureAssets.sourcePngBytes,
sourcePngSha256: textureAssets.sourcePngSha256,
sourceTextureSha256: textureAssets.sourceDdsSha256,
sourceAlphaRole: textureAssets.sourceAlphaRole,
sourceAlphaStats: textureAssets.sourceAlphaStats,
});
provenance.push({
sourceModel: report.sourceModel,
sourceResourceSha256,
nativeReportSha256: sha256(reportRecord.reportBytes),
nativeReportFile: path.relative(projectRoot, reportRecord.reportFile).split(path.sep).join("/"),
nativeMaterialName: row.name,
nativeMaterialIndex: Number(row.name.match(/(\d+)$/)?.[1]),
nativeTexture: row.sourceTexture,
imageName: textureStem(row.stagedTexture),
sourceTextureSha256: textureAssets.sourceDdsSha256,
sourceTexturePath: path.relative(projectRoot, textureAssets.sourceDdsPath).split(path.sep).join("/"),
sourcePngSha256: textureAssets.sourcePngSha256,
sourcePngPath: path.relative(projectRoot, textureAssets.sourcePngPath).split(path.sep).join("/"),
sourceAlphaRange: textureAssets.sourceAlphaStats.range,
sourceAlphaRole: textureAssets.sourceAlphaRole,
cacheMatchedDdsPaths: textureAssets.byteMatchedDdsPaths.map((file) => path.relative(projectRoot, file).split(path.sep).join("/")),
groups: groupNames,
});
}
}
if (!overrides.length) throw new Error(`${plan.slug} has no native gate-waterfall rows to stage.`);
return { overrides, provenance };
}
async function run(command, args, label, cwd = projectRoot) {
const result = spawnSync(command, args, { cwd, encoding: "utf8", windowsHide: true, maxBuffer: 8 * 1024 * 1024 });
if (result.status !== 0) {
throw new Error(`${label} failed (${result.status ?? result.error?.message}).\n${result.stderr || result.stdout || "No command output."}`);
}
return { stdout: result.stdout.trim(), stderr: result.stderr.trim() };
}
async function stagePlan(plan, outputRoot, cacheIndex, sourceOnly) {
const sourceResources = plan.sourceResources ?? [plan.primaryResource];
const reports = await findReportDirectories(sourceResources);
const sourceFile = path.join(projectRoot, `runewaker-export-work/${plan.slug}/converted/${plan.slug}-visual.glb`);
if (!await exists(sourceFile)) throw new Error(`${plan.slug} converted visual GLB is missing.`);
const sourceBuffer = await readFile(sourceFile);
const sourceFileSha256 = sha256(sourceBuffer);
const { overrides, provenance } = await makeOverrides(plan, reports, cacheIndex, sourceBuffer);
const patchResult = patchNativeAlphaBindings(sourceBuffer, overrides);
const dungeonOutputRoot = path.join(outputRoot, plan.slug);
const sourceDirectory = path.join(dungeonOutputRoot, "source");
await mkdir(sourceDirectory, { recursive: true });
const stagedSource = path.join(sourceDirectory, `${plan.slug}-visual.alpha-fixed.glb`);
await writeFile(stagedSource, patchResult.buffer);
const semanticReport = await applyNativeMaterialSemantics(stagedSource, stagedSource);
const stagedSourceBuffer = await readFile(stagedSource);
const stagedSourceSha256 = await sha256File(stagedSource);
const report = {
schemaVersion: 1,
dungeonId: plan.slug,
sourceVisual: {
file: path.relative(projectRoot, sourceFile).split(path.sep).join("/"),
sha256: sourceFileSha256,
bytes: sourceBuffer.length,
},
stagedSource: {
file: path.relative(projectRoot, stagedSource).split(path.sep).join("/"),
sha256: stagedSourceSha256,
bytes: stagedSourceBuffer.length,
},
sourceTextureProvenance: provenance,
alphaPatch: patchResult.report,
nativeMaterialSemantics: semanticReport,
optimized: null,
limitations: [
"This is a review-stage material/image semantics patch. No shipping or canonical GLB was edited.",
"The patch binds source native material rows to matching OBJ group names and requires byte-identical source PNG images embedded in the current GLB.",
"KTX2 alpha is checked only when the optimized/compressed review output is generated.",
],
};
if (!sourceOnly) {
const optimizedFile = path.join(dungeonOutputRoot, `${plan.slug}-visual.optimized.glb`);
const reportFile = path.join(dungeonOutputRoot, "texture-quality-report.json");
const gltfTransformCli = path.join(projectRoot, "node_modules/@gltf-transform/cli/bin/cli.js");
const optimize = await run(process.execPath, [
gltfTransformCli,
"optimize",
stagedSource,
optimizedFile,
"--compress",
"meshopt",
"--meshopt-level",
"high",
"--instance",
"true",
"--instance-min",
"3",
"--simplify",
"false",
"--texture-compress",
"false",
], `optimize ${plan.slug} review GLB`);
const compress = await run(process.execPath, [
path.join(scriptDirectory, "compress-environment-textures.mjs"),
optimizedFile,
"--concurrency",
"1",
"--jobs",
"2",
"--report",
reportFile,
], `compress ${plan.slug} review GLB textures`);
report.optimized = {
file: path.relative(projectRoot, optimizedFile).split(path.sep).join("/"),
sha256: await sha256File(optimizedFile),
bytes: (await readFile(optimizedFile)).length,
optimizeOutput: optimize.stdout,
compressionOutput: compress.stdout,
textureQualityReport: path.relative(projectRoot, reportFile).split(path.sep).join("/"),
gateWaterfallKtxAlpha: await decodeGateWaterfallAlpha(
optimizedFile,
dungeonOutputRoot,
new Set(provenance.map((entry) => entry.imageName.toLowerCase())),
),
};
}
const reportFile = path.join(dungeonOutputRoot, "native-waterfall-alpha-stage-report.json");
await writeFile(reportFile, `${JSON.stringify(report, null, 2)}\n`, "utf8");
return report;
}
async function main() {
const { values, flags } = parseArguments(process.argv.slice(2));
const slugArgument = values.get("--slug") ?? "all";
const outputRoot = path.resolve(projectRoot, values.get("--output-root") ?? defaultOutputRoot);
const sourceOnly = flags.has("--source-only");
const verifyOptimized = flags.has("--verify-optimized");
const selectedPlans = slugArgument === "all" ? dungeonPlans : dungeonPlans.filter((plan) => plan.slug === slugArgument);
if (!selectedPlans.length) throw new Error(`Unknown dungeon slug: ${slugArgument}`);
await mkdir(outputRoot, { recursive: true });
if (verifyOptimized) {
for (const plan of selectedPlans) {
const dungeonOutputRoot = path.join(outputRoot, plan.slug);
const reportFile = path.join(dungeonOutputRoot, "native-waterfall-alpha-stage-report.json");
const report = JSON.parse(await readFile(reportFile, "utf8"));
if (!report.optimized?.file) throw new Error(`${plan.slug} has no recorded optimized output.`);
const optimizedFile = path.resolve(projectRoot, report.optimized.file);
const aliases = new Set(report.sourceTextureProvenance.map((entry) => (
entry.imageName ?? path.posix.basename(entry.sourcePngPath).replace(/\.png$/iu, "")
).toLowerCase()));
report.optimized.gateWaterfallKtxAlpha = await decodeGateWaterfallAlpha(optimizedFile, dungeonOutputRoot, aliases);
await writeFile(reportFile, `${JSON.stringify(report, null, 2)}\n`, "utf8");
console.log(`${plan.slug}: decoded ${report.optimized.gateWaterfallKtxAlpha.length} KTX2 waterfall images; source alpha survived`);
}
return;
}
const cacheIndex = await loadResourceCacheIndex();
const results = [];
for (const plan of selectedPlans) {
const result = await stagePlan(plan, outputRoot, cacheIndex, sourceOnly);
results.push({
dungeonId: plan.slug,
reportFile: path.relative(projectRoot, path.join(outputRoot, plan.slug, "native-waterfall-alpha-stage-report.json")).split(path.sep).join("/"),
bindingCount: result.alphaPatch.bindingCount,
sourceResourceCount: new Set(result.sourceTextureProvenance.map((entry) => entry.sourceModel)).size,
stagedSource: result.stagedSource,
optimized: result.optimized,
});
console.log(`${plan.slug}: ${result.alphaPatch.bindingCount} native alpha bindings staged`);
}
console.log(JSON.stringify({ outputRoot: path.relative(projectRoot, outputRoot).split(path.sep).join("/"), results }, null, 2));
}
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
await main();
}