Files
2026-10-06 22:25:34 -04:00

473 lines
22 KiB
JavaScript

import { spawn } from "node:child_process";
import { createHash } from "node:crypto";
import {
copyFile,
mkdir,
mkdtemp,
readFile,
rm,
stat,
writeFile,
} from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import sharp from "sharp";
import { descriptorSourceMatrix, normalizeWdbResourcePath } from "../lib/environment-assembly.mjs";
import { loadEnvironmentRecipe, projectRoot } from "../lib/recipe.mjs";
import { convertNativeTerrainChunks, sourceTerrainToThreeMatrix } from "./terrain-geometry.mjs";
import { writeNativeTerrainGlbs } from "./native-terrain-glb.mjs";
let workDirectory;
const texconvDefault = path.resolve(process.env.TEXCONV_BIN ?? path.join(
process.env.LOCALAPPDATA ?? "",
"Microsoft",
"WinGet",
"Packages",
"Microsoft.DirectXTex.Texconv_Microsoft.Winget.Source_8wekyb3d8bbwe",
"texconv.exe",
));
const argv = process.argv.slice(2);
function option(name, fallback) {
const equals = argv.find((value) => value.startsWith(name + "="));
if (equals) return equals.slice(name.length + 1);
const index = argv.indexOf(name);
if (index >= 0 && argv[index + 1] && !argv[index + 1].startsWith("--")) return argv[index + 1];
return fallback;
}
function projectOption(name, fallback) {
const value = option(name, fallback);
return path.resolve(projectRoot, value);
}
function hashBytes(data) {
return createHash("sha256").update(data).digest("hex");
}
async function hashFile(file) {
return hashBytes(await readFile(file));
}
function ensureInside(parent, candidate, label) {
const relative = path.relative(path.resolve(parent), path.resolve(candidate));
if (!relative || (relative !== ".." && !relative.startsWith(".." + path.sep) && !path.isAbsolute(relative))) {
return path.resolve(candidate);
}
throw new Error(`${label} resolves outside the allowed directory: ${candidate}`);
}
function sourceRelativeFile(sourceDirectory, relative, label) {
if (typeof relative !== "string" || !relative.trim()) throw new Error(`${label} is required.`);
return ensureInside(sourceDirectory, path.resolve(sourceDirectory, relative.replaceAll("/", path.sep)), label);
}
function typedArrayFromBuffer(buffer, Type, bytesPerElement, label) {
if (buffer.byteLength % bytesPerElement !== 0) {
throw new Error(`${label} has a byte length that is not aligned to ${bytesPerElement}.`);
}
const copy = new Uint8Array(buffer.byteLength);
copy.set(buffer);
return new Type(copy.buffer);
}
async function readJson(file, label) {
try {
return JSON.parse(await readFile(file, "utf8"));
} catch (error) {
throw new Error(`Could not read ${label} ${file}: ${error.message}`);
}
}
async function loadTerrainChunks(manifest, sourceDirectory) {
if (!Array.isArray(manifest.chunks) || manifest.chunks.length === 0) {
throw new Error("Native terrain manifest must contain chunks.");
}
const chunks = [];
for (const row of manifest.chunks) {
const files = row.geometryFiles;
if (!files) throw new Error(`${row.id} has no geometryFiles record.`);
const positionsFile = sourceRelativeFile(sourceDirectory, files.heightfieldVerts ?? files.positions, `${row.id} positions`);
const uvsFile = sourceRelativeFile(sourceDirectory, files.normalizedXZUvs ?? files.uvs, `${row.id} normalized XZ UVs`);
const indicesFile = sourceRelativeFile(sourceDirectory, files.exactIndices ?? files.indices, `${row.id} exact indices`);
const normalsFile = sourceRelativeFile(sourceDirectory, files.nativeNormals ?? files.normals, `${row.id} native normals`);
const triangleFlagsFile = sourceRelativeFile(sourceDirectory, files.triangleVisibilityFlags, `${row.id} triangle flags`);
const vertexFlagsFile = files.vertexFlags
? sourceRelativeFile(sourceDirectory, files.vertexFlags, `${row.id} vertex flags`)
: null;
const cellFlagsFile = files.cellVisibilityFlags
? sourceRelativeFile(sourceDirectory, files.cellVisibilityFlags, `${row.id} cell flags`)
: null;
const [positionBytes, uvBytes, indexBytes, normalBytes, triangleFlagsBytes] = await Promise.all([
readFile(positionsFile), readFile(uvsFile), readFile(indicesFile), readFile(normalsFile), readFile(triangleFlagsFile),
]);
const vertexFlagsBytes = vertexFlagsFile ? await readFile(vertexFlagsFile) : null;
const cellFlagsBytes = cellFlagsFile ? await readFile(cellFlagsFile) : null;
const triangleVisibilityFlags = new Uint8Array(triangleFlagsBytes);
chunks.push({
id: row.id,
coordinate: row.coordinate,
dimension: Number(row.dimension),
gridSize: Number(row.gridSize),
serializedOrigin: row.serializedOrigin,
positions: typedArrayFromBuffer(positionBytes, Float32Array, 4, `${row.id} positions`),
uvs: typedArrayFromBuffer(uvBytes, Float32Array, 4, `${row.id} UVs`),
indices: typedArrayFromBuffer(indexBytes, Uint16Array, 2, `${row.id} indices`),
normals: typedArrayFromBuffer(normalBytes, Float32Array, 4, `${row.id} native normals`),
triangleVisibilityFlags,
triangleVisibilitySha256: hashBytes(triangleFlagsBytes),
vertexFlagsSha256: vertexFlagsBytes ? hashBytes(vertexFlagsBytes) : null,
cellVisibilityFlagsSha256: cellFlagsBytes ? hashBytes(cellFlagsBytes) : null,
visibleSourceTriangles: Number(row.visibleSourceTriangleCount),
maskedSourceTriangles: Number(row.maskedSourceTriangleCount),
layers: row.layers,
splatMaps: row.splatMaps,
material: row.material,
});
}
return chunks;
}
function selectPrimaryDescriptor(wdbReport, recipe) {
const wanted = normalizeWdbResourcePath(recipe.source.model);
const matches = (wdbReport.descriptors ?? []).filter((row) => normalizeWdbResourcePath(row.resource) === wanted);
if (matches.length === 0) throw new Error(`WDB report has no descriptor for ${recipe.source.model}.`);
if (matches.length === 1) return matches[0];
const expected = recipe.source.primaryPlacementTranslation;
if (!Array.isArray(expected) || expected.length !== 3) {
throw new Error("Multiple primary-resource placements require recipe.source.primaryPlacementTranslation.");
}
const match = matches.find((row) => row.translation?.every((value, index) => (
Math.abs(Number(value) - Number(expected[index])) <= 0.0001
)));
if (!match) throw new Error("Configured primary WDB descriptor was not found in the WDB report.");
return match;
}
function run(executable, argumentsList, label) {
return new Promise((resolve, reject) => {
console.log(`[runewaker-terrain] ${label}`);
const child = spawn(executable, argumentsList, {
cwd: projectRoot,
stdio: "inherit",
windowsHide: true,
});
child.once("error", reject);
child.once("exit", (code) => {
if (code === 0) resolve();
else reject(new Error(`${label} exited with code ${code}.`));
});
});
}
async function verifyExists(file, label) {
try {
await stat(file);
} catch {
throw new Error(`${label} does not exist: ${file}`);
}
}
async function copyIfIdenticalOrMissing(source, target) {
try {
const [sourceHash, targetHash] = await Promise.all([hashFile(source), hashFile(target)]);
if (sourceHash !== targetHash) {
throw new Error(`Refusing to overwrite different texture-cache content at ${target}.`);
}
return targetHash;
} catch (error) {
if (error.code !== "ENOENT") throw error;
await mkdir(path.dirname(target), { recursive: true });
await copyFile(source, target);
return hashFile(target);
}
}
function basenameForPng(sourceDDS) {
return path.basename(sourceDDS).replace(/\.dds$/i, ".png");
}
async function decodeTerrainImages(chunks, sourceDirectory, tempDirectory, texconv, rawTextureDirectory, preparedTextureDirectory) {
await mkdir(tempDirectory, { recursive: true });
await mkdir(rawTextureDirectory, { recursive: true });
await mkdir(preparedTextureDirectory, { recursive: true });
const uniqueTiles = new Map();
const uniqueWeights = new Map();
for (const chunk of chunks) {
for (const layer of chunk.layers ?? []) {
if (!uniqueTiles.has(layer.sourceDDS)) uniqueTiles.set(layer.sourceDDS, layer);
}
for (const weight of chunk.splatMaps ?? []) {
if (!uniqueWeights.has(weight.linearPngFile)) uniqueWeights.set(weight.linearPngFile, weight);
}
}
const textureBySource = new Map();
const textureReport = [];
let textureIndex = 0;
for (const [sourceDDS, layer] of uniqueTiles) {
const ddsFile = sourceRelativeFile(sourceDirectory, sourceDDS, `terrain tile ${sourceDDS}`);
await verifyExists(ddsFile, `Terrain source tile ${sourceDDS}`);
const rawTarget = path.join(rawTextureDirectory, path.basename(sourceDDS));
const ddsHash = await copyIfIdenticalOrMissing(ddsFile, rawTarget);
const destination = path.join(tempDirectory, `tile-${String(textureIndex).padStart(2, "0")}`);
await mkdir(destination, { recursive: true });
await run(texconv, ["-ft", "PNG", "-o", destination, "-y", "--permissive", ddsFile], `decode source terrain tile ${sourceDDS}`);
const decodedPng = path.join(destination, basenameForPng(sourceDDS));
await verifyExists(decodedPng, `Converted terrain tile ${sourceDDS}`);
const opaquePng = await sharp(decodedPng)
.removeAlpha()
.ensureAlpha(1)
.png({ compressionLevel: 9, adaptiveFiltering: false, palette: false })
.toBuffer();
const image = await sharp(opaquePng).raw().toBuffer({ resolveWithObject: true });
if (image.info.channels !== 4) throw new Error(`${sourceDDS} did not convert to RGBA PNG.`);
for (let alphaOffset = 3; alphaOffset < image.data.length; alphaOffset += 4) {
if (image.data[alphaOffset] !== 255) throw new Error(`${sourceDDS} terrain alpha was not forced opaque.`);
}
if (Number(layer.width) > 0 && image.info.width !== Number(layer.width)) {
throw new Error(`${sourceDDS} converted width ${image.info.width} does not match source ${layer.width}.`);
}
if (Number(layer.height) > 0 && image.info.height !== Number(layer.height)) {
throw new Error(`${sourceDDS} converted height ${image.info.height} does not match source ${layer.height}.`);
}
const imageName = basenameForPng(sourceDDS);
const preparedTarget = path.join(preparedTextureDirectory, imageName);
const opaqueTemp = path.join(destination, "opaque-" + imageName);
await writeFile(opaqueTemp, opaquePng);
const pngHash = await copyIfIdenticalOrMissing(opaqueTemp, preparedTarget);
textureBySource.set(sourceDDS, {
kind: "tile",
image: opaquePng,
imageName,
sourceDDS,
ddsSha256: ddsHash,
pngSha256: pngHash,
width: image.info.width,
height: image.info.height,
alphaPolicy: "source tile alpha ignored; export PNG alpha forced to 255; material OPAQUE",
});
textureReport.push({
kind: "tile",
sourceDDS,
sourceClientPath: layer.sourceClientPath ?? null,
imageName,
width: image.info.width,
height: image.info.height,
sourceDdsSha256: ddsHash,
preparedPngSha256: pngHash,
alphaPolicy: "source tile alpha ignored; export PNG alpha forced to 255",
});
textureIndex += 1;
}
for (const [linearPngFile, weight] of uniqueWeights) {
const sourceDDS = sourceRelativeFile(sourceDirectory, weight.splatDDS, `splat DDS ${weight.splatDDS}`);
const sourcePng = sourceRelativeFile(sourceDirectory, linearPngFile, `linear splat PNG ${linearPngFile}`);
await verifyExists(sourceDDS, `Weight map DDS ${weight.splatDDS}`);
await verifyExists(sourcePng, `Linear weight PNG ${linearPngFile}`);
const weightImageName = path.basename(linearPngFile);
const rawTarget = path.join(rawTextureDirectory, weightImageName.replace(/\.png$/i, ".dds"));
const ddsHash = await copyIfIdenticalOrMissing(sourceDDS, rawTarget);
const preparedTarget = path.join(preparedTextureDirectory, path.basename(linearPngFile));
const pngHash = await copyIfIdenticalOrMissing(sourcePng, preparedTarget);
const bytes = await readFile(sourcePng);
const image = await sharp(bytes).metadata();
if (Number(weight.width) > 0 && image.width !== Number(weight.width)) {
throw new Error(`${linearPngFile} width ${image.width} does not match source ${weight.width}.`);
}
if (Number(weight.height) > 0 && image.height !== Number(weight.height)) {
throw new Error(`${linearPngFile} height ${image.height} does not match source ${weight.height}.`);
}
textureBySource.set(linearPngFile, {
kind: "weight",
image: bytes,
imageName: path.basename(linearPngFile),
sourceDDS: weight.splatDDS,
sourceCompressedFile: weight.sourceCompressedFile,
ddsSha256: ddsHash,
pngSha256: pngHash,
width: image.width,
height: image.height,
channelLayerIds: weight.channelLayerIds,
colorSpace: "linear RGB; referenced only by metallicRoughnessTexture and occlusionTexture",
});
textureReport.push({
kind: "weight",
sourceDDS: weight.splatDDS,
sourceCompressedFile: weight.sourceCompressedFile,
preparedLinearPng: linearPngFile,
imageName: path.basename(linearPngFile),
width: image.width,
height: image.height,
channelLayerIds: weight.channelLayerIds,
sourceDdsSha256: ddsHash,
preparedPngSha256: pngHash,
colorSpace: "linear RGB; source DXT1 alpha ignored",
});
}
return { textureBySource, textureReport };
}
async function main() {
const recipeFallback = path.join(projectRoot, "scripts", "runewaker-pipeline", "recipes", "sand-vortex.json");
const { file: recipeFile, recipe } = await loadEnvironmentRecipe(argv, recipeFallback);
const slug = recipe.slug;
const title = recipe.title ?? slug;
if (recipe.source?.nativeTerrain?.kind !== "runewaker-wdb-terrain") {
throw new Error(`${title} recipe source.nativeTerrain.kind must be 'runewaker-wdb-terrain'.`);
}
workDirectory = path.join(projectRoot, "runewaker-export-work", slug);
const defaultManifest = path.join(workDirectory, "native-terrain-source", "terrain-native-source.json");
const defaultWdbReport = path.join(workDirectory, "raw", "wdb-report.json");
const defaultOutputDirectory = path.join(workDirectory, "converted", "terrain");
const manifestFile = projectOption("--manifest", defaultManifest);
const wdbReportFile = projectOption("--wdb-report", defaultWdbReport);
const outputDirectory = projectOption("--output-directory", defaultOutputDirectory);
const texconv = path.resolve(option("--texconv", texconvDefault));
const sourceDirectory = path.dirname(manifestFile);
const stageDirectory = await mkdtemp(path.join(workDirectory, ".native-terrain-stage-"));
const stageVisual = path.join(stageDirectory, `${slug}-terrain-visual.glb`);
const stageCollision = path.join(stageDirectory, `${slug}-terrain-collision.glb`);
const stageTextures = path.join(stageDirectory, "texconv");
const rawTextureDirectory = path.join(workDirectory, "raw", "textures");
const preparedTextureDirectory = path.join(workDirectory, "prepared", "textures");
try {
await Promise.all([
verifyExists(manifestFile, "Native terrain manifest"),
verifyExists(wdbReportFile, `${title} WDB report`),
verifyExists(texconv, "DirectXTex texconv"),
]);
const [manifest, wdbReport] = await Promise.all([
readJson(manifestFile, "native terrain manifest"),
readJson(wdbReportFile, `${title} WDB report`),
]);
if (manifest.schemaVersion !== 1 || !manifest.source?.terrainHeader) {
throw new Error("Unsupported native terrain manifest schema or missing source terrain header.");
}
const descriptor = selectPrimaryDescriptor(wdbReport, recipe);
// Validate the primary descriptor before deriving its inverse transform.
descriptorSourceMatrix(descriptor);
const metersPerSourceUnit = Number(recipe.coordinateSystem.metersPerSourceUnit);
const sourceToThree = sourceTerrainToThreeMatrix(descriptor, metersPerSourceUnit);
const sourceChunks = await loadTerrainChunks(manifest, sourceDirectory);
const converted = convertNativeTerrainChunks(sourceChunks, sourceToThree);
const textures = await decodeTerrainImages(
sourceChunks,
sourceDirectory,
stageTextures,
texconv,
rawTextureDirectory,
preparedTextureDirectory,
);
const outputDirectoryResolved = ensureInside(projectRoot, outputDirectory, "Terrain output directory");
await mkdir(outputDirectoryResolved, { recursive: true });
const visualFile = path.join(outputDirectoryResolved, `${slug}-terrain-visual.glb`);
const collisionFile = path.join(outputDirectoryResolved, `${slug}-terrain-collision.glb`);
await writeNativeTerrainGlbs({
converted,
textureSources: textures,
provenance: {
dungeonId: slug,
dungeonTitle: title,
sourceManifestSha256: await hashFile(manifestFile),
sourceWdbSha256: manifest.source.wdbSha256,
primaryPlacementGuid: descriptor.guid,
metersPerSourceUnit,
partitionVisualChunks: recipe.source.nativeTerrain.partitionVisualChunks === true,
...(recipe.source.nativeTerrain?.cliffProjection?.kind === "source-scaled-side-projection" ? { cliffProjection: {
sourceFromAsset: sourceToThree.clone().invert().toArray(),
chunkSizeSourceUnits: manifest.source.terrainHeader.chunkDimension * manifest.source.terrainHeader.gridSize,
} } : {}),
textureSourceDDS: textures.textureReport.map((texture) => texture.sourceDDS),
},
visualFile: stageVisual,
collisionFile: stageCollision,
});
await mkdir(path.dirname(visualFile), { recursive: true });
await Promise.all([
copyFile(stageVisual, visualFile),
copyFile(stageCollision, collisionFile),
]);
const visualBytes = await readFile(visualFile);
const collisionBytes = await readFile(collisionFile);
const report = {
schemaVersion: 1,
dungeonId: recipe.slug,
generatedAt: new Date().toISOString(),
source: {
manifest: path.relative(projectRoot, manifestFile).split(path.sep).join("/"),
manifestSha256: await hashFile(manifestFile),
wdbReport: path.relative(projectRoot, wdbReportFile).split(path.sep).join("/"),
wdbSha256: manifest.source.wdbSha256,
terrainChunkCount: sourceChunks.length,
primaryPlacementGuid: descriptor.guid,
primaryPlacement: descriptor,
geometry: "native WDB heightfield; exact source grid topology; no resampling",
waterPayloadPolicy: manifest.source.waterPayloadPolicy,
normals: "source terrain shader vertexData RGB decoded, inverse-transpose transformed, and preserved per chunk",
sourceTriangleVisibility: "retained as per-primitive base64 Uint8Array in visual GLB extras and summarized below",
materialProjection: recipe.source.nativeTerrain?.cliffProjection ?? { kind: "native-xz" },
textures: textures.textureReport,
},
coordinateSystem: {
source: "Runewaker left-handed Y-up world coordinates",
transform: "inverse primary WDB descriptor TRS, then reflected-Z meter bridge",
metersPerSourceUnit,
},
geometry: converted.report,
visual: {
file: path.basename(visualFile),
fileRelativeToReport: "same directory",
sha256: hashBytes(visualBytes),
bytes: visualBytes.byteLength,
triangles: converted.report.sourceTriangleSlots,
meshObjects: recipe.source.nativeTerrain.partitionVisualChunks === true ? converted.chunks.length : 1,
partition: recipe.source.nativeTerrain.partitionVisualChunks === true ? "native-chunk-meshes" : "shared-grid-mesh",
vertices: converted.vertexCount,
primitives: converted.chunks.length,
sourceVisibleTriangles: converted.report.visibleSourceTriangles,
sourceMaskedTriangleSlotsRetained: converted.report.maskedSourceTriangles,
},
collision: {
file: path.basename(collisionFile),
fileRelativeToReport: "same directory",
sha256: hashBytes(collisionBytes),
bytes: collisionBytes.byteLength,
triangles: converted.report.collisionTriangles,
meshObjects: 1,
excludedSourceMaskedOrDegenerateTriangles: converted.report.excludedCollisionTriangles,
},
standardTextureCache: {
rawDirectory: path.relative(projectRoot, rawTextureDirectory).split(path.sep).join("/"),
preparedDirectory: path.relative(projectRoot, preparedTextureDirectory).split(path.sep).join("/"),
imageNamesMatchGlbTextures: true,
},
};
const reportFile = path.join(outputDirectoryResolved, "conversion-report.json");
await writeFile(reportFile, JSON.stringify(report, null, 2) + "\n", "utf8");
console.log(`[${slug}-terrain] visual: ${path.relative(projectRoot, visualFile).split(path.sep).join("/")} (${report.visual.triangles} source slots)`);
console.log(`[${slug}-terrain] collision: ${path.relative(projectRoot, collisionFile).split(path.sep).join("/")} (${report.collision.triangles} visible triangles)`);
console.log(`[${slug}-terrain] report: ${path.relative(projectRoot, reportFile).split(path.sep).join("/")}`);
} finally {
const absoluteStage = path.resolve(stageDirectory);
const absoluteWork = path.resolve(workDirectory);
const relativeStage = path.relative(absoluteWork, absoluteStage);
if (relativeStage && !relativeStage.startsWith("..") && !path.isAbsolute(relativeStage)) {
await rm(absoluteStage, { recursive: true, force: true });
}
}
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
main().catch((error) => {
console.error(`[sand-vortex-terrain] ${error.stack ?? error.message}`);
process.exitCode = 1;
});
}