import { spawn } from "node:child_process"; import { access, copyFile, mkdir, readFile, readdir, stat, unlink, writeFile, } from "node:fs/promises"; import path from "node:path"; import process from "node:process"; import { fileURLToPath } from "node:url"; import sharp from "sharp"; const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); const projectRoot = path.resolve(scriptDirectory, "../.."); const gltfTransformCli = path.join(projectRoot, "node_modules/@gltf-transform/cli/bin/cli.js"); const genericCompressor = path.join(projectRoot, "scripts/asset-pipeline/compress-ktx2.mjs"); const KTX2_IDENTIFIER = Buffer.from([0xab, 0x4b, 0x54, 0x58, 0x20, 0x32, 0x30, 0xbb, 0x0d, 0x0a, 0x1a, 0x0a]); const REVIEWED_TEXTURE_NAMES = [ "obj_forge01_lava", "lava", "magma_01", "magic_magma", "magic_magma_02", "floor_m_02", "lymor grave_relief_01", "treasure", "wretched_tunnel", "rom-barren-wretched-tunnel-u0-v0", "rom-barren-wretched-tunnel-u3-v0", "rom-barren-wretched-tunnel-u0-v1", "rom-barren-wretched-tunnel-u3-v1", "rom-barren-wretched-tunnel-wall-detail-u1-v0", "echoes_of_the_sea_01", "echoes_of_the_sea_02", "echoes_of_the_sea_03", "echoes_of_the_sea_04", "elves_vine", ]; const REVIEWED_TEXTURE_NAME_SET = new Set(REVIEWED_TEXTURE_NAMES.map((name) => name.toLowerCase())); const TEMP_TARGET_NAME_PREFIX = "__rom_high_quality__"; const TEMP_TARGET_NAME_SUFFIX = "__"; const REVIEWED_TEXTURE_PATTERN = `${TEMP_TARGET_NAME_PREFIX}{${REVIEWED_TEXTURE_NAMES.join(",")}}${TEMP_TARGET_NAME_SUFFIX}`; function argumentValue(args, name, fallback) { const index = args.indexOf(name); return index >= 0 && args[index + 1] ? args[index + 1] : fallback; } function positionalArguments(args) { const valueOptions = new Set(["--concurrency", "--jobs", "--report"]); const result = []; for (let index = 0; index < args.length; index += 1) { if (valueOptions.has(args[index])) { index += 1; continue; } if (!args[index].startsWith("--")) result.push(args[index]); } return result; } async function exists(file) { try { await access(file); return true; } catch { return false; } } async function collectGlbs(target) { const resolved = path.resolve(projectRoot, target); const details = await stat(resolved); if (details.isFile()) { if (!resolved.toLowerCase().endsWith(".glb")) { throw new Error(`Expected a .glb input, received ${path.relative(projectRoot, resolved)}.`); } return [resolved]; } const files = []; for (const entry of await readdir(resolved, { withFileTypes: true })) { const child = path.join(resolved, entry.name); if (entry.isDirectory()) files.push(...await collectGlbs(child)); else if (entry.isFile() && !entry.name.startsWith(".") && entry.name.toLowerCase().endsWith(".glb")) { files.push(child); } } return files; } function parseGlb(buffer, label) { if (buffer.length < 20 || buffer.toString("utf8", 0, 4) !== "glTF") { throw new Error(`${label} is not a binary glTF file.`); } const chunks = []; let json; let jsonChunkIndex = -1; let binaryChunk; let offset = 12; while (offset + 8 <= buffer.length) { const length = buffer.readUInt32LE(offset); const type = buffer.readUInt32LE(offset + 4); const dataStart = offset + 8; const dataEnd = dataStart + length; if (dataEnd > buffer.length) throw new Error(`${label} contains a truncated GLB chunk.`); const data = buffer.subarray(dataStart, dataEnd); chunks.push({ type, data }); if (type === 0x4e4f534a) { jsonChunkIndex = chunks.length - 1; json = JSON.parse(data.toString("utf8").replace(/\0+$/u, "").trim()); } if (type === 0x004e4942) binaryChunk = data; offset = dataEnd; } if (!json || !binaryChunk) throw new Error(`${label} must contain JSON and BIN chunks.`); return { json, chunks, jsonChunkIndex, binaryChunk }; } function serializeGlb(source, json, chunks, jsonChunkIndex) { const encodedJson = Buffer.from(JSON.stringify(json)); const paddedJson = Buffer.alloc(Math.ceil(encodedJson.length / 4) * 4, 0x20); encodedJson.copy(paddedJson); const updatedChunks = chunks.map((chunk, index) => ( index === jsonChunkIndex ? { ...chunk, data: paddedJson } : chunk )); const totalLength = 12 + updatedChunks.reduce((total, chunk) => total + 8 + chunk.data.length, 0); const output = Buffer.alloc(totalLength); source.copy(output, 0, 0, 12); output.writeUInt32LE(totalLength, 8); let offset = 12; for (const chunk of updatedChunks) { output.writeUInt32LE(chunk.data.length, offset); output.writeUInt32LE(chunk.type, offset + 4); chunk.data.copy(output, offset + 8); offset += 8 + chunk.data.length; } return output; } function patchGlbJson(source, transform) { const parsed = parseGlb(source, "temporary GLB"); transform(parsed.json); return serializeGlb(source, parsed.json, parsed.chunks, parsed.jsonChunkIndex); } function basenameWithoutExtension(value) { const basename = path.posix.basename(String(value ?? "").replaceAll("\\", "/")); return basename.replace(/\.[^.]+$/u, ""); } function canonicalTextureName(value) { return basenameWithoutExtension(value).replace(/^[0-9a-f]{8}_/iu, ""); } function imageIndexForTexture(texture) { return texture?.extensions?.KHR_texture_basisu?.source ?? texture?.source; } function imageBytes(parsed, image, label) { if (image.bufferView === undefined) { throw new Error(`${label} has an external or missing image buffer; embedded GLB images are required.`); } const view = parsed.json.bufferViews?.[image.bufferView]; if (!view) throw new Error(`${label} references missing buffer view ${image.bufferView}.`); const start = (view.byteOffset ?? 0); return parsed.binaryChunk.subarray(start, start + view.byteLength); } function textureSlotUsage(json) { const baseColor = new Set(); for (const material of json.materials ?? []) { const index = material.pbrMetallicRoughness?.baseColorTexture?.index; if (Number.isInteger(index)) baseColor.add(index); } return baseColor; } function inspectKtx2(bytes, label) { if (bytes.length < 48 || !bytes.subarray(0, 12).equals(KTX2_IDENTIFIER)) { throw new Error(`${label} declares image/ktx2 but has an invalid KTX2 header.`); } const vkFormat = bytes.readUInt32LE(12); const supercompressionScheme = bytes.readUInt32LE(44); return { width: bytes.readUInt32LE(20), height: bytes.readUInt32LE(24), mipLevels: Math.max(1, bytes.readUInt32LE(40)), vkFormat, supercompressionScheme, codec: supercompressionScheme === 1 ? "ETC1S" : vkFormat === 0 ? "UASTC" : "unknown", }; } async function imageDimensions(parsed, image, label) { const bytes = imageBytes(parsed, image, label); if (image.mimeType === "image/ktx2") return inspectKtx2(bytes, label); const metadata = await sharp(bytes).metadata(); if (!metadata.width || !metadata.height) throw new Error(`${label} has no readable dimensions.`); return { width: metadata.width, height: metadata.height, mipLevels: 0, codec: "source" }; } async function textureInventory(buffer, label) { const parsed = parseGlb(buffer, label); const baseColorTextures = textureSlotUsage(parsed.json); const textures = []; for (const [textureIndex, texture] of (parsed.json.textures ?? []).entries()) { const imageIndex = imageIndexForTexture(texture); const image = parsed.json.images?.[imageIndex]; if (!image) throw new Error(`${label} texture ${textureIndex} has no source image.`); const imageName = canonicalTextureName(image.name ?? image.uri); const bytes = imageBytes(parsed, image, `${label} image ${image.name ?? imageIndex}`); const dimensions = image.mimeType === "image/ktx2" ? inspectKtx2(bytes, `${label} image ${image.name ?? imageIndex}`) : await imageDimensions(parsed, image, `${label} image ${image.name ?? imageIndex}`); textures.push({ textureIndex, imageIndex, name: imageName, imageName: image.name ?? image.uri ?? `image-${imageIndex}`, mimeType: image.mimeType, isBaseColor: baseColorTextures.has(textureIndex), dimensions, alphaModes: (parsed.json.materials ?? []) .filter((material) => material.pbrMetallicRoughness?.baseColorTexture?.index === textureIndex) .map((material) => material.alphaMode ?? "OPAQUE"), }); } return { parsed, textures }; } function normalizeTargetTextureNames(sourceBuffer, label) { const parsed = parseGlb(sourceBuffer, label); const baseColorTextures = textureSlotUsage(parsed.json); // Decide from immutable image names before touching aliases. Several // textures can reference one image while carrying independent samplers; // mutating the image name on the first alias must not make the next alias // look like a non-target. const originalImageNames = (parsed.json.images ?? []).map((image) => canonicalTextureName(image.name ?? image.uri)); const baseColorImageIndexes = new Set(); for (const textureIndex of baseColorTextures) { const imageIndex = imageIndexForTexture(parsed.json.textures?.[textureIndex]); if (Number.isInteger(imageIndex)) baseColorImageIndexes.add(imageIndex); } const selectedImageNames = new Map(); for (const imageIndex of baseColorImageIndexes) { const name = originalImageNames[imageIndex]; if (name && REVIEWED_TEXTURE_NAME_SET.has(name.toLowerCase())) selectedImageNames.set(imageIndex, name); } const selected = []; for (const textureIndex of baseColorTextures) { const texture = parsed.json.textures?.[textureIndex]; if (!texture) continue; const imageIndex = imageIndexForTexture(texture); const image = parsed.json.images?.[imageIndex]; const normalized = originalImageNames[imageIndex]; if (!image || !normalized) continue; if (selectedImageNames.has(imageIndex)) { const temporaryName = `${TEMP_TARGET_NAME_PREFIX}${normalized}${TEMP_TARGET_NAME_SUFFIX}`; texture.name = temporaryName; selected.push({ textureIndex, imageIndex, name: normalized, mimeType: image.mimeType }); } else { // The CLI matcher uses micromatch's `contains: true` option, so bare // names like "lava" would also match "lava_roughness". Keep non-target // base-color names out of the reviewed-name matcher for this one pass. const temporaryName = `__rom_quality_exclude_${textureIndex}__`; texture.name = temporaryName; } } for (const imageIndex of baseColorImageIndexes) { const image = parsed.json.images?.[imageIndex]; if (!image) continue; const targetName = selectedImageNames.get(imageIndex); image.name = targetName ? `${TEMP_TARGET_NAME_PREFIX}${targetName}${TEMP_TARGET_NAME_SUFFIX}` : `__rom_quality_exclude_image_${imageIndex}__`; } return { buffer: selected.length ? serializeGlb(sourceBuffer, parsed.json, parsed.chunks, parsed.jsonChunkIndex) : sourceBuffer, selected, }; } function runProcess(executable, args, environment, label) { return new Promise((resolve, reject) => { const child = spawn(executable, args, { cwd: projectRoot, env: environment, windowsHide: true, stdio: ["ignore", "pipe", "pipe"], }); let output = ""; child.stdout.on("data", (chunk) => { output += chunk; }); child.stderr.on("data", (chunk) => { output += chunk; }); child.once("error", reject); child.once("exit", (code) => { if (code === 0) resolve(output); else reject(new Error(`${label} failed with exit code ${code}.\n${output.trim()}`)); }); }); } async function findToktxDirectory() { const executable = process.platform === "win32" ? "toktx.exe" : "toktx"; const candidates = [ process.env.KTX_BIN_DIR, path.join(projectRoot, ".tools/ktx/bin"), ...(process.env.PATH ?? "").split(path.delimiter), ].filter(Boolean); for (const candidate of candidates) { if (await exists(path.join(candidate, executable))) return path.resolve(candidate); } throw new Error("toktx was not found. Run scripts/asset-pipeline/bootstrap-ktx.ps1 or set KTX_BIN_DIR."); } function fullMipLevelCount(width, height) { return Math.floor(Math.log2(Math.max(width, height))) + 1; } function sameOrderedValues(left, right) { return left.length === right.length && left.every((value, index) => value === right[index]); } async function compressEnvironmentGlb(file, { jobs, ktxBinDirectory }) { const sourceBuffer = await readFile(file); const before = sourceBuffer.length; const label = path.relative(projectRoot, file).replaceAll("\\", "/"); const inputInventory = await textureInventory(sourceBuffer, label); const normalized = normalizeTargetTextureNames(sourceBuffer, label); const selectedTargets = normalized.selected; for (const target of selectedTargets) { if (target.mimeType !== "image/png" && target.mimeType !== "image/jpeg" && target.mimeType !== "image/ktx2") { throw new Error(`${label}: reviewed target ${target.name} has unsupported source type ${target.mimeType}.`); } const source = inputInventory.textures.find((entry) => entry.textureIndex === target.textureIndex); if (target.mimeType === "image/ktx2" && (source?.dimensions.codec !== "UASTC" || source.dimensions.supercompressionScheme !== 2)) { throw new Error( `${label}: reviewed target ${target.name} is already KTX2 but is not UASTC+Zstandard. ` + "Rebuild from its prepared PNG source to apply the reviewed codec policy.", ); } } const directory = path.dirname(file); const stem = path.basename(file, path.extname(file)); const nonce = `${process.pid}-${Math.random().toString(36).slice(2, 8)}`; const namedInput = path.join(directory, `.${stem}.${nonce}.named.glb`); const uastcOutput = path.join(directory, `.${stem}.${nonce}.uastc.glb`); const temporaryOutput = path.join(directory, `.${stem}.${nonce}.mixed.glb`); const environment = { ...process.env, KTX_BIN_DIR: ktxBinDirectory, PATH: `${ktxBinDirectory}${path.delimiter}${process.env.PATH ?? ""}`, }; try { await writeFile(namedInput, normalized.buffer); await runProcess(process.execPath, [ gltfTransformCli, "uastc", namedInput, uastcOutput, "--pattern", REVIEWED_TEXTURE_PATTERN, "--slots", "baseColorTexture", "--level", "2", "--zstd", "18", "--jobs", String(jobs), ], environment, `UASTC pass for ${label}`); const selectiveInventory = await textureInventory(await readFile(uastcOutput), label); for (const target of selectedTargets) { const output = selectiveInventory.textures.find((entry) => entry.textureIndex === target.textureIndex); if (output?.dimensions.codec !== "UASTC") { throw new Error(`${label}: UASTC pattern pass did not encode reviewed target ${target.name} ` + `(texture ${output?.textureIndex ?? "missing"}, image ${output?.imageName ?? "missing"}, ` + `${output?.mimeType ?? "missing"}, codec ${output?.dimensions?.codec ?? "missing"}; ` + `output=${JSON.stringify(selectiveInventory.textures.map((entry) => ({ textureIndex: entry.textureIndex, imageIndex: entry.imageIndex, imageName: entry.imageName, mimeType: entry.mimeType, codec: entry.dimensions.codec })))}.`); } } for (const source of inputInventory.textures) { if (!source.isBaseColor || REVIEWED_TEXTURE_NAME_SET.has(source.name.toLowerCase())) continue; const output = selectiveInventory.textures.find((entry) => entry.textureIndex === source.textureIndex); if (source.mimeType !== "image/ktx2" && output?.mimeType === "image/ktx2") { throw new Error( `${label}: selective UASTC pattern matched non-target base-color ${source.name} ` + "before the ETC1S fallback pass.", ); } } await runProcess(process.execPath, [ genericCompressor, uastcOutput, "--force", "--concurrency", "1", "--jobs", String(jobs), ], environment, `ETC1S fallback pass for ${label}`); const encodedBuffer = await readFile(uastcOutput); const outputBuffer = patchGlbJson(encodedBuffer, (json) => { for (const [textureIndex, originalTexture] of inputInventory.parsed.json.textures.entries()) { const outputTexture = json.textures?.[textureIndex]; if (!outputTexture) throw new Error(`${label}: a texture was removed during compression.`); if (originalTexture.name === undefined) delete outputTexture.name; else outputTexture.name = originalTexture.name; } for (const [imageIndex, originalImage] of inputInventory.parsed.json.images.entries()) { const outputImage = json.images?.[imageIndex]; if (!outputImage) throw new Error(`${label}: an image was removed during compression.`); if (originalImage.name === undefined) delete outputImage.name; else outputImage.name = originalImage.name; } }); const outputInventory = await textureInventory(outputBuffer, label); const outputImageNames = outputInventory.textures.map((entry) => `${entry.imageIndex}:${entry.name}`); const inputImageNames = inputInventory.textures.map((entry) => `${entry.imageIndex}:${entry.name}`); if (!sameOrderedValues(inputImageNames, outputImageNames)) { throw new Error(`${label}: texture/image linkage or ordering changed during compression.`); } if (outputInventory.textures.length !== inputInventory.textures.length) { throw new Error(`${label}: texture count changed during compression.`); } for (const [textureIndex, sourceTexture] of inputInventory.parsed.json.textures.entries()) { if (outputInventory.parsed.json.textures?.[textureIndex]?.name !== sourceTexture.name) { throw new Error(`${label}: texture name ${textureIndex} changed during compression.`); } } for (const [imageIndex, sourceImage] of inputInventory.parsed.json.images.entries()) { if (outputInventory.parsed.json.images?.[imageIndex]?.name !== sourceImage.name) { throw new Error(`${label}: image name ${imageIndex} changed during compression.`); } } for (const source of inputInventory.textures) { const output = outputInventory.textures.find((entry) => entry.textureIndex === source.textureIndex); if (!output || output.mimeType !== "image/ktx2") { throw new Error(`${label}: texture ${source.imageName} was not KTX2-compressed.`); } if (source.dimensions.width !== output.dimensions.width || source.dimensions.height !== output.dimensions.height) { throw new Error( `${label}: texture ${source.imageName} changed dimensions ` + `${source.dimensions.width}x${source.dimensions.height} -> ` + `${output.dimensions.width}x${output.dimensions.height}.`, ); } const expectedMips = fullMipLevelCount(source.dimensions.width, source.dimensions.height); if (output.dimensions.mipLevels !== expectedMips) { throw new Error(`${label}: texture ${source.imageName} has ${output.dimensions.mipLevels} mips; expected ${expectedMips}.`); } if (source.isBaseColor && !REVIEWED_TEXTURE_NAME_SET.has(source.name.toLowerCase()) && output.dimensions.codec !== "ETC1S") { throw new Error( `${label}: non-target base-color ${source.name} did not remain ETC1S ` + `(image ${output.imageName}, observed ${output.dimensions.codec}, ` + `supercompression=${output.dimensions.supercompressionScheme}).`, ); } } const materialModes = (buffer) => parseGlb(buffer, label).json.materials?.map((material) => ({ name: material.name ?? "", alphaMode: material.alphaMode ?? "OPAQUE", alphaCutoff: material.alphaCutoff ?? null, })) ?? []; if (JSON.stringify(materialModes(sourceBuffer)) !== JSON.stringify(materialModes(outputBuffer))) { throw new Error(`${label}: material alpha mode/cutoff changed during compression.`); } const targetOutputs = selectedTargets.map((target) => { const output = outputInventory.textures.find((entry) => entry.textureIndex === target.textureIndex); if (output?.dimensions.codec !== "UASTC" || output.dimensions.supercompressionScheme !== 2) { throw new Error(`${label}: selective UASTC encoder did not upgrade reviewed texture ${target.name}.`); } return { name: target.name, image: output.imageName, width: output.dimensions.width, height: output.dimensions.height, supercompressionScheme: output.dimensions.supercompressionScheme, }; }); await writeFile(temporaryOutput, outputBuffer); await copyFile(temporaryOutput, file); const after = (await stat(file)).size; return { file: label, status: targetOutputs.length ? "mixed-quality-compressed" : "etc1s-compressed-no-targets", beforeBytes: before, afterBytes: after, textureCounts: { total: outputInventory.textures.length, etc1s: outputInventory.textures.filter((entry) => entry.dimensions.codec === "ETC1S").length, uastc: outputInventory.textures.filter((entry) => entry.dimensions.codec === "UASTC").length, baseColorTargets: targetOutputs.length, }, uastcTargets: targetOutputs, }; } finally { for (const temporaryFile of [namedInput, uastcOutput, temporaryOutput]) { try { await unlink(temporaryFile); } catch (error) { if (error?.code !== "ENOENT") throw error; } } } } async function main() { const args = process.argv.slice(2); const jobs = Math.max(1, Number.parseInt(argumentValue(args, "--jobs", "2"), 10)); const concurrency = Math.max(1, Number.parseInt(argumentValue(args, "--concurrency", "1"), 10)); const reportPath = argumentValue(args, "--report", null); const targets = positionalArguments(args); if (!targets.length) { throw new Error("Provide one or more GLB files or directories; this wrapper does not scan the workspace implicitly."); } const ktxBinDirectory = await findToktxDirectory(); const files = [...new Set((await Promise.all(targets.map(collectGlbs))).flat())].sort(); if (!files.length) throw new Error("No GLB inputs were found."); console.log( `RoM environment texture policy: ${files.length} GLBs, UASTC targets=${REVIEWED_TEXTURE_NAMES.length}, ` + `UASTC level 2 + Zstandard 18 (RDO disabled by CLI default), ETC1S fallback, concurrency=${concurrency}, jobs=${jobs}.`, ); const results = []; const errors = []; let cursor = 0; const workers = Array.from({ length: Math.min(concurrency, files.length) }, async () => { while (cursor < files.length) { const file = files[cursor++]; try { const result = await compressEnvironmentGlb(file, { jobs, ktxBinDirectory }); results.push(result); console.log( `[${results.length + errors.length}/${files.length}] ${result.status}: ${result.file} ` + `${result.textureCounts.uastc} UASTC / ${result.textureCounts.etc1s} ETC1S`, ); } catch (error) { errors.push({ file, message: error.message }); console.error(`Texture compression failed: ${path.relative(projectRoot, file)}\n${error.message}`); } } }); await Promise.all(workers); const totals = results.reduce((total, result) => ({ files: total.files + 1, textures: total.textures + result.textureCounts.total, etc1s: total.etc1s + result.textureCounts.etc1s, uastc: total.uastc + result.textureCounts.uastc, baseColorTargets: total.baseColorTargets + result.textureCounts.baseColorTargets, beforeBytes: total.beforeBytes + result.beforeBytes, afterBytes: total.afterBytes + result.afterBytes, }), { files: 0, textures: 0, etc1s: 0, uastc: 0, baseColorTargets: 0, beforeBytes: 0, afterBytes: 0 }); console.log( `RoM texture compression complete: ${results.length} files, ${totals.baseColorTargets} reviewed UASTC targets, ` + `${totals.etc1s} ETC1S / ${totals.uastc} UASTC textures, ${errors.length} failed.`, ); if (reportPath) { const absoluteReportPath = path.resolve(projectRoot, reportPath); await mkdir(path.dirname(absoluteReportPath), { recursive: true }); await writeFile(absoluteReportPath, `${JSON.stringify({ schemaVersion: 1, generatedAt: new Date().toISOString(), policy: { reviewedBaseColorNames: REVIEWED_TEXTURE_NAMES, targetCodec: "UASTC level 2, Zstandard 18, RDO disabled", fallbackCodec: "ETC1S existing pipeline defaults", preservesSourceDimensionsAndFullMipChains: true, preservesMaterialAlphaModes: true, }, totals, files: results, errors: errors.map(({ file, message }) => ({ file: path.relative(projectRoot, file).replaceAll("\\", "/"), message, })), }, null, 2)}\n`, "utf8"); } if (errors.length) process.exitCode = 1; } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { await main(); }