#!/usr/bin/env node import { createHash } from "node:crypto"; import { spawn } from "node:child_process"; import { access, mkdir, readFile, readdir, stat, writeFile, } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { NodeIO } from "@gltf-transform/core"; import { ALL_EXTENSIONS } from "@gltf-transform/extensions"; import { MeshoptDecoder } from "meshoptimizer"; const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const extractedRoot = path.resolve(process.env.ASCENSION_EXTRACTED_ROOT ?? path.join(projectRoot, "../LadiksMPQEditor")); const catalogFile = path.resolve( process.env.MANASTORM_CATALOG ?? path.join(projectRoot, "src/game/generated/manastormCatalog.json"), ); const workRoot = path.resolve( process.env.MANASTORM_PIPELINE_WORK ?? path.join(projectRoot, "..", "HealerMan-Storage", "pipeline-work", "manastorm"), ); const shippingRoot = path.join(projectRoot, "public/assets/game/manastorm"); const registryFile = path.join(projectRoot, "src/game/generated/manastormAssetPackages.json"); const epochCatalogFile = path.join( projectRoot, "src/game/generated/epochDungeonClientCatalog.json", ); const archivedOpenWorldFile = path.join( projectRoot, "open-world-assets/manastorm/archive.json", ); async function exists(file) { try { await access(file); return true; } catch { return false; } } async function readJson(file) { return JSON.parse(await readFile(file, "utf8")); } async function archivedOpenWorldMaps() { if (!await exists(archivedOpenWorldFile)) return new Map(); const archive = await readJson(archivedOpenWorldFile); const packages = Array.isArray(archive.packages) ? archive.packages : []; return new Map( packages .filter((assetPackage) => Number.isInteger(assetPackage.mapId)) .map((assetPackage) => [assetPackage.mapId, assetPackage]), ); } async function writeJson(file, value) { await mkdir(path.dirname(file), { recursive: true }); await writeFile(file, `${JSON.stringify(value, null, 2)}\n`, "utf8"); } function slugPart(value) { return String(value ?? "") .normalize("NFKD") .replace(/[\u0300-\u036f]/g, "") .toLowerCase() .replace(/[^a-z0-9]+/g, "-") .replace(/^-|-$/g, "") || "unknown"; } function mapSlug(map) { return `${map.mapId}-${slugPart(map.directory || map.name)}`; } async function sha256(file) { const hash = createHash("sha256"); hash.update(await readFile(file)); return hash.digest("hex"); } function relativeProjectPath(file) { return path.relative(projectRoot, file).replace(/\\/g, "/"); } let glbIo; async function inspectGlb(file) { await MeshoptDecoder.ready; glbIo ??= new NodeIO() .registerExtensions(ALL_EXTENSIONS) .registerDependencies({ "meshopt.decoder": MeshoptDecoder }); const document = await glbIo.read(file); const root = document.getRoot(); if (!root.listScenes().length) throw new Error(`${relativeProjectPath(file)} has no scene.`); for (const accessor of root.listAccessors()) { const values = accessor.getArray(); if (values && !Array.from(values).every(Number.isFinite)) { throw new Error(`${relativeProjectPath(file)} contains non-finite accessor data.`); } } const triangleCount = root.listMeshes().reduce((total, mesh) => ( total + mesh.listPrimitives().reduce((meshTotal, primitive) => { const indices = primitive.getIndices()?.getCount(); const positions = primitive.getAttribute("POSITION")?.getCount() ?? 0; return meshTotal + Math.floor((indices ?? positions) / 3); }, 0) ), 0); if (!triangleCount) throw new Error(`${relativeProjectPath(file)} contains no triangles.`); return { triangleCount }; } async function directoryChecksum(directory) { const files = []; const queue = [directory]; while (queue.length) { const current = queue.pop(); for (const entry of await readdir(current, { withFileTypes: true })) { const target = path.join(current, entry.name); if (entry.isDirectory()) queue.push(target); else files.push(target); } } files.sort((left, right) => left.localeCompare(right)); const hash = createHash("sha256"); for (const file of files) { hash.update(path.relative(directory, file).replace(/\\/g, "/")); hash.update(await readFile(file)); } return hash.digest("hex"); } async function caseInsensitiveChild(parent, child) { if (!await exists(parent)) return null; const entries = await readdir(parent, { withFileTypes: true }); const match = entries.find((entry) => entry.name.toLowerCase() === child.toLowerCase()); return match ? path.join(parent, match.name) : null; } async function extractedArchiveRoots() { const roots = []; for (const entry of await readdir(extractedRoot, { withFileTypes: true })) { if (!entry.isDirectory()) continue; const direct = path.join(extractedRoot, entry.name); roots.push(direct); if (entry.name.toLowerCase() === "client-current") { for (const child of await readdir(direct, { withFileTypes: true })) { if (child.isDirectory()) roots.push(path.join(direct, child.name)); } } } return roots; } async function findMapDirectories(directory) { if (!directory) return []; const results = []; for (const root of await extractedArchiveRoots()) { const world = await caseInsensitiveChild(root, "world"); const maps = world ? await caseInsensitiveChild(world, "maps") : null; const target = maps ? await caseInsensitiveChild(maps, directory) : null; if (target) results.push(target); } return results; } async function summarizeSourceDirectory(directory) { const summary = { files: 0, bytes: 0, adt: 0, wdt: 0, wdl: 0, wmo: 0 }; const queue = [directory]; while (queue.length) { const current = queue.pop(); for (const entry of await readdir(current, { withFileTypes: true })) { const file = path.join(current, entry.name); if (entry.isDirectory()) { queue.push(file); continue; } const info = await stat(file); const extension = path.extname(entry.name).slice(1).toLowerCase(); summary.files += 1; summary.bytes += info.size; if (extension in summary) summary[extension] += 1; } } return summary; } async function loadCatalog() { if (!await exists(catalogFile)) { throw new Error(`Missing generated Manastorm catalog: ${catalogFile}`); } const catalog = await readJson(catalogFile); if (catalog.schemaVersion !== 1 || !Array.isArray(catalog.maps)) { throw new Error("The generated Manastorm catalog has an unsupported schema."); } if (!await exists(epochCatalogFile)) return catalog; const epoch = await readJson(epochCatalogFile); const mapsById = new Map(catalog.maps.map((map) => [map.mapId, map])); for (const map of epoch.maps ?? []) { if (mapsById.has(map.mapId)) continue; mapsById.set(map.mapId, { id: `epoch:map:${map.mapId}`, mapId: map.mapId, name: map.title || map.name, directory: map.directory, instanceType: map.instanceType, flags: map.flags, expansionId: map.expansionId, maxPlayers: map.maxPlayers, sourceRowIds: [`epoch:Map.dbc:${map.mapId}`], encounterIds: (epoch.encounters ?? []) .filter((encounter) => encounter.mapId === map.mapId) .map((encounter) => `epoch:encounter:${encounter.encounterId}`), }); } return { ...catalog, catalogId: `${catalog.catalogId}+project-epoch`, maps: [...mapsById.values()].sort((left, right) => left.mapId - right.mapId), }; } async function discoverSources() { const catalog = await loadCatalog(); const maps = []; for (const map of catalog.maps) { const candidates = await findMapDirectories(map.directory); const sources = []; for (const candidate of candidates) { sources.push({ path: path.relative(extractedRoot, candidate).replace(/\\/g, "/"), ...await summarizeSourceDirectory(candidate), }); } maps.push({ id: map.id, mapId: map.mapId, name: map.name, directory: map.directory, slug: mapSlug(map), encounterCount: map.encounterIds?.length ?? 0, status: sources.length ? "source-found" : "source-missing", sources, }); } const snapshot = { schemaVersion: 1, catalogId: catalog.catalogId, generatedAt: "deterministic", sourceRoot: path.basename(extractedRoot), counts: { maps: maps.length, sourceFound: maps.filter((map) => map.sources.length).length, sourceMissing: maps.filter((map) => !map.sources.length).length, }, maps, }; await writeJson(path.join(workRoot, "source-snapshot.json"), snapshot); return snapshot; } async function scaffoldRecipes() { const catalog = await loadCatalog(); const snapshotFile = path.join(workRoot, "source-snapshot.json"); const snapshot = await exists(snapshotFile) ? await readJson(snapshotFile) : await discoverSources(); const byMapId = new Map(snapshot.maps.map((map) => [map.mapId, map])); const recipes = []; for (const map of catalog.maps) { const source = byMapId.get(map.mapId); const slug = mapSlug(map); const recipe = { schemaVersion: 1, id: `manastorm:asset-recipe:${map.mapId}`, slug, mapId: map.mapId, location: map.name, mapDirectory: map.directory, encounterIds: map.encounterIds ?? [], sourceCandidates: source?.sources?.map((candidate) => candidate.path) ?? [], shipping: { visual: `${slug}/visual.glb`, collision: `${slug}/collision.glb`, navigation: `${slug}/navigation.glb`, }, transform: { position: [0, 0, 0], rotation: [0, 0, 0], scale: 1, fromWow: "three(x,y,z)=(-wow.x,wow.z,wow.y)", }, review: { sourceSelection: source?.sources?.length === 1 ? "selected" : "required", playerAnchor: "required", trashAnchors: "required", bossAnchor: "required", portalAnchor: "required", resurrectionAnchor: "required", navmeshProjection: "required", }, }; recipes.push(recipe); await writeJson(path.join(workRoot, "recipes", `${slug}.json`), recipe); } const result = { schemaVersion: 1, status: recipes.every((recipe) => recipe.sourceCandidates.length) ? "review-required" : "blocked", recipes: recipes.length, sourceBlocked: recipes.filter((recipe) => !recipe.sourceCandidates.length).map((recipe) => recipe.slug), }; await writeJson(path.join(workRoot, "recipe-report.json"), result); return result; } function run(command, args) { return new Promise((resolve, reject) => { const child = spawn(command, args, { cwd: projectRoot, stdio: "inherit", windowsHide: true, }); child.once("error", reject); child.once("exit", (code) => code === 0 ? resolve() : reject(new Error(`${path.basename(command)} exited with code ${code}.`))); }); } function roleChunks(assets, role) { const value = assets?.[role]; return Array.isArray(value) ? value : value ? [value] : []; } async function loadStagedAssetSet(staging, phase) { const manifestName = phase === "source" ? "stage-source.json" : "optimized-manifest.json"; const manifestFile = path.join(staging, manifestName); if (await exists(manifestFile)) { const manifest = await readJson(manifestFile); const visual = roleChunks(manifest.assets, "visual"); const collision = roleChunks(manifest.assets, "collision"); const navigation = roleChunks(manifest.assets, "navigation"); if (!visual.length || !collision.length || navigation.length !== 1) { throw new Error(`${manifestName} must contain visual/collision arrays and one navigation chunk.`); } return { manifest, visual, collision, navigation: navigation[0] }; } const suffix = phase === "source" ? ".source" : ""; return { manifest: null, visual: [{ id: "visual", path: `visual${suffix}.glb` }], collision: [{ id: "collision", path: `collision${suffix}.glb` }], navigation: { id: "navigation", path: `navigation${suffix}.glb` }, }; } function optimizedChunkName(role, index, count) { return count === 1 ? `${role}.glb` : `${role}-${String(index).padStart(3, "0")}.glb`; } async function mapWithConcurrency(items, concurrency, worker) { const results = new Array(items.length); let nextIndex = 0; async function consume() { while (nextIndex < items.length) { const index = nextIndex; nextIndex += 1; results[index] = await worker(items[index], index); } } await Promise.all( Array.from( { length: Math.min(concurrency, items.length) }, () => consume(), ), ); return results; } async function optimizeStage(slug) { const staging = path.join(workRoot, "staging", slug); const sourceSet = await loadStagedAssetSet(staging, "source"); const chunks = { visual: sourceSet.visual, collision: sourceSet.collision, navigation: [sourceSet.navigation], }; const blockers = []; for (const [role, entries] of Object.entries(chunks)) { for (const entry of entries) { const file = path.join(staging, entry.path); if (!await exists(file)) blockers.push(`Missing ${role} source ${path.relative(projectRoot, file)}.`); } } if (blockers.length) return { status: "blocked", slug, blockers }; const cli = path.join(projectRoot, "node_modules/@gltf-transform/cli/bin/cli.js"); const optimized = { visual: [], collision: [], navigation: null }; for (const role of ["visual", "collision", "navigation"]) { const entries = chunks[role]; const descriptors = await mapWithConcurrency(entries, 4, async (entry, index) => { const source = path.join(staging, entry.path); const outputName = optimizedChunkName(role, index, entries.length); const output = path.join(staging, outputName); await run(process.execPath, [ cli, "meshopt", source, output, "--level", "high", ]); if (role === "visual") { await run(process.execPath, [ path.join(projectRoot, "scripts/asset-pipeline/compress-ktx2.mjs"), output, "--concurrency", "1", "--jobs", "2", ]); } const geometry = await inspectGlb(output); const descriptor = { ...entry, path: outputName, sourcePath: entry.path, sourceChecksum: entry.checksum ?? await sha256(source), checksum: await sha256(output), byteLength: (await stat(output)).size, triangleCount: geometry.triangleCount, compression: role === "visual" ? "meshopt+ktx2" : "meshopt", }; return descriptor; }); if (role === "navigation") optimized.navigation = descriptors[0]; else optimized[role] = descriptors; } const optimizedManifest = { schemaVersion: 1, slug, mapId: sourceSet.manifest?.mapId, assets: optimized, }; await writeJson(path.join(staging, "optimized-manifest.json"), optimizedManifest); return { status: "review-required", slug, assets: optimized, compression: "meshopt+ktx2-visuals", }; } function validAnchor(anchor) { return anchor && typeof anchor.id === "string" && ["player", "trash", "boss", "portal", "resurrection"].includes(anchor.kind) && Array.isArray(anchor.position) && anchor.position.length === 3 && anchor.position.every(Number.isFinite) && Number.isFinite(anchor.navmeshDistance) && anchor.navmeshDistance >= 0 && anchor.navmeshDistance <= 2 && ["client", "observed", "authored"].includes(anchor.provenance) && (anchor.provenance !== "authored" || typeof anchor.source === "string" && anchor.source.trim()); } async function rebuildRegistry() { const packages = []; if (await exists(shippingRoot)) { for (const entry of await readdir(shippingRoot, { withFileTypes: true })) { if (!entry.isDirectory()) continue; const manifest = path.join(shippingRoot, entry.name, "stage-pack.json"); if (await exists(manifest)) packages.push(await readJson(manifest)); } } packages.sort((left, right) => left.id.localeCompare(right.id)); const registry = { schemaVersion: 1, packages }; await writeJson(registryFile, registry); return registry; } async function buildStagePack(slug) { const recipeFile = path.join(workRoot, "recipes", `${slug}.json`); const anchorsFile = path.join(workRoot, "anchors", `${slug}.json`); if (!await exists(recipeFile)) return { status: "blocked", slug, blockers: ["Stage recipe is missing."] }; if (!await exists(anchorsFile)) return { status: "blocked", slug, blockers: ["Reviewed stage anchors are missing."] }; const recipe = await readJson(recipeFile); const archivedMaps = await archivedOpenWorldMaps(); if (archivedMaps.has(recipe.mapId)) { return { status: "blocked", slug, blockers: [ `Map ${recipe.mapId} is archived in open-world-assets and excluded from shipping.`, ], }; } const anchorReview = await readJson(anchorsFile); const anchors = Array.isArray(anchorReview.anchors) ? anchorReview.anchors : []; const blockers = []; if (!["approved", "provisional"].includes(anchorReview.reviewStatus)) { blockers.push("Stage anchors are neither approved nor explicitly provisional."); } if (!anchors.every(validAnchor)) blockers.push("Stage anchors contain invalid or non-navmesh-projected entries."); for (const kind of ["player", "boss", "portal", "resurrection"]) { if (!anchors.some((anchor) => anchor.kind === kind)) blockers.push(`Stage has no ${kind} anchor.`); } const staging = path.join(workRoot, "staging", slug); const optimizedSet = await loadStagedAssetSet(staging, "optimized"); const stagedChunks = { visual: optimizedSet.visual, collision: optimizedSet.collision, navigation: [optimizedSet.navigation], }; for (const [role, entries] of Object.entries(stagedChunks)) { for (const entry of entries) { if (!await exists(path.join(staging, entry.path))) { blockers.push(`Stage ${role} GLB ${entry.path} is missing.`); } } } if (blockers.length) return { status: "blocked", slug, blockers: [...new Set(blockers)].sort() }; const target = path.join(shippingRoot, slug); await mkdir(target, { recursive: true }); const chunks = { visual: [], collision: [], navigation: null }; for (const role of ["visual", "collision", "navigation"]) { const entries = stagedChunks[role]; for (const entry of entries) { const file = path.join(staging, entry.path); const output = path.join(target, path.basename(entry.path)); await writeFile(output, await readFile(file)); const geometry = await inspectGlb(output); if (role === "collision" && geometry.triangleCount > 100_000) { blockers.push(`Stage collision chunk ${entry.id} has ${geometry.triangleCount} triangles; limit is 100000.`); continue; } const chunk = { id: entry.id ?? role, url: `/assets/game/manastorm/${slug}/${path.basename(output)}`, checksum: await sha256(output), byteLength: (await stat(output)).size, triangleCount: geometry.triangleCount, compression: entry.compression ?? "meshopt", ...(entry.transform ? { transform: entry.transform } : {}), ...(entry.provenance ? { provenance: entry.provenance } : {}), }; if (role === "navigation") chunks.navigation = chunk; else chunks[role].push(chunk); } } if (blockers.length) return { status: "blocked", slug, blockers: [...new Set(blockers)].sort() }; const sourceChecksums = {}; for (const candidate of recipe.sourceCandidates) { const sourceDirectory = path.resolve(extractedRoot, candidate); sourceChecksums[candidate] = await directoryChecksum(sourceDirectory); } const pack = { schemaVersion: 1, id: `manastorm:assets:${recipe.mapId}`, mapId: recipe.mapId, location: recipe.location, visual: chunks.visual, collision: chunks.collision, navigation: chunks.navigation, transform: recipe.transform, anchors, review: { anchors: anchorReview.reviewStatus, warnings: anchorReview.warnings ?? [], }, source: { clientSnapshot: recipe.clientSnapshot ?? "ascension-area-52-installed-client", mapDirectory: recipe.mapDirectory, checksums: sourceChecksums, }, }; await writeJson(path.join(target, "stage-pack.json"), pack); await rebuildRegistry(); return { status: "green", slug, packageId: pack.id }; } async function validateAll() { const catalog = await loadCatalog(); const registry = await rebuildRegistry(); const archivedMaps = await archivedOpenWorldMaps(); const archivedMapIds = new Set(archivedMaps.keys()); const shippingCatalogMaps = catalog.maps.filter((map) => !archivedMapIds.has(map.mapId)); const duplicateValues = (values) => [ ...new Set(values.filter((value, index) => values.indexOf(value) !== index)), ].sort((left, right) => ( typeof left === "number" && typeof right === "number" ? left - right : String(left).localeCompare(String(right)) )); const catalogMapIds = new Set(catalog.maps.map((map) => map.mapId)); const duplicateMapIds = duplicateValues(registry.packages.map((assetPackage) => assetPackage.mapId)); const duplicatePackageIds = duplicateValues(registry.packages.map((assetPackage) => assetPackage.id)); const unexpectedMapIds = [ ...new Set( registry.packages .map((assetPackage) => assetPackage.mapId) .filter((mapId) => !catalogMapIds.has(mapId)), ), ].sort((left, right) => left - right); const archivedShippingMapIds = registry.packages .map((assetPackage) => assetPackage.mapId) .filter((mapId) => archivedMapIds.has(mapId)) .sort((left, right) => left - right); const collectionBlockers = [ ...(registry.packages.length !== shippingCatalogMaps.length ? [`Expected ${shippingCatalogMaps.length} shipping packages, found ${registry.packages.length}.`] : []), ...(duplicateMapIds.length ? [`Duplicate package map IDs: ${duplicateMapIds.join(", ")}.`] : []), ...(duplicatePackageIds.length ? [`Duplicate package IDs: ${duplicatePackageIds.join(", ")}.`] : []), ...(unexpectedMapIds.length ? [`Packages reference map IDs outside the catalog: ${unexpectedMapIds.join(", ")}.`] : []), ...(archivedShippingMapIds.length ? [`Archived open-world map IDs remain in the shipping root: ${archivedShippingMapIds.join(", ")}.`] : []), ]; const byMap = new Map(registry.packages.map((assetPackage) => [assetPackage.mapId, assetPackage])); const maps = []; for (const map of shippingCatalogMaps) { const assetPackage = byMap.get(map.mapId); const blockers = []; if (!assetPackage) blockers.push("No shipping stage pack."); if (assetPackage) { const chunks = [...assetPackage.visual, ...assetPackage.collision, assetPackage.navigation]; for (const chunk of chunks) { const file = path.join(projectRoot, "public", chunk.url.replace(/^\//, "")); if (!await exists(file)) blockers.push(`Missing ${chunk.url}.`); else { if (await sha256(file) !== chunk.checksum) blockers.push(`${chunk.url} checksum mismatch.`); try { const geometry = await inspectGlb(file); if (chunk.triangleCount !== geometry.triangleCount) { blockers.push(`${chunk.url} triangle count mismatch.`); } if (assetPackage.collision.includes(chunk) && geometry.triangleCount > 100_000) { blockers.push(`${chunk.url} exceeds the 100000-triangle collision limit.`); } } catch (error) { blockers.push(`${chunk.url} is not a valid decoded GLB: ${error.message}`); } } } if (!assetPackage.anchors.every(validAnchor)) blockers.push("Invalid anchors."); } maps.push({ mapId: map.mapId, name: map.name, packageId: assetPackage?.id ?? null, blockers }); } const report = { schemaVersion: 1, status: maps.every((map) => !map.blockers.length) && !collectionBlockers.length ? "green" : "blocked", counts: { requiredMaps: maps.length, archivedMaps: archivedMapIds.size, packages: registry.packages.length, shippingMaps: maps.filter((map) => map.packageId).length, blockedMaps: maps.filter((map) => map.blockers.length).length, }, collectionBlockers, archivedMaps: catalog.maps .filter((map) => archivedMapIds.has(map.mapId)) .map((map) => ({ mapId: map.mapId, name: map.name, folder: archivedMaps.get(map.mapId)?.folder ?? null, })), maps, }; await writeJson(path.join(workRoot, "validation-report.json"), report); return report; } const [, , command, argument] = process.argv; try { const result = command === "discover" ? await discoverSources() : command === "scaffold" ? await scaffoldRecipes() : command === "optimize" && argument ? await optimizeStage(argument) : command === "pack" && argument ? await buildStagePack(argument) : command === "validate" ? await validateAll() : command === "registry" ? await rebuildRegistry() : null; if (!result) { throw new Error("Usage: node scripts/manastorm-pipeline/cli.mjs "); } console.log(JSON.stringify(result, null, 2)); if (result.status === "blocked") process.exitCode = 1; } catch (error) { console.error(JSON.stringify({ status: "error", command, argument, error: error instanceof Error ? error.message : String(error), }, null, 2)); process.exitCode = 1; }