Release Healer Man 0.1.21

scripts/runewaker-pipeline/build-environment.mjs

@@ -14,6 +14,11 @@ import {
   projectRoot,
   sourceRootFor,
 } from "./lib/recipe.mjs";
+import {
+  assembleInstancedGlb,
+  buildWdbPlacementPlan,
+  normalizeWdbResourcePath,
+} from "./lib/environment-assembly.mjs";
 
 const argv = process.argv.slice(2);
 const fallbackRecipe = path.join(pipelineDirectory, "recipes", "forsaken-abbey.json");
@@ -23,6 +28,7 @@ const workDirectory = path.join(projectRoot, "runewaker-export-work", slug);
 const rawDirectory = path.join(workDirectory, "raw");
 const preparedDirectory = path.join(workDirectory, "prepared");
 const convertedDirectory = path.join(workDirectory, "converted");
+const resourceCacheDirectory = path.join(projectRoot, "runewaker-export-work", "resource-cache");
 const shippingDirectory = path.join(projectRoot, "src", "assets", "game", "dungeons", slug);
 const sourceRoot = sourceRootFor(recipe);
 const resourceRoot = path.join(sourceRoot, recipe.source.resourceRoot);
@@ -45,6 +51,10 @@ const blenderScript = path.join(pipelineDirectory, "blender", "convert-runewaker
 const gltfTransform = path.join(projectRoot, "node_modules", "@gltf-transform", "cli", "bin", "cli.js");
 const ktx2Compressor = path.join(projectRoot, "scripts", "asset-pipeline", "compress-ktx2.mjs");
 const khronosValidator = path.join(projectRoot, "scripts", "manastorm-assets", "validate-package-glbs.cjs");
+const fullWdbAssembly = !recipe.source.proceduralTerrainFallback
+  && recipe.source.wdbAssembly?.mode !== "primary-only"
+  && !argv.includes("--primary-only");
+const collisionChunkTriangleTarget = 90_000;
 
 function run(executable, argumentsList, label) {
   return new Promise((resolve, reject) => {
@@ -67,6 +77,15 @@ async function requirePath(file, guidance) {
   catch { throw new Error(guidance + "\nMissing: " + file); }
 }
 
+async function exists(file) {
+  try {
+    await stat(file);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 async function sha256(file) {
   return createHash("sha256").update(await readFile(file)).digest("hex");
 }
@@ -148,7 +167,7 @@ function sourcePointToPrimaryLocal(sourcePosition, placement) {
 
 async function appendSourceAnchorSupport() {
   const support = recipe.source.sourceAnchorSupport;
-  if (!support) return;
+  if (!support || fullWdbAssembly) return;
   const { placement } = await primaryPlacement();
   const half = Number(support.halfExtentSourceUnits ?? 60);
   const thickness = Number(support.thicknessSourceUnits ?? 3);
@@ -185,6 +204,128 @@ async function appendSourceAnchorSupport() {
   ]);
 }
 
+function resourceSourceFile(resource) {
+  return path.join(resourceRoot, ...normalizeWdbResourcePath(resource).split("\\"));
+}
+
+async function prepareCachedResourceTextures(raw, prepared) {
+  const sourceTextures = path.join(raw, "textures");
+  const preparedTextures = path.join(prepared, "textures");
+  await mkdir(preparedTextures, { recursive: true });
+  const ddsFiles = await exists(sourceTextures)
+    ? (await readdir(sourceTextures))
+      .filter((file) => file.toLowerCase().endsWith(".dds"))
+      .sort()
+      .map((file) => path.join(sourceTextures, file))
+    : [];
+  if (ddsFiles.length) {
+    await run(
+      texconv,
+      ["-ft", "PNG", "-o", preparedTextures, "-y", "--permissive", ...ddsFiles],
+      "convert " + ddsFiles.length + " cached DDS textures",
+    );
+  }
+  await copyFile(path.join(raw, "scene.obj"), path.join(prepared, "scene.obj"));
+  await copyFile(path.join(raw, "export-report.json"), path.join(prepared, "export-report.json"));
+  const sourceMtl = await readFile(path.join(raw, "scene.mtl"), "utf8");
+  await writeFile(
+    path.join(prepared, "scene.mtl"),
+    sourceMtl.replace(/^(map_Kd\s+.+)\.dds\s*$/gim, "$1.png"),
+    "utf8",
+  );
+}
+
+async function cachedResourceAsset(resourceRecord) {
+  const sourceFile = resourceSourceFile(resourceRecord.resource);
+  if (!await exists(sourceFile)) {
+    throw new Error("source ROS is missing: " + projectPath(sourceFile));
+  }
+  const sourceSha256 = await sha256(sourceFile);
+  const cacheRoot = path.join(resourceCacheDirectory, resourceRecord.cacheKey);
+  const raw = path.join(cacheRoot, "raw");
+  const prepared = path.join(cacheRoot, "prepared");
+  const converted = path.join(cacheRoot, "converted");
+  const manifestFile = path.join(cacheRoot, "cache-manifest.json");
+  if (!argv.includes("--force-resource-cache") && await exists(manifestFile)) {
+    const cached = JSON.parse(await readFile(manifestFile, "utf8"));
+    const outputs = [cached.visual?.file, ...(cached.collision ?? []).map((entry) => entry.file)]
+      .filter(Boolean)
+      .map((file) => path.join(cacheRoot, file));
+    if (cached.source?.resource === resourceRecord.normalizedResource
+      && cached.source?.sha256 === sourceSha256
+      && cached.coordinateSystem?.metersPerSourceUnit === recipe.coordinateSystem.metersPerSourceUnit
+      && (await Promise.all(outputs.map(exists))).every(Boolean)) {
+      return {
+        ...cached,
+        cacheHit: true,
+        visualFile: path.join(cacheRoot, cached.visual.file),
+        collision: cached.collision.map((entry) => ({
+          ...entry,
+          absoluteFile: path.join(cacheRoot, entry.file),
+        })),
+      };
+    }
+  }
+  await Promise.all([
+    mkdir(raw, { recursive: true }),
+    mkdir(prepared, { recursive: true }),
+    mkdir(converted, { recursive: true }),
+  ]);
+  await run(
+    nativeExporter,
+    [resourceRoot, resourceRecord.resource.replaceAll("/", "\\"), raw],
+    "export cached WDB resource " + resourceRecord.resource,
+  );
+  await prepareCachedResourceTextures(raw, prepared);
+  await run(
+    blender,
+    [
+      "--background", "--factory-startup", "--python", blenderScript, "--",
+      path.join(prepared, "scene.obj"),
+      converted,
+      resourceRecord.cacheKey,
+      String(recipe.coordinateSystem.metersPerSourceUnit),
+    ],
+    "convert cached WDB resource " + resourceRecord.resource,
+  );
+  const conversion = JSON.parse(await readFile(path.join(converted, "conversion-report.json"), "utf8"));
+  const manifest = {
+    schemaVersion: 1,
+    source: {
+      resource: resourceRecord.normalizedResource,
+      file: projectPath(sourceFile),
+      sha256: sourceSha256,
+    },
+    coordinateSystem: {
+      nativeBridge: recipe.coordinateSystem.nativeBridge,
+      metersPerSourceUnit: recipe.coordinateSystem.metersPerSourceUnit,
+    },
+    visual: {
+      file: path.relative(
+        cacheRoot,
+        path.join(converted, resourceRecord.cacheKey + "-visual.glb"),
+      ).split(path.sep).join("/"),
+      triangles: conversion.visual.triangles,
+      meshObjects: conversion.visual.meshObjects,
+      bounds: conversion.visual.bounds,
+    },
+    collision: conversion.collision.map((entry) => ({
+      ...entry,
+      file: path.relative(cacheRoot, path.join(converted, entry.file)).split(path.sep).join("/"),
+    })),
+  };
+  await writeFile(manifestFile, JSON.stringify(manifest, null, 2) + "\n", "utf8");
+  return {
+    ...manifest,
+    cacheHit: false,
+    visualFile: path.join(cacheRoot, manifest.visual.file),
+    collision: manifest.collision.map((entry) => ({
+      ...entry,
+      absoluteFile: path.join(cacheRoot, entry.file),
+    })),
+  };
+}
+
 async function prepareTextures() {
   const sourceTextures = path.join(rawDirectory, "textures");
   const preparedTextures = path.join(preparedDirectory, "textures");
@@ -252,8 +393,208 @@ function cloneNodeTree(document, node) {
   return clone;
 }
 
+function groupCollisionRecords(records) {
+  const groups = [];
+  let current = [];
+  let triangles = 0;
+  for (const record of records) {
+    if (current.length && triangles + record.triangles > collisionChunkTriangleTarget) {
+      groups.push({ records: current, triangles });
+      current = [];
+      triangles = 0;
+    }
+    current.push(record);
+    triangles += record.triangles;
+  }
+  if (current.length) groups.push({ records: current, triangles });
+  return groups;
+}
+
+async function assembleAllWdbPlacements() {
+  const wdbReport = JSON.parse(await readFile(path.join(rawDirectory, "wdb-report.json"), "utf8"));
+  const plan = buildWdbPlacementPlan(wdbReport, recipe);
+  const primaryResource = normalizeWdbResourcePath(plan.primaryPlacement.resource);
+  const primaryConversion = JSON.parse(
+    await readFile(path.join(convertedDirectory, "conversion-report.json"), "utf8"),
+  );
+  const assets = new Map();
+  assets.set(primaryResource, {
+    cacheHit: true,
+    cacheKind: "primary-conversion",
+    visualFile: path.join(convertedDirectory, slug + "-visual.glb"),
+    visual: primaryConversion.visual,
+    collision: primaryConversion.collision.map((entry) => ({
+      ...entry,
+      absoluteFile: path.join(convertedDirectory, entry.file),
+    })),
+  });
+  const failures = [];
+  for (const resource of plan.resources) {
+    if (resource.normalizedResource === primaryResource) continue;
+    try {
+      assets.set(resource.normalizedResource, await cachedResourceAsset(resource));
+    } catch (error) {
+      failures.push({
+        resource: resource.resource,
+        normalizedResource: resource.normalizedResource,
+        placementCount: resource.placementCount,
+        reason: error.message,
+      });
+    }
+  }
+  const unavailable = new Set(failures.map((failure) => failure.normalizedResource));
+  for (const resource of plan.resources) {
+    const asset = assets.get(resource.normalizedResource);
+    resource.assetStatus = asset ? "available" : "unsupported";
+    resource.cacheHit = asset?.cacheHit ?? false;
+    resource.cacheKind = asset?.cacheKind ?? "normalized-path-cache-with-source-hash-validation";
+    if (!asset) resource.blocker = failures.find((entry) => (
+      entry.normalizedResource === resource.normalizedResource
+    ))?.reason;
+  }
+  for (const placement of [...plan.staticPlacements, ...plan.dynamicPlacements]) {
+    placement.assetStatus = unavailable.has(placement.normalizedResource) ? "unsupported" : "available";
+    placement.assetId = "wdb-resource:" + placement.resourceCacheKey;
+  }
+
+  const staticPlacements = plan.staticPlacements.filter((placement) => (
+    placement.assetStatus === "available"
+  ));
+  if (!staticPlacements.length) throw new Error("WDB assembly produced no available static placements.");
+  const visualRecords = staticPlacements.map((placement) => ({
+    placement,
+    assetFile: assets.get(placement.normalizedResource).visualFile,
+  }));
+  const assembledVisualFile = path.join(convertedDirectory, slug + "-visual.wdb-assembled.glb");
+  const visualAssembly = await assembleInstancedGlb(visualRecords, assembledVisualFile);
+  await copyFile(assembledVisualFile, path.join(convertedDirectory, slug + "-visual.glb"));
+
+  const collisionRecords = staticPlacements.flatMap((placement) => (
+    assets.get(placement.normalizedResource).collision.map((collision) => ({
+      placement,
+      assetFile: collision.absoluteFile,
+      triangles: Number(collision.triangles ?? collision.sourceTriangleEstimate ?? 0),
+    }))
+  )).sort((left, right) => (
+    left.placement.id.localeCompare(right.placement.id)
+    || left.assetFile.localeCompare(right.assetFile)
+  ));
+  const collision = [];
+  for (const [index, group] of groupCollisionRecords(collisionRecords).entries()) {
+    const file = slug + "-wdb-collision-" + String(index).padStart(3, "0") + ".glb";
+    await assembleInstancedGlb(group.records, path.join(convertedDirectory, file));
+    collision.push({
+      file,
+      triangles: group.triangles,
+      sourceTriangleEstimate: group.triangles,
+      placementCount: group.records.length,
+      assembledFromWdb: true,
+    });
+  }
+
+  const dynamicDirectory = path.join(convertedDirectory, "dynamic");
+  const dynamicAssets = [];
+  const dynamicResources = plan.resources.filter((resource) => (
+    resource.dynamicPlacementCount > 0 && resource.assetStatus === "available"
+  ));
+  if (dynamicResources.length) await mkdir(dynamicDirectory, { recursive: true });
+  for (const resource of dynamicResources) {
+    const asset = assets.get(resource.normalizedResource);
+    const visualName = resource.cacheKey + "-visual.glb";
+    await copyFile(asset.visualFile, path.join(dynamicDirectory, visualName));
+    dynamicAssets.push({
+      role: "dynamic-visual",
+      file: "dynamic/" + visualName,
+      assetId: "wdb-resource:" + resource.cacheKey,
+      resource: resource.resource,
+    });
+    for (const [index, sourceCollision] of asset.collision.entries()) {
+      const collisionName = resource.cacheKey + "-collision-" + String(index).padStart(3, "0") + ".glb";
+      await copyFile(sourceCollision.absoluteFile, path.join(dynamicDirectory, collisionName));
+      dynamicAssets.push({
+        role: "dynamic-collision",
+        file: "dynamic/" + collisionName,
+        assetId: "wdb-resource:" + resource.cacheKey,
+        resource: resource.resource,
+        triangles: Number(sourceCollision.triangles ?? 0),
+      });
+    }
+  }
+
+  plan.resourceFailures = failures;
+  plan.buildSummary = {
+    staticPlacements: staticPlacements.length,
+    dynamicPlacements: plan.dynamicPlacements.filter((placement) => (
+      placement.assetStatus === "available"
+    )).length,
+    unavailablePlacements: [...plan.staticPlacements, ...plan.dynamicPlacements]
+      .filter((placement) => placement.assetStatus === "unsupported").length,
+    uniqueResources: plan.resources.length,
+    availableResources: assets.size,
+    unavailableResources: failures.length,
+    resourceCacheHits: [...assets.values()].filter((asset) => (
+      asset.cacheHit && asset.cacheKind !== "primary-conversion"
+    )).length,
+  };
+  const placementManifestFile = path.join(convertedDirectory, "wdb-placement-manifest.json");
+  await writeFile(placementManifestFile, JSON.stringify(plan, null, 2) + "\n", "utf8");
+
+  const visualTriangles = staticPlacements.reduce((total, placement) => (
+    total + Number(assets.get(placement.normalizedResource).visual.triangles ?? 0)
+  ), 0);
+  const visualMeshObjects = staticPlacements.reduce((total, placement) => (
+    total + Number(assets.get(placement.normalizedResource).visual.meshObjects ?? 0)
+  ), 0);
+  const conversion = {
+    schemaVersion: 2,
+    dungeonId: slug,
+    status: failures.length || plan.reviewRequiredPlacements.length
+      ? "review-required"
+      : "source-packaged",
+    environmentMode: "runewaker-wdb-assembly",
+    source: {
+      wdb: recipe.source.wdb,
+      primaryModel: recipe.source.model,
+      descriptorCount: plan.sourceDescriptorCount,
+      placementManifest: "wdb-placement-manifest.json",
+    },
+    coordinateTransform: plan.coordinateTransform,
+    visual: {
+      file: slug + "-visual.glb",
+      meshObjects: visualMeshObjects,
+      triangles: visualTriangles,
+      sourceTriangles: visualTriangles,
+      bounds: visualAssembly.bounds,
+      assembledWdbPlacements: staticPlacements.length,
+      uniqueResources: new Set(staticPlacements.map((placement) => placement.normalizedResource)).size,
+    },
+    collision,
+    collisionPolicy: {
+      kind: "wdb-static-placement-trimesh",
+      maximumTrianglesPerChunk: 100_000,
+      workingChunkTarget: collisionChunkTriangleTarget,
+      excludesReviewedDynamicPlacements: true,
+      includesUnreviewedDynamicCandidates: true,
+    },
+    dynamicAssets,
+    preview: slug + "-preview.png",
+    editableSource: slug + ".source.blend",
+    wdbAssembly: plan.buildSummary,
+    warnings: [
+      "The preview and editable Blender source cover the primary ROS only; the shipping GLB is assembled from the WDB manifest.",
+      "Name-based dynamic candidates remain static until a reviewed dynamic-placement rule supplies source evidence.",
+      ...failures.map((failure) => failure.resource + ": " + failure.reason),
+    ],
+  };
+  await writeFile(
+    path.join(convertedDirectory, "conversion-report.json"),
+    JSON.stringify(conversion, null, 2) + "\n",
+    "utf8",
+  );
+}
+
 async function assembleRepeatedPlacements() {
-  const translations = assemblyTranslations();
+  const translations = fullWdbAssembly ? [] : assemblyTranslations();
   if (!translations.length) return;
   const io = new NodeIO().registerExtensions(ALL_EXTENSIONS);
   const visualFile = path.join(convertedDirectory, slug + "-visual.glb");
@@ -385,7 +726,7 @@ async function bakeNavigation() {
 
 async function appendSourceAnchorNavigation(navigationFile, reportFile) {
   const support = recipe.source.sourceAnchorSupport;
-  if (!support) return;
+  if (!support || fullWdbAssembly) return;
   const { placement } = await primaryPlacement();
   const half = Number(support.halfExtentSourceUnits ?? 60);
   const meters = recipe.coordinateSystem.metersPerSourceUnit;
@@ -556,13 +897,29 @@ async function stageShippingPackage() {
   const files = [
     { role: "visual", source: path.join(convertedDirectory, slug + "-visual.optimized.glb"), target: path.join(shippingDirectory, slug + "-visual.glb") },
     ...conversion.collision.map((chunk) => ({ role: "collision", source: path.join(convertedDirectory, chunk.file), target: path.join(shippingDirectory, chunk.file) })),
+    ...(conversion.dynamicAssets ?? []).map((asset) => ({
+      role: asset.role,
+      source: path.join(convertedDirectory, asset.file),
+      target: path.join(shippingDirectory, asset.file),
+      assetId: asset.assetId,
+      resource: asset.resource,
+    })),
     { role: "navigation", source: path.join(convertedDirectory, slug + "-navigation.glb"), target: path.join(shippingDirectory, slug + "-navigation.glb") },
   ];
-  for (const entry of files) await copyFile(entry.source, entry.target);
+  for (const entry of files) {
+    await mkdir(path.dirname(entry.target), { recursive: true });
+    await copyFile(entry.source, entry.target);
+  }
   for (const report of ["conversion-report.json", "optimization-report.json", "recast-report.json", slug + "-preview.png"]) {
     await copyFile(path.join(convertedDirectory, report), path.join(shippingDirectory, report));
   }
   await copyFile(path.join(rawDirectory, "wdb-report.json"), path.join(shippingDirectory, "wdb-report.json"));
+  if (fullWdbAssembly) {
+    await copyFile(
+      path.join(convertedDirectory, "wdb-placement-manifest.json"),
+      path.join(shippingDirectory, "wdb-placement-manifest.json"),
+    );
+  }
   const server = await sourceMetadata();
   const { report, placement } = await primaryPlacement();
   const sourceEntrance = [server.playerSpawn.x, server.playerSpawn.y, server.playerSpawn.z];
@@ -604,12 +961,21 @@ async function stageShippingPackage() {
       } : {}),
       server,
       wdbReport: { descriptorCount: report.descriptorCount, file: "wdb-report.json", primaryPlacement: placement },
+      ...(fullWdbAssembly ? {
+        wdbAssembly: {
+          mode: "all-ros",
+          placementManifest: "wdb-placement-manifest.json",
+          ...conversion.wdbAssembly,
+        },
+      } : {}),
     },
     coordinateSystem: recipe.coordinateSystem,
     entrance,
     assets: await Promise.all(files.map(async (entry) => ({
       role: entry.role,
       fileName: path.basename(entry.target),
+      relativeFile: path.relative(shippingDirectory, entry.target).split(path.sep).join("/"),
+      ...(entry.assetId ? { assetId: entry.assetId, sourceResource: entry.resource } : {}),
       size: (await stat(entry.target)).size,
       sha256: await sha256(entry.target),
     }))),
@@ -634,7 +1000,8 @@ await buildNativeExporterIfNeeded();
 await exportSource();
 await prepareTextures();
 await convertEnvironment();
-await assembleRepeatedPlacements();
+if (fullWdbAssembly) await assembleAllWdbPlacements();
+else await assembleRepeatedPlacements();
 await optimizeVisual();
 await bakeNavigation();
 await stageShippingPackage();