Files
healer-man/scripts/runewaker-pipeline/promote-hoto-surface-v2.mjs
T
2026-10-06 22:25:34 -04:00

379 lines
22 KiB
JavaScript

import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { constants } from "node:fs";
import { copyFile, mkdir, readFile, rename, stat, writeFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
// This promoter intentionally handles only the HOTO v1 -> v2 metadata update.
// It has no effect unless invoked with the explicit --apply flag.
const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const reviewDir = "artifacts/playtest/rom-deep-review/uv-stretch-review";
const stageDir = `${reviewDir}/staged`;
const stageReportPath = `${stageDir}/hoto-surface-quality-v2-metadata-stage-report.json`;
const visualProofPath = `${reviewDir}/hoto-surface-v2-visual-proof.json`;
const sourceLayoutProofPath = `${stageDir}/hoto-surface-v2-source-layout-proof.json`;
const independentLayoutProofPath = `${stageDir}/hoto-v2-root-source-layout-proof.json`;
const textureQualityProofPath = `${stageDir}/compressed-source-texture-fidelity-policy.json`;
const policyId = "rom-hoto-source-rock-detail-v2";
const sourceModel = "model/dungeon/echoes_of_the_sea/act_echoes_of_the_sea.ros";
const sourceRosSha256 = "a0bd77c09045f8bfa1507484bbb2c2ff0cc72d397df4f769fc4e6d5aab14f8a2";
const canonicalOldSha256 = "1015a8483096c551b4600f772f5f4cc888b3a7ab93dd58fe199cf113488e0078";
const canonicalCandidateSha256 = "5f4329e5dce0651ec30861d3140c130ddf646aac920670aaa6ba901682d258b6";
const optimizedOldSha256 = "f745c41b834647ff660db96665a9c474772162db2382c5a5ffa1a41e6a7e841c";
const optimizedCandidateSha256 = "b80b60cba6e5a0eb3e13bd7e048f9408f21bf870c0c81fc8de5c195d3a3eaee3";
const reviewedProofHashes = new Map([
[stageReportPath, "a896070024f0e03eaef952357e76a3da04b23490da0d12f57fe795eeb253e958"],
[visualProofPath, "da90fb979266efd65ac4a9dd3e8a2447c9c5cc5200520fed60d5547482eb0d51"],
[sourceLayoutProofPath, "d508f442ce3e95b6eed1f628cf4bfad250434f37b15b538c55e5c078b1610297"],
[independentLayoutProofPath, "d508f442ce3e95b6eed1f628cf4bfad250434f37b15b538c55e5c078b1610297"],
[textureQualityProofPath, "ce7d787e502c6283e6bcd64d8c801dfc6724e084d7ff807a3a67063a833e54ca"],
]);
const metadataBaselines = new Map([
["src/assets/game/dungeons/hoto/import-metadata.json", "1e2525fc5c9320125beeb5f5536d8be258c63b0b1bff0fcf958321f8da1bf362"],
["src/assets/game/dungeons/hoto/optimization-report.json", "a23ab1161a743b679eceeefe576ca55c00fb7f672ec071aaa30e7b669bfe7c44"],
["src/assets/game/dungeons/hoto/conversion-report.json", "2abb3bda4b758f05c993f80bf09bc880e7d0d6b78d33c257d6da229713ddf723"],
["runewaker-export-work/hoto/converted/optimization-report.json", "317ec5195ac2f8e8f6c96860cf92905564c5274deacef5c5ffde28165909dbeb"],
["runewaker-export-work/hoto/converted/conversion-report.json", "2abb3bda4b758f05c993f80bf09bc880e7d0d6b78d33c257d6da229713ddf723"],
]);
const canonicalFile = "runewaker-export-work/hoto/converted/hoto-visual.glb";
const optimizedFile = "runewaker-export-work/hoto/converted/hoto-visual.optimized.glb";
const shippingFile = "src/assets/game/dungeons/hoto/hoto-visual.glb";
const targetMaterialNames = [
"runewaker_material_96", "runewaker_material_53", "runewaker_material_26", "runewaker_material_18",
];
const carrierNames = ["rom-detail-hoto-floor:source-tile-carrier", "rom-detail-hoto-wall:source-tile-carrier"];
const approvedMetadataKeys = ["runewakerSurfaceDetail", "runewakerSurfaceDetailCarrier"];
const reviewedScreenshotHashes = new Map([
[`${reviewDir}/hoto-surface-v2-entry-shipped.png`, "b410b8866b3b0732ebe8888eae7c6dea4f9529c8272d6ff47c85d14292363b0c"],
[`${reviewDir}/hoto-surface-v2-entry-candidate.png`, "e68f92d523198700265774c6ad07a8bd0e1372fbf45f4dd28e71787b3062bcb2"],
[`${reviewDir}/hoto-surface-v2-material26-low-density-floor-shipped.png`, "63a1e7832767c952ae927e6a675e8c660a25d6d6de12b94ffb4a54e1339b7224"],
[`${reviewDir}/hoto-surface-v2-material26-low-density-floor-candidate.png`, "6cc22165911aba6835ab0d23e8bf16b1ef5b0992099bcab177d9d57ca11bb841"],
]);
const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
const readRepo = (file) => readFile(path.resolve(repo, file));
const readJson = async (file) => JSON.parse((await readRepo(file)).toString("utf8"));
const repoPath = (file) => path.resolve(repo, file);
function parseGlb(bytes, label) {
assert.equal(bytes.toString("ascii", 0, 4), "glTF", `${label} is not a GLB.`);
assert.equal(bytes.readUInt32LE(4), 2, `${label} is not glTF 2.0.`);
assert.equal(bytes.readUInt32LE(8), bytes.length, `${label} header length is invalid.`);
const chunks = [];
for (let offset = 12; offset < bytes.length;) {
assert.ok(offset + 8 <= bytes.length, `${label} has a truncated chunk header.`);
const length = bytes.readUInt32LE(offset);
const type = bytes.readUInt32LE(offset + 4);
const data = Buffer.from(bytes.subarray(offset + 8, offset + 8 + length));
assert.equal(data.length, length, `${label} has a truncated chunk.`);
chunks.push({ type, data });
offset += 8 + length;
}
const jsonChunk = chunks.find((chunk) => chunk.type === 0x4e4f534a);
assert.ok(jsonChunk, `${label} has no JSON chunk.`);
return {
chunks,
json: JSON.parse(jsonChunk.data.toString("utf8").trim()),
nonJsonChunks: chunks.filter((chunk) => chunk.type !== 0x4e4f534a),
};
}
function withoutApprovedMetadata(json) {
const copy = structuredClone(json);
for (const material of copy.materials ?? []) {
if (!material.extras) continue;
if (targetMaterialNames.includes(material.name)) delete material.extras.runewakerSurfaceDetail;
if (carrierNames.includes(material.name)) delete material.extras.runewakerSurfaceDetailCarrier;
}
return copy;
}
function verifyOnlyReviewedMetadataChanged(source, candidate, label, stageEntry) {
const before = parseGlb(source, `${label} baseline`);
const after = parseGlb(candidate, `${label} candidate`);
assert.deepEqual(withoutApprovedMetadata(after.json), withoutApprovedMetadata(before.json),
`${label} JSON changed outside the six approved HOTO metadata fields.`);
const beforeChunks = before.nonJsonChunks.map((chunk) => [chunk.type, digest(chunk.data), chunk.data.length]);
const afterChunks = after.nonJsonChunks.map((chunk) => [chunk.type, digest(chunk.data), chunk.data.length]);
assert.deepEqual(afterChunks, beforeChunks, `${label} non-JSON GLB chunks changed.`);
assert.equal(stageEntry.nonJsonChunksByteIdentical, true);
assert.deepEqual(stageEntry.nonJsonChunks.map((chunk) => [Number(chunk.type), chunk.sha256, chunk.bytes]), beforeChunks,
`${label} stage report does not match the byte-identity proof.`);
const materialByName = new Map((after.json.materials ?? []).map((material) => [material.name, material]));
return { before, after, materialByName };
}
function expectedCarrierMetadata(variantName, sourceProof) {
const source = sourceProof;
return {
version: 1,
variantName,
policyId,
sourceResourceSha256: sourceRosSha256,
sourceMaterial: source.nativeMaterial,
canonicalMaterialName: source.canonicalMaterialName,
sourceTexturePath: source.sourceTexture,
sourceTextureName: source.sourceImage,
sourcePreparedPngSha256: source.sourcePreparedPngSha256,
};
}
const stageReportBytes = await readRepo(stageReportPath);
const visualProofBytes = await readRepo(visualProofPath);
for (const [file, expectedSha] of reviewedProofHashes) {
assert.equal(digest(await readRepo(file)), expectedSha, `${file} differs from the frozen reviewed proof.`);
}
const stageReport = JSON.parse(stageReportBytes.toString("utf8"));
const visualProof = JSON.parse(visualProofBytes.toString("utf8"));
assert.equal(stageReport.status, "staged-metadata-only-not-promoted");
assert.equal(stageReport.policyId, policyId);
assert.equal(stageReport.source.sourceModel, sourceModel);
assert.equal(stageReport.source.sourceRosSha256, sourceRosSha256);
assert.equal(stageReport.freshPolicySmoke.status, "applied");
assert.equal(stageReport.migrationSmoke.status, "passed");
assert.equal(stageReport.migrationSmoke.firstApplication, "migrated-v1");
assert.equal(stageReport.migrationSmoke.secondApplication, "already-applied");
assert.deepEqual(stageReport.metadataScope.materials, targetMaterialNames);
assert.deepEqual(stageReport.metadataScope.carriers, carrierNames);
assert.deepEqual(stageReport.metadataScope.changedFields, approvedMetadataKeys);
assert.equal(stageReport.metadataScope.decodedGeometryUvTexturesAndAllNonJsonChunksUnchanged, true);
assert.equal(stageReport.metadataScope.level1CompressorOrGeometryOptimizerRun, false);
assert.equal(stageReport.historicalVisualEvidence.v1FinalScreenshots, "superseded-and-unavailable");
const sourceRosBytes = await readFile(path.resolve(repo, stageReport.source.sourceRos));
assert.equal(digest(sourceRosBytes), sourceRosSha256, "The pinned HOTO source ROS changed.");
const recipe = await readJson("scripts/runewaker-pipeline/recipes/hoto.json");
assert.deepEqual(recipe.visualSurfaceQualityPolicies, [policyId], "HOTO recipe does not select the durable v2 policy.");
// Verify every source image identity that the staged surface policy cites.
const sourcePngs = new Map();
for (const metadata of Object.values(stageReport.freshPolicySmoke.targetMetadata)) {
const proof = metadata.sourceProof;
assert.equal(proof.sourceModelSha256, sourceRosSha256);
for (const source of [proof, proof.floorDetailSource, proof.wallDetailSource].filter(Boolean)) {
const imageName = source.sourceTextureName ?? source.sourceImage;
const sha256 = source.sourcePreparedPngSha256;
if (!imageName || !sha256) continue;
const prior = sourcePngs.get(imageName);
if (prior) assert.equal(prior, sha256);
else sourcePngs.set(imageName, sha256);
}
}
for (const [imageName, expectedSha] of sourcePngs) {
const png = await readRepo(`runewaker-export-work/hoto/prepared/textures/${imageName}.png`);
assert.equal(digest(png), expectedSha, `Prepared source image ${imageName} changed.`);
}
const canonicalSource = await readRepo(canonicalFile);
const optimizedSource = await readRepo(optimizedFile);
const shippingSource = await readRepo(shippingFile);
const canonicalCandidate = await readRepo(stageReport.canonical.candidate);
const optimizedCandidate = await readRepo(stageReport.optimized.candidate);
assert.equal(digest(canonicalSource), canonicalOldSha256);
assert.equal(digest(optimizedSource), optimizedOldSha256);
assert.equal(digest(shippingSource), optimizedOldSha256);
assert.equal(digest(canonicalCandidate), canonicalCandidateSha256);
assert.equal(digest(optimizedCandidate), optimizedCandidateSha256);
assert.equal(canonicalCandidate.length, stageReport.canonical.candidateBytes);
assert.equal(optimizedCandidate.length, stageReport.optimized.candidateBytes);
assert.equal(stageReport.canonical.originalSha256, canonicalOldSha256);
assert.equal(stageReport.optimized.originalSha256, optimizedOldSha256);
assert.equal(stageReport.shipping.sha256, optimizedOldSha256);
assert.equal(stageReport.canonical.candidateSha256, canonicalCandidateSha256);
assert.equal(stageReport.optimized.candidateSha256, optimizedCandidateSha256);
const canonical = verifyOnlyReviewedMetadataChanged(canonicalSource, canonicalCandidate, "canonical HOTO", stageReport.canonical);
const optimized = verifyOnlyReviewedMetadataChanged(optimizedSource, optimizedCandidate, "optimized HOTO", stageReport.optimized);
const targetMetadata = stageReport.freshPolicySmoke.targetMetadata;
for (const name of targetMaterialNames) {
assert.deepEqual(canonical.materialByName.get(name)?.extras?.runewakerSurfaceDetail, targetMetadata[name],
`Canonical ${name} metadata differs from the source-pinned policy output.`);
assert.deepEqual(optimized.materialByName.get(name)?.extras?.runewakerSurfaceDetail, targetMetadata[name],
`Optimized ${name} metadata differs from the source-pinned policy output.`);
}
const carriers = [
[carrierNames[0], "rom-detail-hoto-floor", targetMetadata.runewaker_material_26.sourceProof.floorDetailSource],
[carrierNames[1], "rom-detail-hoto-wall", targetMetadata.runewaker_material_18.sourceProof.wallDetailSource],
];
for (const [name, variantName, sourceProof] of carriers) {
const expected = expectedCarrierMetadata(variantName, sourceProof);
for (const document of [canonical, optimized]) {
assert.deepEqual(document.materialByName.get(name)?.extras?.runewakerSurfaceDetailCarrier, expected,
`${name} carrier source provenance changed.`);
}
}
assert.equal(canonical.after.json.images.length, 17, "Canonical candidate changed the source image inventory.");
assert.equal(optimized.after.json.images.length, 17, "Optimized candidate changed the source image inventory.");
const sourceLayoutProof = await readJson(sourceLayoutProofPath);
const independentLayoutProof = await readJson(independentLayoutProofPath);
for (const proof of [sourceLayoutProof, independentLayoutProof]) {
assert.equal(digest(await readRepo(proof.beforeFile)), proof.beforeSha256);
assert.equal(proof.beforeSha256, canonicalOldSha256);
assert.equal(proof.afterSha256, canonicalCandidateSha256);
assert.equal(proof.hierarchyAndTransformsPreserved, true);
assert.equal(proof.orderedSourceTriangleCornersPreserved, true);
assert.equal(proof.allSourceAttributesExceptAdaptedUv1Preserved, true);
assert.equal(proof.originalSourceImagesPreserved, true);
assert.equal(proof.triangleCount, 189855);
assert.equal(proof.originalSourceImageCount, 17);
assert.deepEqual(proof.addedSourceRegionImages, []);
}
const textureQualityProof = await readJson(textureQualityProofPath);
const historicalCodec = textureQualityProof.results.find((result) => result.id === "hoto");
assert.ok(historicalCodec, "Historical HOTO codec proof is missing.");
assert.equal(historicalCodec.glbSha256, optimizedOldSha256);
assert.equal(historicalCodec.textures.length, 4);
assert.ok(historicalCodec.textures.every((texture) => texture.maeAlpha === 0));
assert.ok(historicalCodec.textures.some((texture) => texture.name === "5672f062_echoes_of_the_sea_02" && texture.width === 1024 && texture.height === 1024));
assert.ok(historicalCodec.textures.some((texture) => texture.name === "b2ababb9_echoes_of_the_sea_03" && texture.width === 1024 && texture.height === 1024));
assert.deepEqual(optimized.after.nonJsonChunks.map((chunk) => [chunk.type, digest(chunk.data)]),
parseGlb(optimizedSource, "old optimized HOTO").nonJsonChunks.map((chunk) => [chunk.type, digest(chunk.data)]),
"The v2 metadata-only candidate no longer carries the historically verified KTX2 bytes.");
assert.equal(visualProof.status, "passed");
assert.equal(visualProof.review?.status, "accepted", "The parent review acceptance is not recorded.");
assert.deepEqual(visualProof.review.views, ["entry", "material26-low-density-floor"]);
const visualRows = visualProof.results;
assert.equal(visualRows.length, 4);
for (const viewName of visualProof.review.views) {
const pair = visualRows.filter((row) => row.view === viewName);
const shipped = pair.find((row) => row.stage === "shipped");
const candidate = pair.find((row) => row.stage === "candidate");
assert.ok(shipped && candidate, `Missing the shipped/candidate ${viewName} view pair.`);
assert.deepEqual(shipped.camera, candidate.camera, `${viewName} camera changed between captures.`);
assert.deepEqual(shipped.errors, []);
assert.deepEqual(candidate.errors, []);
assert.equal(shipped.glbSha256, optimizedOldSha256);
assert.equal(candidate.glbSha256, optimizedCandidateSha256);
assert.equal(shipped.screenshot, `${reviewDir}/hoto-surface-v2-${viewName}-shipped.png`);
assert.equal(candidate.screenshot, `${reviewDir}/hoto-surface-v2-${viewName}-candidate.png`);
for (const row of [shipped, candidate]) {
const screenshot = await readRepo(row.screenshot);
const expectedSha = reviewedScreenshotHashes.get(row.screenshot);
assert.ok(expectedSha, `No reviewed screenshot hash is pinned for ${row.screenshot}.`);
assert.equal(row.screenshotSha256, expectedSha, `${row.screenshot} report hash changed.`);
assert.equal(digest(screenshot), expectedSha, `${row.screenshot} image bytes changed.`);
assert.ok(screenshot.length > 1000);
}
assert.ok(candidate.candidateResponses.some((response) => response.status === 200 && response.bodySha256 === optimizedCandidateSha256));
assert.ok(candidate.timing?.sampleFrames?.averageFps > 0);
}
const visualSummary = visualRows.map((row) => ({
view: row.view, stage: row.stage, camera: row.camera, screenshot: row.screenshot,
screenshotSha256: row.screenshotSha256, glbSha256: row.glbSha256, drawCalls: row.drawCalls, timing: row.timing,
}));
const review = {
status: "reviewed-metadata-only",
policyId,
sourceModel,
sourceRosSha256,
stageReport: stageReportPath,
sourceLayoutProof: sourceLayoutProofPath,
independentSourceLayoutProof: independentLayoutProofPath,
historicalTextureQualityProof: textureQualityProofPath,
visualProof: visualProofPath,
visualReview: visualProof.review,
historicalVisualEvidence: stageReport.historicalVisualEvidence,
canonical: { file: canonicalFile, beforeSha256: canonicalOldSha256, candidate: stageReport.canonical.candidate, candidateSha256: canonicalCandidateSha256, candidateBytes: canonicalCandidate.length },
optimized: { file: optimizedFile, beforeSha256: optimizedOldSha256, candidate: stageReport.optimized.candidate, candidateSha256: optimizedCandidateSha256, candidateBytes: optimizedCandidate.length },
shipping: { file: shippingFile, beforeSha256: optimizedOldSha256, afterSha256: optimizedCandidateSha256 },
nonJsonChunksByteIdentical: true,
metadataOnlyFields: approvedMetadataKeys,
migrationSmoke: stageReport.migrationSmoke,
visualRows: visualSummary,
};
const metadataEdits = [];
for (const [file, expectedSha] of metadataBaselines) {
const bytes = await readRepo(file);
assert.equal(digest(bytes), expectedSha, `${file} changed after this promoter was reviewed.`);
const data = JSON.parse(bytes.toString("utf8"));
if (file.endsWith("import-metadata.json")) {
const visualAsset = data.assets?.find((asset) => asset.role === "visual");
assert.equal(visualAsset?.sha256, optimizedOldSha256);
assert.equal(visualAsset.size, shippingSource.length);
visualAsset.sha256 = optimizedCandidateSha256;
visualAsset.size = optimizedCandidate.length;
data.continuationReview = { ...data.continuationReview, surfaceQuality: review };
} else if (file.endsWith("optimization-report.json")) {
assert.equal(data.source?.checksum, canonicalOldSha256);
assert.equal(data.optimized?.checksum, optimizedOldSha256);
data.source = { ...data.source, checksum: canonicalCandidateSha256, size: canonicalCandidate.length };
data.optimized = { ...data.optimized, checksum: optimizedCandidateSha256, size: optimizedCandidate.length };
data.supersededSurfaceQualityV1 ??= data.visualUvAdaptations?.surfaceQuality;
data.visualUvAdaptations ??= {};
data.visualUvAdaptations.surfaceQuality = review;
data.surfaceQuality = review;
// textureQuality remains a historical report for the old optimized SHA;
// the candidate's KTX2 and all other BIN bytes are proven identical above.
} else {
data.continuationIntegrityProof = {
...data.continuationIntegrityProof,
surfaceQuality: review,
};
}
metadataEdits.push({ file, expectedSha, bytes: Buffer.from(`${JSON.stringify(data, null, 2)}\n`, "utf8") });
}
const apply = process.argv.includes("--apply");
const preview = {
ready: true,
applied: false,
dungeonId: "hoto",
policyId,
assetCopies: [
{ file: canonicalFile, sha256: canonicalOldSha256, candidateSha256: canonicalCandidateSha256, bytes: canonicalCandidate.length },
{ file: optimizedFile, sha256: optimizedOldSha256, candidateSha256: optimizedCandidateSha256, bytes: optimizedCandidate.length },
{ file: shippingFile, sha256: optimizedOldSha256, candidateSha256: optimizedCandidateSha256, bytes: optimizedCandidate.length },
],
metadataFiles: metadataEdits.map(({ file, expectedSha, bytes }) => ({ file, sha256: expectedSha, candidateBytes: bytes.length })),
review,
};
if (!apply) {
console.log(JSON.stringify(preview, null, 2));
process.exit(0);
}
const backupRoot = `${reviewDir}/pre-hoto-surface-v2-promotion-backup`;
const proofPath = `${reviewDir}/hoto-surface-v2-promotion-proof.json`;
for (const pathToCheck of [backupRoot, proofPath]) {
await stat(repoPath(pathToCheck)).then(
() => assert.fail(`Refusing to reuse existing promotion output: ${pathToCheck}`),
(error) => assert.equal(error.code, "ENOENT", `Unable to verify promotion output path ${pathToCheck}.`),
);
}
await mkdir(repoPath(backupRoot), { recursive: false });
const protectedFiles = [canonicalFile, optimizedFile, shippingFile, ...metadataEdits.map((edit) => edit.file)];
for (const file of protectedFiles) {
const backupPath = repoPath(path.join(backupRoot, file).replaceAll("\\", "/"));
await mkdir(path.dirname(backupPath), { recursive: true });
await copyFile(repoPath(file), backupPath, constants.COPYFILE_EXCL);
}
// Stage replacement bytes to exclusive sibling files before swapping any live
// path. The reviewed backup above is the recovery point if a later rename fails.
const replacements = [
{ file: canonicalFile, bytes: canonicalCandidate },
{ file: optimizedFile, bytes: optimizedCandidate },
{ file: shippingFile, bytes: optimizedCandidate },
...metadataEdits,
];
const stagedReplacements = [];
for (let index = 0; index < replacements.length; index += 1) {
const replacement = replacements[index];
const temp = `${replacement.file}.surface-v2-${process.pid}-${index}.tmp`;
await stat(repoPath(temp)).then(
() => assert.fail(`Refusing to overwrite a staged promotion file: ${temp}`),
(error) => assert.equal(error.code, "ENOENT", `Unable to verify staged replacement path ${temp}.`),
);
if (replacement.bytes) await writeFile(repoPath(temp), replacement.bytes, { flag: "wx" });
else await copyFile(repoPath(replacement.candidate), repoPath(temp), constants.COPYFILE_EXCL);
stagedReplacements.push({ file: replacement.file, temp });
}
for (const replacement of stagedReplacements) await rename(repoPath(replacement.temp), repoPath(replacement.file));
await writeFile(repoPath(proofPath), `${JSON.stringify({ ...preview, applied: true, backupRoot: backupRoot, proofPath }, null, 2)}\n`, { flag: "wx" });
console.log(JSON.stringify({ ...preview, applied: true, backupRoot, proofPath }, null, 2));