253 lines
10 KiB
JavaScript
253 lines
10 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import { createHash } from "node:crypto";
|
|
import fs from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import test from "node:test";
|
|
import { gzipSync } from "node:zlib";
|
|
import { ARCHIVE_SPELL_SOURCE, createArchiveSpellCacheApi } from "./archive-spells.mjs";
|
|
|
|
const fixtureSourceSha = "b".repeat(64);
|
|
|
|
function sha256(bytes) {
|
|
return createHash("sha256").update(bytes).digest("hex");
|
|
}
|
|
|
|
function gitBlobSha(bytes) {
|
|
return createHash("sha1")
|
|
.update(Buffer.from("blob " + bytes.byteLength + "\0", "utf8"))
|
|
.update(bytes)
|
|
.digest("hex");
|
|
}
|
|
|
|
function spellRow(id, options = {}) {
|
|
const row = {};
|
|
for (let field = 0; field < 234; field += 1) row["f" + field] = 0;
|
|
row.f0 = id;
|
|
row.f41 = 1;
|
|
row.f47 = 1.5;
|
|
row.f71 = options.effectType ?? 0;
|
|
row.f110 = options.miscValue ?? 0;
|
|
row.f116 = options.triggerSpellId ?? 0;
|
|
row.f136 = options.name ?? "Test spell " + id;
|
|
row.f136i = id * 10;
|
|
row.f153 = "Passive";
|
|
row.f153i = id * 10 + 1;
|
|
row.f170 = "Description for " + id;
|
|
row.f170i = id * 10 + 2;
|
|
row.f187 = "Alternate " + id;
|
|
row.f187i = id * 10 + 3;
|
|
row.f229 = 0.25;
|
|
return row;
|
|
}
|
|
|
|
function makeFixture({ sourceDbcSha256 = fixtureSourceSha } = {}) {
|
|
const shardRows = [
|
|
{ lo: 1, hi: 99, rows: [spellRow(1, { triggerSpellId: 101 })] },
|
|
{ lo: 100, hi: 199, rows: [
|
|
spellRow(101, { effectType: 168, miscValue: 202 }),
|
|
spellRow(155),
|
|
] },
|
|
{ lo: 200, hi: 299, rows: [spellRow(202)] },
|
|
{ lo: 300, hi: 399, rows: [spellRow(350)] },
|
|
];
|
|
const shards = [];
|
|
const archiveFiles = new Map();
|
|
for (const group of shardRows) {
|
|
const name = String(group.lo).padStart(8, "0") + "-" + String(group.hi).padStart(8, "0") + ".jsonl.gz";
|
|
const bytes = gzipSync(Buffer.from(group.rows.map((row) => JSON.stringify(row)).join("\n") + "\n", "utf8"), { level: 9, mtime: 0 });
|
|
archiveFiles.set(name, bytes);
|
|
shards.push({
|
|
file: name,
|
|
format: "gzip",
|
|
lo: group.lo,
|
|
hi: group.hi,
|
|
plainBytes: 1,
|
|
rows: group.rows.length,
|
|
sha256: sha256(bytes),
|
|
storedBytes: bytes.byteLength,
|
|
});
|
|
}
|
|
|
|
const index = {
|
|
file: "Spell.dbc",
|
|
format: "gzip",
|
|
rows: 5,
|
|
columns: 234,
|
|
table: "Spell",
|
|
sourceSha256: sourceDbcSha256,
|
|
shardKey: { column: "f0", maxRows: 5000, mode: "column", unsigned: true },
|
|
shards,
|
|
};
|
|
const columnInfo = {
|
|
table: "Spell",
|
|
columnCount: 234,
|
|
header: { records: 5 },
|
|
source: { winner: "Data/patch-T.MPQ", sha256: sourceDbcSha256 },
|
|
columns: Array.from({ length: 234 }, (_, index) => ({
|
|
index,
|
|
inferred: index === 47 || index === 229 ? "float" : [136, 153, 170, 187].includes(index) ? "string" : "int",
|
|
})),
|
|
};
|
|
const indexBytes = Buffer.from(JSON.stringify(index), "utf8");
|
|
const colinfoBytes = Buffer.from(JSON.stringify(columnInfo), "utf8");
|
|
const variantPath = "raw/tables/Spell/variants/data-patch-t-mpq";
|
|
const source = {
|
|
id: "fixture-global-spell-base",
|
|
repository: "example/coa-datamine",
|
|
revision: "a".repeat(40),
|
|
shortRevision: "aaaaaaaaaaaa",
|
|
capturedAt: "2026-08-12T21:02:51+00:00",
|
|
variantPath,
|
|
tableIndexPath: variantPath + "/index.json",
|
|
tableIndexSha256: sha256(indexBytes),
|
|
tableIndexGitBlobSha: gitBlobSha(indexBytes),
|
|
columnInfoPath: variantPath + "/Spell.colinfo.json",
|
|
columnInfoSha256: sha256(colinfoBytes),
|
|
columnInfoGitBlobSha: gitBlobSha(colinfoBytes),
|
|
sourceDbcSha256,
|
|
rows: 5,
|
|
columns: 234,
|
|
};
|
|
const prefix = "https://raw.githubusercontent.com/" + source.repository + "/" + source.revision + "/";
|
|
const metadataFiles = new Map([
|
|
[prefix + source.tableIndexPath, indexBytes],
|
|
[prefix + source.columnInfoPath, colinfoBytes],
|
|
]);
|
|
for (const [name, bytes] of archiveFiles) metadataFiles.set(prefix + variantPath + "/" + name, bytes);
|
|
return { source, archiveFiles, files: metadataFiles, prefix, variantPath, indexBytes, colinfoBytes };
|
|
}
|
|
|
|
function fakeFetch(fileMap, calls) {
|
|
return async (url) => {
|
|
calls.push(url);
|
|
const bytes = fileMap.get(url);
|
|
if (!bytes) return { ok: false, status: 404 };
|
|
return {
|
|
ok: true,
|
|
arrayBuffer: async () => bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength),
|
|
};
|
|
};
|
|
}
|
|
|
|
async function withTempCache(run) {
|
|
const cacheRoot = await fs.mkdtemp(path.join(os.tmpdir(), "coa-archive-spells-"));
|
|
try {
|
|
await run(cacheRoot);
|
|
} finally {
|
|
await fs.rm(cacheRoot, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
test("pins and reads the global patch-T source and closes trigger and effect-168 references", async () => {
|
|
assert.equal(ARCHIVE_SPELL_SOURCE.variantPath, "raw/tables/Spell/variants/data-patch-t-mpq");
|
|
assert.equal(ARCHIVE_SPELL_SOURCE.sourceDbcSha256, "a1e4285cdbc8a269a1b8a3f4b304ee43eb4d1bfa5a98511639868c4ff6097623");
|
|
|
|
await withTempCache(async (cacheRoot) => {
|
|
const fixture = makeFixture();
|
|
const api = createArchiveSpellCacheApi(fixture.source);
|
|
const calls = [];
|
|
const fetchImpl = fakeFetch(fixture.files, calls);
|
|
const downloaded = await api.downloadArchiveSpells({ cacheRoot, spellIds: [1], fetchImpl });
|
|
|
|
assert.deepEqual(downloaded.records.map((row) => row.f0), [1, 101, 202]);
|
|
assert.deepEqual(downloaded.missingIds, []);
|
|
assert.deepEqual(downloaded.fieldTypes.slice(0, 3), ["int", "int", "int"]);
|
|
assert.equal(downloaded.fieldTypes[47], "float");
|
|
assert.equal(downloaded.fieldTypes[136], "string");
|
|
assert.equal(downloaded.source.path, fixture.variantPath);
|
|
assert.equal(downloaded.source.sourceDbcSha256, fixture.source.sourceDbcSha256);
|
|
assert.equal(downloaded.source.originalRecordCount, 5);
|
|
assert.equal(downloaded.source.bytes, downloaded.files.reduce((sum, file) => sum + file.size, 0));
|
|
assert.equal(downloaded.files.length, 3);
|
|
assert.ok(downloaded.files.every((file) => file.archivePath.startsWith(fixture.variantPath + "/")));
|
|
assert.equal(calls.filter((url) => url.includes("/variants/") && url.endsWith(".jsonl.gz")).length, 3);
|
|
assert.ok(!calls.some((url) => url.includes("/variants/") && url.includes("00000300-")));
|
|
|
|
const offline = await api.readArchiveSpells({ cacheRoot });
|
|
assert.deepEqual(offline.records, downloaded.records);
|
|
assert.deepEqual(offline.fieldTypes, downloaded.fieldTypes);
|
|
assert.deepEqual(offline.missingIds, []);
|
|
});
|
|
});
|
|
|
|
test("resumes selected shards, repairs tampered cached bytes, and reports absent ids", async () => {
|
|
await withTempCache(async (cacheRoot) => {
|
|
const fixture = makeFixture();
|
|
const api = createArchiveSpellCacheApi(fixture.source);
|
|
const calls = [];
|
|
const fetchImpl = fakeFetch(fixture.files, calls);
|
|
await api.downloadArchiveSpells({ cacheRoot, spellIds: [1], fetchImpl });
|
|
const firstCallCount = calls.length;
|
|
|
|
const resumed = await api.downloadArchiveSpells({ cacheRoot, spellIds: [350, 999], fetchImpl });
|
|
assert.deepEqual(resumed.records.map((row) => row.f0), [1, 101, 202, 350]);
|
|
assert.deepEqual(resumed.missingIds, [999]);
|
|
assert.equal(calls.length, firstCallCount + 1);
|
|
|
|
const cachedShard = path.join(cacheRoot, "archive-" + fixture.source.shortRevision, "spell-base", "shards", "00000001-00000099.jsonl.gz");
|
|
await fs.writeFile(cachedShard, Buffer.from("tampered"));
|
|
await assert.rejects(api.readArchiveSpells({ cacheRoot }), /SHA-256 mismatch/);
|
|
|
|
const beforeRepair = calls.length;
|
|
const repaired = await api.downloadArchiveSpells({ cacheRoot, spellIds: [350], fetchImpl });
|
|
assert.deepEqual(repaired.records.map((row) => row.f0), [1, 101, 202, 350]);
|
|
assert.equal(calls.length, beforeRepair + 1);
|
|
const raw = await fs.readFile(cachedShard);
|
|
assert.equal(sha256(raw), fixture.archiveFiles.get("00000001-00000099.jsonl.gz") && sha256(fixture.archiveFiles.get("00000001-00000099.jsonl.gz")));
|
|
});
|
|
});
|
|
|
|
test("reuses pin-matching remote-audit BASE metadata and shards without a network fetch", async () => {
|
|
await withTempCache(async (cacheRoot) => {
|
|
const fixture = makeFixture();
|
|
const archiveRoot = path.join(cacheRoot, "archive-" + fixture.source.shortRevision);
|
|
const remoteAudit = path.join(archiveRoot, "remote-audit");
|
|
const shardDirectory = path.join(remoteAudit, "base-spell-shards");
|
|
await fs.mkdir(shardDirectory, { recursive: true });
|
|
await fs.writeFile(path.join(remoteAudit, "raw_tables_Spell_variants_data-patch-t-mpq_index.json"), fixture.indexBytes);
|
|
await fs.writeFile(path.join(remoteAudit, "base_Spell.colinfo.json"), fixture.colinfoBytes);
|
|
for (const name of [
|
|
"00000001-00000099.jsonl.gz",
|
|
"00000100-00000199.jsonl.gz",
|
|
"00000200-00000299.jsonl.gz",
|
|
]) {
|
|
await fs.writeFile(path.join(shardDirectory, name), fixture.archiveFiles.get(name));
|
|
}
|
|
|
|
const calls = [];
|
|
const result = await createArchiveSpellCacheApi(fixture.source).downloadArchiveSpells({
|
|
cacheRoot,
|
|
spellIds: [1],
|
|
fetchImpl: async (url) => {
|
|
calls.push(url);
|
|
throw new Error("unexpected network request");
|
|
},
|
|
});
|
|
assert.equal(calls.length, 0);
|
|
assert.deepEqual(result.records.map((row) => row.f0), [1, 101, 202]);
|
|
assert.deepEqual(result.files.map((file) => file.path), [
|
|
"00000001-00000099.jsonl.gz",
|
|
"00000100-00000199.jsonl.gz",
|
|
"00000200-00000299.jsonl.gz",
|
|
]);
|
|
});
|
|
});
|
|
|
|
test("rejects an overlay index even when its metadata hashes match the supplied bytes", async () => {
|
|
await withTempCache(async (cacheRoot) => {
|
|
const fixture = makeFixture();
|
|
const overlayFixture = makeFixture({ sourceDbcSha256: "c".repeat(64) });
|
|
const wrongIndexUrl = fixture.prefix + fixture.source.tableIndexPath;
|
|
fixture.files.set(wrongIndexUrl, overlayFixture.files.get(wrongIndexUrl));
|
|
const overlayBytes = fixture.files.get(wrongIndexUrl);
|
|
fixture.source.tableIndexSha256 = sha256(overlayBytes);
|
|
fixture.source.tableIndexGitBlobSha = gitBlobSha(overlayBytes);
|
|
const api = createArchiveSpellCacheApi(fixture.source);
|
|
await assert.rejects(
|
|
api.downloadArchiveSpells({ cacheRoot, spellIds: [1], fetchImpl: fakeFetch(fixture.files, []) }),
|
|
/pinned BASE patch-T Spell\.dbc source/,
|
|
);
|
|
});
|
|
});
|