import { createHash } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { gunzipSync } from "node:zlib"; export const ARCHIVE_SPELL_SOURCE = Object.freeze({ id: "august-global-spell-base", repository: "srhinos/coa-datamine", revision: "5078f9ac22541c45c17a866c23aac6c12073de23", shortRevision: "5078f9ac2254", capturedAt: "2026-08-12T21:02:51+00:00", variantPath: "raw/tables/Spell/variants/data-patch-t-mpq", tableIndexPath: "raw/tables/Spell/variants/data-patch-t-mpq/index.json", tableIndexSha256: "3f3bcf86285607f73bc1cab248c57e12abb1d858c015cbd06cfc15d8ec0aea84", tableIndexGitBlobSha: "fadd592b46f7d6d413713d79dff78018c14595bb", columnInfoPath: "raw/tables/Spell/variants/data-patch-t-mpq/Spell.colinfo.json", columnInfoSha256: "ba7ecede4d45c93ac66dd5917e5f9c34c81ac7da818e4a3c33b5cc80d5e1cf20", columnInfoGitBlobSha: "ad3acfe6b3894a7990b81050972d707a6c0e7b28", sourceDbcSha256: "a1e4285cdbc8a269a1b8a3f4b304ee43eb4d1bfa5a98511639868c4ff6097623", rows: 209206, columns: 234, }); const cacheSchemaVersion = 1; const manifestName = "manifest.json"; function sha256(bytes) { return createHash("sha256").update(bytes).digest("hex"); } function gitBlobSha(bytes) { return createHash("sha1") .update(Buffer.from("blob " + bytes.byteLength + "\0", "utf8")) .update(bytes) .digest("hex"); } function parseJson(bytes, label) { try { return JSON.parse(bytes.toString("utf8")); } catch (error) { throw new Error("Archive Spell JSON is invalid at " + label + ": " + error.message, { cause: error }); } } function assertSafeRelativePath(value, label) { if (typeof value !== "string" || value.length === 0) throw new TypeError(label + " must be a non-empty path"); const normalized = value.replaceAll("\\", "/"); if (normalized.startsWith("/") || /^[A-Za-z]:/.test(normalized) || normalized.split("/").some((part) => part === "" || part === "." || part === "..")) { throw new Error(label + " is not a safe relative path: " + value); } return normalized; } function validateSource(source) { const required = [ "id", "repository", "revision", "shortRevision", "capturedAt", "variantPath", "tableIndexPath", "tableIndexSha256", "tableIndexGitBlobSha", "columnInfoPath", "columnInfoSha256", "columnInfoGitBlobSha", "sourceDbcSha256", ]; for (const key of required) { if (typeof source[key] !== "string" || source[key].length === 0) { throw new TypeError("source." + key + " must be a non-empty string"); } } if (!/^[0-9a-f]{40}$/i.test(source.revision) || !/^[0-9a-f]{7,40}$/i.test(source.shortRevision)) { throw new TypeError("source revision values must be hexadecimal Git revisions"); } for (const key of ["tableIndexSha256", "columnInfoSha256", "sourceDbcSha256"]) { if (!/^[0-9a-f]{64}$/i.test(source[key])) throw new TypeError("source." + key + " must be a SHA-256 digest"); } for (const key of ["tableIndexGitBlobSha", "columnInfoGitBlobSha"]) { if (!/^[0-9a-f]{40}$/i.test(source[key])) throw new TypeError("source." + key + " must be a Git blob SHA"); } if (!Number.isSafeInteger(source.rows) || source.rows < 1) throw new TypeError("source.rows must be a positive integer"); if (!Number.isSafeInteger(source.columns) || source.columns < 1) throw new TypeError("source.columns must be a positive integer"); source.variantPath = assertSafeRelativePath(source.variantPath, "source.variantPath"); source.tableIndexPath = assertSafeRelativePath(source.tableIndexPath, "source.tableIndexPath"); source.columnInfoPath = assertSafeRelativePath(source.columnInfoPath, "source.columnInfoPath"); if (source.tableIndexPath !== source.variantPath + "/index.json" || source.columnInfoPath !== source.variantPath + "/Spell.colinfo.json") { throw new Error("Archive Spell metadata paths must belong to the pinned BASE variant"); } return Object.freeze({ ...source }); } function sourceRecord(source) { return { id: source.id, repository: "https://github.com/" + source.repository, revision: source.revision, capturedAt: source.capturedAt, variantPath: source.variantPath, tableIndexPath: source.tableIndexPath, tableIndexSha256: source.tableIndexSha256, tableIndexGitBlobSha: source.tableIndexGitBlobSha, columnInfoPath: source.columnInfoPath, columnInfoSha256: source.columnInfoSha256, columnInfoGitBlobSha: source.columnInfoGitBlobSha, sourceDbcSha256: source.sourceDbcSha256, rows: source.rows, columns: source.columns, }; } function resultSource(source, files) { const storedBytes = files.reduce((total, file) => total + file.size, 0); return { id: source.id, path: source.variantPath, sha256: source.tableIndexSha256, bytes: storedBytes, storedBytes, sourceDbcSha256: source.sourceDbcSha256, originalRecordCount: source.rows, pin: source.revision, capturedAt: source.capturedAt, tableIndexPath: source.tableIndexPath, tableIndexGitBlobSha: source.tableIndexGitBlobSha, columnInfoPath: source.columnInfoPath, columnInfoSha256: source.columnInfoSha256, columnInfoGitBlobSha: source.columnInfoGitBlobSha, }; } function assertPinnedBytes(bytes, sha256Pin, gitBlobPin, label) { const actualSha = sha256(bytes); const actualBlob = gitBlobSha(bytes); if (actualSha !== sha256Pin.toLowerCase()) { throw new Error(label + " SHA-256 mismatch: expected " + sha256Pin + ", found " + actualSha); } if (actualBlob !== gitBlobPin.toLowerCase()) { throw new Error(label + " Git blob mismatch: expected " + gitBlobPin + ", found " + actualBlob); } } function validateIndex(index, source) { if (!index || typeof index !== "object" || Array.isArray(index)) throw new Error("Archive Spell index is not an object"); if (index.table !== "Spell" || index.file !== "Spell.dbc") throw new Error("Archive Spell index is not the Spell table"); if (index.sourceSha256 !== source.sourceDbcSha256) { throw new Error("Archive Spell index is not the pinned BASE patch-T Spell.dbc source"); } if (index.rows !== source.rows || index.columns !== source.columns) { throw new Error("Archive Spell index row or column count does not match the pinned source"); } if (index.format !== "gzip" || index.shardKey?.column !== "f0" || index.shardKey?.mode !== "column" || index.shardKey?.unsigned !== true) { throw new Error("Archive Spell index does not use the pinned unsigned f0 shard layout"); } if (!Array.isArray(index.shards) || index.shards.length === 0) throw new Error("Archive Spell index has no shard list"); const names = new Set(); const ranges = []; for (const shard of index.shards) { const name = assertSafeRelativePath(shard?.file, "Archive Spell shard path"); const match = /^([0-9]+)-([0-9]+)\.jsonl\.gz$/.exec(name); if (!match || name.includes("/")) throw new Error("Archive Spell index has an invalid shard filename: " + name); if (names.has(name)) throw new Error("Archive Spell index repeats shard " + name); names.add(name); const lo = Number(match[1]); const hi = Number(match[2]); if (lo !== shard.lo || hi !== shard.hi || !Number.isSafeInteger(lo) || !Number.isSafeInteger(hi) || hi < lo) { throw new Error("Archive Spell shard range disagrees with filename: " + name); } if (!Number.isSafeInteger(shard.rows) || shard.rows < 0 || !/^[0-9a-f]{64}$/i.test(shard.sha256 ?? "") || shard.format !== "gzip") { throw new Error("Archive Spell index has invalid shard metadata for " + name); } ranges.push({ lo, hi, name }); } ranges.sort((a, b) => a.lo - b.lo || a.hi - b.hi); for (let index = 1; index < ranges.length; index += 1) { if (ranges[index].lo <= ranges[index - 1].hi) throw new Error("Archive Spell index has overlapping shard ranges"); } return index; } function fieldTypesFromColumnInfo(columnInfo, source) { const columns = columnInfo?.columns; if (!columnInfo || typeof columnInfo !== "object" || !Array.isArray(columns)) { throw new Error("Pinned BASE Spell.colinfo.json has no columns array"); } if (columnInfo.table !== "Spell" || columnInfo.columnCount !== source.columns || columns.length !== source.columns || columnInfo.source?.winner !== "Data/patch-T.MPQ" || columnInfo.source?.sha256 !== source.sourceDbcSha256 || columnInfo.header?.records !== source.rows) { throw new Error("Pinned BASE Spell.colinfo.json does not describe all Spell fields"); } const fieldTypes = Array(source.columns); for (const column of columns) { const index = Number(column?.index); const type = column?.inferred; if (!Number.isSafeInteger(index) || index < 0 || index >= source.columns || fieldTypes[index] !== undefined) { throw new Error("Pinned BASE Spell.colinfo.json has an invalid or duplicate field index"); } if (!["int", "float", "string", "unknown"].includes(type)) { throw new Error("Pinned BASE Spell.colinfo.json has an unsupported inferred type at f" + index); } fieldTypes[index] = type; } if (fieldTypes.some((type) => type === undefined)) throw new Error("Pinned BASE Spell.colinfo.json is missing a field type"); return fieldTypes; } function parseShard(bytes, shard, source) { if (sha256(bytes) !== shard.sha256.toLowerCase()) { throw new Error("Cached Archive Spell shard SHA-256 mismatch: " + shard.file); } let text; try { text = gunzipSync(bytes).toString("utf8"); } catch (error) { throw new Error("Archive Spell shard is not valid gzip data: " + shard.file, { cause: error }); } const rows = []; const ids = new Set(); for (const [lineIndex, line] of text.split("\n").entries()) { if (line.length === 0) continue; const row = parseJson(Buffer.from(line, "utf8"), shard.file + ":" + (lineIndex + 1)); if (!row || typeof row !== "object" || Array.isArray(row)) throw new Error("Archive Spell row is not an object in " + shard.file); const id = Number(row.f0); if (!Number.isSafeInteger(id) || id < shard.lo || id > shard.hi || ids.has(id)) { throw new Error("Archive Spell row has an invalid or duplicate f0 in " + shard.file); } for (let field = 0; field < source.columns; field += 1) { if (!Object.hasOwn(row, "f" + field)) throw new Error("Archive Spell row is missing f" + field + " in " + shard.file); } ids.add(id); rows.push(row); } if (rows.length !== shard.rows) throw new Error("Archive Spell shard row count mismatch: " + shard.file); rows.sort((a, b) => Number(a.f0) - Number(b.f0)); return rows; } function positiveSpellIds(input) { if (!Array.isArray(input)) throw new TypeError("spellIds must be an array"); const ids = input.map((value) => Number(value)); if (ids.some((id) => !Number.isSafeInteger(id) || id <= 0)) { throw new TypeError("spellIds must contain positive safe integers"); } return [...new Set(ids)].sort((a, b) => a - b); } function childrenFromRow(row) { const children = []; for (let effect = 0; effect < 3; effect += 1) { const triggerId = Number(row["f" + (116 + effect)]); if (Number.isSafeInteger(triggerId) && triggerId > 0) children.push(triggerId); if (Number(row["f" + (71 + effect)]) === 168) { const itemSpellId = Number(row["f" + (110 + effect)]); if (Number.isSafeInteger(itemSpellId) && itemSpellId > 0) children.push(itemSpellId); } } return children; } function descriptorForId(index, id) { return index.shards.find((shard) => id >= shard.lo && id <= shard.hi) ?? null; } function validateManifest(manifest, source, index) { if (!manifest || typeof manifest !== "object" || Array.isArray(manifest)) throw new Error("Archive Spell cache manifest is not an object"); if (manifest.schemaVersion !== cacheSchemaVersion) throw new Error("Unsupported Archive Spell cache schema"); if (JSON.stringify(manifest.source) !== JSON.stringify(sourceRecord(source))) { throw new Error("Archive Spell cache manifest points at an unexpected source or patch variant"); } if (!Array.isArray(manifest.requestedIds) || !Array.isArray(manifest.files)) { throw new Error("Archive Spell cache manifest is missing requestedIds or files"); } const requestedIds = positiveSpellIds(manifest.requestedIds); if (requestedIds.length !== manifest.requestedIds.length || requestedIds.some((id, position) => id !== manifest.requestedIds[position])) { throw new Error("Archive Spell cache manifest requested ids must be sorted and unique"); } const byFile = new Map(index.shards.map((shard) => [shard.file, shard])); const names = new Set(); for (const file of manifest.files) { const name = assertSafeRelativePath(file?.path, "Archive Spell manifest shard path"); const shard = byFile.get(name); if (!shard || names.has(name) || file.sha256 !== shard.sha256 || file.archivePath !== source.variantPath + "/" + name || file.lo !== shard.lo || file.hi !== shard.hi || file.rows !== shard.rows || !/^[0-9a-f]{40}$/i.test(file.gitBlobSha ?? "") || !Number.isSafeInteger(file.size) || file.size < 0) { throw new Error("Archive Spell cache manifest contains an unknown or inconsistent shard: " + name); } names.add(name); } return { requestedIds, files: manifest.files }; } export function createArchiveSpellCacheApi(sourceInput = ARCHIVE_SPELL_SOURCE) { const source = validateSource({ ...sourceInput }); function archiveDirectory(cacheRoot) { if (typeof cacheRoot !== "string" || cacheRoot.length === 0) throw new TypeError("cacheRoot must be a non-empty directory path"); return path.join(path.resolve(cacheRoot), "archive-" + source.shortRevision, "spell-base"); } function rawUrl(filePath) { const encodedPath = filePath.split("/").map(encodeURIComponent).join("/"); return "https://raw.githubusercontent.com/" + source.repository + "/" + source.revision + "/" + encodedPath; } async function responseBytes(fetchImpl, url) { let response; try { response = await fetchImpl(url, { headers: { Accept: "application/json, application/gzip" } }); } catch (error) { throw new Error("Archive Spell source unavailable at " + url + ": " + error.message, { cause: error }); } if (!response || response.ok !== true) { const status = response?.status === undefined ? "no response" : "HTTP " + response.status; throw new Error("Archive Spell source unavailable at " + url + " (" + status + ")"); } if (typeof response.arrayBuffer === "function") return Buffer.from(await response.arrayBuffer()); if (typeof response.text === "function") return Buffer.from(await response.text(), "utf8"); throw new Error("Archive Spell source response has no readable body at " + url); } async function writeAtomic(filePath, bytes) { await fs.mkdir(path.dirname(filePath), { recursive: true }); const tempPath = filePath + ".tmp-" + process.pid; await fs.writeFile(tempPath, bytes); await fs.rename(tempPath, filePath); } async function cachedPinnedBytes(filePath, shaPin, blobPin) { try { const bytes = await fs.readFile(filePath); assertPinnedBytes(bytes, shaPin, blobPin, path.basename(filePath)); return bytes; } catch (error) { if (error?.code === "ENOENT") return null; if (error?.code && error.code !== "ENOENT") throw error; return null; } } async function ensureMetadata(cacheRoot, fetchImpl) { const root = archiveDirectory(cacheRoot); const indexCachePath = path.join(root, "Spell.index.json"); const colinfoCachePath = path.join(root, "Spell.colinfo.json"); let [indexBytes, colinfoBytes] = await Promise.all([ cachedPinnedBytes(indexCachePath, source.tableIndexSha256, source.tableIndexGitBlobSha), cachedPinnedBytes(colinfoCachePath, source.columnInfoSha256, source.columnInfoGitBlobSha), ]); if (!indexBytes) { indexBytes = await cachedPinnedBytes( path.join(path.dirname(root), "remote-audit", "raw_tables_Spell_variants_data-patch-t-mpq_index.json"), source.tableIndexSha256, source.tableIndexGitBlobSha, ); if (indexBytes) await writeAtomic(indexCachePath, indexBytes); } if (!colinfoBytes) { colinfoBytes = await cachedPinnedBytes( path.join(path.dirname(root), "remote-audit", "base_Spell.colinfo.json"), source.columnInfoSha256, source.columnInfoGitBlobSha, ); if (colinfoBytes) await writeAtomic(colinfoCachePath, colinfoBytes); } if (!indexBytes) { indexBytes = await responseBytes(fetchImpl, rawUrl(source.tableIndexPath)); assertPinnedBytes(indexBytes, source.tableIndexSha256, source.tableIndexGitBlobSha, "BASE Spell index"); await writeAtomic(indexCachePath, indexBytes); } if (!colinfoBytes) { colinfoBytes = await responseBytes(fetchImpl, rawUrl(source.columnInfoPath)); assertPinnedBytes(colinfoBytes, source.columnInfoSha256, source.columnInfoGitBlobSha, "BASE Spell.colinfo.json"); await writeAtomic(colinfoCachePath, colinfoBytes); } const index = validateIndex(parseJson(indexBytes, source.tableIndexPath), source); const columnInfo = parseJson(colinfoBytes, source.columnInfoPath); const fieldTypes = fieldTypesFromColumnInfo(columnInfo, source); return { root, index, fieldTypes }; } async function readShard(root, shard, sourceInputForRows) { const shardPath = path.join(root, "shards", shard.file); const bytes = await fs.readFile(shardPath); return { bytes, rows: parseShard(bytes, shard, sourceInputForRows) }; } async function loadManifest(root, index) { const manifestPath = path.join(root, manifestName); let bytes; try { bytes = await fs.readFile(manifestPath); } catch (error) { if (error?.code === "ENOENT") return null; throw error; } return validateManifest(parseJson(bytes, manifestPath), source, index); } async function closeSpellIds({ index, requestedIds, root, fetchImpl, allowFetch }) { const records = new Map(); const missing = new Set(); const selectedShards = new Map(); const loadedRows = new Map(); const pending = [...requestedIds]; const visited = new Set(); for (let cursor = 0; cursor < pending.length; cursor += 1) { const id = pending[cursor]; if (visited.has(id)) continue; visited.add(id); const descriptor = descriptorForId(index, id); if (!descriptor) { missing.add(id); continue; } selectedShards.set(descriptor.file, descriptor); let rows = loadedRows.get(descriptor.file); if (!rows) { const localPath = path.join(root, "shards", descriptor.file); let bytes = null; let localBytesFound = false; const candidatePaths = allowFetch ? [localPath, path.join(path.dirname(root), "remote-audit", "base-spell-shards", descriptor.file)] : [localPath]; for (const candidatePath of candidatePaths) { try { const candidate = await fs.readFile(candidatePath); if (candidatePath === localPath) localBytesFound = true; if (sha256(candidate) !== descriptor.sha256.toLowerCase()) { if (!allowFetch && candidatePath === localPath) { throw new Error("Cached Archive Spell shard SHA-256 mismatch: " + descriptor.file); } continue; } bytes = candidate; if (candidatePath !== localPath) await writeAtomic(localPath, bytes); break; } catch (error) { if (error?.code !== "ENOENT") throw error; } } if (!bytes && allowFetch) { bytes = await responseBytes(fetchImpl, rawUrl(source.variantPath + "/" + descriptor.file)); if (sha256(bytes) !== descriptor.sha256.toLowerCase()) { throw new Error("BASE Archive Spell shard SHA-256 mismatch: " + descriptor.file); } await writeAtomic(localPath, bytes); } if (!bytes) { const detail = localBytesFound ? " has invalid cached bytes" : " is missing"; throw new Error("Archive Spell cache shard " + descriptor.file + detail + "; run downloadArchiveSpells first"); } rows = parseShard(bytes, descriptor, source); loadedRows.set(descriptor.file, rows); } const row = rows.find((candidate) => Number(candidate.f0) === id); if (!row) { missing.add(id); continue; } records.set(id, row); for (const child of childrenFromRow(row)) if (!visited.has(child)) pending.push(child); } return { records: [...records.values()].sort((a, b) => Number(a.f0) - Number(b.f0)), missingIds: [...missing].sort((a, b) => a - b), selectedShards, }; } function manifestFile(shard, bytes, source) { return { path: shard.file, archivePath: source.variantPath + "/" + shard.file, lo: shard.lo, hi: shard.hi, rows: shard.rows, sha256: shard.sha256, gitBlobSha: gitBlobSha(bytes), size: bytes.byteLength, }; } async function readArchiveSpells({ cacheRoot }) { const root = archiveDirectory(cacheRoot); let indexBytes; let colinfoBytes; try { [indexBytes, colinfoBytes] = await Promise.all([ fs.readFile(path.join(root, "Spell.index.json")), fs.readFile(path.join(root, "Spell.colinfo.json")), ]); } catch (error) { if (error?.code === "ENOENT") throw new Error("Archive Spell cache metadata is missing; run downloadArchiveSpells first"); throw error; } assertPinnedBytes(indexBytes, source.tableIndexSha256, source.tableIndexGitBlobSha, "Cached BASE Spell index"); assertPinnedBytes(colinfoBytes, source.columnInfoSha256, source.columnInfoGitBlobSha, "Cached BASE Spell.colinfo.json"); const index = validateIndex(parseJson(indexBytes, source.tableIndexPath), source); const fieldTypes = fieldTypesFromColumnInfo(parseJson(colinfoBytes, source.columnInfoPath), source); const manifest = await loadManifest(root, index); if (!manifest) throw new Error("Archive Spell cache manifest is missing; run downloadArchiveSpells first"); const closure = await closeSpellIds({ index, requestedIds: manifest.requestedIds, root, allowFetch: false }); const declared = new Map(manifest.files.map((file) => [file.path, file])); if (declared.size !== closure.selectedShards.size || [...closure.selectedShards.keys()].some((name) => !declared.has(name))) { throw new Error("Archive Spell cache manifest does not cover the complete selected spell closure"); } const files = []; for (const shard of [...closure.selectedShards.values()].sort((a, b) => a.lo - b.lo)) { const { bytes } = await readShard(root, shard, source); const file = manifestFile(shard, bytes, source); const expected = declared.get(shard.file); if (file.gitBlobSha !== expected.gitBlobSha || file.size !== expected.size) { throw new Error("Archive Spell cached shard does not match its manifest: " + shard.file); } files.push(file); } return { source: resultSource(source, files), fieldTypes, records: closure.records, files, missingIds: closure.missingIds, }; } async function downloadArchiveSpells({ cacheRoot, spellIds, fetchImpl = globalThis.fetch }) { if (typeof fetchImpl !== "function") throw new TypeError("fetchImpl must be a function"); const newIds = positiveSpellIds(spellIds); const { root, index, fieldTypes } = await ensureMetadata(cacheRoot, fetchImpl); const previous = await loadManifest(root, index); const requestedIds = positiveSpellIds([...(previous?.requestedIds ?? []), ...newIds]); const closure = await closeSpellIds({ index, requestedIds, root, fetchImpl, allowFetch: true }); const files = []; for (const shard of [...closure.selectedShards.values()].sort((a, b) => a.lo - b.lo)) { const { bytes } = await readShard(root, shard, source); files.push(manifestFile(shard, bytes, source)); } const manifest = { schemaVersion: cacheSchemaVersion, source: sourceRecord(source), requestedIds, files, }; await writeAtomic(path.join(root, manifestName), Buffer.from(JSON.stringify(manifest, null, 2) + "\n", "utf8")); return { source: resultSource(source, files), fieldTypes, records: closure.records, files, missingIds: closure.missingIds, }; } return Object.freeze({ downloadArchiveSpells, readArchiveSpells }); } const defaultArchiveSpellApi = createArchiveSpellCacheApi(); export function downloadArchiveSpells({ cacheRoot, spellIds, fetchImpl = globalThis.fetch }) { return defaultArchiveSpellApi.downloadArchiveSpells({ cacheRoot, spellIds, fetchImpl }); } export function readArchiveSpells({ cacheRoot }) { return defaultArchiveSpellApi.readArchiveSpells({ cacheRoot }); }