forked from phenom/healer-man
567 lines
25 KiB
JavaScript
567 lines
25 KiB
JavaScript
import { createHash } from "node:crypto";
|
|
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { gunzipSync } from "node:zlib";
|
|
|
|
export const ARCHIVE_SPELL_SOURCE = Object.freeze({
|
|
id: "august-global-spell-base",
|
|
repository: "srhinos/coa-datamine",
|
|
revision: "5078f9ac22541c45c17a866c23aac6c12073de23",
|
|
shortRevision: "5078f9ac2254",
|
|
capturedAt: "2026-08-12T21:02:51+00:00",
|
|
variantPath: "raw/tables/Spell/variants/data-patch-t-mpq",
|
|
tableIndexPath: "raw/tables/Spell/variants/data-patch-t-mpq/index.json",
|
|
tableIndexSha256: "3f3bcf86285607f73bc1cab248c57e12abb1d858c015cbd06cfc15d8ec0aea84",
|
|
tableIndexGitBlobSha: "fadd592b46f7d6d413713d79dff78018c14595bb",
|
|
columnInfoPath: "raw/tables/Spell/variants/data-patch-t-mpq/Spell.colinfo.json",
|
|
columnInfoSha256: "ba7ecede4d45c93ac66dd5917e5f9c34c81ac7da818e4a3c33b5cc80d5e1cf20",
|
|
columnInfoGitBlobSha: "ad3acfe6b3894a7990b81050972d707a6c0e7b28",
|
|
sourceDbcSha256: "a1e4285cdbc8a269a1b8a3f4b304ee43eb4d1bfa5a98511639868c4ff6097623",
|
|
rows: 209206,
|
|
columns: 234,
|
|
});
|
|
|
|
const cacheSchemaVersion = 1;
|
|
const manifestName = "manifest.json";
|
|
|
|
function sha256(bytes) {
|
|
return createHash("sha256").update(bytes).digest("hex");
|
|
}
|
|
|
|
function gitBlobSha(bytes) {
|
|
return createHash("sha1")
|
|
.update(Buffer.from("blob " + bytes.byteLength + "\0", "utf8"))
|
|
.update(bytes)
|
|
.digest("hex");
|
|
}
|
|
|
|
function parseJson(bytes, label) {
|
|
try {
|
|
return JSON.parse(bytes.toString("utf8"));
|
|
} catch (error) {
|
|
throw new Error("Archive Spell JSON is invalid at " + label + ": " + error.message, { cause: error });
|
|
}
|
|
}
|
|
|
|
function assertSafeRelativePath(value, label) {
|
|
if (typeof value !== "string" || value.length === 0) throw new TypeError(label + " must be a non-empty path");
|
|
const normalized = value.replaceAll("\\", "/");
|
|
if (normalized.startsWith("/") || /^[A-Za-z]:/.test(normalized)
|
|
|| normalized.split("/").some((part) => part === "" || part === "." || part === "..")) {
|
|
throw new Error(label + " is not a safe relative path: " + value);
|
|
}
|
|
return normalized;
|
|
}
|
|
|
|
function validateSource(source) {
|
|
const required = [
|
|
"id", "repository", "revision", "shortRevision", "capturedAt", "variantPath",
|
|
"tableIndexPath", "tableIndexSha256", "tableIndexGitBlobSha", "columnInfoPath",
|
|
"columnInfoSha256", "columnInfoGitBlobSha", "sourceDbcSha256",
|
|
];
|
|
for (const key of required) {
|
|
if (typeof source[key] !== "string" || source[key].length === 0) {
|
|
throw new TypeError("source." + key + " must be a non-empty string");
|
|
}
|
|
}
|
|
if (!/^[0-9a-f]{40}$/i.test(source.revision) || !/^[0-9a-f]{7,40}$/i.test(source.shortRevision)) {
|
|
throw new TypeError("source revision values must be hexadecimal Git revisions");
|
|
}
|
|
for (const key of ["tableIndexSha256", "columnInfoSha256", "sourceDbcSha256"]) {
|
|
if (!/^[0-9a-f]{64}$/i.test(source[key])) throw new TypeError("source." + key + " must be a SHA-256 digest");
|
|
}
|
|
for (const key of ["tableIndexGitBlobSha", "columnInfoGitBlobSha"]) {
|
|
if (!/^[0-9a-f]{40}$/i.test(source[key])) throw new TypeError("source." + key + " must be a Git blob SHA");
|
|
}
|
|
if (!Number.isSafeInteger(source.rows) || source.rows < 1) throw new TypeError("source.rows must be a positive integer");
|
|
if (!Number.isSafeInteger(source.columns) || source.columns < 1) throw new TypeError("source.columns must be a positive integer");
|
|
source.variantPath = assertSafeRelativePath(source.variantPath, "source.variantPath");
|
|
source.tableIndexPath = assertSafeRelativePath(source.tableIndexPath, "source.tableIndexPath");
|
|
source.columnInfoPath = assertSafeRelativePath(source.columnInfoPath, "source.columnInfoPath");
|
|
if (source.tableIndexPath !== source.variantPath + "/index.json"
|
|
|| source.columnInfoPath !== source.variantPath + "/Spell.colinfo.json") {
|
|
throw new Error("Archive Spell metadata paths must belong to the pinned BASE variant");
|
|
}
|
|
return Object.freeze({ ...source });
|
|
}
|
|
|
|
function sourceRecord(source) {
|
|
return {
|
|
id: source.id,
|
|
repository: "https://github.com/" + source.repository,
|
|
revision: source.revision,
|
|
capturedAt: source.capturedAt,
|
|
variantPath: source.variantPath,
|
|
tableIndexPath: source.tableIndexPath,
|
|
tableIndexSha256: source.tableIndexSha256,
|
|
tableIndexGitBlobSha: source.tableIndexGitBlobSha,
|
|
columnInfoPath: source.columnInfoPath,
|
|
columnInfoSha256: source.columnInfoSha256,
|
|
columnInfoGitBlobSha: source.columnInfoGitBlobSha,
|
|
sourceDbcSha256: source.sourceDbcSha256,
|
|
rows: source.rows,
|
|
columns: source.columns,
|
|
};
|
|
}
|
|
|
|
function resultSource(source, files) {
|
|
const storedBytes = files.reduce((total, file) => total + file.size, 0);
|
|
return {
|
|
id: source.id,
|
|
path: source.variantPath,
|
|
sha256: source.tableIndexSha256,
|
|
bytes: storedBytes,
|
|
storedBytes,
|
|
sourceDbcSha256: source.sourceDbcSha256,
|
|
originalRecordCount: source.rows,
|
|
pin: source.revision,
|
|
capturedAt: source.capturedAt,
|
|
tableIndexPath: source.tableIndexPath,
|
|
tableIndexGitBlobSha: source.tableIndexGitBlobSha,
|
|
columnInfoPath: source.columnInfoPath,
|
|
columnInfoSha256: source.columnInfoSha256,
|
|
columnInfoGitBlobSha: source.columnInfoGitBlobSha,
|
|
};
|
|
}
|
|
|
|
function assertPinnedBytes(bytes, sha256Pin, gitBlobPin, label) {
|
|
const actualSha = sha256(bytes);
|
|
const actualBlob = gitBlobSha(bytes);
|
|
if (actualSha !== sha256Pin.toLowerCase()) {
|
|
throw new Error(label + " SHA-256 mismatch: expected " + sha256Pin + ", found " + actualSha);
|
|
}
|
|
if (actualBlob !== gitBlobPin.toLowerCase()) {
|
|
throw new Error(label + " Git blob mismatch: expected " + gitBlobPin + ", found " + actualBlob);
|
|
}
|
|
}
|
|
|
|
function validateIndex(index, source) {
|
|
if (!index || typeof index !== "object" || Array.isArray(index)) throw new Error("Archive Spell index is not an object");
|
|
if (index.table !== "Spell" || index.file !== "Spell.dbc") throw new Error("Archive Spell index is not the Spell table");
|
|
if (index.sourceSha256 !== source.sourceDbcSha256) {
|
|
throw new Error("Archive Spell index is not the pinned BASE patch-T Spell.dbc source");
|
|
}
|
|
if (index.rows !== source.rows || index.columns !== source.columns) {
|
|
throw new Error("Archive Spell index row or column count does not match the pinned source");
|
|
}
|
|
if (index.format !== "gzip" || index.shardKey?.column !== "f0"
|
|
|| index.shardKey?.mode !== "column" || index.shardKey?.unsigned !== true) {
|
|
throw new Error("Archive Spell index does not use the pinned unsigned f0 shard layout");
|
|
}
|
|
if (!Array.isArray(index.shards) || index.shards.length === 0) throw new Error("Archive Spell index has no shard list");
|
|
const names = new Set();
|
|
const ranges = [];
|
|
for (const shard of index.shards) {
|
|
const name = assertSafeRelativePath(shard?.file, "Archive Spell shard path");
|
|
const match = /^([0-9]+)-([0-9]+)\.jsonl\.gz$/.exec(name);
|
|
if (!match || name.includes("/")) throw new Error("Archive Spell index has an invalid shard filename: " + name);
|
|
if (names.has(name)) throw new Error("Archive Spell index repeats shard " + name);
|
|
names.add(name);
|
|
const lo = Number(match[1]);
|
|
const hi = Number(match[2]);
|
|
if (lo !== shard.lo || hi !== shard.hi || !Number.isSafeInteger(lo) || !Number.isSafeInteger(hi) || hi < lo) {
|
|
throw new Error("Archive Spell shard range disagrees with filename: " + name);
|
|
}
|
|
if (!Number.isSafeInteger(shard.rows) || shard.rows < 0
|
|
|| !/^[0-9a-f]{64}$/i.test(shard.sha256 ?? "") || shard.format !== "gzip") {
|
|
throw new Error("Archive Spell index has invalid shard metadata for " + name);
|
|
}
|
|
ranges.push({ lo, hi, name });
|
|
}
|
|
ranges.sort((a, b) => a.lo - b.lo || a.hi - b.hi);
|
|
for (let index = 1; index < ranges.length; index += 1) {
|
|
if (ranges[index].lo <= ranges[index - 1].hi) throw new Error("Archive Spell index has overlapping shard ranges");
|
|
}
|
|
return index;
|
|
}
|
|
|
|
function fieldTypesFromColumnInfo(columnInfo, source) {
|
|
const columns = columnInfo?.columns;
|
|
if (!columnInfo || typeof columnInfo !== "object" || !Array.isArray(columns)) {
|
|
throw new Error("Pinned BASE Spell.colinfo.json has no columns array");
|
|
}
|
|
if (columnInfo.table !== "Spell" || columnInfo.columnCount !== source.columns || columns.length !== source.columns
|
|
|| columnInfo.source?.winner !== "Data/patch-T.MPQ"
|
|
|| columnInfo.source?.sha256 !== source.sourceDbcSha256
|
|
|| columnInfo.header?.records !== source.rows) {
|
|
throw new Error("Pinned BASE Spell.colinfo.json does not describe all Spell fields");
|
|
}
|
|
const fieldTypes = Array(source.columns);
|
|
for (const column of columns) {
|
|
const index = Number(column?.index);
|
|
const type = column?.inferred;
|
|
if (!Number.isSafeInteger(index) || index < 0 || index >= source.columns || fieldTypes[index] !== undefined) {
|
|
throw new Error("Pinned BASE Spell.colinfo.json has an invalid or duplicate field index");
|
|
}
|
|
if (!["int", "float", "string", "unknown"].includes(type)) {
|
|
throw new Error("Pinned BASE Spell.colinfo.json has an unsupported inferred type at f" + index);
|
|
}
|
|
fieldTypes[index] = type;
|
|
}
|
|
if (fieldTypes.some((type) => type === undefined)) throw new Error("Pinned BASE Spell.colinfo.json is missing a field type");
|
|
return fieldTypes;
|
|
}
|
|
|
|
function parseShard(bytes, shard, source) {
|
|
if (sha256(bytes) !== shard.sha256.toLowerCase()) {
|
|
throw new Error("Cached Archive Spell shard SHA-256 mismatch: " + shard.file);
|
|
}
|
|
let text;
|
|
try {
|
|
text = gunzipSync(bytes).toString("utf8");
|
|
} catch (error) {
|
|
throw new Error("Archive Spell shard is not valid gzip data: " + shard.file, { cause: error });
|
|
}
|
|
const rows = [];
|
|
const ids = new Set();
|
|
for (const [lineIndex, line] of text.split("\n").entries()) {
|
|
if (line.length === 0) continue;
|
|
const row = parseJson(Buffer.from(line, "utf8"), shard.file + ":" + (lineIndex + 1));
|
|
if (!row || typeof row !== "object" || Array.isArray(row)) throw new Error("Archive Spell row is not an object in " + shard.file);
|
|
const id = Number(row.f0);
|
|
if (!Number.isSafeInteger(id) || id < shard.lo || id > shard.hi || ids.has(id)) {
|
|
throw new Error("Archive Spell row has an invalid or duplicate f0 in " + shard.file);
|
|
}
|
|
for (let field = 0; field < source.columns; field += 1) {
|
|
if (!Object.hasOwn(row, "f" + field)) throw new Error("Archive Spell row is missing f" + field + " in " + shard.file);
|
|
}
|
|
ids.add(id);
|
|
rows.push(row);
|
|
}
|
|
if (rows.length !== shard.rows) throw new Error("Archive Spell shard row count mismatch: " + shard.file);
|
|
rows.sort((a, b) => Number(a.f0) - Number(b.f0));
|
|
return rows;
|
|
}
|
|
|
|
function positiveSpellIds(input) {
|
|
if (!Array.isArray(input)) throw new TypeError("spellIds must be an array");
|
|
const ids = input.map((value) => Number(value));
|
|
if (ids.some((id) => !Number.isSafeInteger(id) || id <= 0)) {
|
|
throw new TypeError("spellIds must contain positive safe integers");
|
|
}
|
|
return [...new Set(ids)].sort((a, b) => a - b);
|
|
}
|
|
|
|
function childrenFromRow(row) {
|
|
const children = [];
|
|
for (let effect = 0; effect < 3; effect += 1) {
|
|
const triggerId = Number(row["f" + (116 + effect)]);
|
|
if (Number.isSafeInteger(triggerId) && triggerId > 0) children.push(triggerId);
|
|
if (Number(row["f" + (71 + effect)]) === 168) {
|
|
const itemSpellId = Number(row["f" + (110 + effect)]);
|
|
if (Number.isSafeInteger(itemSpellId) && itemSpellId > 0) children.push(itemSpellId);
|
|
}
|
|
}
|
|
return children;
|
|
}
|
|
|
|
function descriptorForId(index, id) {
|
|
return index.shards.find((shard) => id >= shard.lo && id <= shard.hi) ?? null;
|
|
}
|
|
|
|
function validateManifest(manifest, source, index) {
|
|
if (!manifest || typeof manifest !== "object" || Array.isArray(manifest)) throw new Error("Archive Spell cache manifest is not an object");
|
|
if (manifest.schemaVersion !== cacheSchemaVersion) throw new Error("Unsupported Archive Spell cache schema");
|
|
if (JSON.stringify(manifest.source) !== JSON.stringify(sourceRecord(source))) {
|
|
throw new Error("Archive Spell cache manifest points at an unexpected source or patch variant");
|
|
}
|
|
if (!Array.isArray(manifest.requestedIds) || !Array.isArray(manifest.files)) {
|
|
throw new Error("Archive Spell cache manifest is missing requestedIds or files");
|
|
}
|
|
const requestedIds = positiveSpellIds(manifest.requestedIds);
|
|
if (requestedIds.length !== manifest.requestedIds.length
|
|
|| requestedIds.some((id, position) => id !== manifest.requestedIds[position])) {
|
|
throw new Error("Archive Spell cache manifest requested ids must be sorted and unique");
|
|
}
|
|
const byFile = new Map(index.shards.map((shard) => [shard.file, shard]));
|
|
const names = new Set();
|
|
for (const file of manifest.files) {
|
|
const name = assertSafeRelativePath(file?.path, "Archive Spell manifest shard path");
|
|
const shard = byFile.get(name);
|
|
if (!shard || names.has(name) || file.sha256 !== shard.sha256
|
|
|| file.archivePath !== source.variantPath + "/" + name
|
|
|| file.lo !== shard.lo || file.hi !== shard.hi || file.rows !== shard.rows
|
|
|| !/^[0-9a-f]{40}$/i.test(file.gitBlobSha ?? "")
|
|
|| !Number.isSafeInteger(file.size) || file.size < 0) {
|
|
throw new Error("Archive Spell cache manifest contains an unknown or inconsistent shard: " + name);
|
|
}
|
|
names.add(name);
|
|
}
|
|
return { requestedIds, files: manifest.files };
|
|
}
|
|
|
|
export function createArchiveSpellCacheApi(sourceInput = ARCHIVE_SPELL_SOURCE) {
|
|
const source = validateSource({ ...sourceInput });
|
|
|
|
function archiveDirectory(cacheRoot) {
|
|
if (typeof cacheRoot !== "string" || cacheRoot.length === 0) throw new TypeError("cacheRoot must be a non-empty directory path");
|
|
return path.join(path.resolve(cacheRoot), "archive-" + source.shortRevision, "spell-base");
|
|
}
|
|
|
|
function rawUrl(filePath) {
|
|
const encodedPath = filePath.split("/").map(encodeURIComponent).join("/");
|
|
return "https://raw.githubusercontent.com/" + source.repository + "/" + source.revision + "/" + encodedPath;
|
|
}
|
|
|
|
async function responseBytes(fetchImpl, url) {
|
|
let response;
|
|
try {
|
|
response = await fetchImpl(url, { headers: { Accept: "application/json, application/gzip" } });
|
|
} catch (error) {
|
|
throw new Error("Archive Spell source unavailable at " + url + ": " + error.message, { cause: error });
|
|
}
|
|
if (!response || response.ok !== true) {
|
|
const status = response?.status === undefined ? "no response" : "HTTP " + response.status;
|
|
throw new Error("Archive Spell source unavailable at " + url + " (" + status + ")");
|
|
}
|
|
if (typeof response.arrayBuffer === "function") return Buffer.from(await response.arrayBuffer());
|
|
if (typeof response.text === "function") return Buffer.from(await response.text(), "utf8");
|
|
throw new Error("Archive Spell source response has no readable body at " + url);
|
|
}
|
|
|
|
async function writeAtomic(filePath, bytes) {
|
|
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
const tempPath = filePath + ".tmp-" + process.pid;
|
|
await fs.writeFile(tempPath, bytes);
|
|
await fs.rename(tempPath, filePath);
|
|
}
|
|
|
|
async function cachedPinnedBytes(filePath, shaPin, blobPin) {
|
|
try {
|
|
const bytes = await fs.readFile(filePath);
|
|
assertPinnedBytes(bytes, shaPin, blobPin, path.basename(filePath));
|
|
return bytes;
|
|
} catch (error) {
|
|
if (error?.code === "ENOENT") return null;
|
|
if (error?.code && error.code !== "ENOENT") throw error;
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function ensureMetadata(cacheRoot, fetchImpl) {
|
|
const root = archiveDirectory(cacheRoot);
|
|
const indexCachePath = path.join(root, "Spell.index.json");
|
|
const colinfoCachePath = path.join(root, "Spell.colinfo.json");
|
|
let [indexBytes, colinfoBytes] = await Promise.all([
|
|
cachedPinnedBytes(indexCachePath, source.tableIndexSha256, source.tableIndexGitBlobSha),
|
|
cachedPinnedBytes(colinfoCachePath, source.columnInfoSha256, source.columnInfoGitBlobSha),
|
|
]);
|
|
if (!indexBytes) {
|
|
indexBytes = await cachedPinnedBytes(
|
|
path.join(path.dirname(root), "remote-audit", "raw_tables_Spell_variants_data-patch-t-mpq_index.json"),
|
|
source.tableIndexSha256,
|
|
source.tableIndexGitBlobSha,
|
|
);
|
|
if (indexBytes) await writeAtomic(indexCachePath, indexBytes);
|
|
}
|
|
if (!colinfoBytes) {
|
|
colinfoBytes = await cachedPinnedBytes(
|
|
path.join(path.dirname(root), "remote-audit", "base_Spell.colinfo.json"),
|
|
source.columnInfoSha256,
|
|
source.columnInfoGitBlobSha,
|
|
);
|
|
if (colinfoBytes) await writeAtomic(colinfoCachePath, colinfoBytes);
|
|
}
|
|
if (!indexBytes) {
|
|
indexBytes = await responseBytes(fetchImpl, rawUrl(source.tableIndexPath));
|
|
assertPinnedBytes(indexBytes, source.tableIndexSha256, source.tableIndexGitBlobSha, "BASE Spell index");
|
|
await writeAtomic(indexCachePath, indexBytes);
|
|
}
|
|
if (!colinfoBytes) {
|
|
colinfoBytes = await responseBytes(fetchImpl, rawUrl(source.columnInfoPath));
|
|
assertPinnedBytes(colinfoBytes, source.columnInfoSha256, source.columnInfoGitBlobSha, "BASE Spell.colinfo.json");
|
|
await writeAtomic(colinfoCachePath, colinfoBytes);
|
|
}
|
|
const index = validateIndex(parseJson(indexBytes, source.tableIndexPath), source);
|
|
const columnInfo = parseJson(colinfoBytes, source.columnInfoPath);
|
|
const fieldTypes = fieldTypesFromColumnInfo(columnInfo, source);
|
|
return { root, index, fieldTypes };
|
|
}
|
|
|
|
async function readShard(root, shard, sourceInputForRows) {
|
|
const shardPath = path.join(root, "shards", shard.file);
|
|
const bytes = await fs.readFile(shardPath);
|
|
return { bytes, rows: parseShard(bytes, shard, sourceInputForRows) };
|
|
}
|
|
|
|
async function loadManifest(root, index) {
|
|
const manifestPath = path.join(root, manifestName);
|
|
let bytes;
|
|
try {
|
|
bytes = await fs.readFile(manifestPath);
|
|
} catch (error) {
|
|
if (error?.code === "ENOENT") return null;
|
|
throw error;
|
|
}
|
|
return validateManifest(parseJson(bytes, manifestPath), source, index);
|
|
}
|
|
|
|
async function closeSpellIds({ index, requestedIds, root, fetchImpl, allowFetch }) {
|
|
const records = new Map();
|
|
const missing = new Set();
|
|
const selectedShards = new Map();
|
|
const loadedRows = new Map();
|
|
const pending = [...requestedIds];
|
|
const visited = new Set();
|
|
for (let cursor = 0; cursor < pending.length; cursor += 1) {
|
|
const id = pending[cursor];
|
|
if (visited.has(id)) continue;
|
|
visited.add(id);
|
|
const descriptor = descriptorForId(index, id);
|
|
if (!descriptor) {
|
|
missing.add(id);
|
|
continue;
|
|
}
|
|
selectedShards.set(descriptor.file, descriptor);
|
|
let rows = loadedRows.get(descriptor.file);
|
|
if (!rows) {
|
|
const localPath = path.join(root, "shards", descriptor.file);
|
|
let bytes = null;
|
|
let localBytesFound = false;
|
|
const candidatePaths = allowFetch
|
|
? [localPath, path.join(path.dirname(root), "remote-audit", "base-spell-shards", descriptor.file)]
|
|
: [localPath];
|
|
for (const candidatePath of candidatePaths) {
|
|
try {
|
|
const candidate = await fs.readFile(candidatePath);
|
|
if (candidatePath === localPath) localBytesFound = true;
|
|
if (sha256(candidate) !== descriptor.sha256.toLowerCase()) {
|
|
if (!allowFetch && candidatePath === localPath) {
|
|
throw new Error("Cached Archive Spell shard SHA-256 mismatch: " + descriptor.file);
|
|
}
|
|
continue;
|
|
}
|
|
bytes = candidate;
|
|
if (candidatePath !== localPath) await writeAtomic(localPath, bytes);
|
|
break;
|
|
} catch (error) {
|
|
if (error?.code !== "ENOENT") throw error;
|
|
}
|
|
}
|
|
if (!bytes && allowFetch) {
|
|
bytes = await responseBytes(fetchImpl, rawUrl(source.variantPath + "/" + descriptor.file));
|
|
if (sha256(bytes) !== descriptor.sha256.toLowerCase()) {
|
|
throw new Error("BASE Archive Spell shard SHA-256 mismatch: " + descriptor.file);
|
|
}
|
|
await writeAtomic(localPath, bytes);
|
|
}
|
|
if (!bytes) {
|
|
const detail = localBytesFound ? " has invalid cached bytes" : " is missing";
|
|
throw new Error("Archive Spell cache shard " + descriptor.file + detail + "; run downloadArchiveSpells first");
|
|
}
|
|
rows = parseShard(bytes, descriptor, source);
|
|
loadedRows.set(descriptor.file, rows);
|
|
}
|
|
const row = rows.find((candidate) => Number(candidate.f0) === id);
|
|
if (!row) {
|
|
missing.add(id);
|
|
continue;
|
|
}
|
|
records.set(id, row);
|
|
for (const child of childrenFromRow(row)) if (!visited.has(child)) pending.push(child);
|
|
}
|
|
return {
|
|
records: [...records.values()].sort((a, b) => Number(a.f0) - Number(b.f0)),
|
|
missingIds: [...missing].sort((a, b) => a - b),
|
|
selectedShards,
|
|
};
|
|
}
|
|
|
|
function manifestFile(shard, bytes, source) {
|
|
return {
|
|
path: shard.file,
|
|
archivePath: source.variantPath + "/" + shard.file,
|
|
lo: shard.lo,
|
|
hi: shard.hi,
|
|
rows: shard.rows,
|
|
sha256: shard.sha256,
|
|
gitBlobSha: gitBlobSha(bytes),
|
|
size: bytes.byteLength,
|
|
};
|
|
}
|
|
|
|
async function readArchiveSpells({ cacheRoot }) {
|
|
const root = archiveDirectory(cacheRoot);
|
|
let indexBytes;
|
|
let colinfoBytes;
|
|
try {
|
|
[indexBytes, colinfoBytes] = await Promise.all([
|
|
fs.readFile(path.join(root, "Spell.index.json")),
|
|
fs.readFile(path.join(root, "Spell.colinfo.json")),
|
|
]);
|
|
} catch (error) {
|
|
if (error?.code === "ENOENT") throw new Error("Archive Spell cache metadata is missing; run downloadArchiveSpells first");
|
|
throw error;
|
|
}
|
|
assertPinnedBytes(indexBytes, source.tableIndexSha256, source.tableIndexGitBlobSha, "Cached BASE Spell index");
|
|
assertPinnedBytes(colinfoBytes, source.columnInfoSha256, source.columnInfoGitBlobSha, "Cached BASE Spell.colinfo.json");
|
|
const index = validateIndex(parseJson(indexBytes, source.tableIndexPath), source);
|
|
const fieldTypes = fieldTypesFromColumnInfo(parseJson(colinfoBytes, source.columnInfoPath), source);
|
|
const manifest = await loadManifest(root, index);
|
|
if (!manifest) throw new Error("Archive Spell cache manifest is missing; run downloadArchiveSpells first");
|
|
const closure = await closeSpellIds({ index, requestedIds: manifest.requestedIds, root, allowFetch: false });
|
|
const declared = new Map(manifest.files.map((file) => [file.path, file]));
|
|
if (declared.size !== closure.selectedShards.size
|
|
|| [...closure.selectedShards.keys()].some((name) => !declared.has(name))) {
|
|
throw new Error("Archive Spell cache manifest does not cover the complete selected spell closure");
|
|
}
|
|
const files = [];
|
|
for (const shard of [...closure.selectedShards.values()].sort((a, b) => a.lo - b.lo)) {
|
|
const { bytes } = await readShard(root, shard, source);
|
|
const file = manifestFile(shard, bytes, source);
|
|
const expected = declared.get(shard.file);
|
|
if (file.gitBlobSha !== expected.gitBlobSha || file.size !== expected.size) {
|
|
throw new Error("Archive Spell cached shard does not match its manifest: " + shard.file);
|
|
}
|
|
files.push(file);
|
|
}
|
|
return {
|
|
source: resultSource(source, files),
|
|
fieldTypes,
|
|
records: closure.records,
|
|
files,
|
|
missingIds: closure.missingIds,
|
|
};
|
|
}
|
|
|
|
async function downloadArchiveSpells({ cacheRoot, spellIds, fetchImpl = globalThis.fetch }) {
|
|
if (typeof fetchImpl !== "function") throw new TypeError("fetchImpl must be a function");
|
|
const newIds = positiveSpellIds(spellIds);
|
|
const { root, index, fieldTypes } = await ensureMetadata(cacheRoot, fetchImpl);
|
|
const previous = await loadManifest(root, index);
|
|
const requestedIds = positiveSpellIds([...(previous?.requestedIds ?? []), ...newIds]);
|
|
const closure = await closeSpellIds({ index, requestedIds, root, fetchImpl, allowFetch: true });
|
|
const files = [];
|
|
for (const shard of [...closure.selectedShards.values()].sort((a, b) => a.lo - b.lo)) {
|
|
const { bytes } = await readShard(root, shard, source);
|
|
files.push(manifestFile(shard, bytes, source));
|
|
}
|
|
const manifest = {
|
|
schemaVersion: cacheSchemaVersion,
|
|
source: sourceRecord(source),
|
|
requestedIds,
|
|
files,
|
|
};
|
|
await writeAtomic(path.join(root, manifestName), Buffer.from(JSON.stringify(manifest, null, 2) + "\n", "utf8"));
|
|
return {
|
|
source: resultSource(source, files),
|
|
fieldTypes,
|
|
records: closure.records,
|
|
files,
|
|
missingIds: closure.missingIds,
|
|
};
|
|
}
|
|
|
|
return Object.freeze({ downloadArchiveSpells, readArchiveSpells });
|
|
}
|
|
|
|
const defaultArchiveSpellApi = createArchiveSpellCacheApi();
|
|
|
|
export function downloadArchiveSpells({ cacheRoot, spellIds, fetchImpl = globalThis.fetch }) {
|
|
return defaultArchiveSpellApi.downloadArchiveSpells({ cacheRoot, spellIds, fetchImpl });
|
|
}
|
|
|
|
export function readArchiveSpells({ cacheRoot }) {
|
|
return defaultArchiveSpellApi.readArchiveSpells({ cacheRoot });
|
|
}
|