"""Shared reverse-engineering lookups for the agents, backed by the project's own image and annotations. Configured per project in a [reverse] table: image = 'R:/.../image.bin' flat image of the executable, the first byte at `base` base = 0x401000 facts = 'R:/.../facts.jsonl' one JSON object per function: address, size, name, calls, data, ... imports = 'R:/.../imports.json' optional: [{module, name, slot}] annotations = 'R:/.../annotations' names.csv, functions.csv, globals.csv, prototypes.h src = 'R:/.../src' optional: the decompiled C files (_.c) Every answer comes from those files, read fresh when they change, so the agents see the team's current names. Nothing here writes anything.""" import csv import json import os import re import struct from pathlib import Path from .util import HarborError MAX_BYTES = 0x4000 def _int(value): if isinstance(value, int): return value text = str(value).strip() return int(text, 16) if text.lower().startswith("0x") or re.fullmatch(r"[0-9A-Fa-f]{6,8}", text) else int(text) class Reverse: def __init__(self, config): if not config or not config.get("image"): raise HarborError("this project has no [reverse] table (image, base, facts, annotations)") self.image = Path(config["image"]) self.base = _int(config.get("base", 0x401000)) self.facts_path = Path(config["facts"]) if config.get("facts") else None self.imports_path = Path(config["imports"]) if config.get("imports") else None self.annotations = Path(config["annotations"]) if config.get("annotations") else None self.src = Path(config["src"]) if config.get("src") else None self.conventions_path = Path(config["conventions"]) if config.get("conventions") else None self.source_files = Path(config["source_files"]) if config.get("source_files") else None self.mac_names = Path(config["mac_names"]) if config.get("mac_names") else None self._cache = {} # ----------------------------------------------------------------- loading (cached by file mtime) def _load(self, key, path, loader): if path is None or not path.exists(): return loader(None) stamp = path.stat().st_mtime hit = self._cache.get(key) if hit and hit[0] == stamp: return hit[1] value = loader(path) self._cache[key] = (stamp, value) return value def _bytes(self): return self._load("image", self.image, lambda p: p.read_bytes() if p else b"") def _facts(self): def load(p): out = {} if p: for line in p.read_text(encoding="utf-8").splitlines(): if line.strip(): d = json.loads(line) out[_int(d["address"])] = d return out return self._load("facts", self.facts_path, load) def _callers(self): facts = self._facts() key = ("callers", id(facts)) if key not in self._cache: callers = {} for addr, d in facts.items(): for c in d.get("calls", []): callers.setdefault(_int(c), []).append(addr) self._cache[key] = callers return self._cache[key] def _imports(self): return self._load("imports", self.imports_path, lambda p: { int(d["slot"]): f"{d['module']}!{d['name']}" for d in json.loads(p.read_text(encoding="utf-8"))} if p else {}) def _csv(self, name): path = self.annotations / name if self.annotations else None def load(p): rows = {} if p: for row in csv.reader(p.read_text(encoding="utf-8").splitlines()): if row and row[0].lower().startswith("0x"): rows[_int(row[0])] = row return rows return self._load(name, path, load) def _prototypes(self): path = self.annotations / "prototypes.h" if self.annotations else None def load(p): out = {} if p: lines = p.read_text(encoding="utf-8").split("\n") i = 0 while i < len(lines): line = lines[i] m = re.match(r"^[A-Za-z_][^(;]*?\b([A-Za-z_]\w*)\s*\(", line) if not line.startswith( ("typedef", "#", "/", " ")) else None j = i + 1 while m and j < len(lines) and lines[j].startswith(" ") and not re.match(r"^[A-Za-z_]", lines[j]): j += 1 if m: out[m.group(1)] = "\n".join(lines[i:j]) i = j if m else i + 1 return out return self._load("prototypes", path, load) # ----------------------------------------------------------------- names def function_at(self, addr): """The function containing addr: (start, size, name), from facts.jsonl, named from names.csv.""" facts = self._facts() if addr in facts: start = addr else: starts = [a for a in facts if a <= addr < a + int(facts[a].get("size") or 0)] if not starts: return None start = max(starts) d = facts[start] return start, int(d.get("size") or 0), self.name_of(start) or d.get("name") def name_of(self, addr): row = self._csv("names.csv").get(addr) if row and len(row) > 1 and row[1]: return row[1] g = self._csv("globals.csv").get(addr) if g and len(g) > 1 and g[1]: return g[1] imp = self._imports().get(addr) if imp: return imp d = self._facts().get(addr) return d.get("name") if d else None def address_of(self, name): for table in ("names.csv", "globals.csv"): for addr, row in self._csv(table).items(): if len(row) > 1 and row[1] == name: return addr for addr, d in self._facts().items(): if d.get("name") == name: return addr return None def resolve(self, target): """'0x4AD850', '4AD850', or a name -> address.""" text = str(target).strip() if re.fullmatch(r"(0x)?[0-9A-Fa-f]{5,8}", text): return int(text, 16) addr = self.address_of(text) if addr is None: addr = self._mac_index().get(text) or self._mac_index().get(text.replace("::", "_")) if addr is None: raise HarborError(f"{target!r} is neither an address nor a known name") return addr def _mac_index(self): """Original names from the Mac build -> PC address: 'CanCarry', 'TradeMenu::DoTrade', 'TradeMenu_DoTrade'.""" def load(p): out = {} if p: for row in csv.DictReader(p.read_text(encoding="utf-8").splitlines()): head = (row.get("demangled") or "").split("(", 1)[0].strip() try: addr = int(row["pc_address"], 16) except (KeyError, ValueError): continue if head: out.setdefault(head, addr) out.setdefault(head.replace("::", "_"), addr) return out return self._load("mac_index", self.mac_names, load) def convention(self, addr): """The calling convention read from the bytes: register inputs (a dataflow pass) and the `ret N` cleanup.""" if not self.conventions_path or not self.conventions_path.exists(): return None module = self._cache.get("conventions_module") if module is None: import importlib.util spec = importlib.util.spec_from_file_location("harbor_conventions", self.conventions_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) self._cache["conventions_module"] = module func = self.function_at(addr) if not func: return None result = module.analyse(self._bytes(), func[0], func[1]) if not result: return None return {"summary": module.describe(result), "registers_in": result["registers_in"], "ret_pops": result["ret_pops"]} def _keyed_csv(self, key, path, column): def load(p): out = {} if p: for row in csv.DictReader(p.read_text(encoding="utf-8").splitlines()): try: out.setdefault(int(row[column], 16), row) except (KeyError, ValueError): pass return out return self._load(key, path, load) def origins(self, addr): """What the other builds say about a function: its original source file (Xbox file/line sites) and its original name with parameter types (the unstripped Mac build). Evidence, not truth.""" out = {} src = self._keyed_csv("source_files", self.source_files, "pc_address").get(addr) if src: out["source_file"] = {k: src.get(k) for k in ("source_file", "lines", "confidence", "xbox_address", "evidence") if src.get(k)} mac = self._keyed_csv("mac_names", self.mac_names, "pc_address").get(addr) if mac: out["mac_name"] = {k: mac.get(k) for k in ("demangled", "mac_symbol", "confidence", "evidence") if mac.get(k)} return out # ----------------------------------------------------------------- tools def read(self, addr, count): data = self._bytes() offset = addr - self.base if offset < 0 or offset >= len(data): raise HarborError(f"0x{addr:X} is outside the image (0x{self.base:X}..0x{self.base + len(data):X})") return data[offset:offset + count] def disassemble(self, target, max_bytes=None): """The whole function (or max_bytes from an address) with call targets, jumps into other functions, import slots and known globals named.""" try: import capstone except ImportError: raise HarborError("capstone is not installed for this Python (py -3.12 -m pip install capstone)") from None addr = self.resolve(target) func = self.function_at(addr) if max_bytes: start, size = addr, min(int(max_bytes), MAX_BYTES) elif func: start, size = func[0], func[1] else: start, size = addr, 0x100 if size > MAX_BYTES: raise HarborError(f"the function is {size} bytes; pass max_bytes (at most {MAX_BYTES}) and walk it in parts") code = self.read(start, size) md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32) lines = [] if func and not max_bytes: lines.append(f"; {func[2] or 'unnamed'} 0x{func[0]:X}..0x{func[0] + func[1]:X} ({func[1]} bytes)") conv = self.convention(func[0]) if conv: lines.append(f"; convention from the bytes: {conv['summary']}; registers in: " f"{', '.join(conv['registers_in']) or 'none'}; ret pops {conv['ret_pops']}") for ins in md.disasm(code, start): note = self._annotate(ins.op_str, ins.mnemonic, start, start + size) lines.append(f"{ins.address:08X} {ins.mnemonic:7} {ins.op_str}{(' ; ' + note) if note else ''}") return "\n".join(lines) def _annotate(self, op_str, mnemonic, lo, hi): notes = [] constant = re.search(r"(dword|qword) ptr \[0x([0-9a-f]{6,8})\]", op_str) if mnemonic.startswith("f") else None if constant: # an x87 constant: show its value as the FPU reads it size = 4 if constant.group(1) == "dword" else 8 try: raw = self.read(int(constant.group(2), 16), size) value = struct.unpack(" 50: raise HarborError("give 1 to 50 addresses") with tempfile.TemporaryDirectory(dir=config.get("debugger_temp")) as tmp: out = Path(tmp) / "match.json" command = [sys.executable, "-B", str(matcher), "--all", "--only", ",".join(wanted), "--json", str(out), "--flags", flags or config.get("match_flags", "-TC -O2")] if decomp: command += ["--decomp", str(decomp)] result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=600, creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0)) if not out.exists(): raise HarborError("the matcher failed: " + (result.stdout + result.stderr).strip()[-800:]) rows = json.loads(out.read_text(encoding="utf-8")) errors = [line for line in result.stdout.splitlines() if "compile errors" in line] return {"results": rows, "summary": result.stdout.strip().splitlines()[0] if result.stdout.strip() else "", **({"compile": errors[0]} if errors else {})} def run_toolkit(config, key, argv, json_to_file=False, timeout=900): """Run the toolkit script configured as [reverse] with argv and return its JSON (stdout, or a temp file passed as `--json ` when json_to_file).""" import subprocess import sys import tempfile script = (config or {}).get(key) if not script or not Path(script).exists(): raise HarborError(f"this project has no {key} script in its [reverse] table") with tempfile.TemporaryDirectory(dir=(config or {}).get("debugger_temp")) as tmp: out = Path(tmp) / "out.json" command = [sys.executable, "-B", str(script), *argv] + (["--json", str(out)] if json_to_file else ["--json"]) result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, cwd=str(Path(script).parent), creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0)) text = out.read_text(encoding="utf-8") if json_to_file and out.exists() else result.stdout try: return json.loads(text) except ValueError: raise HarborError(f"{key} failed: " + (result.stdout + result.stderr).strip()[-800:]) from None def match_program(config, targets=None, decomp=None): """Build every readable function into one /GL + /LTCG image the way the game was built (stand-in callers, stubs for the rest, globals at their original addresses) and check the targets against the original bytes. Slow: a build takes 1.5 to 9 minutes depending on load.""" import subprocess import sys import tempfile config = config or {} script = config.get("ltcg") if not script or not Path(script).exists(): raise HarborError("this project has no ltcg script in its [reverse] table") wanted = [t.strip() for t in re.split(r"[,\s]+", str(targets or "")) if t.strip()] extra = ["--decomp", str(decomp)] if decomp else [] run = lambda argv, timeout: subprocess.run([sys.executable, "-B", str(script), *argv], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, cwd=str(Path(script).parent), creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0)) built = run(["build", *extra], 1800) if built.returncode != 0: raise HarborError("the whole-program build failed: " + (built.stdout + built.stderr).strip()[-800:]) with tempfile.TemporaryDirectory(dir=config.get("debugger_temp")) as tmp: out = Path(tmp) / "check.json" checked = run(["check", *extra, "--json", str(out)] + (["--only", ",".join(wanted)] if wanted else []), 900) if not out.exists(): raise HarborError("the whole-program check failed: " + (checked.stdout + checked.stderr).strip()[-800:]) rows = json.loads(out.read_text(encoding="utf-8")) counts = {} for r in rows: counts[r.get("status")] = counts.get(r.get("status"), 0) + 1 keep = ("address", "name", "status", "first_difference", "similarity", "registers_in", "ours_registers_in", "ret_pops", "ours_ret_pops", "relocation_problems", "note") return {"counts": counts, "results": [{k: r[k] for k in keep if k in r} for r in rows] if wanted else [{k: r[k] for k in keep if k in r} for r in rows if r.get("status") != "differs"][:200]}