"""Shared reverse-engineering lookups for the agents, backed by the project's own image and annotations.

Configured per project in a [reverse] table:
    image = 'R:/.../image.bin'        flat image of the executable, the first byte at `base`
    base = 0x401000
    facts = 'R:/.../facts.jsonl'      one JSON object per function: address, size, name, calls, data, ...
    imports = 'R:/.../imports.json'   optional: [{module, name, slot}]
    annotations = 'R:/.../annotations' names.csv, functions.csv, globals.csv, prototypes.h
    src = 'R:/.../src'                optional: the decompiled C files (<ADDR>_<Name>.c)
    sections = 'R:/.../sections.json' optional: [{name, start, end}] (default: sections.json beside the image),
                                      used to find the vtables and pointer tables a function sits in

Every answer comes from those files, read fresh when they change, so the agents see the team's current names.
Nothing here writes anything."""
import csv
import json
import os
import re
import struct
from pathlib import Path

from .util import HarborError

MAX_BYTES = 0x4000


def _int(value):
    if isinstance(value, int):
        return value
    text = str(value).strip()
    return int(text, 16) if text.lower().startswith("0x") or re.fullmatch(r"[0-9A-Fa-f]{6,8}", text) else int(text)


class Reverse:
    def __init__(self, config):
        if not config or not config.get("image"):
            raise HarborError("this project has no [reverse] table (image, base, facts, annotations)")
        self.image = Path(config["image"])
        self.base = _int(config.get("base", 0x401000))
        self.facts_path = Path(config["facts"]) if config.get("facts") else None
        self.imports_path = Path(config["imports"]) if config.get("imports") else None
        self.annotations = Path(config["annotations"]) if config.get("annotations") else None
        self.src = Path(config["src"]) if config.get("src") else None
        self.conventions_path = Path(config["conventions"]) if config.get("conventions") else None
        self.source_files = Path(config["source_files"]) if config.get("source_files") else None
        self.mac_names = Path(config["mac_names"]) if config.get("mac_names") else None
        self.sections_path = Path(config["sections"]) if config.get("sections") else None
        self._cache = {}

    # ----------------------------------------------------------------- loading (cached by file mtime)
    def _load(self, key, path, loader):
        if path is None or not path.exists():
            return loader(None)
        stamp = path.stat().st_mtime
        hit = self._cache.get(key)
        if hit and hit[0] == stamp:
            return hit[1]
        value = loader(path)
        self._cache[key] = (stamp, value)
        return value

    def _bytes(self):
        return self._load("image", self.image, lambda p: p.read_bytes() if p else b"")

    def _facts(self):
        def load(p):
            out = {}
            if p:
                for line in p.read_text(encoding="utf-8").splitlines():
                    if line.strip():
                        d = json.loads(line)
                        out[_int(d["address"])] = d
            return out
        return self._load("facts", self.facts_path, load)

    def _callers(self):
        facts = self._facts()
        key = ("callers", id(facts))
        if key not in self._cache:
            callers = {}
            for addr, d in facts.items():
                for c in d.get("calls", []):
                    callers.setdefault(_int(c), []).append(addr)
            self._cache[key] = callers
        return self._cache[key]

    def _imports(self):
        return self._load("imports", self.imports_path, lambda p: {
            int(d["slot"]): f"{d['module']}!{d['name']}" for d in json.loads(p.read_text(encoding="utf-8"))} if p else {})

    def _csv(self, name):
        path = self.annotations / name if self.annotations else None

        def load(p):
            rows = {}
            if p:
                for row in csv.reader(p.read_text(encoding="utf-8").splitlines()):
                    if row and row[0].lower().startswith("0x"):
                        rows[_int(row[0])] = row
            return rows
        return self._load(name, path, load)

    def _prototypes(self):
        path = self.annotations / "prototypes.h" if self.annotations else None

        def load(p):
            out = {}
            if p:
                lines = p.read_text(encoding="utf-8").split("\n")
                i = 0
                while i < len(lines):
                    line = lines[i]
                    m = re.match(r"^[A-Za-z_][^(;]*?\b([A-Za-z_]\w*)\s*\(", line) if not line.startswith(
                        ("typedef", "#", "/", " ")) else None
                    j = i + 1
                    while m and j < len(lines) and lines[j].startswith(" ") and not re.match(r"^[A-Za-z_]", lines[j]):
                        j += 1
                    if m:
                        out[m.group(1)] = "\n".join(lines[i:j])
                    i = j if m else i + 1
            return out
        return self._load("prototypes", path, load)

    # ----------------------------------------------------------------- names
    def function_at(self, addr):
        """The function containing addr: (start, size, name), from facts.jsonl, named from names.csv."""
        facts = self._facts()
        if addr in facts:
            start = addr
        else:
            starts = [a for a in facts if a <= addr < a + int(facts[a].get("size") or 0)]
            if not starts:
                return None
            start = max(starts)
        d = facts[start]
        return start, int(d.get("size") or 0), self.name_of(start) or d.get("name")

    def name_of(self, addr):
        row = self._csv("names.csv").get(addr)
        if row and len(row) > 1 and row[1]:
            return row[1]
        g = self._csv("globals.csv").get(addr)
        if g and len(g) > 1 and g[1]:
            return g[1]
        imp = self._imports().get(addr)
        if imp:
            return imp
        d = self._facts().get(addr)
        return d.get("name") if d else None

    def address_of(self, name):
        for table in ("names.csv", "globals.csv"):
            for addr, row in self._csv(table).items():
                if len(row) > 1 and row[1] == name:
                    return addr
        for addr, d in self._facts().items():
            if d.get("name") == name:
                return addr
        return None

    def resolve(self, target):
        """'0x4AD850', '4AD850', or a name -> address."""
        text = str(target).strip()
        if re.fullmatch(r"(0x)?[0-9A-Fa-f]{5,8}", text):
            return int(text, 16)
        addr = self.address_of(text)
        if addr is None:
            addr = self._mac_index().get(text) or self._mac_index().get(text.replace("::", "_"))
        if addr is None:
            raise HarborError(f"{target!r} is neither an address nor a known name")
        return addr

    def _mac_index(self):
        """Original names from the Mac build -> PC address: 'CanCarry', 'TradeMenu::DoTrade', 'TradeMenu_DoTrade'."""
        def load(p):
            out = {}
            if p:
                for row in csv.DictReader(p.read_text(encoding="utf-8").splitlines()):
                    head = (row.get("demangled") or "").split("(", 1)[0].strip()
                    try:
                        addr = int(row["pc_address"], 16)
                    except (KeyError, ValueError):
                        continue
                    if head:
                        out.setdefault(head, addr)
                        out.setdefault(head.replace("::", "_"), addr)
            return out
        return self._load("mac_index", self.mac_names, load)

    def convention(self, addr):
        """The calling convention read from the bytes: register inputs (a dataflow pass) and the `ret N` cleanup."""
        if not self.conventions_path or not self.conventions_path.exists():
            return None
        module = self._cache.get("conventions_module")
        if module is None:
            import importlib.util
            spec = importlib.util.spec_from_file_location("harbor_conventions", self.conventions_path)
            module = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(module)
            self._cache["conventions_module"] = module
        func = self.function_at(addr)
        if not func:
            return None
        result = module.analyse(self._bytes(), func[0], func[1])
        if not result:
            return None
        return {"summary": module.describe(result), "registers_in": result["registers_in"],
                "ret_pops": result["ret_pops"]}

    def _keyed_csv(self, key, path, column):
        def load(p):
            out = {}
            if p:
                for row in csv.DictReader(p.read_text(encoding="utf-8").splitlines()):
                    try:
                        out.setdefault(int(row[column], 16), row)
                    except (KeyError, ValueError):
                        pass
            return out
        return self._load(key, path, load)

    def origins(self, addr):
        """What the other builds say about a function: its original source file (Xbox file/line sites) and its
        original name with parameter types (the unstripped Mac build). Evidence, not truth."""
        out = {}
        src = self._keyed_csv("source_files", self.source_files, "pc_address").get(addr)
        if src:
            out["source_file"] = {k: src.get(k) for k in ("source_file", "lines", "confidence", "xbox_address",
                                                         "evidence") if src.get(k)}
        mac = self._keyed_csv("mac_names", self.mac_names, "pc_address").get(addr)
        if mac:
            out["mac_name"] = {k: mac.get(k) for k in ("demangled", "mac_symbol", "confidence", "evidence")
                               if mac.get(k)}
        return out

    # ----------------------------------------------------------------- tools
    def read(self, addr, count):
        data = self._bytes()
        offset = addr - self.base
        if offset < 0 or offset >= len(data):
            raise HarborError(f"0x{addr:X} is outside the image (0x{self.base:X}..0x{self.base + len(data):X})")
        return data[offset:offset + count]

    def disassemble(self, target, max_bytes=None):
        """The whole function (or max_bytes from an address) with call targets, jumps into other functions,
        import slots and known globals named."""
        try:
            import capstone
        except ImportError:
            raise HarborError("capstone is not installed for this Python (py -3.12 -m pip install capstone)") from None
        addr = self.resolve(target)
        func = self.function_at(addr)
        if max_bytes:
            start, size = addr, min(int(max_bytes), MAX_BYTES)
        elif func:
            start, size = func[0], func[1]
        else:
            start, size = addr, 0x100
        if size > MAX_BYTES:
            raise HarborError(f"the function is {size} bytes; pass max_bytes (at most {MAX_BYTES}) and walk it in parts")
        code = self.read(start, size)
        md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_32)
        md.detail = True
        lines = []
        if func and not max_bytes:
            lines.append(f"; {func[2] or 'unnamed'} 0x{func[0]:X}..0x{func[0] + func[1]:X} ({func[1]} bytes)")
            conv = self.convention(func[0])
            if conv:
                lines.append(f"; convention from the bytes: {conv['summary']}; registers in: "
                             f"{', '.join(conv['registers_in']) or 'none'}; ret pops {conv['ret_pops']}")
        frames = self._stack_depths(md, func) if func else {}
        if frames:
            lines.append("; stack notes: argN = the Nth stack argument (entry esp + 4N), local = below the return "
                         "address, ~ = after a call whose cleanup was inferred from the pushes")
        for ins in md.disasm(code, start):
            notes = [n for n in (self._frame_note(ins, frames.get(ins.address)),
                                 self._annotate(ins.op_str, ins.mnemonic, start, start + size)) if n]
            lines.append(f"{ins.address:08X}  {ins.mnemonic:7} {ins.op_str}{('    ; ' + ', '.join(notes)) if notes else ''}")
        return "\n".join(lines)

    def _callee_pops(self, target):
        """Bytes a directly called function pops on return (its `ret N`), or None when not known."""
        key = ("pops", target)
        if key not in self._cache:
            pops = None
            try:
                head = self.read(target, 5)
                if head == b"\x3d\x00\x10\x00\x00":           # cmp eax, 0x1000: the stack probe (_alloca_probe)
                    pops = "probe"
                else:
                    conv = self.convention(target) if self.function_at(target) and \
                        self.function_at(target)[0] == target else None
                    if conv and len(conv["ret_pops"]) == 1:
                        pops = conv["ret_pops"][0]
            except Exception:  # noqa: BLE001 - unknown cleanup is reported as inferred
                pops = None
            self._cache[key] = pops
        return self._cache[key]

    def _stack_depths(self, md, func):
        """{address: (bytes pushed since entry or None, ebp's depth or None, inferred)} over the function's control
        flow, so an [esp+N] or [ebp+N] operand can be named as the argument or local it is."""
        import capstone
        from capstone import x86
        start, size = func[0], func[1]
        if size > 0x8000:
            return {}
        insns = list(md.disasm(self.read(start, size), start))
        index = {ins.address: i for i, ins in enumerate(insns)}
        state = {start: (0, None, 0, False)}       # depth, ebp depth, bytes pushed since the last call, inferred
        work = [start]
        while work:
            addr = work.pop()
            i = index.get(addr)
            while i is not None and i < len(insns):
                ins = insns[i]
                d, e, p, guess = state[ins.address]
                ops, m, text = ins.operands, ins.mnemonic, ins.op_str
                imm = ops[1].imm if len(ops) == 2 and ops[1].type == x86.X86_OP_IMM else None
                if m == "push":
                    d, p = (d + 4 if d is not None else None), p + 4
                elif m == "pushal":
                    d = d + 32 if d is not None else None
                elif m == "pop":
                    d, p = (d - 4 if d is not None else None), max(0, p - 4)
                    if text == "ebp":
                        e = None
                elif m == "popal":
                    d = d - 32 if d is not None else None
                elif m == "sub" and text.startswith("esp, ") and imm is not None:
                    d = d + imm if d is not None else None
                elif m == "add" and text.startswith("esp, ") and imm is not None:
                    d, p = (d - imm if d is not None else None), max(0, p - imm)
                elif m == "and" and text.startswith("esp, "):
                    d = None                                   # aligned: esp no longer tracks the entry
                elif m == "mov" and text == "ebp, esp":
                    e = d
                elif m == "mov" and text == "esp, ebp":
                    d = e
                elif m == "leave":
                    d, e = (e - 4 if e is not None else None), None
                elif m == "lea" and text.startswith("esp, [ebp") and e is not None:
                    d = e - ops[1].mem.disp
                elif m == "call":
                    pops = None
                    if ops and ops[0].type == x86.X86_OP_IMM:
                        pops = self._callee_pops(ops[0].imm)
                    if pops == "probe":                        # mov eax, N / call _alloca_probe reserves N bytes
                        prev = insns[i - 1] if i else None
                        pv = prev.operands[1].imm if prev is not None and prev.mnemonic == "mov" and \
                            prev.op_str.startswith("eax, ") and prev.operands[1].type == x86.X86_OP_IMM else None
                        d = d + pv if d is not None and pv is not None else None
                    elif pops is not None:
                        d = d - pops if d is not None else None
                    else:
                        nxt = insns[i + 1] if i + 1 < len(insns) else None
                        cleaned = nxt is not None and nxt.mnemonic == "add" and nxt.op_str.startswith("esp, ")
                        if not cleaned and p:
                            d, guess = (d - p if d is not None else None), True   # the callee popped what was pushed
                    p = 0
                succ = []
                if ins.group(capstone.CS_GRP_RET):
                    pass
                elif m == "jmp":
                    if ops and ops[0].type == x86.X86_OP_IMM and ops[0].imm in index:
                        succ.append(ops[0].imm)
                else:
                    if ins.group(capstone.CS_GRP_JUMP) and ops and ops[0].type == x86.X86_OP_IMM \
                            and ops[0].imm in index:
                        succ.append(ops[0].imm)
                    if i + 1 < len(insns):
                        succ.append(insns[i + 1].address)
                fall = insns[i + 1].address if i + 1 < len(insns) else None
                i = None
                for s in succ:
                    if s in state:
                        continue
                    state[s] = (d, e, p, guess)
                    if s == fall:
                        i = index[s]
                    else:
                        work.append(s)
        return state

    @staticmethod
    def _frame_note(ins, frame):
        """'esp+0x10 = arg2' style notes for the stack operands of one instruction."""
        if not frame:
            return ""
        from capstone import x86
        d, e, _, guess = frame
        notes = []
        for op in ins.operands:
            if op.type != x86.X86_OP_MEM or op.mem.index:
                continue
            base = ins.reg_name(op.mem.base) if op.mem.base else ""
            if base == "esp" and d is not None:
                off = op.mem.disp - d
            elif base == "ebp" and e is not None:
                off = op.mem.disp - e
            else:
                continue
            if off == 0:
                what = "return address"
            elif off >= 4:
                k, r = divmod(off - 4, 4)
                what = f"arg{k + 1}" + (f"+{r}" if r else "")
            else:
                what = f"local entry-0x{-off:X}"
            notes.append(("~" if guess else "") + what)
        return ", ".join(notes)

    def _annotate(self, op_str, mnemonic, lo, hi):
        notes = []
        constant = re.search(r"(dword|qword) ptr \[0x([0-9a-f]{6,8})\]", op_str) if mnemonic.startswith("f") else None
        if constant:                                       # an x87 constant: show its value as the FPU reads it
            size = 4 if constant.group(1) == "dword" else 8
            try:
                raw = self.read(int(constant.group(2), 16), size)
                value = struct.unpack("<f" if size == 4 else "<d", raw)[0]
                notes.append(f"= {value!r}{'f' if size == 4 else ''}")
            except HarborError:
                pass
        for m in re.finditer(r"0x([0-9a-f]{6,8})", op_str):
            value = int(m.group(1), 16)
            if lo <= value < hi and not mnemonic.startswith("call"):
                continue                                   # a jump inside this function needs no name
            name = self.name_of(value)
            if name:
                notes.append(f"0x{value:X} {name}")
            elif mnemonic.startswith("call") or mnemonic.startswith("j"):
                func = self.function_at(value)
                if func and func[0] != value:
                    notes.append(f"0x{value:X} inside {func[2] or hex(func[0])}+0x{value - func[0]:X}")
        return ", ".join(notes)

    def xrefs(self, target):
        """Callers and callees of a function, and the data it touches, named."""
        addr = self.resolve(target)
        func = self.function_at(addr)
        if not func:
            raise HarborError(f"0x{addr:X} is not inside a known function")
        d = self._facts()[func[0]]
        named = lambda a: {"address": f"0x{a:X}", "name": self.name_of(a)}
        return {"function": named(func[0]) | {"size": func[1]},
                "callers": [named(a) for a in sorted(set(self._callers().get(func[0], [])))],
                "calls": [named(_int(a)) for a in d.get("calls", [])],
                "data": [named(_int(a)) for a in d.get("data", [])],
                "imports": d.get("imports", []), "strings": d.get("strings", [])}

    def bytes_at(self, target, count=16, as_type="hex"):
        """Raw bytes at an address, also read as hex, int32, uint32, float, double or a C string."""
        addr = self.resolve(target)
        count = max(1, min(int(count), 4096))
        raw = self.read(addr, count if as_type != "string" else 4096)
        if as_type == "string":
            text = raw.split(b"\0", 1)[0]
            return {"address": f"0x{addr:X}", "string": text.decode("latin-1"), "length": len(text)}
        if as_type in ("int32", "uint32", "float"):
            fmt = {"int32": "i", "uint32": "I", "float": "f"}[as_type]
            values = list(struct.unpack_from(f"<{len(raw) // 4}{fmt}", raw))
            return {"address": f"0x{addr:X}", as_type: [repr(v) if as_type == "float" else v for v in values],
                    "hex": raw[:len(values) * 4].hex(" ")}
        if as_type == "double":
            values = list(struct.unpack_from(f"<{len(raw) // 8}d", raw))
            return {"address": f"0x{addr:X}", "double": [repr(v) for v in values], "hex": raw[:len(values) * 8].hex(" ")}
        return {"address": f"0x{addr:X}", "hex": raw.hex(" "), "name": self.name_of(addr)}

    def lookup(self, target):
        """Everything the team has written about an address or name: names.csv, functions.csv and globals.csv rows,
        the prototype, the decompiled file and its status."""
        addr = self.resolve(target)
        out = {"address": f"0x{addr:X}"}
        func = self.function_at(addr)
        if func:
            out["function"] = {"start": f"0x{func[0]:X}", "size": func[1], "name": func[2]}
            conv = self.convention(func[0])
            if conv:
                out["convention_from_bytes"] = conv
            out.update(self.origins(func[0]))
        for table, key in (("names.csv", "names_row"), ("functions.csv", "functions_row"), ("globals.csv", "globals_row")):
            row = self._csv(table).get(addr)
            if row:
                out[key] = ",".join(row) if len(",".join(row)) < 4000 else ",".join(row)[:4000] + "..."
        name = self.name_of(addr)
        if name and name in self._prototypes():
            out["prototype"] = self._prototypes()[name]
        if self.src and self.src.exists() and func:
            prefix = f"{func[0]:06X}_"
            for path in self.src.rglob(f"{prefix}*.c"):
                head = path.read_text(encoding="utf-8", errors="replace").split("\n")[:6]
                status = next((l.split("status:", 1)[1].strip() for l in head if "status:" in l), "?")
                out["file"] = {"path": str(path.relative_to(self.src.parent)).replace(os.sep, "/"), "status": status}
                break
        return out

    # ----------------------------------------------------------------- one-call summary
    def _src_index(self):
        """address -> path of src/**/<ADDR>_<Name>.c (one directory walk, reused for 30 seconds)."""
        import time
        hit = self._cache.get("src_index")
        if hit and time.monotonic() - hit[0] < 30:
            return hit[1]
        out = {}
        if self.src and self.src.exists():
            for folder, _, files in os.walk(self.src):
                for name in files:
                    m = re.match(r"([0-9A-Fa-f]{6,8})_.*\.c$", name)
                    if m:
                        out.setdefault(int(m.group(1), 16), Path(folder) / name)
        self._cache["src_index"] = (time.monotonic(), out)
        return out

    def status_of(self, addr):
        """The decompiled file's status on main ('readable', 'raw', ...), or None when there is no file."""
        path = self._src_index().get(addr)
        if not path:
            return None
        try:
            with path.open(encoding="utf-8", errors="replace") as f:
                head = [next(f, "") for _ in range(8)]
        except OSError:
            return None
        return next((l.split("status:", 1)[1].strip() for l in head if "status:" in l), "raw")

    def _sections(self):
        """[(name, start, end)] from sections.json beside the image (or [reverse] sections), else []."""
        path = self.sections_path or self.image.with_name("sections.json")
        return self._load("sections", path, lambda p: [
            (s["name"], _int(s["start"]), _int(s["end"])) for s in json.loads(p.read_text(encoding="utf-8"))] if p else [])

    def _code_data_refs(self):
        facts = self._facts()
        key = ("data_refs", id(facts))
        if key not in self._cache:
            self._cache[key] = {_int(a) for d in facts.values() for a in d.get("data", [])}
        return self._cache[key]

    def _mac_vtables(self):
        return self._load("mac_vtables", self.mac_names.with_name("vtables.csv") if self.mac_names else None,
                          lambda p: {int(r["pc_vtable"], 16): r.get("class") for r in
                                     csv.DictReader(p.read_text(encoding="utf-8").splitlines())
                                     if (r.get("pc_vtable") or "").startswith("0x")} if p else {})

    def vtable_slots(self, addr):
        """Where the function's address sits in a table of code pointers outside .text: the table start (walked back
        over function addresses to the entry the code references), the slot index, and what the table is."""
        data = self._bytes()
        facts = self._facts()
        sections = self._sections()
        text = next(((lo, hi) for name, lo, hi in sections if name == ".text"), None)
        tables = [(lo, hi) for name, lo, hi in sections if name in (".rdata", ".data")]
        if not tables:                                     # no section list: everything past the last function
            end = max((a + int(d.get("size") or 0) for a, d in facts.items()), default=self.base)
            tables = [(end, self.base + len(data))]
        refs, mac = self._code_data_refs(), self._mac_vtables()
        is_code = (lambda v: v in facts) if facts else (lambda v: text and text[0] <= v < text[1])
        needle = struct.pack("<I", addr)
        out = []
        for lo, hi in tables:
            pos = data.find(needle, max(0, lo - self.base), hi - self.base)
            while pos >= 0 and len(out) < 40:
                if pos % 4 == 0:
                    start = pos
                    while start - 4 >= lo - self.base and (start + self.base) not in refs and \
                            (start + self.base) not in mac and \
                            is_code(struct.unpack_from("<I", data, start - 4)[0]) and pos - start < 4 * 400:
                        start -= 4
                    table = start + self.base
                    slot = (pos - start) // 4
                    name = self.name_of(table)
                    kind = "vtable" if table in refs or table in mac or (name and "vtable" in name.lower()) \
                        else "pointer table"
                    out.append({"table": f"0x{table:X}", "slot": slot, "offset": f"0x{slot * 4:X}", "kind": kind,
                                **({"table_name": name} if name else {}),
                                **({"mac_class": mac[table]} if table in mac else {})})
                pos = data.find(needle, pos + 1, hi - self.base)
        return out

    def summary(self, target):
        """Everything a worker needs before starting on a function, in one call: name, prototype, size, convention,
        whether it is readable on main, callers and callees (named, with their status on main), globals and strings,
        the vtable slots it fills, and the Mac/Xbox name hints."""
        addr = self.resolve(target)
        func = self.function_at(addr)
        if not func:
            raise HarborError(f"0x{addr:X} is not inside a known function")
        start, size, name = func
        d = self._facts()[start]

        def fn(a):
            status = self.status_of(a)
            return {"address": f"0x{a:X}", "name": self.name_of(a), "status": status or "no file",
                    "readable": status == "readable"}

        out = {"address": f"0x{start:X}", "name": name, "size": size, "end": f"0x{start + size:X}"}
        if addr != start:
            out["asked"] = f"0x{addr:X} (+0x{addr - start:X})"
        proto = self._prototypes().get(name) if name else None
        out["prototype"] = proto
        conv = self.convention(start)
        out["convention"] = conv["summary"] if conv else None
        if conv:
            out["registers_in"], out["ret_pops"] = conv["registers_in"], conv["ret_pops"]
        status = self.status_of(start)
        out["status"] = status or "no file"
        out["readable"] = status == "readable"
        path = self._src_index().get(start)
        if path and self.src:
            out["file"] = str(path.relative_to(self.src.parent)).replace(os.sep, "/")
        row = self._csv("names.csv").get(start)
        if row and len(row) > 3 and row[3]:
            out["note"] = row[3] if len(row[3]) < 600 else row[3][:600] + "..."
        callers = sorted(set(self._callers().get(start, [])))
        callees = list(dict.fromkeys(_int(a) for a in d.get("calls", [])))
        out["callers"] = [fn(a) for a in callers]
        out["callees"] = [fn(a) for a in callees]
        if d.get("imports"):
            out["imports"] = d["imports"]
        glob = self._csv("globals.csv")
        out["globals"] = [{"address": f"0x{a:X}", "name": self.name_of(a),
                           **({"type": glob[a][2]} if a in glob and len(glob[a]) > 2 and glob[a][2] else {})}
                          for a in dict.fromkeys(_int(x) for x in d.get("data", []))]
        out["strings"] = d.get("strings", [])
        out["vtable_slots"] = self.vtable_slots(start)
        out["hints"] = self.origins(start)
        out["counts"] = {"callers": len(callers), "callers_readable": sum(c["readable"] for c in out["callers"]),
                         "callees": len(callees), "callees_readable": sum(c["readable"] for c in out["callees"])}
        return out


def format_summary(s):
    """Reverse.summary() as text for the command line."""
    mark = lambda f: f"{f['address']} {f['name'] or '(unnamed)'} [{'readable' if f['readable'] else f['status']}]"
    lines = [f"{s['name'] or '(unnamed)'} {s['address']}..{s['end']} ({s['size']} bytes)"
             + (f", asked {s['asked']}" if s.get("asked") else ""),
             f"status on main: {s['status']}" + (f" ({s['file']})" if s.get("file") else ""),
             f"convention: {s['convention'] or 'unknown'}",
             "prototype: " + (s["prototype"] or "none in prototypes.h")]
    c = s["counts"]
    lines.append(f"callers ({c['callers']}, {c['callers_readable']} readable):")
    lines += [f"  {mark(f)}" for f in s["callers"]] or ["  none"]
    lines.append(f"callees ({c['callees']}, {c['callees_readable']} readable):")
    lines += [f"  {mark(f)}" for f in s["callees"]] or ["  none"]
    if s.get("imports"):
        lines.append("imports: " + ", ".join(s["imports"]))
    lines.append(f"globals ({len(s['globals'])}):")
    lines += [f"  {g['address']} {g['name'] or '(unnamed)'}" + (f" : {g['type']}" if g.get("type") else "")
              for g in s["globals"]] or ["  none"]
    if s["strings"]:
        lines.append("strings: " + ", ".join(repr(x) for x in s["strings"]))
    lines.append("vtable slots:" + ("" if s["vtable_slots"] else " none"))
    lines += [f"  {v['kind']} {v['table']}" + (f" {v['table_name']}" if v.get("table_name") else "")
              + (f" (Mac class {v['mac_class']})" if v.get("mac_class") else "") + f" slot {v['slot']} (+{v['offset']})"
              for v in s["vtable_slots"]]
    h = s["hints"]
    if h.get("mac_name"):
        lines.append(f"Mac name: {h['mac_name'].get('demangled')} ({h['mac_name'].get('confidence')})")
    if h.get("source_file"):
        sf = h["source_file"]
        lines.append(f"Xbox source file: {sf.get('source_file')}" + (f" lines {sf['lines']}" if sf.get("lines") else "")
                     + f" ({sf.get('confidence')})")
    if not h:
        lines.append("Mac/Xbox hints: none")
    return "\n".join(lines)


class Ghidra:
    """A small client for the shared, read-only Ghidra MCP server (GhidrAssistMCP over streamable HTTP), so every
    agent reaches Ghidra through Harbor's own tools. Configured by `ghidra_url` in the [reverse] table."""

    def __init__(self, config):
        self.url = (config or {}).get("ghidra_url")
        if not self.url:
            raise HarborError("this project has no ghidra_url in its [reverse] table")
        self.session = None
        self.counter = 0

    def _post(self, method, params=None, notify=False, timeout=120):
        import urllib.error
        import urllib.request
        body = {"jsonrpc": "2.0", "method": method, **({"params": params} if params is not None else {})}
        if not notify:
            self.counter += 1
            body["id"] = self.counter
        headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"}
        if self.session:
            headers["Mcp-Session-Id"] = self.session
        request = urllib.request.Request(self.url, json.dumps(body).encode(), headers=headers, method="POST")
        try:
            with urllib.request.urlopen(request, timeout=timeout) as response:
                self.session = response.headers.get("Mcp-Session-Id") or self.session
                text = response.read().decode("utf-8", "replace")
        except urllib.error.HTTPError as error:
            if error.code in (400, 404) and self.session and method != "initialize":
                self.session = None                       # the server restarted: start a new session once
                self._initialize()
                return self._post(method, params, notify, timeout)
            raise HarborError(f"the Ghidra server answered {error.code}: {error.read()[:300]!r}") from None
        except OSError as error:
            raise HarborError(f"the shared Ghidra server at {self.url} is not answering ({error}); start it with "
                              "R:/Workspace/Tools/re/ghidra-serve.ps1, or ask the coordinator") from None
        if notify or not text.strip():
            return None
        if not text.lstrip().startswith("{"):
            text = next((line[5:] for line in text.splitlines() if line.startswith("data:")), "{}")
        answer = json.loads(text)
        if "error" in answer:
            raise HarborError(f"Ghidra: {answer['error'].get('message', answer['error'])}")
        return answer.get("result")

    def _initialize(self):
        self._post("initialize", {"protocolVersion": "2025-06-18", "capabilities": {},
                                  "clientInfo": {"name": "harbor", "version": "2"}})
        self._post("notifications/initialized", notify=True)

    def _ensure(self):
        if not self.session:
            self._initialize()

    def tools(self):
        """The server's tools: name, description and argument schema."""
        self._ensure()
        return [{"name": t["name"], "description": t.get("description", ""),
                 "arguments": (t.get("inputSchema") or {}).get("properties", {}),
                 "required": (t.get("inputSchema") or {}).get("required", [])}
                for t in self._post("tools/list")["tools"]]

    def call(self, tool, arguments=None, wait=180):
        """Call one tool; long-running ones (the decompiler) are polled until they finish or `wait` seconds pass."""
        import time
        self._ensure()
        result = self._post("tools/call", {"name": tool, "arguments": arguments or {}}, timeout=wait + 30)
        text = "\n".join(c.get("text", "") for c in result.get("content", []) if isinstance(c, dict))
        task = re.search(r"Task ID: ([0-9a-f-]{36})", text) if "async execution" in text else None
        deadline = time.monotonic() + wait
        while task and time.monotonic() < deadline:
            time.sleep(1)
            result = self._post("tools/call", {"name": "get_task_status", "arguments": {"task_id": task.group(1)}})
            text = "\n".join(c.get("text", "") for c in result.get("content", []) if isinstance(c, dict))
            if not re.search(r"Status: (RUNNING|PENDING|QUEUED)", text):
                break
        if result.get("isError"):
            raise HarborError(text.strip() or f"Ghidra refused {tool}")
        return re.sub(r"^\[Context\] Operating on: [^\n]*\n+", "", text)


def _hexdump_bytes(text):
    """The bytes of a GhidrAssistMCP get_data_at hexdump ('401000  53 55 8b ...  |SU.|')."""
    out = bytearray()
    for line in text.splitlines():
        m = re.match(r"^\s*[0-9A-Fa-f]{4,16}\s{2}((?:[0-9A-Fa-f]{2}\s{1,2})+)", line + " ")
        if m:
            out += bytes.fromhex("".join(m.group(1).split()))
    return bytes(out)


def ghidra_check(config, samples=16, sample_len=128):
    """Is the shared Ghidra server serving our program? Read-only: compares the program's address range with the
    project image (base and size), the SHA-256 of the file Ghidra imported (when that path is readable here) with
    the image's, and sample bytes read through Ghidra (get_data_at) with the image at spread-out addresses."""
    import hashlib
    reverse = Reverse(config)
    image = reverse._bytes()
    if not image:
        raise HarborError(f"the project image {reverse.image} is missing")
    base = reverse.base
    g = Ghidra(config)
    info = g.call("get_binary_info", {})
    binaries = g.call("list_binaries", {})
    field = lambda label, text: (re.search(rf"^\s*{label}:\s*(.+?)\s*$", text, re.M) or [None, None])[1]
    lo, hi = field("Min Address", info), field("Max Address", info)
    checks = []
    want_lo, want_hi = base, base + len(image) - 1
    got = (int(lo, 16), int(hi, 16)) if lo and hi else None
    checks.append({"check": "address range", "ok": got == (want_lo, want_hi),
                   "expected": f"0x{want_lo:X}..0x{want_hi:X} ({len(image)} bytes)",
                   "ghidra": f"0x{got[0]:X}..0x{got[1]:X} ({got[1] - got[0] + 1} bytes)" if got else info.strip()[:200]})
    sha = hashlib.sha256(image).hexdigest()
    exe = field("Executable Path", binaries)
    local = Path(exe[1:] if exe and re.match(r"^/[A-Za-z]:/", exe) else exe) if exe else None
    if local and local.is_file():
        theirs = hashlib.sha256(local.read_bytes()).hexdigest()
        checks.append({"check": "sha256 of the imported file", "ok": theirs == sha, "expected": sha,
                       "ghidra": theirs, "file": str(local)})
    else:
        checks.append({"check": "sha256 of the imported file", "ok": None, "expected": sha,
                       "ghidra": f"not readable here ({exe or 'no path reported'})"})
    step = max(sample_len, (len(image) - sample_len) // max(1, samples - 1)) & ~0xF
    bad = []
    for i in range(samples):
        addr = base + min(i * step, len(image) - sample_len)
        theirs = _hexdump_bytes(g.call("get_data_at", {"address": f"0x{addr:X}", "len": sample_len}))
        ours = image[addr - base:addr - base + sample_len]
        if theirs != ours:
            bad.append(f"0x{addr:X}")
    checks.append({"check": f"{samples} byte samples of {sample_len}", "ok": not bad,
                   **({"differ_at": bad} if bad else {})})
    ok = all(c["ok"] is not False for c in checks)
    return {"ok": ok, "program": field("Name", info), "image": str(reverse.image), "image_sha256": sha,
            "image_base": f"0x{base:X}", "image_size": len(image), "functions_in_ghidra": field("Function Count", info),
            "checks": checks}


class Debugger:
    """A per-agent x64dbg session through the x64dbg-automate MCP server (stdio), started on first use and kept for
    the life of this Harbor process, so breakpoints and the paused game survive between calls. Configured by
    `debugger_command`, `debugger_path` (x32dbg.exe) and `debugger_temp` in the [reverse] table. The plugin and the
    client find each other through a session file in TEMP, so both get the same TEMP."""

    def __init__(self, config):
        config = config or {}
        self.command = config.get("debugger_command")
        if not self.command:
            raise HarborError("this project has no debugger_command in its [reverse] table")
        self.env = dict(os.environ)
        if config.get("debugger_path"):
            self.env["X64DBG_PATH"] = str(config["debugger_path"])
        if config.get("debugger_temp"):
            self.env["TEMP"] = self.env["TMP"] = str(config["debugger_temp"])
        self.proc = None
        self.counter = 0
        self.lock = __import__("threading").Lock()

    def _start(self):
        import atexit
        import queue
        import subprocess
        import threading
        command = self.command if isinstance(self.command, list) else [self.command]
        self.proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                     stderr=subprocess.DEVNULL, env=self.env, text=True, encoding="utf-8",
                                     creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
        self.answers = queue.Queue()

        def read(stream, answers):
            for line in stream:
                if line.strip():
                    try:
                        answers.put(json.loads(line))
                    except ValueError:
                        pass
            answers.put(None)
        threading.Thread(target=read, args=(self.proc.stdout, self.answers), daemon=True).start()
        self._rpc("initialize", {"protocolVersion": "2025-06-18", "capabilities": {},
                                 "clientInfo": {"name": "harbor", "version": "2"}})
        self._send({"jsonrpc": "2.0", "method": "notifications/initialized"})
        atexit.register(self.close)

    def _send(self, body):
        self.proc.stdin.write(json.dumps(body) + "\n")
        self.proc.stdin.flush()

    def _rpc(self, method, params, timeout=120):
        import queue
        self.counter += 1
        self._send({"jsonrpc": "2.0", "id": self.counter, "method": method, "params": params})
        while True:
            try:
                answer = self.answers.get(timeout=timeout)
            except queue.Empty:
                raise HarborError(f"the debugger did not answer {method} within {timeout}s") from None
            if answer is None:
                self.proc = None
                raise HarborError("the debugger's MCP server exited")
            if answer.get("id") == self.counter:
                if "error" in answer:
                    raise HarborError(f"debugger: {answer['error'].get('message', answer['error'])}")
                return answer.get("result")

    def tools(self):
        with self.lock:
            if self.proc is None or self.proc.poll() is not None:
                self._start()
            return [{"name": t["name"], "description": t.get("description", ""),
                     "arguments": (t.get("inputSchema") or {}).get("properties", {}),
                     "required": (t.get("inputSchema") or {}).get("required", [])}
                    for t in self._rpc("tools/list", {})["tools"]]

    def call(self, tool, arguments=None, timeout=300):
        with self.lock:
            if self.proc is None or self.proc.poll() is not None:
                self._start()
            result = self._rpc("tools/call", {"name": tool, "arguments": arguments or {}}, timeout=timeout)
        text = "\n".join(c.get("text", "") for c in result.get("content", []) if isinstance(c, dict))
        if result.get("isError"):
            raise HarborError(text.strip() or f"the debugger refused {tool}")
        return text

    def close(self):
        """End the debuggee and the debugger with the Harbor process, so no paused game is left behind."""
        if self.proc and self.proc.poll() is None:
            try:
                self._rpc("tools/call", {"name": "terminate_session", "arguments": {}}, timeout=15)
            except HarborError:
                pass
            self.proc.terminate()
        self.proc = None


def match_functions(config, targets, decomp=None, flags=None):
    """Compile functions with the game's compiler (Visual C++ 2003) and compare them with the original bytes.
    targets: addresses ("0x4ADC70, 0x412F70"); decomp: the checkout or worktree whose src/ holds them."""
    import subprocess
    import sys
    import tempfile
    config = config or {}
    matcher = config.get("matcher")
    if not matcher:
        raise HarborError("this project has no matcher in its [reverse] table")
    wanted = [t.strip() for t in re.split(r"[,\s]+", str(targets)) if t.strip()]
    if not wanted or len(wanted) > 50:
        raise HarborError("give 1 to 50 addresses")
    with tempfile.TemporaryDirectory(dir=config.get("debugger_temp")) as tmp:
        out = Path(tmp) / "match.json"
        command = [sys.executable, "-B", str(matcher), "--all", "--only", ",".join(wanted), "--json", str(out),
                   "--flags", flags or config.get("match_flags", "-TC -O2")]
        if decomp:
            command += ["--decomp", str(decomp)]
        result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace",
                                timeout=600, creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
        if not out.exists():
            raise HarborError("the matcher failed: " + (result.stdout + result.stderr).strip()[-800:])
        rows = json.loads(out.read_text(encoding="utf-8"))
    errors = [line for line in result.stdout.splitlines() if "compile errors" in line]
    return {"results": rows, "summary": result.stdout.strip().splitlines()[0] if result.stdout.strip() else "",
            **({"compile": errors[0]} if errors else {})}


def run_toolkit(config, key, argv, json_to_file=False, timeout=900):
    """Run the toolkit script configured as [reverse] <key> with argv and return its JSON (stdout, or a temp file
    passed as `--json <file>` when json_to_file)."""
    import subprocess
    import sys
    import tempfile
    script = (config or {}).get(key)
    if not script or not Path(script).exists():
        raise HarborError(f"this project has no {key} script in its [reverse] table")
    with tempfile.TemporaryDirectory(dir=(config or {}).get("debugger_temp")) as tmp:
        out = Path(tmp) / "out.json"
        command = [sys.executable, "-B", str(script), *argv] + (["--json", str(out)] if json_to_file else ["--json"])
        result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace",
                                timeout=timeout, cwd=str(Path(script).parent),
                                creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
        text = out.read_text(encoding="utf-8") if json_to_file and out.exists() else result.stdout
        try:
            return json.loads(text)
        except ValueError:
            raise HarborError(f"{key} failed: " + (result.stdout + result.stderr).strip()[-800:]) from None


def match_program(config, targets=None, decomp=None):
    """Build every readable function into one /GL + /LTCG image the way the game was built (stand-in callers, stubs
    for the rest, globals at their original addresses) and check the targets against the original bytes. Slow: a
    build takes 1.5 to 9 minutes depending on load."""
    import subprocess
    import sys
    import tempfile
    config = config or {}
    script = config.get("ltcg")
    if not script or not Path(script).exists():
        raise HarborError("this project has no ltcg script in its [reverse] table")
    wanted = [t.strip() for t in re.split(r"[,\s]+", str(targets or "")) if t.strip()]
    extra = ["--decomp", str(decomp)] if decomp else []
    run = lambda argv, timeout: subprocess.run([sys.executable, "-B", str(script), *argv], capture_output=True,
                                               text=True, encoding="utf-8", errors="replace", timeout=timeout,
                                               cwd=str(Path(script).parent),
                                               creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
    # one build per worktree at a time: two runs on the same checkout write the same build folder
    lock_dir = Path(config.get("debugger_temp") or tempfile.gettempdir())
    lock = lock_dir / ("ltcg-" + re.sub(r"[^A-Za-z0-9]+", "_", str(decomp or "main")) + ".lock")
    with _exclusive(lock, wait_seconds=1800):
        built = run(["build", *extra], 1800)
        if built.returncode != 0:
            raise HarborError("the whole-program build failed: " + (built.stdout + built.stderr).strip()[-800:])
        with tempfile.TemporaryDirectory(dir=config.get("debugger_temp")) as tmp:
            out = Path(tmp) / "check.json"
            checked = run(["check", *extra, "--json", str(out)] + (["--only", ",".join(wanted)] if wanted else []), 900)
            if not out.exists():
                raise HarborError("the whole-program check failed: " + (checked.stdout + checked.stderr).strip()[-800:])
            rows = json.loads(out.read_text(encoding="utf-8"))
    counts = {}
    for r in rows:
        counts[r.get("status")] = counts.get(r.get("status"), 0) + 1
    keep = ("address", "name", "status", "first_difference", "similarity", "registers_in", "ours_registers_in",
            "ret_pops", "ours_ret_pops", "relocation_problems", "note")
    return {"counts": counts, "results": [{k: r[k] for k in keep if k in r} for r in rows] if wanted else
            [{k: r[k] for k in keep if k in r} for r in rows if r.get("status") != "differs"][:200]}


def behaviour_check(config, targets, decomp=None, precision=24):
    """Run the original machine code and our compiled C of each function in an emulator from the same inputs and
    compare results, memory writes and calls (diffrun.py). targets: 1 to 30 addresses."""
    import subprocess
    import tempfile
    config = config or {}
    command = config.get("diffrun_command")
    if not command:
        raise HarborError("this project has no diffrun_command in its [reverse] table")
    wanted = [t.strip() for t in re.split(r"[,\s]+", str(targets)) if t.strip()]
    if not wanted or len(wanted) > 30:
        raise HarborError("give 1 to 30 addresses")
    command = command if isinstance(command, list) else [command]
    with tempfile.TemporaryDirectory(dir=config.get("debugger_temp")) as tmp:
        out = Path(tmp) / "behaviour.json"
        argv = [*command, *wanted, "--json", str(out), "--precision", str(int(precision)), "--jobs", "4"]
        if decomp:
            argv += ["--decomp", str(decomp)]
        result = subprocess.run(argv, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=1500,
                                cwd=str(Path(command[-1]).parent),
                                creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
        if not out.exists():
            raise HarborError("the emulator check failed: " + (result.stdout + result.stderr).strip()[-800:])
        rows = json.loads(out.read_text(encoding="utf-8"))
    keep = ("address", "name", "verdict", "reason", "weak", "counts", "notes")
    return [{k: r[k] for k in keep if k in r} for r in rows]


def pid_alive(pid):
    """True while process `pid` runs (Windows: OpenProcess and GetExitCodeProcess; never signals it)."""
    try:
        pid = int(pid)
    except (TypeError, ValueError):
        return False
    if os.name != "nt":
        try:
            os.kill(pid, 0)
            return True
        except OSError:
            return False
    import ctypes
    kernel = ctypes.windll.kernel32
    handle = kernel.OpenProcess(0x1000, False, pid)        # PROCESS_QUERY_LIMITED_INFORMATION
    if not handle:
        return False
    try:
        code = ctypes.c_ulong()
        return bool(kernel.GetExitCodeProcess(handle, ctypes.byref(code))) and code.value == 259   # STILL_ACTIVE
    finally:
        kernel.CloseHandle(handle)


class _exclusive:
    """A lock file shared between processes: wait for it, hold it, remove it (stale after two hours)."""

    def __init__(self, path, wait_seconds=1800):
        self.path, self.wait = Path(path), wait_seconds

    def __enter__(self):
        import time
        deadline = time.monotonic() + self.wait
        while True:
            try:
                fd = os.open(str(self.path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
                os.write(fd, str(os.getpid()).encode())
                os.close(fd)
                return self
            except FileExistsError:
                try:
                    holder = self.path.read_text(encoding="utf-8").strip()
                    age = time.time() - self.path.stat().st_mtime
                    if (holder and not pid_alive(holder)) or (not holder and age > 5) or age > 7200:
                        self.path.unlink()
                        continue
                except FileNotFoundError:
                    continue
                if time.monotonic() > deadline:
                    raise HarborError(f"another whole-program build on this worktree is still running ({self.path.name}); try again later") from None
                time.sleep(5)

    def __exit__(self, *exc):
        try:
            self.path.unlink()
        except FileNotFoundError:
            pass