diff --git a/harbor/decomp_api.py b/harbor/decomp_api.py new file mode 100644 index 0000000..ddd607b --- /dev/null +++ b/harbor/decomp_api.py @@ -0,0 +1,200 @@ +"""The web board's Decomp view: progress tiers, a function inspector, each agent's next functions, deferrals and the +state of the reverse-engineering tools. Everything here reads; nothing writes. + +It runs on its own Harbor connection with its own lock and short caches, because the answers take seconds (they scan +src/ and score whole bands) and must not hold up the chat.""" +import json +import socket +import subprocess +import threading +import time +from pathlib import Path + +from .core import Harbor +from .queue import FunctionQueue +from .reverse import run_toolkit +from .util import HarborError + +PROGRESS_TTL = 120 +QUEUE_TTL = 600 + + +class DecompAPI: + def __init__(self, project): + self.project = project + self.lock = threading.Lock() + self.cache = {} + self._local = threading.local() + + @property + def h(self): + # SQLite connections belong to the thread that opened them, and the server answers on many threads + if getattr(self._local, "h", None) is None: + self._local.h = Harbor(self.project) + return self._local.h + + def _cached(self, key, ttl, fn): + hit = self.cache.get(key) + if hit and time.monotonic() - hit[0] < ttl: + return hit[1] + value = fn() + self.cache[key] = (time.monotonic(), value) + return value + + def _queue(self): + return FunctionQueue(self.h) + + # ------------------------------------------------------------------ answers + def progress(self): + with self.lock: + return self._cached("progress", PROGRESS_TTL, lambda: self._queue().progress()) + + def queues(self, n=6): + def build(): + q = self._queue() + similar = q.similar_readable() + out = [] + for agent in self.project.ai_names: + if not q.bands(agent): + continue + try: + rows = q.next_functions(agent, n, similar_readable=similar) + except HarborError as error: + rows, note = [], str(error) + else: + note = None + out.append({"agent": agent, "bands": [[f"0x{lo:X}", f"0x{hi:X}"] for lo, hi in q.bands(agent)], + "next": rows, **({"note": note} if note else {})}) + return out + with self.lock: + return self._cached("queues", QUEUE_TTL, build) + + def deferred(self): + with self.lock: + q = self._queue() + return [{"address": f"0x{a:X}", "name": q.r.name_of(a), "reason": reason, "evidence": evidence, + "author": author} for a, (reason, evidence, author) in sorted(q.deferrals().items())] + + def function(self, target): + """Everything about one function, for the inspector.""" + with self.lock: + q = self._queue() + r = q.r + info = r.lookup(target) + addr = int(info.get("function", {}).get("start", info["address"]), 16) + statuses = self._cached("statuses", PROGRESS_TTL, q.statuses) + status, filename = statuses.get(addr, ("none", None)) + info["status"] = status + if filename: + info["src_file"] = filename + for key, label in (("match_results", "match"), ("diffrun_results", "behaviour")): + row = q._results(key).get(addr) + if row: + info[label] = {k: row[k] for k in ("status", "verdict", "similarity", "first_difference", + "registers_in", "ours_registers_in", "reason", "weak") + if k in row} + deferral = q.deferrals().get(addr) + if deferral: + info["deferred"] = {"reason": deferral[0], "evidence": deferral[1], "by": deferral[2]} + info["category"] = q._category(addr) + library = q._results("library_results") + if q.is_library(addr, library, q.library_ranges()): + info["library"] = True + try: + info["disassembly"] = r.disassemble(f"0x{addr:X}") if int(info.get("function", {}).get("size", 0)) <= 0x1800 \ + else r.disassemble(f"0x{addr:X}", 0x600) + "\n; ... (first 0x600 bytes of a large function)" + except HarborError as error: + info["disassembly"] = f"; {error}" + try: + similar = run_toolkit(self.project.reverse, "similar", [f"0x{addr:X}", "--k", "6"], timeout=60) + info["similar"] = similar.get("results", []) + except (HarborError, subprocess.TimeoutExpired, OSError): + info["similar"] = [] + return info + + def tools(self): + """Is each reverse-engineering tool in place, and how fresh are its results?""" + config = self.project.reverse or {} + out = [] + + def fresh(path): + p = Path(path) + if not p.exists(): + return None + return round((time.time() - p.stat().st_mtime) / 60) + + url = config.get("ghidra_url", "") + port = int(url.rsplit(":", 1)[1].split("/")[0]) if url.count(":") >= 2 else 0 + up = False + if port: + try: + with socket.create_connection(("127.0.0.1", port), timeout=1): + up = True + except OSError: + up = False + decomp = Path(config.get("annotations", "")).parent if config.get("annotations") else None + head = applied = None + if decomp and (decomp / ".git").exists(): + try: + head = subprocess.run(["git", "-C", str(decomp), "rev-parse", "HEAD"], capture_output=True, text=True, + timeout=10).stdout.strip() or None + except (OSError, subprocess.TimeoutExpired): + head = None + applied_file = Path(config.get("debugger_temp", "")).parent / "ghidra-mcp" / "applied-commit.txt" \ + if config.get("debugger_temp") else None + if applied_file and applied_file.exists(): + applied = applied_file.read_text(encoding="utf-8").strip() + out.append({"tool": "Shared Ghidra session", "ok": up and (not head or head == applied), + "detail": ("up" if up else "down") + (f"; names from main {applied[:8]}" if applied else "") + + ("" if not head or head == applied else f"; main is at {head[:8]}: run ghidra-serve.ps1 -IfStale")}) + compiler = Path(r"C:\Program Files (x86)\Microsoft Visual C++ Toolkit 2003\bin\cl.exe") + out.append({"tool": "Game compiler (Visual C++ 2003 Toolkit)", "ok": compiler.exists(), + "detail": "cl 13.10.3052 installed" if compiler.exists() else "not found"}) + if config.get("debugger_path"): + out.append({"tool": "Debugger (x32dbg + automation plugin)", "ok": Path(config["debugger_path"]).exists(), + "detail": config["debugger_path"]}) + for key, label in (("match_results", "Whole-program match results"), ("diffrun_results", "Emulator results"), + ("library_results", "Library sweep"), ("similar_clusters", "Near-duplicate clusters"), + ("source_files", "Source files from the Xbox build"), ("mac_names", "Names from the Mac build")): + if config.get(key): + age = fresh(config[key]) + out.append({"tool": label, "ok": age is not None, + "detail": "missing" if age is None else f"updated {age} min ago"}) + return out + + +_APIS = {} +_LOCK = threading.Lock() + + +def api_for(project): + with _LOCK: + if project.name not in _APIS: + _APIS[project.name] = DecompAPI(project) + return _APIS[project.name] + + +def handle(project, path, query): + """GET /api/decomp/: returns (status, body).""" + api = api_for(project) + what = path.rsplit("/", 1)[-1] + if not (project.reverse or {}).get("facts"): + return 404, {"error": "this project has no [reverse] table"} + if what == "progress": + return 200, api.progress() + if what == "queues": + return 200, api.queues() + if what == "deferred": + return 200, api.deferred() + if what == "function": + target = (query.get("target") or "").strip() + if not target: + return 400, {"error": "give an address or a name"} + return 200, api.function(target) + if what == "tools": + return 200, api.tools() + return 404, {"error": "not found"} + + +def _json_safe(value): + return json.loads(json.dumps(value, default=str)) diff --git a/harbor/queue.py b/harbor/queue.py index b417ed9..4c69984 100644 --- a/harbor/queue.py +++ b/harbor/queue.py @@ -129,14 +129,14 @@ class FunctionQueue: return addr in library or any(lo <= addr < hi for lo, hi in ranges) # ------------------------------------------------------------------ queue - def score(self, addr, facts, named, rejected, similar_readable=frozenset()): + def score(self, addr, facts, named, rejected, similar_readable=frozenset(), with_convention=True): d = facts[addr] size = int(d.get("size") or 0) calls = [int(c, 16) if isinstance(c, str) else c for c in d.get("calls", [])] named_share = (sum(1 for c in calls if c in named) / len(calls)) if calls else 1.0 score = 1.0 / (1.0 + size / 250.0) # small first score *= 0.5 + 0.5 * named_share # callees with names make readable code possible - conv = self.r.convention(addr) if size <= 4000 else None + conv = self.r.convention(addr) if with_convention and size <= 4000 else None if conv and conv["summary"].startswith("custom"): score *= 0.75 # register arguments: prototypes.h work too if self._category(addr) not in ("game", "unsorted"): @@ -186,9 +186,12 @@ class FunctionQueue: head = self.r.read(addr, 10) if int(facts[addr].get("size") or 0) == 10 else b"" if head[:1] and 0xB8 <= head[0] <= 0xBF and head[5:6] == b"\xe9": continue # mov reg, imm32; jmp: an at-exit destructor thunk or an SEH handler stub (compiler's) - score, reasons = self.score(addr, facts, named, rejected, similar_readable) + score, reasons = self.score(addr, facts, named, rejected, similar_readable, with_convention=False) ranked.append((score, addr, reasons)) ranked.sort(reverse=True) + # the convention read from the bytes only lowers a score, and it is the slow part: read it for the head only + head = [(*self.score(a, facts, named, rejected, similar_readable), a) for _, a, _ in ranked[:max(4 * n, 40)]] + ranked = sorted(((s, a, why) for s, why, a in head), reverse=True) return [{"address": f"0x{a:X}", "name": self.r.name_of(a), "score": round(s, 4), "why": ", ".join(why), "file": statuses.get(a, (None, None))[1]} for s, a, why in ranked[:n]] @@ -207,7 +210,9 @@ class FunctionQueue: facts, statuses, deferred = self.r._facts(), self.statuses(), self.deferrals() exact = {a for a, row in self._results("match_results").items() if str(row.get("status")) in ("match", "exact", "exact-relaxed")} - verified = {a for a, row in self._results("diffrun_results").items() if str(row.get("verdict")) == "same"} + # behaves the same in the emulator, with at least one trial that ran to the end in both (not 'weak') + verified = {a for a, row in self._results("diffrun_results").items() + if str(row.get("verdict")) == "same" and not row.get("weak")} library = set(self._results("library_results")) ranges = self.library_ranges() tiers = {t: {"functions": 0, "bytes": 0} for t in TIER_ORDER + ("library", "engine")} diff --git a/harbor/reverse.py b/harbor/reverse.py index 9e59969..f1a3a94 100644 --- a/harbor/reverse.py +++ b/harbor/reverse.py @@ -161,10 +161,29 @@ class Reverse: if re.fullmatch(r"(0x)?[0-9A-Fa-f]{5,8}", text): return int(text, 16) addr = self.address_of(text) + if addr is None: + addr = self._mac_index().get(text) or self._mac_index().get(text.replace("::", "_")) if addr is None: raise HarborError(f"{target!r} is neither an address nor a known name") return addr + def _mac_index(self): + """Original names from the Mac build -> PC address: 'CanCarry', 'TradeMenu::DoTrade', 'TradeMenu_DoTrade'.""" + def load(p): + out = {} + if p: + for row in csv.DictReader(p.read_text(encoding="utf-8").splitlines()): + head = (row.get("demangled") or "").split("(", 1)[0].strip() + try: + addr = int(row["pc_address"], 16) + except (KeyError, ValueError): + continue + if head: + out.setdefault(head, addr) + out.setdefault(head.replace("::", "_"), addr) + return out + return self._load("mac_index", self.mac_names, load) + def convention(self, addr): """The calling convention read from the bytes: register inputs (a dataflow pass) and the `ret N` cleanup.""" if not self.conventions_path or not self.conventions_path.exists(): diff --git a/harbor/server.py b/harbor/server.py index 5c19e54..756888e 100644 --- a/harbor/server.py +++ b/harbor/server.py @@ -330,6 +330,11 @@ class Handler(http.server.BaseHTTPRequestHandler): "stopped": any(h.stopped(a) for a in h.project.ai_names)}) return self._reply(200, out) h, lock = REGISTRY.get(name) + if path.startswith("/api/decomp/"): + # its own connection and lock: these answers take seconds and must not hold up the chat + from . import decomp_api + status, body = decomp_api.handle(h.project, path, query) + return self._reply(status, body) if path.startswith("/files/"): filename = unquote(path[len("/files/"):]) if not re.fullmatch(r"[0-9a-f]{12}-[A-Za-z0-9._-]{1,80}", filename): diff --git a/harbor/web/app.html b/harbor/web/app.html index 87ed4c3..54ca2f1 100644 --- a/harbor/web/app.html +++ b/harbor/web/app.html @@ -55,6 +55,20 @@ body { overflow: hidden; } .content { flex: 1; min-height: 0; overflow-y: auto; padding: 24px 32px 56px; } .view { display: none; max-width: 1380px; margin: 0 auto; gap: 20px; } .view.on { display: grid; } +.tierbar { display: flex; height: 22px; border-radius: 7px; overflow: hidden; background: var(--surface-3); gap: 2px; } +.tierbar span { display: block; height: 100%; min-width: 3px; } +.tier-exact { background: var(--green); } .tier-verified { background: var(--blue); } .tier-readable { background: var(--violet); } +.tier-deferred { background: var(--amber); } .tier-raw { background: var(--line-2); } +.tier-library, .tier-engine { background: repeating-linear-gradient(45deg, var(--surface-3) 0 4px, var(--line-2) 4px 8px); } +.tier-key { display: inline-block; width: 10px; height: 10px; border-radius: 3px; margin-right: 8px; vertical-align: -1px; } +td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; } +.fn-head { display: flex; flex-wrap: wrap; gap: 8px 12px; align-items: center; margin-top: 16px; } +.fn-head h3 { margin: 0; font-size: 16px; } +.fn-facts { display: grid; grid-template-columns: max-content minmax(0, 1fr); gap: 7px 16px; font-size: 13px; margin: 14px 0 0; } +.fn-facts dt { color: var(--muted); } .fn-facts dd { margin: 0; min-width: 0; overflow-wrap: anywhere; } +.disasm { white-space: pre; font: 11.5px/1.5 var(--mono); max-height: 440px; overflow: auto; padding: 10px 12px; border-radius: 9px; background: var(--surface-3); margin: 8px 0 0; } +.band { padding: 12px 18px; border-top: 1px solid var(--line); } .band:first-child { border-top: 0; } +.band .cell-who { gap: 8px; } .band ul { margin: 8px 0 0; } .cols { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 460px), 1fr)); gap: 20px; align-items: start; } .sub-h { font-size: 11px; letter-spacing: .8px; text-transform: uppercase; color: var(--faint); font-weight: 700; margin: 20px 0 8px; } .flush th:first-child, .flush td:first-child { padding-left: 18px; } @@ -431,6 +445,7 @@ button.chip:hover { color: var(--accent-ink); background: var(--accent-soft); } + @@ -622,12 +637,44 @@ button.chip:hover { color: var(--accent-ink); background: var(--accent-soft); } + +
+
+

Progress by function

+
+
+
+
+

Inspect a function

original name, source file, convention, match and near-duplicates
+
+ +
+
+
+
+
+

Next up in each band

what next_functions hands out, easiest first
+
+
+
+

Deferred

+
+
+
+
+

Notifications

+
+

Reverse-engineering tools

+
+

Health and connections

@@ -709,7 +756,8 @@ async function get(path){const r=await fetch(path,{cache:'no-store'});const j=aw const VIEWS={overview:['Overview','Who is working, what needs you, and what happened','home'],launch:['Launch','Bring agents online one careful step at a time','rocket'], tasks:['Tasks','The shared queue that idle agents take work from','tasks'],review:['Review','Submitted work, acknowledgements and claims','review'], activity:['Activity','Live tool calls, the team pulse and the scoreboard','activity'],knowledge:['Knowledge','Search every message, pinned decisions and threads','book'], - health:['Health','Harbor in every agent app, the database, backups and the server','heart'],chat:['Chat','The team’s conversation','chat']}; + health:['Health','Harbor in every agent app, the database, backups and the server','heart'], + decomp:['Decomp','Progress by function, the function inspector and what each agent takes next','terminal'],chat:['Chat','The team’s conversation','chat']}; function show(view){if(!VIEWS[view])view='overview';current=view; document.querySelectorAll('.view').forEach(v=>v.classList.toggle('on',v.id==='view-'+view)); document.querySelectorAll('.nav-btn[data-go]').forEach(b=>{if(b.dataset.go===view)b.setAttribute('aria-current','page');else b.removeAttribute('aria-current')}); @@ -722,6 +770,8 @@ function show(view){if(!VIEWS[view])view='overview';current=view; if(data&&view==='overview')renderDigest(); if(data&&view==='activity'){renderPulse();renderFeed()} if(data&&view==='launch')renderLaunch(); + if(data&&view==='decomp')renderDecomp(); + if(view==='health')renderTools(); if(view==='health'&&!doctorRan)runDoctor()} function paintTitle(){document.title=(unseen&&document.hidden?'('+unseen+') ':'')+(data?data.project.title+' · ':'')+(current?VIEWS[current][0]:'Harbor')} @@ -962,6 +1012,52 @@ el('searchForm').onsubmit=async e=>{e.preventDefault();const q=el('q').value.tri el('menuBtn').onclick=()=>el('shell').classList.add('open'); el('scrim').onclick=()=>el('shell').classList.remove('open'); window.addEventListener('hashchange',()=>{const v=location.hash.slice(1);if(v&&v!==current)show(v)}); +const TIERS=[['exact','Exact match','byte-identical with the game’s own compiler'],['verified','Verified','behaves the same as the game in the emulator'], + ['readable','Readable','readable C that passed review'],['deferred','Deferred','set aside with a reason after failed attempts'],['raw','Raw','not decompiled yet'], + ['library','Library','C runtime, D3DX9 and other linked libraries: never decompiled'],['engine','Engine','Gamebryo engine code: names only']]; +const fmt=n=>Number(n||0).toLocaleString('en-US'); +function renderDecomp(){renderProgress();renderNextUp();renderDeferred()} +async function renderProgress(){let p;try{p=await get('api/decomp/progress')}catch(e){put('progress','
'+esc(e.message)+'
');return} + const t=p.tiers,own=['exact','verified','readable','deferred','raw'].reduce((a,k)=>a+t[k].functions,0),done=t.exact.functions+t.verified.functions+t.readable.functions; + el('progressSum').textContent=fmt(done)+' of '+fmt(own)+' game functions decompiled'; + put('progress','' + +'' + +TIERS.map(([k,l,d])=>'').join('')+'
TierFunctionsBytesMeaning
'+esc(l)+''+fmt(t[k].functions)+''+fmt(t[k].bytes)+''+esc(d)+'
' + +(p.sources&&!p.sources.diffrun_results?'
The emulator check has not reported yet, so Verified is still empty.
':''))} +async function renderNextUp(){if(!el('nextUp')._html)put('nextUp',empty('clock','Ranking the bands…','This takes a few seconds the first time.'));let q; + try{q=await get('api/decomp/queues')}catch(e){put('nextUp','
'+esc(e.message)+'
');return} + put('nextUp',q.length?q.map(a=>'
'+avatar(a.agent,'xs')+''+esc(a.agent)+''+esc(a.bands.map(b=>b.join('–')).join(', '))+'
' + +(a.next.length?'
    '+a.next.map(f=>'
  • '+esc(f.address)+' '+esc(f.name||(f.file||'').replace(/\.c$/,''))+'
    '+esc(f.why)+'
    • ').join('')+'
' + :'
'+esc(a.note||'Nothing left in this band.')+'
')+'
').join(''):empty('list','No bands yet','The coordinator gives each agent an address band.'))} +async function renderDeferred(){let d;try{d=await get('api/decomp/deferred')}catch(e){return}el('deferCount').textContent=d.length||''; + put('deferred',d.length?''+d.map(x=>'').join('')+'
FunctionReasonEvidence
'+esc(x.address)+'
'+esc(x.name||'')+'
'+esc(x.reason)+'
'+esc(x.author)+'
'+esc(x.evidence)+'
' + :empty('check','Nothing deferred','After two failed attempts an agent sets a function aside here, with the reason and what it found.'))} +async function inspectFn(){const target=el('fnTarget').value.trim();if(!target)return;put('fnInfo',empty('search','Looking it up…',''));let f; + try{f=await get('api/decomp/function?target='+encodeURIComponent(target))}catch(e){put('fnInfo','
'+esc(e.message)+'
');return} + const fn=f.function||{},mac=f.mac_name,src=f.source_file,m=f.match,b=f.behaviour,c=f.convention_from_bytes; + const row=(k,v)=>v?'
'+esc(k)+'
'+v+'
':''; + const note=f.names_row?String(f.names_row).split(',').slice(3).join(',').replace(/^"|"$/g,'').replace(/""/g,'"'):''; + const exact=m&&['exact','exact-relaxed','match'].includes(m.status),near=m&&m.status==='exact-but-targets-differ'; + put('fnInfo','

'+esc(fn.start||f.address)+'

'+esc(fn.name||'unnamed')+'' + +''+esc(f.status)+''+(f.library?'library':'') + +(m?''+(exact?'exact match':near?'same bytes, different targets':esc(m.status)+(m.similarity!=null?' · '+Math.round(m.similarity*100)+'% similar':''))+'':'') + +(b?'emulator: '+esc(b.verdict)+'':'') + +''+esc(fn.size??'?')+' bytes · '+esc(f.category||'')+'
' + +'
'+row('Original name',mac?''+esc(mac.demangled)+' '+esc(mac.confidence)+' confidence, from the Mac build':'') + +row('Source file',src?''+esc(src.source_file)+''+(src.lines?' line '+esc(src.lines):'')+' '+esc(src.confidence)+' confidence, from the Xbox build':'') + +row('Convention',c?esc(c.summary)+(c.ret_pops&&c.ret_pops.some(x=>x)?' · cleans up '+esc(c.ret_pops.join('/'))+' bytes':''):'') + +row('Decompiled file',f.src_file?''+esc(f.src_file)+'':'') + +row('Deferred',f.deferred?''+esc(f.deferred.reason)+' '+esc(f.deferred.evidence):'') + +row('Emulator',b&&b.verdict!=='same'?esc(b.reason||''):b&&b.weak?'same, but no trial ran to the end in both':'') + +row('Prototype',f.prototype?''+esc(f.prototype)+'':'')+row('Team note',note?esc(note.slice(0,700)):'')+'
' + +(f.similar&&f.similar.length?'

Near-duplicates

    '+f.similar.map(s=>'
  • '+esc(s.address)+' '+esc(s.name||'')+' '+Math.round(s.score*100)+'% alike · '+esc(s.status)+'
    • ').join('')+'
':'') + +'
Disassembly
'+esc(f.disassembly||'')+'
')} +async function renderTools(){let t;try{t=await get('api/decomp/tools')}catch(e){put('retools','
'+esc(e.message)+'
');return} + const bad=t.filter(x=>!x.ok).length;el('reSum').textContent=bad?bad+' to check':'all in place'; + put('retools',''+t.map(x=>'').join('')+'
StatusTool
'+(x.ok?'ok':'check')+''+esc(x.tool)+'
'+esc(x.detail)+'
')} +el('fnForm').addEventListener('submit',e=>{e.preventDefault();inspectFn()}); +el('refreshDecomp').addEventListener('click',renderDecomp); +document.addEventListener('click',e=>{const a=e.target.closest('[data-fn]');if(!a)return;e.preventDefault();if(current!=='decomp')show('decomp');el('fnTarget').value=a.dataset.fn;inspectFn();el('h-inspect').scrollIntoView({behavior:'smooth',block:'start'})}); document.addEventListener('keydown',e=>{if(e.ctrlKey||e.metaKey||e.altKey||e.target.closest('input,textarea,select,[contenteditable]')||document.querySelector('dialog[open]'))return; const keys=Object.keys(VIEWS).filter(k=>k!=='chat');const i=Number(e.key)-1;if(i>=0&&i{if(!document.hidden){unseen=0; const startQ=new URLSearchParams(location.search).get('q'); const onBoard=/\/board\/?$/.test(location.pathname); show(startQ?'knowledge':(location.hash.slice(1)||(onBoard?store.get('board.view','overview'):'chat'))); -refresh().then(()=>{renderDigest();if(current==='activity'){renderPulse();renderFeed()}if(current==='launch')renderLaunch();if(startQ){el('q').value=startQ;el('searchForm').requestSubmit()}}); +refresh().then(()=>{renderDigest();if(current==='activity'){renderPulse();renderFeed()}if(current==='launch')renderLaunch();if(current==='decomp')renderDecomp();if(current==='health')renderTools();if(startQ){el('q').value=startQ;el('searchForm').requestSubmit()}}); setInterval(refresh,3000); setInterval(()=>{if(current==='overview')renderDigest()},30000); setInterval(()=>{if(current==='activity')renderPulse()},60000);