From 71e6d6cfdad13af29d2d172eb9b7c5147c366768 Mon Sep 17 00:00:00 2001 From: KeelrLeek <156356974+KeelrLeek@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:43:38 -0500 Subject: [PATCH] logbook: an All code bar (game + engine + platform, by bytes, with function counts) next to the game-code bar, so engine work shows too --- harbor/progress_map.py | 646 ++++---- harbor/web/app.html | 3278 ++++++++++++++++++++-------------------- 2 files changed, 1964 insertions(+), 1960 deletions(-) diff --git a/harbor/progress_map.py b/harbor/progress_map.py index 87935cc..41a58f7 100644 --- a/harbor/progress_map.py +++ b/harbor/progress_map.py @@ -1,321 +1,325 @@ -"""The Logbook's progress map, in the style of decomp.dev: one unit per original source file, its functions and how -far each has come, plus the headline share of game code decompiled and the Code and Data bars. - -GET /api/logbook/progress-map answers with one compact JSON document. It is computed once and kept until one of its -inputs changes (main's HEAD, the Logbook's symbol table and statistics, the match, emulator and library results, -facts, names, source files, globals, or a deferral note); `?have=` answers {"version", "same": true} when -the caller already holds the current one, so a page that polls does not download or re-lay-out anything. - -Definitions follow the Logbook exactly (open-seas tools/progress/progress.py, which writes stats.json): - - the symbol table is <[logbook] stats_repo>/re/symbols/pc102.csv: its category and status per function; - - counted ("game code") = category game and status not superseded or wontfix; - - decompiled = status decompiled, reimplemented or verified, so percent = decompiled game bytes / game bytes, - the Logbook's game_bytes_decompiled_percent. -Decompiled functions are split further, best first: exact (byte match in the [reverse] match_results), verified -(behaves the same in the emulator, diffrun_results, not 'weak'), readable. The rest: deferred (a live DEFERRED note), -raw, and, not counted, unsorted, engine (Gamebryo, named only) and library (C runtime, D3DX9, platform wrappers). -"results" says which commit the exact and verified tiers are from (harbor.results_refresh keeps them current). -Without a stats_repo the categories come from names.csv / facts and readable from the src headers (the queue's view). - -Units, first that applies: the source file from the Xbox build's assert sites ([reverse] source_files); the D3DX9 block -(platform/d3dx9) or the import-thunk stubs from library_ranges; the unit facts gives (engine/, -platform/); otherwise - plus its 64 KB address block, labelled "(no source file)".""" -import csv -import hashlib -import json -import subprocess -import threading -import time -from pathlib import Path - -from .queue import FunctionQueue - -STATUSES = ("exact", "verified", "readable", "deferred", "raw", "unsorted", "engine", "library") -DECOMPILED = ("decompiled", "reimplemented", "verified") -EXCLUDED = ("superseded", "wontfix") -UNIT_FIELDS = ("name", "kind", "lo", "hi", "bytes", "counted", "exact", "verified", "readable", "functions") -FUNCTION_FIELDS = ("unit", "address", "size", "status", "name") -KEY_TTL = 15 # seconds a computed input key is trusted before the files and HEAD are looked at again - - -def _int(text): - return int(str(text), 16) if str(text).lower().startswith("0x") else int(text) - - -def _mtime(path): - try: - return Path(path).stat().st_mtime_ns if path else None - except OSError: - return None - - -def _git(repo, *args): - try: - out = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, timeout=20, - encoding="utf-8", errors="replace") - except (OSError, subprocess.TimeoutExpired): - return "" - return out.stdout.strip() if out.returncode == 0 else "" - - -def _utc(iso): - import datetime - try: - return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - except ValueError: - return None - - -class ProgressMap: - def __init__(self, project, queue_factory): - self.project = project - self.queue_factory = queue_factory # -> FunctionQueue on a connection this thread may use - self.lock = threading.Lock() - self.value = None # (key, document) - self.checked = (0.0, None) # (monotonic time, key) - self.computations = 0 - - # ------------------------------------------------------------------ inputs - def _config(self): - return self.project.reverse or {}, getattr(self.project, "logbook", None) or {} - - def _symbols_path(self): - _, lb = self._config() - return Path(lb["stats_repo"]) / "re" / "symbols" / "pc102.csv" if lb.get("stats_repo") else None - - def _decomp(self): - rev, lb = self._config() - if lb.get("decomp"): - return Path(lb["decomp"]) - return Path(rev["annotations"]).parent if rev.get("annotations") else None - - def _logbook_dir(self): - from . import logbook - return logbook.folder(self.project) - - def key(self, q): - rev, _ = self._config() - decomp = self._decomp() - head = _git(decomp, "rev-parse", "HEAD") if decomp and (decomp / ".git").exists() else "" - paths = [self._symbols_path(), self._logbook_dir() / "stats.json", self._logbook_dir() / "history.json", - self._logbook_dir() / "results.json"] - paths += [rev.get(k) for k in ("facts", "match_results", "diffrun_results", "library_results", - "library_ranges", "source_files")] - if rev.get("annotations"): - paths += [Path(rev["annotations"]) / "names.csv", Path(rev["annotations"]) / "globals.csv"] - try: - note = q.h.conn.execute("SELECT MAX(id) FROM notes").fetchone()[0] - except Exception: - note = None - raw = json.dumps([head, note, [str(p) for p in paths], [_mtime(p) for p in paths]]) - return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16], head - - # ------------------------------------------------------------------ answer - def get(self, have=None): - with self.lock: - q = self.queue_factory() - now = time.monotonic() - if self.value and now - self.checked[0] < KEY_TTL: - key = self.checked[1] - else: - key, head = self.key(q) - self.checked = (now, key) - if not self.value or self.value[0] != key: - document = self.compute(q, head) - if document is None and self.value: # the symbol table is being rewritten: keep the last - document = self.value[1] - elif document is None: - document = self.compute(q, head, use_symbols=False) - document["version"] = key - self.value = (key, document) - self.computations += 1 - if have and have == self.value[0]: - return {"version": have, "same": True} - return self.value[1] - - # ------------------------------------------------------------------ the computation - def _symbols(self, expected): - """pc102.csv as {address: (category, status, name)}, or None while it is missing or half written.""" - path = self._symbols_path() - if not path or not path.exists(): - return None - try: - with path.open(encoding="utf-8", newline="") as stream: - rows = {_int(r["address"]): (r["category"], r["status"], r.get("name") or "") - for r in csv.DictReader(stream)} - except (OSError, ValueError, KeyError): - return None - return rows if len(rows) >= 0.98 * expected else None - - def _source_files(self, path): - out = {} - if path and Path(path).exists(): - with open(path, encoding="utf-8", newline="") as stream: - for row in csv.DictReader(stream): - if row.get("source_file"): - out[_int(row["pc_address"])] = row["source_file"] - return out - - def _all_ranges(self, path): - """[(lo, hi, label)] of the library_ranges csv (the D3DX9 block and the compiler's stubs).""" - out = [] - if path and Path(path).exists(): - with open(path, encoding="utf-8", newline="") as stream: - for r in csv.DictReader(stream): - verdict = r.get("verdict", "") - label = "platform/d3dx9" if verdict.startswith("D3DX") else \ - "Import thunks and unwind funclets" if "stubs" in verdict else None - if label: - out.append((_int(r["start"]), _int(r["end"]), label)) - return out - - def _data(self, facts, rev): - """Named globals against the distinct .data/.rdata addresses the code touches (needs sections.json).""" - facts_path = Path(rev["facts"]) if rev.get("facts") else None - sections_path = facts_path.parent / "sections.json" if facts_path else None - globals_path = Path(rev["annotations"]) / "globals.csv" if rev.get("annotations") else None - if not sections_path or not sections_path.exists() or not globals_path or not globals_path.exists(): - return None - spans = [(s["start"], s["end"]) for s in json.loads(sections_path.read_text(encoding="utf-8")) - if s.get("name") in (".data", ".rdata")] - touched = set() - for d in facts.values(): - for ref in d.get("data", []): - a = _int(ref) - if any(lo <= a < hi for lo, hi in spans): - touched.add(a) - with globals_path.open(encoding="utf-8", newline="") as stream: - named = {_int(r["address"]) for r in csv.DictReader(line for line in stream if not line.startswith("#")) - if r.get("address") and r.get("name")} - if not touched: - return None - return {"addresses": len(touched), "named": len(touched & named), "globals": len(named), - "bytes": sum(hi - lo for lo, hi in spans)} - - def compute(self, q, head, use_symbols=True): - rev, _ = self._config() - facts = q.r._facts() - symbols = self._symbols(len(facts)) if use_symbols else None - if use_symbols and self._symbols_path() and symbols is None and self._symbols_path().exists(): - return None - names = q.r._csv("names.csv") - statuses = None if symbols else q.statuses() - exact = {a for a, row in q._results("match_results").items() - if str(row.get("status")) in ("match", "exact", "exact-relaxed")} - verified = {a for a, row in q._results("diffrun_results").items() - if str(row.get("verdict")) == "same" and not row.get("weak")} - library = set(q._results("library_results")) - ranges = self._all_ranges(rev.get("library_ranges")) - lib_ranges = [(lo, hi) for lo, hi, _ in ranges] - deferred = q.deferrals() - sources = self._source_files(rev.get("source_files")) - - units, unit_index, functions = [], {}, [] - game_functions = game_done = 0 - for addr in sorted(facts): - d = facts[addr] - size = int(d.get("size") or 0) - if symbols and addr in symbols: - category, status, sym_name = symbols[addr] - else: - category = q._category(addr) - status = "decompiled" if (statuses or {}).get(addr, ("raw",))[0] == "readable" else "classified" - sym_name = "" - row = names.get(addr) - name = (row[1] if row and len(row) > 1 and row[1] else "") or sym_name or d.get("name") or f"FUN_{addr:06X}" - counted = category == "game" and status not in EXCLUDED - done = status in DECOMPILED - is_lib = q.is_library(addr, library, lib_ranges) or category in ("library", "platform", "crt") - if counted or (done and not is_lib) or (category == "unsorted" and not is_lib): - tier = "exact" if done and addr in exact else "verified" if done and addr in verified else \ - "readable" if done else "deferred" if addr in deferred else \ - "raw" if counted else "unsorted" - elif is_lib: - tier = "library" - else: - tier = "engine" - - source = sources.get(addr) - in_range = next((label for lo, hi, label in ranges if lo <= addr < hi), None) - if source: - unit = source - elif in_range: - unit = in_range - elif d.get("unit"): - unit = d["unit"] - else: - unit = f"{category} · 0x{addr & ~0xFFFF:06X} block (no source file)" - if unit not in unit_index: - unit_index[unit] = len(units) - units.append({"name": unit, "lo": addr, "hi": addr + size, "bytes": 0, "counted": 0, "exact": 0, - "verified": 0, "readable": 0, "functions": 0, "kinds": {}}) - u = units[unit_index[unit]] - u["hi"] = max(u["hi"], addr + size) - u["bytes"] += size - u["functions"] += 1 - kind = "game" if counted else "library" if tier == "library" else "engine" if category == "engine" \ - else "unsorted" if category == "unsorted" else "library" - u["kinds"][kind] = u["kinds"].get(kind, 0) + size - if counted: - game_functions += 1 - game_done += done - u["counted"] += size - if tier in ("exact", "verified", "readable"): - u[tier] += size - functions.append([unit_index[unit], addr, size, STATUSES.index(tier), name]) - - for u in units: - kinds = u.pop("kinds") - u["kind"] = max(kinds, key=kinds.get) if kinds else "unsorted" # what most of its bytes are - - code = {k: sum(u[k] for u in units) for k in ("exact", "verified", "readable")} - code["bytes"] = sum(u["counted"] for u in units) - code["functions"], code["decompiled_functions"] = game_functions, game_done - percent = round(100 * (code["exact"] + code["verified"] + code["readable"]) / code["bytes"], 2) \ - if code["bytes"] else 0.0 - - from . import logbook - stats = logbook._read(self.project, "stats.json", None) or {} - history = sorted(logbook._read(self.project, "history.json", []), key=lambda p: p.get("at", "")) - decomp = self._decomp() - commit = {"sha": stats.get("commit"), "at": None} - if commit["sha"] and decomp: - commit["at"] = _utc(_git(decomp, "log", "-1", "--format=%cI", commit["sha"])) or None - main = {"sha": head[:7] if head else None, - "at": _utc(_git(decomp, "log", "-1", "--format=%cI", head)) if head else None} - from . import results_refresh - return { - "project": self.project.title or self.project.name, - "results": results_refresh.freshness(self.project), - "source": "logbook symbol table" if symbols else "src headers and names.csv", - "commit": commit, "main": main, "updated_at": stats.get("updated_at"), - "percent": percent, "logbook_percent": stats.get("game_bytes_decompiled_percent"), - "code": code, "data": self._data(facts, rev), - "history": [[p.get("at"), p.get("decompiled"), p.get("commit"), p.get("game_bytes")] - for p in history if p.get("decompiled") is not None], - "statuses": list(STATUSES), - "unit_fields": list(UNIT_FIELDS), - "units": [[u[k] for k in UNIT_FIELDS] for u in units], - "function_fields": list(FUNCTION_FIELDS), - "functions": functions, - } - - -_MAPS = {} -_LOCK = threading.Lock() - - -def map_for(project): - with _LOCK: - if project.name not in _MAPS: - from . import decomp_api - api = decomp_api.api_for(project) - _MAPS[project.name] = ProgressMap(project, lambda: FunctionQueue(api.h)) - return _MAPS[project.name] - - -def handle(project, query): - if not (project.reverse or {}).get("facts"): - return 404, {"error": "this project has no [reverse] table"} - return 200, map_for(project).get(query.get("have")) +"""The Logbook's progress map, in the style of decomp.dev: one unit per original source file, its functions and how +far each has come, plus the headline share of game code decompiled and the Code and Data bars. + +GET /api/logbook/progress-map answers with one compact JSON document. It is computed once and kept until one of its +inputs changes (main's HEAD, the Logbook's symbol table and statistics, the match, emulator and library results, +facts, names, source files, globals, or a deferral note); `?have=` answers {"version", "same": true} when +the caller already holds the current one, so a page that polls does not download or re-lay-out anything. + +Definitions follow the Logbook exactly (open-seas tools/progress/progress.py, which writes stats.json): + - the symbol table is <[logbook] stats_repo>/re/symbols/pc102.csv: its category and status per function; + - counted ("game code") = category game and status not superseded or wontfix; + - decompiled = status decompiled, reimplemented or verified, so percent = decompiled game bytes / game bytes, + the Logbook's game_bytes_decompiled_percent. +Decompiled functions are split further, best first: exact (byte match in the [reverse] match_results), verified +(behaves the same in the emulator, diffrun_results, not 'weak'), readable. The rest: deferred (a live DEFERRED note), +raw, and, not counted, unsorted, engine (Gamebryo, named only) and library (C runtime, D3DX9, platform wrappers). +"results" says which commit the exact and verified tiers are from (harbor.results_refresh keeps them current). +Without a stats_repo the categories come from names.csv / facts and readable from the src headers (the queue's view). + +Units, first that applies: the source file from the Xbox build's assert sites ([reverse] source_files); the D3DX9 block +(platform/d3dx9) or the import-thunk stubs from library_ranges; the unit facts gives (engine/, +platform/); otherwise + plus its 64 KB address block, labelled "(no source file)".""" +import csv +import hashlib +import json +import subprocess +import threading +import time +from pathlib import Path + +from .queue import FunctionQueue + +STATUSES = ("exact", "verified", "readable", "deferred", "raw", "unsorted", "engine", "library") +DECOMPILED = ("decompiled", "reimplemented", "verified") +EXCLUDED = ("superseded", "wontfix") +UNIT_FIELDS = ("name", "kind", "lo", "hi", "bytes", "counted", "exact", "verified", "readable", "functions") +FUNCTION_FIELDS = ("unit", "address", "size", "status", "name") +KEY_TTL = 15 # seconds a computed input key is trusted before the files and HEAD are looked at again + + +def _int(text): + return int(str(text), 16) if str(text).lower().startswith("0x") else int(text) + + +def _mtime(path): + try: + return Path(path).stat().st_mtime_ns if path else None + except OSError: + return None + + +def _git(repo, *args): + try: + out = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, timeout=20, + encoding="utf-8", errors="replace") + except (OSError, subprocess.TimeoutExpired): + return "" + return out.stdout.strip() if out.returncode == 0 else "" + + +def _utc(iso): + import datetime + try: + return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + except ValueError: + return None + + +class ProgressMap: + def __init__(self, project, queue_factory): + self.project = project + self.queue_factory = queue_factory # -> FunctionQueue on a connection this thread may use + self.lock = threading.Lock() + self.value = None # (key, document) + self.checked = (0.0, None) # (monotonic time, key) + self.computations = 0 + + # ------------------------------------------------------------------ inputs + def _config(self): + return self.project.reverse or {}, getattr(self.project, "logbook", None) or {} + + def _symbols_path(self): + _, lb = self._config() + return Path(lb["stats_repo"]) / "re" / "symbols" / "pc102.csv" if lb.get("stats_repo") else None + + def _decomp(self): + rev, lb = self._config() + if lb.get("decomp"): + return Path(lb["decomp"]) + return Path(rev["annotations"]).parent if rev.get("annotations") else None + + def _logbook_dir(self): + from . import logbook + return logbook.folder(self.project) + + def key(self, q): + rev, _ = self._config() + decomp = self._decomp() + head = _git(decomp, "rev-parse", "HEAD") if decomp and (decomp / ".git").exists() else "" + paths = [self._symbols_path(), self._logbook_dir() / "stats.json", self._logbook_dir() / "history.json", + self._logbook_dir() / "results.json"] + paths += [rev.get(k) for k in ("facts", "match_results", "diffrun_results", "library_results", + "library_ranges", "source_files")] + if rev.get("annotations"): + paths += [Path(rev["annotations"]) / "names.csv", Path(rev["annotations"]) / "globals.csv"] + try: + note = q.h.conn.execute("SELECT MAX(id) FROM notes").fetchone()[0] + except Exception: + note = None + raw = json.dumps([head, note, [str(p) for p in paths], [_mtime(p) for p in paths]]) + return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16], head + + # ------------------------------------------------------------------ answer + def get(self, have=None): + with self.lock: + q = self.queue_factory() + now = time.monotonic() + if self.value and now - self.checked[0] < KEY_TTL: + key = self.checked[1] + else: + key, head = self.key(q) + self.checked = (now, key) + if not self.value or self.value[0] != key: + document = self.compute(q, head) + if document is None and self.value: # the symbol table is being rewritten: keep the last + document = self.value[1] + elif document is None: + document = self.compute(q, head, use_symbols=False) + document["version"] = key + self.value = (key, document) + self.computations += 1 + if have and have == self.value[0]: + return {"version": have, "same": True} + return self.value[1] + + # ------------------------------------------------------------------ the computation + def _symbols(self, expected): + """pc102.csv as {address: (category, status, name)}, or None while it is missing or half written.""" + path = self._symbols_path() + if not path or not path.exists(): + return None + try: + with path.open(encoding="utf-8", newline="") as stream: + rows = {_int(r["address"]): (r["category"], r["status"], r.get("name") or "") + for r in csv.DictReader(stream)} + except (OSError, ValueError, KeyError): + return None + return rows if len(rows) >= 0.98 * expected else None + + def _source_files(self, path): + out = {} + if path and Path(path).exists(): + with open(path, encoding="utf-8", newline="") as stream: + for row in csv.DictReader(stream): + if row.get("source_file"): + out[_int(row["pc_address"])] = row["source_file"] + return out + + def _all_ranges(self, path): + """[(lo, hi, label)] of the library_ranges csv (the D3DX9 block and the compiler's stubs).""" + out = [] + if path and Path(path).exists(): + with open(path, encoding="utf-8", newline="") as stream: + for r in csv.DictReader(stream): + verdict = r.get("verdict", "") + label = "platform/d3dx9" if verdict.startswith("D3DX") else \ + "Import thunks and unwind funclets" if "stubs" in verdict else None + if label: + out.append((_int(r["start"]), _int(r["end"]), label)) + return out + + def _data(self, facts, rev): + """Named globals against the distinct .data/.rdata addresses the code touches (needs sections.json).""" + facts_path = Path(rev["facts"]) if rev.get("facts") else None + sections_path = facts_path.parent / "sections.json" if facts_path else None + globals_path = Path(rev["annotations"]) / "globals.csv" if rev.get("annotations") else None + if not sections_path or not sections_path.exists() or not globals_path or not globals_path.exists(): + return None + spans = [(s["start"], s["end"]) for s in json.loads(sections_path.read_text(encoding="utf-8")) + if s.get("name") in (".data", ".rdata")] + touched = set() + for d in facts.values(): + for ref in d.get("data", []): + a = _int(ref) + if any(lo <= a < hi for lo, hi in spans): + touched.add(a) + with globals_path.open(encoding="utf-8", newline="") as stream: + named = {_int(r["address"]) for r in csv.DictReader(line for line in stream if not line.startswith("#")) + if r.get("address") and r.get("name")} + if not touched: + return None + return {"addresses": len(touched), "named": len(touched & named), "globals": len(named), + "bytes": sum(hi - lo for lo, hi in spans)} + + def compute(self, q, head, use_symbols=True): + rev, _ = self._config() + facts = q.r._facts() + symbols = self._symbols(len(facts)) if use_symbols else None + if use_symbols and self._symbols_path() and symbols is None and self._symbols_path().exists(): + return None + names = q.r._csv("names.csv") + statuses = None if symbols else q.statuses() + exact = {a for a, row in q._results("match_results").items() + if str(row.get("status")) in ("match", "exact", "exact-relaxed")} + verified = {a for a, row in q._results("diffrun_results").items() + if str(row.get("verdict")) == "same" and not row.get("weak")} + library = set(q._results("library_results")) + ranges = self._all_ranges(rev.get("library_ranges")) + lib_ranges = [(lo, hi) for lo, hi, _ in ranges] + deferred = q.deferrals() + sources = self._source_files(rev.get("source_files")) + + units, unit_index, functions = [], {}, [] + game_functions = game_done = 0 + for addr in sorted(facts): + d = facts[addr] + size = int(d.get("size") or 0) + if symbols and addr in symbols: + category, status, sym_name = symbols[addr] + else: + category = q._category(addr) + status = "decompiled" if (statuses or {}).get(addr, ("raw",))[0] == "readable" else "classified" + sym_name = "" + row = names.get(addr) + name = (row[1] if row and len(row) > 1 and row[1] else "") or sym_name or d.get("name") or f"FUN_{addr:06X}" + counted = category == "game" and status not in EXCLUDED + done = status in DECOMPILED + is_lib = q.is_library(addr, library, lib_ranges) or category in ("library", "platform", "crt") + if counted or (done and not is_lib) or (category == "unsorted" and not is_lib): + tier = "exact" if done and addr in exact else "verified" if done and addr in verified else \ + "readable" if done else "deferred" if addr in deferred else \ + "raw" if counted else "unsorted" + elif is_lib: + tier = "library" + else: + tier = "engine" + + source = sources.get(addr) + in_range = next((label for lo, hi, label in ranges if lo <= addr < hi), None) + if source: + unit = source + elif in_range: + unit = in_range + elif d.get("unit"): + unit = d["unit"] + else: + unit = f"{category} · 0x{addr & ~0xFFFF:06X} block (no source file)" + if unit not in unit_index: + unit_index[unit] = len(units) + units.append({"name": unit, "lo": addr, "hi": addr + size, "bytes": 0, "counted": 0, "exact": 0, + "verified": 0, "readable": 0, "functions": 0, "kinds": {}}) + u = units[unit_index[unit]] + u["hi"] = max(u["hi"], addr + size) + u["bytes"] += size + u["functions"] += 1 + kind = "game" if counted else "library" if tier == "library" else "engine" if category == "engine" \ + else "unsorted" if category == "unsorted" else "library" + u["kinds"][kind] = u["kinds"].get(kind, 0) + size + if counted: + game_functions += 1 + game_done += done + u["counted"] += size + if tier in ("exact", "verified", "readable"): + u[tier] += size + functions.append([unit_index[unit], addr, size, STATUSES.index(tier), name]) + + for u in units: + kinds = u.pop("kinds") + u["kind"] = max(kinds, key=kinds.get) if kinds else "unsorted" # what most of its bytes are + + code = {k: sum(u[k] for u in units) for k in ("exact", "verified", "readable")} + code["bytes"] = sum(u["counted"] for u in units) + code["functions"], code["decompiled_functions"] = game_functions, game_done + percent = round(100 * (code["exact"] + code["verified"] + code["readable"]) / code["bytes"], 2) \ + if code["bytes"] else 0.0 + + from . import logbook + stats = logbook._read(self.project, "stats.json", None) or {} + history = sorted(logbook._read(self.project, "history.json", []), key=lambda p: p.get("at", "")) + decomp = self._decomp() + commit = {"sha": stats.get("commit"), "at": None} + if commit["sha"] and decomp: + commit["at"] = _utc(_git(decomp, "log", "-1", "--format=%cI", commit["sha"])) or None + main = {"sha": head[:7] if head else None, + "at": _utc(_git(decomp, "log", "-1", "--format=%cI", head)) if head else None} + from . import results_refresh + return { + "project": self.project.title or self.project.name, + "results": results_refresh.freshness(self.project), + "source": "logbook symbol table" if symbols else "src headers and names.csv", + "commit": commit, "main": main, "updated_at": stats.get("updated_at"), + "percent": percent, "logbook_percent": stats.get("game_bytes_decompiled_percent"), + "code": code, "data": self._data(facts, rev), + # every function in the executable (game, engine, platform, CRT), so engine work shows too + "all_code": {"bytes": stats.get("bytes"), "functions": stats.get("functions"), + "decompiled_bytes": ((stats.get("by_status") or {}).get("decompiled") or {}).get("bytes"), + "decompiled_functions": ((stats.get("by_status") or {}).get("decompiled") or {}).get("functions")}, + "history": [[p.get("at"), p.get("decompiled"), p.get("commit"), p.get("game_bytes")] + for p in history if p.get("decompiled") is not None], + "statuses": list(STATUSES), + "unit_fields": list(UNIT_FIELDS), + "units": [[u[k] for k in UNIT_FIELDS] for u in units], + "function_fields": list(FUNCTION_FIELDS), + "functions": functions, + } + + +_MAPS = {} +_LOCK = threading.Lock() + + +def map_for(project): + with _LOCK: + if project.name not in _MAPS: + from . import decomp_api + api = decomp_api.api_for(project) + _MAPS[project.name] = ProgressMap(project, lambda: FunctionQueue(api.h)) + return _MAPS[project.name] + + +def handle(project, query): + if not (project.reverse or {}).get("facts"): + return 404, {"error": "this project has no [reverse] table"} + return 200, map_for(project).get(query.get("have")) diff --git a/harbor/web/app.html b/harbor/web/app.html index e7ecb1a..b0e9e33 100644 --- a/harbor/web/app.html +++ b/harbor/web/app.html @@ -1,1639 +1,1639 @@ - - - - - - -Harbor - - - - - - - -
- -
- -
-
- - -

Harbor

-
- - - -
-
- -
- -
-
-
-
-
-
-
Your team’s conversationConnecting…
- -
-
-
- - -
- - -
-
- -
-
Enter to send · Shift + Enter for a new lineCentral time
-
-
- - -
- -
- - -
-

Crew

-
-
-
-

What happened

-
-
-
-
- - -
-
    -
  1. 1
    Open the appSend the agent any message; it reads its brief and shows up as connected.
    2. -
  2. 2
    Roll callAsk it to answer "Here". Works while it is stopped.
    3. -
  3. 3
    Trial runLet one agent work for 15 to 60 minutes; the stop returns by itself.
    4. -
  4. 4
    ResumeWatch the live feed, then let it work for real.
    5. -
-
-

Agents

-
-
-
-
- - -
-
-

New task

-
-
- - - - -
-
Bulk add: one task per line ("0x445310 RouteCellCost" or a plain title); addresses go to their band owner -
-
- -
-
-
-
-

Queue

-
-
-
-
- - -
-
-

Review queue

-
-
-
-
-

Waiting for acknowledgement

-
-
-
-

Claims

-
-
-
-
- - -
-
-

Team pulse

tool calls and messages, 5-minute steps -
-
-
-
-
-

Live feed

the latest tool calls, from the hooks
-
    -
    -
    -

    Scoreboard

    -

    Landed in the last 24 hours

      -
      -
      -
      - - -
      -
      -
      -
      - -
      A scope like fn:0x445310 shows everything known about it: claims, tasks, reviews, notes and experts.
      -
      -
      -
      -
      -
      -

      Pinned and decisions

      -
      -
      -
      -

      Threads

      -
        -
        -
        -
        - - -
        -
        -
        Milestone -

        Decompiling Sid Meier’s Pirates! (2004) into readable C, then rebuilding it in Rust, charted function by function against the PC 1.0.2 executable.

        -
        Loading…
        -
        -
        -

        The project is – decompiled

        -
        -
        -
        -
        -
        -
        -
        -
        -
        -
        exact matchverified in the emulatorreadablerawdeferredunsorted (not counted)engine and libraries: named, not decompiled (not counted)
        -
        A unit is an original source file (from the Xbox build’s asserts) or, without one, a 64 KB address block. Area is code bytes; a unit’s green is the share of its game-code bytes decompiled. Click a unit to see its functions.
        - -
        -
        -
        -
        -

        Chart of the executable

        bytes per category along the code addresses
        -
        -
        0x401000code address →
        -
        -
        -
        -
        -

        Where the code stands

        -
        -
        -
        -

        Trend

        -
        -
        classifiedgame code decompileddone (functions handled)
        -
        -
        -
        -
        -
        -

        Milestones

        -
        -
        -
        -

        Ship’s log

        -
          -
          -
          -
          - - -
          -
          -

          Progress by function

          -
          -
          -
          -
          -

          Inspect a function

          original name, source file, convention, match and near-duplicates
          -
          - -
          -
          -
          -
          -
          -

          Next up in each band

          what next_functions hands out, easiest first
          -
          -
          -
          -

          Deferred

          -
          -
          -
          -
          - - -
          -
          -

          Notifications

          -
          -
          -
          -

          Reverse-engineering tools

          -
          -
          -
          -

          Health and connections

          -
          -
          -
          -
          -
          -
          -
          - - -
          -

          Your availability

          -
          -
          - - - -
          - - - - -
          -
          Agents see this at the top of every brief.
          -
          -           - -

          Restart prompt

          -
          -
          Paste this into a fresh session for the agent.
          -           - -

          Thread

          - - - -
          -
          -
          - -
          - - -
          -
          -           - - -

          While you were away

          -
          -
          -           - - - - - + + + + + + +Harbor + + + + + + + +
          + +
          + +
          +
          + + +

          Harbor

          +
          + + + +
          +
          + +
          + +
          +
          +
          +
          +
          +
          +
          Your team’s conversationConnecting…
          + +
          +
          +
          + + +
          + + +
          +
          + +
          +
          Enter to send · Shift + Enter for a new lineCentral time
          +
          +
          + + +
          + +
          + + +
          +

          Crew

          +
          +
          +
          +

          What happened

          +
          +
          +
          +
          + + +
          +
            +
          1. 1
            Open the appSend the agent any message; it reads its brief and shows up as connected.
            2. +
          2. 2
            Roll callAsk it to answer "Here". Works while it is stopped.
            3. +
          3. 3
            Trial runLet one agent work for 15 to 60 minutes; the stop returns by itself.
            4. +
          4. 4
            ResumeWatch the live feed, then let it work for real.
            5. +
          +
          +

          Agents

          +
          +
          +
          +
          + + +
          +
          +

          New task

          +
          +
          + + + + +
          +
          Bulk add: one task per line ("0x445310 RouteCellCost" or a plain title); addresses go to their band owner +
          +
          + +
          +
          +
          +
          +

          Queue

          +
          +
          +
          +
          + + +
          +
          +

          Review queue

          +
          +
          +
          +
          +

          Waiting for acknowledgement

          +
          +
          +
          +

          Claims

          +
          +
          +
          +
          + + +
          +
          +

          Team pulse

          tool calls and messages, 5-minute steps +
          +
          +
          +
          +
          +

          Live feed

          the latest tool calls, from the hooks
          +
            +
            +
            +

            Scoreboard

            +

            Landed in the last 24 hours

              +
              +
              +
              + + +
              +
              +
              +
              + +
              A scope like fn:0x445310 shows everything known about it: claims, tasks, reviews, notes and experts.
              +
              +
              +
              +
              +
              +

              Pinned and decisions

              +
              +
              +
              +

              Threads

              +
                +
                +
                +
                + + +
                +
                +
                Milestone +

                Decompiling Sid Meier’s Pirates! (2004) into readable C, then rebuilding it in Rust, charted function by function against the PC 1.0.2 executable.

                +
                Loading…
                +
                +
                +

                The project is – decompiled

                +
                +
                +
                +
                +
                +
                +
                +
                +
                +
                exact matchverified in the emulatorreadablerawdeferredunsorted (not counted)engine and libraries: named, not decompiled (not counted)
                +
                A unit is an original source file (from the Xbox build’s asserts) or, without one, a 64 KB address block. Area is code bytes; a unit’s green is the share of its game-code bytes decompiled. Click a unit to see its functions.
                + +
                +
                +
                +
                +

                Chart of the executable

                bytes per category along the code addresses
                +
                +
                0x401000code address →
                +
                +
                +
                +
                +

                Where the code stands

                +
                +
                +
                +

                Trend

                +
                +
                classifiedgame code decompileddone (functions handled)
                +
                +
                +
                +
                +
                +

                Milestones

                +
                +
                +
                +

                Ship’s log

                +
                  +
                  +
                  +
                  + + +
                  +
                  +

                  Progress by function

                  +
                  +
                  +
                  +
                  +

                  Inspect a function

                  original name, source file, convention, match and near-duplicates
                  +
                  + +
                  +
                  +
                  +
                  +
                  +

                  Next up in each band

                  what next_functions hands out, easiest first
                  +
                  +
                  +
                  +

                  Deferred

                  +
                  +
                  +
                  +
                  + + +
                  +
                  +

                  Notifications

                  +
                  +
                  +
                  +

                  Reverse-engineering tools

                  +
                  +
                  +
                  +

                  Health and connections

                  +
                  +
                  +
                  +
                  +
                  +
                  +
                  + + +
                  +

                  Your availability

                  +
                  +
                  + + + +
                  + + + + +
                  +
                  Agents see this at the top of every brief.
                  +
                  +                   + +

                  Restart prompt

                  +
                  +
                  Paste this into a fresh session for the agent.
                  +                   + +

                  Thread

                  + + + +
                  +
                  +
                  + +
                  + + +
                  +
                  +                   + + +

                  While you were away

                  +
                  +
                  +                   + + + + +