From d5eb82261eadee70f2f550f87e826878d94271d6 Mon Sep 17 00:00:00 2001 From: KeelrLeek <156356974+KeelrLeek@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:46:42 -0500 Subject: [PATCH] Logbook progress map in the style of decomp.dev: " is NN.NN% decompiled", a layered Code bar (exact solid, verified lighter, readable hatched, with per-segment tooltips), a Data bar (named globals of the data addresses the code touches), Previous / Next / Latest through the history reports, a Units dropdown and filter (text, <70%, >10kb), and a canvas treemap of units (original source files, else 64 KB blocks) that drills into functions - GET /api/logbook/progress-map: one compact document, cached until an input changes (main's HEAD, the Logbook symbol table and statistics, match/emulator/library results, facts, names, source files, globals, deferral notes); ?have= answers {"same": true} - definitions follow the Logbook's progress.py (pc102.csv category game, status decompiled or later), so the headline equals game_bytes_decompiled_percent; engine, library and unsorted code are shown but not counted - history points now record the commit and the game-code bytes - the page lays out once per load, filter or size change, hit-tests hover on the computed rectangles, builds the 600-entry units list only when opened, and polls once a minute only while the Logbook is visible --- harbor/logbook.py | 7 +- harbor/progress_map.py | 317 +++++++++++++++++++++++++++++++++++++++++ harbor/server.py | 5 + harbor/web/app.html | 150 ++++++++++++++++++- tests/test_logbook.py | 81 ++++++++++- 5 files changed, 556 insertions(+), 4 deletions(-) create mode 100644 harbor/progress_map.py diff --git a/harbor/logbook.py b/harbor/logbook.py index 5c7f4b3..6a0433f 100644 --- a/harbor/logbook.py +++ b/harbor/logbook.py @@ -4,7 +4,8 @@ executable, the trend, milestones and the ship's log. (It replaced a separately One JSON file per collection in the project's logbook folder ([logbook] dir, default /logbook/): stats.json the latest statistics (functions, bytes, by_category, percentages, commit, updated_at) map.json the chart of the executable (low, high, cells: "game,engine,platform,crt,unsorted,done" per cell) - history.json [{at, classified, decompiled, weighted, verified}] one point per statistics report + history.json [{at, classified, done, decompiled, weighted, verified, commit, game_bytes}] one point per + statistics report (commit and game_bytes since 2026-10-06) milestones.json [{id, code, title, phase, state, order, items: [{text, done}]}] log.json [{id, at, kind, title, detail}] the ship's log, newest last Writes replace the file atomically. Nothing here talks to the network.""" @@ -93,7 +94,9 @@ def set_stats(project, stats, history_point=True): "done": round(100 * handled / total, 2) if total else 0, "decompiled": stats.get("game_bytes_decompiled_percent", 0), "weighted": stats.get("weighted_percent", 0), - "verified": stats.get("game_bytes_verified_percent", 0)}) + "verified": stats.get("game_bytes_verified_percent", 0), + **({"commit": stats["commit"]} if stats.get("commit") else {}), + **({"game_bytes": cats["game"]["bytes"]} if (cats.get("game") or {}).get("bytes") else {})}) _write(project, "history.json", history) return stats diff --git a/harbor/progress_map.py b/harbor/progress_map.py new file mode 100644 index 0000000..e182415 --- /dev/null +++ b/harbor/progress_map.py @@ -0,0 +1,317 @@ +"""The Logbook's progress map, in the style of decomp.dev: one unit per original source file, its functions and how +far each has come, plus the headline share of game code decompiled and the Code and Data bars. + +GET /api/logbook/progress-map answers with one compact JSON document. It is computed once and kept until one of its +inputs changes (main's HEAD, the Logbook's symbol table and statistics, the match, emulator and library results, +facts, names, source files, globals, or a deferral note); `?have=` answers {"version", "same": true} when +the caller already holds the current one, so a page that polls does not download or re-lay-out anything. + +Definitions follow the Logbook exactly (open-seas tools/progress/progress.py, which writes stats.json): + - the symbol table is <[logbook] stats_repo>/re/symbols/pc102.csv: its category and status per function; + - counted ("game code") = category game and status not superseded or wontfix; + - decompiled = status decompiled, reimplemented or verified, so percent = decompiled game bytes / game bytes, + the Logbook's game_bytes_decompiled_percent. +Decompiled functions are split further, best first: exact (byte match in the [reverse] match_results), verified +(behaves the same in the emulator, diffrun_results, not 'weak'), readable. The rest: deferred (a live DEFERRED note), +raw, and, not counted, unsorted, engine (Gamebryo, named only) and library (C runtime, D3DX9, platform wrappers). +Without a stats_repo the categories come from names.csv / facts and readable from the src headers (the queue's view). + +Units, first that applies: the source file from the Xbox build's assert sites ([reverse] source_files); the D3DX9 block +(platform/d3dx9) or the import-thunk stubs from library_ranges; the unit facts gives (engine/, +platform/); otherwise + plus its 64 KB address block, labelled "(no source file)".""" +import csv +import hashlib +import json +import subprocess +import threading +import time +from pathlib import Path + +from .queue import FunctionQueue + +STATUSES = ("exact", "verified", "readable", "deferred", "raw", "unsorted", "engine", "library") +DECOMPILED = ("decompiled", "reimplemented", "verified") +EXCLUDED = ("superseded", "wontfix") +UNIT_FIELDS = ("name", "kind", "lo", "hi", "bytes", "counted", "exact", "verified", "readable", "functions") +FUNCTION_FIELDS = ("unit", "address", "size", "status", "name") +KEY_TTL = 15 # seconds a computed input key is trusted before the files and HEAD are looked at again + + +def _int(text): + return int(str(text), 16) if str(text).lower().startswith("0x") else int(text) + + +def _mtime(path): + try: + return Path(path).stat().st_mtime_ns if path else None + except OSError: + return None + + +def _git(repo, *args): + try: + out = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, timeout=20, + encoding="utf-8", errors="replace") + except (OSError, subprocess.TimeoutExpired): + return "" + return out.stdout.strip() if out.returncode == 0 else "" + + +def _utc(iso): + import datetime + try: + return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + except ValueError: + return None + + +class ProgressMap: + def __init__(self, project, queue_factory): + self.project = project + self.queue_factory = queue_factory # -> FunctionQueue on a connection this thread may use + self.lock = threading.Lock() + self.value = None # (key, document) + self.checked = (0.0, None) # (monotonic time, key) + self.computations = 0 + + # ------------------------------------------------------------------ inputs + def _config(self): + return self.project.reverse or {}, getattr(self.project, "logbook", None) or {} + + def _symbols_path(self): + _, lb = self._config() + return Path(lb["stats_repo"]) / "re" / "symbols" / "pc102.csv" if lb.get("stats_repo") else None + + def _decomp(self): + rev, lb = self._config() + if lb.get("decomp"): + return Path(lb["decomp"]) + return Path(rev["annotations"]).parent if rev.get("annotations") else None + + def _logbook_dir(self): + from . import logbook + return logbook.folder(self.project) + + def key(self, q): + rev, _ = self._config() + decomp = self._decomp() + head = _git(decomp, "rev-parse", "HEAD") if decomp and (decomp / ".git").exists() else "" + paths = [self._symbols_path(), self._logbook_dir() / "stats.json", self._logbook_dir() / "history.json"] + paths += [rev.get(k) for k in ("facts", "match_results", "diffrun_results", "library_results", + "library_ranges", "source_files")] + if rev.get("annotations"): + paths += [Path(rev["annotations"]) / "names.csv", Path(rev["annotations"]) / "globals.csv"] + try: + note = q.h.conn.execute("SELECT MAX(id) FROM notes").fetchone()[0] + except Exception: + note = None + raw = json.dumps([head, note, [str(p) for p in paths], [_mtime(p) for p in paths]]) + return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16], head + + # ------------------------------------------------------------------ answer + def get(self, have=None): + with self.lock: + q = self.queue_factory() + now = time.monotonic() + if self.value and now - self.checked[0] < KEY_TTL: + key = self.checked[1] + else: + key, head = self.key(q) + self.checked = (now, key) + if not self.value or self.value[0] != key: + document = self.compute(q, head) + if document is None and self.value: # the symbol table is being rewritten: keep the last + document = self.value[1] + elif document is None: + document = self.compute(q, head, use_symbols=False) + document["version"] = key + self.value = (key, document) + self.computations += 1 + if have and have == self.value[0]: + return {"version": have, "same": True} + return self.value[1] + + # ------------------------------------------------------------------ the computation + def _symbols(self, expected): + """pc102.csv as {address: (category, status, name)}, or None while it is missing or half written.""" + path = self._symbols_path() + if not path or not path.exists(): + return None + try: + with path.open(encoding="utf-8", newline="") as stream: + rows = {_int(r["address"]): (r["category"], r["status"], r.get("name") or "") + for r in csv.DictReader(stream)} + except (OSError, ValueError, KeyError): + return None + return rows if len(rows) >= 0.98 * expected else None + + def _source_files(self, path): + out = {} + if path and Path(path).exists(): + with open(path, encoding="utf-8", newline="") as stream: + for row in csv.DictReader(stream): + if row.get("source_file"): + out[_int(row["pc_address"])] = row["source_file"] + return out + + def _all_ranges(self, path): + """[(lo, hi, label)] of the library_ranges csv (the D3DX9 block and the compiler's stubs).""" + out = [] + if path and Path(path).exists(): + with open(path, encoding="utf-8", newline="") as stream: + for r in csv.DictReader(stream): + verdict = r.get("verdict", "") + label = "platform/d3dx9" if verdict.startswith("D3DX") else \ + "Import thunks and unwind funclets" if "stubs" in verdict else None + if label: + out.append((_int(r["start"]), _int(r["end"]), label)) + return out + + def _data(self, facts, rev): + """Named globals against the distinct .data/.rdata addresses the code touches (needs sections.json).""" + facts_path = Path(rev["facts"]) if rev.get("facts") else None + sections_path = facts_path.parent / "sections.json" if facts_path else None + globals_path = Path(rev["annotations"]) / "globals.csv" if rev.get("annotations") else None + if not sections_path or not sections_path.exists() or not globals_path or not globals_path.exists(): + return None + spans = [(s["start"], s["end"]) for s in json.loads(sections_path.read_text(encoding="utf-8")) + if s.get("name") in (".data", ".rdata")] + touched = set() + for d in facts.values(): + for ref in d.get("data", []): + a = _int(ref) + if any(lo <= a < hi for lo, hi in spans): + touched.add(a) + with globals_path.open(encoding="utf-8", newline="") as stream: + named = {_int(r["address"]) for r in csv.DictReader(line for line in stream if not line.startswith("#")) + if r.get("address") and r.get("name")} + if not touched: + return None + return {"addresses": len(touched), "named": len(touched & named), "globals": len(named), + "bytes": sum(hi - lo for lo, hi in spans)} + + def compute(self, q, head, use_symbols=True): + rev, _ = self._config() + facts = q.r._facts() + symbols = self._symbols(len(facts)) if use_symbols else None + if use_symbols and self._symbols_path() and symbols is None and self._symbols_path().exists(): + return None + names = q.r._csv("names.csv") + statuses = None if symbols else q.statuses() + exact = {a for a, row in q._results("match_results").items() + if str(row.get("status")) in ("match", "exact", "exact-relaxed")} + verified = {a for a, row in q._results("diffrun_results").items() + if str(row.get("verdict")) == "same" and not row.get("weak")} + library = set(q._results("library_results")) + ranges = self._all_ranges(rev.get("library_ranges")) + lib_ranges = [(lo, hi) for lo, hi, _ in ranges] + deferred = q.deferrals() + sources = self._source_files(rev.get("source_files")) + + units, unit_index, functions = [], {}, [] + game_functions = game_done = 0 + for addr in sorted(facts): + d = facts[addr] + size = int(d.get("size") or 0) + if symbols and addr in symbols: + category, status, sym_name = symbols[addr] + else: + category = q._category(addr) + status = "decompiled" if (statuses or {}).get(addr, ("raw",))[0] == "readable" else "classified" + sym_name = "" + row = names.get(addr) + name = (row[1] if row and len(row) > 1 and row[1] else "") or sym_name or d.get("name") or f"FUN_{addr:06X}" + counted = category == "game" and status not in EXCLUDED + done = status in DECOMPILED + is_lib = q.is_library(addr, library, lib_ranges) or category in ("library", "platform", "crt") + if counted or (done and not is_lib) or (category == "unsorted" and not is_lib): + tier = "exact" if done and addr in exact else "verified" if done and addr in verified else \ + "readable" if done else "deferred" if addr in deferred else \ + "raw" if counted else "unsorted" + elif is_lib: + tier = "library" + else: + tier = "engine" + + source = sources.get(addr) + in_range = next((label for lo, hi, label in ranges if lo <= addr < hi), None) + if source: + unit = source + elif in_range: + unit = in_range + elif d.get("unit"): + unit = d["unit"] + else: + unit = f"{category} · 0x{addr & ~0xFFFF:06X} block (no source file)" + if unit not in unit_index: + unit_index[unit] = len(units) + units.append({"name": unit, "lo": addr, "hi": addr + size, "bytes": 0, "counted": 0, "exact": 0, + "verified": 0, "readable": 0, "functions": 0, "kinds": {}}) + u = units[unit_index[unit]] + u["hi"] = max(u["hi"], addr + size) + u["bytes"] += size + u["functions"] += 1 + kind = "game" if counted else "library" if tier == "library" else "engine" if category == "engine" \ + else "unsorted" if category == "unsorted" else "library" + u["kinds"][kind] = u["kinds"].get(kind, 0) + size + if counted: + game_functions += 1 + game_done += done + u["counted"] += size + if tier in ("exact", "verified", "readable"): + u[tier] += size + functions.append([unit_index[unit], addr, size, STATUSES.index(tier), name]) + + for u in units: + kinds = u.pop("kinds") + u["kind"] = max(kinds, key=kinds.get) if kinds else "unsorted" # what most of its bytes are + + code = {k: sum(u[k] for u in units) for k in ("exact", "verified", "readable")} + code["bytes"] = sum(u["counted"] for u in units) + code["functions"], code["decompiled_functions"] = game_functions, game_done + percent = round(100 * (code["exact"] + code["verified"] + code["readable"]) / code["bytes"], 2) \ + if code["bytes"] else 0.0 + + from . import logbook + stats = logbook._read(self.project, "stats.json", None) or {} + history = sorted(logbook._read(self.project, "history.json", []), key=lambda p: p.get("at", "")) + decomp = self._decomp() + commit = {"sha": stats.get("commit"), "at": None} + if commit["sha"] and decomp: + commit["at"] = _utc(_git(decomp, "log", "-1", "--format=%cI", commit["sha"])) or None + main = {"sha": head[:7] if head else None, + "at": _utc(_git(decomp, "log", "-1", "--format=%cI", head)) if head else None} + return { + "project": self.project.title or self.project.name, + "source": "logbook symbol table" if symbols else "src headers and names.csv", + "commit": commit, "main": main, "updated_at": stats.get("updated_at"), + "percent": percent, "logbook_percent": stats.get("game_bytes_decompiled_percent"), + "code": code, "data": self._data(facts, rev), + "history": [[p.get("at"), p.get("decompiled"), p.get("commit"), p.get("game_bytes")] + for p in history if p.get("decompiled") is not None], + "statuses": list(STATUSES), + "unit_fields": list(UNIT_FIELDS), + "units": [[u[k] for k in UNIT_FIELDS] for u in units], + "function_fields": list(FUNCTION_FIELDS), + "functions": functions, + } + + +_MAPS = {} +_LOCK = threading.Lock() + + +def map_for(project): + with _LOCK: + if project.name not in _MAPS: + from . import decomp_api + api = decomp_api.api_for(project) + _MAPS[project.name] = ProgressMap(project, lambda: FunctionQueue(api.h)) + return _MAPS[project.name] + + +def handle(project, query): + if not (project.reverse or {}).get("facts"): + return 404, {"error": "this project has no [reverse] table"} + return 200, map_for(project).get(query.get("have")) diff --git a/harbor/server.py b/harbor/server.py index 82f81e1..2586e31 100644 --- a/harbor/server.py +++ b/harbor/server.py @@ -330,6 +330,11 @@ class Handler(http.server.BaseHTTPRequestHandler): "stopped": any(h.stopped(a) for a in h.project.ai_names)}) return self._reply(200, out) h, lock = REGISTRY.get(name) + if path == "/api/logbook/progress-map": + # its own connection and lock, like the Decomp view; cached until an input changes + from . import progress_map + status, body = progress_map.handle(h.project, query) + return self._reply(status, body) if path == "/api/logbook": from . import logbook return self._reply(200, logbook.read_all(h.project, log_limit=min(int(query.get("limit", 80)), 500))) diff --git a/harbor/web/app.html b/harbor/web/app.html index 477dcd5..66f8271 100644 --- a/harbor/web/app.html +++ b/harbor/web/app.html @@ -56,6 +56,25 @@ body { overflow: hidden; } .view { display: none; max-width: 1380px; margin: 0 auto; gap: 20px; } .view.on { display: grid; } .lb { --lb-game: var(--blue); --lb-engine: var(--violet); --lb-platform: var(--amber); --lb-crt: var(--muted); --lb-unsorted: var(--line-2); --lb-done: var(--green); } +/* progress map (decomp.dev style) */ +.lb { --pm-exact: var(--green); --pm-verified: color-mix(in srgb, var(--green) 62%, var(--surface)); --pm-readable: color-mix(in srgb, color-mix(in srgb, var(--green) 52%, #d8c22e) 68%, var(--surface)); + --pm-raw: var(--line-2); --pm-deferred: color-mix(in srgb, var(--line-2) 62%, var(--amber)); --pm-unsorted: color-mix(in srgb, var(--line-2) 50%, var(--surface-3)); + --pm-engine: color-mix(in srgb, var(--blue) 30%, var(--line-2)); --pm-library: color-mix(in srgb, var(--blue) 22%, var(--line-2)); } +.pm-title { margin: 0 0 14px; font: 700 22px/1.25 var(--font-display); letter-spacing: -.4px; } .pm-title b { color: var(--green); font-variant-numeric: tabular-nums; } +.pm-bars { display: grid; gap: 12px; } +.pm-bar-l { display: flex; align-items: baseline; gap: 6px; font-size: 13px; margin-bottom: 5px; flex-wrap: wrap; } .pm-bar-l .grow { flex: 1; } .pm-bar-l .mono { font: 600 12.5px var(--mono); font-variant-numeric: tabular-nums; } +.pm-bar { display: flex; height: 14px; border-radius: 7px; overflow: hidden; background: var(--surface-3); } +.pm-seg { display: block; height: 100%; min-width: 2px; cursor: default; } +.pm-seg.pm-exact { background: var(--pm-exact); } .pm-seg.pm-verified { background: var(--pm-verified); } +.pm-seg.pm-readable { background: repeating-linear-gradient(135deg, var(--pm-readable) 0 4px, color-mix(in srgb, var(--pm-readable) 45%, var(--surface-3)) 4px 8px); } +.pm-seg.pm-named { background: var(--blue); } +.pm-commit { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 10px; margin-top: 12px; font-size: 13px; color: var(--muted); } .pm-commit b { color: var(--text); font-weight: 600; } .pm-commit .grow { flex: 1; } +.pm-tools { display: flex; flex-wrap: wrap; gap: 8px 10px; align-items: center; margin-top: 16px; } .pm-tools select { max-width: min(100%, 340px); } .pm-tools input { flex: 1 1 220px; min-width: 0; } +.pm-crumb { display: flex; flex-wrap: wrap; gap: 6px 10px; align-items: center; margin-top: 12px; font-size: 13px; } .pm-crumb:empty { display: none; } .pm-crumb .pm-bar-row { flex: 1 1 100%; } +.pm-wrap { position: relative; margin-top: 12px; } canvas#pmMap { width: 100%; height: 560px; display: block; border-radius: 9px; cursor: pointer; } +.pm-tip { position: fixed; z-index: 50; pointer-events: none; max-width: 340px; padding: 8px 10px; border-radius: 8px; background: var(--surface); color: var(--text); border: 1px solid var(--line-2); box-shadow: var(--shadow-3); font-size: 12.5px; line-height: 1.45; } +.pm-tip .mono { font-family: var(--mono); font-size: 11.5px; color: var(--muted); } .pm-tip b { overflow-wrap: anywhere; } +@media (max-width: 700px) { canvas#pmMap { height: 380px; } .pm-title { font-size: 19px; } } .lb-head { display: flex; flex-wrap: wrap; align-items: flex-end; justify-content: space-between; gap: 10px 20px; } .lb-head .chip { display: inline-block; font: 600 11.5px/1 var(--font, inherit); letter-spacing: .06em; text-transform: uppercase; color: var(--accent-ink); background: var(--accent-soft); border-radius: 999px; padding: 6px 10px; } .lb-head p { margin: 8px 0 0; color: var(--muted); max-width: 70ch; } @@ -677,6 +696,22 @@ button.chip:hover { color: var(--accent-ink); background: var(--accent-soft); }
Milestone

Decompiling Sid Meier’s Pirates! (2004) into readable C, then rebuilding it in Rust, charted function by function against the PC 1.0.2 executable.

Loading…
+
+
+

The project is – decompiled

+
+
+
+
+
+
+
+
+
exact matchverified in the emulatorreadablerawdeferredunsorted (not counted)engine and libraries: named, not decompiled (not counted)
+
A unit is an original source file (from the Xbox build’s asserts) or, without one, a 64 KB address block. Area is code bytes; a unit’s green is the share of its game-code bytes decompiled. Click a unit to see its functions.
+ +
+

Chart of the executable

bytes per category along the code addresses
@@ -1154,7 +1189,7 @@ async function renderLogbook(){try{LB=await get('api/logbook')}catch(e){put('lbS [fmt(uns),'functions not sorted yet','each one is either game code (to decompile) or engine/library code (to name)']] .map(([v,l,n])=>'
'+esc(v)+'
'+esc(l)+'
'+esc(n)+'
').join('')); put('lbLegend',LB_CATS.map(([k,n])=>''+esc(n)+'').join('')+'decompiled share under each column'); - drawLbMap();renderLbWhere(s);renderLbTrend(LB.history);renderLbMilestones(LB.milestones); + drawLbMap();pmLoad().then(()=>{if(PM.doc)pmLayout()});renderLbWhere(s);renderLbTrend(LB.history);renderLbMilestones(LB.milestones); el('lbLogCount').textContent=LB.log_total?LB.log_total+' entries':''; put('lbLog',LB.log.length?LB.log.map(e=>'
  • '+esc(e.kind)+''+esc(e.title)+'
    '+esc(e.detail)+'
    • ').join(''):'
  • No entries yet.
    • ')} async function renderLbWhere(s){const total=s.bytes||0,cat=k=>(s.by_category||{})[k]||{functions:0,bytes:0},uns=cat('unsorted'),sorted=total?100*(total-uns.bytes)/total:0; @@ -1182,6 +1217,119 @@ function renderLbMilestones(list){let last='';put('lbMilestones',list.length?lis const items=m.items||[],done=items.filter(i=>i.done).length,li=items.map(i=>'
  • '+esc(i.text)+'
    • ').join(''),open=/progress/i.test(m.state||''); return head+'

    '+esc(m.code)+''+esc(m.title)+''+done+'/'+items.length+(m.state?' · '+esc(m.state):'')+'

    '+(open?'
      '+li+'
    ':'
    Show '+items.length+' steps
      '+li+'
    ')+'
    '}).join(''):empty('flag','No milestones yet',''))} window.addEventListener('resize',()=>{if(current==='logbook')drawLbMap()}); +/* progress map (decomp.dev style): one compact document per change, laid out once per load, filter or resize, drawn on a canvas */ +const PM={doc:null,version:null,unit:null,hist:null,rects:[],items:[],hover:-1,base:null,colors:null,busy:false,topFilter:''}; +const PM_LABEL={exact:'exact match',verified:'verified in the emulator',readable:'readable',deferred:'deferred',raw:'raw',unsorted:'unsorted (not counted)',engine:'engine: named, not decompiled',library:'library: named, not decompiled'}; +const PM_KIND={game:'game code',unsorted:'unsorted',engine:'engine',library:'library'}; +const kB=b=>b>=1e6?(b/1e6).toFixed(2)+' MB':(b/1e3).toFixed(2)+' kB'; +const hx=a=>'0x'+Number(a).toString(16).toUpperCase(); +const pc=(b,t)=>(t?100*b/t:0).toFixed(2); +function rel(iso){const t=Date.parse(iso||'');if(!t)return 'at an unknown time';const s=(Date.now()-t)/1000;if(s<90)return 'just now';const m=s/60;if(m<90)return Math.round(m)+' minutes ago';const h=m/60;if(h<36)return Math.round(h)+' hours ago';return Math.round(h/24)+' days ago'} +async function pmLoad(){if(PM.busy)return;PM.busy=true;try{ + const d=await get('api/logbook/progress-map'+(PM.version?'?have='+encodeURIComponent(PM.version):''));if(d.same)return; + const U=d.unit_fields;d.units=d.units.map((u,i)=>{const o={i,fns:[]};U.forEach((k,j)=>o[k]=u[j]);o.pct=o.counted?100*(o.exact+o.verified+o.readable)/o.counted:null;o.lname=o.name.toLowerCase();return o}); + d.functions.forEach(f=>{const o={unit:f[0],address:f[1],size:f[2],status:d.statuses[f[3]],name:f[4]};o.lname=o.name.toLowerCase();d.units[o.unit].fns.push(o)});delete d.functions; + PM.doc=d;PM.version=d.version;if(PM.unit!=null&&!d.units[PM.unit])PM.unit=null; + PM.latest=(()=>{const h=d.history;for(let i=h.length-1;i>=0;i--)if(h[i][0]===d.updated_at)return i;return h.length})(); + pmUnitsSelect();pmHeader();pmLayout()} + catch(e){put('pmBars','
    '+esc(e.message)+'
    ')}finally{PM.busy=false}} +function pmBar(label,size,segs,total,extra){const sum=segs.reduce((a,s)=>a+s[1],0); + return '
    '+esc(label)+'('+esc(size)+')'+(extra?''+esc(extra)+'':'')+''+pc(sum,total)+'%
    ' + +'
    '} +function pmCodeSegs(x,total){return [['pm-exact',x.exact,pc(x.exact,total)+'% perfect match ('+kB(x.exact)+')'], + ['pm-verified',x.verified,pc(x.verified,total)+'% decompiled, verified in the emulator, not exact ('+kB(x.verified)+')'], + ['pm-readable',x.readable,pc(x.readable,total)+'% decompiled, readable only ('+kB(x.readable)+')']]} +function pmHeader(){const d=PM.doc;if(!d)return;const h=d.history,pt=PM.hist!=null?h[PM.hist]:null,c=d.code; + el('pmProject').textContent=d.project; + if(!pt){el('pmPct').textContent=d.percent.toFixed(2)+'%'; + put('pmBars',pmBar('Code',kB(c.bytes),pmCodeSegs(c,c.bytes),c.bytes,'game code · '+fmt(c.decompiled_functions)+' of '+fmt(c.functions)+' functions') + +(d.data?pmBar('Data',fmt(d.data.addresses)+' addresses',[['pm-named',d.data.named,pc(d.data.named,d.data.addresses)+'% of the data addresses the code touches have a name in globals.csv ('+fmt(d.data.named)+' of '+fmt(d.data.addresses)+')']],d.data.addresses,'globals named'):'')); + const at=d.commit.at||d.updated_at,moved=d.main.sha&&d.commit.sha&&d.main.sha.slice(0,7)!==String(d.commit.sha).slice(0,7); + put('pmCommit','Updated '+esc(rel(at))+' in commit '+esc(d.commit.sha||'–')+''+(moved?' · main has moved on to '+esc(d.main.sha)+'; the statistics follow within minutes':'')); + el('pmHistNote').textContent=d.percent!==d.logbook_percent&&d.logbook_percent!=null?'The Logbook statistics say '+d.logbook_percent+'%; they are being rebuilt.':''} + else{const p=pt[1],gb=pt[3];el('pmPct').textContent=Number(p).toFixed(2)+'%'; + put('pmBars',pmBar('Code',gb?kB(gb):'size not recorded',[['pm-readable',p,Number(p).toFixed(2)+'% decompiled'+(gb?' ('+kB(gb*p/100)+')':'')+'; the split into exact, verified and readable was not recorded']],100,'game code')); + put('pmCommit','Recorded '+esc(rel(pt[0]))+''+(pt[2]?' in commit '+esc(pt[2])+'':' · commit not recorded')+' · report '+(PM.hist+1)+' of '+h.length+''); + el('pmHistNote').textContent='An earlier report: the headline and the Code bar are from then. The map and the Data bar show Latest, since per-function history is not stored.'} + el('pmPrev').disabled=!h.length||PM.hist===0||(PM.hist==null&&PM.latest<1);el('pmNext').disabled=pt==null;el('pmLatest').disabled=pt==null} +function pmStep(k){const d=PM.doc;if(!d)return;const L=PM.latest;let i=PM.hist==null?L:PM.hist; + i=k==='latest'?L:k==='prev'?Math.max(0,i-1):i+1;PM.hist=i>=L?null:i;pmHeader()} +function pmFilter(){const conds=[],words=[],cmp=(op,a,b)=>op==='<'?a'?a>b:a>=b; + for(const t of (el('pmFilter').value||'').trim().toLowerCase().split(/\s+/).filter(Boolean)){let m; + if((m=t.match(/^([<>]=?)(\d+(?:\.\d+)?)%$/)))conds.push(['pct',m[1],+m[2]]); + else if((m=t.match(/^([<>]=?)(\d+(?:\.\d+)?)(b|k|kb|m|mb)?$/)))conds.push(['size',m[1],+m[2]*({b:1,k:1e3,kb:1e3,m:1e6,mb:1e6}[m[3]||'b'])]); + else words.push(t)} + return (name,kind,pct,size,more)=>conds.every(([k,op,v])=>k==='pct'?pct!=null&&cmp(op,pct,v):cmp(op,size,v))&&words.every(w=>name.includes(w)||kind===w||(more?more(w):false))} +function pmLayout(){const d=PM.doc,c=el('pmMap');if(!d||!c||!c.clientWidth)return;const test=pmFilter();let items,of; + if(PM.unit==null){items=d.units.filter(u=>u.bytes>0&&test(u.lname,u.kind,u.pct,u.bytes,w=>u.fns.some(f=>f.lname.includes(w))));of=d.units.length+' units'} + else{const u=d.units[PM.unit];items=u.fns.filter(f=>f.size>0&&test(f.lname,f.status,['exact','verified','readable'].includes(f.status)?100:['engine','library'].includes(f.status)?null:0,f.size));of=u.fns.length+' functions'} + const sz=x=>x.fns?x.bytes:x.size;items.sort((a,b)=>sz(b)-sz(a)); + PM.items=items;PM.w=c.clientWidth;PM.h=c.clientHeight;PM.rects=squarify(items.map(sz),0,0,PM.w,PM.h);PM.hover=-1; + el('pmCount').textContent=items.length+' of '+of+' · '+kB(items.reduce((a,x)=>a+sz(x),0));pmCrumb();pmPaint()} +function squarify(vals,x,y,w,h){const out=new Array(vals.length),total=vals.reduce((a,b)=>a+b,0);if(!total||w<=0||h<=0)return out; + const scale=w*h/total,worst=(s,mn,mx,side)=>Math.max(side*side*mx/(s*s),s*s/(side*side*mn));let i=0; + while(ibest)break;sum=s2;mn=m2;mx=M2;best=w2;j++} + if(w>=h){const cw=sum/h;let yy=y;for(let k=i;k{probe.style.color='var('+v+')';const s=getComputedStyle(probe).color;g.clearRect(0,0,1,1);g.fillStyle='#808080';g.fillStyle=s;g.fillRect(0,0,1,1);const p=g.getImageData(0,0,1,1).data;return [p[0],p[1],p[2]]}; + const C={};['exact','verified','readable','deferred','raw','unsorted','engine','library'].forEach(k=>C[k]=rgb('--pm-'+k));C.bg=rgb('--surface');C.text=rgb('--text');probe.remove(); + return PM.colors=C} +const pmCss=a=>'rgb('+a.map(v=>Math.round(v)).join(',')+')'; +function pmFill(it,C){if(!it.fns)return C[it.status];if(it.kind!=='game'||it.pct==null)return C[it.kind==='unsorted'?'unsorted':it.kind==='engine'?'engine':'library']; + const t=it.pct<=0?0:.18+.82*it.pct/100;return C.raw.map((v,k)=>v+(C.exact[k]-v)*t)} +function pmPaint(){const c=el('pmMap');if(!c||!c.clientWidth)return;const r=window.devicePixelRatio||1,W=c.clientWidth,H=c.clientHeight,C=pmColors(); + c.width=Math.round(W*r);c.height=Math.round(H*r);const off=PM.base||(PM.base=document.createElement('canvas'));off.width=c.width;off.height=c.height; + const g=off.getContext('2d');g.setTransform(r,0,0,r,0,0);g.fillStyle=pmCss(C.bg);g.fillRect(0,0,W,H);g.textBaseline='top'; + const font=getComputedStyle(c).fontFamily; + PM.items.forEach((it,i)=>{const q=PM.rects[i];if(!q||q[2]<.5||q[3]<.5)return;const [x,y,w,h]=q,f=pmFill(it,C);g.fillStyle=pmCss(f);g.fillRect(x+.5,y+.5,Math.max(.5,w-1),Math.max(.5,h-1)); + if(w<44||h<17)return;const light=(.299*f[0]+.587*f[1]+.114*f[2])>150;g.fillStyle=light?'rgba(0,0,0,.82)':'rgba(255,255,255,.92)'; + g.save();g.beginPath();g.rect(x+1,y+1,w-2,h-2);g.clip();g.font='600 11.5px '+font;g.fillText(it.fns?it.name.split('/').pop():it.name,x+5,y+4); + if(h>32){g.font='11px '+font;g.globalAlpha=.8;g.fillText(it.fns?(it.pct!=null&&it.kind==='game'?it.pct.toFixed(1)+'%':PM_KIND[it.kind])+' · '+kB(it.bytes):PM_LABEL[it.status].split(':')[0]+' · '+it.size+' B',x+5,y+18)}g.restore()}); + pmBlit()} +function pmBlit(){const c=el('pmMap');if(!PM.base||!c.width)return;const g=c.getContext('2d'),r=window.devicePixelRatio||1;g.setTransform(1,0,0,1,0,0);g.drawImage(PM.base,0,0); + const q=PM.rects[PM.hover];if(q){g.setTransform(r,0,0,r,0,0);g.strokeStyle=pmCss(pmColors().text);g.lineWidth=2;g.strokeRect(q[0]+1,q[1]+1,Math.max(1,q[2]-2),Math.max(1,q[3]-2))}} +function pmTipHtml(it){if(it.fns){const g=it.kind==='game'&&it.pct!=null; + return ''+esc(it.name)+'
    '+hx(it.lo)+'–'+hx(it.hi)+' · '+kB(it.bytes)+' · '+fmt(it.functions)+' functions
    ' + +(g?'
    '+it.pct.toFixed(2)+'% of '+kB(it.counted)+' game code decompiled
    exact '+kB(it.exact)+' · verified '+kB(it.verified)+' · readable '+kB(it.readable)+'
    ' + :'
    '+esc(PM_KIND[it.kind])+(it.kind==='unsorted'?': not counted until sorted':': named, not decompiled; not counted')+'
    ')+'
    click to see its functions
    '} + return ''+esc(it.name)+'
    '+hx(it.address)+'–'+hx(it.address+it.size)+' · '+fmt(it.size)+' bytes
    '+esc(PM_LABEL[it.status])+'
    click to inspect in the Decomp view
    '} +function pmTip(html,x,y){const t=el('pmTip');if(html==null){t.hidden=true;t._for=null;return}if(t._for!==html){t.innerHTML=html;t._for=html}t.hidden=false; + const w=t.offsetWidth,h=t.offsetHeight;t.style.left=Math.max(8,Math.min(x+14,innerWidth-w-8))+'px';t.style.top=(y+16+h>innerHeight?y-h-10:y+16)+'px'} +function pmCrumb(){const d=PM.doc;if(PM.unit==null){put('pmCrumb','');return}const u=d.units[PM.unit]; + put('pmCrumb',''+esc(u.name)+''+esc(PM_KIND[u.kind])+''+hx(u.lo)+'–'+hx(u.hi)+' · '+fmt(u.functions)+' functions · '+kB(u.bytes)+'' + +(u.counted?pmBar('Game code in this unit',kB(u.counted),pmCodeSegs(u,u.counted),u.counted):''))} +function pmDrill(i){if(PM.unit==null)PM.topFilter=el('pmFilter').value;PM.unit=i;el('pmFilter').value='';if(el('pmUnits')._filled)el('pmUnits').value=String(i);else pmUnitsSelect();pmTip(null);pmLayout()} +function pmBack(){PM.unit=null;el('pmFilter').value=PM.topFilter||'';if(el('pmUnits')._filled)el('pmUnits').value='';else pmUnitsSelect();pmTip(null);pmLayout()} +function pmUnitsSelect(){const d=PM.doc,s=el('pmUnits'),u=PM.unit!=null?d.units[PM.unit]:null;s._filled=false; /* the full list is built only when someone opens it */ + put(s,''+(u?'':''));s.value=u?String(u.i):''} +function pmFillUnits(){const d=PM.doc,s=el('pmUnits');if(!d||s._filled)return;s._filled=true;const groups={game:[],unsorted:[],engine:[],library:[]};d.units.forEach(u=>(groups[u.kind]||groups.library).push(u)); + put(s,''+[['game','Game code'],['unsorted','Unsorted'],['engine','Engine (Gamebryo)'],['library','Libraries']].map(([k,l])=>groups[k].length?'' + +groups[k].sort((a,b)=>a.name.localeCompare(b.name)).map(u=>'').join('')+'':'').join('')); + s.value=PM.unit==null?'':String(PM.unit)} +(()=>{const c=el('pmMap');let pending=null;document.body.appendChild(el('pmTip')); /* fixed position must be relative to the viewport, not a card */ + const hit=(x,y)=>{for(let i=0;i=q[0]&&x=q[1]&&y{const b=c.getBoundingClientRect(),i=hit(e.clientX-b.left,e.clientY-b.top); + if(i!==PM.hover){PM.hover=i;if(!pending)pending=requestAnimationFrame(()=>{pending=null;pmBlit()})}pmTip(i<0?null:pmTipHtml(PM.items[i]),e.clientX,e.clientY)}); + c.addEventListener('mouseleave',()=>{PM.hover=-1;pmBlit();pmTip(null)}); + c.addEventListener('click',e=>{const b=c.getBoundingClientRect(),i=hit(e.clientX-b.left,e.clientY-b.top);if(i<0)return;const it=PM.items[i]; + if(it.fns)return pmDrill(it.i);pmTip(null);show('decomp');el('fnTarget').value=hx(it.address);inspectFn();el('h-inspect').scrollIntoView({block:'start'})}); + const card=el('pmCard');card.addEventListener('mouseover',e=>{const s=e.target.closest('[data-tip]');if(s)pmTip(esc(s.dataset.tip),e.clientX,e.clientY)}); + card.addEventListener('mousemove',e=>{const s=e.target.closest('[data-tip]');if(s)pmTip(esc(s.dataset.tip),e.clientX,e.clientY)}); + card.addEventListener('mouseout',e=>{if(e.target.closest('[data-tip]'))pmTip(null)}); + card.addEventListener('click',e=>{if(e.target.closest('#pmBack'))pmBack()}); + ['pointerdown','focus','keydown'].forEach(t=>el('pmUnits').addEventListener(t,pmFillUnits)); + el('pmUnits').addEventListener('change',e=>{const v=e.target.value;if(v==='')pmBack();else pmDrill(Number(v))}); + let ft=0;el('pmFilter').addEventListener('input',()=>{clearTimeout(ft);ft=setTimeout(pmLayout,180)}); + el('pmPrev').onclick=()=>pmStep('prev');el('pmNext').onclick=()=>pmStep('next');el('pmLatest').onclick=()=>pmStep('latest'); + let rt=0;new ResizeObserver(()=>{clearTimeout(rt);rt=setTimeout(()=>{if(current==='logbook'&&c.clientWidth&&(c.clientWidth!==PM.w||c.clientHeight!==PM.h))pmLayout()},120)}).observe(c); /* only when its size really changes */ + const recolor=()=>{PM.colors=null;if(current==='logbook'&&PM.doc)pmPaint()}; + new MutationObserver(recolor).observe(document.documentElement,{attributes:true,attributeFilter:['data-theme','data-palette']}); + matchMedia('(prefers-color-scheme: dark)').addEventListener('change',recolor); + every(()=>{if(current==='logbook')return pmLoad()},60000,null)})(); el('fnForm').addEventListener('submit',e=>{e.preventDefault();inspectFn()}); el('refreshDecomp').addEventListener('click',renderDecomp); document.addEventListener('click',e=>{const a=e.target.closest('[data-fn]');if(!a)return;e.preventDefault();if(current!=='decomp')show('decomp');el('fnTarget').value=a.dataset.fn;inspectFn();el('h-inspect').scrollIntoView({behavior:'smooth',block:'start'})}); diff --git a/tests/test_logbook.py b/tests/test_logbook.py index dd77e0e..d201fc4 100644 --- a/tests/test_logbook.py +++ b/tests/test_logbook.py @@ -5,7 +5,8 @@ import subprocess import unittest from helpers import HarborCase -from harbor import logbook, logbook_auto +from harbor import logbook, logbook_auto, progress_map +from harbor.queue import FunctionQueue from harbor.util import HarborError @@ -58,5 +59,83 @@ class LogbookAuto(HarborCase): self.assertFalse(logbook_auto.refresh(self.h.project)["changed"]) +class ProgressMapTests(HarborCase): + """GET /api/logbook/progress-map: units, tiers and the headline, by the Logbook's own definitions.""" + def setUp(self): + super().setUp() + d = self.dir + (d / "image.bin").write_bytes(b"\xc3" * 16) + facts = [{"address": "0x401000", "size": 100, "unit": "", "data": ["0x500010", "0x500020"]}, + {"address": "0x401100", "size": 50, "unit": ""}, {"address": "0x401200", "size": 30, "unit": ""}, + {"address": "0x402000", "size": 40, "unit": "engine/NiFoo"}, + {"address": "0x403000", "size": 20, "unit": ""}, {"address": "0x404000", "size": 10, "unit": ""}] + (d / "facts.jsonl").write_text("\n".join(json.dumps(f) for f in facts), encoding="utf-8") + (d / "sections.json").write_text(json.dumps([{"name": ".text", "start": 0x401000, "end": 0x500000}, + {"name": ".data", "start": 0x500000, "end": 0x501000}]), + encoding="utf-8") + ann = d / "annotations" + ann.mkdir() + (ann / "names.csv").write_text("address,name,category,note\n0x00401000,First,game,x\n", encoding="utf-8") + (ann / "globals.csv").write_text("address,name,type,comment\n0x00500010,g_one,int,x\n", encoding="utf-8") + symbols = d / "stats" / "re" / "symbols" + symbols.mkdir(parents=True) + rows = [("0x401000", "First", "game", "decompiled"), ("0x401100", "Second", "game", "decompiled"), + ("0x401200", "", "game", "classified"), ("0x402000", "NiFoo_Bar", "engine", "classified"), + ("0x403000", "", "platform", "classified"), ("0x404000", "", "unsorted", "unknown")] + (symbols / "pc102.csv").write_text("address,end,size,name,category,unit,status,xbox,notes\n" + "".join( + f"{a},,0,{n},{c},,{s},,\n" for a, n, c, s in rows), encoding="utf-8") + (d / "check.json").write_text(json.dumps([{"address": "0x401000", "status": "exact"}]), encoding="utf-8") + (d / "diffrun.json").write_text(json.dumps([{"address": "0x401100", "verdict": "same"}]), encoding="utf-8") + (d / "sources.csv").write_text("pc_address,pc_name,source_file,lines,xbox_address,confidence,kind,evidence\n" + "0x401000,,a.cpp,,,high,,\n0x401100,,a.cpp,,,high,,\n", encoding="utf-8") + self.h.project.reverse = {"image": str(d / "image.bin"), "facts": str(d / "facts.jsonl"), + "annotations": str(ann), "match_results": str(d / "check.json"), + "diffrun_results": str(d / "diffrun.json"), "source_files": str(d / "sources.csv")} + self.h.project.logbook = {"dir": str(d / "logbook"), "stats_repo": str(d / "stats")} + logbook.set_stats(self.h.project, {"functions": 6, "commit": "abc1234", "game_bytes_decompiled_percent": 83.33, + "by_category": {"game": {"functions": 3, "bytes": 180}}}) + self.old_ttl, progress_map.KEY_TTL = progress_map.KEY_TTL, 0 + self.map = progress_map.ProgressMap(self.h.project, lambda: FunctionQueue(self.h)) + + def tearDown(self): + progress_map.KEY_TTL = self.old_ttl + progress_map._MAPS.clear() + super().tearDown() + + def test_headline_units_tiers_and_cache(self): + doc = self.map.get() + self.assertEqual((doc["percent"], doc["logbook_percent"]), (83.33, 83.33)) # 150 of 180 game bytes + self.assertEqual(doc["code"], {"exact": 100, "verified": 50, "readable": 0, "bytes": 180, "functions": 3, + "decompiled_functions": 2}) + self.assertEqual((doc["data"]["named"], doc["data"]["addresses"]), (1, 2)) + self.assertEqual(doc["commit"]["sha"], "abc1234") + self.assertEqual(doc["history"][-1][2:], ["abc1234", 180]) # history points keep the commit + units = {u[0]: dict(zip(doc["unit_fields"], u)) for u in doc["units"]} + self.assertEqual(set(units), {"a.cpp", "game · 0x400000 block (no source file)", "engine/NiFoo", + "platform · 0x400000 block (no source file)", + "unsorted · 0x400000 block (no source file)"}) + self.assertEqual((units["a.cpp"]["kind"], units["a.cpp"]["counted"], units["a.cpp"]["exact"]), + ("game", 150, 100)) + self.assertEqual(units["engine/NiFoo"]["kind"], "engine") + status = {f[1]: doc["statuses"][f[3]] for f in doc["functions"]} + self.assertEqual(status, {0x401000: "exact", 0x401100: "verified", 0x401200: "raw", 0x402000: "engine", + 0x403000: "library", 0x404000: "unsorted"}) + self.assertEqual({f[1]: f[4] for f in doc["functions"]}[0x401000], "First") + self.assertEqual(self.map.get(doc["version"]), {"version": doc["version"], "same": True}) + self.assertEqual(self.map.computations, 1) + FunctionQueue(self.h).defer("Claude", "0x401200", "x87", "precision differs") # a new note: recomputed + doc2 = self.map.get(doc["version"]) + self.assertNotEqual(doc2["version"], doc["version"]) + self.assertEqual(doc2["statuses"][next(f for f in doc2["functions"] if f[1] == 0x401200)[3]], "deferred") + + def test_endpoint(self): + progress_map._MAPS[self.h.project.name] = self.map + status, body = progress_map.handle(self.h.project, {}) + self.assertEqual((status, body["percent"]), (200, 83.33)) + self.assertEqual(progress_map.handle(self.h.project, {"have": body["version"]})[1]["same"], True) + self.h.project.reverse = {} + self.assertEqual(progress_map.handle(self.h.project, {})[0], 404) + + if __name__ == "__main__": unittest.main()