Logbook progress map in the style of decomp.dev: "<project> is NN.NN% decompiled", a layered Code bar (exact solid, verified lighter, readable hatched, with per-segment tooltips), a Data bar (named globals of the data addresses the code touches), Previous / Next / Latest through the history reports, a Units dropdown and filter (text, <70%, >10kb), and a canvas treemap of units (original source files, else 64 KB blocks) that drills into functions
- GET /api/logbook/progress-map: one compact document, cached until an input changes (main's HEAD, the Logbook symbol table and statistics, match/emulator/library results, facts, names, source files, globals, deferral notes); ?have=<version> answers {"same": true}
- definitions follow the Logbook's progress.py (pc102.csv category game, status decompiled or later), so the headline equals game_bytes_decompiled_percent; engine, library and unsorted code are shown but not counted
- history points now record the commit and the game-code bytes
- the page lays out once per load, filter or size change, hit-tests hover on the computed rectangles, builds the 600-entry units list only when opened, and polls once a minute only while the Logbook is visible
This commit is contained in:
KeelrLeek committed
1 parent
315b63b571
commit
d5eb82261e
5 files changed
+556
-4
No files matched your search
@@ -0,0 +1,317 @@
|
||||
"""The Logbook's progress map, in the style of decomp.dev: one unit per original source file, its functions and how
|
||||
far each has come, plus the headline share of game code decompiled and the Code and Data bars.
|
||||
|
||||
GET /api/logbook/progress-map answers with one compact JSON document. It is computed once and kept until one of its
|
||||
inputs changes (main's HEAD, the Logbook's symbol table and statistics, the match, emulator and library results,
|
||||
facts, names, source files, globals, or a deferral note); `?have=<version>` answers {"version", "same": true} when
|
||||
the caller already holds the current one, so a page that polls does not download or re-lay-out anything.
|
||||
|
||||
Definitions follow the Logbook exactly (open-seas tools/progress/progress.py, which writes stats.json):
|
||||
- the symbol table is <[logbook] stats_repo>/re/symbols/pc102.csv: its category and status per function;
|
||||
- counted ("game code") = category game and status not superseded or wontfix;
|
||||
- decompiled = status decompiled, reimplemented or verified, so percent = decompiled game bytes / game bytes,
|
||||
the Logbook's game_bytes_decompiled_percent.
|
||||
Decompiled functions are split further, best first: exact (byte match in the [reverse] match_results), verified
|
||||
(behaves the same in the emulator, diffrun_results, not 'weak'), readable. The rest: deferred (a live DEFERRED note),
|
||||
raw, and, not counted, unsorted, engine (Gamebryo, named only) and library (C runtime, D3DX9, platform wrappers).
|
||||
Without a stats_repo the categories come from names.csv / facts and readable from the src headers (the queue's view).
|
||||
|
||||
Units, first that applies: the source file from the Xbox build's assert sites ([reverse] source_files); the D3DX9 block
|
||||
(platform/d3dx9) or the import-thunk stubs from library_ranges; the unit facts gives (engine/<class>,
|
||||
platform/<library>); otherwise
|
||||
<category> plus its 64 KB address block, labelled "(no source file)"."""
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from .queue import FunctionQueue
|
||||
|
||||
STATUSES = ("exact", "verified", "readable", "deferred", "raw", "unsorted", "engine", "library")
|
||||
DECOMPILED = ("decompiled", "reimplemented", "verified")
|
||||
EXCLUDED = ("superseded", "wontfix")
|
||||
UNIT_FIELDS = ("name", "kind", "lo", "hi", "bytes", "counted", "exact", "verified", "readable", "functions")
|
||||
FUNCTION_FIELDS = ("unit", "address", "size", "status", "name")
|
||||
KEY_TTL = 15 # seconds a computed input key is trusted before the files and HEAD are looked at again
|
||||
|
||||
|
||||
def _int(text):
|
||||
return int(str(text), 16) if str(text).lower().startswith("0x") else int(text)
|
||||
|
||||
|
||||
def _mtime(path):
|
||||
try:
|
||||
return Path(path).stat().st_mtime_ns if path else None
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _git(repo, *args):
|
||||
try:
|
||||
out = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, timeout=20,
|
||||
encoding="utf-8", errors="replace")
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return ""
|
||||
return out.stdout.strip() if out.returncode == 0 else ""
|
||||
|
||||
|
||||
def _utc(iso):
|
||||
import datetime
|
||||
try:
|
||||
return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
class ProgressMap:
|
||||
def __init__(self, project, queue_factory):
|
||||
self.project = project
|
||||
self.queue_factory = queue_factory # -> FunctionQueue on a connection this thread may use
|
||||
self.lock = threading.Lock()
|
||||
self.value = None # (key, document)
|
||||
self.checked = (0.0, None) # (monotonic time, key)
|
||||
self.computations = 0
|
||||
|
||||
# ------------------------------------------------------------------ inputs
|
||||
def _config(self):
|
||||
return self.project.reverse or {}, getattr(self.project, "logbook", None) or {}
|
||||
|
||||
def _symbols_path(self):
|
||||
_, lb = self._config()
|
||||
return Path(lb["stats_repo"]) / "re" / "symbols" / "pc102.csv" if lb.get("stats_repo") else None
|
||||
|
||||
def _decomp(self):
|
||||
rev, lb = self._config()
|
||||
if lb.get("decomp"):
|
||||
return Path(lb["decomp"])
|
||||
return Path(rev["annotations"]).parent if rev.get("annotations") else None
|
||||
|
||||
def _logbook_dir(self):
|
||||
from . import logbook
|
||||
return logbook.folder(self.project)
|
||||
|
||||
def key(self, q):
|
||||
rev, _ = self._config()
|
||||
decomp = self._decomp()
|
||||
head = _git(decomp, "rev-parse", "HEAD") if decomp and (decomp / ".git").exists() else ""
|
||||
paths = [self._symbols_path(), self._logbook_dir() / "stats.json", self._logbook_dir() / "history.json"]
|
||||
paths += [rev.get(k) for k in ("facts", "match_results", "diffrun_results", "library_results",
|
||||
"library_ranges", "source_files")]
|
||||
if rev.get("annotations"):
|
||||
paths += [Path(rev["annotations"]) / "names.csv", Path(rev["annotations"]) / "globals.csv"]
|
||||
try:
|
||||
note = q.h.conn.execute("SELECT MAX(id) FROM notes").fetchone()[0]
|
||||
except Exception:
|
||||
note = None
|
||||
raw = json.dumps([head, note, [str(p) for p in paths], [_mtime(p) for p in paths]])
|
||||
return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16], head
|
||||
|
||||
# ------------------------------------------------------------------ answer
|
||||
def get(self, have=None):
|
||||
with self.lock:
|
||||
q = self.queue_factory()
|
||||
now = time.monotonic()
|
||||
if self.value and now - self.checked[0] < KEY_TTL:
|
||||
key = self.checked[1]
|
||||
else:
|
||||
key, head = self.key(q)
|
||||
self.checked = (now, key)
|
||||
if not self.value or self.value[0] != key:
|
||||
document = self.compute(q, head)
|
||||
if document is None and self.value: # the symbol table is being rewritten: keep the last
|
||||
document = self.value[1]
|
||||
elif document is None:
|
||||
document = self.compute(q, head, use_symbols=False)
|
||||
document["version"] = key
|
||||
self.value = (key, document)
|
||||
self.computations += 1
|
||||
if have and have == self.value[0]:
|
||||
return {"version": have, "same": True}
|
||||
return self.value[1]
|
||||
|
||||
# ------------------------------------------------------------------ the computation
|
||||
def _symbols(self, expected):
|
||||
"""pc102.csv as {address: (category, status, name)}, or None while it is missing or half written."""
|
||||
path = self._symbols_path()
|
||||
if not path or not path.exists():
|
||||
return None
|
||||
try:
|
||||
with path.open(encoding="utf-8", newline="") as stream:
|
||||
rows = {_int(r["address"]): (r["category"], r["status"], r.get("name") or "")
|
||||
for r in csv.DictReader(stream)}
|
||||
except (OSError, ValueError, KeyError):
|
||||
return None
|
||||
return rows if len(rows) >= 0.98 * expected else None
|
||||
|
||||
def _source_files(self, path):
|
||||
out = {}
|
||||
if path and Path(path).exists():
|
||||
with open(path, encoding="utf-8", newline="") as stream:
|
||||
for row in csv.DictReader(stream):
|
||||
if row.get("source_file"):
|
||||
out[_int(row["pc_address"])] = row["source_file"]
|
||||
return out
|
||||
|
||||
def _all_ranges(self, path):
|
||||
"""[(lo, hi, label)] of the library_ranges csv (the D3DX9 block and the compiler's stubs)."""
|
||||
out = []
|
||||
if path and Path(path).exists():
|
||||
with open(path, encoding="utf-8", newline="") as stream:
|
||||
for r in csv.DictReader(stream):
|
||||
verdict = r.get("verdict", "")
|
||||
label = "platform/d3dx9" if verdict.startswith("D3DX") else \
|
||||
"Import thunks and unwind funclets" if "stubs" in verdict else None
|
||||
if label:
|
||||
out.append((_int(r["start"]), _int(r["end"]), label))
|
||||
return out
|
||||
|
||||
def _data(self, facts, rev):
|
||||
"""Named globals against the distinct .data/.rdata addresses the code touches (needs sections.json)."""
|
||||
facts_path = Path(rev["facts"]) if rev.get("facts") else None
|
||||
sections_path = facts_path.parent / "sections.json" if facts_path else None
|
||||
globals_path = Path(rev["annotations"]) / "globals.csv" if rev.get("annotations") else None
|
||||
if not sections_path or not sections_path.exists() or not globals_path or not globals_path.exists():
|
||||
return None
|
||||
spans = [(s["start"], s["end"]) for s in json.loads(sections_path.read_text(encoding="utf-8"))
|
||||
if s.get("name") in (".data", ".rdata")]
|
||||
touched = set()
|
||||
for d in facts.values():
|
||||
for ref in d.get("data", []):
|
||||
a = _int(ref)
|
||||
if any(lo <= a < hi for lo, hi in spans):
|
||||
touched.add(a)
|
||||
with globals_path.open(encoding="utf-8", newline="") as stream:
|
||||
named = {_int(r["address"]) for r in csv.DictReader(line for line in stream if not line.startswith("#"))
|
||||
if r.get("address") and r.get("name")}
|
||||
if not touched:
|
||||
return None
|
||||
return {"addresses": len(touched), "named": len(touched & named), "globals": len(named),
|
||||
"bytes": sum(hi - lo for lo, hi in spans)}
|
||||
|
||||
def compute(self, q, head, use_symbols=True):
|
||||
rev, _ = self._config()
|
||||
facts = q.r._facts()
|
||||
symbols = self._symbols(len(facts)) if use_symbols else None
|
||||
if use_symbols and self._symbols_path() and symbols is None and self._symbols_path().exists():
|
||||
return None
|
||||
names = q.r._csv("names.csv")
|
||||
statuses = None if symbols else q.statuses()
|
||||
exact = {a for a, row in q._results("match_results").items()
|
||||
if str(row.get("status")) in ("match", "exact", "exact-relaxed")}
|
||||
verified = {a for a, row in q._results("diffrun_results").items()
|
||||
if str(row.get("verdict")) == "same" and not row.get("weak")}
|
||||
library = set(q._results("library_results"))
|
||||
ranges = self._all_ranges(rev.get("library_ranges"))
|
||||
lib_ranges = [(lo, hi) for lo, hi, _ in ranges]
|
||||
deferred = q.deferrals()
|
||||
sources = self._source_files(rev.get("source_files"))
|
||||
|
||||
units, unit_index, functions = [], {}, []
|
||||
game_functions = game_done = 0
|
||||
for addr in sorted(facts):
|
||||
d = facts[addr]
|
||||
size = int(d.get("size") or 0)
|
||||
if symbols and addr in symbols:
|
||||
category, status, sym_name = symbols[addr]
|
||||
else:
|
||||
category = q._category(addr)
|
||||
status = "decompiled" if (statuses or {}).get(addr, ("raw",))[0] == "readable" else "classified"
|
||||
sym_name = ""
|
||||
row = names.get(addr)
|
||||
name = (row[1] if row and len(row) > 1 and row[1] else "") or sym_name or d.get("name") or f"FUN_{addr:06X}"
|
||||
counted = category == "game" and status not in EXCLUDED
|
||||
done = status in DECOMPILED
|
||||
is_lib = q.is_library(addr, library, lib_ranges) or category in ("library", "platform", "crt")
|
||||
if counted or (done and not is_lib) or (category == "unsorted" and not is_lib):
|
||||
tier = "exact" if done and addr in exact else "verified" if done and addr in verified else \
|
||||
"readable" if done else "deferred" if addr in deferred else \
|
||||
"raw" if counted else "unsorted"
|
||||
elif is_lib:
|
||||
tier = "library"
|
||||
else:
|
||||
tier = "engine"
|
||||
|
||||
source = sources.get(addr)
|
||||
in_range = next((label for lo, hi, label in ranges if lo <= addr < hi), None)
|
||||
if source:
|
||||
unit = source
|
||||
elif in_range:
|
||||
unit = in_range
|
||||
elif d.get("unit"):
|
||||
unit = d["unit"]
|
||||
else:
|
||||
unit = f"{category} · 0x{addr & ~0xFFFF:06X} block (no source file)"
|
||||
if unit not in unit_index:
|
||||
unit_index[unit] = len(units)
|
||||
units.append({"name": unit, "lo": addr, "hi": addr + size, "bytes": 0, "counted": 0, "exact": 0,
|
||||
"verified": 0, "readable": 0, "functions": 0, "kinds": {}})
|
||||
u = units[unit_index[unit]]
|
||||
u["hi"] = max(u["hi"], addr + size)
|
||||
u["bytes"] += size
|
||||
u["functions"] += 1
|
||||
kind = "game" if counted else "library" if tier == "library" else "engine" if category == "engine" \
|
||||
else "unsorted" if category == "unsorted" else "library"
|
||||
u["kinds"][kind] = u["kinds"].get(kind, 0) + size
|
||||
if counted:
|
||||
game_functions += 1
|
||||
game_done += done
|
||||
u["counted"] += size
|
||||
if tier in ("exact", "verified", "readable"):
|
||||
u[tier] += size
|
||||
functions.append([unit_index[unit], addr, size, STATUSES.index(tier), name])
|
||||
|
||||
for u in units:
|
||||
kinds = u.pop("kinds")
|
||||
u["kind"] = max(kinds, key=kinds.get) if kinds else "unsorted" # what most of its bytes are
|
||||
|
||||
code = {k: sum(u[k] for u in units) for k in ("exact", "verified", "readable")}
|
||||
code["bytes"] = sum(u["counted"] for u in units)
|
||||
code["functions"], code["decompiled_functions"] = game_functions, game_done
|
||||
percent = round(100 * (code["exact"] + code["verified"] + code["readable"]) / code["bytes"], 2) \
|
||||
if code["bytes"] else 0.0
|
||||
|
||||
from . import logbook
|
||||
stats = logbook._read(self.project, "stats.json", None) or {}
|
||||
history = sorted(logbook._read(self.project, "history.json", []), key=lambda p: p.get("at", ""))
|
||||
decomp = self._decomp()
|
||||
commit = {"sha": stats.get("commit"), "at": None}
|
||||
if commit["sha"] and decomp:
|
||||
commit["at"] = _utc(_git(decomp, "log", "-1", "--format=%cI", commit["sha"])) or None
|
||||
main = {"sha": head[:7] if head else None,
|
||||
"at": _utc(_git(decomp, "log", "-1", "--format=%cI", head)) if head else None}
|
||||
return {
|
||||
"project": self.project.title or self.project.name,
|
||||
"source": "logbook symbol table" if symbols else "src headers and names.csv",
|
||||
"commit": commit, "main": main, "updated_at": stats.get("updated_at"),
|
||||
"percent": percent, "logbook_percent": stats.get("game_bytes_decompiled_percent"),
|
||||
"code": code, "data": self._data(facts, rev),
|
||||
"history": [[p.get("at"), p.get("decompiled"), p.get("commit"), p.get("game_bytes")]
|
||||
for p in history if p.get("decompiled") is not None],
|
||||
"statuses": list(STATUSES),
|
||||
"unit_fields": list(UNIT_FIELDS),
|
||||
"units": [[u[k] for k in UNIT_FIELDS] for u in units],
|
||||
"function_fields": list(FUNCTION_FIELDS),
|
||||
"functions": functions,
|
||||
}
|
||||
|
||||
|
||||
_MAPS = {}
|
||||
_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def map_for(project):
|
||||
with _LOCK:
|
||||
if project.name not in _MAPS:
|
||||
from . import decomp_api
|
||||
api = decomp_api.api_for(project)
|
||||
_MAPS[project.name] = ProgressMap(project, lambda: FunctionQueue(api.h))
|
||||
return _MAPS[project.name]
|
||||
|
||||
|
||||
def handle(project, query):
|
||||
if not (project.reverse or {}).get("facts"):
|
||||
return 404, {"error": "this project has no [reverse] table"}
|
||||
return 200, map_for(project).get(query.get("have"))
|
||||
Reference in new issue
Block a user