326 lines
16 KiB
Python
326 lines
16 KiB
Python
"""The Logbook's progress map, in the style of decomp.dev: one unit per original source file, its functions and how
|
|
far each has come, plus the headline share of game code decompiled and the Code and Data bars.
|
|
|
|
GET /api/logbook/progress-map answers with one compact JSON document. It is computed once and kept until one of its
|
|
inputs changes (main's HEAD, the Logbook's symbol table and statistics, the match, emulator and library results,
|
|
facts, names, source files, globals, or a deferral note); `?have=<version>` answers {"version", "same": true} when
|
|
the caller already holds the current one, so a page that polls does not download or re-lay-out anything.
|
|
|
|
Definitions follow the Logbook exactly (open-seas tools/progress/progress.py, which writes stats.json):
|
|
- the symbol table is <[logbook] stats_repo>/re/symbols/pc102.csv: its category and status per function;
|
|
- counted ("game code") = category game and status not superseded or wontfix;
|
|
- decompiled = status decompiled, reimplemented or verified, so percent = decompiled game bytes / game bytes,
|
|
the Logbook's game_bytes_decompiled_percent.
|
|
Decompiled functions are split further, best first: exact (byte match in the [reverse] match_results), verified
|
|
(behaves the same in the emulator, diffrun_results, not 'weak'), readable. The rest: deferred (a live DEFERRED note),
|
|
raw, and, not counted, unsorted, engine (Gamebryo, named only) and library (C runtime, D3DX9, platform wrappers).
|
|
"results" says which commit the exact and verified tiers are from (harbor.results_refresh keeps them current).
|
|
Without a stats_repo the categories come from names.csv / facts and readable from the src headers (the queue's view).
|
|
|
|
Units, first that applies: the source file from the Xbox build's assert sites ([reverse] source_files); the D3DX9 block
|
|
(platform/d3dx9) or the import-thunk stubs from library_ranges; the unit facts gives (engine/<class>,
|
|
platform/<library>); otherwise
|
|
<category> plus its 64 KB address block, labelled "(no source file)"."""
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from .queue import FunctionQueue
|
|
|
|
STATUSES = ("exact", "verified", "readable", "deferred", "raw", "unsorted", "engine", "library")
|
|
DECOMPILED = ("decompiled", "reimplemented", "verified")
|
|
EXCLUDED = ("superseded", "wontfix")
|
|
UNIT_FIELDS = ("name", "kind", "lo", "hi", "bytes", "counted", "exact", "verified", "readable", "functions")
|
|
FUNCTION_FIELDS = ("unit", "address", "size", "status", "name")
|
|
KEY_TTL = 15 # seconds a computed input key is trusted before the files and HEAD are looked at again
|
|
|
|
|
|
def _int(text):
|
|
return int(str(text), 16) if str(text).lower().startswith("0x") else int(text)
|
|
|
|
|
|
def _mtime(path):
|
|
try:
|
|
return Path(path).stat().st_mtime_ns if path else None
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def _git(repo, *args):
|
|
try:
|
|
out = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, timeout=20,
|
|
encoding="utf-8", errors="replace")
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
return ""
|
|
return out.stdout.strip() if out.returncode == 0 else ""
|
|
|
|
|
|
def _utc(iso):
|
|
import datetime
|
|
try:
|
|
return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
class ProgressMap:
|
|
def __init__(self, project, queue_factory):
|
|
self.project = project
|
|
self.queue_factory = queue_factory # -> FunctionQueue on a connection this thread may use
|
|
self.lock = threading.Lock()
|
|
self.value = None # (key, document)
|
|
self.checked = (0.0, None) # (monotonic time, key)
|
|
self.computations = 0
|
|
|
|
# ------------------------------------------------------------------ inputs
|
|
def _config(self):
|
|
return self.project.reverse or {}, getattr(self.project, "logbook", None) or {}
|
|
|
|
def _symbols_path(self):
|
|
_, lb = self._config()
|
|
return Path(lb["stats_repo"]) / "re" / "symbols" / "pc102.csv" if lb.get("stats_repo") else None
|
|
|
|
def _decomp(self):
|
|
rev, lb = self._config()
|
|
if lb.get("decomp"):
|
|
return Path(lb["decomp"])
|
|
return Path(rev["annotations"]).parent if rev.get("annotations") else None
|
|
|
|
def _logbook_dir(self):
|
|
from . import logbook
|
|
return logbook.folder(self.project)
|
|
|
|
def key(self, q):
|
|
rev, _ = self._config()
|
|
decomp = self._decomp()
|
|
head = _git(decomp, "rev-parse", "HEAD") if decomp and (decomp / ".git").exists() else ""
|
|
paths = [self._symbols_path(), self._logbook_dir() / "stats.json", self._logbook_dir() / "history.json",
|
|
self._logbook_dir() / "results.json"]
|
|
paths += [rev.get(k) for k in ("facts", "match_results", "diffrun_results", "library_results",
|
|
"library_ranges", "source_files")]
|
|
if rev.get("annotations"):
|
|
paths += [Path(rev["annotations"]) / "names.csv", Path(rev["annotations"]) / "globals.csv"]
|
|
try:
|
|
note = q.h.conn.execute("SELECT MAX(id) FROM notes").fetchone()[0]
|
|
except Exception:
|
|
note = None
|
|
raw = json.dumps([head, note, [str(p) for p in paths], [_mtime(p) for p in paths]])
|
|
return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16], head
|
|
|
|
# ------------------------------------------------------------------ answer
|
|
def get(self, have=None):
|
|
with self.lock:
|
|
q = self.queue_factory()
|
|
now = time.monotonic()
|
|
if self.value and now - self.checked[0] < KEY_TTL:
|
|
key = self.checked[1]
|
|
else:
|
|
key, head = self.key(q)
|
|
self.checked = (now, key)
|
|
if not self.value or self.value[0] != key:
|
|
document = self.compute(q, head)
|
|
if document is None and self.value: # the symbol table is being rewritten: keep the last
|
|
document = self.value[1]
|
|
elif document is None:
|
|
document = self.compute(q, head, use_symbols=False)
|
|
document["version"] = key
|
|
self.value = (key, document)
|
|
self.computations += 1
|
|
if have and have == self.value[0]:
|
|
return {"version": have, "same": True}
|
|
return self.value[1]
|
|
|
|
# ------------------------------------------------------------------ the computation
|
|
def _symbols(self, expected):
|
|
"""pc102.csv as {address: (category, status, name)}, or None while it is missing or half written."""
|
|
path = self._symbols_path()
|
|
if not path or not path.exists():
|
|
return None
|
|
try:
|
|
with path.open(encoding="utf-8", newline="") as stream:
|
|
rows = {_int(r["address"]): (r["category"], r["status"], r.get("name") or "")
|
|
for r in csv.DictReader(stream)}
|
|
except (OSError, ValueError, KeyError):
|
|
return None
|
|
return rows if len(rows) >= 0.98 * expected else None
|
|
|
|
def _source_files(self, path):
|
|
out = {}
|
|
if path and Path(path).exists():
|
|
with open(path, encoding="utf-8", newline="") as stream:
|
|
for row in csv.DictReader(stream):
|
|
if row.get("source_file"):
|
|
out[_int(row["pc_address"])] = row["source_file"]
|
|
return out
|
|
|
|
def _all_ranges(self, path):
|
|
"""[(lo, hi, label)] of the library_ranges csv (the D3DX9 block and the compiler's stubs)."""
|
|
out = []
|
|
if path and Path(path).exists():
|
|
with open(path, encoding="utf-8", newline="") as stream:
|
|
for r in csv.DictReader(stream):
|
|
verdict = r.get("verdict", "")
|
|
label = "platform/d3dx9" if verdict.startswith("D3DX") else \
|
|
"Import thunks and unwind funclets" if "stubs" in verdict else None
|
|
if label:
|
|
out.append((_int(r["start"]), _int(r["end"]), label))
|
|
return out
|
|
|
|
def _data(self, facts, rev):
|
|
"""Named globals against the distinct .data/.rdata addresses the code touches (needs sections.json)."""
|
|
facts_path = Path(rev["facts"]) if rev.get("facts") else None
|
|
sections_path = facts_path.parent / "sections.json" if facts_path else None
|
|
globals_path = Path(rev["annotations"]) / "globals.csv" if rev.get("annotations") else None
|
|
if not sections_path or not sections_path.exists() or not globals_path or not globals_path.exists():
|
|
return None
|
|
spans = [(s["start"], s["end"]) for s in json.loads(sections_path.read_text(encoding="utf-8"))
|
|
if s.get("name") in (".data", ".rdata")]
|
|
touched = set()
|
|
for d in facts.values():
|
|
for ref in d.get("data", []):
|
|
a = _int(ref)
|
|
if any(lo <= a < hi for lo, hi in spans):
|
|
touched.add(a)
|
|
with globals_path.open(encoding="utf-8", newline="") as stream:
|
|
named = {_int(r["address"]) for r in csv.DictReader(line for line in stream if not line.startswith("#"))
|
|
if r.get("address") and r.get("name")}
|
|
if not touched:
|
|
return None
|
|
return {"addresses": len(touched), "named": len(touched & named), "globals": len(named),
|
|
"bytes": sum(hi - lo for lo, hi in spans)}
|
|
|
|
def compute(self, q, head, use_symbols=True):
|
|
rev, _ = self._config()
|
|
facts = q.r._facts()
|
|
symbols = self._symbols(len(facts)) if use_symbols else None
|
|
if use_symbols and self._symbols_path() and symbols is None and self._symbols_path().exists():
|
|
return None
|
|
names = q.r._csv("names.csv")
|
|
statuses = None if symbols else q.statuses()
|
|
exact = {a for a, row in q._results("match_results").items()
|
|
if str(row.get("status")) in ("match", "exact", "exact-relaxed")}
|
|
verified = {a for a, row in q._results("diffrun_results").items()
|
|
if str(row.get("verdict")) == "same" and not row.get("weak")}
|
|
library = set(q._results("library_results"))
|
|
ranges = self._all_ranges(rev.get("library_ranges"))
|
|
lib_ranges = [(lo, hi) for lo, hi, _ in ranges]
|
|
deferred = q.deferrals()
|
|
sources = self._source_files(rev.get("source_files"))
|
|
|
|
units, unit_index, functions = [], {}, []
|
|
game_functions = game_done = 0
|
|
for addr in sorted(facts):
|
|
d = facts[addr]
|
|
size = int(d.get("size") or 0)
|
|
if symbols and addr in symbols:
|
|
category, status, sym_name = symbols[addr]
|
|
else:
|
|
category = q._category(addr)
|
|
status = "decompiled" if (statuses or {}).get(addr, ("raw",))[0] == "readable" else "classified"
|
|
sym_name = ""
|
|
row = names.get(addr)
|
|
name = (row[1] if row and len(row) > 1 and row[1] else "") or sym_name or d.get("name") or f"FUN_{addr:06X}"
|
|
counted = category == "game" and status not in EXCLUDED
|
|
done = status in DECOMPILED
|
|
is_lib = q.is_library(addr, library, lib_ranges) or category in ("library", "platform", "crt")
|
|
if counted or (done and not is_lib) or (category == "unsorted" and not is_lib):
|
|
tier = "exact" if done and addr in exact else "verified" if done and addr in verified else \
|
|
"readable" if done else "deferred" if addr in deferred else \
|
|
"raw" if counted else "unsorted"
|
|
elif is_lib:
|
|
tier = "library"
|
|
else:
|
|
tier = "engine"
|
|
|
|
source = sources.get(addr)
|
|
in_range = next((label for lo, hi, label in ranges if lo <= addr < hi), None)
|
|
if source:
|
|
unit = source
|
|
elif in_range:
|
|
unit = in_range
|
|
elif d.get("unit"):
|
|
unit = d["unit"]
|
|
else:
|
|
unit = f"{category} · 0x{addr & ~0xFFFF:06X} block (no source file)"
|
|
if unit not in unit_index:
|
|
unit_index[unit] = len(units)
|
|
units.append({"name": unit, "lo": addr, "hi": addr + size, "bytes": 0, "counted": 0, "exact": 0,
|
|
"verified": 0, "readable": 0, "functions": 0, "kinds": {}})
|
|
u = units[unit_index[unit]]
|
|
u["hi"] = max(u["hi"], addr + size)
|
|
u["bytes"] += size
|
|
u["functions"] += 1
|
|
kind = "game" if counted else "library" if tier == "library" else "engine" if category == "engine" \
|
|
else "unsorted" if category == "unsorted" else "library"
|
|
u["kinds"][kind] = u["kinds"].get(kind, 0) + size
|
|
if counted:
|
|
game_functions += 1
|
|
game_done += done
|
|
u["counted"] += size
|
|
if tier in ("exact", "verified", "readable"):
|
|
u[tier] += size
|
|
functions.append([unit_index[unit], addr, size, STATUSES.index(tier), name])
|
|
|
|
for u in units:
|
|
kinds = u.pop("kinds")
|
|
u["kind"] = max(kinds, key=kinds.get) if kinds else "unsorted" # what most of its bytes are
|
|
|
|
code = {k: sum(u[k] for u in units) for k in ("exact", "verified", "readable")}
|
|
code["bytes"] = sum(u["counted"] for u in units)
|
|
code["functions"], code["decompiled_functions"] = game_functions, game_done
|
|
percent = round(100 * (code["exact"] + code["verified"] + code["readable"]) / code["bytes"], 2) \
|
|
if code["bytes"] else 0.0
|
|
|
|
from . import logbook
|
|
stats = logbook._read(self.project, "stats.json", None) or {}
|
|
history = sorted(logbook._read(self.project, "history.json", []), key=lambda p: p.get("at", ""))
|
|
decomp = self._decomp()
|
|
commit = {"sha": stats.get("commit"), "at": None}
|
|
if commit["sha"] and decomp:
|
|
commit["at"] = _utc(_git(decomp, "log", "-1", "--format=%cI", commit["sha"])) or None
|
|
main = {"sha": head[:7] if head else None,
|
|
"at": _utc(_git(decomp, "log", "-1", "--format=%cI", head)) if head else None}
|
|
from . import results_refresh
|
|
return {
|
|
"project": self.project.title or self.project.name,
|
|
"results": results_refresh.freshness(self.project),
|
|
"source": "logbook symbol table" if symbols else "src headers and names.csv",
|
|
"commit": commit, "main": main, "updated_at": stats.get("updated_at"),
|
|
"percent": percent, "logbook_percent": stats.get("game_bytes_decompiled_percent"),
|
|
"code": code, "data": self._data(facts, rev),
|
|
# every function in the executable (game, engine, platform, CRT), so engine work shows too
|
|
"all_code": {"bytes": stats.get("bytes"), "functions": stats.get("functions"),
|
|
"decompiled_bytes": ((stats.get("by_status") or {}).get("decompiled") or {}).get("bytes"),
|
|
"decompiled_functions": ((stats.get("by_status") or {}).get("decompiled") or {}).get("functions")},
|
|
"history": [[p.get("at"), p.get("decompiled"), p.get("commit"), p.get("game_bytes")]
|
|
for p in history if p.get("decompiled") is not None],
|
|
"statuses": list(STATUSES),
|
|
"unit_fields": list(UNIT_FIELDS),
|
|
"units": [[u[k] for k in UNIT_FIELDS] for u in units],
|
|
"function_fields": list(FUNCTION_FIELDS),
|
|
"functions": functions,
|
|
}
|
|
|
|
|
|
_MAPS = {}
|
|
_LOCK = threading.Lock()
|
|
|
|
|
|
def map_for(project):
|
|
with _LOCK:
|
|
if project.name not in _MAPS:
|
|
from . import decomp_api
|
|
api = decomp_api.api_for(project)
|
|
_MAPS[project.name] = ProgressMap(project, lambda: FunctionQueue(api.h))
|
|
return _MAPS[project.name]
|
|
|
|
|
|
def handle(project, query):
|
|
if not (project.reverse or {}).get("facts"):
|
|
return 404, {"error": "this project has no [reverse] table"}
|
|
return 200, map_for(project).get(query.get("have"))
|