phenom/HarborFork
1
0 Fork 0
forked from Keelrleek/Harbor
Code Pull Requests Activity

next_functions: skip functions already in an open review item (by scope, or the addresses its title names), so agents are not offered work they or others have already done

Browse Source
This commit is contained in:
KeelrLeek committed 2026-10-07 08:47:21 -05:00
1 parent 9b6ea3bc57
commit 3d9ac4ca9a
1 file changed
+269 -252
Download Patch File Download Diff File Expand all files Collapse all files
harbor/queue.py
+269 -252
View File
@@ -1,252 +1,269 @@
"""Which functions to decompile next, which to set aside, and how far the project is, per function. """Which functions to decompile next, which to set aside, and how far the project is, per function.
- next_functions(agent): the functions in the agent's bands that are not decompiled, not deferred and not claimed by - next_functions(agent): the functions in the agent's bands that are not decompiled, not deferred and not claimed by
someone else, easiest first. Ease is scored from the bytes and the team's records: small functions, callees that someone else, easiest first. Ease is scored from the bytes and the team's records: small functions, callees that
already have names, stack-only conventions, game code, few earlier rejections and a readable near-duplicate rank already have names, stack-only conventions, game code, few earlier rejections and a readable near-duplicate rank
higher. A function rejected `attempt_cap` times or more is left out (defer it or ask the coordinator). higher. A function rejected `attempt_cap` times or more is left out (defer it or ask the coordinator).
- defer(agent, address, reason, evidence): set a function aside with a reason slug; it becomes a note on fn:0x... - defer(agent, address, reason, evidence): set a function aside with a reason slug; it becomes a note on fn:0x...
(`DEFERRED <reason>: <evidence>`) so everyone sees it, and the queue skips it until the note is replaced. (`DEFERRED <reason>: <evidence>`) so everyone sees it, and the queue skips it until the note is replaced.
- progress(): every function in a tier, best first: exact (byte match), verified (behaves the same in the emulator), - progress(): every function in a tier, best first: exact (byte match), verified (behaves the same in the emulator),
readable, deferred, raw, plus library and engine functions counted apart. readable, deferred, raw, plus library and engine functions counted apart.
Configured from the project's [reverse] table (facts, annotations, src, conventions) plus optional result files: Configured from the project's [reverse] table (facts, annotations, src, conventions) plus optional result files:
match_results (match71.py --json), diffrun_results (diffrun.py --json), library_results (libsweep csv).""" match_results (match71.py --json), diffrun_results (diffrun.py --json), library_results (libsweep csv)."""
import csv import csv
import json import json
import re import re
from pathlib import Path from pathlib import Path
from .reverse import Reverse from .reverse import Reverse
from .util import HarborError from .util import HarborError, parse_scope
DEFER_REASONS = ("register-args", "needs-type", "callee-unnamed", "inline", "x87", "layout", "too-large", DEFER_REASONS = ("register-args", "needs-type", "callee-unnamed", "inline", "x87", "layout", "too-large",
"engine-template", "data-not-code", "other") "engine-template", "data-not-code", "other")
READABLE = " * status: readable" READABLE = " * status: readable"
TIER_ORDER = ("exact", "verified", "readable", "deferred", "raw") TIER_ORDER = ("exact", "verified", "readable", "deferred", "raw")
class FunctionQueue: class FunctionQueue:
def __init__(self, harbor): def __init__(self, harbor):
self.h = harbor self.h = harbor
self.config = harbor.project.reverse or {} self.config = harbor.project.reverse or {}
self.r = Reverse(self.config) self.r = Reverse(self.config)
self.cap = int(self.config.get("attempt_cap", 2)) self.cap = int(self.config.get("attempt_cap", 2))
self.min_size = int(self.config.get("queue_min_size", 8)) self.min_size = int(self.config.get("queue_min_size", 8))
# [[lo, hi, "label"], ...] ranked low, e.g. the compiler-generated static initializers at the end of .text # [[lo, hi, "label"], ...] ranked low, e.g. the compiler-generated static initializers at the end of .text
self.low_priority = [(int(str(lo), 0), int(str(hi), 0), label) self.low_priority = [(int(str(lo), 0), int(str(hi), 0), label)
for lo, hi, label in self.config.get("low_priority_ranges", [])] for lo, hi, label in self.config.get("low_priority_ranges", [])]
# ------------------------------------------------------------------ records # ------------------------------------------------------------------ records
def statuses(self): def statuses(self):
"""address -> (status, file name) from the headers of src/**/<ADDR>_<Name>.c.""" """address -> (status, file name) from the headers of src/**/<ADDR>_<Name>.c."""
out = {} out = {}
src = self.r.src src = self.r.src
if not src or not src.exists(): if not src or not src.exists():
return out return out
for path in src.rglob("*.c"): for path in src.rglob("*.c"):
m = re.match(r"([0-9A-F]{6,8})_", path.name) m = re.match(r"([0-9A-F]{6,8})_", path.name)
if not m: if not m:
continue continue
try: try:
with path.open(encoding="utf-8", errors="replace") as f: with path.open(encoding="utf-8", errors="replace") as f:
head = [next(f, "") for _ in range(8)] head = [next(f, "") for _ in range(8)]
except (FileNotFoundError, PermissionError): except (FileNotFoundError, PermissionError):
continue # a landing on main moved or rewrote it while we scanned; its new path is found continue # a landing on main moved or rewrote it while we scanned; its new path is found
status = "readable" if any(line.startswith(READABLE) for line in head) else \ status = "readable" if any(line.startswith(READABLE) for line in head) else \
next((line.split("status:", 1)[1].strip() for line in head if "status:" in line), "raw") next((line.split("status:", 1)[1].strip() for line in head if "status:" in line), "raw")
out[int(m.group(1), 16)] = (status, path.name) out[int(m.group(1), 16)] = (status, path.name)
return out return out
def rejections(self): def rejections(self):
"""address -> how many review items on fn:0x... were rejected or sent back.""" """address -> how many review items on fn:0x... were rejected or sent back."""
out = {} out = {}
rows = self.h.conn.execute( rows = self.h.conn.execute(
"SELECT i.scope, COUNT(*) AS n FROM item_events e JOIN items i ON i.id = e.item_id " "SELECT i.scope, COUNT(*) AS n FROM item_events e JOIN items i ON i.id = e.item_id "
"WHERE e.to_state IN ('rejected', 'changes_requested') AND i.scope LIKE 'fn:0x%' GROUP BY i.scope") "WHERE e.to_state IN ('rejected', 'changes_requested') AND i.scope LIKE 'fn:0x%' GROUP BY i.scope")
for row in rows: for row in rows:
try: try:
out[int(row["scope"][3:], 16)] = row["n"] out[int(row["scope"][3:], 16)] = row["n"]
except ValueError: except ValueError:
pass pass
return out return out
def deferrals(self): def deferrals(self):
"""address -> (reason, evidence, author) for live DEFERRED notes.""" """address -> (reason, evidence, author) for live DEFERRED notes."""
out = {} out = {}
for row in self.h.conn.execute("SELECT scope, body, author FROM notes WHERE replaced_by IS NULL " for row in self.h.conn.execute("SELECT scope, body, author FROM notes WHERE replaced_by IS NULL "
"AND body LIKE 'DEFERRED %' AND scope LIKE 'fn:0x%'"): "AND body LIKE 'DEFERRED %' AND scope LIKE 'fn:0x%'"):
m = re.match(r"DEFERRED ([\w-]+): (.*)", row["body"], re.S) m = re.match(r"DEFERRED ([\w-]+): (.*)", row["body"], re.S)
if m: if m:
out[int(row["scope"][3:], 16)] = (m.group(1), m.group(2).strip(), row["author"]) out[int(row["scope"][3:], 16)] = (m.group(1), m.group(2).strip(), row["author"])
return out return out
def bands(self, agent): def bands(self, agent):
return [(row["lo"], row["hi"]) for row in self.h.conn.execute( return [(row["lo"], row["hi"]) for row in self.h.conn.execute(
"SELECT lo, hi FROM claims WHERE kind='band' AND agent=? AND released_at IS NULL", (agent,)) "SELECT lo, hi FROM claims WHERE kind='band' AND agent=? AND released_at IS NULL", (agent,))
if row["lo"] is not None] if row["lo"] is not None]
def claimed_by_others(self, agent): def claimed_by_others(self, agent):
return [(row["lo"], row["hi"]) for row in self.h.conn.execute( return [(row["lo"], row["hi"]) for row in self.h.conn.execute(
"SELECT lo, hi FROM claims WHERE kind IN ('draft','edit','naming') AND agent<>? AND released_at IS NULL", "SELECT lo, hi FROM claims WHERE kind IN ('draft','edit','naming') AND agent<>? AND released_at IS NULL",
(agent,)) if row["lo"] is not None] (agent,)) if row["lo"] is not None]
def _named(self): def in_review(self):
names = self.r._csv("names.csv") """(lo, hi) of every item not yet landed or closed, from any author: work already done on a branch"""
return {a for a, row in names.items() if len(row) > 1 and row[1] and not re.match(r"(FUN|sub|thunk)_", row[1])} out = []
for row in self.h.conn.execute("SELECT scope, title FROM items WHERE state IN "
def _category(self, addr): "('submitted','in_review','approved','changes_requested','held')"):
row = self.r._csv("names.csv").get(addr) lo = None
if row and len(row) > 2 and row[2]: if row["scope"]:
return row[2] try:
return (self.r._facts().get(addr) or {}).get("category", "unsorted") _, lo, hi = parse_scope(row["scope"])
except HarborError:
def _results(self, key): lo = None
path = self.config.get(key) if lo is not None:
if not path or not Path(path).exists(): out.append((lo, hi))
return {} else: # no scope given: the addresses its title names
if str(path).endswith(".csv"): out += [(int(a, 16), int(a, 16)) for a in re.findall(r"\b0x([0-9A-Fa-f]{6})\b", row["title"] or "")]
with open(path, encoding="utf-8") as f: return out
return {int(row["address"], 16): row for row in csv.DictReader(f)}
return {int(row["address"], 16): row for row in json.loads(Path(path).read_text(encoding="utf-8"))} def _named(self):
names = self.r._csv("names.csv")
def similar_readable(self): return {a for a, row in names.items() if len(row) > 1 and row[1] and not re.match(r"(FUN|sub|thunk)_", row[1])}
"""Raw functions with a readable near-duplicate (from the [reverse] similar_clusters file)."""
path = self.config.get("similar_clusters") def _category(self, addr):
if not path or not Path(path).exists(): row = self.r._csv("names.csv").get(addr)
return frozenset() if row and len(row) > 2 and row[2]:
data = json.loads(Path(path).read_text(encoding="utf-8")) return row[2]
return frozenset(int(m["address"], 16) for c in data.get("clusters", []) for m in c.get("members", []) return (self.r._facts().get(addr) or {}).get("category", "unsorted")
if m.get("best_readable") and m.get("status") != "readable")
def _results(self, key):
def library_ranges(self): path = self.config.get(key)
"""[(start, end)] of statically linked library blocks (the [reverse] library_ranges csv).""" if not path or not Path(path).exists():
path = self.config.get("library_ranges") return {}
if not path or not Path(path).exists(): if str(path).endswith(".csv"):
return [] with open(path, encoding="utf-8") as f:
with open(path, encoding="utf-8") as f: return {int(row["address"], 16): row for row in csv.DictReader(f)}
return [(int(r["start"], 16), int(r["end"], 16)) for r in csv.DictReader(f) return {int(row["address"], 16): row for row in json.loads(Path(path).read_text(encoding="utf-8"))}
if r.get("verdict", "").startswith("D3DX") or "stubs" in r.get("verdict", "")]
# the D3DX9 block, and the compiler's import thunks and exception-unwind funclets: nothing to decompile def similar_readable(self):
"""Raw functions with a readable near-duplicate (from the [reverse] similar_clusters file)."""
def is_library(self, addr, library, ranges): path = self.config.get("similar_clusters")
return addr in library or any(lo <= addr < hi for lo, hi in ranges) if not path or not Path(path).exists():
return frozenset()
# ------------------------------------------------------------------ queue data = json.loads(Path(path).read_text(encoding="utf-8"))
def score(self, addr, facts, named, rejected, similar_readable=frozenset(), with_convention=True): return frozenset(int(m["address"], 16) for c in data.get("clusters", []) for m in c.get("members", [])
d = facts[addr] if m.get("best_readable") and m.get("status") != "readable")
size = int(d.get("size") or 0)
calls = [int(c, 16) if isinstance(c, str) else c for c in d.get("calls", [])] def library_ranges(self):
named_share = (sum(1 for c in calls if c in named) / len(calls)) if calls else 1.0 """[(start, end)] of statically linked library blocks (the [reverse] library_ranges csv)."""
score = 1.0 / (1.0 + size / 250.0) # small first path = self.config.get("library_ranges")
score *= 0.5 + 0.5 * named_share # callees with names make readable code possible if not path or not Path(path).exists():
conv = self.r.convention(addr) if with_convention and size <= 4000 else None return []
if conv and conv["summary"].startswith("custom"): with open(path, encoding="utf-8") as f:
score *= 0.75 # register arguments: prototypes.h work too return [(int(r["start"], 16), int(r["end"], 16)) for r in csv.DictReader(f)
if self._category(addr) not in ("game", "unsorted"): if r.get("verdict", "").startswith("D3DX") or "stubs" in r.get("verdict", "")]
score *= 0.6 # the D3DX9 block, and the compiler's import thunks and exception-unwind funclets: nothing to decompile
score *= 0.5 ** rejected.get(addr, 0)
low = next((label for lo, hi, label in self.low_priority if lo <= addr < hi), None) def is_library(self, addr, library, ranges):
if low: return addr in library or any(lo <= addr < hi for lo, hi in ranges)
score *= 0.2 # e.g. the compiler's static initializers
reasons_low = [low] # ------------------------------------------------------------------ queue
else: def score(self, addr, facts, named, rejected, similar_readable=frozenset(), with_convention=True):
reasons_low = [] d = facts[addr]
if addr in similar_readable: size = int(d.get("size") or 0)
score *= 1.5 # a readable near-duplicate to start from calls = [int(c, 16) if isinstance(c, str) else c for c in d.get("calls", [])]
reasons = [f"{size} bytes", f"{round(named_share * 100)}% of {len(calls)} callees named"] named_share = (sum(1 for c in calls if c in named) / len(calls)) if calls else 1.0
if conv: score = 1.0 / (1.0 + size / 250.0) # small first
reasons.append(conv["summary"]) score *= 0.5 + 0.5 * named_share # callees with names make readable code possible
if rejected.get(addr): conv = self.r.convention(addr) if with_convention and size <= 4000 else None
reasons.append(f"rejected {rejected[addr]}x") if conv and conv["summary"].startswith("custom"):
if addr in similar_readable: score *= 0.75 # register arguments: prototypes.h work too
reasons.append("has a readable near-duplicate") if self._category(addr) not in ("game", "unsorted"):
reasons += reasons_low score *= 0.6
return score, reasons score *= 0.5 ** rejected.get(addr, 0)
low = next((label for lo, hi, label in self.low_priority if lo <= addr < hi), None)
def next_functions(self, agent, n=10, similar_readable=frozenset()): if low:
bands = self.bands(agent) score *= 0.2 # e.g. the compiler's static initializers
if not bands: reasons_low = [low]
raise HarborError(f"{agent} has no band; ask the coordinator for one") else:
taken = self.claimed_by_others(agent) reasons_low = []
statuses, deferred, rejected = self.statuses(), self.deferrals(), self.rejections() if addr in similar_readable:
facts, named = self.r._facts(), self._named() score *= 1.5 # a readable near-duplicate to start from
library = self._results("library_results") reasons = [f"{size} bytes", f"{round(named_share * 100)}% of {len(calls)} callees named"]
ranges = self.library_ranges() if conv:
ranked = [] reasons.append(conv["summary"])
for addr in facts: if rejected.get(addr):
if not any(lo <= addr <= hi for lo, hi in bands) or any(lo <= addr <= hi for lo, hi in taken): reasons.append(f"rejected {rejected[addr]}x")
continue if addr in similar_readable:
if statuses.get(addr, ("raw",))[0] == "readable" or addr in deferred or self.is_library(addr, library, ranges): reasons.append("has a readable near-duplicate")
continue reasons += reasons_low
if rejected.get(addr, 0) >= self.cap: return score, reasons
continue
if self._category(addr) in ("library", "platform"): def next_functions(self, agent, n=10, similar_readable=frozenset()):
continue bands = self.bands(agent)
if int(facts[addr].get("size") or 0) < self.min_size or \ if not bands:
re.match(r"(nullsub|thunk)_", self.r.name_of(addr) or "") or \ raise HarborError(f"{agent} has no band; ask the coordinator for one")
re.match(r"thunk_", (statuses.get(addr) or (None, ""))[1].split("_", 1)[-1]): taken = self.claimed_by_others(agent) + self.in_review() # claimed elsewhere, or already in an item
continue # stubs and thunks are named in bulk, not decompiled statuses, deferred, rejected = self.statuses(), self.deferrals(), self.rejections()
head = self.r.read(addr, 10) if int(facts[addr].get("size") or 0) == 10 else b"" facts, named = self.r._facts(), self._named()
if head[:1] and 0xB8 <= head[0] <= 0xBF and head[5:6] == b"\xe9": library = self._results("library_results")
continue # mov reg, imm32; jmp: an at-exit destructor thunk or an SEH handler stub (compiler's) ranges = self.library_ranges()
score, reasons = self.score(addr, facts, named, rejected, similar_readable, with_convention=False) ranked = []
ranked.append((score, addr, reasons)) for addr in facts:
ranked.sort(reverse=True) if not any(lo <= addr <= hi for lo, hi in bands) or any(lo <= addr <= hi for lo, hi in taken):
# the convention read from the bytes only lowers a score, and it is the slow part: read it for the head only continue
head = [(*self.score(a, facts, named, rejected, similar_readable), a) for _, a, _ in ranked[:max(4 * n, 40)]] if statuses.get(addr, ("raw",))[0] == "readable" or addr in deferred or self.is_library(addr, library, ranges):
ranked = sorted(((s, a, why) for s, why, a in head), reverse=True) continue
return [{"address": f"0x{a:X}", "name": self.r.name_of(a), "score": round(s, 4), "why": ", ".join(why), if rejected.get(addr, 0) >= self.cap:
"file": statuses.get(a, (None, None))[1]} for s, a, why in ranked[:n]] continue
if self._category(addr) in ("library", "platform"):
def defer(self, agent, address, reason, evidence): continue
if reason not in DEFER_REASONS: if int(facts[addr].get("size") or 0) < self.min_size or \
raise HarborError("reason must be one of " + ", ".join(DEFER_REASONS)) re.match(r"(nullsub|thunk)_", self.r.name_of(addr) or "") or \
if not (evidence or "").strip(): re.match(r"thunk_", (statuses.get(addr) or (None, ""))[1].split("_", 1)[-1]):
raise HarborError("say what you found (evidence), so the next person does not start from zero") continue # stubs and thunks are named in bulk, not decompiled
addr = self.r.resolve(address) head = self.r.read(addr, 10) if int(facts[addr].get("size") or 0) == 10 else b""
func = self.r.function_at(addr) if head[:1] and 0xB8 <= head[0] <= 0xBF and head[5:6] == b"\xe9":
start = func[0] if func else addr continue # mov reg, imm32; jmp: an at-exit destructor thunk or an SEH handler stub (compiler's)
return self.h.note(agent, f"fn:0x{start:X}", f"DEFERRED {reason}: {evidence.strip()}") score, reasons = self.score(addr, facts, named, rejected, similar_readable, with_convention=False)
ranked.append((score, addr, reasons))
# ------------------------------------------------------------------ progress ranked.sort(reverse=True)
def progress(self): # the convention read from the bytes only lowers a score, and it is the slow part: read it for the head only
facts, statuses, deferred = self.r._facts(), self.statuses(), self.deferrals() head = [(*self.score(a, facts, named, rejected, similar_readable), a) for _, a, _ in ranked[:max(4 * n, 40)]]
exact = {a for a, row in self._results("match_results").items() ranked = sorted(((s, a, why) for s, why, a in head), reverse=True)
if str(row.get("status")) in ("match", "exact", "exact-relaxed")} return [{"address": f"0x{a:X}", "name": self.r.name_of(a), "score": round(s, 4), "why": ", ".join(why),
# behaves the same in the emulator, with at least one trial that ran to the end in both (not 'weak') "file": statuses.get(a, (None, None))[1]} for s, a, why in ranked[:n]]
verified = {a for a, row in self._results("diffrun_results").items()
if str(row.get("verdict")) == "same" and not row.get("weak")} def defer(self, agent, address, reason, evidence):
library = set(self._results("library_results")) if reason not in DEFER_REASONS:
ranges = self.library_ranges() raise HarborError("reason must be one of " + ", ".join(DEFER_REASONS))
tiers = {t: {"functions": 0, "bytes": 0} for t in TIER_ORDER + ("library", "engine")} if not (evidence or "").strip():
for addr, d in facts.items(): raise HarborError("say what you found (evidence), so the next person does not start from zero")
size = int(d.get("size") or 0) addr = self.r.resolve(address)
category = self._category(addr) func = self.r.function_at(addr)
readable = statuses.get(addr, ("raw",))[0] == "readable" start = func[0] if func else addr
if self.is_library(addr, library, ranges) or category in ("library", "platform", "crt"): return self.h.note(agent, f"fn:0x{start:X}", f"DEFERRED {reason}: {evidence.strip()}")
tier = "library"
elif category == "engine" and not readable: # ------------------------------------------------------------------ progress
tier = "engine" def progress(self):
elif addr in exact: facts, statuses, deferred = self.r._facts(), self.statuses(), self.deferrals()
tier = "exact" exact = {a for a, row in self._results("match_results").items()
elif addr in verified: if str(row.get("status")) in ("match", "exact", "exact-relaxed")}
tier = "verified" # behaves the same in the emulator, with at least one trial that ran to the end in both (not 'weak')
elif readable: verified = {a for a, row in self._results("diffrun_results").items()
tier = "readable" if str(row.get("verdict")) == "same" and not row.get("weak")}
elif addr in deferred: library = set(self._results("library_results"))
tier = "deferred" ranges = self.library_ranges()
else: tiers = {t: {"functions": 0, "bytes": 0} for t in TIER_ORDER + ("library", "engine")}
tier = "raw" for addr, d in facts.items():
tiers[tier]["functions"] += 1 size = int(d.get("size") or 0)
tiers[tier]["bytes"] += size category = self._category(addr)
sources = {k: bool(self.config.get(k) and Path(self.config[k]).exists()) readable = statuses.get(addr, ("raw",))[0] == "readable"
for k in ("match_results", "diffrun_results", "library_results")} if self.is_library(addr, library, ranges) or category in ("library", "platform", "crt"):
return {"tiers": tiers, "deferred_reasons": _count(r for r, _, _ in deferred.values()), "sources": sources} tier = "library"
elif category == "engine" and not readable:
tier = "engine"
def _count(values): elif addr in exact:
out = {} tier = "exact"
for v in values: elif addr in verified:
out[v] = out.get(v, 0) + 1 tier = "verified"
return out elif readable:
tier = "readable"
elif addr in deferred:
tier = "deferred"
else:
tier = "raw"
tiers[tier]["functions"] += 1
tiers[tier]["bytes"] += size
sources = {k: bool(self.config.get(k) and Path(self.config[k]).exists())
for k in ("match_results", "diffrun_results", "library_results")}
return {"tiers": tiers, "deferred_reasons": _count(r for r, _, _ in deferred.values()), "sources": sources}
def _count(values):
out = {}
for v in values:
out[v] = out.get(v, 0) + 1
return out