Keelrleek/Harbor
1
1 Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

logbook: statistics run on a clean detached snapshot of main ([logbook] snapshot), not the integrator's live checkout, which is often mid-batch; a failed run retries on the next tick instead of waiting for main to move

Browse Source
This commit is contained in:
KeelrLeek committed 2026-10-07 10:35:55 -05:00
1 parent d7464aaa40
commit a3fc48802c
1 file changed
+169 -152
Download Patch File Download Diff File Expand all files Collapse all files
harbor/logbook_auto.py
+169 -152
View File
@@ -1,152 +1,169 @@
"""Keeps the Logbook current without anyone writing to it: when the decomp repository's main branch moves, """Keeps the Logbook current without anyone writing to it: when the decomp repository's main branch moves,
1. each new commit on main becomes a ship's log entry (its subject and the first lines of its message), and 1. each new commit on main becomes a ship's log entry (its subject and the first lines of its message), and
2. the statistics, the chart of the executable and a trend point are rebuilt with the open-seas progress tools, 2. the statistics, the chart of the executable and a trend point are rebuilt with the open-seas progress tools,
run on a scratch copy of that repository (the real one stays untouched). run on a scratch copy of that repository (the real one stays untouched).
Configured per project in a [logbook] table: Configured per project in a [logbook] table:
auto = true auto = true
decomp = 'R:/.../open-seas-decomp' the repository whose main branch is followed decomp = 'R:/.../open-seas-decomp' the repository whose main branch is followed
stats_repo = 'R:/.../stats-work/repo' a scratch copy of open-seas holding tools/progress/*.py stats_repo = 'R:/.../stats-work/repo' a scratch copy of open-seas holding tools/progress/*.py
interval_minutes = 10 how often the server looks (default 10) interval_minutes = 10 how often the server looks (default 10)
Milestones and narrative entries stay with people: they need judgement. Milestones and narrative entries stay with people: they need judgement.
State (the last commit seen) is kept in the logbook folder as auto.json.""" State (the last commit seen) is kept in the logbook folder as auto.json."""
import json import json
import os import os
import re import re
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import time import time
import traceback import traceback
from pathlib import Path from pathlib import Path
from . import logbook from . import logbook
MAX_ENTRIES = 25 MAX_ENTRIES = 25
def _git(repo, *args): def _git(repo, *args):
result = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, encoding="utf-8", result = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, encoding="utf-8",
errors="replace", timeout=60) errors="replace", timeout=60)
if result.returncode != 0: if result.returncode != 0:
raise RuntimeError(f"git {' '.join(args)}: {result.stderr.strip()[:300]}") raise RuntimeError(f"git {' '.join(args)}: {result.stderr.strip()[:300]}")
return result.stdout return result.stdout
def _state(project): def _state(project):
path = logbook.folder(project) / "auto.json" path = logbook.folder(project) / "auto.json"
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
def _save_state(project, state): def _save_state(project, state):
path = logbook.folder(project) / "auto.json" path = logbook.folder(project) / "auto.json"
path.write_text(json.dumps(state, indent=1), encoding="utf-8") path.write_text(json.dumps(state, indent=1), encoding="utf-8")
def _utc(iso): def _utc(iso):
"""git's %cI (local time with an offset) as UTC, the Logbook's format.""" """git's %cI (local time with an offset) as UTC, the Logbook's format."""
import datetime import datetime
return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _kind(files): def _kind(files):
if files and all(f.startswith("annotations/") for f in files): if files and all(f.startswith("annotations/") for f in files):
return "naming" return "naming"
if any(f.startswith("src/") for f in files): if any(f.startswith("src/") for f in files):
return "progress" return "progress"
return "note" return "note"
def log_commits(project, repo, since, head): def log_commits(project, repo, since, head):
"""One ship's log entry per commit on main after `since` (oldest first); a summary when there are many.""" """One ship's log entry per commit on main after `since` (oldest first); a summary when there are many."""
raw = _git(repo, "log", "--reverse", "--no-merges", "--format=%H%x1f%cI%x1f%s%x1f%b%x1e", f"{since}..{head}") raw = _git(repo, "log", "--reverse", "--no-merges", "--format=%H%x1f%cI%x1f%s%x1f%b%x1e", f"{since}..{head}")
# str.strip() counts \x1f as whitespace and would eat an empty body's field, so trim newlines only and pad # str.strip() counts \x1f as whitespace and would eat an empty body's field, so trim newlines only and pad
commits = [(c.strip("\r\n").split("\x1f") + [""] * 4)[:4] for c in raw.split("\x1e") if c.strip("\r\n")] commits = [(c.strip("\r\n").split("\x1f") + [""] * 4)[:4] for c in raw.split("\x1e") if c.strip("\r\n")]
added = [] added = []
if len(commits) > MAX_ENTRIES: if len(commits) > MAX_ENTRIES:
subjects = "\n".join(f"- {c[2]} ({c[0][:8]})" for c in commits[-40:]) subjects = "\n".join(f"- {c[2]} ({c[0][:8]})" for c in commits[-40:])
added.append(logbook.add_entry(project, f"{len(commits)} commits landed on main", added.append(logbook.add_entry(project, f"{len(commits)} commits landed on main",
subjects, kind="progress", at=_utc(commits[-1][1]))) subjects, kind="progress", at=_utc(commits[-1][1])))
return added return added
for sha, when, subject, body in commits: for sha, when, subject, body in commits:
files = [f for f in _git(repo, "show", "--name-only", "--format=", sha).splitlines() if f.strip()] files = [f for f in _git(repo, "show", "--name-only", "--format=", sha).splitlines() if f.strip()]
lines = [l for l in body.strip().splitlines() if l.strip() and not l.lower().startswith("co-authored-by")] lines = [l for l in body.strip().splitlines() if l.strip() and not l.lower().startswith("co-authored-by")]
detail = "\n".join(lines[:8]) if lines else subject detail = "\n".join(lines[:8]) if lines else subject
detail += f"\n(main {sha[:8]}, {len(files)} file{'s' if len(files) != 1 else ''})" detail += f"\n(main {sha[:8]}, {len(files)} file{'s' if len(files) != 1 else ''})"
added.append(logbook.add_entry(project, subject, detail, kind=_kind(files), at=_utc(when))) added.append(logbook.add_entry(project, subject, detail, kind=_kind(files), at=_utc(when)))
return added return added
def rebuild_stats(project, repo, stats_repo, head): def rebuild_stats(project, repo, stats_repo, head):
"""Run the open-seas progress tools on the scratch copy and store their statistics and chart.""" """Run the open-seas progress tools on the scratch copy and store their statistics and chart."""
tools = Path(stats_repo) / "tools" / "progress" tools = Path(stats_repo) / "tools" / "progress"
# its own data dir: the agents' compiles hold the shared decomp-build\pirates.h open # its own data dir: the agents' compiles hold the shared decomp-build\pirates.h open
env = dict(os.environ, OPENSEAS_DATA=os.environ.get("HARBOR_LOGBOOK_DATA", r"R:\Workspace\Data\openseas-logbook")) env = dict(os.environ, OPENSEAS_DATA=os.environ.get("HARBOR_LOGBOOK_DATA", r"R:\Workspace\Data\openseas-logbook"))
# the readable build finds the compiler through vswhere, which a server started without a shell's PATH lacks # the readable build finds the compiler through vswhere, which a server started without a shell's PATH lacks
installer = r"C:\Program Files (x86)\Microsoft Visual Studio\Installer" installer = r"C:\Program Files (x86)\Microsoft Visual Studio\Installer"
if os.path.isdir(installer) and installer.lower() not in env.get("PATH", "").lower(): if os.path.isdir(installer) and installer.lower() not in env.get("PATH", "").lower():
env["PATH"] = installer + os.pathsep + env.get("PATH", "") env["PATH"] = installer + os.pathsep + env.get("PATH", "")
run = lambda *argv: subprocess.run([sys.executable, "-B", *map(str, argv)], cwd=str(stats_repo), env=env, run = lambda *argv: subprocess.run([sys.executable, "-B", *map(str, argv)], cwd=str(stats_repo), env=env,
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=900) capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=900)
sync = run(tools / "sync_decomp.py", "--decomp", repo) sync = run(tools / "sync_decomp.py", "--decomp", repo)
if sync.returncode != 0: if sync.returncode != 0:
raise RuntimeError("sync_decomp: " + (sync.stdout + sync.stderr).strip()[-400:]) raise RuntimeError("sync_decomp: " + (sync.stdout + sync.stderr).strip()[-400:])
report = run(tools / "progress.py") report = run(tools / "progress.py")
if report.returncode != 0: if report.returncode != 0:
raise RuntimeError("progress: " + (report.stdout + report.stderr).strip()[-400:]) raise RuntimeError("progress: " + (report.stdout + report.stderr).strip()[-400:])
with tempfile.TemporaryDirectory() as out: with tempfile.TemporaryDirectory() as out:
payload = run(tools / "dashboard_payload.py", out) payload = run(tools / "dashboard_payload.py", out)
if payload.returncode != 0: if payload.returncode != 0:
raise RuntimeError("dashboard_payload: " + (payload.stdout + payload.stderr).strip()[-400:]) raise RuntimeError("dashboard_payload: " + (payload.stdout + payload.stderr).strip()[-400:])
stats = json.loads((Path(out) / "stats_current.json").read_text(encoding="utf-8")) stats = json.loads((Path(out) / "stats_current.json").read_text(encoding="utf-8"))
chart = json.loads((Path(out) / "stats_map.json").read_text(encoding="utf-8")) chart = json.loads((Path(out) / "stats_map.json").read_text(encoding="utf-8"))
stats["commit"] = head[:7] stats["commit"] = head[:7]
stats["updated_at"] = logbook.now() stats["updated_at"] = logbook.now()
logbook.set_stats(project, stats) logbook.set_stats(project, stats)
logbook.set_map(project, chart) logbook.set_map(project, chart)
return {"classified": stats.get("classified_percent"), "weighted": stats.get("weighted_percent")} return {"classified": stats.get("classified_percent"), "weighted": stats.get("weighted_percent")}
def refresh(project, force=False): def snapshot_at(config, repo, head):
"""Bring the Logbook up to main. Returns what changed.""" """A clean checkout of `head` for the statistics: [logbook] snapshot, a detached worktree of the decomp repository
config = getattr(project, "logbook", None) or {} that only this refresher uses. The live checkout is the integrator's and is often mid-batch with uncommitted
repo, stats_repo = config.get("decomp"), config.get("stats_repo") files, which the statistics tools rightly refuse. Without a snapshot setting, the live checkout is used."""
if not repo: snapshot = config.get("snapshot")
return {"changed": False, "reason": "no [logbook] decomp"} if not snapshot:
head = _git(repo, "rev-parse", "HEAD").strip() return repo
state = _state(project) path = Path(snapshot)
if state.get("last_head") == head and not force: if not (path / ".git").exists():
return {"changed": False, "head": head[:8]} _git(repo, "worktree", "add", "--detach", str(path), head)
out = {"changed": True, "head": head[:8]} else:
if state.get("last_head"): _git(str(path), "checkout", "--detach", "--force", head)
try: return str(path)
out["entries"] = len(log_commits(project, repo, state["last_head"], head))
except RuntimeError as error: # e.g. the old commit vanished after a rewrite
out["log_error"] = str(error) def refresh(project, force=False):
if stats_repo: """Bring the Logbook up to main. Returns what changed."""
try: config = getattr(project, "logbook", None) or {}
out["stats"] = rebuild_stats(project, repo, stats_repo, head) repo, stats_repo = config.get("decomp"), config.get("stats_repo")
except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as error: if not repo:
out["stats_error"] = str(error) return {"changed": False, "reason": "no [logbook] decomp"}
_save_state(project, {**state, "last_head": head, "last_refresh": logbook.now(), head = _git(repo, "rev-parse", "HEAD").strip()
**({"last_error": out.get("stats_error") or out.get("log_error")} state = _state(project)
if out.get("stats_error") or out.get("log_error") else {"last_error": None})}) if state.get("last_head") == head and not force:
return out return {"changed": False, "head": head[:8]}
out = {"changed": True, "head": head[:8]}
if state.get("last_head"):
def loop(registry): try:
"""Server thread: refresh every project with [logbook] auto = true.""" out["entries"] = len(log_commits(project, repo, state["last_head"], head))
while True: except RuntimeError as error: # e.g. the old commit vanished after a rewrite
wait = 600 out["log_error"] = str(error)
for name in registry.names(): if stats_repo:
try: try:
h, _ = registry.get(name) out["stats"] = rebuild_stats(project, snapshot_at(config, repo, head), stats_repo, head)
config = getattr(h.project, "logbook", None) or {} except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as error:
if config.get("auto"): out["stats_error"] = str(error)
wait = min(wait, int(config.get("interval_minutes", 10)) * 60) # a failed statistics run is retried next time, instead of waiting for main to move again
refresh(h.project) done_head = state.get("last_head") if out.get("stats_error") else head
except Exception: _save_state(project, {**state, "last_head": done_head, "last_refresh": logbook.now(),
traceback.print_exc() **({"last_error": out.get("stats_error") or out.get("log_error")}
time.sleep(max(60, wait)) if out.get("stats_error") or out.get("log_error") else {"last_error": None})})
return out
def loop(registry):
"""Server thread: refresh every project with [logbook] auto = true."""
while True:
wait = 600
for name in registry.names():
try:
h, _ = registry.get(name)
config = getattr(h.project, "logbook", None) or {}
if config.get("auto"):
wait = min(wait, int(config.get("interval_minutes", 10)) * 60)
refresh(h.project)
except Exception:
traceback.print_exc()
time.sleep(max(60, wait))