From a3fc48802cf13dad547352437e890bacd6f8fa7a Mon Sep 17 00:00:00 2001 From: KeelrLeek <156356974+KeelrLeek@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:35:55 -0500 Subject: [PATCH] logbook: statistics run on a clean detached snapshot of main ([logbook] snapshot), not the integrator's live checkout, which is often mid-batch; a failed run retries on the next tick instead of waiting for main to move --- harbor/logbook_auto.py | 321 ++++++++++++++++++++++------------------- 1 file changed, 169 insertions(+), 152 deletions(-) diff --git a/harbor/logbook_auto.py b/harbor/logbook_auto.py index 92c3124..8b7c58e 100644 --- a/harbor/logbook_auto.py +++ b/harbor/logbook_auto.py @@ -1,152 +1,169 @@ -"""Keeps the Logbook current without anyone writing to it: when the decomp repository's main branch moves, - - 1. each new commit on main becomes a ship's log entry (its subject and the first lines of its message), and - 2. the statistics, the chart of the executable and a trend point are rebuilt with the open-seas progress tools, - run on a scratch copy of that repository (the real one stays untouched). - -Configured per project in a [logbook] table: - auto = true - decomp = 'R:/.../open-seas-decomp' the repository whose main branch is followed - stats_repo = 'R:/.../stats-work/repo' a scratch copy of open-seas holding tools/progress/*.py - interval_minutes = 10 how often the server looks (default 10) -Milestones and narrative entries stay with people: they need judgement. - -State (the last commit seen) is kept in the logbook folder as auto.json.""" -import json -import os -import re -import subprocess -import sys -import tempfile -import time -import traceback -from pathlib import Path - -from . import logbook - -MAX_ENTRIES = 25 - - -def _git(repo, *args): - result = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, encoding="utf-8", - errors="replace", timeout=60) - if result.returncode != 0: - raise RuntimeError(f"git {' '.join(args)}: {result.stderr.strip()[:300]}") - return result.stdout - - -def _state(project): - path = logbook.folder(project) / "auto.json" - return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} - - -def _save_state(project, state): - path = logbook.folder(project) / "auto.json" - path.write_text(json.dumps(state, indent=1), encoding="utf-8") - - -def _utc(iso): - """git's %cI (local time with an offset) as UTC, the Logbook's format.""" - import datetime - return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def _kind(files): - if files and all(f.startswith("annotations/") for f in files): - return "naming" - if any(f.startswith("src/") for f in files): - return "progress" - return "note" - - -def log_commits(project, repo, since, head): - """One ship's log entry per commit on main after `since` (oldest first); a summary when there are many.""" - raw = _git(repo, "log", "--reverse", "--no-merges", "--format=%H%x1f%cI%x1f%s%x1f%b%x1e", f"{since}..{head}") - # str.strip() counts \x1f as whitespace and would eat an empty body's field, so trim newlines only and pad - commits = [(c.strip("\r\n").split("\x1f") + [""] * 4)[:4] for c in raw.split("\x1e") if c.strip("\r\n")] - added = [] - if len(commits) > MAX_ENTRIES: - subjects = "\n".join(f"- {c[2]} ({c[0][:8]})" for c in commits[-40:]) - added.append(logbook.add_entry(project, f"{len(commits)} commits landed on main", - subjects, kind="progress", at=_utc(commits[-1][1]))) - return added - for sha, when, subject, body in commits: - files = [f for f in _git(repo, "show", "--name-only", "--format=", sha).splitlines() if f.strip()] - lines = [l for l in body.strip().splitlines() if l.strip() and not l.lower().startswith("co-authored-by")] - detail = "\n".join(lines[:8]) if lines else subject - detail += f"\n(main {sha[:8]}, {len(files)} file{'s' if len(files) != 1 else ''})" - added.append(logbook.add_entry(project, subject, detail, kind=_kind(files), at=_utc(when))) - return added - - -def rebuild_stats(project, repo, stats_repo, head): - """Run the open-seas progress tools on the scratch copy and store their statistics and chart.""" - tools = Path(stats_repo) / "tools" / "progress" - # its own data dir: the agents' compiles hold the shared decomp-build\pirates.h open - env = dict(os.environ, OPENSEAS_DATA=os.environ.get("HARBOR_LOGBOOK_DATA", r"R:\Workspace\Data\openseas-logbook")) - # the readable build finds the compiler through vswhere, which a server started without a shell's PATH lacks - installer = r"C:\Program Files (x86)\Microsoft Visual Studio\Installer" - if os.path.isdir(installer) and installer.lower() not in env.get("PATH", "").lower(): - env["PATH"] = installer + os.pathsep + env.get("PATH", "") - run = lambda *argv: subprocess.run([sys.executable, "-B", *map(str, argv)], cwd=str(stats_repo), env=env, - capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=900) - sync = run(tools / "sync_decomp.py", "--decomp", repo) - if sync.returncode != 0: - raise RuntimeError("sync_decomp: " + (sync.stdout + sync.stderr).strip()[-400:]) - report = run(tools / "progress.py") - if report.returncode != 0: - raise RuntimeError("progress: " + (report.stdout + report.stderr).strip()[-400:]) - with tempfile.TemporaryDirectory() as out: - payload = run(tools / "dashboard_payload.py", out) - if payload.returncode != 0: - raise RuntimeError("dashboard_payload: " + (payload.stdout + payload.stderr).strip()[-400:]) - stats = json.loads((Path(out) / "stats_current.json").read_text(encoding="utf-8")) - chart = json.loads((Path(out) / "stats_map.json").read_text(encoding="utf-8")) - stats["commit"] = head[:7] - stats["updated_at"] = logbook.now() - logbook.set_stats(project, stats) - logbook.set_map(project, chart) - return {"classified": stats.get("classified_percent"), "weighted": stats.get("weighted_percent")} - - -def refresh(project, force=False): - """Bring the Logbook up to main. Returns what changed.""" - config = getattr(project, "logbook", None) or {} - repo, stats_repo = config.get("decomp"), config.get("stats_repo") - if not repo: - return {"changed": False, "reason": "no [logbook] decomp"} - head = _git(repo, "rev-parse", "HEAD").strip() - state = _state(project) - if state.get("last_head") == head and not force: - return {"changed": False, "head": head[:8]} - out = {"changed": True, "head": head[:8]} - if state.get("last_head"): - try: - out["entries"] = len(log_commits(project, repo, state["last_head"], head)) - except RuntimeError as error: # e.g. the old commit vanished after a rewrite - out["log_error"] = str(error) - if stats_repo: - try: - out["stats"] = rebuild_stats(project, repo, stats_repo, head) - except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as error: - out["stats_error"] = str(error) - _save_state(project, {**state, "last_head": head, "last_refresh": logbook.now(), - **({"last_error": out.get("stats_error") or out.get("log_error")} - if out.get("stats_error") or out.get("log_error") else {"last_error": None})}) - return out - - -def loop(registry): - """Server thread: refresh every project with [logbook] auto = true.""" - while True: - wait = 600 - for name in registry.names(): - try: - h, _ = registry.get(name) - config = getattr(h.project, "logbook", None) or {} - if config.get("auto"): - wait = min(wait, int(config.get("interval_minutes", 10)) * 60) - refresh(h.project) - except Exception: - traceback.print_exc() - time.sleep(max(60, wait)) +"""Keeps the Logbook current without anyone writing to it: when the decomp repository's main branch moves, + + 1. each new commit on main becomes a ship's log entry (its subject and the first lines of its message), and + 2. the statistics, the chart of the executable and a trend point are rebuilt with the open-seas progress tools, + run on a scratch copy of that repository (the real one stays untouched). + +Configured per project in a [logbook] table: + auto = true + decomp = 'R:/.../open-seas-decomp' the repository whose main branch is followed + stats_repo = 'R:/.../stats-work/repo' a scratch copy of open-seas holding tools/progress/*.py + interval_minutes = 10 how often the server looks (default 10) +Milestones and narrative entries stay with people: they need judgement. + +State (the last commit seen) is kept in the logbook folder as auto.json.""" +import json +import os +import re +import subprocess +import sys +import tempfile +import time +import traceback +from pathlib import Path + +from . import logbook + +MAX_ENTRIES = 25 + + +def _git(repo, *args): + result = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, encoding="utf-8", + errors="replace", timeout=60) + if result.returncode != 0: + raise RuntimeError(f"git {' '.join(args)}: {result.stderr.strip()[:300]}") + return result.stdout + + +def _state(project): + path = logbook.folder(project) / "auto.json" + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} + + +def _save_state(project, state): + path = logbook.folder(project) / "auto.json" + path.write_text(json.dumps(state, indent=1), encoding="utf-8") + + +def _utc(iso): + """git's %cI (local time with an offset) as UTC, the Logbook's format.""" + import datetime + return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _kind(files): + if files and all(f.startswith("annotations/") for f in files): + return "naming" + if any(f.startswith("src/") for f in files): + return "progress" + return "note" + + +def log_commits(project, repo, since, head): + """One ship's log entry per commit on main after `since` (oldest first); a summary when there are many.""" + raw = _git(repo, "log", "--reverse", "--no-merges", "--format=%H%x1f%cI%x1f%s%x1f%b%x1e", f"{since}..{head}") + # str.strip() counts \x1f as whitespace and would eat an empty body's field, so trim newlines only and pad + commits = [(c.strip("\r\n").split("\x1f") + [""] * 4)[:4] for c in raw.split("\x1e") if c.strip("\r\n")] + added = [] + if len(commits) > MAX_ENTRIES: + subjects = "\n".join(f"- {c[2]} ({c[0][:8]})" for c in commits[-40:]) + added.append(logbook.add_entry(project, f"{len(commits)} commits landed on main", + subjects, kind="progress", at=_utc(commits[-1][1]))) + return added + for sha, when, subject, body in commits: + files = [f for f in _git(repo, "show", "--name-only", "--format=", sha).splitlines() if f.strip()] + lines = [l for l in body.strip().splitlines() if l.strip() and not l.lower().startswith("co-authored-by")] + detail = "\n".join(lines[:8]) if lines else subject + detail += f"\n(main {sha[:8]}, {len(files)} file{'s' if len(files) != 1 else ''})" + added.append(logbook.add_entry(project, subject, detail, kind=_kind(files), at=_utc(when))) + return added + + +def rebuild_stats(project, repo, stats_repo, head): + """Run the open-seas progress tools on the scratch copy and store their statistics and chart.""" + tools = Path(stats_repo) / "tools" / "progress" + # its own data dir: the agents' compiles hold the shared decomp-build\pirates.h open + env = dict(os.environ, OPENSEAS_DATA=os.environ.get("HARBOR_LOGBOOK_DATA", r"R:\Workspace\Data\openseas-logbook")) + # the readable build finds the compiler through vswhere, which a server started without a shell's PATH lacks + installer = r"C:\Program Files (x86)\Microsoft Visual Studio\Installer" + if os.path.isdir(installer) and installer.lower() not in env.get("PATH", "").lower(): + env["PATH"] = installer + os.pathsep + env.get("PATH", "") + run = lambda *argv: subprocess.run([sys.executable, "-B", *map(str, argv)], cwd=str(stats_repo), env=env, + capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=900) + sync = run(tools / "sync_decomp.py", "--decomp", repo) + if sync.returncode != 0: + raise RuntimeError("sync_decomp: " + (sync.stdout + sync.stderr).strip()[-400:]) + report = run(tools / "progress.py") + if report.returncode != 0: + raise RuntimeError("progress: " + (report.stdout + report.stderr).strip()[-400:]) + with tempfile.TemporaryDirectory() as out: + payload = run(tools / "dashboard_payload.py", out) + if payload.returncode != 0: + raise RuntimeError("dashboard_payload: " + (payload.stdout + payload.stderr).strip()[-400:]) + stats = json.loads((Path(out) / "stats_current.json").read_text(encoding="utf-8")) + chart = json.loads((Path(out) / "stats_map.json").read_text(encoding="utf-8")) + stats["commit"] = head[:7] + stats["updated_at"] = logbook.now() + logbook.set_stats(project, stats) + logbook.set_map(project, chart) + return {"classified": stats.get("classified_percent"), "weighted": stats.get("weighted_percent")} + + +def snapshot_at(config, repo, head): + """A clean checkout of `head` for the statistics: [logbook] snapshot, a detached worktree of the decomp repository + that only this refresher uses. The live checkout is the integrator's and is often mid-batch with uncommitted + files, which the statistics tools rightly refuse. Without a snapshot setting, the live checkout is used.""" + snapshot = config.get("snapshot") + if not snapshot: + return repo + path = Path(snapshot) + if not (path / ".git").exists(): + _git(repo, "worktree", "add", "--detach", str(path), head) + else: + _git(str(path), "checkout", "--detach", "--force", head) + return str(path) + + +def refresh(project, force=False): + """Bring the Logbook up to main. Returns what changed.""" + config = getattr(project, "logbook", None) or {} + repo, stats_repo = config.get("decomp"), config.get("stats_repo") + if not repo: + return {"changed": False, "reason": "no [logbook] decomp"} + head = _git(repo, "rev-parse", "HEAD").strip() + state = _state(project) + if state.get("last_head") == head and not force: + return {"changed": False, "head": head[:8]} + out = {"changed": True, "head": head[:8]} + if state.get("last_head"): + try: + out["entries"] = len(log_commits(project, repo, state["last_head"], head)) + except RuntimeError as error: # e.g. the old commit vanished after a rewrite + out["log_error"] = str(error) + if stats_repo: + try: + out["stats"] = rebuild_stats(project, snapshot_at(config, repo, head), stats_repo, head) + except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as error: + out["stats_error"] = str(error) + # a failed statistics run is retried next time, instead of waiting for main to move again + done_head = state.get("last_head") if out.get("stats_error") else head + _save_state(project, {**state, "last_head": done_head, "last_refresh": logbook.now(), + **({"last_error": out.get("stats_error") or out.get("log_error")} + if out.get("stats_error") or out.get("log_error") else {"last_error": None})}) + return out + + +def loop(registry): + """Server thread: refresh every project with [logbook] auto = true.""" + while True: + wait = 600 + for name in registry.names(): + try: + h, _ = registry.get(name) + config = getattr(h.project, "logbook", None) or {} + if config.get("auto"): + wait = min(wait, int(config.get("interval_minutes", 10)) * 60) + refresh(h.project) + except Exception: + traceback.print_exc() + time.sleep(max(60, wait))