"""Keeps the Logbook current without anyone writing to it: when the decomp repository's main branch moves, 1. each new commit on main becomes a ship's log entry (its subject and the first lines of its message), and 2. the statistics, the chart of the executable and a trend point are rebuilt with the open-seas progress tools, run on a scratch copy of that repository (the real one stays untouched). Configured per project in a [logbook] table: auto = true decomp = 'R:/.../open-seas-decomp' the repository whose main branch is followed stats_repo = 'R:/.../stats-work/repo' a scratch copy of open-seas holding tools/progress/*.py interval_minutes = 10 how often the server looks (default 10) lander_lock = 'R:/.../lander-busy.lock' wait while the lander runs its gates (harbor.lander_gate) The progress tools run below normal priority. Milestones and narrative entries stay with people: they need judgement. State (the last commit seen) is kept in the logbook folder as auto.json.""" import json import os import re import subprocess import sys import tempfile import time import traceback from pathlib import Path from . import lander_gate, logbook MAX_ENTRIES = 25 STATS_TIMEOUT = 1800 # each progress tool; sync_decomp compiles every readable file, below normal priority def _git(repo, *args): result = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=60) if result.returncode != 0: raise RuntimeError(f"git {' '.join(args)}: {result.stderr.strip()[:300]}") return result.stdout def _state(project): path = logbook.folder(project) / "auto.json" return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} def _save_state(project, state): path = logbook.folder(project) / "auto.json" path.write_text(json.dumps(state, indent=1), encoding="utf-8") def _utc(iso): """git's %cI (local time with an offset) as UTC, the Logbook's format.""" import datetime return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def _kind(files): if files and all(f.startswith("annotations/") for f in files): return "naming" if any(f.startswith("src/") for f in files): return "progress" return "note" def log_commits(project, repo, since, head): """One ship's log entry per commit on main after `since` (oldest first); a summary when there are many.""" raw = _git(repo, "log", "--reverse", "--no-merges", "--format=%H%x1f%cI%x1f%s%x1f%b%x1e", f"{since}..{head}") # str.strip() counts \x1f as whitespace and would eat an empty body's field, so trim newlines only and pad commits = [(c.strip("\r\n").split("\x1f") + [""] * 4)[:4] for c in raw.split("\x1e") if c.strip("\r\n")] added = [] if len(commits) > MAX_ENTRIES: subjects = "\n".join(f"- {c[2]} ({c[0][:8]})" for c in commits[-40:]) added.append(logbook.add_entry(project, f"{len(commits)} commits landed on main", subjects, kind="progress", at=_utc(commits[-1][1]))) return added for sha, when, subject, body in commits: files = [f for f in _git(repo, "show", "--name-only", "--format=", sha).splitlines() if f.strip()] lines = [l for l in body.strip().splitlines() if l.strip() and not l.lower().startswith("co-authored-by")] detail = "\n".join(lines[:8]) + "\n" if lines else "" # the board shows the title already detail += f"(main {sha[:8]}, {len(files)} file{'s' if len(files) != 1 else ''})" added.append(logbook.add_entry(project, subject, detail, kind=_kind(files), at=_utc(when))) return added def rebuild_stats(project, repo, stats_repo, head): """Run the open-seas progress tools on the scratch copy and store their statistics and chart.""" tools = Path(stats_repo) / "tools" / "progress" # its own data dir: the agents' compiles hold the shared decomp-build\pirates.h open env = dict(os.environ, OPENSEAS_DATA=os.environ.get("HARBOR_LOGBOOK_DATA", r"R:\Workspace\Data\openseas-logbook")) # the readable build finds the compiler through vswhere, which a server started without a shell's PATH lacks installer = r"C:\Program Files (x86)\Microsoft Visual Studio\Installer" if os.path.isdir(installer) and installer.lower() not in env.get("PATH", "").lower(): env["PATH"] = installer + os.pathsep + env.get("PATH", "") # below normal priority (cl.exe inherits it), so the lander's and the agents' compiles go first; a timeout ends # the whole tree, not only python (an orphaned cl.exe went on compiling all of main at normal priority) run = lambda *argv: lander_gate.run_low([sys.executable, "-B", *map(str, argv)], STATS_TIMEOUT, cwd=str(stats_repo), env=env, encoding="utf-8", errors="replace") sync = run(tools / "sync_decomp.py", "--decomp", repo) if sync.returncode != 0: raise RuntimeError("sync_decomp: " + (sync.stdout + sync.stderr).strip()[-400:]) report = run(tools / "progress.py") if report.returncode != 0: raise RuntimeError("progress: " + (report.stdout + report.stderr).strip()[-400:]) with tempfile.TemporaryDirectory() as out: payload = run(tools / "dashboard_payload.py", out) if payload.returncode != 0: raise RuntimeError("dashboard_payload: " + (payload.stdout + payload.stderr).strip()[-400:]) stats = json.loads((Path(out) / "stats_current.json").read_text(encoding="utf-8")) chart = json.loads((Path(out) / "stats_map.json").read_text(encoding="utf-8")) stats["commit"] = head[:7] stats["updated_at"] = logbook.now() logbook.set_stats(project, stats) logbook.set_map(project, chart) return {"classified": stats.get("classified_percent"), "weighted": stats.get("weighted_percent")} def snapshot_at(config, repo, head): """A clean checkout of `head` for the statistics: [logbook] snapshot, a detached worktree of the decomp repository that only this refresher uses. The live checkout is the integrator's and is often mid-batch with uncommitted files, which the statistics tools rightly refuse. Without a snapshot setting, the live checkout is used.""" snapshot = config.get("snapshot") if not snapshot: return repo path = Path(snapshot) if not (path / ".git").exists(): _git(repo, "worktree", "add", "--detach", str(path), head) else: _git(str(path), "checkout", "--detach", "--force", head) return str(path) def refresh(project, force=False): """Bring the Logbook up to main. Returns what changed.""" config = getattr(project, "logbook", None) or {} repo, stats_repo = config.get("decomp"), config.get("stats_repo") if not repo: return {"changed": False, "reason": "no [logbook] decomp"} head = _git(repo, "rev-parse", "HEAD").strip() state = _state(project) if state.get("last_head") == head and not force: return {"changed": False, "head": head[:8]} busy = None if force else lander_gate.lander_busy(project) if busy: # its gates compile all of main: look again next time return {"changed": False, "head": head[:8], "waiting": busy} out = {"changed": True, "head": head[:8]} # last_head is where the statistics are; logged_head where the ship's log is (a failed statistics run is retried, # and must not log the same commits twice) logged = state.get("logged_head") or state.get("last_head") if logged and logged != head: try: out["entries"] = len(log_commits(project, repo, logged, head)) except RuntimeError as error: # e.g. the old commit vanished after a rewrite out["log_error"] = str(error) if logged: state["logged_head"] = head if stats_repo: try: out["stats"] = rebuild_stats(project, snapshot_at(config, repo, head), stats_repo, head) except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as error: out["stats_error"] = str(error) # a failed statistics run is retried next time, instead of waiting for main to move again done_head = state.get("last_head") if out.get("stats_error") else head _save_state(project, {**state, "last_head": done_head, "last_refresh": logbook.now(), **({"last_error": out.get("stats_error") or out.get("log_error")} if out.get("stats_error") or out.get("log_error") else {"last_error": None})}) return out def loop(registry): """Server thread: refresh every project with [logbook] auto = true.""" while True: wait = 600 for name in registry.names(): try: h, _ = registry.get(name) config = getattr(h.project, "logbook", None) or {} if config.get("auto"): wait = min(wait, int(config.get("interval_minutes", 10)) * 60) refresh(h.project) except Exception: traceback.print_exc() time.sleep(max(60, wait))