183 lines
9.3 KiB
Python
183 lines
9.3 KiB
Python
"""Keeps the Logbook current without anyone writing to it: when the decomp repository's main branch moves,
|
|
|
|
1. each new commit on main becomes a ship's log entry (its subject and the first lines of its message), and
|
|
2. the statistics, the chart of the executable and a trend point are rebuilt with the open-seas progress tools,
|
|
run on a scratch copy of that repository (the real one stays untouched).
|
|
|
|
Configured per project in a [logbook] table:
|
|
auto = true
|
|
decomp = 'R:/.../open-seas-decomp' the repository whose main branch is followed
|
|
stats_repo = 'R:/.../stats-work/repo' a scratch copy of open-seas holding tools/progress/*.py
|
|
interval_minutes = 10 how often the server looks (default 10)
|
|
lander_lock = 'R:/.../lander-busy.lock' wait while the lander runs its gates (harbor.lander_gate)
|
|
The progress tools run below normal priority.
|
|
Milestones and narrative entries stay with people: they need judgement.
|
|
|
|
State (the last commit seen) is kept in the logbook folder as auto.json."""
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import traceback
|
|
from pathlib import Path
|
|
|
|
from . import lander_gate, logbook
|
|
|
|
MAX_ENTRIES = 25
|
|
STATS_TIMEOUT = 1800 # each progress tool; sync_decomp compiles every readable file, below normal priority
|
|
|
|
|
|
def _git(repo, *args):
|
|
result = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, encoding="utf-8",
|
|
errors="replace", timeout=60)
|
|
if result.returncode != 0:
|
|
raise RuntimeError(f"git {' '.join(args)}: {result.stderr.strip()[:300]}")
|
|
return result.stdout
|
|
|
|
|
|
def _state(project):
|
|
path = logbook.folder(project) / "auto.json"
|
|
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
|
|
|
|
|
|
def _save_state(project, state):
|
|
path = logbook.folder(project) / "auto.json"
|
|
path.write_text(json.dumps(state, indent=1), encoding="utf-8")
|
|
|
|
|
|
def _utc(iso):
|
|
"""git's %cI (local time with an offset) as UTC, the Logbook's format."""
|
|
import datetime
|
|
return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
def _kind(files):
|
|
if files and all(f.startswith("annotations/") for f in files):
|
|
return "naming"
|
|
if any(f.startswith("src/") for f in files):
|
|
return "progress"
|
|
return "note"
|
|
|
|
|
|
def log_commits(project, repo, since, head):
|
|
"""One ship's log entry per commit on main after `since` (oldest first); a summary when there are many."""
|
|
raw = _git(repo, "log", "--reverse", "--no-merges", "--format=%H%x1f%cI%x1f%s%x1f%b%x1e", f"{since}..{head}")
|
|
# str.strip() counts \x1f as whitespace and would eat an empty body's field, so trim newlines only and pad
|
|
commits = [(c.strip("\r\n").split("\x1f") + [""] * 4)[:4] for c in raw.split("\x1e") if c.strip("\r\n")]
|
|
added = []
|
|
if len(commits) > MAX_ENTRIES:
|
|
subjects = "\n".join(f"- {c[2]} ({c[0][:8]})" for c in commits[-40:])
|
|
added.append(logbook.add_entry(project, f"{len(commits)} commits landed on main",
|
|
subjects, kind="progress", at=_utc(commits[-1][1])))
|
|
return added
|
|
for sha, when, subject, body in commits:
|
|
files = [f for f in _git(repo, "show", "--name-only", "--format=", sha).splitlines() if f.strip()]
|
|
lines = [l for l in body.strip().splitlines() if l.strip() and not l.lower().startswith("co-authored-by")]
|
|
detail = "\n".join(lines[:8]) + "\n" if lines else "" # the board shows the title already
|
|
detail += f"(main {sha[:8]}, {len(files)} file{'s' if len(files) != 1 else ''})"
|
|
added.append(logbook.add_entry(project, subject, detail, kind=_kind(files), at=_utc(when)))
|
|
return added
|
|
|
|
|
|
def rebuild_stats(project, repo, stats_repo, head):
|
|
"""Run the open-seas progress tools on the scratch copy and store their statistics and chart."""
|
|
tools = Path(stats_repo) / "tools" / "progress"
|
|
# its own data dir: the agents' compiles hold the shared decomp-build\pirates.h open
|
|
env = dict(os.environ, OPENSEAS_DATA=os.environ.get("HARBOR_LOGBOOK_DATA", r"R:\Workspace\Data\openseas-logbook"))
|
|
# the readable build finds the compiler through vswhere, which a server started without a shell's PATH lacks
|
|
installer = r"C:\Program Files (x86)\Microsoft Visual Studio\Installer"
|
|
if os.path.isdir(installer) and installer.lower() not in env.get("PATH", "").lower():
|
|
env["PATH"] = installer + os.pathsep + env.get("PATH", "")
|
|
# below normal priority (cl.exe inherits it), so the lander's and the agents' compiles go first; a timeout ends
|
|
# the whole tree, not only python (an orphaned cl.exe went on compiling all of main at normal priority)
|
|
run = lambda *argv: lander_gate.run_low([sys.executable, "-B", *map(str, argv)], STATS_TIMEOUT,
|
|
cwd=str(stats_repo), env=env, encoding="utf-8", errors="replace")
|
|
sync = run(tools / "sync_decomp.py", "--decomp", repo)
|
|
if sync.returncode != 0:
|
|
raise RuntimeError("sync_decomp: " + (sync.stdout + sync.stderr).strip()[-400:])
|
|
report = run(tools / "progress.py")
|
|
if report.returncode != 0:
|
|
raise RuntimeError("progress: " + (report.stdout + report.stderr).strip()[-400:])
|
|
with tempfile.TemporaryDirectory() as out:
|
|
payload = run(tools / "dashboard_payload.py", out)
|
|
if payload.returncode != 0:
|
|
raise RuntimeError("dashboard_payload: " + (payload.stdout + payload.stderr).strip()[-400:])
|
|
stats = json.loads((Path(out) / "stats_current.json").read_text(encoding="utf-8"))
|
|
chart = json.loads((Path(out) / "stats_map.json").read_text(encoding="utf-8"))
|
|
stats["commit"] = head[:7]
|
|
stats["updated_at"] = logbook.now()
|
|
logbook.set_stats(project, stats)
|
|
logbook.set_map(project, chart)
|
|
return {"classified": stats.get("classified_percent"), "weighted": stats.get("weighted_percent")}
|
|
|
|
|
|
def snapshot_at(config, repo, head):
|
|
"""A clean checkout of `head` for the statistics: [logbook] snapshot, a detached worktree of the decomp repository
|
|
that only this refresher uses. The live checkout is the integrator's and is often mid-batch with uncommitted
|
|
files, which the statistics tools rightly refuse. Without a snapshot setting, the live checkout is used."""
|
|
snapshot = config.get("snapshot")
|
|
if not snapshot:
|
|
return repo
|
|
path = Path(snapshot)
|
|
if not (path / ".git").exists():
|
|
_git(repo, "worktree", "add", "--detach", str(path), head)
|
|
else:
|
|
_git(str(path), "checkout", "--detach", "--force", head)
|
|
return str(path)
|
|
|
|
|
|
def refresh(project, force=False):
|
|
"""Bring the Logbook up to main. Returns what changed."""
|
|
config = getattr(project, "logbook", None) or {}
|
|
repo, stats_repo = config.get("decomp"), config.get("stats_repo")
|
|
if not repo:
|
|
return {"changed": False, "reason": "no [logbook] decomp"}
|
|
head = _git(repo, "rev-parse", "HEAD").strip()
|
|
state = _state(project)
|
|
if state.get("last_head") == head and not force:
|
|
return {"changed": False, "head": head[:8]}
|
|
busy = None if force else lander_gate.lander_busy(project)
|
|
if busy: # its gates compile all of main: look again next time
|
|
return {"changed": False, "head": head[:8], "waiting": busy}
|
|
out = {"changed": True, "head": head[:8]}
|
|
# last_head is where the statistics are; logged_head where the ship's log is (a failed statistics run is retried,
|
|
# and must not log the same commits twice)
|
|
logged = state.get("logged_head") or state.get("last_head")
|
|
if logged and logged != head:
|
|
try:
|
|
out["entries"] = len(log_commits(project, repo, logged, head))
|
|
except RuntimeError as error: # e.g. the old commit vanished after a rewrite
|
|
out["log_error"] = str(error)
|
|
if logged:
|
|
state["logged_head"] = head
|
|
if stats_repo:
|
|
try:
|
|
out["stats"] = rebuild_stats(project, snapshot_at(config, repo, head), stats_repo, head)
|
|
except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as error:
|
|
out["stats_error"] = str(error)
|
|
# a failed statistics run is retried next time, instead of waiting for main to move again
|
|
done_head = state.get("last_head") if out.get("stats_error") else head
|
|
_save_state(project, {**state, "last_head": done_head, "last_refresh": logbook.now(),
|
|
**({"last_error": out.get("stats_error") or out.get("log_error")}
|
|
if out.get("stats_error") or out.get("log_error") else {"last_error": None})})
|
|
return out
|
|
|
|
|
|
def loop(registry):
|
|
"""Server thread: refresh every project with [logbook] auto = true."""
|
|
while True:
|
|
wait = 600
|
|
for name in registry.names():
|
|
try:
|
|
h, _ = registry.get(name)
|
|
config = getattr(h.project, "logbook", None) or {}
|
|
if config.get("auto"):
|
|
wait = min(wait, int(config.get("interval_minutes", 10)) * 60)
|
|
refresh(h.project)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
time.sleep(max(60, wait))
|