logbook: statistics run on a clean detached snapshot of main ([logbook] snapshot), not the integrator's live checkout, which is often mid-batch; a failed run retries on the next tick instead of waiting for main to move
This commit is contained in:
KeelrLeek committed
1 parent
d7464aaa40
commit
a3fc48802c
1 file changed
+169
-152
+169
-152
@@ -1,152 +1,169 @@
|
||||
"""Keeps the Logbook current without anyone writing to it: when the decomp repository's main branch moves,
|
||||
|
||||
1. each new commit on main becomes a ship's log entry (its subject and the first lines of its message), and
|
||||
2. the statistics, the chart of the executable and a trend point are rebuilt with the open-seas progress tools,
|
||||
run on a scratch copy of that repository (the real one stays untouched).
|
||||
|
||||
Configured per project in a [logbook] table:
|
||||
auto = true
|
||||
decomp = 'R:/.../open-seas-decomp' the repository whose main branch is followed
|
||||
stats_repo = 'R:/.../stats-work/repo' a scratch copy of open-seas holding tools/progress/*.py
|
||||
interval_minutes = 10 how often the server looks (default 10)
|
||||
Milestones and narrative entries stay with people: they need judgement.
|
||||
|
||||
State (the last commit seen) is kept in the logbook folder as auto.json."""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
|
||||
from . import logbook
|
||||
|
||||
MAX_ENTRIES = 25
|
||||
|
||||
|
||||
def _git(repo, *args):
|
||||
result = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, encoding="utf-8",
|
||||
errors="replace", timeout=60)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"git {' '.join(args)}: {result.stderr.strip()[:300]}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def _state(project):
|
||||
path = logbook.folder(project) / "auto.json"
|
||||
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
|
||||
|
||||
|
||||
def _save_state(project, state):
|
||||
path = logbook.folder(project) / "auto.json"
|
||||
path.write_text(json.dumps(state, indent=1), encoding="utf-8")
|
||||
|
||||
|
||||
def _utc(iso):
|
||||
"""git's %cI (local time with an offset) as UTC, the Logbook's format."""
|
||||
import datetime
|
||||
return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _kind(files):
|
||||
if files and all(f.startswith("annotations/") for f in files):
|
||||
return "naming"
|
||||
if any(f.startswith("src/") for f in files):
|
||||
return "progress"
|
||||
return "note"
|
||||
|
||||
|
||||
def log_commits(project, repo, since, head):
|
||||
"""One ship's log entry per commit on main after `since` (oldest first); a summary when there are many."""
|
||||
raw = _git(repo, "log", "--reverse", "--no-merges", "--format=%H%x1f%cI%x1f%s%x1f%b%x1e", f"{since}..{head}")
|
||||
# str.strip() counts \x1f as whitespace and would eat an empty body's field, so trim newlines only and pad
|
||||
commits = [(c.strip("\r\n").split("\x1f") + [""] * 4)[:4] for c in raw.split("\x1e") if c.strip("\r\n")]
|
||||
added = []
|
||||
if len(commits) > MAX_ENTRIES:
|
||||
subjects = "\n".join(f"- {c[2]} ({c[0][:8]})" for c in commits[-40:])
|
||||
added.append(logbook.add_entry(project, f"{len(commits)} commits landed on main",
|
||||
subjects, kind="progress", at=_utc(commits[-1][1])))
|
||||
return added
|
||||
for sha, when, subject, body in commits:
|
||||
files = [f for f in _git(repo, "show", "--name-only", "--format=", sha).splitlines() if f.strip()]
|
||||
lines = [l for l in body.strip().splitlines() if l.strip() and not l.lower().startswith("co-authored-by")]
|
||||
detail = "\n".join(lines[:8]) if lines else subject
|
||||
detail += f"\n(main {sha[:8]}, {len(files)} file{'s' if len(files) != 1 else ''})"
|
||||
added.append(logbook.add_entry(project, subject, detail, kind=_kind(files), at=_utc(when)))
|
||||
return added
|
||||
|
||||
|
||||
def rebuild_stats(project, repo, stats_repo, head):
|
||||
"""Run the open-seas progress tools on the scratch copy and store their statistics and chart."""
|
||||
tools = Path(stats_repo) / "tools" / "progress"
|
||||
# its own data dir: the agents' compiles hold the shared decomp-build\pirates.h open
|
||||
env = dict(os.environ, OPENSEAS_DATA=os.environ.get("HARBOR_LOGBOOK_DATA", r"R:\Workspace\Data\openseas-logbook"))
|
||||
# the readable build finds the compiler through vswhere, which a server started without a shell's PATH lacks
|
||||
installer = r"C:\Program Files (x86)\Microsoft Visual Studio\Installer"
|
||||
if os.path.isdir(installer) and installer.lower() not in env.get("PATH", "").lower():
|
||||
env["PATH"] = installer + os.pathsep + env.get("PATH", "")
|
||||
run = lambda *argv: subprocess.run([sys.executable, "-B", *map(str, argv)], cwd=str(stats_repo), env=env,
|
||||
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=900)
|
||||
sync = run(tools / "sync_decomp.py", "--decomp", repo)
|
||||
if sync.returncode != 0:
|
||||
raise RuntimeError("sync_decomp: " + (sync.stdout + sync.stderr).strip()[-400:])
|
||||
report = run(tools / "progress.py")
|
||||
if report.returncode != 0:
|
||||
raise RuntimeError("progress: " + (report.stdout + report.stderr).strip()[-400:])
|
||||
with tempfile.TemporaryDirectory() as out:
|
||||
payload = run(tools / "dashboard_payload.py", out)
|
||||
if payload.returncode != 0:
|
||||
raise RuntimeError("dashboard_payload: " + (payload.stdout + payload.stderr).strip()[-400:])
|
||||
stats = json.loads((Path(out) / "stats_current.json").read_text(encoding="utf-8"))
|
||||
chart = json.loads((Path(out) / "stats_map.json").read_text(encoding="utf-8"))
|
||||
stats["commit"] = head[:7]
|
||||
stats["updated_at"] = logbook.now()
|
||||
logbook.set_stats(project, stats)
|
||||
logbook.set_map(project, chart)
|
||||
return {"classified": stats.get("classified_percent"), "weighted": stats.get("weighted_percent")}
|
||||
|
||||
|
||||
def refresh(project, force=False):
|
||||
"""Bring the Logbook up to main. Returns what changed."""
|
||||
config = getattr(project, "logbook", None) or {}
|
||||
repo, stats_repo = config.get("decomp"), config.get("stats_repo")
|
||||
if not repo:
|
||||
return {"changed": False, "reason": "no [logbook] decomp"}
|
||||
head = _git(repo, "rev-parse", "HEAD").strip()
|
||||
state = _state(project)
|
||||
if state.get("last_head") == head and not force:
|
||||
return {"changed": False, "head": head[:8]}
|
||||
out = {"changed": True, "head": head[:8]}
|
||||
if state.get("last_head"):
|
||||
try:
|
||||
out["entries"] = len(log_commits(project, repo, state["last_head"], head))
|
||||
except RuntimeError as error: # e.g. the old commit vanished after a rewrite
|
||||
out["log_error"] = str(error)
|
||||
if stats_repo:
|
||||
try:
|
||||
out["stats"] = rebuild_stats(project, repo, stats_repo, head)
|
||||
except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as error:
|
||||
out["stats_error"] = str(error)
|
||||
_save_state(project, {**state, "last_head": head, "last_refresh": logbook.now(),
|
||||
**({"last_error": out.get("stats_error") or out.get("log_error")}
|
||||
if out.get("stats_error") or out.get("log_error") else {"last_error": None})})
|
||||
return out
|
||||
|
||||
|
||||
def loop(registry):
|
||||
"""Server thread: refresh every project with [logbook] auto = true."""
|
||||
while True:
|
||||
wait = 600
|
||||
for name in registry.names():
|
||||
try:
|
||||
h, _ = registry.get(name)
|
||||
config = getattr(h.project, "logbook", None) or {}
|
||||
if config.get("auto"):
|
||||
wait = min(wait, int(config.get("interval_minutes", 10)) * 60)
|
||||
refresh(h.project)
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
time.sleep(max(60, wait))
|
||||
"""Keeps the Logbook current without anyone writing to it: when the decomp repository's main branch moves,
|
||||
|
||||
1. each new commit on main becomes a ship's log entry (its subject and the first lines of its message), and
|
||||
2. the statistics, the chart of the executable and a trend point are rebuilt with the open-seas progress tools,
|
||||
run on a scratch copy of that repository (the real one stays untouched).
|
||||
|
||||
Configured per project in a [logbook] table:
|
||||
auto = true
|
||||
decomp = 'R:/.../open-seas-decomp' the repository whose main branch is followed
|
||||
stats_repo = 'R:/.../stats-work/repo' a scratch copy of open-seas holding tools/progress/*.py
|
||||
interval_minutes = 10 how often the server looks (default 10)
|
||||
Milestones and narrative entries stay with people: they need judgement.
|
||||
|
||||
State (the last commit seen) is kept in the logbook folder as auto.json."""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
|
||||
from . import logbook
|
||||
|
||||
MAX_ENTRIES = 25
|
||||
|
||||
|
||||
def _git(repo, *args):
|
||||
result = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, encoding="utf-8",
|
||||
errors="replace", timeout=60)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"git {' '.join(args)}: {result.stderr.strip()[:300]}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def _state(project):
|
||||
path = logbook.folder(project) / "auto.json"
|
||||
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
|
||||
|
||||
|
||||
def _save_state(project, state):
|
||||
path = logbook.folder(project) / "auto.json"
|
||||
path.write_text(json.dumps(state, indent=1), encoding="utf-8")
|
||||
|
||||
|
||||
def _utc(iso):
|
||||
"""git's %cI (local time with an offset) as UTC, the Logbook's format."""
|
||||
import datetime
|
||||
return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _kind(files):
|
||||
if files and all(f.startswith("annotations/") for f in files):
|
||||
return "naming"
|
||||
if any(f.startswith("src/") for f in files):
|
||||
return "progress"
|
||||
return "note"
|
||||
|
||||
|
||||
def log_commits(project, repo, since, head):
|
||||
"""One ship's log entry per commit on main after `since` (oldest first); a summary when there are many."""
|
||||
raw = _git(repo, "log", "--reverse", "--no-merges", "--format=%H%x1f%cI%x1f%s%x1f%b%x1e", f"{since}..{head}")
|
||||
# str.strip() counts \x1f as whitespace and would eat an empty body's field, so trim newlines only and pad
|
||||
commits = [(c.strip("\r\n").split("\x1f") + [""] * 4)[:4] for c in raw.split("\x1e") if c.strip("\r\n")]
|
||||
added = []
|
||||
if len(commits) > MAX_ENTRIES:
|
||||
subjects = "\n".join(f"- {c[2]} ({c[0][:8]})" for c in commits[-40:])
|
||||
added.append(logbook.add_entry(project, f"{len(commits)} commits landed on main",
|
||||
subjects, kind="progress", at=_utc(commits[-1][1])))
|
||||
return added
|
||||
for sha, when, subject, body in commits:
|
||||
files = [f for f in _git(repo, "show", "--name-only", "--format=", sha).splitlines() if f.strip()]
|
||||
lines = [l for l in body.strip().splitlines() if l.strip() and not l.lower().startswith("co-authored-by")]
|
||||
detail = "\n".join(lines[:8]) if lines else subject
|
||||
detail += f"\n(main {sha[:8]}, {len(files)} file{'s' if len(files) != 1 else ''})"
|
||||
added.append(logbook.add_entry(project, subject, detail, kind=_kind(files), at=_utc(when)))
|
||||
return added
|
||||
|
||||
|
||||
def rebuild_stats(project, repo, stats_repo, head):
|
||||
"""Run the open-seas progress tools on the scratch copy and store their statistics and chart."""
|
||||
tools = Path(stats_repo) / "tools" / "progress"
|
||||
# its own data dir: the agents' compiles hold the shared decomp-build\pirates.h open
|
||||
env = dict(os.environ, OPENSEAS_DATA=os.environ.get("HARBOR_LOGBOOK_DATA", r"R:\Workspace\Data\openseas-logbook"))
|
||||
# the readable build finds the compiler through vswhere, which a server started without a shell's PATH lacks
|
||||
installer = r"C:\Program Files (x86)\Microsoft Visual Studio\Installer"
|
||||
if os.path.isdir(installer) and installer.lower() not in env.get("PATH", "").lower():
|
||||
env["PATH"] = installer + os.pathsep + env.get("PATH", "")
|
||||
run = lambda *argv: subprocess.run([sys.executable, "-B", *map(str, argv)], cwd=str(stats_repo), env=env,
|
||||
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=900)
|
||||
sync = run(tools / "sync_decomp.py", "--decomp", repo)
|
||||
if sync.returncode != 0:
|
||||
raise RuntimeError("sync_decomp: " + (sync.stdout + sync.stderr).strip()[-400:])
|
||||
report = run(tools / "progress.py")
|
||||
if report.returncode != 0:
|
||||
raise RuntimeError("progress: " + (report.stdout + report.stderr).strip()[-400:])
|
||||
with tempfile.TemporaryDirectory() as out:
|
||||
payload = run(tools / "dashboard_payload.py", out)
|
||||
if payload.returncode != 0:
|
||||
raise RuntimeError("dashboard_payload: " + (payload.stdout + payload.stderr).strip()[-400:])
|
||||
stats = json.loads((Path(out) / "stats_current.json").read_text(encoding="utf-8"))
|
||||
chart = json.loads((Path(out) / "stats_map.json").read_text(encoding="utf-8"))
|
||||
stats["commit"] = head[:7]
|
||||
stats["updated_at"] = logbook.now()
|
||||
logbook.set_stats(project, stats)
|
||||
logbook.set_map(project, chart)
|
||||
return {"classified": stats.get("classified_percent"), "weighted": stats.get("weighted_percent")}
|
||||
|
||||
|
||||
def snapshot_at(config, repo, head):
|
||||
"""A clean checkout of `head` for the statistics: [logbook] snapshot, a detached worktree of the decomp repository
|
||||
that only this refresher uses. The live checkout is the integrator's and is often mid-batch with uncommitted
|
||||
files, which the statistics tools rightly refuse. Without a snapshot setting, the live checkout is used."""
|
||||
snapshot = config.get("snapshot")
|
||||
if not snapshot:
|
||||
return repo
|
||||
path = Path(snapshot)
|
||||
if not (path / ".git").exists():
|
||||
_git(repo, "worktree", "add", "--detach", str(path), head)
|
||||
else:
|
||||
_git(str(path), "checkout", "--detach", "--force", head)
|
||||
return str(path)
|
||||
|
||||
|
||||
def refresh(project, force=False):
|
||||
"""Bring the Logbook up to main. Returns what changed."""
|
||||
config = getattr(project, "logbook", None) or {}
|
||||
repo, stats_repo = config.get("decomp"), config.get("stats_repo")
|
||||
if not repo:
|
||||
return {"changed": False, "reason": "no [logbook] decomp"}
|
||||
head = _git(repo, "rev-parse", "HEAD").strip()
|
||||
state = _state(project)
|
||||
if state.get("last_head") == head and not force:
|
||||
return {"changed": False, "head": head[:8]}
|
||||
out = {"changed": True, "head": head[:8]}
|
||||
if state.get("last_head"):
|
||||
try:
|
||||
out["entries"] = len(log_commits(project, repo, state["last_head"], head))
|
||||
except RuntimeError as error: # e.g. the old commit vanished after a rewrite
|
||||
out["log_error"] = str(error)
|
||||
if stats_repo:
|
||||
try:
|
||||
out["stats"] = rebuild_stats(project, snapshot_at(config, repo, head), stats_repo, head)
|
||||
except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as error:
|
||||
out["stats_error"] = str(error)
|
||||
# a failed statistics run is retried next time, instead of waiting for main to move again
|
||||
done_head = state.get("last_head") if out.get("stats_error") else head
|
||||
_save_state(project, {**state, "last_head": done_head, "last_refresh": logbook.now(),
|
||||
**({"last_error": out.get("stats_error") or out.get("log_error")}
|
||||
if out.get("stats_error") or out.get("log_error") else {"last_error": None})})
|
||||
return out
|
||||
|
||||
|
||||
def loop(registry):
|
||||
"""Server thread: refresh every project with [logbook] auto = true."""
|
||||
while True:
|
||||
wait = 600
|
||||
for name in registry.names():
|
||||
try:
|
||||
h, _ = registry.get(name)
|
||||
config = getattr(h.project, "logbook", None) or {}
|
||||
if config.get("auto"):
|
||||
wait = min(wait, int(config.get("interval_minutes", 10)) * 60)
|
||||
refresh(h.project)
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
time.sleep(max(60, wait))
|
||||
Reference in new issue
Block a user