diff --git a/harbor/cli.py b/harbor/cli.py index ba4e093..0422f26 100644 --- a/harbor/cli.py +++ b/harbor/cli.py @@ -262,6 +262,13 @@ def build_parser(): p.add_argument("--max-bytes", type=int, dest="max_bytes") p.add_argument("--count", type=int, default=16) p.add_argument("--as", dest="as_type", default="hex", choices=["hex", "int32", "uint32", "float", "double", "string"]) + p = sub.add_parser("logbook", help="the project Logbook: add a log entry, set statistics, the chart or a milestone") + p.add_argument("action", choices=["add", "stats", "map", "milestone", "show", "import", "refresh"]) + p.add_argument("--title") + p.add_argument("--kind", default="note") + p.add_argument("--detail", help="the entry text (or --file)") + p.add_argument("--file", help="a text file (add) or a JSON file (stats, map, milestone) or an export folder (import)") + p.add_argument("--no-history", action="store_true", help="stats: do not add a trend point") p = sub.add_parser("next-functions", help="functions in your bands to decompile next, easiest first") p.add_argument("-n", type=int, default=10) p = sub.add_parser("defer", help="set a function aside with a reason and evidence") @@ -718,6 +725,34 @@ def main(argv=None): else: res = r.lookup(args.target) out(res, json.dumps(res, indent=1)) + elif cmd == "logbook": + from . import logbook + project = harbor.project + if args.action in ("add", "stats", "map", "milestone", "import", "refresh") and who() not in harbor.coordinators: + raise HarborError("the Logbook is written by a coordinator") + text = Path(args.file).read_text(encoding="utf-8") if args.file and args.action != "import" else None + if args.action == "add": + res = logbook.add_entry(project, args.title, args.detail or text, kind=args.kind) + out(res, f"logged {res['id']}: {res['title']}") + elif args.action == "stats": + res = logbook.set_stats(project, json.loads(text), history_point=not args.no_history) + out(res, "statistics updated") + elif args.action == "map": + res = logbook.set_map(project, json.loads(text)) + out(res, f"chart updated ({res['cells']} cells)") + elif args.action == "milestone": + res = logbook.upsert_milestone(project, json.loads(text)) + out(res, f"milestone {res['id']} saved") + elif args.action == "refresh": + from . import logbook_auto + res = logbook_auto.refresh(project, force=True) + out(res, json.dumps(res)) + elif args.action == "import": + res = logbook.import_export(project, args.file) + out(res, json.dumps(res)) + else: + res = logbook.read_all(project, log_limit=10) + out(res, "\n".join(f"{e['at'][:16]} [{e['kind']}] {e['title']}" for e in res["log"])) elif cmd in ("next-functions", "defer", "progress"): from .queue import FunctionQueue q = FunctionQueue(harbor) diff --git a/harbor/logbook.py b/harbor/logbook.py new file mode 100644 index 0000000..33beba1 --- /dev/null +++ b/harbor/logbook.py @@ -0,0 +1,130 @@ +"""The project's Logbook, kept by Harbor and shown on the board's Logbook view: headline statistics, the chart of the +executable, the trend, milestones and the ship's log. (It replaced a separately published page on 2026-10-05.) + +One JSON file per collection in the project's logbook folder ([logbook] dir, default /logbook/): + stats.json the latest statistics (functions, bytes, by_category, percentages, commit, updated_at) + map.json the chart of the executable (low, high, cells: "game,engine,platform,crt,unsorted,done" per cell) + history.json [{at, classified, decompiled, weighted, verified}] one point per statistics report + milestones.json [{id, code, title, phase, state, order, items: [{text, done}]}] + log.json [{id, at, kind, title, detail}] the ship's log, newest last +Writes replace the file atomically. Nothing here talks to the network.""" +import datetime +import json +import os +import threading +from pathlib import Path + +from . import projects +from .util import HarborError + +KINDS = ("milestone", "tooling", "progress", "decision", "finding", "naming", "review", "setback", "note") +_LOCK = threading.Lock() + + +def folder(project): + configured = (getattr(project, "logbook", None) or {}).get("dir") + path = Path(configured) if configured else projects.harbor_home() / "logbook" / project.name + path.mkdir(parents=True, exist_ok=True) + return path + + +def _read(project, name, default): + path = folder(project) / name + if not path.exists(): + return default + try: + return json.loads(path.read_text(encoding="utf-8")) + except ValueError: + raise HarborError(f"{path} is not valid JSON") from None + + +def _write(project, name, value): + path = folder(project) / name + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(value, indent=1, ensure_ascii=False), encoding="utf-8") + os.replace(tmp, path) + + +def now(): + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def read_all(project, log_limit=80): + log = _read(project, "log.json", []) + return {"stats": _read(project, "stats.json", None), "map": _read(project, "map.json", None), + "history": sorted(_read(project, "history.json", []), key=lambda p: p.get("at", "")), + "milestones": sorted(_read(project, "milestones.json", []), key=lambda m: m.get("order", 0)), + "log": sorted(log, key=lambda e: e.get("at", ""), reverse=True)[:log_limit], "log_total": len(log), + "auto": _read(project, "auto.json", {})} + + +def add_entry(project, title, detail, kind="note", at=None): + title, detail = (title or "").strip(), (detail or "").strip() + if not title or not detail: + raise HarborError("a log entry needs a title and a detail") + if kind not in KINDS: + raise HarborError("kind must be one of " + ", ".join(KINDS)) + with _LOCK: + log = _read(project, "log.json", []) + at = at or now() + stem = at[:16].replace("-", "").replace(":", "").replace("T", "-") + n = 1 + sum(1 for e in log if str(e.get("id", "")).startswith(stem)) + entry = {"id": f"{stem}-{n}", "at": at, "kind": kind, "title": title, "detail": detail} + log.append(entry) + _write(project, "log.json", log) + return entry + + +def set_stats(project, stats, history_point=True): + """Replace the statistics; also record a trend point (classified, decompiled, weighted, verified).""" + if not isinstance(stats, dict) or "functions" not in stats: + raise HarborError("statistics need at least 'functions'") + stats = {**stats, "updated_at": stats.get("updated_at") or now()} + with _LOCK: + _write(project, "stats.json", stats) + if history_point: + history = _read(project, "history.json", []) + history.append({"at": stats["updated_at"], "classified": stats.get("classified_percent", 0), + "decompiled": stats.get("game_bytes_decompiled_percent", 0), + "weighted": stats.get("weighted_percent", 0), + "verified": stats.get("game_bytes_verified_percent", 0)}) + _write(project, "history.json", history) + return stats + + +def set_map(project, chart): + if not isinstance(chart, dict) or "cells" not in chart: + raise HarborError("the chart needs 'cells'") + with _LOCK: + _write(project, "map.json", chart) + return {"cells": len(chart["cells"])} + + +def upsert_milestone(project, milestone): + if not isinstance(milestone, dict) or not milestone.get("id"): + raise HarborError("a milestone needs an id") + with _LOCK: + milestones = [m for m in _read(project, "milestones.json", []) if m.get("id") != milestone["id"]] + milestones.append(milestone) + _write(project, "milestones.json", sorted(milestones, key=lambda m: m.get("order", 0))) + return milestone + + +def import_export(project, export_dir): + """One-time import of the published page's database export (one JSON file per document).""" + export = Path(export_dir) + + def docs(collection): + return [json.loads(p.read_text(encoding="utf-8")) for p in sorted((export / collection).glob("*.json"))] + with _LOCK: + log = [{**d, "id": d.get("id") or p.stem} for d, p in zip(docs("log"), sorted((export / "log").glob("*.json")))] + _write(project, "log.json", sorted(log, key=lambda e: e.get("at", ""))) + _write(project, "history.json", sorted(docs("history"), key=lambda e: e.get("at", ""))) + _write(project, "milestones.json", sorted(docs("milestones"), key=lambda m: m.get("order", 0))) + if (export / "stats" / "current.json").exists(): + _write(project, "stats.json", json.loads((export / "stats" / "current.json").read_text(encoding="utf-8"))) + if (export / "stats" / "map.json").exists(): + _write(project, "map.json", json.loads((export / "stats" / "map.json").read_text(encoding="utf-8"))) + data = read_all(project, log_limit=0) + return {"log": data["log_total"], "history": len(data["history"]), "milestones": len(data["milestones"]), + "stats": bool(data["stats"]), "map": bool(data["map"])} diff --git a/harbor/logbook_auto.py b/harbor/logbook_auto.py new file mode 100644 index 0000000..a78f807 --- /dev/null +++ b/harbor/logbook_auto.py @@ -0,0 +1,144 @@ +"""Keeps the Logbook current without anyone writing to it: when the decomp repository's main branch moves, + + 1. each new commit on main becomes a ship's log entry (its subject and the first lines of its message), and + 2. the statistics, the chart of the executable and a trend point are rebuilt with the open-seas progress tools, + run on a scratch copy of that repository (the real one stays untouched). + +Configured per project in a [logbook] table: + auto = true + decomp = 'R:/.../open-seas-decomp' the repository whose main branch is followed + stats_repo = 'R:/.../stats-work/repo' a scratch copy of open-seas holding tools/progress/*.py + interval_minutes = 10 how often the server looks (default 10) +Milestones and narrative entries stay with people: they need judgement. + +State (the last commit seen) is kept in the logbook folder as auto.json.""" +import json +import re +import subprocess +import sys +import tempfile +import time +import traceback +from pathlib import Path + +from . import logbook + +MAX_ENTRIES = 25 + + +def _git(repo, *args): + result = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, encoding="utf-8", + errors="replace", timeout=60) + if result.returncode != 0: + raise RuntimeError(f"git {' '.join(args)}: {result.stderr.strip()[:300]}") + return result.stdout + + +def _state(project): + path = logbook.folder(project) / "auto.json" + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} + + +def _save_state(project, state): + path = logbook.folder(project) / "auto.json" + path.write_text(json.dumps(state, indent=1), encoding="utf-8") + + +def _utc(iso): + """git's %cI (local time with an offset) as UTC, the Logbook's format.""" + import datetime + return datetime.datetime.fromisoformat(iso).astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _kind(files): + if files and all(f.startswith("annotations/") for f in files): + return "naming" + if any(f.startswith("src/") for f in files): + return "progress" + return "note" + + +def log_commits(project, repo, since, head): + """One ship's log entry per commit on main after `since` (oldest first); a summary when there are many.""" + raw = _git(repo, "log", "--reverse", "--no-merges", "--format=%H%x1f%cI%x1f%s%x1f%b%x1e", f"{since}..{head}") + commits = [c.strip().split("\x1f") for c in raw.split("\x1e") if c.strip()] + added = [] + if len(commits) > MAX_ENTRIES: + subjects = "\n".join(f"- {c[2]} ({c[0][:8]})" for c in commits[-40:]) + added.append(logbook.add_entry(project, f"{len(commits)} commits landed on main", + subjects, kind="progress", at=_utc(commits[-1][1]))) + return added + for sha, when, subject, body in commits: + files = [f for f in _git(repo, "show", "--name-only", "--format=", sha).splitlines() if f.strip()] + lines = [l for l in body.strip().splitlines() if l.strip() and not l.lower().startswith("co-authored-by")] + detail = "\n".join(lines[:8]) if lines else subject + detail += f"\n(main {sha[:8]}, {len(files)} file{'s' if len(files) != 1 else ''})" + added.append(logbook.add_entry(project, subject, detail, kind=_kind(files), at=_utc(when))) + return added + + +def rebuild_stats(project, repo, stats_repo, head): + """Run the open-seas progress tools on the scratch copy and store their statistics and chart.""" + tools = Path(stats_repo) / "tools" / "progress" + run = lambda *argv: subprocess.run([sys.executable, "-B", *map(str, argv)], cwd=str(stats_repo), + capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=900) + sync = run(tools / "sync_decomp.py", "--decomp", repo) + if sync.returncode != 0: + raise RuntimeError("sync_decomp: " + (sync.stdout + sync.stderr).strip()[-400:]) + report = run(tools / "progress.py") + if report.returncode != 0: + raise RuntimeError("progress: " + (report.stdout + report.stderr).strip()[-400:]) + with tempfile.TemporaryDirectory() as out: + payload = run(tools / "dashboard_payload.py", out) + if payload.returncode != 0: + raise RuntimeError("dashboard_payload: " + (payload.stdout + payload.stderr).strip()[-400:]) + stats = json.loads((Path(out) / "stats_current.json").read_text(encoding="utf-8")) + chart = json.loads((Path(out) / "stats_map.json").read_text(encoding="utf-8")) + stats["commit"] = head[:7] + stats["updated_at"] = logbook.now() + logbook.set_stats(project, stats) + logbook.set_map(project, chart) + return {"classified": stats.get("classified_percent"), "weighted": stats.get("weighted_percent")} + + +def refresh(project, force=False): + """Bring the Logbook up to main. Returns what changed.""" + config = getattr(project, "logbook", None) or {} + repo, stats_repo = config.get("decomp"), config.get("stats_repo") + if not repo: + return {"changed": False, "reason": "no [logbook] decomp"} + head = _git(repo, "rev-parse", "HEAD").strip() + state = _state(project) + if state.get("last_head") == head and not force: + return {"changed": False, "head": head[:8]} + out = {"changed": True, "head": head[:8]} + if state.get("last_head"): + try: + out["entries"] = len(log_commits(project, repo, state["last_head"], head)) + except RuntimeError as error: # e.g. the old commit vanished after a rewrite + out["log_error"] = str(error) + if stats_repo: + try: + out["stats"] = rebuild_stats(project, repo, stats_repo, head) + except (RuntimeError, OSError, ValueError, subprocess.TimeoutExpired) as error: + out["stats_error"] = str(error) + _save_state(project, {**state, "last_head": head, "last_refresh": logbook.now(), + **({"last_error": out.get("stats_error") or out.get("log_error")} + if out.get("stats_error") or out.get("log_error") else {"last_error": None})}) + return out + + +def loop(registry): + """Server thread: refresh every project with [logbook] auto = true.""" + while True: + wait = 600 + for name in registry.names(): + try: + h, _ = registry.get(name) + config = getattr(h.project, "logbook", None) or {} + if config.get("auto"): + wait = min(wait, int(config.get("interval_minutes", 10)) * 60) + refresh(h.project) + except Exception: + traceback.print_exc() + time.sleep(max(60, wait)) diff --git a/harbor/projects.py b/harbor/projects.py index bea984a..16d9b0c 100644 --- a/harbor/projects.py +++ b/harbor/projects.py @@ -94,6 +94,7 @@ class Project: policy: dict = field(default_factory=dict) autopilot: bool = False # running agents keep working instead of ending their turns (hooks) shared_files: tuple = () # glob patterns (repo-relative) every agent edits and the integrator merges: not collisions + logbook: dict = field(default_factory=dict) # [logbook]: dir, auto, decomp, stats_repo (harbor.logbook_auto) reverse: dict = field(default_factory=dict) # [reverse]: image, base, facts, imports, annotations, src (harbor.reverse) @property @@ -209,6 +210,7 @@ def project_from_dict(data, source=None, home=None): templates=_templates(data.get("templates", {}), names), policy=_policy(data.get("policy"), names), autopilot=bool(data.get("autopilot", False)), shared_files=tuple(str(p).replace("\\", "/") for p in data.get("shared_files", [])), + logbook=dict(data.get("logbook", {})), reverse=dict(data.get("reverse", {}))) diff --git a/harbor/server.py b/harbor/server.py index 756888e..82f81e1 100644 --- a/harbor/server.py +++ b/harbor/server.py @@ -330,6 +330,9 @@ class Handler(http.server.BaseHTTPRequestHandler): "stopped": any(h.stopped(a) for a in h.project.ai_names)}) return self._reply(200, out) h, lock = REGISTRY.get(name) + if path == "/api/logbook": + from . import logbook + return self._reply(200, logbook.read_all(h.project, log_limit=min(int(query.get("limit", 80)), 500))) if path.startswith("/api/decomp/"): # its own connection and lock: these answers take seconds and must not hold up the chat from . import decomp_api @@ -548,6 +551,8 @@ def main(argv=None): threading.Thread(target=backup_loop, daemon=True).start() threading.Thread(target=git_loop, daemon=True).start() threading.Thread(target=toast_loop, daemon=True).start() + from . import logbook_auto + threading.Thread(target=logbook_auto.loop, args=(REGISTRY,), daemon=True).start() print(f"Harbor on http://127.0.0.1:{PORT} - projects: {', '.join(REGISTRY.names())}", flush=True) server.serve_forever() diff --git a/harbor/web/app.html b/harbor/web/app.html index 54ca2f1..e2443e7 100644 --- a/harbor/web/app.html +++ b/harbor/web/app.html @@ -55,6 +55,37 @@ body { overflow: hidden; } .content { flex: 1; min-height: 0; overflow-y: auto; padding: 24px 32px 56px; } .view { display: none; max-width: 1380px; margin: 0 auto; gap: 20px; } .view.on { display: grid; } +.lb { --lb-game: var(--blue); --lb-engine: var(--violet); --lb-platform: var(--amber); --lb-crt: var(--muted); --lb-unsorted: var(--line-2); --lb-done: var(--green); } +.lb-head { display: flex; flex-wrap: wrap; align-items: flex-end; justify-content: space-between; gap: 10px 20px; } +.lb-head .chip { display: inline-block; font: 600 11.5px/1 var(--font, inherit); letter-spacing: .06em; text-transform: uppercase; color: var(--accent-ink); background: var(--accent-soft); border-radius: 999px; padding: 6px 10px; } +.lb-head p { margin: 8px 0 0; color: var(--muted); max-width: 70ch; } +.lb-stamp { font: 12px/1.5 var(--mono); color: var(--muted); text-align: right; } .lb-stamp b { color: var(--text); font-weight: 600; } +.lb-figs { display: grid; grid-template-columns: repeat(auto-fit, minmax(190px, 1fr)); gap: 12px; } +.lb-fig { background: var(--surface); border: 1px solid var(--line); border-radius: var(--r-2, 12px); padding: 14px 16px; } +.lb-fig .v { font: 700 26px/1.1 var(--mono); font-variant-numeric: tabular-nums; } .lb-fig .l { color: var(--muted); font-size: 13px; margin-top: 4px; } .lb-fig .n { color: var(--faint); font: 11.5px/1.35 var(--mono); margin-top: 6px; } +canvas#lbMap { width: 100%; height: 150px; display: block; } +.lb-axis { display: flex; justify-content: space-between; font: 11.5px var(--mono); color: var(--faint); margin-top: 6px; } +.lb-legend { display: flex; flex-wrap: wrap; gap: 6px 16px; margin-top: 12px; font-size: 12.5px; color: var(--muted); } +.lb-sw { display: inline-block; width: 10px; height: 10px; border-radius: 3px; margin-right: 6px; vertical-align: -1px; } +.lb-stack { display: flex; height: 24px; border-radius: 7px; overflow: hidden; background: var(--surface-3); gap: 2px; margin-top: 10px; } +.lb-stack i { display: block; height: 100%; min-width: 2px; } +.lb-hatch { background: repeating-linear-gradient(135deg, var(--surface-3) 0 5px, var(--line-2) 5px 10px) !important; } +.lb-cat { display: grid; grid-template-columns: 14px minmax(0, 1fr) auto; gap: 2px 10px; padding: 10px 0; border-top: 1px solid var(--line); align-items: start; } +.lb-cat .sw { width: 12px; height: 12px; border-radius: 3px; margin-top: 4px; } .lb-cat .plan { color: var(--muted); font-size: 12.5px; } +.lb-cat .nums { text-align: right; font: 12.5px/1.45 var(--mono); font-variant-numeric: tabular-nums; white-space: nowrap; color: var(--muted); } .lb-cat .nums b { color: var(--text); } +.lb-cat .sub { grid-column: 2 / 4; margin-top: 8px; } +.lb-big { font: 700 26px/1.1 var(--mono); font-variant-numeric: tabular-nums; } +svg#lbTrend { width: 100%; height: 110px; display: block; } +.lb-ms h3 { font-size: 14px; margin: 0 0 6px; display: flex; gap: 10px; align-items: baseline; flex-wrap: wrap; } .lb-ms h3 .code { font-family: var(--mono); color: var(--accent-ink); } +.lb-ms { padding: 10px 0; border-top: 1px solid var(--line); } .lb-ms:first-child { border-top: 0; } +.lb-ms ul { list-style: none; margin: 6px 0 0; padding: 0; display: grid; gap: 4px; } .lb-ms li { display: grid; grid-template-columns: 16px 1fr; gap: 6px; font-size: 13px; color: var(--muted); } +.lb-ms li::before { content: "○"; } .lb-ms li.done { color: var(--text); } .lb-ms li.done::before { content: "●"; color: var(--green); } +.lb-phase { font-weight: 700; color: var(--accent-ink); margin-top: 10px; font-size: 13px; letter-spacing: .04em; text-transform: uppercase; } +.lb-log { list-style: none; margin: 0; padding: 0; } .lb-log li { display: grid; grid-template-columns: 104px minmax(0, 1fr); gap: 12px; padding: 12px 0; border-top: 1px solid var(--line); } +.lb-log li:first-child { border-top: 0; } .lb-log time { font: 11.5px/1.5 var(--mono); color: var(--faint); } +.lb-log .kind { font: 700 10px/1 inherit; letter-spacing: .08em; text-transform: uppercase; color: var(--accent-ink); margin-right: 8px; } +.lb-log .t { font-weight: 600; } .lb-log .d { color: var(--muted); font-size: 13px; white-space: pre-wrap; overflow-wrap: anywhere; margin-top: 4px; } +@media (max-width: 560px) { .lb-log li { grid-template-columns: 1fr; gap: 2px; } .lb-stamp { text-align: left; } } .tierbar { display: flex; height: 22px; border-radius: 7px; overflow: hidden; background: var(--surface-3); gap: 2px; } .tierbar span { display: block; height: 100%; min-width: 3px; } .tier-exact { background: var(--green); } .tier-verified { background: var(--blue); } .tier-readable { background: var(--violet); } @@ -442,6 +473,7 @@ button.chip:hover { color: var(--accent-ink); background: var(--accent-soft); }