forked from Keelrleek/Harbor
KeelrLeek
dbfd1720c5
Messages, threads, acknowledgements, claims, bands, tasks, review items and stop flags for a team of AI agents; the chat page and board; host hooks and MCP server; autopilot; tools/watch.py. Put under version control after the weekend's changes so they have history and can be rolled back. Local only.
374 lines
20 KiB
Python
374 lines
20 KiB
Python
"""Regression tests for the correctness review of rounds five and six."""
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import threading
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
os.environ["HARBOR_NO_TOASTS"] = "1"
|
|
from helpers import ROOT, HarborCase, make_home, run_module, temp_dir
|
|
from harbor import notify, projects
|
|
from harbor.core import Harbor, HarborError
|
|
|
|
|
|
class StopState(HarborCase):
|
|
def test_02_trial_for_one_agent_while_everyone_is_stopped(self):
|
|
self.h.set_flag("stop:all", "on", "User", "web")
|
|
self.clock.t += 60
|
|
self.h.set_flag("stop:Codex", "off", "User", "web", until="30m")
|
|
self.assertEqual((self.h.stopped("Codex"), bool(self.h.trial("Codex"))), ([], True))
|
|
self.assertTrue(self.h.stopped("Kimi"))
|
|
self.clock.t += 60
|
|
self.h.set_flag("stop:all", "on", "Claude", "cli") # an agent's re-stop changes nothing
|
|
self.assertFalse(self.h.stopped("Codex"))
|
|
self.h.set_flag("stop:all", "on", "User", "web") # the human pressing it again is a
|
|
self.assertTrue(self.h.stopped("Codex")) # new decision: it covers everyone
|
|
|
|
def test_03_coordinator_stop_and_lift_keep_the_trial_end(self):
|
|
self.h.set_flag("stop:Codex", "on", "User", "web")
|
|
self.h.set_flag("stop:Codex", "off", "User", "web", until="30m")
|
|
self.h.set_flag("stop:Codex", "on", "Claude", "cli")
|
|
self.h.set_flag("stop:Codex", "off", "Claude", "cli")
|
|
self.assertIsNotNone(self.h.trial("Codex"))
|
|
self.clock.t += 31 * 60
|
|
self.assertTrue(self.h.stopped("Codex")) # the trial still ends
|
|
|
|
def test_04_ended_trial_is_a_stop_everywhere(self):
|
|
(self.dir / "x").mkdir()
|
|
self.h.set_flag("stop:Codex", "on", "User", "web")
|
|
self.h.set_flag("stop:Codex", "off", "User", "web", until="1m")
|
|
self.clock.t += 120
|
|
self.assertTrue(self.h.stopped("Codex"))
|
|
self.assertIn("trial run ended", self.h.stopped("Codex")[0]["reason"])
|
|
launch = {a["agent"]: a for a in self.h.launch_status()}
|
|
self.assertEqual(launch["Codex"]["state"], "stopped")
|
|
|
|
def test_07_launch_shows_team_trials(self):
|
|
self.h.set_flag("stop:all", "on", "User", "web")
|
|
self.clock.t += 60
|
|
self.h.set_flag("stop:all", "off", "User", "web", until="15m")
|
|
self.assertEqual({a["agent"]: a for a in self.h.launch_status()}["Kimi"]["state"], "trial")
|
|
|
|
|
|
class RollCalls(HarborCase):
|
|
def test_06_second_roll_call_keeps_the_first(self):
|
|
self.h.rollcall("User", "web")
|
|
codex_q = next(q for q in self.h.questions(to="Codex"))
|
|
self.h.answer("Codex", codex_q["id"], "Here")
|
|
self.h.rollcall("User", "web", agents=["Kimi"])
|
|
rows = {r["agent"]: r for r in self.h.rollcall_status()["agents"]}
|
|
self.assertEqual(rows["Codex"]["answer"], "Here")
|
|
self.assertEqual(len(rows), 5)
|
|
self.assertEqual(len([q for q in self.h.questions(to="Kimi")]), 1) # the older one is closed
|
|
|
|
|
|
class Notifications(HarborCase):
|
|
def test_05_backlog_over_fifty(self):
|
|
notify.set_enabled(self.h, True)
|
|
for i in range(60):
|
|
self.h.send("Codex", "All", f"broadcast {i}")
|
|
self.h.send("Codex", ["User"], "the one that matters")
|
|
pending, top = notify.pending_for_human(self.h)
|
|
self.assertEqual([m["body"] for m in pending], ["the one that matters"])
|
|
|
|
|
|
class FlowFixes(unittest.TestCase):
|
|
def setUp(self):
|
|
self.dir = temp_dir()
|
|
|
|
def tearDown(self):
|
|
shutil.rmtree(self.dir, ignore_errors=True)
|
|
|
|
def project(self, **extra):
|
|
data = {"name": "f", "coordinators": ["Lead"], "database": str(self.dir / "f.db"),
|
|
"agents": [{"name": "Lead"}, {"name": "Ann"}, {"name": "Bob"}], **extra}
|
|
return projects.project_from_dict(data)
|
|
|
|
def test_08_09_bulk_and_templates_by_a_worker(self):
|
|
h = Harbor(self.project(templates={"t": {"steps": [{"name": "a", "title": "Fix {name} in {scope}",
|
|
"assignee": "Bob"}]}}))
|
|
h.claim("Lead", "range:0x440000-0x47FFFF", kind="band", on_behalf_of="Bob")
|
|
r = h.bulk_tasks("Ann", "0x445310 One\n0x445320 Two")
|
|
self.assertEqual([t["assignee"] for t in r["created"]], [None, None]) # no routing to someone else
|
|
with self.assertRaises(HarborError):
|
|
h.run_template("Ann", "t", scope="fn:0x445400")
|
|
self.assertEqual(len(h.tasks()), 2) # nothing half-created
|
|
task = h.run_template("Lead", "t", scope="fn:0x445400")[0]
|
|
self.assertEqual(task["title"], "Fix {name} in fn:0x445400")
|
|
h.close()
|
|
|
|
def test_11_policy_checked_at_load(self):
|
|
p = self.project(policy={"land_requires": "Ann"})
|
|
self.assertEqual(p.policy["land_requires"], ["Ann"])
|
|
for bad in ({"land_requires": ["Ghost"]}, {"all_per_hour": "lots"}):
|
|
with self.assertRaises(projects.ProjectError):
|
|
self.project(policy=bad)
|
|
h = Harbor(self.project(policy={"land_requires": ["Ann"]}))
|
|
item = h.submit("Ann", "x", commit="abc")
|
|
h.transition("Lead", item["id"], "approved")
|
|
h.transition("Lead", item["id"], "landed", commit="b4") # the author is not required
|
|
h.close()
|
|
|
|
def test_10_rebase_does_not_look_new(self):
|
|
env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@x", "GIT_COMMITTER_NAME": "t",
|
|
"GIT_COMMITTER_EMAIL": "t@x", "GIT_AUTHOR_DATE": "2026-01-01T10:00:00", "GIT_COMMITTER_DATE": "2026-01-01T10:00:00"}
|
|
repo = self.dir / "r"
|
|
repo.mkdir()
|
|
g = lambda *a, e=env: subprocess.run(["git", "-C", str(repo), *a], check=True, capture_output=True, env=e)
|
|
g("init", "-q", "-b", "main")
|
|
(repo / "a").write_text("a")
|
|
g("add", "a")
|
|
g("commit", "-q", "-m", "base")
|
|
g("checkout", "-q", "-b", "work")
|
|
(repo / "b").write_text("b")
|
|
g("add", "b")
|
|
g("commit", "-q", "-m", "draft")
|
|
h = Harbor(projects.project_from_dict({"name": "g", "database": str(self.dir / "g.db"),
|
|
"agents": [{"name": "Ann", "workdir": str(repo)}]}))
|
|
h.clock = lambda: 1767268800.0 + 3600 * 5 # 2026-01-01 15:00 UTC
|
|
sha = h.git_status("Ann", max_age=0)["last_commit"]["sha"]
|
|
h.submit("Ann", "draft", commit=sha, reviewers=["User"])
|
|
h.clock = lambda: 1767268800.0 + 3600 * 6
|
|
g("checkout", "-q", "main")
|
|
(repo / "c").write_text("c")
|
|
g("add", "c")
|
|
g("commit", "-q", "-m", "main moves", e={**env, "GIT_AUTHOR_DATE": "2026-01-01T16:00:00",
|
|
"GIT_COMMITTER_DATE": "2026-01-01T16:00:00"})
|
|
g("checkout", "-q", "work")
|
|
g("rebase", "-q", "main", e={**env, "GIT_COMMITTER_DATE": "2026-01-01T16:30:00"})
|
|
self.assertEqual(h.git_status("Ann", max_age=0)["unsubmitted"], [])
|
|
h.close()
|
|
|
|
|
|
class Pages(unittest.TestCase):
|
|
def test_01_no_code_hidden_behind_a_line_comment(self):
|
|
for page in ("app.html",):
|
|
scripts = re.findall(r"<script>([\s\S]*?)</script>", (ROOT / "harbor" / "web" / page).read_text(encoding="utf-8"))
|
|
self.assertTrue(scripts, page)
|
|
for line in "\n".join(scripts).splitlines():
|
|
code = re.sub(r"'(?:\\.|[^'\\])*'|\"(?:\\.|[^\"\\])*\"|`(?:\\.|[^`\\])*`|/(?:\\.|[^/\\\n])+/[gimsuy]*", "", line)
|
|
if "//" in code:
|
|
after = code.split("//", 1)[1]
|
|
self.assertFalse(re.search(r"\b(const|let|var|function|if|return)\b", after), (page, line[:120]))
|
|
|
|
|
|
class Race(HarborCase):
|
|
def test_answer_race_for_real(self):
|
|
q = self.h.ask("Codex", "Ship?", ["Yes", "No"], fallback="Claude")
|
|
self.h.set_presence("User", "web", "away")
|
|
results, gate = [], threading.Barrier(2)
|
|
|
|
def answer(who, value, origin):
|
|
h = Harbor(self.project, clock=self.clock)
|
|
try:
|
|
gate.wait()
|
|
results.append(h.answer(who, q["id"], value, origin=origin)["answer"])
|
|
except HarborError as error:
|
|
results.append(f"refused: {error}")
|
|
finally:
|
|
h.close()
|
|
threads = [threading.Thread(target=answer, args=a) for a in (("Claude", "Yes", "cli"), ("User", "No", "web"))]
|
|
[t.start() for t in threads]
|
|
[t.join() for t in threads]
|
|
self.assertEqual(sum(1 for r in results if not r.startswith("refused")), 1, results)
|
|
self.assertEqual(len([m for m in self.h.inbox("Codex", unread_only=False) if m["kind"] == "answer"]), 1)
|
|
|
|
|
|
class LaunchFindings(unittest.TestCase):
|
|
"""What the first launch with the real agent apps showed."""
|
|
|
|
def gate(self, command=None, tool="shell"):
|
|
from harbor import hooks
|
|
payload = {"tool_name": tool}
|
|
if command is not None:
|
|
payload["tool_input"] = {"command": command}
|
|
return hooks._tool_is_harbor(payload)
|
|
|
|
def test_messages_may_hold_shell_characters_inside_double_quotes(self):
|
|
run = "py -3.12 -B R:/Workspace/Tools/Harbor/run.py --as Kimi "
|
|
for c in [run + 'send Claude --thread launch "Kimi is online (through the command; MCP was blocked) & ready"',
|
|
'harbor ack 1708 --note "ok, I\'m stopped <as asked> {no work} @Claude ^"',
|
|
'harbor send Claude "it\u2019s fine"', "harbor send Claude 'plain words'"]:
|
|
self.assertTrue(self.gate(c), c)
|
|
for c in ['harbor send All "$(id)"', 'harbor send All "a" ; rm x', "harbor send All '\"'; rm x; '\"'",
|
|
'harbor send All "x\\" ; rm y ; \\"z"', 'harbor send All "100%"', 'harbor send All "line\nRemove-Item x"',
|
|
'harbor send All "a\u201d ; rm x ; \u201cb"', "harbor send All 'it (x)'", 'harbor send All "unterminated',
|
|
'harbor send All "`whoami`"', 'harbor send All \\"x\\"']:
|
|
self.assertFalse(self.gate(c), c)
|
|
|
|
def test_every_hosts_harbor_tool_names_and_tool_loaders(self):
|
|
for name in ("mcp__plugin-harbor-open-seas_harbor__send", "mcp__harbor-open-seas__brief", "harbor-open-seas.inbox",
|
|
"mcp__harbor__wait_for_messages", "harbor_ack", "select_tools", "ToolSearch"):
|
|
self.assertTrue(self.gate(tool=name), name)
|
|
for name in ("mcp__plugin-harbor-open-seas_harbor__land", "mcp__plugin-harbor-open-seas_harbor__next_task",
|
|
"mcp__other__send", "my_harbor_tool", "Harbor", "mcp__plugin-harbor-open-seas_harbor__resume"):
|
|
self.assertFalse(self.gate(tool=name), name)
|
|
|
|
def test_the_stop_message_says_sending_is_allowed(self):
|
|
from harbor import hooks
|
|
|
|
class FakeHarbor:
|
|
human = "User"
|
|
text = hooks._stop_text(FakeHarbor(), "Kimi", [{"key": "stop:Kimi", "set_by": "User", "set_at": "now", "reason": None}])
|
|
self.assertIn("send", text)
|
|
self.assertIn("double quotes", text)
|
|
|
|
|
|
class AllocationWhileStopped(HarborCase):
|
|
def test_coordinator_records_bands_for_stopped_agents_but_no_work(self):
|
|
self.h.set_flag("stop:Codex", "on", "User", "web")
|
|
self.h.claim("Claude", "range:0x401000-0x43FFFF", kind="band", on_behalf_of="Codex")
|
|
self.h.claim("Claude", "fn:0x401200", kind="naming", on_behalf_of="Codex")
|
|
with self.assertRaises(HarborError):
|
|
self.h.claim("Claude", "fn:0x401300", kind="draft", on_behalf_of="Codex")
|
|
with self.assertRaises(HarborError):
|
|
self.h.claim("Codex", "fn:0x401400")
|
|
|
|
|
|
class BulkNotice(HarborCase):
|
|
def test_bulk_sends_one_notice_per_assignee(self):
|
|
self.h.claim("Claude", "range:0x401000-0x43FFFF", kind="band", on_behalf_of="Codex")
|
|
before = len(self.h.inbox("Codex", unread_only=False, limit=500, mark_read=False))
|
|
r = self.h.bulk_tasks("Claude", "0x401100 One\n0x401200 Two\n0x401300 Three")
|
|
self.assertEqual([t["assignee"] for t in r["created"]], ["Codex"] * 3)
|
|
new = self.h.inbox("Codex", unread_only=False, limit=500, mark_read=False)[before:]
|
|
self.assertEqual(len(new), 1)
|
|
self.assertIn("3 new tasks", new[0]["body"])
|
|
|
|
|
|
class Autopilot(HarborCase):
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.h.project.autopilot = True
|
|
|
|
def test_running_agents_keep_going_and_stopped_or_stuck_ones_rest(self):
|
|
self.h.claim("Claude", "range:0x401000-0x43FFFF", kind="band", on_behalf_of="Codex")
|
|
t = self.h.add_task("Claude", "Make 0x401100 readable", scope="fn:0x401100", assignee="Codex")
|
|
self.assertIn(f"#{t['id']}", self.h.continuation("Codex")) # offers the next task
|
|
self.h.take_task("Codex", t["id"])
|
|
self.assertIn(f"Continue task #{t['id']}", self.h.continuation("Codex"))
|
|
self.assertIsNotNone(self.h.continuation("Codex"))
|
|
self.assertIsNone(self.h.continuation("Codex")) # three idle turn ends: rest
|
|
self.assertEqual([a["agent"] for a in self.h.attention() if a["kind"] == "resting"], ["Codex"])
|
|
self.h.activity("Codex", "Edit", "codex", target="src/x.c") # real work resets the guard
|
|
self.assertIsNotNone(self.h.continuation("Codex"))
|
|
self.assertEqual([a for a in self.h.attention() if a["kind"] == "resting"], [])
|
|
self.h.set_flag("stop:Codex", "on", "User", "web")
|
|
self.assertIsNone(self.h.continuation("Codex")) # the human's stop rests it
|
|
|
|
def test_band_fallback_and_reviewer_work(self):
|
|
self.h.claim("Claude", "range:0x401000-0x43FFFF", kind="band", on_behalf_of="Codex")
|
|
self.assertIn("own band", self.h.continuation("Codex"))
|
|
self.assertIn("wait_for_messages", self.h.continuation("Kimi"))
|
|
item = self.h.submit("Codex", "Readable 0x401100", commit="abc123")
|
|
self.assertIn(f"#{item['id']}", self.h.continuation("Kimi"))
|
|
|
|
|
|
class LandMany(HarborCase):
|
|
def test_batch_landing_sends_one_message_per_author_and_one_win(self):
|
|
ids = [self.h.submit("Codex", f"Readable 0x40{i}000", commit=f"c{i}")["id"] for i in range(3)]
|
|
before = self.h.conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0]
|
|
landed = self.h.land_many("Claude", ids, "abc1234", note="readable check passed")
|
|
self.assertEqual([i["state"] for i in landed], ["landed"] * 3)
|
|
self.assertEqual(self.h.conn.execute("SELECT COUNT(*) FROM messages").fetchone()[0] - before, 2)
|
|
with self.assertRaises(HarborError):
|
|
self.h.land_many("Codex", ids, "abc1234")
|
|
|
|
|
|
class AuthorRefreshesCommit(HarborCase):
|
|
def test_author_hands_in_a_newer_commit_while_submitted_or_approved(self):
|
|
item = self.h.submit("Codex", "Readable 0x401000", commit="c1")
|
|
self.h.transition("Codex", item["id"], "submitted", note="newer", commit="c2")
|
|
self.assertEqual(self.h.item(item["id"])[0]["commit_hash"], "c2")
|
|
self.h.transition("Kimi", item["id"], "approved")
|
|
self.h.transition("Codex", item["id"], "submitted", note="one more fix", commit="c3")
|
|
self.assertEqual(self.h.item(item["id"])[0]["state"], "submitted") # back for review with c3
|
|
with self.assertRaises(HarborError):
|
|
self.h.transition("Codex", item["id"], "submitted") # no commit: nothing to refresh
|
|
with self.assertRaises(HarborError):
|
|
self.h.transition("Kimi", item["id"], "submitted", commit="c4") # only the author
|
|
|
|
|
|
class ProjectRefresh(unittest.TestCase):
|
|
def test_edits_to_the_project_file_apply_without_a_restart(self):
|
|
d = temp_dir()
|
|
try:
|
|
path = d / "p.toml"
|
|
body = 'name = "p"\ndatabase = \'{db}\'\nreviewers = ["Ann"]\n[[agents]]\nname = "Ann"\n[[agents]]\nname = "Bob"\n'
|
|
path.write_text(body.format(db=d / "p.db"), encoding="utf-8")
|
|
h = Harbor(projects.load_file(path))
|
|
self.assertFalse(h.refresh_project())
|
|
path.write_text(body.format(db=d / "p.db").replace('reviewers = ["Ann"]', 'reviewers = ["Bob"]\nautopilot = true'),
|
|
encoding="utf-8")
|
|
os.utime(path, (os.path.getmtime(path) + 5,) * 2)
|
|
self.assertTrue(h.refresh_project())
|
|
self.assertEqual((h.project.reviewers, h.project.autopilot), (("Bob",), True))
|
|
path.write_text("this is not toml [", encoding="utf-8")
|
|
os.utime(path, (os.path.getmtime(path) + 10,) * 2)
|
|
self.assertFalse(h.refresh_project()) # keeps the last good one
|
|
self.assertEqual(h.project.reviewers, ("Bob",))
|
|
h.close()
|
|
finally:
|
|
shutil.rmtree(d, ignore_errors=True)
|
|
|
|
|
|
class SharedFiles(HarborCase):
|
|
def test_shared_annotation_files_are_not_collisions(self):
|
|
self.h.project.shared_files = ("annotations/*.csv", "annotations/prototypes.h")
|
|
self.h.record_touches("Codex", ["annotations/names.csv", "annotations/prototypes.h", "src/a.c"])
|
|
cols = self.h.record_touches("Kimi", ["annotations/names.csv", "annotations/prototypes.h", "src/a.c"])
|
|
self.assertEqual([c["path"] for c in cols], ["src/a.c"])
|
|
self.assertEqual([c["path"] for c in self.h.recent_collisions()], ["src/a.c"])
|
|
|
|
|
|
class WorkingWithoutHooks(HarborCase):
|
|
def test_harbor_calls_and_file_scans_count_as_work_but_not_as_hooks(self):
|
|
self.h.activity("DeepSeek", "harbor submit", "deepseek", channel="mcp")
|
|
row = next(a for a in self.h.agents() if a["name"] == "DeepSeek")
|
|
self.assertTrue(self.h.working(row))
|
|
self.assertIsNone({r["agent"]: r for r in self.h.launch_status()}["DeepSeek"]["hooks"])
|
|
self.h.activity("DeepSeek", "edit", "", target="src/x.c", channel="files")
|
|
self.assertIn("files", {c["channel"] for c in self.h.connections()["DeepSeek"]})
|
|
|
|
|
|
class UnreadQuestions(HarborCase):
|
|
def test_unread_questions_to_an_agent_need_attention_until_read(self):
|
|
q = self.h.send("DeepSeek", ["Claude"], "May I take the 0x4C slice?", kind="question")
|
|
self.h.send("DeepSeek", ["All"], "status for everyone", kind="request")
|
|
kinds = lambda: [a for a in self.h.attention() if a["kind"] == "unread_questions"]
|
|
self.assertEqual(kinds(), []) # not yet 20 minutes old
|
|
self.clock.t += 25 * 60
|
|
found = kinds()
|
|
self.assertEqual([(a["agent"], f"#{q['id']}" in a["text"]) for a in found], [("Claude", True)])
|
|
self.h.inbox("Claude") # reading it clears the item
|
|
self.assertEqual(kinds(), [])
|
|
|
|
|
|
class AttentionWithoutFalseAlarms(HarborCase):
|
|
def test_stopped_agents_owe_no_acks_and_file_scopes_carry_their_address(self):
|
|
from harbor.util import parse_scope
|
|
self.assertEqual(parse_scope("file:src/game/_unassigned/4C6540_UnkPediaTextHelper.c")[1:], (0x4C6540, 0x4C6540))
|
|
self.assertEqual(parse_scope("file:annotations/names.csv")[1:], (None, None))
|
|
m = self.h.send("Codex", ["Kimi"], "please confirm", ack_required=True)
|
|
self.clock.t += 120 * 60
|
|
self.assertTrue(any(a["kind"] == "ack_overdue" for a in self.h.attention()))
|
|
self.h.set_flag("stop:Kimi", "on", "Claude", "cli")
|
|
self.assertFalse(any(a["kind"] == "ack_overdue" and f"#{m['id']}" in a["text"] for a in self.h.attention()))
|
|
|
|
|
|
class HooksVisible(HarborCase):
|
|
def test_mcp_without_hooks_needs_attention(self):
|
|
self.h.connect("Codex", "mcp", "codex")
|
|
no_hooks = lambda: [a["agent"] for a in self.h.attention() if a["kind"] == "no_hooks"]
|
|
launch = lambda: {r["agent"]: r for r in self.h.launch_status()}["Codex"]["hooks"]
|
|
self.assertEqual((no_hooks(), launch()), (["Codex"], None))
|
|
self.h.connect("Codex", "hook", "codex")
|
|
self.assertEqual(no_hooks(), [])
|
|
self.assertTrue(launch())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|