hooks: any agent's hook restarts the project's web server when it is down ([server] port and start; at most once a minute, detached), so the board and chat no longer depend on the coordinator's session

harbor/hooks.py

@@ -200,6 +200,42 @@ def _context(host, event, text):
         print(text)
 
 
+def ensure_server(project, every=60):
+    """Restart the project's web server when it is down: [server] port and start (a command). Any agent's hook
+    does it, at most once a minute, so the board and chat come back without the coordinator's session. Detached,
+    so the server outlives the hook (and the app) that started it."""
+    port, start = project.server.get("port"), project.server.get("start")
+    if not port or not start:
+        return
+    stamp = project.db.parent / "server-check.stamp"
+    try:
+        import time
+        if stamp.exists() and time.time() - stamp.stat().st_mtime < every:
+            return
+        stamp.touch()
+        import socket
+        with socket.socket() as connection:
+            connection.settimeout(0.3)
+            if connection.connect_ex(("127.0.0.1", int(port))) == 0:
+                return
+        import subprocess
+        flags = 0
+        if os.name == "nt":
+            flags = (subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
+                     | subprocess.CREATE_BREAKAWAY_FROM_JOB | subprocess.CREATE_NO_WINDOW)
+        command = start if isinstance(start, list) else [start]
+        try:
+            subprocess.Popen(command, creationflags=flags, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
+                             stderr=subprocess.DEVNULL, close_fds=True)
+        except OSError:                                  # the job may refuse breakaway: start it without
+            subprocess.Popen(command, creationflags=flags & ~getattr(subprocess, "CREATE_BREAKAWAY_FROM_JOB", 0),
+                             stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+                             close_fds=True)
+        print(f"harbor: the web server on port {port} was down; restarting it", file=sys.stderr)
+    except Exception as error:                           # never break the host because of this
+        print(f"harbor: server check failed: {error}", file=sys.stderr)
+
+
 def run(event, project_spec, agent, host="generic"):
     payload = _payload()
     try:
@@ -216,6 +252,7 @@ def run(event, project_spec, agent, host="generic"):
         agent = project.session_identity(agent or os.environ.get("HARBOR_AGENT"), cwd)   # the session's own folder
         if not agent or agent not in project.ai_names:
             return 0
+        ensure_server(project)
         harbor = Harbor(project, busy_ms=1500, write_attempts=2)
         try:
             return _run(event, harbor, agent, host, payload)