v0.8.2: build fixes, WebUI security and correctness

Build: <fcntl.h> in tui.c, format-truncation in http_srv.c and /web.

WebUI:
- refuse cross-site requests (Origin must be the page itself) in the
  default no-token mode: any page could POST approval=yolo and a prompt
- session/workspace/undo/compact/attach/model changes are 409 while a
  turn runs (use-after-free, messages landing in the wrong session)
- ask_user returned "dismissed" at once instead of waiting
- cancel and shutdown wake a turn parked on a permission or question;
  SIGTERM mid-turn no longer hangs or crashes
- a page opened while a question is pending still gets it;
  Last-Event-ID resumes the stream; event seq matches the SSE id
- token mode: page assets load without the token, images via cookie
- --web-open works, --web-port/--web-token are validated
- frontend: gate keys ignored while typing, clear-images button,
  reasoning no longer wiped while streaming

HTTP server: auth and size cap before reading the body, strict
Content-Length, NUL in url_decode, 413 on keep-alive, idle timeout,
absolute-form only without a leading '/', case-insensitive Connection,
header deadline, 400 for long request lines, unlink before close,
poll() instead of select(), SO_EXCLUSIVEADDRUSE on Windows.

Markdown sanitizer escapes quotes in attribute values.

/web: passes this session's provider/model/base/effort/approval (key
via the environment), keeps output off the TUI, shows the real URL and
keeps CURRENT on the TUI's session.

Tests: scripts/e2e_webui_ask.py, scripts/e2e_webui_http.py (run from
e2e_webui.sh), sanitizer quote test.

scripts/e2e_webui.sh

@@ -47,6 +47,19 @@ BASE="http://127.0.0.1:$PORT"
 echo "   port $PORT"
 
 fail() { echo "FAIL: $1"; echo "--- events"; cat "$EVENTS" 2>/dev/null | head -40; exit 1; }
+# SIGTERM must shut the server down cleanly, even mid-turn (128+ = a crash)
+stop_server() {
+  kill "$1" 2>/dev/null || return 0
+  n=0
+  while kill -0 "$1" 2>/dev/null && [ $n -lt 200 ]; do n=$((n + 1)); sleep 0.2; done
+  if kill -0 "$1" 2>/dev/null; then
+    kill -9 "$1" 2>/dev/null || true
+    fail "$2 hung on SIGTERM"
+  fi
+  st=0
+  wait "$1" 2>/dev/null || st=$?
+  [ "$st" -lt 128 ] || fail "$2 died with status $st on SIGTERM"
+}
 
 # 1. the shell page and the state endpoint
 curl -fsS --max-time 5 "$BASE/" | grep -q "<title>CST-Agent" || fail "GET / is not the shell page"
@@ -160,7 +173,7 @@ fi
 # 7. the interactive approval path: a fresh server in ask mode must raise a
 # permission event, and answering it must let the turn continue.
 echo "== permission round trip (-A ask)"
-kill "$SRV_PID" 2>/dev/null || true   # the first server's replay is spent
+stop_server "$SRV_PID" "the first server"   # its replay is spent
 rm -rf "$WORK/pw"; mkdir -p "$WORK/pw"
 P_EVENTS="$WORK/permission-events.txt"
 P_LOG="$WORK/permission-server.log"   # its own log: the port parse must not
@@ -206,7 +219,7 @@ else
     i=$((i + 1)); sleep 0.2
   done
   grep -q "event: permission_done" "$P_EVENTS" || fail "the answer was not acknowledged"
-  kill "$P_PID" 2>/dev/null || true
+  stop_server "$P_PID" "the ask-mode server"
   kill "$P_CURL" 2>/dev/null || true
   echo "   answered request $PID_REQ with 'allow once'"
 fi
@@ -260,4 +273,15 @@ done
 kill -0 "$T_PID" 2> /dev/null && fail "the server ignored /api/quit"
 wait "$T_PID" 2>/dev/null || true
 
+# 9. ask_user waits for the answer, and the HTTP hardening checks (both
+# need a scripted model: python3)
+if command -v python3 > /dev/null; then
+  echo "== ask_user round trip"
+  CST_AGENT="$BIN" python3 "$ROOT/scripts/e2e_webui_ask.py" > "$WORK/ask.txt" 2>&1 ||
+    { cat "$WORK/ask.txt"; fail "the ask_user round trip"; }
+  echo "== HTTP hardening"
+  CST_AGENT="$BIN" python3 "$ROOT/scripts/e2e_webui_http.py" > "$WORK/http.txt" 2>&1 ||
+    { cat "$WORK/http.txt"; fail "the HTTP hardening checks"; }
+fi
+
 echo "== OK: (events: $(grep -c '^event: ' "$EVENTS"))"

scripts/e2e_webui_ask.py

@@ -0,0 +1,164 @@
+"""WebUI ask_user round trip against a fake OpenAI-compatible API.
+
+  python3 scripts/e2e_webui_ask.py      (CST_AGENT= picks the binary)
+
+Checks: the turn really waits for the answer, a page opened while the
+question is open still receives it, and Last-Event-ID resumes the stream.
+"""
+import json, os, shutil, subprocess, sys, tempfile, threading, time, urllib.request
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+BIN = os.environ.get("CST_AGENT", os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "cst-agent"))
+REQS = []
+
+
+class H(BaseHTTPRequestHandler):
+    def log_message(self, *a):
+        pass
+
+    def do_GET(self):
+        body = json.dumps({"data": [{"id": "deepseek-flash"}]}).encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_POST(self):
+        n = int(self.headers.get("Content-Length", 0))
+        REQS.append(json.loads(self.rfile.read(n)))
+        self.send_response(200)
+        self.send_header("Content-Type", "text/event-stream")
+        self.end_headers()
+        base = {"id": "x", "object": "chat.completion.chunk", "model": "deepseek-flash"}
+
+        def ev(o):
+            self.wfile.write(b"data: " + json.dumps({**base, **o}).encode() + b"\n\n")
+            self.wfile.flush()
+        if len(REQS) == 1:
+            args = json.dumps({"question": "Pick a colour?", "options": "red\nblue"})
+            ev({"choices": [{"index": 0, "delta": {"role": "assistant", "tool_calls": [
+                {"index": 0, "id": "call_1", "type": "function",
+                 "function": {"name": "ask_user", "arguments": args}}]}}]})
+            ev({"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]})
+        else:
+            ev({"choices": [{"index": 0, "delta": {"role": "assistant", "content": "done"}}]})
+            ev({"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]})
+        self.wfile.write(b"data: [DONE]\n\n")
+        self.wfile.flush()
+
+
+fails = 0
+
+
+def check(ok, msg):
+    global fails
+    print(("PASS " if ok else "FAIL ") + msg)
+    fails += 0 if ok else 1
+
+
+class Sse:
+    def __init__(self, base, last_id=None):
+        self.lines = []
+        req = urllib.request.Request(base + "/api/events")
+        if last_id is not None:
+            req.add_header("Last-Event-ID", str(last_id))
+        self.r = urllib.request.urlopen(req, timeout=60)
+        threading.Thread(target=self.pump, daemon=True).start()
+
+    def pump(self):
+        try:
+            for raw in self.r:
+                self.lines.append(raw.decode())
+        except Exception:
+            pass
+
+    def events(self, name):
+        out = []
+        for i, l in enumerate(self.lines):
+            if l.strip() == "event: " + name and i + 1 < len(self.lines):
+                out.append(json.loads(self.lines[i + 1][6:]))
+        return out
+
+    def ids(self):
+        return [int(l[4:]) for l in self.lines if l.startswith("id: ")]
+
+    def close(self):
+        try:
+            self.r.close()
+        except Exception:
+            pass
+
+
+def wait(fn, t=15):
+    end = time.time() + t
+    while time.time() < end:
+        v = fn()
+        if v:
+            return v
+        time.sleep(0.1)
+    return None
+
+
+def post(base, path, obj):
+    req = urllib.request.Request(base + path, data=json.dumps(obj).encode(),
+                                 headers={"Content-Type": "application/json"})
+    try:
+        with urllib.request.urlopen(req, timeout=5) as r:
+            return r.status
+    except urllib.error.HTTPError as e:
+        return e.code
+
+
+api = ThreadingHTTPServer(("127.0.0.1", 0), H)
+threading.Thread(target=api.serve_forever, daemon=True).start()
+ws = tempfile.mkdtemp(prefix="cst-ask-")
+log = open(ws + "/server.log", "w+")
+p = subprocess.Popen([BIN, "--web", "--web-port", "0", "--workspace", ws, "--new",
+                      "--api-key", "sk-x", "--provider", "deepseek", "--model", "deepseek-flash",
+                      "--api-base", "http://127.0.0.1:%d/v1" % api.server_address[1],
+                      "--protocol", "native-eos", "--approval", "auto"],
+                     stdout=log, stderr=subprocess.STDOUT)
+try:
+    def port():
+        log.seek(0)
+        for l in log.read().splitlines():
+            if l.startswith("webui: http://127.0.0.1:"):
+                return l.split(":")[3].split("/")[0]
+    pt = wait(port)
+    base = "http://127.0.0.1:%s" % pt
+    a = Sse(base)
+    time.sleep(0.3)
+    check(post(base, "/api/turn", {"text": "ask me"}) == 202, "turn accepted")
+    ask = wait(lambda: a.events("ask_user"))
+    check(bool(ask), "ask_user event raised")
+    time.sleep(1.5)
+    check(len(REQS) == 1, "the turn waits for the answer (requests so far: %d)" % len(REQS))
+    # a page opened now must still see the open question
+    b = Sse(base)
+    check(bool(wait(lambda: b.events("ask_user"), 5)), "a late page gets the open question")
+    # reconnect with Last-Event-ID: nothing before that id is resent
+    last = a.ids()[-1]
+    c = Sse(base, last_id=last - 1)
+    got = wait(lambda: c.ids(), 5)
+    check(bool(got) and got[0] == last, "Last-Event-ID resumes after that id (%s, want %d)" % (got[:1] if got else got, last))
+    if ask:
+        check(post(base, "/api/ask_user", {"id": ask[0]["id"], "text": "blue"}) == 200, "answer accepted")
+    end = wait(lambda: a.events("turn_end"), 20)
+    check(bool(end) and end[0].get("rc") == 0, "turn ended rc=0")
+    tool_msgs = [m for m in (REQS[1]["messages"] if len(REQS) > 1 else []) if m.get("role") == "tool"]
+    check(bool(tool_msgs) and "blue" in json.dumps(tool_msgs), "the answer reached the model: %s" % json.dumps(tool_msgs)[:200])
+    for s in (a, b, c):
+        s.close()
+finally:
+    p.terminate()
+    try:
+        p.wait(20)
+    except subprocess.TimeoutExpired:
+        p.kill()
+        check(False, "server hung on SIGTERM")
+    check(p.returncode is not None and p.returncode < 128 and p.returncode >= 0, "clean exit (%s)" % p.returncode)
+    log.close()
+    shutil.rmtree(ws, ignore_errors=True)
+print("FAILED:", fails)
+sys.exit(1 if fails else 0)

scripts/e2e_webui_http.py

@@ -0,0 +1,234 @@
+"""WebUI HTTP hardening checks over raw sockets, against a scripted model.
+
+  python3 scripts/e2e_webui_http.py      (CST_AGENT= picks the binary)
+
+Cross-site requests, request parsing (Content-Length, NUL, long lines,
+keep-alive), auth before the body, changes refused while a turn runs,
+cancel waking a pending question, token mode and shutdown.
+"""
+import json, os, re, shutil, socket, subprocess, sys, tempfile, threading, time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+BIN = os.environ.get("CST_AGENT", os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "cst-agent"))
+REQS = []
+
+
+class Api(BaseHTTPRequestHandler):
+    """First call: ask_user. Later calls: a plain answer."""
+
+    def log_message(self, *a):
+        pass
+
+    def do_GET(self):
+        body = json.dumps({"data": [{"id": "deepseek-flash"}]}).encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_POST(self):
+        n = int(self.headers.get("Content-Length", 0))
+        REQS.append(json.loads(self.rfile.read(n)))
+        self.send_response(200)
+        self.send_header("Content-Type", "text/event-stream")
+        self.end_headers()
+        base = {"id": "x", "object": "chat.completion.chunk", "model": "deepseek-flash"}
+
+        def ev(o):
+            self.wfile.write(b"data: " + json.dumps({**base, **o}).encode() + b"\n\n")
+            self.wfile.flush()
+        if len(REQS) == 1:
+            args = json.dumps({"question": "Pick?", "options": "a\nb"})
+            ev({"choices": [{"index": 0, "delta": {"role": "assistant", "tool_calls": [
+                {"index": 0, "id": "call_1", "type": "function",
+                 "function": {"name": "ask_user", "arguments": args}}]}}]})
+            ev({"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]})
+        else:
+            ev({"choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]})
+            ev({"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]})
+        self.wfile.write(b"data: [DONE]\n\n")
+        self.wfile.flush()
+
+
+fails = 0
+
+
+def check(ok, msg):
+    global fails
+    print(("PASS " if ok else "FAIL ") + msg)
+    fails += 0 if ok else 1
+
+
+def raw(port, data, timeout=5, read_all=True):
+    """Send bytes, return everything read until EOF/timeout."""
+    s = socket.create_connection(("127.0.0.1", port), timeout=timeout)
+    s.sendall(data)
+    out = b""
+    try:
+        while True:
+            b = s.recv(65536)
+            if not b:
+                break
+            out += b
+            if not read_all:
+                break
+    except socket.timeout:
+        pass
+    s.close()
+    return out.decode("latin1")
+
+
+def status(resp):
+    # a pipelined response starts right after the previous body, not on a new line
+    return [int(m) for m in re.findall(r"HTTP/1\.1 (\d{3}) ", resp)]
+
+
+def req(method, path, headers=None, body=b"", host=None):
+    h = {"Host": host or "127.0.0.1:%d" % PORT, "Connection": "close"}
+    h.update(headers or {})
+    if body or method == "POST":
+        h["Content-Length"] = str(len(body))
+    head = "%s %s HTTP/1.1\r\n" % (method, path) + "".join("%s: %s\r\n" % kv for kv in h.items()) + "\r\n"
+    return head.encode() + body
+
+
+def start(ws, *extra):
+    log = open(os.path.join(ws, "server.log"), "w+")
+    p = subprocess.Popen([BIN, "--web", "--web-port", "0", "--workspace", ws, "--new",
+                          "--api-key", "sk-x", "--provider", "deepseek", "--model", "deepseek-flash",
+                          "--api-base", "http://127.0.0.1:%d/v1" % api.server_address[1],
+                          "--protocol", "native-eos", "--approval", "auto", *extra],
+                         stdout=log, stderr=subprocess.STDOUT)
+    end = time.time() + 10
+    while time.time() < end:
+        log.seek(0)
+        for l in log.read().splitlines():
+            if l.startswith("webui: http://127.0.0.1:"):
+                return p, int(l.split(":")[3].split("/")[0]), log
+        time.sleep(0.1)
+    raise SystemExit("server did not start")
+
+
+def stop(p, name):
+    p.terminate()
+    try:
+        p.wait(20)
+    except subprocess.TimeoutExpired:
+        p.kill()
+        check(False, name + " hung on SIGTERM")
+        return
+    check(0 <= p.returncode < 128, "%s exits cleanly (%s)" % (name, p.returncode))
+
+
+api = ThreadingHTTPServer(("127.0.0.1", 0), Api)
+threading.Thread(target=api.serve_forever, daemon=True).start()
+root = tempfile.mkdtemp(prefix="cst-http-")
+try:
+    ws = os.path.join(root, "a")
+    os.makedirs(ws)
+    p, PORT, log = start(ws)
+    me = "http://127.0.0.1:%d" % PORT
+    cfg = b'{"approval":"ask"}'
+    # 1. cross-site requests
+    r = raw(PORT, req("POST", "/api/config", {"Origin": "http://evil.example", "Content-Type": "text/plain"}, cfg))
+    check(status(r)[:1] == [401], "a cross-site POST is refused (%s)" % status(r))
+    r = raw(PORT, req("POST", "/api/config", {"Origin": "null"}, cfg))
+    check(status(r)[:1] == [401], "Origin: null is refused")
+    r = raw(PORT, req("POST", "/api/config", {"Origin": me, "Content-Type": "application/json"}, b'{"approval":"auto"}'))
+    check(status(r)[:1] == [200], "our own page may POST (%s)" % status(r))
+    r = raw(PORT, req("POST", "/api/config", {}, b'{"approval":"auto"}'))
+    check(status(r)[:1] == [200], "a client without Origin (curl) may POST")
+    r = raw(PORT, req("GET", "/api/attach/img-0000.png", host="evil.example"))
+    check(status(r)[:1] == [401], "session images need the Host check too (%s)" % status(r))
+    r = raw(PORT, req("GET", "/app.js"))
+    check(status(r)[:1] == [200], "assets are served")
+    # 2. request parsing
+    r = raw(PORT, req("GET", "/api/state%00x"))
+    check(status(r)[:1] == [400], "an encoded NUL in the path is a 400 (%s)" % status(r))
+    smuggle = b"GET /api/sessions HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"
+    r = raw(PORT, ("POST /api/cancel HTTP/1.1\r\nHost: 127.0.0.1:%d\r\nContent-Length: abc\r\n\r\n" % PORT).encode() + smuggle)
+    check(status(r) == [400], "Content-Length: abc is a 400, nothing smuggled (%s)" % status(r))
+    r = raw(PORT, ("POST /api/cancel HTTP/1.1\r\nHost: 127.0.0.1:%d\r\nContent-Length: 0\r\nContent-Length: %d\r\n\r\n"
+                   % (PORT, len(smuggle))).encode() + smuggle)
+    check(status(r) == [400], "two different Content-Lengths are a 400 (%s)" % status(r))
+    r = raw(PORT, ("GET /%s HTTP/1.1\r\nHost: x\r\n\r\n" % ("a" * 9000)).encode())
+    check(status(r)[:1] == [400], "an over-long request line is a 400 (%s)" % status(r))
+    r = raw(PORT, req("GET", "/api/state?x=http://a/b"))
+    check(status(r)[:1] == [200], "'://' in the query leaves the path alone (%s)" % status(r))
+    r = raw(PORT, req("GET", "/api/state", {"Connection": "Close"}), timeout=3)
+    check("Connection: close" in r, "Connection: Close (any case) closes")
+    ka = ("GET /api/state HTTP/1.1\r\nHost: 127.0.0.1:%d\r\n\r\n" % PORT).encode()
+    big = ("POST /api/turn HTTP/1.1\r\nHost: 127.0.0.1:%d\r\nContent-Length: 5000000\r\n\r\n" % PORT).encode()
+    r = raw(PORT, ka + big)
+    check(status(r) == [200, 413], "a 413 after a keep-alive request still arrives (%s)" % status(r))
+    # 3. a pending question: changes are refused, approval may change, cancel wakes it
+    events = []
+
+    def sse():
+        s = socket.create_connection(("127.0.0.1", PORT))
+        s.sendall(("GET /api/events HTTP/1.1\r\nHost: 127.0.0.1:%d\r\n\r\n" % PORT).encode())
+        buf = b""
+        s.settimeout(60)
+        try:
+            while True:
+                b = s.recv(65536)
+                if not b:
+                    break
+                buf += b
+                events[:] = [l[7:] for l in buf.decode("latin1").split("\n") if l.startswith("event: ")]
+        except Exception:
+            pass
+    threading.Thread(target=sse, daemon=True).start()
+    time.sleep(0.4)
+    r = raw(PORT, req("POST", "/api/turn", {"Content-Type": "application/json"}, b'{"text":"go"}'))
+    check(status(r)[:1] == [202], "turn accepted")
+    end = time.time() + 15
+    while time.time() < end and "ask_user" not in events:
+        time.sleep(0.1)
+    check("ask_user" in events, "the question is open")
+    for path, body in (("/api/sessions/new", b""), ("/api/workspace", json.dumps({"path": root}).encode()),
+                       ("/api/compact", b""), ("/api/undo", b""), ("/api/config", b'{"model":"x"}')):
+        r = raw(PORT, req("POST", path, {"Content-Type": "application/json"}, body))
+        check(status(r)[:1] == [409], "%s is refused mid-turn (%s)" % (path, status(r)))
+    r = raw(PORT, req("POST", "/api/config", {"Content-Type": "application/json"}, b'{"approval":"yolo"}'))
+    check(status(r)[:1] == [200], "approval may change mid-turn")
+    r = raw(PORT, req("POST", "/api/cancel", {"Content-Type": "application/json"}, b"{}"))
+    end = time.time() + 10
+    while time.time() < end and "turn_end" not in events:
+        time.sleep(0.1)
+    check("turn_end" in events, "cancel ends a turn parked on a question")
+    r = raw(PORT, req("POST", "/api/sessions/new", {"Content-Type": "application/json"}))
+    check(status(r)[:1] == [200], "after the turn, a new session is fine (%s)" % status(r))
+    stop(p, "the server")
+
+    # 4. token mode: the page loads, the API and images do not without it
+    ws2 = os.path.join(root, "b")
+    os.makedirs(ws2)
+    p, PORT, log = start(ws2, "--web-token", "tok-x")
+    r = raw(PORT, req("GET", "/app.js"))
+    check(status(r)[:1] == [200], "token mode: app.js loads without the token (%s)" % status(r))
+    r = raw(PORT, req("GET", "/api/state"))
+    check(status(r)[:1] == [401], "token mode: the API needs it")
+    r = raw(PORT, req("GET", "/api/attach/img-0000.png"))
+    check(status(r)[:1] == [401], "token mode: images need it")
+    r = raw(PORT, req("GET", "/api/state", {"Cookie": "cst_token=tok-x"}))
+    check(status(r)[:1] == [200], "token mode: the cookie works")
+    t0 = time.time()
+    r = raw(PORT, ("POST /api/attach HTTP/1.1\r\nHost: 127.0.0.1:%d\r\nContent-Length: 33554431\r\n"
+                   "Expect: 100-continue\r\n\r\n" % PORT).encode(), timeout=5)
+    check(status(r) == [401] and time.time() - t0 < 3,
+          "no token, no body read: 401 at once (%s, %.1fs)" % (status(r), time.time() - t0))
+    stop(p, "the token server")
+
+    # 5. option checks
+    rc = subprocess.run([BIN, "--web", "--web-port", "70000", "--workspace", ws2],
+                        capture_output=True, timeout=10).returncode
+    check(rc != 0, "--web-port 70000 is refused")
+    rc = subprocess.run([BIN, "--web", "--web-token", "x" * 80, "--web-port", "0", "--workspace", ws2,
+                         "--api-key", "k"], capture_output=True, timeout=10).returncode
+    check(rc != 0, "an over-long --web-token is refused")
+finally:
+    shutil.rmtree(root, ignore_errors=True)
+print("FAILED:", fails)
+sys.exit(1 if fails else 0)