Usability pass: tool registry, TUI polish, saved keys and settings
ci / linux (clang) (push) Canceled after 0s
ci / linux (gcc) (push) Canceled after 0s
ci / sanitizers (push) Canceled after 0s
ci / musl (push) Canceled after 0s
ci / macos (push) Canceled after 0s

Tools
- Table-driven tool registry with permission classes and default/minimal
  profiles; new tools: edit_file, list_dir, glob, grep, delete_file,
  move_file, todo_write, web_fetch, ask_user (plus read_file paging).
- Built-in Myers diff; write tools return unified diffs.
- Persistent permission rules (.agent/permissions.json), per-turn undo
  snapshots (.agent/undo), AGENTS.md project notes, auto-compact at 80%
  of the model context, cancellable run_shell with process-group kill.

TUI
- Table-driven slash commands: /provider /key /effort /status /tools
  /permissions /yolo /undo /retry /diff /export /copy /rename /delete
  /init /max-steps /todo, with / and @ completion.
- Permission modal with diff and prefix rules, ask_user modal, todo
  panel, header with model/effort/context, bracketed paste, line editing,
  wide-char cursor, NO_COLOR and ASCII fallback; Esc never quits.

Startup and configuration
- API keys are entered in the TUI (hidden) and saved per provider in
  <config>/credentials (0600); env variables are only a fallback.
- Last provider, per-provider model and reasoning effort are remembered
  in <config>/settings.json; --effort/-e and /effort send
  reasoning_effort (low|medium|high|max) to models that accept it.
- DeepSeek preset uses deepseek-flash / deepseek-v4-pro.
- Session flags --new/--continue/--resume/--list-sessions, --yolo,
  --confirm.

Compatibility and tests
- ncurses detection, macOS/Homebrew support, make install, XDG config
  dir, POSIX cleanups; Gitea CI (gcc/clang/asan/musl/macOS).
- Unit tests for diff, fs tools, registry, undo/perms and config;
  e2e fixtures multi_file_refactor and ask_or_todo; vendored cJSON.
This commit is contained in:
WHC
2026-10-05 01:47:01 +08:00
parent 9877ff48b6
commit 2fde898da0
105 changed files with 19348 additions and 3786 deletions
+186
View File
@@ -0,0 +1,186 @@
#!/usr/bin/env python3
"""Summarize e2e runs (scripts/e2e_fixture.sh / run_matrix.sh output).
Usage: scripts/summarize.py <out-dir> [--md summary.md] [--csv results.csv]
Every run directory holds result.json and metrics.jsonl. Per run we derive:
steps model calls in the turn
llm_s sum of stream durations (request start -> stream end)
tool_s sum of tool execution time
overlap_s tool time that ran while the model was still streaming
(eager protocols; 0 for end-of-stream execution)
confirm_ms mean call open -> call confirmed (args complete)
wait_ms mean call confirmed -> execution start
ctl_errors contract errors (model left the CTL/XML grammar)
tokens prompt / completion / cached prompt tokens
"""
import csv
import json
import os
import statistics
import sys
from collections import defaultdict
FIELDS = [
"run", "fixture", "protocol", "provider", "model", "outcome", "pass",
"wall_s", "turn_s", "steps", "calls", "calls_failed", "llm_s", "tool_s",
"overlap_s", "confirm_ms", "wait_ms", "ctl_errors", "retries",
"prompt_tokens", "completion_tokens", "cached_tokens",
]
def load_jsonl(path):
rows = []
if not os.path.exists(path):
return rows
with open(path, encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
rows.append(json.loads(line))
except json.JSONDecodeError:
pass
return rows
def analyze(run_dir):
with open(os.path.join(run_dir, "result.json"), encoding="utf-8") as f:
res = json.load(f)
if res.get("outcome") in ("baseline_only", "unexpected_green_fixture"):
return None
m = load_jsonl(os.path.join(run_dir, "metrics.jsonl"))
steps = [r for r in m if r.get("type") == "step"]
calls = [r for r in m if r.get("type") == "call"]
turns = [r for r in m if r.get("type") == "turn"]
end_of = {(s["turn"], s["step"]): s.get("t_stream_end", 0) for s in steps}
overlap = 0.0
confirm, wait = [], []
tool_ms = 0.0
for c in calls:
start, end = c.get("t_exec_start", -1), c.get("t_exec_end", -1)
if start >= 0 and end >= start:
tool_ms += end - start
stream_end = end_of.get((c["turn"], c["step"]))
if stream_end is not None and start < stream_end:
overlap += min(end, stream_end) - start
if c.get("t_call_open", -1) >= 0 and c.get("t_call_ready", -1) >= 0:
confirm.append(c["t_call_ready"] - c["t_call_open"])
if c.get("t_call_ready", -1) >= 0 and start >= 0:
wait.append(start - c["t_call_ready"])
model = res.get("model", "")
if model in ("", "default") and steps:
model = steps[0].get("model", model)
return {
"run": res.get("run", os.path.basename(run_dir)),
"fixture": res.get("fixture", ""),
"protocol": res.get("protocol", ""),
"provider": res.get("provider", ""),
"model": model,
"outcome": res.get("outcome", ""),
"pass": 1 if res.get("outcome") == "pass" else 0,
"wall_s": res.get("wall_s", 0),
"turn_s": round(sum(t.get("t_total", 0) for t in turns) / 1000, 3),
"steps": len(steps),
"calls": len(calls),
"calls_failed": sum(1 for c in calls if c.get("exit", 0) != 0
or c.get("status") != "ok"),
"llm_s": round(sum(s.get("t_stream_end", 0) for s in steps) / 1000, 3),
"tool_s": round(tool_ms / 1000, 3),
"overlap_s": round(overlap / 1000, 3),
"confirm_ms": round(statistics.mean(confirm), 1) if confirm else "",
"wait_ms": round(statistics.mean(wait), 1) if wait else "",
"ctl_errors": sum(1 for s in steps if s.get("parse_error")),
"retries": sum(s.get("retries", 0) for s in steps),
"prompt_tokens": sum(s.get("prompt_tokens", 0) for s in steps),
"completion_tokens": sum(s.get("completion_tokens", 0) for s in steps),
"cached_tokens": sum(s.get("cached_tokens", 0) for s in steps),
}
def mean(rows, key, nd=2):
vals = [r[key] for r in rows if r[key] != ""]
if not vals:
return "-"
v = statistics.mean(vals)
return int(round(v)) if nd == 0 else round(v, nd)
def table(rows, keys, title):
groups = defaultdict(list)
for r in rows:
groups[tuple(r[k] for k in keys)].append(r)
cols = keys + ["n", "pass%", "steps", "wall_s", "llm_s", "tool_s",
"overlap_s", "wait_ms", "ctl_err", "tok_in", "tok_out"]
out = [f"### {title}", "", "| " + " | ".join(cols) + " |",
"|" + "---|" * len(cols)]
for g in sorted(groups):
rs = groups[g]
n = len(rs)
cells = list(g) + [
n,
round(100.0 * sum(r["pass"] for r in rs) / n),
mean(rs, "steps", 1),
mean(rs, "wall_s"),
mean(rs, "llm_s"),
mean(rs, "tool_s"),
mean(rs, "overlap_s"),
mean(rs, "wait_ms", 0),
mean(rs, "ctl_errors", 2),
mean(rs, "prompt_tokens", 0),
mean(rs, "completion_tokens", 0),
]
out.append("| " + " | ".join(str(c) for c in cells) + " |")
out.append("")
return "\n".join(out)
def main(argv):
if len(argv) < 2:
print(__doc__)
return 64
root = argv[1]
md_path = os.path.join(root, "summary.md")
csv_path = os.path.join(root, "results.csv")
if "--md" in argv:
md_path = argv[argv.index("--md") + 1]
if "--csv" in argv:
csv_path = argv[argv.index("--csv") + 1]
rows = []
for name in sorted(os.listdir(root)):
d = os.path.join(root, name)
if os.path.isfile(os.path.join(d, "result.json")):
r = analyze(d)
if r:
rows.append(r)
if not rows:
print(f"no runs under {root}")
return 1
with open(csv_path, "w", newline="", encoding="utf-8") as f:
w = csv.DictWriter(f, fieldnames=FIELDS)
w.writeheader()
w.writerows(rows)
md = [f"## Results ({len(rows)} runs)", "",
"wait_ms = call confirmed -> execution start (~0 for eager "
"protocols, large when buffered until stream end). overlap_s = "
"tool time hidden behind generation.", "",
table(rows, ["model", "protocol"], "By model x protocol"),
table(rows, ["model", "protocol", "fixture"],
"By model x protocol x fixture")]
text = "\n".join(md)
with open(md_path, "w", encoding="utf-8") as f:
f.write(text)
print(text)
print(f"wrote {csv_path} and {md_path}")
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))