Usability pass: tool registry, TUI polish, saved keys and settings
Tools - Table-driven tool registry with permission classes and default/minimal profiles; new tools: edit_file, list_dir, glob, grep, delete_file, move_file, todo_write, web_fetch, ask_user (plus read_file paging). - Built-in Myers diff; write tools return unified diffs. - Persistent permission rules (.agent/permissions.json), per-turn undo snapshots (.agent/undo), AGENTS.md project notes, auto-compact at 80% of the model context, cancellable run_shell with process-group kill. TUI - Table-driven slash commands: /provider /key /effort /status /tools /permissions /yolo /undo /retry /diff /export /copy /rename /delete /init /max-steps /todo, with / and @ completion. - Permission modal with diff and prefix rules, ask_user modal, todo panel, header with model/effort/context, bracketed paste, line editing, wide-char cursor, NO_COLOR and ASCII fallback; Esc never quits. Startup and configuration - API keys are entered in the TUI (hidden) and saved per provider in <config>/credentials (0600); env variables are only a fallback. - Last provider, per-provider model and reasoning effort are remembered in <config>/settings.json; --effort/-e and /effort send reasoning_effort (low|medium|high|max) to models that accept it. - DeepSeek preset uses deepseek-flash / deepseek-v4-pro. - Session flags --new/--continue/--resume/--list-sessions, --yolo, --confirm. Compatibility and tests - ncurses detection, macOS/Homebrew support, make install, XDG config dir, POSIX cleanups; Gitea CI (gcc/clang/asan/musl/macOS). - Unit tests for diff, fs tools, registry, undo/perms and config; e2e fixtures multi_file_refactor and ask_or_todo; vendored cJSON.
This commit is contained in:
WHC
Executable
+148
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env bash
|
||||
# One end-to-end run: copy fixtures/<name> to a temp workspace, run cst-agent
|
||||
# headless on its TASK.md, then judge with the fixture's check.sh.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/e2e_fixture.sh <fixture> [--protocol P] [--provider NAME]
|
||||
# [--model M] [--max-steps N] [--tools minimal|default] [--out DIR]
|
||||
# [--record] [--baseline-only]
|
||||
#
|
||||
# --tools defaults to minimal (the four tools the paper experiments used).
|
||||
#
|
||||
# Artifacts go to <out>/<fixture>__<protocol>__<model>__<stamp>/:
|
||||
# result.json agent.log session.jsonl metrics.jsonl check.log [sse/]
|
||||
#
|
||||
# Environment:
|
||||
# CST_AGENT_BIN agent binary (default: <repo>/cst-agent, built if missing)
|
||||
# API key CST_API_KEY, or the provider variable (DEEPSEEK_API_KEY,
|
||||
# OPENAI_API_KEY, MINIMAX_API_KEY), or the credentials file
|
||||
# E2E_TIMEOUT seconds before the agent is killed (default 900; needs
|
||||
# `timeout` or `gtimeout` from coreutils)
|
||||
set -uo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
FIXTURE=""
|
||||
PROTOCOL="${CST_PROTOCOL:-native-eos}"
|
||||
PROVIDER="${CST_PROVIDER:-}"
|
||||
MODEL="${CST_MODEL:-}"
|
||||
MAX_STEPS=""
|
||||
TOOLS="${CST_E2E_TOOLS:-minimal}"
|
||||
OUT="$ROOT/out/e2e"
|
||||
RECORD=0
|
||||
BASELINE_ONLY=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--protocol) PROTOCOL="$2"; shift 2 ;;
|
||||
--provider) PROVIDER="$2"; shift 2 ;;
|
||||
--model) MODEL="$2"; shift 2 ;;
|
||||
--max-steps) MAX_STEPS="$2"; shift 2 ;;
|
||||
--tools) TOOLS="$2"; shift 2 ;;
|
||||
--out) OUT="$2"; shift 2 ;;
|
||||
--record) RECORD=1; shift ;;
|
||||
--baseline-only) BASELINE_ONLY=1; shift ;;
|
||||
-h|--help) sed -n '2,21p' "$0"; exit 0 ;;
|
||||
-*) echo "unknown option: $1" >&2; exit 64 ;;
|
||||
*) FIXTURE="$1"; shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
FIX="$ROOT/fixtures/$FIXTURE"
|
||||
if [[ -z "$FIXTURE" || ! -f "$FIX/TASK.md" || ! -x "$FIX/check.sh" ]]; then
|
||||
echo "usage: $0 <fixture> [options] (fixture needs TASK.md + check.sh)" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
BIN="${CST_AGENT_BIN:-$ROOT/cst-agent}"
|
||||
if [[ ! -x "$BIN" ]]; then
|
||||
make -s -C "$ROOT" cst-agent >/dev/null || { echo "build failed" >&2; exit 65; }
|
||||
fi
|
||||
|
||||
STAMP="$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
||||
MODEL_TAG="${MODEL:-default}"
|
||||
RUN="$OUT/${FIXTURE}__${PROTOCOL}__${MODEL_TAG//\//_}__${STAMP}"
|
||||
mkdir -p "$RUN"
|
||||
WORK="$(mktemp -d /tmp/cst-e2e.XXXXXX)"
|
||||
SCRATCH="$(mktemp -d /tmp/cst-e2e-base.XXXXXX)"
|
||||
trap 'rm -rf "$WORK" "$SCRATCH"' EXIT
|
||||
|
||||
copy_fixture() {
|
||||
cp -a "$FIX/." "$1/"
|
||||
rm -rf "$1/out" "$1/.agent" "$1/.baseline_green_ok"
|
||||
}
|
||||
|
||||
json_str() { python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$1"; }
|
||||
|
||||
write_result() {
|
||||
local outcome="$1" agent_rc="$2" check_rc="$3" wall="$4"
|
||||
cat >"$RUN/result.json" <<EOF
|
||||
{"fixture": $(json_str "$FIXTURE"), "protocol": $(json_str "$PROTOCOL"),
|
||||
"provider": $(json_str "${PROVIDER:-default}"), "model": $(json_str "$MODEL_TAG"),
|
||||
"tools": $(json_str "$TOOLS"),
|
||||
"run": $(json_str "$(basename "$RUN")"), "baseline_rc": $BASE_RC,
|
||||
"agent_rc": $agent_rc, "check_rc": $check_rc, "wall_s": $wall,
|
||||
"outcome": $(json_str "$outcome")}
|
||||
EOF
|
||||
echo "outcome=$outcome agent_rc=$agent_rc check_rc=$check_rc wall_s=$wall -> $RUN"
|
||||
}
|
||||
|
||||
# Baseline in a throwaway copy: the task must not already be done.
|
||||
copy_fixture "$SCRATCH"
|
||||
"$SCRATCH/check.sh" >"$RUN/baseline.log" 2>&1
|
||||
BASE_RC=$?
|
||||
if [[ "$BASE_RC" -eq 0 && ! -e "$FIX/.baseline_green_ok" ]]; then
|
||||
write_result unexpected_green_fixture 0 0 0
|
||||
exit 2
|
||||
fi
|
||||
if [[ "$BASELINE_ONLY" -eq 1 ]]; then
|
||||
write_result baseline_only 0 "$BASE_RC" 0
|
||||
exit 0
|
||||
fi
|
||||
|
||||
copy_fixture "$WORK"
|
||||
args=(--headless --workspace "$WORK" --session "$WORK/.agent/e2e.jsonl"
|
||||
--protocol "$PROTOCOL" --tools "$TOOLS")
|
||||
[[ -n "$PROVIDER" ]] && args+=(--provider "$PROVIDER")
|
||||
[[ -n "$MODEL" ]] && args+=(--model "$MODEL")
|
||||
[[ -n "$MAX_STEPS" ]] && args+=(--max-steps "$MAX_STEPS")
|
||||
mkdir -p "$WORK/.agent"
|
||||
|
||||
export CST_AUTO_APPROVE=1
|
||||
export CST_METRICS_PATH="$RUN/metrics.jsonl"
|
||||
if [[ "$RECORD" -eq 1 ]]; then
|
||||
export CST_RECORD_SSE="$RUN/sse"
|
||||
fi
|
||||
|
||||
now() { python3 -c 'import time; print(time.time())'; }
|
||||
with_timeout() {
|
||||
if command -v timeout >/dev/null 2>&1; then
|
||||
timeout "${E2E_TIMEOUT:-900}" "$@"
|
||||
elif command -v gtimeout >/dev/null 2>&1; then
|
||||
gtimeout "${E2E_TIMEOUT:-900}" "$@"
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
t0=$(now)
|
||||
with_timeout "$BIN" "${args[@]}" --prompt "$(cat "$FIX/TASK.md")" \
|
||||
>"$RUN/agent.log" 2>&1
|
||||
AGENT_RC=$?
|
||||
t1=$(now)
|
||||
WALL=$(python3 -c "print(round($t1-$t0,2))")
|
||||
cp -f "$WORK/.agent/e2e.jsonl" "$RUN/session.jsonl" 2>/dev/null || true
|
||||
|
||||
(cd "$WORK" && ./check.sh) >"$RUN/check.log" 2>&1
|
||||
CHECK_RC=$?
|
||||
|
||||
if [[ "$CHECK_RC" -eq 0 ]]; then
|
||||
OUTCOME=pass
|
||||
elif [[ "$AGENT_RC" -eq 124 ]]; then
|
||||
OUTCOME=timeout
|
||||
elif [[ "$AGENT_RC" -ne 0 ]]; then
|
||||
OUTCOME=agent_error
|
||||
else
|
||||
OUTCOME=fail
|
||||
fi
|
||||
write_result "$OUTCOME" "$AGENT_RC" "$CHECK_RC" "$WALL"
|
||||
[[ "$OUTCOME" == pass ]]
|
||||
Reference in New Issue
Block a user