WHCandCursor
2602806aea
Rewrite the homepage for day-to-day use, move paper/eval drafts under docs/archive, and add a docs index so theory no longer leads the project. Co-authored-by: Cursor <cursoragent@cursor.com>
149 lines
4.6 KiB
Bash
Executable File
149 lines
4.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# One end-to-end run: copy fixtures/<name> to a temp workspace, run cst-agent
|
|
# headless on its TASK.md, then judge with the fixture's check.sh.
|
|
#
|
|
# Usage:
|
|
# scripts/e2e_fixture.sh <fixture> [--protocol P] [--provider NAME]
|
|
# [--model M] [--max-steps N] [--tools minimal|default] [--out DIR]
|
|
# [--record] [--baseline-only]
|
|
#
|
|
# --tools defaults to minimal (read_file / run_shell / write_file / apply_patch).
|
|
#
|
|
# Artifacts go to <out>/<fixture>__<protocol>__<model>__<stamp>/:
|
|
# result.json agent.log session.jsonl metrics.jsonl check.log [sse/]
|
|
#
|
|
# Environment:
|
|
# CST_AGENT_BIN agent binary (default: <repo>/cst-agent, built if missing)
|
|
# API key CST_API_KEY, or the provider variable (DEEPSEEK_API_KEY,
|
|
# OPENAI_API_KEY, MINIMAX_API_KEY), or the credentials file
|
|
# E2E_TIMEOUT seconds before the agent is killed (default 900; needs
|
|
# `timeout` or `gtimeout` from coreutils)
|
|
set -uo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
FIXTURE=""
|
|
PROTOCOL="${CST_PROTOCOL:-native-eos}"
|
|
PROVIDER="${CST_PROVIDER:-}"
|
|
MODEL="${CST_MODEL:-}"
|
|
MAX_STEPS=""
|
|
TOOLS="${CST_E2E_TOOLS:-minimal}"
|
|
OUT="$ROOT/out/e2e"
|
|
RECORD=0
|
|
BASELINE_ONLY=0
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--protocol) PROTOCOL="$2"; shift 2 ;;
|
|
--provider) PROVIDER="$2"; shift 2 ;;
|
|
--model) MODEL="$2"; shift 2 ;;
|
|
--max-steps) MAX_STEPS="$2"; shift 2 ;;
|
|
--tools) TOOLS="$2"; shift 2 ;;
|
|
--out) OUT="$2"; shift 2 ;;
|
|
--record) RECORD=1; shift ;;
|
|
--baseline-only) BASELINE_ONLY=1; shift ;;
|
|
-h|--help) sed -n '2,21p' "$0"; exit 0 ;;
|
|
-*) echo "unknown option: $1" >&2; exit 64 ;;
|
|
*) FIXTURE="$1"; shift ;;
|
|
esac
|
|
done
|
|
|
|
FIX="$ROOT/fixtures/$FIXTURE"
|
|
if [[ -z "$FIXTURE" || ! -f "$FIX/TASK.md" || ! -x "$FIX/check.sh" ]]; then
|
|
echo "usage: $0 <fixture> [options] (fixture needs TASK.md + check.sh)" >&2
|
|
exit 64
|
|
fi
|
|
|
|
BIN="${CST_AGENT_BIN:-$ROOT/cst-agent}"
|
|
if [[ ! -x "$BIN" ]]; then
|
|
make -s -C "$ROOT" cst-agent >/dev/null || { echo "build failed" >&2; exit 65; }
|
|
fi
|
|
|
|
STAMP="$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
|
MODEL_TAG="${MODEL:-default}"
|
|
RUN="$OUT/${FIXTURE}__${PROTOCOL}__${MODEL_TAG//\//_}__${STAMP}"
|
|
mkdir -p "$RUN"
|
|
WORK="$(mktemp -d /tmp/cst-e2e.XXXXXX)"
|
|
SCRATCH="$(mktemp -d /tmp/cst-e2e-base.XXXXXX)"
|
|
trap 'rm -rf "$WORK" "$SCRATCH"' EXIT
|
|
|
|
copy_fixture() {
|
|
cp -a "$FIX/." "$1/"
|
|
rm -rf "$1/out" "$1/.agent" "$1/.baseline_green_ok"
|
|
}
|
|
|
|
json_str() { python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$1"; }
|
|
|
|
write_result() {
|
|
local outcome="$1" agent_rc="$2" check_rc="$3" wall="$4"
|
|
cat >"$RUN/result.json" <<EOF
|
|
{"fixture": $(json_str "$FIXTURE"), "protocol": $(json_str "$PROTOCOL"),
|
|
"provider": $(json_str "${PROVIDER:-default}"), "model": $(json_str "$MODEL_TAG"),
|
|
"tools": $(json_str "$TOOLS"),
|
|
"run": $(json_str "$(basename "$RUN")"), "baseline_rc": $BASE_RC,
|
|
"agent_rc": $agent_rc, "check_rc": $check_rc, "wall_s": $wall,
|
|
"outcome": $(json_str "$outcome")}
|
|
EOF
|
|
echo "outcome=$outcome agent_rc=$agent_rc check_rc=$check_rc wall_s=$wall -> $RUN"
|
|
}
|
|
|
|
# Baseline in a throwaway copy: the task must not already be done.
|
|
copy_fixture "$SCRATCH"
|
|
"$SCRATCH/check.sh" >"$RUN/baseline.log" 2>&1
|
|
BASE_RC=$?
|
|
if [[ "$BASE_RC" -eq 0 && ! -e "$FIX/.baseline_green_ok" ]]; then
|
|
write_result unexpected_green_fixture 0 0 0
|
|
exit 2
|
|
fi
|
|
if [[ "$BASELINE_ONLY" -eq 1 ]]; then
|
|
write_result baseline_only 0 "$BASE_RC" 0
|
|
exit 0
|
|
fi
|
|
|
|
copy_fixture "$WORK"
|
|
args=(--headless --workspace "$WORK" --session "$WORK/.agent/e2e.jsonl"
|
|
--protocol "$PROTOCOL" --tools "$TOOLS")
|
|
[[ -n "$PROVIDER" ]] && args+=(--provider "$PROVIDER")
|
|
[[ -n "$MODEL" ]] && args+=(--model "$MODEL")
|
|
[[ -n "$MAX_STEPS" ]] && args+=(--max-steps "$MAX_STEPS")
|
|
mkdir -p "$WORK/.agent"
|
|
|
|
export CST_AUTO_APPROVE=1
|
|
export CST_METRICS_PATH="$RUN/metrics.jsonl"
|
|
if [[ "$RECORD" -eq 1 ]]; then
|
|
export CST_RECORD_SSE="$RUN/sse"
|
|
fi
|
|
|
|
now() { python3 -c 'import time; print(time.time())'; }
|
|
with_timeout() {
|
|
if command -v timeout >/dev/null 2>&1; then
|
|
timeout "${E2E_TIMEOUT:-900}" "$@"
|
|
elif command -v gtimeout >/dev/null 2>&1; then
|
|
gtimeout "${E2E_TIMEOUT:-900}" "$@"
|
|
else
|
|
"$@"
|
|
fi
|
|
}
|
|
|
|
t0=$(now)
|
|
with_timeout "$BIN" "${args[@]}" --prompt "$(cat "$FIX/TASK.md")" \
|
|
>"$RUN/agent.log" 2>&1
|
|
AGENT_RC=$?
|
|
t1=$(now)
|
|
WALL=$(python3 -c "print(round($t1-$t0,2))")
|
|
cp -f "$WORK/.agent/e2e.jsonl" "$RUN/session.jsonl" 2>/dev/null || true
|
|
|
|
(cd "$WORK" && ./check.sh) >"$RUN/check.log" 2>&1
|
|
CHECK_RC=$?
|
|
|
|
if [[ "$CHECK_RC" -eq 0 ]]; then
|
|
OUTCOME=pass
|
|
elif [[ "$AGENT_RC" -eq 124 ]]; then
|
|
OUTCOME=timeout
|
|
elif [[ "$AGENT_RC" -ne 0 ]]; then
|
|
OUTCOME=agent_error
|
|
else
|
|
OUTCOME=fail
|
|
fi
|
|
write_result "$OUTCOME" "$AGENT_RC" "$CHECK_RC" "$WALL"
|
|
[[ "$OUTCOME" == pass ]]
|