WHCandCursor
2602806aea
Rewrite the homepage for day-to-day use, move paper/eval drafts under docs/archive, and add a docs index so theory no longer leads the project. Co-authored-by: Cursor <cursoragent@cursor.com>
64 lines
2.0 KiB
Bash
Executable File
64 lines
2.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Experiment matrix: protocol x model x fixture x N runs, then summarize.
|
|
#
|
|
# Usage:
|
|
# scripts/run_matrix.sh [-p "native-eos native-inc ctl ctl-heredoc xml"]
|
|
# [-m "deepseek:deepseek-flash openai:gpt-4.1-mini"]
|
|
# [-f "mini_calc makefile_tab big_file test_fix_test"]
|
|
# [-n 3] [-o out/matrix-<stamp>] [-r] [-t minimal|default]
|
|
#
|
|
# -m provider:model pairs (model may be empty: "deepseek:")
|
|
# -r also record raw SSE streams per run (for replay experiments)
|
|
# -t tool profile (default minimal)
|
|
#
|
|
# Each run is scripts/e2e_fixture.sh; results land in <out>/<run>/ and the
|
|
# summary in <out>/results.csv and <out>/summary.md.
|
|
set -uo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
PROTOCOLS="native-eos native-inc ctl ctl-heredoc xml"
|
|
MODELS="deepseek:deepseek-flash"
|
|
FIXTURES="mini_calc makefile_tab big_file test_fix_test"
|
|
N=3
|
|
OUT="$ROOT/out/matrix-$(date -u +%Y%m%dT%H%M%SZ)"
|
|
RECORD=()
|
|
TOOLS=minimal
|
|
|
|
while getopts "p:m:f:n:o:rt:h" opt; do
|
|
case "$opt" in
|
|
p) PROTOCOLS="$OPTARG" ;;
|
|
m) MODELS="$OPTARG" ;;
|
|
f) FIXTURES="$OPTARG" ;;
|
|
n) N="$OPTARG" ;;
|
|
o) OUT="$OPTARG" ;;
|
|
r) RECORD=(--record) ;;
|
|
t) TOOLS="$OPTARG" ;;
|
|
h|*) sed -n '2,16p' "$0"; exit 0 ;;
|
|
esac
|
|
done
|
|
|
|
mkdir -p "$OUT"
|
|
if [[ -z "${CST_AGENT_BIN:-}" ]]; then
|
|
make -s -C "$ROOT" cst-agent >/dev/null || { echo "build failed" >&2; exit 65; }
|
|
fi
|
|
|
|
total=0
|
|
for pm in $MODELS; do
|
|
for proto in $PROTOCOLS; do
|
|
for fx in $FIXTURES; do
|
|
for ((i = 1; i <= N; i++)); do
|
|
total=$((total + 1))
|
|
provider="${pm%%:*}"
|
|
model="${pm#*:}"
|
|
margs=(--provider "$provider")
|
|
[[ -n "$model" ]] && margs+=(--model "$model")
|
|
printf '[%d] %-10s %-12s %-14s #%d ' "$total" "$provider" "$proto" "$fx" "$i"
|
|
"$ROOT/scripts/e2e_fixture.sh" "$fx" --protocol "$proto" --tools "$TOOLS" "${margs[@]}" \
|
|
--out "$OUT" "${RECORD[@]}" | sed 's/ -> .*//'
|
|
done
|
|
done
|
|
done
|
|
done
|
|
|
|
python3 "$ROOT/scripts/summarize.py" "$OUT"
|