43 lines
2.4 KiB
Bash
43 lines
2.4 KiB
Bash
#!/usr/bin/env bash
|
|
# Optional paid real-model evaluation. Pass normal provider/model CLI options.
|
|
# WORKER_EVAL_CONFIG may point to a workers.json to evaluate orchestration.
|
|
set -euo pipefail
|
|
eval_bin="${CST_AGENT_BIN:-$(pwd)/cst-agent}"
|
|
eval_root=$(mktemp -d -t cst-worker-eval.XXXXXX)
|
|
printf 'Results: %s\n' "$eval_root"
|
|
eval_failed=0
|
|
for eval_case in bounded_edit truncated_read; do
|
|
eval_ws="$eval_root/$eval_case"
|
|
mkdir -p "$eval_ws/.agent"
|
|
if [[ -n "${WORKER_EVAL_CONFIG:-}" ]]; then
|
|
cp "$WORKER_EVAL_CONFIG" "$eval_ws/.agent/workers.json"
|
|
fi
|
|
if [[ "$eval_case" == bounded_edit ]]; then
|
|
printf 'int add(int a,int b){return a-b;}\n' > "$eval_ws/calc.c"
|
|
printf 'int add(int,int);int main(void){return add(2,3)!=5;}\n' > "$eval_ws/check.c"
|
|
cp "$eval_ws/check.c" "$eval_ws/.agent/check.original"
|
|
eval_prompt='Fix add in calc.c to add its arguments. Only change calc.c. Verify by compiling calc.c and check.c with cc and executing the result. Report the exact checks and result. If the compiler fails, inspect the error and adjust the invocation. Delegate the implementation when orchestration is enabled.'
|
|
else
|
|
awk 'BEGIN { for(i=1;i<=9000;i++) print "line " i; print "FINAL_MARKER=worker-cache-check-7391" }' > "$eval_ws/large.txt"
|
|
eval_prompt='Read large.txt and report its exact FINAL_MARKER value. Use tools, handle truncated output by reading an appropriate later range, and do not modify any files. Delegate research when orchestration is enabled.'
|
|
fi
|
|
eval_start=$(date +%s)
|
|
if "$eval_bin" "$@" --headless --new --workspace "$eval_ws" --prompt "$eval_prompt" \
|
|
> "$eval_ws/.agent/answer.txt" 2> "$eval_ws/.agent/stderr.txt"; then
|
|
if [[ "$eval_case" == bounded_edit ]]; then
|
|
if cmp -s "$eval_ws/check.c" "$eval_ws/.agent/check.original" &&
|
|
cc "$eval_ws/calc.c" "$eval_ws/check.c" -o "$eval_ws/.agent/check" &&
|
|
"$eval_ws/.agent/check"; then eval_result=PASS; else eval_result=FAIL; fi
|
|
else
|
|
if rg -q 'worker-cache-check-7391' "$eval_ws/.agent/answer.txt"; then
|
|
eval_result=PASS
|
|
else eval_result=FAIL; fi
|
|
fi
|
|
else eval_result=FAIL; fi
|
|
[[ "$eval_result" == PASS ]] || eval_failed=1
|
|
printf '%s %s %ss\n' "$eval_case" "$eval_result" "$(( $(date +%s) - eval_start ))" |
|
|
tee -a "$eval_root/results.txt"
|
|
done
|
|
printf 'Compare results, transcripts and *.runs.jsonl across fixed model versions/configurations.\n'
|
|
exit "$eval_failed"
|