CST-agent/scripts/e2e_fixture.sh

#!/usr/bin/env bash
# One end-to-end run: copy fixtures/<name> to a temp workspace, run cst-agent
# headless on its TASK.md, then judge with the fixture's check.sh.
#
# Usage:
#   scripts/e2e_fixture.sh <fixture> [--protocol P] [--provider NAME]
#       [--model M] [--max-steps N] [--tools minimal|default] [--out DIR]
#       [--record] [--baseline-only]
#
# --tools defaults to minimal (the four tools the paper experiments used).
#
# Artifacts go to <out>/<fixture>__<protocol>__<model>__<stamp>/:
#   result.json  agent.log  session.jsonl  metrics.jsonl  check.log  [sse/]
#
# Environment:
#   CST_AGENT_BIN   agent binary (default: <repo>/cst-agent, built if missing)
#   API key         CST_API_KEY, or the provider variable (DEEPSEEK_API_KEY,
#                   OPENAI_API_KEY, MINIMAX_API_KEY), or the credentials file
#   E2E_TIMEOUT     seconds before the agent is killed (default 900; needs
#                   `timeout` or `gtimeout` from coreutils)
set -uo pipefail

ROOT="$(cd "$(dirname "$0")/.." && pwd)"
FIXTURE=""
PROTOCOL="${CST_PROTOCOL:-native-eos}"
PROVIDER="${CST_PROVIDER:-}"
MODEL="${CST_MODEL:-}"
MAX_STEPS=""
TOOLS="${CST_E2E_TOOLS:-minimal}"
OUT="$ROOT/out/e2e"
RECORD=0
BASELINE_ONLY=0

while [[ $# -gt 0 ]]; do
  case "$1" in
    --protocol) PROTOCOL="$2"; shift 2 ;;
    --provider) PROVIDER="$2"; shift 2 ;;
    --model) MODEL="$2"; shift 2 ;;
    --max-steps) MAX_STEPS="$2"; shift 2 ;;
    --tools) TOOLS="$2"; shift 2 ;;
    --out) OUT="$2"; shift 2 ;;
    --record) RECORD=1; shift ;;
    --baseline-only) BASELINE_ONLY=1; shift ;;
    -h|--help) sed -n '2,21p' "$0"; exit 0 ;;
    -*) echo "unknown option: $1" >&2; exit 64 ;;
    *) FIXTURE="$1"; shift ;;
  esac
done

FIX="$ROOT/fixtures/$FIXTURE"
if [[ -z "$FIXTURE" || ! -f "$FIX/TASK.md" || ! -x "$FIX/check.sh" ]]; then
  echo "usage: $0 <fixture> [options]  (fixture needs TASK.md + check.sh)" >&2
  exit 64
fi

BIN="${CST_AGENT_BIN:-$ROOT/cst-agent}"
if [[ ! -x "$BIN" ]]; then
  make -s -C "$ROOT" cst-agent >/dev/null || { echo "build failed" >&2; exit 65; }
fi

STAMP="$(date -u +%Y%m%dT%H%M%SZ)-$$"
MODEL_TAG="${MODEL:-default}"
RUN="$OUT/${FIXTURE}__${PROTOCOL}__${MODEL_TAG//\//_}__${STAMP}"
mkdir -p "$RUN"
WORK="$(mktemp -d /tmp/cst-e2e.XXXXXX)"
SCRATCH="$(mktemp -d /tmp/cst-e2e-base.XXXXXX)"
trap 'rm -rf "$WORK" "$SCRATCH"' EXIT

copy_fixture() {
  cp -a "$FIX/." "$1/"
  rm -rf "$1/out" "$1/.agent" "$1/.baseline_green_ok"
}

json_str() { python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$1"; }

write_result() {
  local outcome="$1" agent_rc="$2" check_rc="$3" wall="$4"
  cat >"$RUN/result.json" <<EOF
{"fixture": $(json_str "$FIXTURE"), "protocol": $(json_str "$PROTOCOL"),
 "provider": $(json_str "${PROVIDER:-default}"), "model": $(json_str "$MODEL_TAG"),
 "tools": $(json_str "$TOOLS"),
 "run": $(json_str "$(basename "$RUN")"), "baseline_rc": $BASE_RC,
 "agent_rc": $agent_rc, "check_rc": $check_rc, "wall_s": $wall,
 "outcome": $(json_str "$outcome")}
EOF
  echo "outcome=$outcome agent_rc=$agent_rc check_rc=$check_rc wall_s=$wall -> $RUN"
}

# Baseline in a throwaway copy: the task must not already be done.
copy_fixture "$SCRATCH"
"$SCRATCH/check.sh" >"$RUN/baseline.log" 2>&1
BASE_RC=$?
if [[ "$BASE_RC" -eq 0 && ! -e "$FIX/.baseline_green_ok" ]]; then
  write_result unexpected_green_fixture 0 0 0
  exit 2
fi
if [[ "$BASELINE_ONLY" -eq 1 ]]; then
  write_result baseline_only 0 "$BASE_RC" 0
  exit 0
fi

copy_fixture "$WORK"
args=(--headless --workspace "$WORK" --session "$WORK/.agent/e2e.jsonl"
      --protocol "$PROTOCOL" --tools "$TOOLS")
[[ -n "$PROVIDER" ]] && args+=(--provider "$PROVIDER")
[[ -n "$MODEL" ]] && args+=(--model "$MODEL")
[[ -n "$MAX_STEPS" ]] && args+=(--max-steps "$MAX_STEPS")
mkdir -p "$WORK/.agent"

export CST_AUTO_APPROVE=1
export CST_METRICS_PATH="$RUN/metrics.jsonl"
if [[ "$RECORD" -eq 1 ]]; then
  export CST_RECORD_SSE="$RUN/sse"
fi

now() { python3 -c 'import time; print(time.time())'; }
with_timeout() {
  if command -v timeout >/dev/null 2>&1; then
    timeout "${E2E_TIMEOUT:-900}" "$@"
  elif command -v gtimeout >/dev/null 2>&1; then
    gtimeout "${E2E_TIMEOUT:-900}" "$@"
  else
    "$@"
  fi
}

t0=$(now)
with_timeout "$BIN" "${args[@]}" --prompt "$(cat "$FIX/TASK.md")" \
  >"$RUN/agent.log" 2>&1
AGENT_RC=$?
t1=$(now)
WALL=$(python3 -c "print(round($t1-$t0,2))")
cp -f "$WORK/.agent/e2e.jsonl" "$RUN/session.jsonl" 2>/dev/null || true

(cd "$WORK" && ./check.sh) >"$RUN/check.log" 2>&1
CHECK_RC=$?

if [[ "$CHECK_RC" -eq 0 ]]; then
  OUTCOME=pass
elif [[ "$AGENT_RC" -eq 124 ]]; then
  OUTCOME=timeout
elif [[ "$AGENT_RC" -ne 0 ]]; then
  OUTCOME=agent_error
else
  OUTCOME=fail
fi
write_result "$OUTCOME" "$AGENT_RC" "$CHECK_RC" "$WALL"
[[ "$OUTCOME" == pass ]]