#!/usr/bin/env bash
# Experiment matrix: protocol x model x fixture x N runs, then summarize.
#
# Usage:
#   scripts/run_matrix.sh [-p "native-eos native-inc ctl ctl-heredoc xml"]
#       [-m "deepseek:deepseek-flash openai:gpt-4.1-mini"]
#       [-f "mini_calc makefile_tab big_file test_fix_test"]
#       [-n 3] [-o out/matrix-<stamp>] [-r] [-t minimal|default]
#
#   -m  provider:model pairs (model may be empty: "deepseek:")
#   -r  also record raw SSE streams per run (for replay experiments)
#   -t  tool profile (default minimal)
#
# Each run is scripts/e2e_fixture.sh; results land in <out>/<run>/ and the
# summary in <out>/results.csv and <out>/summary.md.
set -uo pipefail

ROOT="$(cd "$(dirname "$0")/.." && pwd)"
PROTOCOLS="native-eos native-inc ctl ctl-heredoc xml"
MODELS="deepseek:deepseek-flash"
FIXTURES="mini_calc makefile_tab big_file test_fix_test"
N=3
OUT="$ROOT/out/matrix-$(date -u +%Y%m%dT%H%M%SZ)"
RECORD=()
TOOLS=minimal

while getopts "p:m:f:n:o:rt:h" opt; do
  case "$opt" in
    p) PROTOCOLS="$OPTARG" ;;
    m) MODELS="$OPTARG" ;;
    f) FIXTURES="$OPTARG" ;;
    n) N="$OPTARG" ;;
    o) OUT="$OPTARG" ;;
    r) RECORD=(--record) ;;
    t) TOOLS="$OPTARG" ;;
    h|*) sed -n '2,16p' "$0"; exit 0 ;;
  esac
done

mkdir -p "$OUT"
if [[ -z "${CST_AGENT_BIN:-}" ]]; then
  make -s -C "$ROOT" cst-agent >/dev/null || { echo "build failed" >&2; exit 65; }
fi

total=0
for pm in $MODELS; do
  for proto in $PROTOCOLS; do
    for fx in $FIXTURES; do
      for ((i = 1; i <= N; i++)); do
        total=$((total + 1))
        provider="${pm%%:*}"
        model="${pm#*:}"
        margs=(--provider "$provider")
        [[ -n "$model" ]] && margs+=(--model "$model")
        printf '[%d] %-10s %-12s %-14s #%d  ' "$total" "$provider" "$proto" "$fx" "$i"
        "$ROOT/scripts/e2e_fixture.sh" "$fx" --protocol "$proto" --tools "$TOOLS" "${margs[@]}" \
          --out "$OUT" "${RECORD[@]}" | sed 's/ -> .*//'
      done
    done
  done
done

python3 "$ROOT/scripts/summarize.py" "$OUT"