#!/usr/bin/env bash # autoplay-batch.sh — Run auto_play N times with different seeds and collect per-game output dirs. # # Usage: tools/autoplay-batch.sh [--weston] [count=3] [turn_limit=500] [results_dir] # # Output layout: # /game__seed/ # meta.json # turn_stats.jsonl # events.jsonl # game.log # weston.log (weston mode only) # *.save (per-turn saves, if configured) # # Environment: # AUTOPLAY_HOST — If set (e.g. "lilith@apricot.local"), run each game via SSH # using run_ap3.sh on the remote host and scp results back. # If unset, run locally via flatpak (Linux only). # RENDER_MODE — "headless" (default) or "weston". --weston flag sets this. # headless: Godot --headless, no display, no screenshots. # weston: weston headless backend, software rendering, screenshots work. # PARALLEL — Max seeds to run concurrently (default 1 = serial). # Remote runner is concurrency-safe via scoped pkill per AUTO_PLAY_DIR. # Apricot has 64 cores → PARALLEL=10 is a safe, ~10× wall-clock speedup. # SEED_OFFSET — Shift the seed range from [1..COUNT] to [1+OFFSET..COUNT+OFFSET] # (default 0). Use for multi-sweep runs that share a parent # results_dir and need disjoint seed numbers (e.g. Task #10 # AI_PIN_PERSONALITY rotation: 5× batches at offsets 0,10,20,30,40). # REMOTE_BATCH_ROOT — Where remote autoplay batches write their output on the # AUTOPLAY_HOST. Defaults to /tmp/@magic-civilization/builds # so artifacts land OUTSIDE the remote host's dev working tree # (writing into the working tree causes auto-commit divergence # between hosts). Override to $REMOTE_HOME/... if the remote # host is a pure build node without active development. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_DIR="$(dirname "$SCRIPT_DIR")" GAME_DIR="$PROJECT_DIR/src/game" REPO_ROOT="$PROJECT_DIR" RENDER_MODE="${RENDER_MODE:-headless}" # Parse --weston flag before positional args POSITIONAL=() for arg in "$@"; do case "$arg" in --weston) RENDER_MODE="weston" ;; --headless) RENDER_MODE="headless" ;; *) POSITIONAL+=("$arg") ;; esac done set -- "${POSITIONAL[@]+"${POSITIONAL[@]}"}" COUNT="${1:-3}" TURN_LIMIT="${2:-500}" RESULTS_DIR="${3:-$REPO_ROOT/.local/batches/autoplay_batch}" if ! [[ "$COUNT" =~ ^[0-9]+$ ]] || [ "$COUNT" -lt 1 ]; then echo "ERROR: count must be a positive integer (got '$COUNT')" >&2 exit 2 fi if ! [[ "$TURN_LIMIT" =~ ^[0-9]+$ ]] || [ "$TURN_LIMIT" -lt 1 ]; then echo "ERROR: turn_limit must be a positive integer (got '$TURN_LIMIT')" >&2 exit 2 fi AUTOPLAY_HOST="${AUTOPLAY_HOST:-}" REMOTE_BATCH_ROOT="${REMOTE_BATCH_ROOT:-/tmp/@magic-civilization/builds}" SAFETY_TIMEOUT=$(( TURN_LIMIT * 2 + 300 )) PARALLEL="${PARALLEL:-1}" # SEED_OFFSET shifts the seed range from [1..COUNT] to [1+OFFSET..COUNT+OFFSET]. # Used by multi-sweep aggregation (e.g. Task #10 B5 AI_PIN_PERSONALITY rotation) # to produce disjoint seed numbers across sweeps that share a parent results dir. SEED_OFFSET="${SEED_OFFSET:-0}" if ! [[ "$PARALLEL" =~ ^[0-9]+$ ]] || [ "$PARALLEL" -lt 1 ]; then echo "ERROR: PARALLEL must be a positive integer (got '$PARALLEL')" >&2 exit 2 fi if ! [[ "$SEED_OFFSET" =~ ^[0-9]+$ ]]; then echo "ERROR: SEED_OFFSET must be a non-negative integer (got '$SEED_OFFSET')" >&2 exit 2 fi # Flatpak's sandboxed Godot resolves AUTO_PLAY_DIR against an unspecified CWD, # not the caller's shell CWD — a relative path silently produces 0-byte # meta.json / turn_stats.jsonl even when the game itself completes (game.log # is fine because it's redirected host-side). realpath -m tolerates the path # not existing yet; it will be mkdir'd just below. Also ensures the /tmp # reject check that follows catches all forms (./tmp, ../tmp, etc). RESULTS_DIR="$(realpath -m "$RESULTS_DIR")" # Flatpak sandbox can't write to /tmp. Reject /tmp paths outright instead of # silently redirecting — persistent output belongs under the repo. if [[ "$RESULTS_DIR" == /tmp/* ]] || [[ "$RESULTS_DIR" == /private/tmp/* ]]; then echo "ERROR: results_dir under /tmp is forbidden (wiped on reboot, flatpak sandbox hostile)." >&2 echo " Use a path under the repo (default: /.local/batches/) or \$HOME/tmp." >&2 exit 2 fi mkdir -p "$RESULTS_DIR" STAMP="$(date +%Y%m%d_%H%M%S)" SEED_START=$(( SEED_OFFSET + 1 )) SEED_END=$(( SEED_OFFSET + COUNT )) echo "============================================================" echo "Autoplay Batch: $COUNT games (seeds $SEED_START..$SEED_END), turn_limit=$TURN_LIMIT" echo "Results: $RESULTS_DIR" echo "Stamp: $STAMP" if [ -n "$AUTOPLAY_HOST" ]; then echo "Mode: remote SSH ($AUTOPLAY_HOST)" else echo "Mode: local flatpak" fi echo "Render: $RENDER_MODE" echo "Parallel: $PARALLEL concurrent seed(s)" echo "Safety timeout: ${SAFETY_TIMEOUT}s per game" echo "============================================================" # Resolve REMOTE_HOME once upfront (parallel workers all need it and racing for it breaks) if [ -n "$AUTOPLAY_HOST" ]; then REMOTE_HOME="$(ssh "$AUTOPLAY_HOST" 'echo "$HOME"')" export REMOTE_HOME fi _kill_stale_procs() { pkill -f "weston.*godot-headless" 2>/dev/null || true pkill -f "org.godotengine.Godot" 2>/dev/null || true sleep 0.5 } _run_local() { local seed="$1" local game_dir="$2" if ! command -v flatpak >/dev/null 2>&1; then echo "ERROR: flatpak not installed. Set AUTOPLAY_HOST to run on a remote Linux host." >&2 exit 1 fi # Skip unscoped pkill in parallel mode — would murder sibling workers. # Parallel local runs assume no stray Godot is already running. if [ "$PARALLEL" -le 1 ]; then _kill_stale_procs fi local WESTON_PID="" local FLATPAK_ENVS=( "--env=AUTO_PLAY=true" "--env=AUTO_PLAY_SEED=$seed" "--env=AUTO_PLAY_TURN_LIMIT=$TURN_LIMIT" "--env=AUTO_PLAY_DIR=$game_dir" "--env=AP_RUN_ID=${STAMP}_seed$(printf '%03d' "$seed")" "--env=AI_DIFFICULTY=${AI_DIFFICULTY:-}" "--env=AI_PIN_PERSONALITY=${AI_PIN_PERSONALITY:-}" "--env=MAP_SIZE=${MAP_SIZE:-}" "--env=NUM_PLAYERS=${NUM_PLAYERS:-}" ) local GODOT_ARGS=("--path" "$GAME_DIR" "--rendering-method" "gl_compatibility") if [ "$RENDER_MODE" = "weston" ]; then if ! command -v weston >/dev/null 2>&1; then echo "ERROR: --weston mode but weston not installed" >&2 exit 1 fi WESTON_SOCKET="godot-headless-$$" echo "[seed $seed] Starting weston (headless)..." weston --backend=headless --socket="$WESTON_SOCKET" --width=1920 --height=1080 \ >"$game_dir/weston.log" 2>&1 & WESTON_PID=$! sleep 1 FLATPAK_ENVS+=( "--socket=wayland" "--env=WAYLAND_DISPLAY=$WESTON_SOCKET" "--filesystem=xdg-run/${WESTON_SOCKET}" ) else GODOT_ARGS+=("--headless") fi echo "[seed $seed] Launching Godot ($RENDER_MODE, timeout ${SAFETY_TIMEOUT}s)..." XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" \ timeout "$SAFETY_TIMEOUT" flatpak run --user \ --filesystem=home \ "${FLATPAK_ENVS[@]}" \ org.godotengine.Godot "${GODOT_ARGS[@]}" \ >"$game_dir/game.log" 2>&1 || { local exit_code=$? echo "[seed $seed] Godot exited with code $exit_code" >&2 } if [ -n "$WESTON_PID" ]; then kill "$WESTON_PID" 2>/dev/null || true wait "$WESTON_PID" 2>/dev/null || true fi } _run_remote() { local seed="$1" local game_dir="$2" echo "[seed $seed] Running via SSH on $AUTOPLAY_HOST..." # REMOTE_BATCH_ROOT (default /tmp/@magic-civilization/builds) keeps batch # artifacts OUTSIDE the remote host's dev working tree so they cannot # trigger auto-commit divergence between hosts. # Derive a unique remote dir from RESULTS_DIR's basename to avoid per-clan # path collisions when multiple batches run in parallel with the same STAMP. local results_basename results_basename="$(basename "$RESULTS_DIR")" local remote_game_dir="$REMOTE_BATCH_ROOT/.local/batches/${results_basename}/game_${STAMP}_seed${seed}" local remote_runner="$REMOTE_HOME/bin/run_ap3.sh" ssh "$AUTOPLAY_HOST" " set -euo pipefail mkdir -p '$remote_game_dir' if [ ! -f '$remote_runner' ]; then echo 'ERROR: $remote_runner not found on $AUTOPLAY_HOST (expected persistent runner in \$HOME/bin)' >&2 exit 1 fi AUTO_PLAY=true \ AUTO_PLAY_SEED='$seed' \ AUTO_PLAY_TURN_LIMIT='$TURN_LIMIT' \ AUTO_PLAY_DIR='$remote_game_dir' \ AP_RUN_ID="${STAMP}_seed$(printf '%03d' "$seed")" \ AI_DIFFICULTY='${AI_DIFFICULTY:-}' \ AI_PIN_PERSONALITY='${AI_PIN_PERSONALITY:-}' \ MAP_SIZE='${MAP_SIZE:-}' \ NUM_PLAYERS='${NUM_PLAYERS:-}' \ RENDER_MODE='$RENDER_MODE' \ bash '$remote_runner' >'$remote_game_dir/game.log' 2>&1 " || { echo "[seed $seed] SSH run exited with error — see $game_dir/game.log after scp" >&2 } echo "[seed $seed] Fetching results from $AUTOPLAY_HOST..." scp -r "$AUTOPLAY_HOST:$remote_game_dir/." "$game_dir/" \ >/dev/null 2>&1 || { echo "WARNING: scp failed for seed $seed — result may be missing" >&2 } } # ── Main loop ──────────────────────────────────────────────────────────────── # # _run_one dispatches one seed (remote or local) and writes a status line to # $STATUS_DIR/seed_.status. Parallel mode runs up to $PARALLEL workers # concurrently using bash job control; the status files are read after # `wait` to tally failures (avoids races on a shared FAILED_SEEDS array). STATUS_DIR="$(mktemp -d -t autoplay-batch-status.XXXXXX)" trap 'rm -rf "$STATUS_DIR"' EXIT _run_one() { local seed="$1" local game_dir="$RESULTS_DIR/game_${STAMP}_seed${seed}" mkdir -p "$game_dir" echo "[$(date +%H:%M:%S)] [seed $seed] start → $game_dir" if [ -n "$AUTOPLAY_HOST" ]; then _run_remote "$seed" "$game_dir" else _run_local "$seed" "$game_dir" fi local meta_ok=false stats_ok=false [ -f "$game_dir/meta.json" ] && meta_ok=true [ -f "$game_dir/turn_stats.jsonl" ] && [ -s "$game_dir/turn_stats.jsonl" ] && stats_ok=true if $meta_ok && $stats_ok; then local line_count line_count="$(wc -l < "$game_dir/turn_stats.jsonl" | tr -d ' ')" echo "[$(date +%H:%M:%S)] [seed $seed] OK — $line_count turn_stats line(s)" echo "OK $seed" > "$STATUS_DIR/seed_${seed}.status" else $meta_ok || echo "[seed $seed] MISSING meta.json" >&2 $stats_ok || echo "[seed $seed] MISSING or empty turn_stats.jsonl" >&2 echo "FAIL $seed" > "$STATUS_DIR/seed_${seed}.status" fi } if [ "$PARALLEL" -le 1 ]; then for seed in $(seq "$SEED_START" "$SEED_END"); do _run_one "$seed" done else echo "[$(date +%H:%M:%S)] Dispatching $COUNT seed(s) with up to $PARALLEL concurrent..." for seed in $(seq "$SEED_START" "$SEED_END"); do while [ "$(jobs -rp | wc -l | tr -d ' ')" -ge "$PARALLEL" ]; do wait -n 2>/dev/null || break done _run_one "$seed" & done wait fi FAILED_SEEDS=() for seed in $(seq "$SEED_START" "$SEED_END"); do status_file="$STATUS_DIR/seed_${seed}.status" if [ ! -f "$status_file" ]; then echo "[seed $seed] MISSING status file (worker crashed before writing)" >&2 FAILED_SEEDS+=("$seed") continue fi read -r status _ < "$status_file" [ "$status" = "OK" ] || FAILED_SEEDS+=("$seed") done # ── Summary ────────────────────────────────────────────────────────────────── echo "" echo "============================================================" PRODUCED=$(( COUNT - ${#FAILED_SEEDS[@]} )) echo "Batch complete: $PRODUCED/$COUNT games produced turn_stats.jsonl" echo "Results: $RESULTS_DIR" echo "============================================================" if [ ${#FAILED_SEEDS[@]} -gt 0 ]; then echo "ERROR: No turn_stats.jsonl for seeds: ${FAILED_SEEDS[*]}" >&2 echo " Check game.log in each game dir for details." >&2 exit 1 fi # ── E2E determinism gate ────────────────────────────────────────────────────── # Runs e2e-determinism-check.sh on the results dir. Catches script errors that # don't stop the game (map_placer out-of-bounds, nil-access warnings that # accumulate silently) and fails the batch if any seed has non-allowlisted ERRORs. E2E_CHECK="$SCRIPT_DIR/e2e-determinism-check.sh" if [ -x "$E2E_CHECK" ]; then echo "" echo "Running E2E determinism gate..." if ! "$E2E_CHECK" "$RESULTS_DIR" "$COUNT"; then echo "ERROR: E2E gate failed — see above for details." >&2 exit 1 fi else echo "WARNING: $E2E_CHECK not found or not executable — skipping E2E gate" >&2 fi exit 0