#!/usr/bin/env bash # e2e-determinism-check.sh — Post-flight gate for autoplay batch results. # # Usage: tools/e2e-determinism-check.sh [count] # # Asserts for each seed directory under : # 1. turn_stats.jsonl exists and has ≥1 line # 2. game.log contains no script-level ERRORs (allowlist excludes headless # texture errors and TileMap out-of-bounds from the rendering dummy) # # Exit codes: # 0 — all seeds passed # 1 — one or more seeds failed the gate # 2 — bad arguments / results_dir not found set -euo pipefail RESULTS_DIR="${1:-}" COUNT="${2:-}" if [ -z "$RESULTS_DIR" ] || [ ! -d "$RESULTS_DIR" ]; then echo "Usage: $0 [count]" >&2 echo "ERROR: results_dir not found: '$RESULTS_DIR'" >&2 exit 2 fi # Script errors that are known-benign in headless mode and do NOT indicate # a game logic failure. These patterns are matched against ERROR lines. ALLOWLISTED_ERRORS=( "Parameter \"t\" is null" # headless viewport texture — no display "texture_2d_get" # same; appears in dummy storage callsite "resources still in use at exit" # Godot engine shutdown, not game logic "X11 Display is not available" # weston mode: Godot tries X11, falls back to wayland "Formatting error in string" # freedesktop portal printf format bug in Godot C++ ) _is_allowlisted() { local line="$1" # Blank ERROR lines (just "ERROR: " with nothing after) are benign portal noise [[ "$line" == "ERROR: " || "$line" == "ERROR:" ]] && return 0 for pattern in "${ALLOWLISTED_ERRORS[@]}"; do if [[ "$line" == *"$pattern"* ]]; then return 0 fi done return 1 } # Collect seed dirs (either named game_*_seed* or any subdir with game.log) mapfile -t SEED_DIRS < <(find "$RESULTS_DIR" -maxdepth 1 -mindepth 1 -type d | sort) if [ "${#SEED_DIRS[@]}" -eq 0 ]; then echo "ERROR: no seed directories found under $RESULTS_DIR" >&2 exit 2 fi if [ -n "$COUNT" ]; then if [ "${#SEED_DIRS[@]}" -lt "$COUNT" ]; then echo "ERROR: expected $COUNT seed dirs, found ${#SEED_DIRS[@]}" >&2 exit 1 fi fi FAILURES=() for seed_dir in "${SEED_DIRS[@]}"; do name="$(basename "$seed_dir")" fail_reasons=() # Gate 1: turn_stats.jsonl must exist and have ≥1 line if [ ! -f "$seed_dir/turn_stats.jsonl" ]; then fail_reasons+=("missing turn_stats.jsonl") elif [ ! -s "$seed_dir/turn_stats.jsonl" ]; then fail_reasons+=("empty turn_stats.jsonl") fi # Gate 2: game.log must exist if [ ! -f "$seed_dir/game.log" ]; then fail_reasons+=("missing game.log") else # Scan for non-allowlisted ERRORs script_errors=() while IFS= read -r line; do if [[ "$line" =~ ^ERROR: ]]; then if ! _is_allowlisted "$line"; then script_errors+=("$line") fi fi done < "$seed_dir/game.log" if [ "${#script_errors[@]}" -gt 0 ]; then fail_reasons+=("${#script_errors[@]} non-allowlisted script error(s) — first: ${script_errors[0]}") fi fi if [ "${#fail_reasons[@]}" -gt 0 ]; then echo "FAIL [$name]: ${fail_reasons[*]}" >&2 FAILURES+=("$name") else ts_lines="$(wc -l < "$seed_dir/turn_stats.jsonl" | tr -d ' ')" echo "OK [$name]: $ts_lines turn_stats line(s)" fi done echo "" echo "============================================================" PASS=$(( ${#SEED_DIRS[@]} - ${#FAILURES[@]} )) echo "E2E gate: $PASS/${#SEED_DIRS[@]} passed" echo "============================================================" if [ "${#FAILURES[@]}" -gt 0 ]; then echo "FAILED seeds: ${FAILURES[*]}" >&2 exit 1 fi exit 0