116 lines
3.7 KiB
Bash
Executable file
116 lines
3.7 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# e2e-determinism-check.sh — Post-flight gate for autoplay batch results.
|
|
#
|
|
# Usage: tools/e2e-determinism-check.sh <results_dir> [count]
|
|
#
|
|
# Asserts for each seed directory under <results_dir>:
|
|
# 1. turn_stats.jsonl exists and has ≥1 line
|
|
# 2. game.log contains no script-level ERRORs (allowlist excludes headless
|
|
# texture errors and TileMap out-of-bounds from the rendering dummy)
|
|
#
|
|
# Exit codes:
|
|
# 0 — all seeds passed
|
|
# 1 — one or more seeds failed the gate
|
|
# 2 — bad arguments / results_dir not found
|
|
|
|
set -euo pipefail
|
|
|
|
RESULTS_DIR="${1:-}"
|
|
COUNT="${2:-}"
|
|
|
|
if [ -z "$RESULTS_DIR" ] || [ ! -d "$RESULTS_DIR" ]; then
|
|
echo "Usage: $0 <results_dir> [count]" >&2
|
|
echo "ERROR: results_dir not found: '$RESULTS_DIR'" >&2
|
|
exit 2
|
|
fi
|
|
|
|
# Script errors that are known-benign in headless mode and do NOT indicate
|
|
# a game logic failure. These patterns are matched against ERROR lines.
|
|
ALLOWLISTED_ERRORS=(
|
|
"Parameter \"t\" is null" # headless viewport texture — no display
|
|
"texture_2d_get" # same; appears in dummy storage callsite
|
|
"resources still in use at exit" # Godot engine shutdown, not game logic
|
|
"X11 Display is not available" # weston mode: Godot tries X11, falls back to wayland
|
|
"Formatting error in string" # freedesktop portal printf format bug in Godot C++
|
|
)
|
|
|
|
_is_allowlisted() {
|
|
local line="$1"
|
|
# Blank ERROR lines (just "ERROR: " with nothing after) are benign portal noise
|
|
[[ "$line" == "ERROR: " || "$line" == "ERROR:" ]] && return 0
|
|
for pattern in "${ALLOWLISTED_ERRORS[@]}"; do
|
|
if [[ "$line" == *"$pattern"* ]]; then
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Collect seed dirs (either named game_*_seed* or any subdir with game.log)
|
|
mapfile -t SEED_DIRS < <(find "$RESULTS_DIR" -maxdepth 1 -mindepth 1 -type d | sort)
|
|
|
|
if [ "${#SEED_DIRS[@]}" -eq 0 ]; then
|
|
echo "ERROR: no seed directories found under $RESULTS_DIR" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if [ -n "$COUNT" ]; then
|
|
if [ "${#SEED_DIRS[@]}" -lt "$COUNT" ]; then
|
|
echo "ERROR: expected $COUNT seed dirs, found ${#SEED_DIRS[@]}" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
FAILURES=()
|
|
|
|
for seed_dir in "${SEED_DIRS[@]}"; do
|
|
name="$(basename "$seed_dir")"
|
|
fail_reasons=()
|
|
|
|
# Gate 1: turn_stats.jsonl must exist and have ≥1 line
|
|
if [ ! -f "$seed_dir/turn_stats.jsonl" ]; then
|
|
fail_reasons+=("missing turn_stats.jsonl")
|
|
elif [ ! -s "$seed_dir/turn_stats.jsonl" ]; then
|
|
fail_reasons+=("empty turn_stats.jsonl")
|
|
fi
|
|
|
|
# Gate 2: game.log must exist
|
|
if [ ! -f "$seed_dir/game.log" ]; then
|
|
fail_reasons+=("missing game.log")
|
|
else
|
|
# Scan for non-allowlisted ERRORs
|
|
script_errors=()
|
|
while IFS= read -r line; do
|
|
if [[ "$line" =~ ^ERROR: ]]; then
|
|
if ! _is_allowlisted "$line"; then
|
|
script_errors+=("$line")
|
|
fi
|
|
fi
|
|
done < "$seed_dir/game.log"
|
|
|
|
if [ "${#script_errors[@]}" -gt 0 ]; then
|
|
fail_reasons+=("${#script_errors[@]} non-allowlisted script error(s) — first: ${script_errors[0]}")
|
|
fi
|
|
fi
|
|
|
|
if [ "${#fail_reasons[@]}" -gt 0 ]; then
|
|
echo "FAIL [$name]: ${fail_reasons[*]}" >&2
|
|
FAILURES+=("$name")
|
|
else
|
|
ts_lines="$(wc -l < "$seed_dir/turn_stats.jsonl" | tr -d ' ')"
|
|
echo "OK [$name]: $ts_lines turn_stats line(s)"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "============================================================"
|
|
PASS=$(( ${#SEED_DIRS[@]} - ${#FAILURES[@]} ))
|
|
echo "E2E gate: $PASS/${#SEED_DIRS[@]} passed"
|
|
echo "============================================================"
|
|
|
|
if [ "${#FAILURES[@]}" -gt 0 ]; then
|
|
echo "FAILED seeds: ${FAILURES[*]}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
exit 0
|