check_harness_runner_exit.sh (3898B)
1 #!/usr/bin/env sh 2 # Runner-exit control: prove that a failing test actually fails the Mojo test 3 # runner, and that infrastructure failures (compiler, loader, missing tool, 4 # pre-body crash) make THIS control fail instead of being mistaken for an 5 # expected failing test. 6 # 7 # For every canary: compile it first, then execute it, then require the exact 8 # expected test identity, failure/pass summary and zero skips. Any compile or 9 # loader error, any missing executable, or any output that does not name the 10 # expected executed test is an infrastructure failure. 11 # 12 # Usage: sh tools/check_harness_runner_exit.sh 13 # sh tools/check_harness_runner_exit.sh --self-test 14 set -eu 15 16 root="$(cd "$(dirname "$0")/.." && pwd)" 17 tmp="$(mktemp -d)" 18 trap 'rm -rf "$tmp"' EXIT 19 20 # run_one <canary.mojo> <expected_test_name> <fail|pass> 21 run_one() { 22 canary="$1" 23 expected_name="$2" 24 expect="$3" 25 bin="$tmp/$(basename "$canary" .mojo).bin" 26 27 if ! mojo build -I "$root/src" -I "$root/tests" "$canary" -o "$bin" \ 28 >"$tmp/build.log" 2>&1; then 29 echo "INFRA: compile failed: $canary" >&2 30 sed 's/^/ /' "$tmp/build.log" >&2 31 return 1 32 fi 33 if [ ! -x "$bin" ]; then 34 echo "INFRA: compile produced no executable: $canary" >&2 35 return 1 36 fi 37 38 if output="$("$bin" 2>&1)"; then 39 rc=0 40 else 41 rc=$? 42 fi 43 44 case "$expect" in 45 fail) 46 if [ "$rc" -eq 0 ]; then 47 echo "CONTROL: $canary did not fail the runner" >&2 48 return 1 49 fi 50 if ! printf '%s' "$output" | grep -q "FAIL .*$expected_name"; then 51 echo "CONTROL: $canary did not report executed test $expected_name" >&2 52 printf '%s\n' "$output" >&2 53 return 1 54 fi 55 if ! printf '%s' "$output" | grep -qE 'failed , 0 skipped'; then 56 echo "CONTROL: $canary produced no failure summary with zero skips" >&2 57 return 1 58 fi 59 ;; 60 pass) 61 if [ "$rc" -ne 0 ]; then 62 echo "CONTROL: passing canary $canary exited $rc" >&2 63 return 1 64 fi 65 if ! printf '%s' "$output" | grep -q '1 passed , 0 failed , 0 skipped'; then 66 echo "CONTROL: passing canary produced no clean pass summary" >&2 67 return 1 68 fi 69 ;; 70 esac 71 return 0 72 } 73 74 self_test() { 75 test_status=0 76 77 # A formatter/runner binary that fails compilation must be detected as 78 # infrastructure failure, never as an expected failing test. 79 bindir="$tmp/bin" 80 mkdir -p "$bindir" 81 printf '#!/bin/sh\necho "error: synthetic compiler failure" >&2\nexit 1\n' \ 82 > "$bindir/mojo" 83 chmod +x "$bindir/mojo" 84 if PATH="$bindir:$PATH" run_one \ 85 "$root/tests/harness_canary_assert_fail.mojo" \ 86 test_canary_assertion_fails fail; then 87 echo "self-test failed: synthetic compile failure accepted" >&2 88 test_status=1 89 fi 90 91 # A loader/pre-body crash must also be rejected. 92 crash="$tmp/prebody_crash.mojo" 93 printf 'def main() raises:\n raise Error("pre-body crash")\n' > "$crash" 94 if run_one "$crash" test_none fail; then 95 echo "self-test failed: pre-body crash accepted" >&2 96 test_status=1 97 fi 98 99 # A missing toolchain must fail the control, not pass it. 100 if PATH="/usr/bin:/bin" run_one \ 101 "$root/tests/harness_canary_assert_fail.mojo" \ 102 test_canary_assertion_fails fail; then 103 echo "self-test failed: missing tool accepted" >&2 104 test_status=1 105 fi 106 107 if [ "$test_status" -eq 0 ]; then 108 echo "harness runner-exit self-test: ok" 109 fi 110 return "$test_status" 111 } 112 113 if [ "${1:-}" = "--self-test" ]; then 114 self_test 115 exit $? 116 fi 117 118 status=0 119 run_one "$root/tests/harness_canary_assert_fail.mojo" \ 120 test_canary_assertion_fails fail || status=1 121 run_one "$root/tests/harness_canary_error_fail.mojo" \ 122 test_canary_generic_error_fails fail || status=1 123 run_one "$root/tests/harness_canary_pass.mojo" \ 124 test_canary_passes pass || status=1 125 126 if [ "$status" -eq 0 ]; then 127 echo "harness runner-exit canaries: ok" 128 fi 129 exit "$status"