commit adb7938c48140089b0541bd5cf44d31ae3c542e2
parent d7dac16d808ca7b156a325ce157911361ef5ed19
Author: triesap <tyson@radroots.org>
Date: Mon, 21 Sep 2026 21:22:09 +0000
test: prove verification controls (C059)
Diffstat:
5 files changed, 204 insertions(+), 58 deletions(-)
diff --git a/pixi.toml b/pixi.toml
@@ -22,6 +22,7 @@ run = "mojo run -I src src/main.mojo"
test-unit = "mojo -I src tests/test_hyf.mojo"
test-harness = "mojo -I src -I tests tests/test_harness_self_test.mojo"
test-harness-runner-exit = "sh tools/check_harness_runner_exit.sh"
+test-harness-runner-exit-self-test = "sh tools/check_harness_runner_exit.sh --self-test"
test-runtime = "mojo -I src tests/test_runtime_paths.mojo"
test-repo-local-process = "mojo -I src tests/test_repo_local_process_contract.mojo"
test-stdio = "mojo -I src tests/test_stdio_contract.mojo"
diff --git a/tests/safe_tempdir.mojo b/tests/safe_tempdir.mojo
@@ -1,14 +1,15 @@
"""Non-swallowing temporary-directory context manager for tests.
-The Mojo 1.0.0b1 `std.tempfile.TemporaryDirectory.__exit__` suppresses an
-exception raised inside its `with` body, which silently masks failing
-assertions. This wrapper delegates enter/exit to the standard implementation
-but discards the suppression return value so body exceptions propagate
-normally.
-
-Use `SafeTempDir` anywhere a test currently uses `TemporaryDirectory` as a
-`with` context manager. The interface is identical: `__enter__` returns the
-temporary directory path as a `String`.
+On this toolchain, using the standard library's temporary-directory helper as a
+`with` context manager suppresses an exception raised inside the body, silently
+masking failing assertions. This wrapper delegates enter/exit to the standard
+implementation but does not reproduce that suppression, so body exceptions
+propagate normally.
+
+This describes observed behavior only; the standard library's exact internal
+mechanism is not asserted here. Use `SafeTempDir` anywhere a test currently
+uses the standard helper as a `with` context manager. The interface is
+identical: `__enter__` returns the temporary directory path as a `String`.
"""
from std.tempfile import TemporaryDirectory as _StdTemporaryDirectory
@@ -24,7 +25,7 @@ struct SafeTempDir:
return self._inner.__enter__()
def __exit__(mut self) raises:
- # Discard the inner return value deliberately: the standard
- # implementation returns a suppression flag that would otherwise swallow
- # an exception raised in the `with` body.
+ # Delegate cleanup. The observed standard behavior suppresses a body
+ # exception at this point; this wrapper's non-suppressing exit lets the
+ # body exception propagate instead.
self._inner.__exit__()
diff --git a/tests/test_harness_self_test.mojo b/tests/test_harness_self_test.mojo
@@ -1,29 +1,38 @@
from std.os.path import exists
-from std.testing import TestSuite, assert_true
+from std.testing import TestSuite, assert_equal, assert_true
from safe_tempdir import SafeTempDir
def test_safe_tempdir_propagates_assertion_failure() raises:
+ var entered_body = False
var caught = False
try:
with SafeTempDir() as temp_dir:
_ = temp_dir
+ entered_body = True
assert_true(False)
except:
caught = True
+ assert_true(entered_body)
assert_true(caught)
-def test_safe_tempdir_propagates_generic_error() raises:
+def test_safe_tempdir_propagates_generic_error_with_exact_identity() raises:
+ var entered_body = False
var caught = False
+ var message = String("")
try:
with SafeTempDir() as temp_dir:
_ = temp_dir
+ entered_body = True
raise Error("intentional generic harness error")
- except:
+ except e:
caught = True
+ message = String(e)
+ assert_true(entered_body)
assert_true(caught)
+ assert_equal(message, "intentional generic harness error")
def test_safe_tempdir_nested_contexts_are_distinct_and_cleaned() raises:
@@ -40,15 +49,36 @@ def test_safe_tempdir_nested_contexts_are_distinct_and_cleaned() raises:
assert_true(not exists(inner_path))
+def test_safe_tempdir_nested_error_propagates_and_cleans_up() raises:
+ var outer_path = String("")
+ var inner_path = String("")
+ var caught = False
+ try:
+ with SafeTempDir() as outer:
+ outer_path = outer
+ with SafeTempDir() as inner:
+ inner_path = inner
+ raise Error("nested harness error")
+ except e:
+ caught = True
+ assert_equal(String(e), "nested harness error")
+ assert_true(caught)
+ assert_true(not exists(outer_path))
+ assert_true(not exists(inner_path))
+
+
def test_safe_tempdir_cleans_up_on_exception() raises:
var path = String("")
+ var entered_body = False
var caught = False
try:
with SafeTempDir() as temp_dir:
path = temp_dir
+ entered_body = True
raise Error("intentional cleanup harness error")
except:
caught = True
+ assert_true(entered_body)
assert_true(caught)
assert_true(not exists(path))
diff --git a/tools/check_format.sh b/tools/check_format.sh
@@ -40,35 +40,60 @@ check() {
self_test() {
test_status=0
- bindir="$tmp/bin"
- mkdir -p "$bindir"
- # Negative control 1: a formatter that crashes must be reported as a
- # formatter error, not silently treated as a formatting diff.
+ # A missing/broken formatter must fail the self-test, never make it pass.
+ if ! command -v mojo >/dev/null 2>&1; then
+ echo "self-test failed: no Mojo formatter available" >&2
+ return 1
+ fi
+
+ # Positive control: an already-formatted input must pass the check.
+ positive="$tmp/positive"
+ mkdir -p "$positive"
+ printf 'def main():\n pass\n' > "$positive/x.mojo"
+ if ! mojo format -q "$positive" >/dev/null 2>&1; then
+ echo "self-test failed: cannot produce a formatted positive case" >&2
+ return 1
+ fi
+ if ! sh "$0" "$positive" >/dev/null 2>&1; then
+ echo "self-test failed: formatted input did not pass" >&2
+ test_status=1
+ fi
+
+ # Unformatted control: must be classified as `unformatted`.
+ unformatted="$tmp/unformatted"
+ mkdir -p "$unformatted"
+ printf 'def main( ): \n pass\n' > "$unformatted/x.mojo"
+ if output="$(sh "$0" "$unformatted" 2>&1)"; then rc=0; else rc=$?; fi
+ if [ "$rc" -eq 0 ] || ! printf '%s' "$output" | grep -q 'unformatted:'; then
+ echo "self-test failed: unformatted input not classified" >&2
+ test_status=1
+ fi
+
+ # Crashing formatter control: must be classified as `formatter-error`.
+ bindir="$tmp/bin-crash"
+ mkdir -p "$bindir"
printf '#!/bin/sh\nexit 42\n' > "$bindir/mojo"
chmod +x "$bindir/mojo"
- src="$tmp/crash_input"
- mkdir -p "$src"
- printf 'def main():\n pass\n' > "$src/x.mojo"
- if out="$(PATH="$bindir:$PATH" sh "$0" "$src" 2>&1)"; then
+ if output="$(PATH="$bindir:$PATH" sh "$0" "$unformatted" 2>&1)"; then
rc=0
else
rc=$?
fi
- if [ "$rc" -eq 0 ] || ! printf '%s' "$out" | grep -q 'formatter-error'; then
- echo "self-test failed: formatter crash was not reported" >&2
+ if [ "$rc" -eq 0 ] || ! printf '%s' "$output" | grep -q 'formatter-error:'; then
+ echo "self-test failed: crashing formatter not classified" >&2
test_status=1
fi
- # Negative control 2: a known-unformatted input must fail the check.
- if command -v mojo >/dev/null 2>&1; then
- src2="$tmp/unformatted_input"
- mkdir -p "$src2"
- printf 'def main( ): \n pass\n' > "$src2/x.mojo"
- if sh "$0" "$src2" >/dev/null 2>&1; then
- echo "self-test failed: unformatted input was not detected" >&2
- test_status=1
- fi
+ # Missing formatter control: must be classified as `formatter-error`.
+ if output="$(PATH="/usr/bin:/bin" sh "$0" "$positive" 2>&1)"; then
+ rc=0
+ else
+ rc=$?
+ fi
+ if [ "$rc" -eq 0 ] || ! printf '%s' "$output" | grep -q 'formatter-error:'; then
+ echo "self-test failed: missing formatter not classified" >&2
+ test_status=1
fi
if [ "$test_status" -eq 0 ]; then
diff --git a/tools/check_harness_runner_exit.sh b/tools/check_harness_runner_exit.sh
@@ -1,39 +1,128 @@
#!/usr/bin/env sh
-# Negative controls for test-harness exception propagation.
+# Runner-exit control: prove that a failing test actually fails the Mojo test
+# runner, and that infrastructure failures (compiler, loader, missing tool,
+# pre-body crash) make THIS control fail instead of being mistaken for an
+# expected failing test.
#
-# A failing assertion or a raised error inside SafeTempDir must make the Mojo
-# test runner exit nonzero. A passing canary must exit zero. This proves the
-# SafeTempDir wrapper does not reintroduce the TemporaryDirectory masking
-# behavior.
+# For every canary: compile it first, then execute it, then require the exact
+# expected test identity, failure/pass summary and zero skips. Any compile or
+# loader error, any missing executable, or any output that does not name the
+# expected executed test is an infrastructure failure.
+#
+# Usage: sh tools/check_harness_runner_exit.sh
+# sh tools/check_harness_runner_exit.sh --self-test
set -eu
root="$(cd "$(dirname "$0")/.." && pwd)"
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+# run_one <canary.mojo> <expected_test_name> <fail|pass>
+run_one() {
+ canary="$1"
+ expected_name="$2"
+ expect="$3"
+ bin="$tmp/$(basename "$canary" .mojo).bin"
+
+ if ! mojo build -I "$root/src" -I "$root/tests" "$canary" -o "$bin" \
+ >"$tmp/build.log" 2>&1; then
+ echo "INFRA: compile failed: $canary" >&2
+ sed 's/^/ /' "$tmp/build.log" >&2
+ return 1
+ fi
+ if [ ! -x "$bin" ]; then
+ echo "INFRA: compile produced no executable: $canary" >&2
+ return 1
+ fi
-run_canary() {
- if mojo -I "$root/src" -I "$root/tests" "$1" >/dev/null 2>&1; then
- printf '0'
+ if output="$("$bin" 2>&1)"; then
+ rc=0
else
- printf '1'
+ rc=$?
fi
+
+ case "$expect" in
+ fail)
+ if [ "$rc" -eq 0 ]; then
+ echo "CONTROL: $canary did not fail the runner" >&2
+ return 1
+ fi
+ if ! printf '%s' "$output" | grep -q "FAIL .*$expected_name"; then
+ echo "CONTROL: $canary did not report executed test $expected_name" >&2
+ printf '%s\n' "$output" >&2
+ return 1
+ fi
+ if ! printf '%s' "$output" | grep -qE 'failed , 0 skipped'; then
+ echo "CONTROL: $canary produced no failure summary with zero skips" >&2
+ return 1
+ fi
+ ;;
+ pass)
+ if [ "$rc" -ne 0 ]; then
+ echo "CONTROL: passing canary $canary exited $rc" >&2
+ return 1
+ fi
+ if ! printf '%s' "$output" | grep -q '1 passed , 0 failed , 0 skipped'; then
+ echo "CONTROL: passing canary produced no clean pass summary" >&2
+ return 1
+ fi
+ ;;
+ esac
+ return 0
}
-assert_fail="$(run_canary "$root/tests/harness_canary_assert_fail.mojo")"
-error_fail="$(run_canary "$root/tests/harness_canary_error_fail.mojo")"
-pass_ok="$(run_canary "$root/tests/harness_canary_pass.mojo")"
+self_test() {
+ test_status=0
-status=0
-if [ "$assert_fail" -eq 0 ]; then
- echo "harness canary: failing assertion did not fail the runner" >&2
- status=1
-fi
-if [ "$error_fail" -eq 0 ]; then
- echo "harness canary: raised error did not fail the runner" >&2
- status=1
-fi
-if [ "$pass_ok" -ne 0 ]; then
- echo "harness canary: passing canary unexpectedly failed" >&2
- status=1
+ # A formatter/runner binary that fails compilation must be detected as
+ # infrastructure failure, never as an expected failing test.
+ bindir="$tmp/bin"
+ mkdir -p "$bindir"
+ printf '#!/bin/sh\necho "error: synthetic compiler failure" >&2\nexit 1\n' \
+ > "$bindir/mojo"
+ chmod +x "$bindir/mojo"
+ if PATH="$bindir:$PATH" run_one \
+ "$root/tests/harness_canary_assert_fail.mojo" \
+ test_canary_assertion_fails fail; then
+ echo "self-test failed: synthetic compile failure accepted" >&2
+ test_status=1
+ fi
+
+ # A loader/pre-body crash must also be rejected.
+ crash="$tmp/prebody_crash.mojo"
+ printf 'def main() raises:\n raise Error("pre-body crash")\n' > "$crash"
+ if run_one "$crash" test_none fail; then
+ echo "self-test failed: pre-body crash accepted" >&2
+ test_status=1
+ fi
+
+ # A missing toolchain must fail the control, not pass it.
+ if PATH="/usr/bin:/bin" run_one \
+ "$root/tests/harness_canary_assert_fail.mojo" \
+ test_canary_assertion_fails fail; then
+ echo "self-test failed: missing tool accepted" >&2
+ test_status=1
+ fi
+
+ if [ "$test_status" -eq 0 ]; then
+ echo "harness runner-exit self-test: ok"
+ fi
+ return "$test_status"
+}
+
+if [ "${1:-}" = "--self-test" ]; then
+ self_test
+ exit $?
fi
+
+status=0
+run_one "$root/tests/harness_canary_assert_fail.mojo" \
+ test_canary_assertion_fails fail || status=1
+run_one "$root/tests/harness_canary_error_fail.mojo" \
+ test_canary_generic_error_fails fail || status=1
+run_one "$root/tests/harness_canary_pass.mojo" \
+ test_canary_passes pass || status=1
+
if [ "$status" -eq 0 ]; then
echo "harness runner-exit canaries: ok"
fi
