test_unittest_report.py (36439B)
1 """Exercise the standalone producer with actual stdlib tests and disposable roots.""" 2 3 from __future__ import annotations 4 5 import importlib 6 import io 7 import json 8 import os 9 import shutil 10 import subprocess 11 import sys 12 import tempfile 13 import unittest 14 from pathlib import Path 15 from unittest.mock import patch 16 17 from scripts.fixture_tool_dispatch import CompilerCommand 18 from scripts.fixture_test_policy import limit 19 20 SECRET = "private exception, skip reason, and subtest parameter" 21 SCRIPTS = Path(__file__).resolve().parent 22 23 24 def observer_code_mutations(failure): 25 for table in ["", "executables", "methods"]: 26 for restored in [False, True]: 27 action = ( 28 "function = type(result).addFailure; original = function.__code__; " 29 ) 30 if restored: 31 action += "self.addCleanup(setattr, function, '__code__', original); " 32 action += "function.__code__ = (lambda self, test, err: self.addSuccess(test)).__code__; " 33 if table: 34 action += ( 35 "import sys; owner = sys.modules['__main__']; " 36 "names = ['CALLBACK_EXECUTABLES', 'CALLBACK_METHODS']; " 37 "previous = {name: getattr(owner, name, None) for name in names}; " 38 ) 39 if restored: 40 action += "self.addCleanup(lambda: [(setattr(owner, name, value) if value is not None else delattr(owner, name)) for name, value in previous.items() if value is not None or hasattr(owner, name)]); " 41 action += ( 42 "baseline = {**getattr(owner, 'CALLBACK_METHODS', {}), 'addFailure': function}; " 43 "owner.CALLBACK_EXECUTABLES = {name: owner.origin.callable_identity(method) for name, method in baseline.items()}; " 44 ) 45 if table == "methods": 46 action += "owner.CALLBACK_METHODS = baseline; " 47 yield ( 48 action, 49 ( 50 "try: " + failure + "\n except AssertionError:\n" 51 " Path(__name__ + '.failure.executed').write_text(self.id())\n" 52 " raise" 53 ), 54 ) 55 56 57 class StandaloneReportTests(unittest.TestCase): 58 def setUp(self): 59 temporary = tempfile.TemporaryDirectory(prefix="tera-unittest-report-") 60 self.addCleanup(temporary.cleanup) 61 self.root = Path(temporary.name).resolve() 62 (self.root / "scripts").mkdir() 63 shutil.copyfile( 64 SCRIPTS / "unittest_report.py", self.root / "scripts/unittest_report.py" 65 ) 66 if (SCRIPTS / "unittest_report_origin.py").is_file(): 67 shutil.copyfile( 68 SCRIPTS / "unittest_report_origin.py", 69 self.root / "scripts/unittest_report_origin.py", 70 ) 71 self.output = self.root / "report.json" 72 73 def module(self, source, name="test_observed"): 74 path = self.root / "scripts" / (name + ".py") 75 path.write_text("import unittest\nSECRET = " + repr(SECRET) + "\n" + source) 76 return "scripts." + name 77 78 def run_process(self, modules, output=None): 79 entry = getattr(self, "report_entry", ["scripts/unittest_report.py"]) 80 output_path = str(output or self.output) 81 arguments = [sys.executable, *entry, "--report", output_path, *modules] 82 record = self.probe_record(arguments, modules) 83 command = CompilerCommand( 84 arguments, 85 self.root, 86 getattr(self, "process_timeout", limit("reporter_child")), 87 env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, 88 budget="reporter_child", 89 ) 90 failure = None 91 try: 92 command.execute() 93 except BaseException as error: 94 failure = error 95 process = command.process 96 if process is None: 97 raise failure 98 result = subprocess.CompletedProcess( 99 arguments, 100 process.returncode, 101 bytes(command.output["stdout"]).decode(), 102 bytes(command.output["stderr"]).decode(), 103 ) 104 if record is not None: 105 (record / "capture.json").write_text(json.dumps(command.record, indent=2)) 106 (record / "settlement.json").write_text( 107 json.dumps( 108 { 109 "pid": process.pid, 110 "reaped": process.poll() is not None, 111 "timed_out": isinstance(failure, subprocess.TimeoutExpired), 112 "group_absent": command.record["group_absent"], 113 "pid_absent": not Path("/proc", str(process.pid)).exists() 114 if sys.platform == "linux" 115 else self.pid_absent(process.pid), 116 } 117 ) 118 ) 119 (record / "exit.txt").write_text(str(result.returncode)) 120 (record / "stdout.txt").write_text(result.stdout) 121 (record / "stderr.txt").write_text(result.stderr) 122 raw = ( 123 self.output.read_bytes() 124 if self.output.is_file() and not self.output.is_symlink() 125 else b"" 126 ) 127 (record / "report.json").write_bytes(raw) 128 if failure: 129 raise failure 130 return result 131 132 @staticmethod 133 def pid_absent(pid): 134 try: 135 os.kill(pid, 0) 136 except ProcessLookupError: 137 return True 138 return False 139 140 def probe_record(self, arguments, modules): 141 evidence = os.environ.get("TERA_REPORT_TEST_EVIDENCE") 142 if not evidence: 143 return None 144 record = Path(tempfile.mkdtemp(prefix="probe-", dir=evidence)) 145 (record / "argv.json").write_text(json.dumps(arguments)) 146 (record / "cwd.txt").write_text(str(self.root)) 147 (record / "test_id.txt").write_text(self.id()) 148 (record / "producer.py").write_bytes( 149 (self.root / "scripts/unittest_report.py").read_bytes() 150 ) 151 for index, module in enumerate(modules[:4096]): 152 if len(module) <= 255 and module.startswith("scripts.test_"): 153 path = self.root / "scripts" / (module[8:] + ".py") 154 if path.is_file() and not path.is_symlink(): 155 (record / ("module-" + str(index) + ".py")).write_bytes( 156 path.read_bytes() 157 ) 158 return record 159 160 def run_report(self, modules, output=None): 161 result = self.run_process(modules, output) 162 raw = self.output.read_bytes() if self.output.is_file() else b"" 163 document = json.loads(raw) if raw else None 164 if document is not None: 165 self.assertNotIn(SECRET, json.dumps(document)) 166 self.assertEqual( 167 set(document), {"schema", "test_ids", "tests_run", "callbacks"} 168 ) 169 return result, document 170 171 def test_genuine_pass_and_successful_subtests_have_one_identity(self): 172 module = self.module(""" 173 class ObservedTests(unittest.TestCase): 174 def test_one(self): 175 for index in range(3): 176 with self.subTest(private=SECRET, index=index): 177 self.assertTrue(True) 178 """) 179 result, document = self.run_report([module]) 180 self.assertEqual(result.returncode, 0, result.stderr) 181 identity = module + ".ObservedTests.test_one" 182 self.assertEqual(document["test_ids"], [identity]) 183 self.assertEqual(document["tests_run"], 1) 184 self.assertEqual( 185 [row["callback"] for row in document["callbacks"]], 186 [ 187 "startTestRun", 188 "startTest", 189 "addSubTest", 190 "addSubTest", 191 "addSubTest", 192 "addSuccess", 193 "stopTest", 194 "stopTestRun", 195 ], 196 ) 197 self.assertEqual( 198 [ 199 row["ordinal"] 200 for row in document["callbacks"] 201 if row["callback"] == "addSubTest" 202 ], 203 [1, 2, 3], 204 ) 205 206 def test_actual_nonpassing_test_outcomes_are_observed(self): 207 modes = { 208 "failure": ("self.fail(SECRET)", "addFailure"), 209 "error": ("raise RuntimeError(SECRET)", "addError"), 210 "skip": ("self.skipTest(SECRET)", "addSkip"), 211 "cleanup": ("self.addCleanup(lambda: self.fail(SECRET))", "addFailure"), 212 "cleanup_error": ("self.addCleanup(lambda: int(SECRET))", "addError"), 213 "expected": ("self.fail(SECRET)", "addExpectedFailure"), 214 "unexpected": ("pass", "addUnexpectedSuccess"), 215 } 216 for mode, (body, callback) in modes.items(): 217 with self.subTest(mode=mode): 218 decorator = ( 219 " @unittest.expectedFailure\n" 220 if mode in {"expected", "unexpected"} 221 else "" 222 ) 223 module = self.module( 224 "class ObservedTests(unittest.TestCase):\n" 225 + decorator 226 + " def test_one(self):\n " 227 + body 228 + "\n" 229 ) 230 self.output.unlink(missing_ok=True) 231 result, document = self.run_report([module]) 232 self.assertNotEqual(result.returncode, 0) 233 self.assertIn( 234 callback, [row["callback"] for row in document["callbacks"]] 235 ) 236 self.assertEqual(document["callbacks"][-1], {"callback": "stopTestRun"}) 237 238 def test_actual_subtest_failure_error_and_skip_redact_parameters(self): 239 for body, callback in [ 240 ("self.fail(SECRET)", "addSubTest"), 241 ("raise RuntimeError(SECRET)", "addSubTest"), 242 ("self.skipTest(SECRET)", "addSkip"), 243 ]: 244 with self.subTest(callback=callback, body=body): 245 module = self.module( 246 "class ObservedTests(unittest.TestCase):\n def test_one(self):\n with self.subTest(private=SECRET):\n " 247 + body 248 + "\n" 249 ) 250 self.output.unlink(missing_ok=True) 251 result, document = self.run_report([module]) 252 self.assertNotEqual(result.returncode, 0) 253 event = next( 254 row for row in document["callbacks"] if row["callback"] == callback 255 ) 256 self.assertEqual(event["test_id"], module + ".ObservedTests.test_one") 257 if callback == "addSubTest": 258 self.assertFalse(event["successful"]) 259 260 def fixture_source(self, scope, phase, skip=False): 261 action = ( 262 "raise unittest.SkipTest(SECRET)" if skip else "raise RuntimeError(SECRET)" 263 ) 264 if scope == "module": 265 fixture = "def " + phase + "():\n " + action + "\n" 266 return ( 267 fixture 268 + "class ObservedTests(unittest.TestCase):\n def test_one(self): pass\n" 269 ) 270 return ( 271 "class ObservedTests(unittest.TestCase):\n @classmethod\n def " 272 + phase 273 + "(cls):\n " 274 + action 275 + "\n def test_one(self): pass\n" 276 ) 277 278 def test_actual_module_and_class_fixture_errors_and_skips(self): 279 for scope, phases in [ 280 ("module", ["setUpModule", "tearDownModule"]), 281 ("class", ["setUpClass", "tearDownClass"]), 282 ]: 283 for phase in phases: 284 for skip in [False, True]: 285 with self.subTest(scope=scope, phase=phase, skip=skip): 286 module = self.module(self.fixture_source(scope, phase, skip)) 287 self.output.unlink(missing_ok=True) 288 result, document = self.run_report([module]) 289 self.assertNotEqual(result.returncode, 0) 290 self.assertIn( 291 "addSkip" if skip else "addError", 292 [row["callback"] for row in document["callbacks"]], 293 ) 294 event = next( 295 row 296 for row in document["callbacks"] 297 if row["callback"] == ("addSkip" if skip else "addError") 298 ) 299 owner = module + (".ObservedTests" if scope == "class" else "") 300 self.assertEqual(event["test_id"], phase + " (" + owner + ")") 301 self.assertEqual( 302 document["callbacks"][-1], {"callback": "stopTestRun"} 303 ) 304 305 def test_actual_class_and_module_cleanup_errors(self): 306 for source in [ 307 "def setUpModule():\n unittest.addModuleCleanup(lambda: int(SECRET))\nclass ObservedTests(unittest.TestCase):\n def test_one(self): pass\n", 308 "class ObservedTests(unittest.TestCase):\n @classmethod\n def setUpClass(cls): cls.addClassCleanup(lambda: int(SECRET))\n def test_one(self): pass\n", 309 ]: 310 with self.subTest(source=source): 311 module = self.module(source) 312 self.output.unlink(missing_ok=True) 313 result, document = self.run_report([module]) 314 self.assertNotEqual(result.returncode, 0) 315 self.assertIn( 316 "addError", [row["callback"] for row in document["callbacks"]] 317 ) 318 319 def test_actual_early_stop_and_interruption_preserve_partial_inventory(self): 320 for body in ["self._outcome.result.stop()", "raise KeyboardInterrupt(SECRET)"]: 321 with self.subTest(body=body): 322 module = self.module( 323 "class ObservedTests(unittest.TestCase):\n def test_one(self):\n " 324 + body 325 + "\n def test_two(self): pass\n" 326 ) 327 self.output.unlink(missing_ok=True) 328 result, document = self.run_report([module]) 329 self.assertNotEqual(result.returncode, 0) 330 self.assertIsNotNone(document, result.stderr) 331 self.assertEqual(len(document["test_ids"]), 2) 332 self.assertEqual(document["tests_run"], 1) 333 self.assertEqual(document["callbacks"][-1], {"callback": "stopTestRun"}) 334 335 def test_multiple_modules_have_real_completed_tests(self): 336 source = ( 337 "class ObservedTests(unittest.TestCase):\n def test_one(self): pass\n" 338 ) 339 modules = [self.module(source), self.module(source, "test_secondary")] 340 result, document = self.run_report(modules) 341 self.assertEqual(result.returncode, 0, result.stderr) 342 self.assertEqual(document["tests_run"], 2) 343 self.assertEqual(len(document["test_ids"]), 2) 344 345 def test_empty_module_and_duplicate_test_inventory_are_rejected(self): 346 sources = [ 347 "pass\n", 348 "class ObservedTests(unittest.TestCase):\n def test_one(self): pass\ndef load_tests(loader, tests, pattern):\n return unittest.TestSuite([ObservedTests('test_one'), ObservedTests('test_one')])\n", 349 ] 350 for source in sources: 351 with self.subTest(source=source): 352 result, _ = self.run_report([self.module(source)]) 353 self.assertNotEqual(result.returncode, 0) 354 355 def test_selectors_are_bounded_canonical_unique_modules(self): 356 module = self.module( 357 "class ObservedTests(unittest.TestCase):\n def test_one(self): pass\n" 358 ) 359 for selectors in [ 360 [module, module], 361 [], 362 ["scripts/test_observed.py"], 363 ["scripts.test_"], 364 ["foreign.test_observed"], 365 ["scripts.test_" + "x" * 300], 366 [module] * 4097, 367 ]: 368 with self.subTest(selectors=selectors[:2]): 369 result, _ = self.run_report(selectors) 370 self.assertNotEqual(result.returncode, 0) 371 372 def test_selected_origins_cannot_be_symlinks_directories_or_foreign_ids(self): 373 module = self.module( 374 "class ObservedTests(unittest.TestCase):\n def test_one(self): pass\n" 375 ) 376 path = self.root / "scripts/test_observed.py" 377 foreign = self.root / "foreign.py" 378 path.rename(foreign) 379 path.symlink_to(foreign) 380 result, _ = self.run_report([module]) 381 self.assertNotEqual(result.returncode, 0) 382 path.unlink() 383 path.mkdir() 384 result, _ = self.run_report([module]) 385 self.assertNotEqual(result.returncode, 0) 386 path.rmdir() 387 self.module( 388 "class ObservedTests(unittest.TestCase):\n def test_one(self): pass\nObservedTests.__module__ = 'foreign.test_observed'\n" 389 ) 390 result, _ = self.run_report([module]) 391 self.assertNotEqual(result.returncode, 0) 392 393 def test_producer_origin_cannot_be_symlinked(self): 394 path = self.root / "scripts/unittest_report.py" 395 foreign = self.root / "foreign.py" 396 path.rename(foreign) 397 path.symlink_to(foreign) 398 result, _ = self.run_report([self.module("pass\n")]) 399 self.assertNotEqual(result.returncode, 0) 400 401 def test_safe_output_is_exclusive_and_rejects_symlink_parents(self): 402 module = self.module( 403 "class ObservedTests(unittest.TestCase):\n def test_one(self): pass\n" 404 ) 405 self.output.write_bytes(b"preserved") 406 result = self.run_process([module]) 407 self.assertNotEqual(result.returncode, 0) 408 self.assertEqual(self.output.read_bytes(), b"preserved") 409 self.output.unlink() 410 self.output.symlink_to(self.root / "foreign.json") 411 result, _ = self.run_report([module]) 412 self.assertNotEqual(result.returncode, 0) 413 self.output.unlink() 414 (self.root / "alias").symlink_to( 415 self.root / "scripts", target_is_directory=True 416 ) 417 result, _ = self.run_report([module], self.root / "alias/report.json") 418 self.assertNotEqual(result.returncode, 0) 419 self.assertFalse((self.root / "scripts/report.json").exists()) 420 421 def test_source_and_output_replacement_during_execution_fail_closed(self): 422 sources = [ 423 "from pathlib import Path\nclass ObservedTests(unittest.TestCase):\n def test_one(self):\n Path(__file__).write_text('pass\\n')\n", 424 "from pathlib import Path\nclass ObservedTests(unittest.TestCase):\n def test_one(self):\n Path('report.json').unlink()\n Path('report.json').write_text('replacement')\n", 425 ] 426 for source in sources: 427 with self.subTest(source=source): 428 self.output.unlink(missing_ok=True) 429 result = self.run_process([self.module(source)]) 430 self.assertNotEqual(result.returncode, 0) 431 432 def test_real_same_inode_output_tampering_is_rejected(self): 433 for action in [ 434 "path.write_bytes(b'PRIVATE_OUTPUT_SENTINEL')", 435 "path.write_bytes(b'PRIVATE_OUTPUT_SENTINEL' * 120000)", 436 "path.write_bytes(b'PRIVATE_OUTPUT_SENTINEL'); path.write_bytes(b'')", 437 ]: 438 with self.subTest(action=action): 439 self.output.unlink(missing_ok=True) 440 module = self.module( 441 "from pathlib import Path\nclass ObservedTests(unittest.TestCase):\n" 442 " def test_one(self):\n path = Path('report.json')\n" 443 " before = path.stat().st_ino\n " + action + "\n" 444 " self.assertEqual(path.stat().st_ino, before)\n" 445 " Path('testcase-passed').write_text('passed')\n" 446 ) 447 result = self.run_process([module]) 448 self.assertEqual((self.root / "testcase-passed").read_text(), "passed") 449 self.assertNotEqual(result.returncode, 0) 450 self.assertNotIn("PRIVATE_OUTPUT_SENTINEL", result.stderr) 451 452 def test_output_settlement_rechecks_content_and_position(self): 453 # Inject a mutation after the real writer returns, before actual close. 454 for action in [ 455 "os.write(output.descriptor, b'PRIVATE_OUTPUT_SENTINEL')", 456 "os.ftruncate(output.descriptor, 0)", 457 "os.pwrite(output.descriptor, b'X', 0)", 458 "os.lseek(output.descriptor, 0, os.SEEK_SET)", 459 ]: 460 with self.subTest(action=action): 461 self.output.unlink(missing_ok=True) 462 module = self.module( 463 "import os, sys\nclass ObservedTests(unittest.TestCase):\n" 464 " def test_one(self):\n" 465 " owner = sys.modules['__main__']\n" 466 " original = owner.ReportOutput.write\n" 467 " def changed(output, document):\n" 468 " original(output, document)\n " 469 + action 470 + "\n owner.ReportOutput.write = changed\n" 471 " from pathlib import Path\n" 472 " Path('testcase-passed').write_text('passed')\n" 473 ) 474 result = self.run_process([module]) 475 self.assertEqual((self.root / "testcase-passed").read_text(), "passed") 476 self.assertNotEqual(result.returncode, 0) 477 self.assertNotIn("PRIVATE_OUTPUT_SENTINEL", result.stderr) 478 479 def test_real_callback_limit_is_exact_and_does_not_append_over_bound(self): 480 for extra in [0, 1]: 481 with self.subTest(extra=extra): 482 module = self.module( 483 "class ObservedTests(unittest.TestCase):\n def test_one(self):\n for _ in range(" 484 + str(4091 + extra) 485 + "):\n with self.subTest(private=SECRET): pass\n" 486 ) 487 self.output.unlink(missing_ok=True) 488 result, document = self.run_report([module]) 489 self.assertEqual(result.returncode == 0, extra == 0, result.stderr) 490 self.assertLessEqual(len(document["callbacks"]), 4096) 491 if extra == 0: 492 self.assertEqual(len(document["callbacks"]), 4096) 493 494 def test_real_inventory_limit_is_separate_from_callback_limit(self): 495 for count in [4096, 4097]: 496 with self.subTest(count=count): 497 module = self.module( 498 "class ObservedTests(unittest.TestCase):\n def test_one(self): pass\ndef load_tests(loader, tests, pattern):\n case = type('ManyTests', (unittest.TestCase,), {'__module__': __name__, **{f'test_{index}': lambda self: None for index in range(" 499 + str(count) 500 + ")}})\n return loader.loadTestsFromTestCase(case)\n" 501 ) 502 self.output.unlink(missing_ok=True) 503 result, document = self.run_report([module]) 504 self.assertNotEqual(result.returncode, 0) 505 if count == 4096: 506 self.assertEqual(len(document["test_ids"]), 4096) 507 self.assertLessEqual(len(document["callbacks"]), 4096) 508 509 def report_shape(self, identity, subtests): 510 callbacks = [ 511 {"callback": "startTestRun"}, 512 {"callback": "startTest", "test_id": identity}, 513 ] 514 callbacks.extend( 515 { 516 "callback": "addSubTest", 517 "test_id": identity, 518 "ordinal": index + 1, 519 "successful": True, 520 } 521 for index in range(subtests) 522 ) 523 callbacks.extend( 524 [ 525 {"callback": "addSuccess", "test_id": identity}, 526 {"callback": "stopTest", "test_id": identity}, 527 {"callback": "stopTestRun"}, 528 ] 529 ) 530 return { 531 "schema": "tera.unittest-callback-report.v1", 532 "test_ids": [identity], 533 "tests_run": 1, 534 "callbacks": callbacks, 535 } 536 537 def test_actual_report_accepts_two_million_bytes_and_rejects_overflow(self): 538 prefix = "scripts.test_observed.ManyTests.test_" 539 for others in range(4): 540 document = self.report_shape(prefix, 0) 541 for index in range(others): 542 other = self.report_shape(prefix + "a_" + str(index), 0) 543 document["test_ids"].insert(index, other["test_ids"][0]) 544 document["callbacks"][1:1] = other["callbacks"][1:-1] 545 document["tests_run"] += 1 546 baseline = len(json.dumps(document, sort_keys=True, separators=(",", ":"))) 547 padding, remainder = divmod(2_000_000 - baseline, 4) 548 if remainder == 0: 549 break 550 self.assertEqual(remainder, 0) 551 for extra in [0, 1]: 552 with self.subTest(extra=extra): 553 source = ( 554 "class ManyTests(unittest.TestCase): pass\nfor index in range(" 555 + str(others) 556 + "):\n setattr(ManyTests, 'test_a_' + str(index), lambda self: None)\nsetattr(ManyTests, 'test_' + 'x' * " 557 + str(padding + extra) 558 + ", lambda self: None)\n" 559 ) 560 module = self.module(source) 561 self.output.unlink(missing_ok=True) 562 result, document = self.run_report([module]) 563 self.assertEqual(result.returncode == 0, extra == 0, result.stderr) 564 self.assertLessEqual(len(self.output.read_bytes()), 2_000_000) 565 if extra == 0: 566 self.assertEqual(len(self.output.read_bytes()), 2_000_000) 567 self.assertEqual(document["tests_run"], others + 1) 568 569 def test_partial_testcase_without_success_is_not_accepted(self): 570 module = self.module( 571 "class ObservedTests(unittest.TestCase):\n def test_one(self): pass\n def run(self, result):\n result.startTest(self)\n result.stopTest(self)\n" 572 ) 573 result, document = self.run_report([module]) 574 self.assertNotEqual(result.returncode, 0) 575 self.assertEqual(document["tests_run"], 1) 576 self.assertNotIn( 577 "addSuccess", [row["callback"] for row in document["callbacks"]] 578 ) 579 580 def test_selected_code_cannot_exit_successfully_during_import(self): 581 result, _ = self.run_report([self.module("raise SystemExit(0)\n")]) 582 self.assertNotEqual(result.returncode, 0) 583 584 def test_selected_module_does_not_execute_forged_cached_bytecode(self): 585 module = self.module( 586 "class ObservedTests(unittest.TestCase):\n def test_one(self): pass\n" 587 ) 588 source = self.root / "scripts/test_observed.py" 589 code = compile("raise SystemExit(0)", str(source), "exec") 590 import importlib._bootstrap_external 591 592 cached = Path(importlib.util.cache_from_source(str(source))) 593 cached.parent.mkdir() 594 cached.write_bytes( 595 importlib._bootstrap_external._code_to_timestamp_pyc( 596 code, int(source.stat().st_mtime), source.stat().st_size 597 ) 598 ) 599 result, document = self.run_report([module]) 600 self.assertEqual(result.returncode, 0, result.stderr) 601 self.assertEqual(document["tests_run"], 1) 602 603 def test_foreign_resolution_is_rejected_before_module_execution(self): 604 (self.root / "foreign").mkdir() 605 (self.root / "foreign/test_secondary.py").write_text( 606 "from pathlib import Path\nPath('foreign-executed').write_text('executed')\n" 607 ) 608 first = self.module( 609 "import scripts\nfrom pathlib import Path\nscripts.__path__ = [str(Path.cwd() / 'foreign')]\nclass ObservedTests(unittest.TestCase):\n def test_one(self): pass\n" 610 ) 611 second = self.module( 612 "class ObservedTests(unittest.TestCase):\n def test_one(self): pass\n", 613 "test_secondary", 614 ) 615 result, _ = self.run_report([first, second]) 616 self.assertNotEqual(result.returncode, 0) 617 self.assertFalse((self.root / "foreign-executed").exists()) 618 619 def test_output_directory_rename_cannot_redirect_descriptor_writes(self): 620 module = self.module( 621 "from pathlib import Path\nclass ObservedTests(unittest.TestCase):\n def test_one(self):\n Path('output').rename('moved')\n Path('output').mkdir()\n Path('output/report.json').write_text('replacement')\n" 622 ) 623 (self.root / "output").mkdir() 624 result = self.run_process([module], self.root / "output/report.json") 625 self.assertNotEqual(result.returncode, 0) 626 self.assertEqual((self.root / "output/report.json").read_text(), "replacement") 627 self.assertEqual((self.root / "moved/report.json").read_bytes(), b"") 628 629 def test_duplicate_run_boundary_and_invalid_test_id_are_nonpassing(self): 630 for source in [ 631 "class ObservedTests(unittest.TestCase):\n def test_one(self): self._outcome.result.startTestRun()\n", 632 "class ObservedTests(unittest.TestCase):\n def test_one(self): pass\n def id(self): return SECRET\n", 633 ]: 634 with self.subTest(source=source): 635 self.output.unlink(missing_ok=True) 636 result, _ = self.run_report([self.module(source)]) 637 self.assertNotEqual(result.returncode, 0) 638 639 640 class OutputIntegrityTests(unittest.TestCase): 641 """Injected timing controls around actual writes of genuine observations.""" 642 643 def setUp(self): 644 self.producer = importlib.import_module("scripts.unittest_report") 645 temporary = tempfile.TemporaryDirectory(prefix="tera-output-integrity-") 646 self.addCleanup(temporary.cleanup) 647 self.root = Path(temporary.name) 648 case = type( 649 "ObservedTests", 650 (unittest.TestCase,), 651 {"__module__": "scripts.test_observed", "test_one": lambda self: None}, 652 )("test_one") 653 suite, budget = unittest.TestSuite([case]), self.producer.ReportBudget() 654 inventory = self.producer.inventory(suite, ["scripts.test_observed"], budget) 655 result, passing = self.producer.observe(suite, inventory, budget, io.StringIO()) 656 self.assertTrue(passing) 657 self.document = { 658 "schema": self.producer.SCHEMA, 659 "test_ids": inventory, 660 "tests_run": result.testsRun, 661 "callbacks": [dict(event) for event in result.callbacks], 662 } 663 664 def output(self, name): 665 path = self.root / (name + ".json") 666 output = self.producer.ReportOutput(str(path)) 667 self.addCleanup(os.close, output.directory) 668 self.addCleanup(os.close, output.descriptor) 669 evidence = os.environ.get("TERA_REPORT_TEST_EVIDENCE") 670 if evidence: 671 record = Path( 672 tempfile.mkdtemp(prefix="injected-" + name + "-", dir=evidence) 673 ) 674 self.addCleanup(shutil.copyfile, path, record / "report.json") 675 return output 676 677 def mutate(self, output, mode): 678 if mode == "append": 679 os.pwrite( 680 output.descriptor, SECRET.encode(), os.fstat(output.descriptor).st_size 681 ) 682 elif mode == "truncate": 683 os.ftruncate(output.descriptor, 0) 684 elif mode == "same_size": 685 os.pwrite(output.descriptor, b"X", 0) 686 else: 687 os.lseek(output.descriptor, 0, os.SEEK_SET) 688 689 def test_actual_write_races_reject_append_truncate_content_and_offset(self): 690 original = os.write 691 for mode in ["append", "truncate", "same_size", "offset"]: 692 with self.subTest(mode=mode): 693 output, calls = self.output("write-" + mode), [] 694 695 def changed(descriptor, raw): 696 count = original(descriptor, raw) 697 if not calls: 698 calls.append(count) 699 self.mutate(output, mode) 700 return count 701 702 with patch.object(self.producer.os, "write", changed): 703 with self.assertRaises(self.producer.ReportError) as failure: 704 output.write(self.document) 705 self.assertNotIn(SECRET, str(failure.exception)) 706 self.assertTrue(calls) 707 708 def test_prewrite_descriptor_offset_is_rejected(self): 709 output = self.output("prewrite-offset") 710 os.lseek(output.descriptor, 3, os.SEEK_SET) 711 with self.assertRaises(self.producer.ReportError): 712 output.write(self.document) 713 self.assertEqual(os.fstat(output.descriptor).st_size, 0) 714 715 def test_actual_short_writes_preserve_exact_emitted_bytes(self): 716 output, original = self.output("short-write"), os.write 717 718 def partial(descriptor, raw): 719 return original(descriptor, raw[:7]) 720 721 with patch.object(self.producer.os, "write", partial): 722 output.write(self.document) 723 output.verify() 724 raw = (self.root / "short-write.json").read_bytes() 725 self.assertEqual( 726 raw, self.producer.ENCODER.encode(self.document).encode("ascii") 727 ) 728 self.assertEqual(os.lseek(output.descriptor, 0, os.SEEK_CUR), len(raw)) 729 730 def test_descriptor_read_races_are_bounded_and_fenced(self): 731 original = os.pread 732 for mode in ["same_size", "offset"]: 733 with self.subTest(mode=mode): 734 output, reads = self.output("read-" + mode), [] 735 736 def changed(descriptor, count, offset): 737 raw = original(descriptor, count, offset) 738 reads.append((count, offset)) 739 if len(reads) == 1: 740 self.mutate(output, mode) 741 return raw 742 743 with patch.object(self.producer.os, "pread", changed): 744 with self.assertRaises(self.producer.ReportError): 745 output.write(self.document) 746 self.assertTrue(reads) 747 self.assertTrue(all(0 < count <= 65536 for count, _ in reads)) 748 749 750 class ObserverBoundTests(unittest.TestCase): 751 def test_byte_budget_accepts_exact_limit_and_rejects_next_byte(self): 752 producer = importlib.import_module("scripts.unittest_report") 753 budget = producer.ReportBudget() 754 budget.size = producer.MAX_BYTES - 1 755 budget.add(1) 756 self.assertEqual(budget.size, producer.MAX_BYTES) 757 with self.assertRaises(producer.ReportError): 758 budget.add(1) 759 self.assertEqual(budget.size, producer.MAX_BYTES) 760 761 def test_observer_never_retains_exception_skip_or_subtest_values(self): 762 producer = importlib.import_module("scripts.unittest_report") 763 module = "scripts.test_observed" 764 765 def method(case): 766 with case.subTest(private=SECRET): 767 case.skipTest(SECRET) 768 769 case_type = type( 770 "ObservedTests", 771 (unittest.TestCase,), 772 {"__module__": module, "test_one": method}, 773 ) 774 case = case_type("test_one") 775 budget = producer.ReportBudget() 776 inventory = producer.inventory(unittest.TestSuite([case]), [module], budget) 777 result, passing = producer.observe( 778 unittest.TestSuite([case]), inventory, budget, io.StringIO() 779 ) 780 self.assertFalse(passing) 781 self.assertEqual(result.skipped, []) 782 self.assertNotIn(SECRET, repr(result.__dict__)) 783 784 def test_imported_module_origin_is_checked_after_loading(self): 785 producer = importlib.import_module("scripts.unittest_report") 786 with tempfile.TemporaryDirectory() as temporary: 787 root = Path(temporary).resolve() 788 (root / "scripts").mkdir() 789 (root / "scripts/test_observed.py").write_text("pass\n") 790 with patch.object(importlib.util, "find_spec") as find: 791 find.return_value.origin = str(root / "foreign.py") 792 with self.assertRaises(producer.ReportError): 793 producer.load_modules(root, ["scripts.test_observed"]) 794 795 796 if __name__ == "__main__": 797 unittest.main()