test_unittest_report_nested.py (36175B)
1 """Actual nested unittest execution, owned ancestry and adversarial controls.""" 2 3 from __future__ import annotations 4 5 import json 6 import os 7 import shutil 8 import stat 9 import tempfile 10 import unittest 11 from pathlib import Path 12 from unittest.mock import patch 13 14 from scripts import test_unittest_report as support 15 16 PASS = """import unittest 17 from pathlib import Path 18 class NestedTests(unittest.TestCase): 19 def test_authored_nested_method_executes(self): 20 Path(__name__ + '.executed').write_text(self.id()) 21 with self.subTest(index=1): 22 self.assertEqual(2 + 2, 4) 23 """ 24 25 26 class NestedReportTests(unittest.TestCase): 27 run_report = support.StandaloneReportTests.run_report 28 pid_absent = staticmethod(support.StandaloneReportTests.pid_absent) 29 30 def setUp(self): 31 support.StandaloneReportTests.setUp(self) 32 self.authored_sources = {} 33 34 def run_process(self, modules, output=None): 35 result = support.StandaloneReportTests.run_process(self, modules, output) 36 record = getattr(self, "last_probe_record", None) 37 if record: 38 for marker in self.root.glob("*.executed"): 39 shutil.copyfile(marker, record / marker.name) 40 return result 41 42 def module(self, source=PASS, name="controls.test_nested", *, initializers=False): 43 relative = Path("scripts", *name.split(".")).with_suffix(".py") 44 path = self.root / relative 45 path.parent.mkdir(parents=True, exist_ok=True) 46 if initializers: 47 for directory in [self.root / "scripts", *path.parents]: 48 if directory.is_relative_to(self.root / "scripts"): 49 (directory / "__init__.py").write_bytes(b"") 50 path.write_text(source) 51 self.authored_sources[path] = path.read_bytes() 52 return "scripts." + name 53 54 def probe_record(self, arguments, modules): 55 del modules 56 evidence = os.environ.get("TERA_REPORT_TEST_EVIDENCE") 57 if not evidence: 58 return None 59 record = Path(tempfile.mkdtemp(prefix="nested-probe-", dir=evidence)) 60 self.last_probe_record = record 61 (record / "argv.json").write_text(json.dumps(arguments)) 62 (record / "cwd.txt").write_text(str(self.root)) 63 (record / "test_id.txt").write_text(self.id()) 64 shapes = [] 65 for path in (self.root / "scripts").rglob("*.py"): 66 relative = path.relative_to(self.root) 67 row = {"path": str(relative), "mode": path.lstat().st_mode} 68 shapes.append(row) 69 self.copy_fixture(path, record / relative, row) 70 (record / "source-shapes.json").write_text(json.dumps(shapes)) 71 return record 72 73 def copy_fixture(self, path, output, row): 74 output.parent.mkdir(parents=True, exist_ok=True) 75 if path.is_symlink(): 76 row["symlink_target"] = os.readlink(path) 77 elif stat.S_ISREG(path.lstat().st_mode): 78 try: 79 output.write_bytes(path.read_bytes()) 80 except PermissionError: 81 row["unreadable"] = True 82 if path in self.authored_sources: 83 output.write_bytes(self.authored_sources[path]) 84 85 def assert_execution(self, module, document): 86 identity = module + ".NestedTests.test_authored_nested_method_executes" 87 self.assertIn(identity, document["test_ids"]) 88 self.assertEqual((self.root / (module + ".executed")).read_text(), identity) 89 90 def test_namespace_nested_authored_test_really_executes(self): 91 module = self.module() 92 result, document = self.run_report([module]) 93 self.assertEqual(result.returncode, 0, result.stderr) 94 self.assertEqual(document["schema"], "tera.unittest-callback-report.v2") 95 self.assert_execution(module, document) 96 97 def test_empty_initializers_and_mixed_flat_nested_selections_execute(self): 98 modules = [ 99 self.module(initializers=True, name="controls.deep.test_nested"), 100 self.module(name="test_flat"), 101 ] 102 result, document = self.run_report(modules) 103 self.assertEqual(result.returncode, 0, result.stderr) 104 self.assertEqual(document["schema"], "tera.unittest-callback-report.v2") 105 self.assertEqual(document["tests_run"], 2) 106 for module in modules: 107 self.assert_execution(module, document) 108 109 def rejected(self, modules): 110 self.output.unlink(missing_ok=True) 111 result, document = self.run_report(modules) 112 self.assertNotEqual(result.returncode, 0) 113 return document 114 115 def test_flat_only_nested_fixture_retains_v1(self): 116 module = self.module(name="test_flat") 117 result, document = self.run_report([module]) 118 self.assertEqual(result.returncode, 0, result.stderr) 119 self.assertEqual(document["schema"], "tera.unittest-callback-report.v1") 120 self.assert_execution(module, document) 121 122 def test_nested_selector_shape_bounds_and_duplicates_fail_closed(self): 123 module = self.module() 124 for names in [ 125 [module, module], 126 [module] * 4097, 127 ["scripts.controls.test_"], 128 ["scripts.controls.9bad.test_nested"], 129 ["scripts..controls.test_nested"], 130 ["scripts.controls.test_nested.method"], 131 ["scripts.controls.test_" + "x" * 256], 132 ["scripts/controls/test_nested.py"], 133 ]: 134 with self.subTest(selector=names[0]): 135 self.rejected(names) 136 137 def test_symlinked_package_directory_is_rejected(self): 138 module = self.module() 139 path = self.root / "scripts/controls" 140 path.rename(self.root / "foreign") 141 path.symlink_to(self.root / "foreign", target_is_directory=True) 142 self.rejected([module]) 143 self.assertFalse((self.root / (module + ".executed")).exists()) 144 145 def test_symlinked_initializer_and_selected_module_are_rejected(self): 146 module = self.module(initializers=True) 147 for name in ["__init__.py", "test_nested.py"]: 148 with self.subTest(name=name): 149 path = self.root / "scripts/controls" / name 150 before = path.read_bytes() 151 foreign = self.root / "foreign.py" 152 foreign.write_bytes(before) 153 path.unlink() 154 path.symlink_to(foreign) 155 self.rejected([module]) 156 path.unlink() 157 path.write_bytes(before) 158 159 def test_missing_unreadable_and_fifo_selected_source_is_rejected(self): 160 module = self.module() 161 path = self.root / "scripts/controls/test_nested.py" 162 before = path.read_bytes() 163 path.unlink() 164 self.rejected([module]) 165 os.mkfifo(path) 166 self.rejected([module]) 167 path.unlink() 168 path.write_bytes(before) 169 path.chmod(0) 170 try: 171 self.rejected([module]) 172 finally: 173 path.chmod(0o600) 174 175 def test_initializer_changes_during_import_never_run_selected_test(self): 176 module = self.module(initializers=True) 177 path = self.root / "scripts/controls/__init__.py" 178 for action in [ 179 "Path(__file__).write_bytes(b'')", 180 "Path('scripts/controls/test_nested.py').write_text('pass\\n')", 181 "__path__ = [str(Path.cwd() / 'foreign')]", 182 "__spec__.origin = str(Path.cwd() / 'foreign.py')", 183 ]: 184 with self.subTest(action=action): 185 self.module(initializers=True) 186 path.write_text("from pathlib import Path\n" + action + "\n") 187 self.rejected([module]) 188 self.assertFalse((self.root / (module + ".executed")).exists()) 189 190 def test_ancestry_source_presence_mode_and_identity_drift_is_rejected(self): 191 for action in [ 192 "Path('scripts/controls/__init__.py').write_text('new = True\\n')", 193 "Path('scripts/controls/__init__.py').chmod(0o400)", 194 "Path('scripts/controls').rename('moved'); Path('scripts/controls').mkdir()", 195 "Path(__file__).write_text('pass\\n')", 196 "sys.modules['scripts.controls'].__path__ = ['foreign']", 197 "sys.modules['scripts.controls'].__spec__.origin = 'foreign.py'", 198 "sys.modules[__name__] = None", 199 "globals()['__name__'] = 'foreign.test_drift'", 200 "__spec__.name = 'foreign.test_drift'", 201 "__spec__.loader = None", 202 ]: 203 with self.subTest(action=action): 204 source = "import sys\n" + PASS.replace( 205 " with self.subTest(index=1):", 206 " " + action + "\n with self.subTest(index=1):", 207 ) 208 module = self.module( 209 source, name="controls.test_drift", initializers=True 210 ) 211 self.rejected([module]) 212 shutil.rmtree(self.root / "scripts/controls") 213 214 def test_empty_initializer_added_during_execution_is_rejected(self): 215 module = self.module( 216 PASS.replace( 217 " with self.subTest(index=1):", 218 " Path('scripts/controls/__init__.py').touch()\n with self.subTest(index=1):", 219 ) 220 ) 221 self.rejected([module]) 222 223 def test_owned_initializer_executes_source_instead_of_forged_bytecode(self): 224 import importlib._bootstrap_external 225 import importlib.util 226 227 module = self.module(initializers=True) 228 initializer = self.root / "scripts/controls/__init__.py" 229 initializer.write_text( 230 "from pathlib import Path\nPath('initializer-executed').write_text('actual source')\n" 231 ) 232 cached = Path(importlib.util.cache_from_source(str(initializer))) 233 cached.parent.mkdir() 234 forged = compile("raise SystemExit(0)", str(initializer), "exec") 235 cached.write_bytes( 236 importlib._bootstrap_external._code_to_timestamp_pyc( 237 forged, 238 int(initializer.stat().st_mtime), 239 initializer.stat().st_size, 240 ) 241 ) 242 result, document = self.run_report([module]) 243 self.assertEqual(result.returncode, 0, result.stderr) 244 self.assertEqual( 245 (self.root / "initializer-executed").read_text(), "actual source" 246 ) 247 self.assert_execution(module, document) 248 249 def test_foreign_package_shadowing_is_rejected_before_foreign_import(self): 250 module = self.module() 251 foreign = self.root / "foreign" 252 foreign.mkdir() 253 (foreign / "test_other.py").write_text( 254 "from pathlib import Path\nPath('foreign-executed').touch()\n" 255 ) 256 first = self.module( 257 "import scripts\nfrom pathlib import Path\nscripts.__path__ = [str(Path.cwd() / 'foreign')]\n" 258 + PASS, 259 name="test_first", 260 ) 261 self.rejected([first, module]) 262 self.assertFalse((self.root / "foreign-executed").exists()) 263 264 def test_nested_partial_nonpassing_duplicate_and_fabricated_lifecycles_reject(self): 265 for source in [ 266 PASS.replace( 267 "self.assertEqual(2 + 2, 4)", "self.skipTest('private reason')" 268 ), 269 PASS.replace("self.assertEqual(2 + 2, 4)", "self.fail('private reason')"), 270 PASS.replace( 271 "self.assertEqual(2 + 2, 4)", "self._outcome.result.startTestRun()" 272 ), 273 PASS 274 + "\ndef load_tests(loader, tests, pattern):\n return unittest.TestSuite([NestedTests('test_authored_nested_method_executes')] * 2)\n", 275 PASS 276 + "\n def id(self): return 'scripts.controls.test_nested.NestedTests.forged'\n", 277 PASS 278 + "\n def run(self, result):\n result.startTest(self)\n result.addSuccess(self)\n result.stopTest(self)\n", 279 PASS.replace("self.assertEqual(2 + 2, 4)", "self._outcome.result.stop()") 280 + "\n def test_second(self): pass\n", 281 ]: 282 with self.subTest(source=source): 283 self.rejected([self.module(source)]) 284 285 def test_monkey_patched_stdlib_testcase_identity_is_rejected(self): 286 module = self.module( 287 PASS 288 + "\nunittest.TestCase.id = lambda self: 'scripts.controls.test_nested.NestedTests.forged'\n" 289 ) 290 self.rejected([module]) 291 292 def test_root_initializer_cannot_prepatch_a_false_testcase_identity(self): 293 module = self.module(initializers=True) 294 (self.root / "scripts/__init__.py").write_text( 295 "import unittest\nunittest.TestCase.id = lambda self: " 296 "'scripts.controls.test_nested.NestedTests.forged'\n" 297 ) 298 self.rejected([module]) 299 300 def test_monkey_patched_observer_cannot_forge_successful_execution(self): 301 source = "import sys\n" + PASS.replace( 302 "self.assertEqual(2 + 2, 4)", 303 "sys.modules['__main__'].CallbackResult.passing = lambda result: True\n" 304 " self.fail('private failure')", 305 ) 306 self.rejected([self.module(source)]) 307 308 def test_instance_callback_shadows_cannot_convert_real_nonpassing_tests(self): 309 controls = [ 310 ( 311 "result.addFailure = lambda test, err: result.addSuccess(test)", 312 "self.fail('private failure')", 313 ), 314 ( 315 "result.__dict__['addFailure'] = lambda test, err: result.addSuccess(test)", 316 "self.fail('private failure')", 317 ), 318 ( 319 "object.__setattr__(result, 'addFailure', lambda test, err: result.addSuccess(test))", 320 "self.fail('private failure')", 321 ), 322 ( 323 "result.addError = lambda test, err: result.addSuccess(test)", 324 "raise RuntimeError('private error')", 325 ), 326 ( 327 "result.addSkip = lambda test, reason: result.addSuccess(test)", 328 "self.skipTest('private skip')", 329 ), 330 ( 331 "result.addExpectedFailure = lambda test, err: result.addSuccess(test)", 332 "self.fail('private expected failure')", 333 ), 334 ( 335 "result.addUnexpectedSuccess = lambda test: result.addSuccess(test)", 336 "self.assertTrue(True)", 337 ), 338 ("result.passing = lambda: True", "self.fail('private failure')"), 339 ( 340 "result.addSubTest = lambda test, subtest, err: result.addSuccess(test)", 341 "with self.subTest(index=1): self.fail('private subtest failure')", 342 ), 343 ] 344 controls.extend(support.observer_code_mutations("self.fail('private failure')")) 345 for name in ["test_observer", "controls.test_observer"]: 346 for action, outcome in controls: 347 with self.subTest(module=name, action=action): 348 source = PASS.replace( 349 " with self.subTest(index=1):\n self.assertEqual(2 + 2, 4)", 350 " result = self._outcome.result\n " 351 + action 352 + "\n " 353 + outcome, 354 ) 355 if ( 356 "addExpectedFailure" in action 357 or "addUnexpectedSuccess" in action 358 ): 359 source = source.replace( 360 " def test_authored", 361 " @unittest.expectedFailure\n def test_authored", 362 ) 363 module = self.module(source, name=name) 364 document = self.rejected([module]) 365 self.assert_execution(module, document) 366 if "function = type(result).addFailure" in action: 367 self.assertEqual( 368 (self.root / (module + ".failure.executed")).read_text(), 369 document["test_ids"][0], 370 ) 371 self.assertNotIn( 372 "addSuccess", 373 [event["callback"] for event in document["callbacks"]], 374 ) 375 376 def test_restored_instance_callback_shadows_remain_rejected(self): 377 for name in ["test_observer", "controls.test_observer"]: 378 for action in [ 379 "result.addFailure = forged", 380 "result.__dict__['addFailure'] = forged", 381 "object.__setattr__(result, 'addFailure', forged)", 382 ]: 383 with self.subTest(module=name, action=action): 384 source = PASS.replace( 385 " with self.subTest(index=1):\n self.assertEqual(2 + 2, 4)", 386 """ result = self._outcome.result 387 original = result.addFailure 388 def forged(test, err): 389 result.__dict__.pop('addFailure', None) 390 result.addFailure = original 391 result.addSuccess(test) 392 """ 393 + action 394 + "\n self.fail('private failure')", 395 ) 396 module = self.module(source, name=name) 397 document = self.rejected([module]) 398 self.assert_execution(module, document) 399 400 def test_unused_assignment_restored_before_callback_is_still_rejected(self): 401 for name in ["test_observer", "controls.test_observer"]: 402 with self.subTest(module=name): 403 source = PASS.replace( 404 " with self.subTest(index=1):", 405 " result = self._outcome.result\n" 406 " original = result.addFailure\n" 407 " result.addFailure = lambda test, err: result.addSuccess(test)\n" 408 " result.addFailure = original\n" 409 " result.rejected = False\n" 410 " with self.subTest(index=1):", 411 ) 412 module = self.module(source, name=name) 413 document = self.rejected([module]) 414 self.assert_execution(module, document) 415 416 def test_unused_instance_dictionary_callback_shadow_is_rejected(self): 417 for name in ["test_observer", "controls.test_observer"]: 418 for method in ["addFailure", "addSkip", "stop", "fixture_id"]: 419 with self.subTest(module=name, method=method): 420 source = PASS.replace( 421 " with self.subTest(index=1):", 422 " self._outcome.result.__dict__['" 423 + method 424 + "'] = lambda *args: None\n" 425 " with self.subTest(index=1):", 426 ) 427 module = self.module(source, name=name) 428 document = self.rejected([module]) 429 self.assert_execution(module, document) 430 431 def test_observed_failure_and_callback_journal_cannot_be_erased(self): 432 resets = [ 433 "result.nonpassing = False", 434 "result.__dict__['nonpassing'] = False", 435 "object.__setattr__(result, 'nonpassing', False)", 436 ] 437 edits = [ 438 "result.callbacks[:] = filtered", 439 "result.callbacks = filtered", 440 "result.__dict__['callbacks'] = filtered", 441 "object.__setattr__(result, 'callbacks', filtered)", 442 "result.callbacks.clear()", 443 "result.callbacks.pop(2)", 444 "del result.callbacks[2]", 445 "result.callbacks[2]['successful'] = True", 446 ] 447 for name in ["test_state", "controls.test_state"]: 448 for reset in resets: 449 for edit in edits: 450 with self.subTest(module=name, reset=reset, edit=edit): 451 source = PASS.replace( 452 " with self.subTest(index=1):\n self.assertEqual(2 + 2, 4)", 453 " result = self._outcome.result\n" 454 " with self.subTest(index=1): self.fail('private failed subtest')\n" 455 " " + reset + "\n" 456 " filtered = [event for event in result.callbacks if event['callback'] != 'addSubTest']\n" 457 " try:\n " + edit + "\n" 458 " except (AttributeError, TypeError): pass\n" 459 " self._outcome.success = True", 460 ) 461 module = self.module(source, name=name) 462 document = self.rejected([module]) 463 self.assert_execution(module, document) 464 self.assertTrue( 465 any( 466 event["callback"] == "addSubTest" 467 and not event["successful"] 468 for event in document["callbacks"] 469 ) 470 ) 471 472 def test_observer_descriptor_getter_and_setter_code_are_fenced(self): 473 for name in ["test_descriptor", "controls.test_descriptor"]: 474 for mode in ["getter", "setter", "journal_getters"]: 475 with self.subTest(module=name, mode=mode): 476 changed = { 477 "getter": "owner.nonpassing.fget.__code__ = false_getter.__code__", 478 "setter": "owner.nonpassing.fset.__code__ = reset_setter.__code__", 479 "journal_getters": "owner.nonpassing.fget.__code__ = false_getter.__code__; owner.callbacks.fget.__code__ = filtered_getter.__code__", 480 }[mode] 481 reset = "result.nonpassing = False" if mode == "setter" else "pass" 482 body = ( 483 """ result = self._outcome.result 484 with self.subTest(index=1): self.fail('private failed subtest') 485 owner = type(result) 486 def replacements(facts): 487 def false_getter(result): return False if facts else False 488 def reset_setter(result, value): facts[result]['nonpassing'] = bool(value) 489 def filtered_getter(result): 490 return tuple(event for event in facts[result]['callbacks'] if event['callback'] != 'addSubTest') 491 return false_getter, reset_setter, filtered_getter 492 false_getter, reset_setter, filtered_getter = replacements(None) 493 """ 494 + changed 495 + "\n " 496 + reset 497 + "\n self._outcome.success = True" 498 ) 499 source = PASS.replace( 500 " with self.subTest(index=1):\n self.assertEqual(2 + 2, 4)", 501 body, 502 ) 503 module = self.module(source, name=name) 504 document = self.rejected([module]) 505 self.assert_execution(module, document) 506 507 def test_instance_run_shadow_cannot_fabricate_unexecuted_test_callbacks(self): 508 for name in ["test_execution", "controls.test_execution"]: 509 for target in ["case", "suite", "method", "dispatch", "suite_call"]: 510 with self.subTest(module=name, target=target): 511 source = ( 512 PASS 513 + """ 514 def load_tests(loader, tests, pattern): 515 case = NestedTests('test_authored_nested_method_executes') 516 suite = unittest.TestSuite([case]) 517 def fabricated(result): 518 result.startTest(case) 519 result.addSuccess(case) 520 result.stopTest(case) 521 """ 522 + { 523 "case": "case.run = fabricated", 524 "suite": "suite.run = fabricated", 525 "method": "setattr(case, case._testMethodName, lambda: None)", 526 "dispatch": "case._callTestMethod = lambda method: None", 527 "suite_call": "class ForgedSuite(unittest.TestSuite):\n def __call__(self, result): fabricated(result)\n suite = ForgedSuite([case])", 528 }[target] 529 + "\n return suite\n" 530 ) 531 module = self.module(source, name=name) 532 self.rejected([module]) 533 self.assertFalse((self.root / (module + ".executed")).exists()) 534 535 def test_setup_dispatch_shadow_cannot_skip_authored_failure(self): 536 for name in ["test_dispatch", "controls.test_dispatch"]: 537 for target, assignment in [ 538 ("self._callTestMethod", "self._callTestMethod = lambda method: None"), 539 ( 540 "self._callTestMethod", 541 "self.__dict__['_callTestMethod'] = lambda method: None", 542 ), 543 ( 544 "type(self).test_authored_failure.__code__", 545 "type(self).test_authored_failure.__code__ = (lambda self: None).__code__", 546 ), 547 ( 548 "unittest.TestCase._callTestMethod.__code__", 549 "unittest.TestCase._callTestMethod.__code__ = (lambda self, method: None).__code__", 550 ), 551 ]: 552 for restored in [False, True]: 553 with self.subTest( 554 module=name, assignment=assignment, restored=restored 555 ): 556 source = ( 557 """import unittest 558 from pathlib import Path 559 class DispatchTests(unittest.TestCase): 560 def setUp(self): 561 Path(__name__ + '.setup.executed').write_text(self.id()) 562 self.addCleanup(lambda: Path(__name__ + '.cleanup.executed').write_text(self.id())) 563 self.original = ORIGINAL 564 ASSIGNMENT 565 def tearDown(self): 566 RESTORE 567 Path(__name__ + '.teardown.executed').write_text(self.id()) 568 def test_authored_failure(self): 569 Path(__name__ + '.body.executed').write_text(self.id()) 570 self.fail('actual authored failure') 571 """.replace("ORIGINAL", target) 572 .replace("ASSIGNMENT", assignment) 573 .replace( 574 "RESTORE", 575 target + " = self.original" if restored else "pass", 576 ) 577 ) 578 module = self.module(source, name=name) 579 document = self.rejected([module]) 580 identity = module + ".DispatchTests.test_authored_failure" 581 self.assertEqual( 582 (self.root / (module + ".setup.executed")).read_text(), 583 identity, 584 ) 585 self.assertEqual( 586 (self.root / (module + ".cleanup.executed")).read_text(), 587 identity, 588 ) 589 self.assertFalse( 590 (self.root / (module + ".body.executed")).exists() 591 ) 592 self.assertIn( 593 "addError", 594 [event["callback"] for event in document["callbacks"]], 595 ) 596 self.assertNotIn( 597 "addSuccess", 598 [event["callback"] for event in document["callbacks"]], 599 ) 600 601 def test_normal_setup_body_teardown_and_cleanup_keep_stdlib_outcomes(self): 602 for name in ["test_fixtures", "controls.test_fixtures"]: 603 for body in [ 604 "self.assertTrue(True)", 605 "self.fail('authored failure')", 606 "self.skipTest('authored skip')", 607 "with self.subTest(index=1): self.fail('authored subtest')", 608 ]: 609 with self.subTest(module=name, body=body): 610 self.output.unlink(missing_ok=True) 611 source = """import unittest 612 from pathlib import Path 613 class FixtureTests(unittest.TestCase): 614 def setUp(self): 615 Path(__name__ + '.setup.executed').write_text(self.id()) 616 self.addCleanup(lambda: Path(__name__ + '.cleanup.executed').write_text(self.id())) 617 def tearDown(self): 618 Path(__name__ + '.teardown.executed').write_text(self.id()) 619 def test_authored_body(self): 620 Path(__name__ + '.body.executed').write_text(self.id()) 621 BODY 622 """.replace("BODY", body) 623 module = self.module(source, name=name) 624 result, document = self.run_report([module]) 625 self.assertEqual( 626 result.returncode == 0, 627 body == "self.assertTrue(True)", 628 result.stderr, 629 ) 630 identity = module + ".FixtureTests.test_authored_body" 631 self.assertEqual(document["test_ids"], [identity]) 632 for part in ["setup", "body", "teardown", "cleanup"]: 633 self.assertEqual( 634 ( 635 self.root / (module + "." + part + ".executed") 636 ).read_text(), 637 identity, 638 ) 639 640 def test_monkey_patched_writer_cannot_export_a_forged_success_report(self): 641 source = ( 642 "import sys\n" 643 + PASS 644 + """ 645 owner = sys.modules['__main__'] 646 original = owner.ReportOutput.write 647 def changed(output, document): 648 document['tests_run'] = 0 649 original(output, document) 650 owner.ReportOutput.write = changed 651 """ 652 ) 653 document = self.rejected([self.module(source)]) 654 self.assertEqual(document["tests_run"], 0) 655 656 def test_actual_nested_report_byte_budget_remains_two_million(self): 657 prefix = "scripts.controls.test_nested.ManyTests.test_" 658 for others in range(4): 659 document = support.StandaloneReportTests.report_shape(self, prefix, 0) 660 for index in range(others): 661 other = support.StandaloneReportTests.report_shape( 662 self, prefix + "a_" + str(index), 0 663 ) 664 document["test_ids"].insert(index, other["test_ids"][0]) 665 document["callbacks"][1:1] = other["callbacks"][1:-1] 666 document["tests_run"] += 1 667 baseline = len(json.dumps(document, sort_keys=True, separators=(",", ":"))) 668 padding, remainder = divmod(2_000_000 - baseline, 4) 669 if remainder == 0: 670 break 671 self.assertEqual(remainder, 0) 672 for extra in [0, 1]: 673 with self.subTest(extra=extra): 674 self.output.unlink(missing_ok=True) 675 source = ( 676 "import unittest\nclass ManyTests(unittest.TestCase): pass\n" 677 "for index in range(" + str(others) + "):\n" 678 " setattr(ManyTests, 'test_a_' + str(index), lambda self: None)\n" 679 "setattr(ManyTests, 'test_' + 'x' * " 680 + str(padding + extra) 681 + ", lambda self: None)\n" 682 ) 683 result, document = self.run_report([self.module(source)]) 684 self.assertEqual(result.returncode == 0, extra == 0, result.stderr) 685 self.assertLessEqual(len(self.output.read_bytes()), 2_000_000) 686 if extra == 0: 687 self.assertEqual(len(self.output.read_bytes()), 2_000_000) 688 689 def test_selected_code_cannot_change_report_budget_constants(self): 690 for action in [ 691 "sys.modules['__main__'].MAX_ITEMS = 5000", 692 "sys.modules['scripts.unittest_report_origin'].MAX_BYTES = 5000000", 693 "sys.modules['__main__'].ENCODER.sort_keys = False", 694 ]: 695 with self.subTest(action=action): 696 source = "import sys\n" + PASS.replace( 697 " with self.subTest(index=1):", 698 " " + action + "\n with self.subTest(index=1):", 699 ) 700 self.rejected([self.module(source)]) 701 702 def test_selected_inventory_cannot_omit_empty_or_foreign_nested_modules(self): 703 first = self.module() 704 for source in [ 705 "pass\n", 706 PASS + "\nNestedTests.__module__ = 'foreign.test_other'\n", 707 ]: 708 with self.subTest(source=source): 709 other = self.module(source, name="other.test_selected") 710 self.rejected([first, other]) 711 712 def test_nested_fixture_failure_has_actual_redacted_module_identity(self): 713 module = self.module( 714 PASS + "\ndef setUpModule(): raise RuntimeError('private fixture error')\n" 715 ) 716 document = self.rejected([module]) 717 event = next( 718 row for row in document["callbacks"] if row["callback"] == "addError" 719 ) 720 self.assertEqual(event["test_id"], "setUpModule (" + module + ")") 721 self.assertNotIn("private fixture error", json.dumps(document)) 722 723 def test_nested_callback_budget_remains_exact(self): 724 for extra in [0, 1]: 725 with self.subTest(extra=extra): 726 self.output.unlink(missing_ok=True) 727 module = self.module( 728 "import unittest\nclass NestedTests(unittest.TestCase):\n" 729 " def test_many(self):\n for _ in range(" 730 + str(4091 + extra) 731 + "):\n with self.subTest(index=1): pass\n" 732 ) 733 result, document = self.run_report([module]) 734 self.assertEqual(result.returncode == 0, extra == 0, result.stderr) 735 self.assertLessEqual(len(document["callbacks"]), 4096) 736 737 def test_initializer_drift_after_report_write_is_rejected(self): 738 source = ( 739 "import sys\n" 740 + PASS 741 + """ 742 owner = sys.modules['__main__'] 743 original = owner.ReportOutput.write 744 def changed(output, document): 745 original(output, document) 746 Path('scripts/controls/__init__.py').write_bytes(b'changed = True\n') 747 owner.ReportOutput.write = changed 748 """.replace("b'changed = True\n'", "b'changed = True\\n'") 749 ) 750 self.rejected([self.module(source, initializers=True)]) 751 752 def test_actual_descriptor_read_race_fences_same_inode_source_bytes(self): 753 from scripts import unittest_report_origin as origin 754 755 module = self.module() 756 path = self.root.joinpath(*module.split(".")).with_suffix(".py") 757 original = os.read 758 calls = [] 759 760 def changed(descriptor, count): 761 raw = original(descriptor, count) 762 calls.append(count) 763 path.write_bytes(b"x" * path.stat().st_size) 764 return raw 765 766 with patch.object(origin.os, "read", changed): 767 with self.assertRaises(origin.ReportError): 768 origin.source_snapshot(self.root, path.relative_to(self.root)) 769 self.assertTrue(calls) 770 771 def test_actual_hostile_long_testcase_identity_rejects_in_bounded_child(self): 772 self.process_timeout = 3 773 source = PASS + "\nNestedTests.__qualname__ = 'test_a.' * 8000 + '!'\n" 774 self.rejected([self.module(source)]) 775 776 def test_actual_hostile_long_fixture_identity_rejects_in_bounded_child(self): 777 self.process_timeout = 3 778 source = ( 779 PASS 780 + """ 781 unittest.suite._ErrorHolder.id = lambda self: 'setUpModule (scripts.' + 'test_a.' * 8000 + '!)' 782 def setUpModule(): raise RuntimeError('synthetic fixture error') 783 """ 784 ) 785 self.rejected([self.module(source)]) 786 787 788 if __name__ == "__main__": 789 unittest.main()