test_unittest_report_bootstrap.py (12139B)
1 """Run genuine before-package and before-fixture dispatch integrity controls.""" 2 3 from __future__ import annotations 4 5 import shutil 6 import unittest 7 8 from scripts import test_unittest_report as support 9 10 BODY = """from pathlib import Path 11 class ActualTests(unittest.TestCase): 12 def setUp(self): 13 Path('setup.executed').write_text(self.id()) 14 def test_authored_body(self): 15 Path('body.executed').write_text(self.id()) 16 with self.subTest(index=1): 17 BODY 18 def tearDown(self): 19 Path('teardown.executed').write_text(self.id()) 20 """ 21 FORGED_RUN = """import unittest 22 def forged_run(self, result=None): 23 result.startTest(self) 24 result.addSuccess(self) 25 result.stopTest(self) 26 return result 27 unittest.TestCase.run = forged_run 28 """ 29 STDLIB_SHADOW = """import sys 30 from pathlib import Path 31 actual = Path(sys._stdlib_dir) / 'unittest' 32 __path__ = [str(actual)] 33 __package__ = 'unittest' 34 exec(compile((actual / '__init__.py').read_bytes(), str(actual / '__init__.py'), 'exec'), globals()) 35 def forged_run(self, result=None): 36 result.startTest(self) 37 result.addSuccess(self) 38 result.stopTest(self) 39 return result 40 TestCase.run = forged_run 41 """ 42 43 44 def fixture_source(kind, restored, target="body"): 45 """Keep replacement/restoration controls on the real fixture lifecycle.""" 46 source = BODY.replace("BODY", "self.fail('actual authored failure')") 47 expression = ( 48 "ActualTests.test_authored_body" 49 if target == "body" 50 else "unittest.TestCase._callTestMethod" 51 ) 52 action = ( 53 f"function = {expression}; original = function.__code__; " 54 "function.__code__ = (lambda self, *args: None).__code__" 55 ) 56 if kind == "class": 57 cleanup = ( 58 "cls.addClassCleanup(setattr, function, '__code__', original); " 59 if restored == "cleanup" 60 else "cls.original = original; " 61 ) 62 fixture = " @classmethod\n def setUpClass(cls):\n " 63 fixture += action.replace( 64 "function.__code__ =", cleanup + "function.__code__ =" 65 ) 66 fixture += "\n" 67 if restored == "teardown": 68 fixture += ( 69 " @classmethod\n def tearDownClass(cls):\n" 70 f" {expression}.__code__ = cls.original\n" 71 ) 72 return source.replace(" def setUp(self):", fixture + " def setUp(self):") 73 fixture = "\ndef setUpModule():\n global original\n " + action + "\n" 74 if restored == "cleanup": 75 fixture += ( 76 " unittest.addModuleCleanup(setattr, function, '__code__', original)\n" 77 ) 78 elif restored == "teardown": 79 fixture += f"def tearDownModule():\n {expression}.__code__ = original\n" 80 return source + fixture 81 82 83 class BootstrapTests(unittest.TestCase): 84 setUp = support.StandaloneReportTests.setUp 85 pid_absent = staticmethod(support.StandaloneReportTests.pid_absent) 86 run_report = support.StandaloneReportTests.run_report 87 88 def probe_record(self, arguments, modules): 89 self.last_record = support.StandaloneReportTests.probe_record( 90 self, arguments, modules 91 ) 92 if self.last_record is not None: 93 shutil.copytree( 94 self.root / "scripts", 95 self.last_record / "scripts_before", 96 symlinks=True, 97 ) 98 return self.last_record 99 100 def run_process(self, modules, output=None): 101 result = support.StandaloneReportTests.run_process(self, modules, output) 102 if self.last_record is not None: 103 shutil.copytree( 104 self.root / "scripts", self.last_record / "scripts_after", symlinks=True 105 ) 106 for marker in self.root.glob("*.executed"): 107 shutil.copy2(marker, self.last_record / marker.name) 108 return result 109 110 def prepare(self, source, *, nested=False, initializer=None): 111 self.output.unlink(missing_ok=True) 112 for marker in self.root.glob("*.executed"): 113 marker.unlink() 114 root_init = self.root / "scripts/__init__.py" 115 root_init.unlink(missing_ok=True) 116 if initializer is not None: 117 root_init.write_text(initializer) 118 name = "scripts.controls.test_actual" if nested else "scripts.test_actual" 119 path = self.root.joinpath(*name.split(".")).with_suffix(".py") 120 path.parent.mkdir(parents=True, exist_ok=True) 121 path.write_text("import unittest\n" + source) 122 return name 123 124 def test_namespace_empty_and_ordinary_root_packages_execute_actual_body(self): 125 ordinary = ( 126 "from pathlib import Path\n" 127 "from . import unittest_report, unittest_report_origin\n" 128 "Path('initializer.executed').write_text('actual initializer')\n" 129 ) 130 for nested in (False, True): 131 for initializer in (None, "", ordinary): 132 with self.subTest(nested=nested, initializer=initializer): 133 module = self.prepare( 134 BODY.replace("BODY", "self.assertTrue(True)"), 135 nested=nested, 136 initializer=initializer, 137 ) 138 result, document = self.run_report([module]) 139 self.assertEqual(result.returncode, 0, result.stderr) 140 expected = module + ".ActualTests.test_authored_body" 141 self.assertEqual(document["test_ids"], [expected]) 142 for name in ("setup", "body", "teardown"): 143 self.assertEqual( 144 (self.root / (name + ".executed")).read_text(), expected 145 ) 146 if initializer: 147 self.assertTrue((self.root / "initializer.executed").is_file()) 148 149 def test_root_initializer_self_mutation_and_forged_run_are_rejected(self): 150 controls = ( 151 FORGED_RUN, 152 "from pathlib import Path\nPath(__file__).write_text('# mutated\\n')\n", 153 "from pathlib import Path\nPath(__file__).chmod(0o600)\n", 154 ) 155 for nested in (False, True): 156 for initializer in controls: 157 with self.subTest(nested=nested, initializer=initializer): 158 module = self.prepare( 159 BODY.replace("BODY", "self.fail('actual failure')"), 160 nested=nested, 161 initializer=initializer, 162 ) 163 result = self.run_process([module]) 164 self.assertNotEqual(result.returncode, 0) 165 self.assertFalse((self.root / "body.executed").exists()) 166 167 def test_class_and_module_fixture_code_cannot_skip_inventoried_body(self): 168 for nested in (False, True): 169 for kind in ("class", "module"): 170 for restored in (None, "cleanup", "teardown"): 171 for target in ("body", "dispatch"): 172 with self.subTest( 173 nested=nested, kind=kind, restored=restored, target=target 174 ): 175 module = self.prepare( 176 fixture_source(kind, restored, target), nested=nested 177 ) 178 result, document = self.run_report([module]) 179 self.assertNotEqual(result.returncode, 0) 180 self.assertFalse((self.root / "body.executed").exists()) 181 self.assertFalse( 182 document 183 and any( 184 event["callback"] == "addSuccess" 185 for event in document["callbacks"] 186 ) 187 ) 188 189 def test_genuine_body_failure_keeps_actual_setup_body_and_teardown(self): 190 for nested in (False, True): 191 with self.subTest(nested=nested): 192 module = self.prepare( 193 BODY.replace("BODY", "self.fail('actual authored failure')"), 194 nested=nested, 195 ) 196 result, document = self.run_report([module]) 197 self.assertNotEqual(result.returncode, 0) 198 self.assertTrue((self.root / "body.executed").exists()) 199 self.assertTrue((self.root / "teardown.executed").exists()) 200 self.assertIn( 201 "addSubTest", [e["callback"] for e in document["callbacks"]] 202 ) 203 204 def test_live_legacy_module_entry_requires_trusted_direct_entry(self): 205 module = self.prepare(BODY.replace("BODY", "self.assertTrue(True)")) 206 self.report_entry = ["-m", "scripts.unittest_report"] 207 result = self.run_process([module]) 208 self.assertNotEqual(result.returncode, 0, result.stderr) 209 self.assertFalse((self.root / "body.executed").exists()) 210 self.assertFalse(self.output.exists() and self.output.stat().st_size) 211 self.assertNotIn("actual authored failure", result.stderr) 212 self.assertIn("direct file entry", result.stderr) 213 214 def test_bootstrap_helper_failure_has_static_safe_diagnostics(self): 215 for source in ( 216 "P139S_SYNTHETIC_DIAGNOSTIC_MARKER = '\n", 217 "raise RuntimeError('P139S_SYNTHETIC_DIAGNOSTIC_MARKER')\n", 218 ): 219 with self.subTest(source=source): 220 module = self.prepare(BODY.replace("BODY", "self.assertTrue(True)")) 221 (self.root / "scripts/unittest_report_origin.py").write_text(source) 222 result = self.run_process([module]) 223 self.assertNotEqual(result.returncode, 0) 224 self.assertEqual(result.stdout, "") 225 self.assertEqual( 226 result.stderr, 227 "unittest-report: bootstrap source or origin rejected\n", 228 ) 229 self.assertFalse((self.root / "body.executed").exists()) 230 self.assertFalse(self.output.exists()) 231 232 def test_local_stdlib_shadow_cannot_skip_actual_authored_failure(self): 233 for nested in (False, True): 234 for dependency in ("unittest", "hashlib", "inspect", "json"): 235 with self.subTest(nested=nested, dependency=dependency): 236 module = self.prepare( 237 BODY.replace("BODY", "self.fail('actual authored failure')"), 238 nested=nested, 239 ) 240 (self.root / "scripts" / (dependency + ".py")).write_text( 241 STDLIB_SHADOW 242 ) 243 result, document = self.run_report([module]) 244 self.assertNotEqual(result.returncode, 0) 245 self.assertTrue((self.root / "body.executed").is_file()) 246 self.assertFalse( 247 any( 248 e["callback"] == "addSuccess" for e in document["callbacks"] 249 ) 250 ) 251 252 def test_library_preload_accepts_real_pyexpat_children_and_rejects_forgery(self): 253 for forged in (None, "alias", "parent_source"): 254 with self.subTest(forged=forged): 255 module = self.prepare(BODY.replace("BODY", "self.assertTrue(True)")) 256 source = "import xml.parsers.expat, sys, types; " 257 if forged == "alias": 258 source += "sys.modules['pyexpat.errors'] = types.ModuleType('pyexpat.errors'); " 259 elif forged == "parent_source": 260 source += "sys.modules['pyexpat'].__spec__.origin = 'foreign.py'; " 261 source += "import scripts.unittest_report" 262 self.report_entry = ["-c", source] 263 result = self.run_process([module]) 264 self.assertEqual(result.returncode == 0, forged is None, result.stderr) 265 self.assertFalse(self.output.exists()) 266 self.assertFalse((self.root / "body.executed").exists()) 267 268 269 if __name__ == "__main__": 270 unittest.main()