field_ios

In-the-field app for Radroots on iOS
git clone https://radroots.dev/git/field_ios.git
Log | Files | Refs | README | LICENSE

test_unittest_report_nested.py (36175B)


      1 """Actual nested unittest execution, owned ancestry and adversarial controls."""
      2 
      3 from __future__ import annotations
      4 
      5 import json
      6 import os
      7 import shutil
      8 import stat
      9 import tempfile
     10 import unittest
     11 from pathlib import Path
     12 from unittest.mock import patch
     13 
     14 from scripts import test_unittest_report as support
     15 
     16 PASS = """import unittest
     17 from pathlib import Path
     18 class NestedTests(unittest.TestCase):
     19     def test_authored_nested_method_executes(self):
     20         Path(__name__ + '.executed').write_text(self.id())
     21         with self.subTest(index=1):
     22             self.assertEqual(2 + 2, 4)
     23 """
     24 
     25 
     26 class NestedReportTests(unittest.TestCase):
     27     run_report = support.StandaloneReportTests.run_report
     28     pid_absent = staticmethod(support.StandaloneReportTests.pid_absent)
     29 
     30     def setUp(self):
     31         support.StandaloneReportTests.setUp(self)
     32         self.authored_sources = {}
     33 
     34     def run_process(self, modules, output=None):
     35         result = support.StandaloneReportTests.run_process(self, modules, output)
     36         record = getattr(self, "last_probe_record", None)
     37         if record:
     38             for marker in self.root.glob("*.executed"):
     39                 shutil.copyfile(marker, record / marker.name)
     40         return result
     41 
     42     def module(self, source=PASS, name="controls.test_nested", *, initializers=False):
     43         relative = Path("scripts", *name.split(".")).with_suffix(".py")
     44         path = self.root / relative
     45         path.parent.mkdir(parents=True, exist_ok=True)
     46         if initializers:
     47             for directory in [self.root / "scripts", *path.parents]:
     48                 if directory.is_relative_to(self.root / "scripts"):
     49                     (directory / "__init__.py").write_bytes(b"")
     50         path.write_text(source)
     51         self.authored_sources[path] = path.read_bytes()
     52         return "scripts." + name
     53 
     54     def probe_record(self, arguments, modules):
     55         del modules
     56         evidence = os.environ.get("TERA_REPORT_TEST_EVIDENCE")
     57         if not evidence:
     58             return None
     59         record = Path(tempfile.mkdtemp(prefix="nested-probe-", dir=evidence))
     60         self.last_probe_record = record
     61         (record / "argv.json").write_text(json.dumps(arguments))
     62         (record / "cwd.txt").write_text(str(self.root))
     63         (record / "test_id.txt").write_text(self.id())
     64         shapes = []
     65         for path in (self.root / "scripts").rglob("*.py"):
     66             relative = path.relative_to(self.root)
     67             row = {"path": str(relative), "mode": path.lstat().st_mode}
     68             shapes.append(row)
     69             self.copy_fixture(path, record / relative, row)
     70         (record / "source-shapes.json").write_text(json.dumps(shapes))
     71         return record
     72 
     73     def copy_fixture(self, path, output, row):
     74         output.parent.mkdir(parents=True, exist_ok=True)
     75         if path.is_symlink():
     76             row["symlink_target"] = os.readlink(path)
     77         elif stat.S_ISREG(path.lstat().st_mode):
     78             try:
     79                 output.write_bytes(path.read_bytes())
     80             except PermissionError:
     81                 row["unreadable"] = True
     82                 if path in self.authored_sources:
     83                     output.write_bytes(self.authored_sources[path])
     84 
     85     def assert_execution(self, module, document):
     86         identity = module + ".NestedTests.test_authored_nested_method_executes"
     87         self.assertIn(identity, document["test_ids"])
     88         self.assertEqual((self.root / (module + ".executed")).read_text(), identity)
     89 
     90     def test_namespace_nested_authored_test_really_executes(self):
     91         module = self.module()
     92         result, document = self.run_report([module])
     93         self.assertEqual(result.returncode, 0, result.stderr)
     94         self.assertEqual(document["schema"], "tera.unittest-callback-report.v2")
     95         self.assert_execution(module, document)
     96 
     97     def test_empty_initializers_and_mixed_flat_nested_selections_execute(self):
     98         modules = [
     99             self.module(initializers=True, name="controls.deep.test_nested"),
    100             self.module(name="test_flat"),
    101         ]
    102         result, document = self.run_report(modules)
    103         self.assertEqual(result.returncode, 0, result.stderr)
    104         self.assertEqual(document["schema"], "tera.unittest-callback-report.v2")
    105         self.assertEqual(document["tests_run"], 2)
    106         for module in modules:
    107             self.assert_execution(module, document)
    108 
    109     def rejected(self, modules):
    110         self.output.unlink(missing_ok=True)
    111         result, document = self.run_report(modules)
    112         self.assertNotEqual(result.returncode, 0)
    113         return document
    114 
    115     def test_flat_only_nested_fixture_retains_v1(self):
    116         module = self.module(name="test_flat")
    117         result, document = self.run_report([module])
    118         self.assertEqual(result.returncode, 0, result.stderr)
    119         self.assertEqual(document["schema"], "tera.unittest-callback-report.v1")
    120         self.assert_execution(module, document)
    121 
    122     def test_nested_selector_shape_bounds_and_duplicates_fail_closed(self):
    123         module = self.module()
    124         for names in [
    125             [module, module],
    126             [module] * 4097,
    127             ["scripts.controls.test_"],
    128             ["scripts.controls.9bad.test_nested"],
    129             ["scripts..controls.test_nested"],
    130             ["scripts.controls.test_nested.method"],
    131             ["scripts.controls.test_" + "x" * 256],
    132             ["scripts/controls/test_nested.py"],
    133         ]:
    134             with self.subTest(selector=names[0]):
    135                 self.rejected(names)
    136 
    137     def test_symlinked_package_directory_is_rejected(self):
    138         module = self.module()
    139         path = self.root / "scripts/controls"
    140         path.rename(self.root / "foreign")
    141         path.symlink_to(self.root / "foreign", target_is_directory=True)
    142         self.rejected([module])
    143         self.assertFalse((self.root / (module + ".executed")).exists())
    144 
    145     def test_symlinked_initializer_and_selected_module_are_rejected(self):
    146         module = self.module(initializers=True)
    147         for name in ["__init__.py", "test_nested.py"]:
    148             with self.subTest(name=name):
    149                 path = self.root / "scripts/controls" / name
    150                 before = path.read_bytes()
    151                 foreign = self.root / "foreign.py"
    152                 foreign.write_bytes(before)
    153                 path.unlink()
    154                 path.symlink_to(foreign)
    155                 self.rejected([module])
    156                 path.unlink()
    157                 path.write_bytes(before)
    158 
    159     def test_missing_unreadable_and_fifo_selected_source_is_rejected(self):
    160         module = self.module()
    161         path = self.root / "scripts/controls/test_nested.py"
    162         before = path.read_bytes()
    163         path.unlink()
    164         self.rejected([module])
    165         os.mkfifo(path)
    166         self.rejected([module])
    167         path.unlink()
    168         path.write_bytes(before)
    169         path.chmod(0)
    170         try:
    171             self.rejected([module])
    172         finally:
    173             path.chmod(0o600)
    174 
    175     def test_initializer_changes_during_import_never_run_selected_test(self):
    176         module = self.module(initializers=True)
    177         path = self.root / "scripts/controls/__init__.py"
    178         for action in [
    179             "Path(__file__).write_bytes(b'')",
    180             "Path('scripts/controls/test_nested.py').write_text('pass\\n')",
    181             "__path__ = [str(Path.cwd() / 'foreign')]",
    182             "__spec__.origin = str(Path.cwd() / 'foreign.py')",
    183         ]:
    184             with self.subTest(action=action):
    185                 self.module(initializers=True)
    186                 path.write_text("from pathlib import Path\n" + action + "\n")
    187                 self.rejected([module])
    188                 self.assertFalse((self.root / (module + ".executed")).exists())
    189 
    190     def test_ancestry_source_presence_mode_and_identity_drift_is_rejected(self):
    191         for action in [
    192             "Path('scripts/controls/__init__.py').write_text('new = True\\n')",
    193             "Path('scripts/controls/__init__.py').chmod(0o400)",
    194             "Path('scripts/controls').rename('moved'); Path('scripts/controls').mkdir()",
    195             "Path(__file__).write_text('pass\\n')",
    196             "sys.modules['scripts.controls'].__path__ = ['foreign']",
    197             "sys.modules['scripts.controls'].__spec__.origin = 'foreign.py'",
    198             "sys.modules[__name__] = None",
    199             "globals()['__name__'] = 'foreign.test_drift'",
    200             "__spec__.name = 'foreign.test_drift'",
    201             "__spec__.loader = None",
    202         ]:
    203             with self.subTest(action=action):
    204                 source = "import sys\n" + PASS.replace(
    205                     "        with self.subTest(index=1):",
    206                     "        " + action + "\n        with self.subTest(index=1):",
    207                 )
    208                 module = self.module(
    209                     source, name="controls.test_drift", initializers=True
    210                 )
    211                 self.rejected([module])
    212                 shutil.rmtree(self.root / "scripts/controls")
    213 
    214     def test_empty_initializer_added_during_execution_is_rejected(self):
    215         module = self.module(
    216             PASS.replace(
    217                 "        with self.subTest(index=1):",
    218                 "        Path('scripts/controls/__init__.py').touch()\n        with self.subTest(index=1):",
    219             )
    220         )
    221         self.rejected([module])
    222 
    223     def test_owned_initializer_executes_source_instead_of_forged_bytecode(self):
    224         import importlib._bootstrap_external
    225         import importlib.util
    226 
    227         module = self.module(initializers=True)
    228         initializer = self.root / "scripts/controls/__init__.py"
    229         initializer.write_text(
    230             "from pathlib import Path\nPath('initializer-executed').write_text('actual source')\n"
    231         )
    232         cached = Path(importlib.util.cache_from_source(str(initializer)))
    233         cached.parent.mkdir()
    234         forged = compile("raise SystemExit(0)", str(initializer), "exec")
    235         cached.write_bytes(
    236             importlib._bootstrap_external._code_to_timestamp_pyc(
    237                 forged,
    238                 int(initializer.stat().st_mtime),
    239                 initializer.stat().st_size,
    240             )
    241         )
    242         result, document = self.run_report([module])
    243         self.assertEqual(result.returncode, 0, result.stderr)
    244         self.assertEqual(
    245             (self.root / "initializer-executed").read_text(), "actual source"
    246         )
    247         self.assert_execution(module, document)
    248 
    249     def test_foreign_package_shadowing_is_rejected_before_foreign_import(self):
    250         module = self.module()
    251         foreign = self.root / "foreign"
    252         foreign.mkdir()
    253         (foreign / "test_other.py").write_text(
    254             "from pathlib import Path\nPath('foreign-executed').touch()\n"
    255         )
    256         first = self.module(
    257             "import scripts\nfrom pathlib import Path\nscripts.__path__ = [str(Path.cwd() / 'foreign')]\n"
    258             + PASS,
    259             name="test_first",
    260         )
    261         self.rejected([first, module])
    262         self.assertFalse((self.root / "foreign-executed").exists())
    263 
    264     def test_nested_partial_nonpassing_duplicate_and_fabricated_lifecycles_reject(self):
    265         for source in [
    266             PASS.replace(
    267                 "self.assertEqual(2 + 2, 4)", "self.skipTest('private reason')"
    268             ),
    269             PASS.replace("self.assertEqual(2 + 2, 4)", "self.fail('private reason')"),
    270             PASS.replace(
    271                 "self.assertEqual(2 + 2, 4)", "self._outcome.result.startTestRun()"
    272             ),
    273             PASS
    274             + "\ndef load_tests(loader, tests, pattern):\n    return unittest.TestSuite([NestedTests('test_authored_nested_method_executes')] * 2)\n",
    275             PASS
    276             + "\n    def id(self): return 'scripts.controls.test_nested.NestedTests.forged'\n",
    277             PASS
    278             + "\n    def run(self, result):\n        result.startTest(self)\n        result.addSuccess(self)\n        result.stopTest(self)\n",
    279             PASS.replace("self.assertEqual(2 + 2, 4)", "self._outcome.result.stop()")
    280             + "\n    def test_second(self): pass\n",
    281         ]:
    282             with self.subTest(source=source):
    283                 self.rejected([self.module(source)])
    284 
    285     def test_monkey_patched_stdlib_testcase_identity_is_rejected(self):
    286         module = self.module(
    287             PASS
    288             + "\nunittest.TestCase.id = lambda self: 'scripts.controls.test_nested.NestedTests.forged'\n"
    289         )
    290         self.rejected([module])
    291 
    292     def test_root_initializer_cannot_prepatch_a_false_testcase_identity(self):
    293         module = self.module(initializers=True)
    294         (self.root / "scripts/__init__.py").write_text(
    295             "import unittest\nunittest.TestCase.id = lambda self: "
    296             "'scripts.controls.test_nested.NestedTests.forged'\n"
    297         )
    298         self.rejected([module])
    299 
    300     def test_monkey_patched_observer_cannot_forge_successful_execution(self):
    301         source = "import sys\n" + PASS.replace(
    302             "self.assertEqual(2 + 2, 4)",
    303             "sys.modules['__main__'].CallbackResult.passing = lambda result: True\n"
    304             "            self.fail('private failure')",
    305         )
    306         self.rejected([self.module(source)])
    307 
    308     def test_instance_callback_shadows_cannot_convert_real_nonpassing_tests(self):
    309         controls = [
    310             (
    311                 "result.addFailure = lambda test, err: result.addSuccess(test)",
    312                 "self.fail('private failure')",
    313             ),
    314             (
    315                 "result.__dict__['addFailure'] = lambda test, err: result.addSuccess(test)",
    316                 "self.fail('private failure')",
    317             ),
    318             (
    319                 "object.__setattr__(result, 'addFailure', lambda test, err: result.addSuccess(test))",
    320                 "self.fail('private failure')",
    321             ),
    322             (
    323                 "result.addError = lambda test, err: result.addSuccess(test)",
    324                 "raise RuntimeError('private error')",
    325             ),
    326             (
    327                 "result.addSkip = lambda test, reason: result.addSuccess(test)",
    328                 "self.skipTest('private skip')",
    329             ),
    330             (
    331                 "result.addExpectedFailure = lambda test, err: result.addSuccess(test)",
    332                 "self.fail('private expected failure')",
    333             ),
    334             (
    335                 "result.addUnexpectedSuccess = lambda test: result.addSuccess(test)",
    336                 "self.assertTrue(True)",
    337             ),
    338             ("result.passing = lambda: True", "self.fail('private failure')"),
    339             (
    340                 "result.addSubTest = lambda test, subtest, err: result.addSuccess(test)",
    341                 "with self.subTest(index=1): self.fail('private subtest failure')",
    342             ),
    343         ]
    344         controls.extend(support.observer_code_mutations("self.fail('private failure')"))
    345         for name in ["test_observer", "controls.test_observer"]:
    346             for action, outcome in controls:
    347                 with self.subTest(module=name, action=action):
    348                     source = PASS.replace(
    349                         "        with self.subTest(index=1):\n            self.assertEqual(2 + 2, 4)",
    350                         "        result = self._outcome.result\n        "
    351                         + action
    352                         + "\n        "
    353                         + outcome,
    354                     )
    355                     if (
    356                         "addExpectedFailure" in action
    357                         or "addUnexpectedSuccess" in action
    358                     ):
    359                         source = source.replace(
    360                             "    def test_authored",
    361                             "    @unittest.expectedFailure\n    def test_authored",
    362                         )
    363                     module = self.module(source, name=name)
    364                     document = self.rejected([module])
    365                     self.assert_execution(module, document)
    366                     if "function = type(result).addFailure" in action:
    367                         self.assertEqual(
    368                             (self.root / (module + ".failure.executed")).read_text(),
    369                             document["test_ids"][0],
    370                         )
    371                     self.assertNotIn(
    372                         "addSuccess",
    373                         [event["callback"] for event in document["callbacks"]],
    374                     )
    375 
    376     def test_restored_instance_callback_shadows_remain_rejected(self):
    377         for name in ["test_observer", "controls.test_observer"]:
    378             for action in [
    379                 "result.addFailure = forged",
    380                 "result.__dict__['addFailure'] = forged",
    381                 "object.__setattr__(result, 'addFailure', forged)",
    382             ]:
    383                 with self.subTest(module=name, action=action):
    384                     source = PASS.replace(
    385                         "        with self.subTest(index=1):\n            self.assertEqual(2 + 2, 4)",
    386                         """        result = self._outcome.result
    387         original = result.addFailure
    388         def forged(test, err):
    389             result.__dict__.pop('addFailure', None)
    390             result.addFailure = original
    391             result.addSuccess(test)
    392         """
    393                         + action
    394                         + "\n        self.fail('private failure')",
    395                     )
    396                     module = self.module(source, name=name)
    397                     document = self.rejected([module])
    398                     self.assert_execution(module, document)
    399 
    400     def test_unused_assignment_restored_before_callback_is_still_rejected(self):
    401         for name in ["test_observer", "controls.test_observer"]:
    402             with self.subTest(module=name):
    403                 source = PASS.replace(
    404                     "        with self.subTest(index=1):",
    405                     "        result = self._outcome.result\n"
    406                     "        original = result.addFailure\n"
    407                     "        result.addFailure = lambda test, err: result.addSuccess(test)\n"
    408                     "        result.addFailure = original\n"
    409                     "        result.rejected = False\n"
    410                     "        with self.subTest(index=1):",
    411                 )
    412                 module = self.module(source, name=name)
    413                 document = self.rejected([module])
    414                 self.assert_execution(module, document)
    415 
    416     def test_unused_instance_dictionary_callback_shadow_is_rejected(self):
    417         for name in ["test_observer", "controls.test_observer"]:
    418             for method in ["addFailure", "addSkip", "stop", "fixture_id"]:
    419                 with self.subTest(module=name, method=method):
    420                     source = PASS.replace(
    421                         "        with self.subTest(index=1):",
    422                         "        self._outcome.result.__dict__['"
    423                         + method
    424                         + "'] = lambda *args: None\n"
    425                         "        with self.subTest(index=1):",
    426                     )
    427                     module = self.module(source, name=name)
    428                     document = self.rejected([module])
    429                     self.assert_execution(module, document)
    430 
    431     def test_observed_failure_and_callback_journal_cannot_be_erased(self):
    432         resets = [
    433             "result.nonpassing = False",
    434             "result.__dict__['nonpassing'] = False",
    435             "object.__setattr__(result, 'nonpassing', False)",
    436         ]
    437         edits = [
    438             "result.callbacks[:] = filtered",
    439             "result.callbacks = filtered",
    440             "result.__dict__['callbacks'] = filtered",
    441             "object.__setattr__(result, 'callbacks', filtered)",
    442             "result.callbacks.clear()",
    443             "result.callbacks.pop(2)",
    444             "del result.callbacks[2]",
    445             "result.callbacks[2]['successful'] = True",
    446         ]
    447         for name in ["test_state", "controls.test_state"]:
    448             for reset in resets:
    449                 for edit in edits:
    450                     with self.subTest(module=name, reset=reset, edit=edit):
    451                         source = PASS.replace(
    452                             "        with self.subTest(index=1):\n            self.assertEqual(2 + 2, 4)",
    453                             "        result = self._outcome.result\n"
    454                             "        with self.subTest(index=1): self.fail('private failed subtest')\n"
    455                             "        " + reset + "\n"
    456                             "        filtered = [event for event in result.callbacks if event['callback'] != 'addSubTest']\n"
    457                             "        try:\n            " + edit + "\n"
    458                             "        except (AttributeError, TypeError): pass\n"
    459                             "        self._outcome.success = True",
    460                         )
    461                         module = self.module(source, name=name)
    462                         document = self.rejected([module])
    463                         self.assert_execution(module, document)
    464                         self.assertTrue(
    465                             any(
    466                                 event["callback"] == "addSubTest"
    467                                 and not event["successful"]
    468                                 for event in document["callbacks"]
    469                             )
    470                         )
    471 
    472     def test_observer_descriptor_getter_and_setter_code_are_fenced(self):
    473         for name in ["test_descriptor", "controls.test_descriptor"]:
    474             for mode in ["getter", "setter", "journal_getters"]:
    475                 with self.subTest(module=name, mode=mode):
    476                     changed = {
    477                         "getter": "owner.nonpassing.fget.__code__ = false_getter.__code__",
    478                         "setter": "owner.nonpassing.fset.__code__ = reset_setter.__code__",
    479                         "journal_getters": "owner.nonpassing.fget.__code__ = false_getter.__code__; owner.callbacks.fget.__code__ = filtered_getter.__code__",
    480                     }[mode]
    481                     reset = "result.nonpassing = False" if mode == "setter" else "pass"
    482                     body = (
    483                         """        result = self._outcome.result
    484         with self.subTest(index=1): self.fail('private failed subtest')
    485         owner = type(result)
    486         def replacements(facts):
    487             def false_getter(result): return False if facts else False
    488             def reset_setter(result, value): facts[result]['nonpassing'] = bool(value)
    489             def filtered_getter(result):
    490                 return tuple(event for event in facts[result]['callbacks'] if event['callback'] != 'addSubTest')
    491             return false_getter, reset_setter, filtered_getter
    492         false_getter, reset_setter, filtered_getter = replacements(None)
    493         """
    494                         + changed
    495                         + "\n        "
    496                         + reset
    497                         + "\n        self._outcome.success = True"
    498                     )
    499                     source = PASS.replace(
    500                         "        with self.subTest(index=1):\n            self.assertEqual(2 + 2, 4)",
    501                         body,
    502                     )
    503                     module = self.module(source, name=name)
    504                     document = self.rejected([module])
    505                     self.assert_execution(module, document)
    506 
    507     def test_instance_run_shadow_cannot_fabricate_unexecuted_test_callbacks(self):
    508         for name in ["test_execution", "controls.test_execution"]:
    509             for target in ["case", "suite", "method", "dispatch", "suite_call"]:
    510                 with self.subTest(module=name, target=target):
    511                     source = (
    512                         PASS
    513                         + """
    514 def load_tests(loader, tests, pattern):
    515     case = NestedTests('test_authored_nested_method_executes')
    516     suite = unittest.TestSuite([case])
    517     def fabricated(result):
    518         result.startTest(case)
    519         result.addSuccess(case)
    520         result.stopTest(case)
    521     """
    522                         + {
    523                             "case": "case.run = fabricated",
    524                             "suite": "suite.run = fabricated",
    525                             "method": "setattr(case, case._testMethodName, lambda: None)",
    526                             "dispatch": "case._callTestMethod = lambda method: None",
    527                             "suite_call": "class ForgedSuite(unittest.TestSuite):\n        def __call__(self, result): fabricated(result)\n    suite = ForgedSuite([case])",
    528                         }[target]
    529                         + "\n    return suite\n"
    530                     )
    531                     module = self.module(source, name=name)
    532                     self.rejected([module])
    533                     self.assertFalse((self.root / (module + ".executed")).exists())
    534 
    535     def test_setup_dispatch_shadow_cannot_skip_authored_failure(self):
    536         for name in ["test_dispatch", "controls.test_dispatch"]:
    537             for target, assignment in [
    538                 ("self._callTestMethod", "self._callTestMethod = lambda method: None"),
    539                 (
    540                     "self._callTestMethod",
    541                     "self.__dict__['_callTestMethod'] = lambda method: None",
    542                 ),
    543                 (
    544                     "type(self).test_authored_failure.__code__",
    545                     "type(self).test_authored_failure.__code__ = (lambda self: None).__code__",
    546                 ),
    547                 (
    548                     "unittest.TestCase._callTestMethod.__code__",
    549                     "unittest.TestCase._callTestMethod.__code__ = (lambda self, method: None).__code__",
    550                 ),
    551             ]:
    552                 for restored in [False, True]:
    553                     with self.subTest(
    554                         module=name, assignment=assignment, restored=restored
    555                     ):
    556                         source = (
    557                             """import unittest
    558 from pathlib import Path
    559 class DispatchTests(unittest.TestCase):
    560     def setUp(self):
    561         Path(__name__ + '.setup.executed').write_text(self.id())
    562         self.addCleanup(lambda: Path(__name__ + '.cleanup.executed').write_text(self.id()))
    563         self.original = ORIGINAL
    564         ASSIGNMENT
    565     def tearDown(self):
    566         RESTORE
    567         Path(__name__ + '.teardown.executed').write_text(self.id())
    568     def test_authored_failure(self):
    569         Path(__name__ + '.body.executed').write_text(self.id())
    570         self.fail('actual authored failure')
    571 """.replace("ORIGINAL", target)
    572                             .replace("ASSIGNMENT", assignment)
    573                             .replace(
    574                                 "RESTORE",
    575                                 target + " = self.original" if restored else "pass",
    576                             )
    577                         )
    578                         module = self.module(source, name=name)
    579                         document = self.rejected([module])
    580                         identity = module + ".DispatchTests.test_authored_failure"
    581                         self.assertEqual(
    582                             (self.root / (module + ".setup.executed")).read_text(),
    583                             identity,
    584                         )
    585                         self.assertEqual(
    586                             (self.root / (module + ".cleanup.executed")).read_text(),
    587                             identity,
    588                         )
    589                         self.assertFalse(
    590                             (self.root / (module + ".body.executed")).exists()
    591                         )
    592                         self.assertIn(
    593                             "addError",
    594                             [event["callback"] for event in document["callbacks"]],
    595                         )
    596                         self.assertNotIn(
    597                             "addSuccess",
    598                             [event["callback"] for event in document["callbacks"]],
    599                         )
    600 
    601     def test_normal_setup_body_teardown_and_cleanup_keep_stdlib_outcomes(self):
    602         for name in ["test_fixtures", "controls.test_fixtures"]:
    603             for body in [
    604                 "self.assertTrue(True)",
    605                 "self.fail('authored failure')",
    606                 "self.skipTest('authored skip')",
    607                 "with self.subTest(index=1): self.fail('authored subtest')",
    608             ]:
    609                 with self.subTest(module=name, body=body):
    610                     self.output.unlink(missing_ok=True)
    611                     source = """import unittest
    612 from pathlib import Path
    613 class FixtureTests(unittest.TestCase):
    614     def setUp(self):
    615         Path(__name__ + '.setup.executed').write_text(self.id())
    616         self.addCleanup(lambda: Path(__name__ + '.cleanup.executed').write_text(self.id()))
    617     def tearDown(self):
    618         Path(__name__ + '.teardown.executed').write_text(self.id())
    619     def test_authored_body(self):
    620         Path(__name__ + '.body.executed').write_text(self.id())
    621         BODY
    622 """.replace("BODY", body)
    623                     module = self.module(source, name=name)
    624                     result, document = self.run_report([module])
    625                     self.assertEqual(
    626                         result.returncode == 0,
    627                         body == "self.assertTrue(True)",
    628                         result.stderr,
    629                     )
    630                     identity = module + ".FixtureTests.test_authored_body"
    631                     self.assertEqual(document["test_ids"], [identity])
    632                     for part in ["setup", "body", "teardown", "cleanup"]:
    633                         self.assertEqual(
    634                             (
    635                                 self.root / (module + "." + part + ".executed")
    636                             ).read_text(),
    637                             identity,
    638                         )
    639 
    640     def test_monkey_patched_writer_cannot_export_a_forged_success_report(self):
    641         source = (
    642             "import sys\n"
    643             + PASS
    644             + """
    645         owner = sys.modules['__main__']
    646         original = owner.ReportOutput.write
    647         def changed(output, document):
    648             document['tests_run'] = 0
    649             original(output, document)
    650         owner.ReportOutput.write = changed
    651 """
    652         )
    653         document = self.rejected([self.module(source)])
    654         self.assertEqual(document["tests_run"], 0)
    655 
    656     def test_actual_nested_report_byte_budget_remains_two_million(self):
    657         prefix = "scripts.controls.test_nested.ManyTests.test_"
    658         for others in range(4):
    659             document = support.StandaloneReportTests.report_shape(self, prefix, 0)
    660             for index in range(others):
    661                 other = support.StandaloneReportTests.report_shape(
    662                     self, prefix + "a_" + str(index), 0
    663                 )
    664                 document["test_ids"].insert(index, other["test_ids"][0])
    665                 document["callbacks"][1:1] = other["callbacks"][1:-1]
    666                 document["tests_run"] += 1
    667             baseline = len(json.dumps(document, sort_keys=True, separators=(",", ":")))
    668             padding, remainder = divmod(2_000_000 - baseline, 4)
    669             if remainder == 0:
    670                 break
    671         self.assertEqual(remainder, 0)
    672         for extra in [0, 1]:
    673             with self.subTest(extra=extra):
    674                 self.output.unlink(missing_ok=True)
    675                 source = (
    676                     "import unittest\nclass ManyTests(unittest.TestCase): pass\n"
    677                     "for index in range(" + str(others) + "):\n"
    678                     "    setattr(ManyTests, 'test_a_' + str(index), lambda self: None)\n"
    679                     "setattr(ManyTests, 'test_' + 'x' * "
    680                     + str(padding + extra)
    681                     + ", lambda self: None)\n"
    682                 )
    683                 result, document = self.run_report([self.module(source)])
    684                 self.assertEqual(result.returncode == 0, extra == 0, result.stderr)
    685                 self.assertLessEqual(len(self.output.read_bytes()), 2_000_000)
    686                 if extra == 0:
    687                     self.assertEqual(len(self.output.read_bytes()), 2_000_000)
    688 
    689     def test_selected_code_cannot_change_report_budget_constants(self):
    690         for action in [
    691             "sys.modules['__main__'].MAX_ITEMS = 5000",
    692             "sys.modules['scripts.unittest_report_origin'].MAX_BYTES = 5000000",
    693             "sys.modules['__main__'].ENCODER.sort_keys = False",
    694         ]:
    695             with self.subTest(action=action):
    696                 source = "import sys\n" + PASS.replace(
    697                     "        with self.subTest(index=1):",
    698                     "        " + action + "\n        with self.subTest(index=1):",
    699                 )
    700                 self.rejected([self.module(source)])
    701 
    702     def test_selected_inventory_cannot_omit_empty_or_foreign_nested_modules(self):
    703         first = self.module()
    704         for source in [
    705             "pass\n",
    706             PASS + "\nNestedTests.__module__ = 'foreign.test_other'\n",
    707         ]:
    708             with self.subTest(source=source):
    709                 other = self.module(source, name="other.test_selected")
    710                 self.rejected([first, other])
    711 
    712     def test_nested_fixture_failure_has_actual_redacted_module_identity(self):
    713         module = self.module(
    714             PASS + "\ndef setUpModule(): raise RuntimeError('private fixture error')\n"
    715         )
    716         document = self.rejected([module])
    717         event = next(
    718             row for row in document["callbacks"] if row["callback"] == "addError"
    719         )
    720         self.assertEqual(event["test_id"], "setUpModule (" + module + ")")
    721         self.assertNotIn("private fixture error", json.dumps(document))
    722 
    723     def test_nested_callback_budget_remains_exact(self):
    724         for extra in [0, 1]:
    725             with self.subTest(extra=extra):
    726                 self.output.unlink(missing_ok=True)
    727                 module = self.module(
    728                     "import unittest\nclass NestedTests(unittest.TestCase):\n"
    729                     "    def test_many(self):\n        for _ in range("
    730                     + str(4091 + extra)
    731                     + "):\n            with self.subTest(index=1): pass\n"
    732                 )
    733                 result, document = self.run_report([module])
    734                 self.assertEqual(result.returncode == 0, extra == 0, result.stderr)
    735                 self.assertLessEqual(len(document["callbacks"]), 4096)
    736 
    737     def test_initializer_drift_after_report_write_is_rejected(self):
    738         source = (
    739             "import sys\n"
    740             + PASS
    741             + """
    742         owner = sys.modules['__main__']
    743         original = owner.ReportOutput.write
    744         def changed(output, document):
    745             original(output, document)
    746             Path('scripts/controls/__init__.py').write_bytes(b'changed = True\n')
    747         owner.ReportOutput.write = changed
    748 """.replace("b'changed = True\n'", "b'changed = True\\n'")
    749         )
    750         self.rejected([self.module(source, initializers=True)])
    751 
    752     def test_actual_descriptor_read_race_fences_same_inode_source_bytes(self):
    753         from scripts import unittest_report_origin as origin
    754 
    755         module = self.module()
    756         path = self.root.joinpath(*module.split(".")).with_suffix(".py")
    757         original = os.read
    758         calls = []
    759 
    760         def changed(descriptor, count):
    761             raw = original(descriptor, count)
    762             calls.append(count)
    763             path.write_bytes(b"x" * path.stat().st_size)
    764             return raw
    765 
    766         with patch.object(origin.os, "read", changed):
    767             with self.assertRaises(origin.ReportError):
    768                 origin.source_snapshot(self.root, path.relative_to(self.root))
    769         self.assertTrue(calls)
    770 
    771     def test_actual_hostile_long_testcase_identity_rejects_in_bounded_child(self):
    772         self.process_timeout = 3
    773         source = PASS + "\nNestedTests.__qualname__ = 'test_a.' * 8000 + '!'\n"
    774         self.rejected([self.module(source)])
    775 
    776     def test_actual_hostile_long_fixture_identity_rejects_in_bounded_child(self):
    777         self.process_timeout = 3
    778         source = (
    779             PASS
    780             + """
    781 unittest.suite._ErrorHolder.id = lambda self: 'setUpModule (scripts.' + 'test_a.' * 8000 + '!)'
    782 def setUpModule(): raise RuntimeError('synthetic fixture error')
    783 """
    784         )
    785         self.rejected([self.module(source)])
    786 
    787 
    788 if __name__ == "__main__":
    789     unittest.main()