field_ios

In-the-field app for Radroots on iOS
git clone https://radroots.dev/git/field_ios.git
Log | Files | Refs | README | LICENSE

test_unittest_report.py (36439B)


      1 """Exercise the standalone producer with actual stdlib tests and disposable roots."""
      2 
      3 from __future__ import annotations
      4 
      5 import importlib
      6 import io
      7 import json
      8 import os
      9 import shutil
     10 import subprocess
     11 import sys
     12 import tempfile
     13 import unittest
     14 from pathlib import Path
     15 from unittest.mock import patch
     16 
     17 from scripts.fixture_tool_dispatch import CompilerCommand
     18 from scripts.fixture_test_policy import limit
     19 
     20 SECRET = "private exception, skip reason, and subtest parameter"
     21 SCRIPTS = Path(__file__).resolve().parent
     22 
     23 
     24 def observer_code_mutations(failure):
     25     for table in ["", "executables", "methods"]:
     26         for restored in [False, True]:
     27             action = (
     28                 "function = type(result).addFailure; original = function.__code__; "
     29             )
     30             if restored:
     31                 action += "self.addCleanup(setattr, function, '__code__', original); "
     32             action += "function.__code__ = (lambda self, test, err: self.addSuccess(test)).__code__; "
     33             if table:
     34                 action += (
     35                     "import sys; owner = sys.modules['__main__']; "
     36                     "names = ['CALLBACK_EXECUTABLES', 'CALLBACK_METHODS']; "
     37                     "previous = {name: getattr(owner, name, None) for name in names}; "
     38                 )
     39                 if restored:
     40                     action += "self.addCleanup(lambda: [(setattr(owner, name, value) if value is not None else delattr(owner, name)) for name, value in previous.items() if value is not None or hasattr(owner, name)]); "
     41                 action += (
     42                     "baseline = {**getattr(owner, 'CALLBACK_METHODS', {}), 'addFailure': function}; "
     43                     "owner.CALLBACK_EXECUTABLES = {name: owner.origin.callable_identity(method) for name, method in baseline.items()}; "
     44                 )
     45                 if table == "methods":
     46                     action += "owner.CALLBACK_METHODS = baseline; "
     47             yield (
     48                 action,
     49                 (
     50                     "try: " + failure + "\n        except AssertionError:\n"
     51                     "            Path(__name__ + '.failure.executed').write_text(self.id())\n"
     52                     "            raise"
     53                 ),
     54             )
     55 
     56 
     57 class StandaloneReportTests(unittest.TestCase):
     58     def setUp(self):
     59         temporary = tempfile.TemporaryDirectory(prefix="tera-unittest-report-")
     60         self.addCleanup(temporary.cleanup)
     61         self.root = Path(temporary.name).resolve()
     62         (self.root / "scripts").mkdir()
     63         shutil.copyfile(
     64             SCRIPTS / "unittest_report.py", self.root / "scripts/unittest_report.py"
     65         )
     66         if (SCRIPTS / "unittest_report_origin.py").is_file():
     67             shutil.copyfile(
     68                 SCRIPTS / "unittest_report_origin.py",
     69                 self.root / "scripts/unittest_report_origin.py",
     70             )
     71         self.output = self.root / "report.json"
     72 
     73     def module(self, source, name="test_observed"):
     74         path = self.root / "scripts" / (name + ".py")
     75         path.write_text("import unittest\nSECRET = " + repr(SECRET) + "\n" + source)
     76         return "scripts." + name
     77 
     78     def run_process(self, modules, output=None):
     79         entry = getattr(self, "report_entry", ["scripts/unittest_report.py"])
     80         output_path = str(output or self.output)
     81         arguments = [sys.executable, *entry, "--report", output_path, *modules]
     82         record = self.probe_record(arguments, modules)
     83         command = CompilerCommand(
     84             arguments,
     85             self.root,
     86             getattr(self, "process_timeout", limit("reporter_child")),
     87             env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"},
     88             budget="reporter_child",
     89         )
     90         failure = None
     91         try:
     92             command.execute()
     93         except BaseException as error:
     94             failure = error
     95         process = command.process
     96         if process is None:
     97             raise failure
     98         result = subprocess.CompletedProcess(
     99             arguments,
    100             process.returncode,
    101             bytes(command.output["stdout"]).decode(),
    102             bytes(command.output["stderr"]).decode(),
    103         )
    104         if record is not None:
    105             (record / "capture.json").write_text(json.dumps(command.record, indent=2))
    106             (record / "settlement.json").write_text(
    107                 json.dumps(
    108                     {
    109                         "pid": process.pid,
    110                         "reaped": process.poll() is not None,
    111                         "timed_out": isinstance(failure, subprocess.TimeoutExpired),
    112                         "group_absent": command.record["group_absent"],
    113                         "pid_absent": not Path("/proc", str(process.pid)).exists()
    114                         if sys.platform == "linux"
    115                         else self.pid_absent(process.pid),
    116                     }
    117                 )
    118             )
    119             (record / "exit.txt").write_text(str(result.returncode))
    120             (record / "stdout.txt").write_text(result.stdout)
    121             (record / "stderr.txt").write_text(result.stderr)
    122             raw = (
    123                 self.output.read_bytes()
    124                 if self.output.is_file() and not self.output.is_symlink()
    125                 else b""
    126             )
    127             (record / "report.json").write_bytes(raw)
    128         if failure:
    129             raise failure
    130         return result
    131 
    132     @staticmethod
    133     def pid_absent(pid):
    134         try:
    135             os.kill(pid, 0)
    136         except ProcessLookupError:
    137             return True
    138         return False
    139 
    140     def probe_record(self, arguments, modules):
    141         evidence = os.environ.get("TERA_REPORT_TEST_EVIDENCE")
    142         if not evidence:
    143             return None
    144         record = Path(tempfile.mkdtemp(prefix="probe-", dir=evidence))
    145         (record / "argv.json").write_text(json.dumps(arguments))
    146         (record / "cwd.txt").write_text(str(self.root))
    147         (record / "test_id.txt").write_text(self.id())
    148         (record / "producer.py").write_bytes(
    149             (self.root / "scripts/unittest_report.py").read_bytes()
    150         )
    151         for index, module in enumerate(modules[:4096]):
    152             if len(module) <= 255 and module.startswith("scripts.test_"):
    153                 path = self.root / "scripts" / (module[8:] + ".py")
    154                 if path.is_file() and not path.is_symlink():
    155                     (record / ("module-" + str(index) + ".py")).write_bytes(
    156                         path.read_bytes()
    157                     )
    158         return record
    159 
    160     def run_report(self, modules, output=None):
    161         result = self.run_process(modules, output)
    162         raw = self.output.read_bytes() if self.output.is_file() else b""
    163         document = json.loads(raw) if raw else None
    164         if document is not None:
    165             self.assertNotIn(SECRET, json.dumps(document))
    166             self.assertEqual(
    167                 set(document), {"schema", "test_ids", "tests_run", "callbacks"}
    168             )
    169         return result, document
    170 
    171     def test_genuine_pass_and_successful_subtests_have_one_identity(self):
    172         module = self.module("""
    173 class ObservedTests(unittest.TestCase):
    174     def test_one(self):
    175         for index in range(3):
    176             with self.subTest(private=SECRET, index=index):
    177                 self.assertTrue(True)
    178 """)
    179         result, document = self.run_report([module])
    180         self.assertEqual(result.returncode, 0, result.stderr)
    181         identity = module + ".ObservedTests.test_one"
    182         self.assertEqual(document["test_ids"], [identity])
    183         self.assertEqual(document["tests_run"], 1)
    184         self.assertEqual(
    185             [row["callback"] for row in document["callbacks"]],
    186             [
    187                 "startTestRun",
    188                 "startTest",
    189                 "addSubTest",
    190                 "addSubTest",
    191                 "addSubTest",
    192                 "addSuccess",
    193                 "stopTest",
    194                 "stopTestRun",
    195             ],
    196         )
    197         self.assertEqual(
    198             [
    199                 row["ordinal"]
    200                 for row in document["callbacks"]
    201                 if row["callback"] == "addSubTest"
    202             ],
    203             [1, 2, 3],
    204         )
    205 
    206     def test_actual_nonpassing_test_outcomes_are_observed(self):
    207         modes = {
    208             "failure": ("self.fail(SECRET)", "addFailure"),
    209             "error": ("raise RuntimeError(SECRET)", "addError"),
    210             "skip": ("self.skipTest(SECRET)", "addSkip"),
    211             "cleanup": ("self.addCleanup(lambda: self.fail(SECRET))", "addFailure"),
    212             "cleanup_error": ("self.addCleanup(lambda: int(SECRET))", "addError"),
    213             "expected": ("self.fail(SECRET)", "addExpectedFailure"),
    214             "unexpected": ("pass", "addUnexpectedSuccess"),
    215         }
    216         for mode, (body, callback) in modes.items():
    217             with self.subTest(mode=mode):
    218                 decorator = (
    219                     "    @unittest.expectedFailure\n"
    220                     if mode in {"expected", "unexpected"}
    221                     else ""
    222                 )
    223                 module = self.module(
    224                     "class ObservedTests(unittest.TestCase):\n"
    225                     + decorator
    226                     + "    def test_one(self):\n        "
    227                     + body
    228                     + "\n"
    229                 )
    230                 self.output.unlink(missing_ok=True)
    231                 result, document = self.run_report([module])
    232                 self.assertNotEqual(result.returncode, 0)
    233                 self.assertIn(
    234                     callback, [row["callback"] for row in document["callbacks"]]
    235                 )
    236                 self.assertEqual(document["callbacks"][-1], {"callback": "stopTestRun"})
    237 
    238     def test_actual_subtest_failure_error_and_skip_redact_parameters(self):
    239         for body, callback in [
    240             ("self.fail(SECRET)", "addSubTest"),
    241             ("raise RuntimeError(SECRET)", "addSubTest"),
    242             ("self.skipTest(SECRET)", "addSkip"),
    243         ]:
    244             with self.subTest(callback=callback, body=body):
    245                 module = self.module(
    246                     "class ObservedTests(unittest.TestCase):\n    def test_one(self):\n        with self.subTest(private=SECRET):\n            "
    247                     + body
    248                     + "\n"
    249                 )
    250                 self.output.unlink(missing_ok=True)
    251                 result, document = self.run_report([module])
    252                 self.assertNotEqual(result.returncode, 0)
    253                 event = next(
    254                     row for row in document["callbacks"] if row["callback"] == callback
    255                 )
    256                 self.assertEqual(event["test_id"], module + ".ObservedTests.test_one")
    257                 if callback == "addSubTest":
    258                     self.assertFalse(event["successful"])
    259 
    260     def fixture_source(self, scope, phase, skip=False):
    261         action = (
    262             "raise unittest.SkipTest(SECRET)" if skip else "raise RuntimeError(SECRET)"
    263         )
    264         if scope == "module":
    265             fixture = "def " + phase + "():\n    " + action + "\n"
    266             return (
    267                 fixture
    268                 + "class ObservedTests(unittest.TestCase):\n    def test_one(self): pass\n"
    269             )
    270         return (
    271             "class ObservedTests(unittest.TestCase):\n    @classmethod\n    def "
    272             + phase
    273             + "(cls):\n        "
    274             + action
    275             + "\n    def test_one(self): pass\n"
    276         )
    277 
    278     def test_actual_module_and_class_fixture_errors_and_skips(self):
    279         for scope, phases in [
    280             ("module", ["setUpModule", "tearDownModule"]),
    281             ("class", ["setUpClass", "tearDownClass"]),
    282         ]:
    283             for phase in phases:
    284                 for skip in [False, True]:
    285                     with self.subTest(scope=scope, phase=phase, skip=skip):
    286                         module = self.module(self.fixture_source(scope, phase, skip))
    287                         self.output.unlink(missing_ok=True)
    288                         result, document = self.run_report([module])
    289                         self.assertNotEqual(result.returncode, 0)
    290                         self.assertIn(
    291                             "addSkip" if skip else "addError",
    292                             [row["callback"] for row in document["callbacks"]],
    293                         )
    294                         event = next(
    295                             row
    296                             for row in document["callbacks"]
    297                             if row["callback"] == ("addSkip" if skip else "addError")
    298                         )
    299                         owner = module + (".ObservedTests" if scope == "class" else "")
    300                         self.assertEqual(event["test_id"], phase + " (" + owner + ")")
    301                         self.assertEqual(
    302                             document["callbacks"][-1], {"callback": "stopTestRun"}
    303                         )
    304 
    305     def test_actual_class_and_module_cleanup_errors(self):
    306         for source in [
    307             "def setUpModule():\n    unittest.addModuleCleanup(lambda: int(SECRET))\nclass ObservedTests(unittest.TestCase):\n    def test_one(self): pass\n",
    308             "class ObservedTests(unittest.TestCase):\n    @classmethod\n    def setUpClass(cls): cls.addClassCleanup(lambda: int(SECRET))\n    def test_one(self): pass\n",
    309         ]:
    310             with self.subTest(source=source):
    311                 module = self.module(source)
    312                 self.output.unlink(missing_ok=True)
    313                 result, document = self.run_report([module])
    314                 self.assertNotEqual(result.returncode, 0)
    315                 self.assertIn(
    316                     "addError", [row["callback"] for row in document["callbacks"]]
    317                 )
    318 
    319     def test_actual_early_stop_and_interruption_preserve_partial_inventory(self):
    320         for body in ["self._outcome.result.stop()", "raise KeyboardInterrupt(SECRET)"]:
    321             with self.subTest(body=body):
    322                 module = self.module(
    323                     "class ObservedTests(unittest.TestCase):\n    def test_one(self):\n        "
    324                     + body
    325                     + "\n    def test_two(self): pass\n"
    326                 )
    327                 self.output.unlink(missing_ok=True)
    328                 result, document = self.run_report([module])
    329                 self.assertNotEqual(result.returncode, 0)
    330                 self.assertIsNotNone(document, result.stderr)
    331                 self.assertEqual(len(document["test_ids"]), 2)
    332                 self.assertEqual(document["tests_run"], 1)
    333                 self.assertEqual(document["callbacks"][-1], {"callback": "stopTestRun"})
    334 
    335     def test_multiple_modules_have_real_completed_tests(self):
    336         source = (
    337             "class ObservedTests(unittest.TestCase):\n    def test_one(self): pass\n"
    338         )
    339         modules = [self.module(source), self.module(source, "test_secondary")]
    340         result, document = self.run_report(modules)
    341         self.assertEqual(result.returncode, 0, result.stderr)
    342         self.assertEqual(document["tests_run"], 2)
    343         self.assertEqual(len(document["test_ids"]), 2)
    344 
    345     def test_empty_module_and_duplicate_test_inventory_are_rejected(self):
    346         sources = [
    347             "pass\n",
    348             "class ObservedTests(unittest.TestCase):\n    def test_one(self): pass\ndef load_tests(loader, tests, pattern):\n    return unittest.TestSuite([ObservedTests('test_one'), ObservedTests('test_one')])\n",
    349         ]
    350         for source in sources:
    351             with self.subTest(source=source):
    352                 result, _ = self.run_report([self.module(source)])
    353                 self.assertNotEqual(result.returncode, 0)
    354 
    355     def test_selectors_are_bounded_canonical_unique_modules(self):
    356         module = self.module(
    357             "class ObservedTests(unittest.TestCase):\n    def test_one(self): pass\n"
    358         )
    359         for selectors in [
    360             [module, module],
    361             [],
    362             ["scripts/test_observed.py"],
    363             ["scripts.test_"],
    364             ["foreign.test_observed"],
    365             ["scripts.test_" + "x" * 300],
    366             [module] * 4097,
    367         ]:
    368             with self.subTest(selectors=selectors[:2]):
    369                 result, _ = self.run_report(selectors)
    370                 self.assertNotEqual(result.returncode, 0)
    371 
    372     def test_selected_origins_cannot_be_symlinks_directories_or_foreign_ids(self):
    373         module = self.module(
    374             "class ObservedTests(unittest.TestCase):\n    def test_one(self): pass\n"
    375         )
    376         path = self.root / "scripts/test_observed.py"
    377         foreign = self.root / "foreign.py"
    378         path.rename(foreign)
    379         path.symlink_to(foreign)
    380         result, _ = self.run_report([module])
    381         self.assertNotEqual(result.returncode, 0)
    382         path.unlink()
    383         path.mkdir()
    384         result, _ = self.run_report([module])
    385         self.assertNotEqual(result.returncode, 0)
    386         path.rmdir()
    387         self.module(
    388             "class ObservedTests(unittest.TestCase):\n    def test_one(self): pass\nObservedTests.__module__ = 'foreign.test_observed'\n"
    389         )
    390         result, _ = self.run_report([module])
    391         self.assertNotEqual(result.returncode, 0)
    392 
    393     def test_producer_origin_cannot_be_symlinked(self):
    394         path = self.root / "scripts/unittest_report.py"
    395         foreign = self.root / "foreign.py"
    396         path.rename(foreign)
    397         path.symlink_to(foreign)
    398         result, _ = self.run_report([self.module("pass\n")])
    399         self.assertNotEqual(result.returncode, 0)
    400 
    401     def test_safe_output_is_exclusive_and_rejects_symlink_parents(self):
    402         module = self.module(
    403             "class ObservedTests(unittest.TestCase):\n    def test_one(self): pass\n"
    404         )
    405         self.output.write_bytes(b"preserved")
    406         result = self.run_process([module])
    407         self.assertNotEqual(result.returncode, 0)
    408         self.assertEqual(self.output.read_bytes(), b"preserved")
    409         self.output.unlink()
    410         self.output.symlink_to(self.root / "foreign.json")
    411         result, _ = self.run_report([module])
    412         self.assertNotEqual(result.returncode, 0)
    413         self.output.unlink()
    414         (self.root / "alias").symlink_to(
    415             self.root / "scripts", target_is_directory=True
    416         )
    417         result, _ = self.run_report([module], self.root / "alias/report.json")
    418         self.assertNotEqual(result.returncode, 0)
    419         self.assertFalse((self.root / "scripts/report.json").exists())
    420 
    421     def test_source_and_output_replacement_during_execution_fail_closed(self):
    422         sources = [
    423             "from pathlib import Path\nclass ObservedTests(unittest.TestCase):\n    def test_one(self):\n        Path(__file__).write_text('pass\\n')\n",
    424             "from pathlib import Path\nclass ObservedTests(unittest.TestCase):\n    def test_one(self):\n        Path('report.json').unlink()\n        Path('report.json').write_text('replacement')\n",
    425         ]
    426         for source in sources:
    427             with self.subTest(source=source):
    428                 self.output.unlink(missing_ok=True)
    429                 result = self.run_process([self.module(source)])
    430                 self.assertNotEqual(result.returncode, 0)
    431 
    432     def test_real_same_inode_output_tampering_is_rejected(self):
    433         for action in [
    434             "path.write_bytes(b'PRIVATE_OUTPUT_SENTINEL')",
    435             "path.write_bytes(b'PRIVATE_OUTPUT_SENTINEL' * 120000)",
    436             "path.write_bytes(b'PRIVATE_OUTPUT_SENTINEL'); path.write_bytes(b'')",
    437         ]:
    438             with self.subTest(action=action):
    439                 self.output.unlink(missing_ok=True)
    440                 module = self.module(
    441                     "from pathlib import Path\nclass ObservedTests(unittest.TestCase):\n"
    442                     "    def test_one(self):\n        path = Path('report.json')\n"
    443                     "        before = path.stat().st_ino\n        " + action + "\n"
    444                     "        self.assertEqual(path.stat().st_ino, before)\n"
    445                     "        Path('testcase-passed').write_text('passed')\n"
    446                 )
    447                 result = self.run_process([module])
    448                 self.assertEqual((self.root / "testcase-passed").read_text(), "passed")
    449                 self.assertNotEqual(result.returncode, 0)
    450                 self.assertNotIn("PRIVATE_OUTPUT_SENTINEL", result.stderr)
    451 
    452     def test_output_settlement_rechecks_content_and_position(self):
    453         # Inject a mutation after the real writer returns, before actual close.
    454         for action in [
    455             "os.write(output.descriptor, b'PRIVATE_OUTPUT_SENTINEL')",
    456             "os.ftruncate(output.descriptor, 0)",
    457             "os.pwrite(output.descriptor, b'X', 0)",
    458             "os.lseek(output.descriptor, 0, os.SEEK_SET)",
    459         ]:
    460             with self.subTest(action=action):
    461                 self.output.unlink(missing_ok=True)
    462                 module = self.module(
    463                     "import os, sys\nclass ObservedTests(unittest.TestCase):\n"
    464                     "    def test_one(self):\n"
    465                     "        owner = sys.modules['__main__']\n"
    466                     "        original = owner.ReportOutput.write\n"
    467                     "        def changed(output, document):\n"
    468                     "            original(output, document)\n            "
    469                     + action
    470                     + "\n        owner.ReportOutput.write = changed\n"
    471                     "        from pathlib import Path\n"
    472                     "        Path('testcase-passed').write_text('passed')\n"
    473                 )
    474                 result = self.run_process([module])
    475                 self.assertEqual((self.root / "testcase-passed").read_text(), "passed")
    476                 self.assertNotEqual(result.returncode, 0)
    477                 self.assertNotIn("PRIVATE_OUTPUT_SENTINEL", result.stderr)
    478 
    479     def test_real_callback_limit_is_exact_and_does_not_append_over_bound(self):
    480         for extra in [0, 1]:
    481             with self.subTest(extra=extra):
    482                 module = self.module(
    483                     "class ObservedTests(unittest.TestCase):\n    def test_one(self):\n        for _ in range("
    484                     + str(4091 + extra)
    485                     + "):\n            with self.subTest(private=SECRET): pass\n"
    486                 )
    487                 self.output.unlink(missing_ok=True)
    488                 result, document = self.run_report([module])
    489                 self.assertEqual(result.returncode == 0, extra == 0, result.stderr)
    490                 self.assertLessEqual(len(document["callbacks"]), 4096)
    491                 if extra == 0:
    492                     self.assertEqual(len(document["callbacks"]), 4096)
    493 
    494     def test_real_inventory_limit_is_separate_from_callback_limit(self):
    495         for count in [4096, 4097]:
    496             with self.subTest(count=count):
    497                 module = self.module(
    498                     "class ObservedTests(unittest.TestCase):\n    def test_one(self): pass\ndef load_tests(loader, tests, pattern):\n    case = type('ManyTests', (unittest.TestCase,), {'__module__': __name__, **{f'test_{index}': lambda self: None for index in range("
    499                     + str(count)
    500                     + ")}})\n    return loader.loadTestsFromTestCase(case)\n"
    501                 )
    502                 self.output.unlink(missing_ok=True)
    503                 result, document = self.run_report([module])
    504                 self.assertNotEqual(result.returncode, 0)
    505                 if count == 4096:
    506                     self.assertEqual(len(document["test_ids"]), 4096)
    507                     self.assertLessEqual(len(document["callbacks"]), 4096)
    508 
    509     def report_shape(self, identity, subtests):
    510         callbacks = [
    511             {"callback": "startTestRun"},
    512             {"callback": "startTest", "test_id": identity},
    513         ]
    514         callbacks.extend(
    515             {
    516                 "callback": "addSubTest",
    517                 "test_id": identity,
    518                 "ordinal": index + 1,
    519                 "successful": True,
    520             }
    521             for index in range(subtests)
    522         )
    523         callbacks.extend(
    524             [
    525                 {"callback": "addSuccess", "test_id": identity},
    526                 {"callback": "stopTest", "test_id": identity},
    527                 {"callback": "stopTestRun"},
    528             ]
    529         )
    530         return {
    531             "schema": "tera.unittest-callback-report.v1",
    532             "test_ids": [identity],
    533             "tests_run": 1,
    534             "callbacks": callbacks,
    535         }
    536 
    537     def test_actual_report_accepts_two_million_bytes_and_rejects_overflow(self):
    538         prefix = "scripts.test_observed.ManyTests.test_"
    539         for others in range(4):
    540             document = self.report_shape(prefix, 0)
    541             for index in range(others):
    542                 other = self.report_shape(prefix + "a_" + str(index), 0)
    543                 document["test_ids"].insert(index, other["test_ids"][0])
    544                 document["callbacks"][1:1] = other["callbacks"][1:-1]
    545                 document["tests_run"] += 1
    546             baseline = len(json.dumps(document, sort_keys=True, separators=(",", ":")))
    547             padding, remainder = divmod(2_000_000 - baseline, 4)
    548             if remainder == 0:
    549                 break
    550         self.assertEqual(remainder, 0)
    551         for extra in [0, 1]:
    552             with self.subTest(extra=extra):
    553                 source = (
    554                     "class ManyTests(unittest.TestCase): pass\nfor index in range("
    555                     + str(others)
    556                     + "):\n    setattr(ManyTests, 'test_a_' + str(index), lambda self: None)\nsetattr(ManyTests, 'test_' + 'x' * "
    557                     + str(padding + extra)
    558                     + ", lambda self: None)\n"
    559                 )
    560                 module = self.module(source)
    561                 self.output.unlink(missing_ok=True)
    562                 result, document = self.run_report([module])
    563                 self.assertEqual(result.returncode == 0, extra == 0, result.stderr)
    564                 self.assertLessEqual(len(self.output.read_bytes()), 2_000_000)
    565                 if extra == 0:
    566                     self.assertEqual(len(self.output.read_bytes()), 2_000_000)
    567                     self.assertEqual(document["tests_run"], others + 1)
    568 
    569     def test_partial_testcase_without_success_is_not_accepted(self):
    570         module = self.module(
    571             "class ObservedTests(unittest.TestCase):\n    def test_one(self): pass\n    def run(self, result):\n        result.startTest(self)\n        result.stopTest(self)\n"
    572         )
    573         result, document = self.run_report([module])
    574         self.assertNotEqual(result.returncode, 0)
    575         self.assertEqual(document["tests_run"], 1)
    576         self.assertNotIn(
    577             "addSuccess", [row["callback"] for row in document["callbacks"]]
    578         )
    579 
    580     def test_selected_code_cannot_exit_successfully_during_import(self):
    581         result, _ = self.run_report([self.module("raise SystemExit(0)\n")])
    582         self.assertNotEqual(result.returncode, 0)
    583 
    584     def test_selected_module_does_not_execute_forged_cached_bytecode(self):
    585         module = self.module(
    586             "class ObservedTests(unittest.TestCase):\n    def test_one(self): pass\n"
    587         )
    588         source = self.root / "scripts/test_observed.py"
    589         code = compile("raise SystemExit(0)", str(source), "exec")
    590         import importlib._bootstrap_external
    591 
    592         cached = Path(importlib.util.cache_from_source(str(source)))
    593         cached.parent.mkdir()
    594         cached.write_bytes(
    595             importlib._bootstrap_external._code_to_timestamp_pyc(
    596                 code, int(source.stat().st_mtime), source.stat().st_size
    597             )
    598         )
    599         result, document = self.run_report([module])
    600         self.assertEqual(result.returncode, 0, result.stderr)
    601         self.assertEqual(document["tests_run"], 1)
    602 
    603     def test_foreign_resolution_is_rejected_before_module_execution(self):
    604         (self.root / "foreign").mkdir()
    605         (self.root / "foreign/test_secondary.py").write_text(
    606             "from pathlib import Path\nPath('foreign-executed').write_text('executed')\n"
    607         )
    608         first = self.module(
    609             "import scripts\nfrom pathlib import Path\nscripts.__path__ = [str(Path.cwd() / 'foreign')]\nclass ObservedTests(unittest.TestCase):\n    def test_one(self): pass\n"
    610         )
    611         second = self.module(
    612             "class ObservedTests(unittest.TestCase):\n    def test_one(self): pass\n",
    613             "test_secondary",
    614         )
    615         result, _ = self.run_report([first, second])
    616         self.assertNotEqual(result.returncode, 0)
    617         self.assertFalse((self.root / "foreign-executed").exists())
    618 
    619     def test_output_directory_rename_cannot_redirect_descriptor_writes(self):
    620         module = self.module(
    621             "from pathlib import Path\nclass ObservedTests(unittest.TestCase):\n    def test_one(self):\n        Path('output').rename('moved')\n        Path('output').mkdir()\n        Path('output/report.json').write_text('replacement')\n"
    622         )
    623         (self.root / "output").mkdir()
    624         result = self.run_process([module], self.root / "output/report.json")
    625         self.assertNotEqual(result.returncode, 0)
    626         self.assertEqual((self.root / "output/report.json").read_text(), "replacement")
    627         self.assertEqual((self.root / "moved/report.json").read_bytes(), b"")
    628 
    629     def test_duplicate_run_boundary_and_invalid_test_id_are_nonpassing(self):
    630         for source in [
    631             "class ObservedTests(unittest.TestCase):\n    def test_one(self): self._outcome.result.startTestRun()\n",
    632             "class ObservedTests(unittest.TestCase):\n    def test_one(self): pass\n    def id(self): return SECRET\n",
    633         ]:
    634             with self.subTest(source=source):
    635                 self.output.unlink(missing_ok=True)
    636                 result, _ = self.run_report([self.module(source)])
    637                 self.assertNotEqual(result.returncode, 0)
    638 
    639 
    640 class OutputIntegrityTests(unittest.TestCase):
    641     """Injected timing controls around actual writes of genuine observations."""
    642 
    643     def setUp(self):
    644         self.producer = importlib.import_module("scripts.unittest_report")
    645         temporary = tempfile.TemporaryDirectory(prefix="tera-output-integrity-")
    646         self.addCleanup(temporary.cleanup)
    647         self.root = Path(temporary.name)
    648         case = type(
    649             "ObservedTests",
    650             (unittest.TestCase,),
    651             {"__module__": "scripts.test_observed", "test_one": lambda self: None},
    652         )("test_one")
    653         suite, budget = unittest.TestSuite([case]), self.producer.ReportBudget()
    654         inventory = self.producer.inventory(suite, ["scripts.test_observed"], budget)
    655         result, passing = self.producer.observe(suite, inventory, budget, io.StringIO())
    656         self.assertTrue(passing)
    657         self.document = {
    658             "schema": self.producer.SCHEMA,
    659             "test_ids": inventory,
    660             "tests_run": result.testsRun,
    661             "callbacks": [dict(event) for event in result.callbacks],
    662         }
    663 
    664     def output(self, name):
    665         path = self.root / (name + ".json")
    666         output = self.producer.ReportOutput(str(path))
    667         self.addCleanup(os.close, output.directory)
    668         self.addCleanup(os.close, output.descriptor)
    669         evidence = os.environ.get("TERA_REPORT_TEST_EVIDENCE")
    670         if evidence:
    671             record = Path(
    672                 tempfile.mkdtemp(prefix="injected-" + name + "-", dir=evidence)
    673             )
    674             self.addCleanup(shutil.copyfile, path, record / "report.json")
    675         return output
    676 
    677     def mutate(self, output, mode):
    678         if mode == "append":
    679             os.pwrite(
    680                 output.descriptor, SECRET.encode(), os.fstat(output.descriptor).st_size
    681             )
    682         elif mode == "truncate":
    683             os.ftruncate(output.descriptor, 0)
    684         elif mode == "same_size":
    685             os.pwrite(output.descriptor, b"X", 0)
    686         else:
    687             os.lseek(output.descriptor, 0, os.SEEK_SET)
    688 
    689     def test_actual_write_races_reject_append_truncate_content_and_offset(self):
    690         original = os.write
    691         for mode in ["append", "truncate", "same_size", "offset"]:
    692             with self.subTest(mode=mode):
    693                 output, calls = self.output("write-" + mode), []
    694 
    695                 def changed(descriptor, raw):
    696                     count = original(descriptor, raw)
    697                     if not calls:
    698                         calls.append(count)
    699                         self.mutate(output, mode)
    700                     return count
    701 
    702                 with patch.object(self.producer.os, "write", changed):
    703                     with self.assertRaises(self.producer.ReportError) as failure:
    704                         output.write(self.document)
    705                 self.assertNotIn(SECRET, str(failure.exception))
    706                 self.assertTrue(calls)
    707 
    708     def test_prewrite_descriptor_offset_is_rejected(self):
    709         output = self.output("prewrite-offset")
    710         os.lseek(output.descriptor, 3, os.SEEK_SET)
    711         with self.assertRaises(self.producer.ReportError):
    712             output.write(self.document)
    713         self.assertEqual(os.fstat(output.descriptor).st_size, 0)
    714 
    715     def test_actual_short_writes_preserve_exact_emitted_bytes(self):
    716         output, original = self.output("short-write"), os.write
    717 
    718         def partial(descriptor, raw):
    719             return original(descriptor, raw[:7])
    720 
    721         with patch.object(self.producer.os, "write", partial):
    722             output.write(self.document)
    723         output.verify()
    724         raw = (self.root / "short-write.json").read_bytes()
    725         self.assertEqual(
    726             raw, self.producer.ENCODER.encode(self.document).encode("ascii")
    727         )
    728         self.assertEqual(os.lseek(output.descriptor, 0, os.SEEK_CUR), len(raw))
    729 
    730     def test_descriptor_read_races_are_bounded_and_fenced(self):
    731         original = os.pread
    732         for mode in ["same_size", "offset"]:
    733             with self.subTest(mode=mode):
    734                 output, reads = self.output("read-" + mode), []
    735 
    736                 def changed(descriptor, count, offset):
    737                     raw = original(descriptor, count, offset)
    738                     reads.append((count, offset))
    739                     if len(reads) == 1:
    740                         self.mutate(output, mode)
    741                     return raw
    742 
    743                 with patch.object(self.producer.os, "pread", changed):
    744                     with self.assertRaises(self.producer.ReportError):
    745                         output.write(self.document)
    746                 self.assertTrue(reads)
    747                 self.assertTrue(all(0 < count <= 65536 for count, _ in reads))
    748 
    749 
    750 class ObserverBoundTests(unittest.TestCase):
    751     def test_byte_budget_accepts_exact_limit_and_rejects_next_byte(self):
    752         producer = importlib.import_module("scripts.unittest_report")
    753         budget = producer.ReportBudget()
    754         budget.size = producer.MAX_BYTES - 1
    755         budget.add(1)
    756         self.assertEqual(budget.size, producer.MAX_BYTES)
    757         with self.assertRaises(producer.ReportError):
    758             budget.add(1)
    759         self.assertEqual(budget.size, producer.MAX_BYTES)
    760 
    761     def test_observer_never_retains_exception_skip_or_subtest_values(self):
    762         producer = importlib.import_module("scripts.unittest_report")
    763         module = "scripts.test_observed"
    764 
    765         def method(case):
    766             with case.subTest(private=SECRET):
    767                 case.skipTest(SECRET)
    768 
    769         case_type = type(
    770             "ObservedTests",
    771             (unittest.TestCase,),
    772             {"__module__": module, "test_one": method},
    773         )
    774         case = case_type("test_one")
    775         budget = producer.ReportBudget()
    776         inventory = producer.inventory(unittest.TestSuite([case]), [module], budget)
    777         result, passing = producer.observe(
    778             unittest.TestSuite([case]), inventory, budget, io.StringIO()
    779         )
    780         self.assertFalse(passing)
    781         self.assertEqual(result.skipped, [])
    782         self.assertNotIn(SECRET, repr(result.__dict__))
    783 
    784     def test_imported_module_origin_is_checked_after_loading(self):
    785         producer = importlib.import_module("scripts.unittest_report")
    786         with tempfile.TemporaryDirectory() as temporary:
    787             root = Path(temporary).resolve()
    788             (root / "scripts").mkdir()
    789             (root / "scripts/test_observed.py").write_text("pass\n")
    790             with patch.object(importlib.util, "find_spec") as find:
    791                 find.return_value.origin = str(root / "foreign.py")
    792                 with self.assertRaises(producer.ReportError):
    793                     producer.load_modules(root, ["scripts.test_observed"])
    794 
    795 
    796 if __name__ == "__main__":
    797     unittest.main()