field_ios

In-the-field app for Radroots on iOS
git clone https://radroots.dev/git/field_ios.git
Log | Files | Refs | README | LICENSE

fixture_tool_dispatch.py (14299B)


      1 """Fresh owned native dispatch for Python test-tool bodies; no product runner."""
      2 
      3 from __future__ import annotations
      4 
      5 import hashlib
      6 import json
      7 import os
      8 import re
      9 import selectors
     10 import shutil
     11 import signal
     12 import stat
     13 import subprocess
     14 import sys
     15 import time
     16 from pathlib import Path
     17 
     18 if __package__:
     19     from .fixture_test_policy import limit, load_policy
     20 else:
     21     from fixture_test_policy import limit, load_policy
     22 
     23 
     24 class FixtureDispatchError(RuntimeError):
     25     """The owned fixture compiler or declared tool could not be established."""
     26 
     27 
     28 class CompilerCommand:
     29     """One deadline, bounded pipes and cleanup before releasing the leader PID."""
     30 
     31     LIMITS = {"stdout": 1024 * 1024, "stderr": 64 * 1024}
     32 
     33     def __init__(
     34         self,
     35         argv: list[str],
     36         cwd: Path,
     37         timeout=None,
     38         *,
     39         env=None,
     40         budget="compiler",
     41         maximum_stdout=1024 * 1024,
     42     ):
     43         policy = load_policy()
     44         timeout = policy["limits"][budget] if timeout is None else timeout
     45         if not 0 < timeout <= policy["limits"][budget]:
     46             raise ValueError("fixture command deadline exceeds its fixed policy budget")
     47         if (
     48             type(maximum_stdout) is not int
     49             or not 0 < maximum_stdout <= self.LIMITS["stdout"]
     50         ):
     51             raise ValueError("command JSON output exceeds its byte bound")
     52         self.LIMITS = {**self.LIMITS, "stdout": maximum_stdout}
     53         self.argv, self.cwd, self.timeout = argv, cwd, timeout
     54         self.environment = env
     55         self.output = {name: bytearray() for name in self.LIMITS}
     56         self.selector = selectors.DefaultSelector()
     57         self.process = None
     58         self.started = time.monotonic()
     59         self.record = {
     60             "argv": argv,
     61             "cwd": str(cwd),
     62             "timeout": timeout,
     63             "policy_id": policy["id"],
     64             "policy_sha256": policy["source_sha256"],
     65             "budget": budget,
     66             "started_ns": time.time_ns(),
     67             "signals": [],
     68             "wait_reaped": False,
     69             "group_absent": False,
     70         }
     71 
     72     def exited(self) -> bool:
     73         state = os.waitid(
     74             os.P_PID, self.process.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT
     75         )
     76         return state is not None and state.si_pid == self.process.pid
     77 
     78     def read_ready(self, timeout: float, *, reject_excess: bool) -> None:
     79         if not self.selector.get_map():
     80             time.sleep(timeout)
     81             return
     82         for key, _ in self.selector.select(timeout):
     83             name = key.data
     84             available = self.LIMITS[name] - len(self.output[name])
     85             size = min(65536, available + 1) if reject_excess else 65536
     86             data = os.read(key.fd, size)
     87             if not data:
     88                 self.selector.unregister(key.fileobj)
     89                 key.fileobj.close()
     90                 continue
     91             self.output[name].extend(data[:available])
     92             if len(data) > available:
     93                 self.record.setdefault("output_excess", name)
     94                 if reject_excess:
     95                     raise FixtureDispatchError(
     96                         "fixture compiler capture limit exceeded"
     97                     )
     98 
     99     def capture(self) -> None:
    100         deadline = self.started + self.timeout
    101         while self.selector.get_map() or not self.exited():
    102             remaining = deadline - time.monotonic()
    103             if remaining <= 0:
    104                 raise subprocess.TimeoutExpired(self.argv, self.timeout)
    105             self.read_ready(min(remaining, 0.05), reject_excess=True)
    106 
    107     def group_present(self) -> bool:
    108         try:
    109             os.killpg(self.process.pid, 0)
    110         except ProcessLookupError:
    111             return False
    112         except PermissionError:
    113             return True
    114         return True
    115 
    116     def send(self, number: int) -> None:
    117         try:
    118             os.killpg(self.process.pid, number)
    119         except ProcessLookupError:
    120             return
    121         except PermissionError:
    122             self.record.setdefault("signal_errors", []).append(
    123                 {"signal": number, "errno": 1}
    124             )
    125             if not self.exited():
    126                 raise
    127             return
    128         self.record["signals"].append({"signal": number, "at_ns": time.time_ns()})
    129 
    130     def drain(self, deadline: float) -> None:
    131         while self.selector.get_map() and time.monotonic() < deadline:
    132             self.read_ready(
    133                 min(0.01, max(0, deadline - time.monotonic())), reject_excess=False
    134             )
    135 
    136     def finish_cleanup(self, deadline: float) -> None:
    137         self.send(signal.SIGKILL)
    138         self.process.wait(timeout=max(0.001, deadline - time.monotonic()))
    139         self.record.update(wait_reaped=True, exit=self.process.returncode)
    140         self.drain(deadline)
    141         while self.group_present() and time.monotonic() < deadline:
    142             time.sleep(0.005)
    143         self.record["group_absent"] = not self.group_present()
    144         if not self.record["group_absent"] or self.selector.get_map():
    145             raise FixtureDispatchError("fixture compiler cleanup incomplete")
    146 
    147     def settle(self) -> None:
    148         deadline = time.monotonic() + limit("leaf_cleanup")
    149         try:
    150             self.send(signal.SIGTERM)
    151             grace = time.monotonic() + 0.2
    152             self.drain(grace)
    153             while self.group_present() and time.monotonic() < grace:
    154                 time.sleep(0.005)
    155         finally:
    156             self.finish_cleanup(deadline)
    157 
    158     def execute(self) -> subprocess.CompletedProcess:
    159         try:
    160             self.process = subprocess.Popen(
    161                 self.argv,
    162                 cwd=self.cwd,
    163                 env=self.environment,
    164                 stdin=subprocess.DEVNULL,
    165                 stdout=subprocess.PIPE,
    166                 stderr=subprocess.PIPE,
    167                 bufsize=0,
    168                 start_new_session=True,
    169             )
    170             self.record.update(pid=self.process.pid, pgid=self.process.pid)
    171             for name in self.LIMITS:
    172                 stream = getattr(self.process, name)
    173                 os.set_blocking(stream.fileno(), False)
    174                 self.selector.register(stream, selectors.EVENT_READ, name)
    175             self.capture()
    176         except BaseException as error:
    177             self.record["error"] = type(error).__name__
    178             raise
    179         finally:
    180             try:
    181                 if self.process is not None:
    182                     self.settle()
    183             except BaseException as error:
    184                 self.record["cleanup_error"] = type(error).__name__
    185                 raise
    186             finally:
    187                 self.selector.close()
    188                 if self.process is not None:
    189                     for name in self.LIMITS:
    190                         getattr(self.process, name).close()
    191                 self.record.update(
    192                     elapsed_seconds=time.monotonic() - self.started,
    193                     ended_ns=time.time_ns(),
    194                 )
    195         return subprocess.CompletedProcess(
    196             self.argv,
    197             self.process.returncode,
    198             bytes(self.output["stdout"]),
    199             bytes(self.output["stderr"]),
    200         )
    201 
    202 
    203 def selector_command(argv, *, maximum_stdout=1024 * 1024, timeout=None):
    204     capture = CompilerCommand(
    205         argv,
    206         Path.cwd(),
    207         timeout,
    208         budget="selector_command",
    209         maximum_stdout=maximum_stdout,
    210     )
    211     try:
    212         result = capture.execute()
    213     except FixtureDispatchError as error:
    214         if capture.record.get("output_excess") == "stdout":
    215             raise ValueError("command JSON output exceeds its byte bound") from error
    216         raise
    217     if result.returncode:
    218         raise subprocess.CalledProcessError(result.returncode, argv)
    219     return result
    220 
    221 
    222 NATIVE_SOURCE = r"""
    223 #include <limits.h>
    224 #include <stdint.h>
    225 #include <stdio.h>
    226 #include <stdlib.h>
    227 #include <string.h>
    228 #include <unistd.h>
    229 #if defined(__APPLE__)
    230 #include <mach-o/dyld.h>
    231 #endif
    232 
    233 static const char *roles[] = {ROLE_LITERALS};
    234 static const char *python = PYTHON_LITERAL;
    235 
    236 static int executing_image(char *resolved) {
    237     char image[PATH_MAX];
    238 #if defined(__APPLE__)
    239     uint32_t size = sizeof(image);
    240     if (_NSGetExecutablePath(image, &size) != 0) return 0;
    241 #elif defined(__linux__)
    242     ssize_t size = readlink("/proc/self/exe", image, sizeof(image) - 1);
    243     if (size < 0 || (size_t)size >= sizeof(image) - 1) return 0;
    244     image[size] = '\0';
    245 #else
    246     return 0;
    247 #endif
    248     return realpath(image, resolved) != NULL;
    249 }
    250 
    251 int main(int argc, char **argv) {
    252     if (argc < 1 || argc > 4096) return 64;
    253     const char *role = strrchr(argv[0], '/');
    254     role = role ? role + 1 : argv[0];
    255     size_t count = sizeof(roles) / sizeof(roles[0]);
    256     size_t index;
    257     for (index = 0; index < count; index++)
    258         if (strcmp(role, roles[index]) == 0) break;
    259     if (index == count) return 64;
    260     char image[PATH_MAX];
    261     if (!executing_image(image)) return 64;
    262     char *basename = strrchr(image, '/');
    263     if (!basename || strcmp(basename + 1, role) != 0) return 64;
    264     *basename = '\0';
    265     const char *path = getenv("PATH");
    266     if (!path || path[0] != '/') return 64;
    267     size_t length = strcspn(path, ":");
    268     if (!length || length >= PATH_MAX) return 64;
    269     if (strlen(image) != length || strncmp(path, image, length) != 0) return 64;
    270     char body[PATH_MAX];
    271     int size = snprintf(body, sizeof(body), "%.*s/%s.body.py", (int)length, path, role);
    272     if (size < 0 || (size_t)size >= sizeof(body)) return 64;
    273     if (access(body, R_OK) != 0) return 66;
    274     char **arguments = calloc((size_t)argc + 2, sizeof(char *));
    275     if (!arguments) return 70;
    276     arguments[0] = (char *)python;
    277     arguments[1] = body;
    278     for (int item = 1; item < argc; item++) arguments[item + 1] = argv[item];
    279     execv(python, arguments);
    280     perror("fixture-tool-dispatch: execv");
    281     free(arguments);
    282     return 71;
    283 }
    284 """
    285 
    286 
    287 def file_identity(path: Path) -> dict:
    288     information = path.stat()
    289     return {
    290         "path": str(path),
    291         "resolved": str(path.resolve()),
    292         "sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
    293         "bytes": information.st_size,
    294         "mode": stat.S_IMODE(information.st_mode),
    295     }
    296 
    297 
    298 def c_literal(value: str) -> str:
    299     return '"' + "".join(f"\\{byte:03o}" for byte in value.encode()) + '"'
    300 
    301 
    302 class FixtureToolDispatcher:
    303     """Compile once per fixture, never execute the target or share tool inodes."""
    304 
    305     def __init__(self, tools: Path, roles: tuple[str, ...]):
    306         self.tools = tools.resolve()
    307         if not 0 < len(roles) <= 16 or len(set(roles)) != len(roles):
    308             raise FixtureDispatchError("fixture roles must be nonempty and unique")
    309         if any(re.fullmatch(r"[a-z][a-z0-9_]{0,31}", role) is None for role in roles):
    310             raise FixtureDispatchError("fixture role name rejected")
    311         self.roles = roles
    312         self.directory = self.tools / ".fixture-dispatch"
    313         self.directory.mkdir()
    314         self.binary = self.directory / "dispatch"
    315         self.record = {
    316             "interpreter": file_identity(Path(sys.executable)),
    317             "roles": list(roles),
    318             "tools": str(self.tools),
    319             "environment_policy": "compiler inherits existing selected developer environment",
    320             "body_policy": "actual interpreter argv is [interpreter, role.body.py, *original argument suffix]",
    321             "image_policy": "actual canonical image parent equals first PATH; actual basename equals argv0 role",
    322         }
    323         self.compile()
    324 
    325     def compile(self) -> None:
    326         compiler = shutil.which("cc")
    327         if compiler is None:
    328             self.record["error"] = "selected native compiler unavailable"
    329             self.save_record()
    330             raise FixtureDispatchError(
    331                 "fixture requires an installed selected native compiler"
    332             )
    333         source = self.directory / "dispatch.c"
    334         source.write_text(
    335             NATIVE_SOURCE.replace(
    336                 "ROLE_LITERALS", ",".join(map(c_literal, self.roles))
    337             ).replace("PYTHON_LITERAL", c_literal(sys.executable))
    338         )
    339         self.record.update(
    340             source=file_identity(source), compiler=file_identity(Path(compiler))
    341         )
    342         argv = [
    343             compiler,
    344             "-Wall",
    345             "-Wextra",
    346             "-Werror",
    347             "-o",
    348             str(self.binary),
    349             str(source),
    350         ]
    351         command = CompilerCommand(argv, self.directory)
    352         try:
    353             result = command.execute()
    354             if result.returncode != 0:
    355                 raise FixtureDispatchError("fixture native compilation failed")
    356             if (
    357                 not self.binary.is_file()
    358                 or self.binary.stat().st_size > 2 * 1024 * 1024
    359             ):
    360                 raise FixtureDispatchError("fixture compiler output rejected")
    361             self.record["binary"] = file_identity(self.binary)
    362         except BaseException as error:
    363             self.record["error"] = type(error).__name__
    364             raise
    365         finally:
    366             self.record["command"] = command.record
    367             for name, raw in command.output.items():
    368                 (self.directory / f"compiler.{name}").write_bytes(raw)
    369             self.save_record()
    370 
    371     def save_record(self) -> None:
    372         (self.directory / "compilation.json").write_text(
    373             json.dumps(self.record, indent=2) + "\n"
    374         )
    375 
    376     def executable(self, name: str, body: str) -> None:
    377         if name not in self.roles or len(body.encode()) > 2 * 1024 * 1024:
    378             raise FixtureDispatchError("undeclared fixture tool or oversized body")
    379         path = self.tools / name
    380         if path.exists() or path.is_symlink():
    381             path.unlink()
    382         body_path = self.tools / f"{name}.body.py"
    383         body_path.write_text(body)
    384         shutil.copyfile(self.binary, path)
    385         path.chmod(0o700)
    386         record = {
    387             "tool": file_identity(path),
    388             "body": file_identity(body_path),
    389             "interpreter": self.record["interpreter"],
    390             "actual_exec_argv_prefix": [sys.executable, str(body_path)],
    391             "argument_suffix": "original native argv[1:] without transformation",
    392         }
    393         (self.directory / f"{name}.json").write_text(
    394             json.dumps(record, indent=2) + "\n"
    395         )