fixture_tool_dispatch.py (14299B)
1 """Fresh owned native dispatch for Python test-tool bodies; no product runner.""" 2 3 from __future__ import annotations 4 5 import hashlib 6 import json 7 import os 8 import re 9 import selectors 10 import shutil 11 import signal 12 import stat 13 import subprocess 14 import sys 15 import time 16 from pathlib import Path 17 18 if __package__: 19 from .fixture_test_policy import limit, load_policy 20 else: 21 from fixture_test_policy import limit, load_policy 22 23 24 class FixtureDispatchError(RuntimeError): 25 """The owned fixture compiler or declared tool could not be established.""" 26 27 28 class CompilerCommand: 29 """One deadline, bounded pipes and cleanup before releasing the leader PID.""" 30 31 LIMITS = {"stdout": 1024 * 1024, "stderr": 64 * 1024} 32 33 def __init__( 34 self, 35 argv: list[str], 36 cwd: Path, 37 timeout=None, 38 *, 39 env=None, 40 budget="compiler", 41 maximum_stdout=1024 * 1024, 42 ): 43 policy = load_policy() 44 timeout = policy["limits"][budget] if timeout is None else timeout 45 if not 0 < timeout <= policy["limits"][budget]: 46 raise ValueError("fixture command deadline exceeds its fixed policy budget") 47 if ( 48 type(maximum_stdout) is not int 49 or not 0 < maximum_stdout <= self.LIMITS["stdout"] 50 ): 51 raise ValueError("command JSON output exceeds its byte bound") 52 self.LIMITS = {**self.LIMITS, "stdout": maximum_stdout} 53 self.argv, self.cwd, self.timeout = argv, cwd, timeout 54 self.environment = env 55 self.output = {name: bytearray() for name in self.LIMITS} 56 self.selector = selectors.DefaultSelector() 57 self.process = None 58 self.started = time.monotonic() 59 self.record = { 60 "argv": argv, 61 "cwd": str(cwd), 62 "timeout": timeout, 63 "policy_id": policy["id"], 64 "policy_sha256": policy["source_sha256"], 65 "budget": budget, 66 "started_ns": time.time_ns(), 67 "signals": [], 68 "wait_reaped": False, 69 "group_absent": False, 70 } 71 72 def exited(self) -> bool: 73 state = os.waitid( 74 os.P_PID, self.process.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT 75 ) 76 return state is not None and state.si_pid == self.process.pid 77 78 def read_ready(self, timeout: float, *, reject_excess: bool) -> None: 79 if not self.selector.get_map(): 80 time.sleep(timeout) 81 return 82 for key, _ in self.selector.select(timeout): 83 name = key.data 84 available = self.LIMITS[name] - len(self.output[name]) 85 size = min(65536, available + 1) if reject_excess else 65536 86 data = os.read(key.fd, size) 87 if not data: 88 self.selector.unregister(key.fileobj) 89 key.fileobj.close() 90 continue 91 self.output[name].extend(data[:available]) 92 if len(data) > available: 93 self.record.setdefault("output_excess", name) 94 if reject_excess: 95 raise FixtureDispatchError( 96 "fixture compiler capture limit exceeded" 97 ) 98 99 def capture(self) -> None: 100 deadline = self.started + self.timeout 101 while self.selector.get_map() or not self.exited(): 102 remaining = deadline - time.monotonic() 103 if remaining <= 0: 104 raise subprocess.TimeoutExpired(self.argv, self.timeout) 105 self.read_ready(min(remaining, 0.05), reject_excess=True) 106 107 def group_present(self) -> bool: 108 try: 109 os.killpg(self.process.pid, 0) 110 except ProcessLookupError: 111 return False 112 except PermissionError: 113 return True 114 return True 115 116 def send(self, number: int) -> None: 117 try: 118 os.killpg(self.process.pid, number) 119 except ProcessLookupError: 120 return 121 except PermissionError: 122 self.record.setdefault("signal_errors", []).append( 123 {"signal": number, "errno": 1} 124 ) 125 if not self.exited(): 126 raise 127 return 128 self.record["signals"].append({"signal": number, "at_ns": time.time_ns()}) 129 130 def drain(self, deadline: float) -> None: 131 while self.selector.get_map() and time.monotonic() < deadline: 132 self.read_ready( 133 min(0.01, max(0, deadline - time.monotonic())), reject_excess=False 134 ) 135 136 def finish_cleanup(self, deadline: float) -> None: 137 self.send(signal.SIGKILL) 138 self.process.wait(timeout=max(0.001, deadline - time.monotonic())) 139 self.record.update(wait_reaped=True, exit=self.process.returncode) 140 self.drain(deadline) 141 while self.group_present() and time.monotonic() < deadline: 142 time.sleep(0.005) 143 self.record["group_absent"] = not self.group_present() 144 if not self.record["group_absent"] or self.selector.get_map(): 145 raise FixtureDispatchError("fixture compiler cleanup incomplete") 146 147 def settle(self) -> None: 148 deadline = time.monotonic() + limit("leaf_cleanup") 149 try: 150 self.send(signal.SIGTERM) 151 grace = time.monotonic() + 0.2 152 self.drain(grace) 153 while self.group_present() and time.monotonic() < grace: 154 time.sleep(0.005) 155 finally: 156 self.finish_cleanup(deadline) 157 158 def execute(self) -> subprocess.CompletedProcess: 159 try: 160 self.process = subprocess.Popen( 161 self.argv, 162 cwd=self.cwd, 163 env=self.environment, 164 stdin=subprocess.DEVNULL, 165 stdout=subprocess.PIPE, 166 stderr=subprocess.PIPE, 167 bufsize=0, 168 start_new_session=True, 169 ) 170 self.record.update(pid=self.process.pid, pgid=self.process.pid) 171 for name in self.LIMITS: 172 stream = getattr(self.process, name) 173 os.set_blocking(stream.fileno(), False) 174 self.selector.register(stream, selectors.EVENT_READ, name) 175 self.capture() 176 except BaseException as error: 177 self.record["error"] = type(error).__name__ 178 raise 179 finally: 180 try: 181 if self.process is not None: 182 self.settle() 183 except BaseException as error: 184 self.record["cleanup_error"] = type(error).__name__ 185 raise 186 finally: 187 self.selector.close() 188 if self.process is not None: 189 for name in self.LIMITS: 190 getattr(self.process, name).close() 191 self.record.update( 192 elapsed_seconds=time.monotonic() - self.started, 193 ended_ns=time.time_ns(), 194 ) 195 return subprocess.CompletedProcess( 196 self.argv, 197 self.process.returncode, 198 bytes(self.output["stdout"]), 199 bytes(self.output["stderr"]), 200 ) 201 202 203 def selector_command(argv, *, maximum_stdout=1024 * 1024, timeout=None): 204 capture = CompilerCommand( 205 argv, 206 Path.cwd(), 207 timeout, 208 budget="selector_command", 209 maximum_stdout=maximum_stdout, 210 ) 211 try: 212 result = capture.execute() 213 except FixtureDispatchError as error: 214 if capture.record.get("output_excess") == "stdout": 215 raise ValueError("command JSON output exceeds its byte bound") from error 216 raise 217 if result.returncode: 218 raise subprocess.CalledProcessError(result.returncode, argv) 219 return result 220 221 222 NATIVE_SOURCE = r""" 223 #include <limits.h> 224 #include <stdint.h> 225 #include <stdio.h> 226 #include <stdlib.h> 227 #include <string.h> 228 #include <unistd.h> 229 #if defined(__APPLE__) 230 #include <mach-o/dyld.h> 231 #endif 232 233 static const char *roles[] = {ROLE_LITERALS}; 234 static const char *python = PYTHON_LITERAL; 235 236 static int executing_image(char *resolved) { 237 char image[PATH_MAX]; 238 #if defined(__APPLE__) 239 uint32_t size = sizeof(image); 240 if (_NSGetExecutablePath(image, &size) != 0) return 0; 241 #elif defined(__linux__) 242 ssize_t size = readlink("/proc/self/exe", image, sizeof(image) - 1); 243 if (size < 0 || (size_t)size >= sizeof(image) - 1) return 0; 244 image[size] = '\0'; 245 #else 246 return 0; 247 #endif 248 return realpath(image, resolved) != NULL; 249 } 250 251 int main(int argc, char **argv) { 252 if (argc < 1 || argc > 4096) return 64; 253 const char *role = strrchr(argv[0], '/'); 254 role = role ? role + 1 : argv[0]; 255 size_t count = sizeof(roles) / sizeof(roles[0]); 256 size_t index; 257 for (index = 0; index < count; index++) 258 if (strcmp(role, roles[index]) == 0) break; 259 if (index == count) return 64; 260 char image[PATH_MAX]; 261 if (!executing_image(image)) return 64; 262 char *basename = strrchr(image, '/'); 263 if (!basename || strcmp(basename + 1, role) != 0) return 64; 264 *basename = '\0'; 265 const char *path = getenv("PATH"); 266 if (!path || path[0] != '/') return 64; 267 size_t length = strcspn(path, ":"); 268 if (!length || length >= PATH_MAX) return 64; 269 if (strlen(image) != length || strncmp(path, image, length) != 0) return 64; 270 char body[PATH_MAX]; 271 int size = snprintf(body, sizeof(body), "%.*s/%s.body.py", (int)length, path, role); 272 if (size < 0 || (size_t)size >= sizeof(body)) return 64; 273 if (access(body, R_OK) != 0) return 66; 274 char **arguments = calloc((size_t)argc + 2, sizeof(char *)); 275 if (!arguments) return 70; 276 arguments[0] = (char *)python; 277 arguments[1] = body; 278 for (int item = 1; item < argc; item++) arguments[item + 1] = argv[item]; 279 execv(python, arguments); 280 perror("fixture-tool-dispatch: execv"); 281 free(arguments); 282 return 71; 283 } 284 """ 285 286 287 def file_identity(path: Path) -> dict: 288 information = path.stat() 289 return { 290 "path": str(path), 291 "resolved": str(path.resolve()), 292 "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), 293 "bytes": information.st_size, 294 "mode": stat.S_IMODE(information.st_mode), 295 } 296 297 298 def c_literal(value: str) -> str: 299 return '"' + "".join(f"\\{byte:03o}" for byte in value.encode()) + '"' 300 301 302 class FixtureToolDispatcher: 303 """Compile once per fixture, never execute the target or share tool inodes.""" 304 305 def __init__(self, tools: Path, roles: tuple[str, ...]): 306 self.tools = tools.resolve() 307 if not 0 < len(roles) <= 16 or len(set(roles)) != len(roles): 308 raise FixtureDispatchError("fixture roles must be nonempty and unique") 309 if any(re.fullmatch(r"[a-z][a-z0-9_]{0,31}", role) is None for role in roles): 310 raise FixtureDispatchError("fixture role name rejected") 311 self.roles = roles 312 self.directory = self.tools / ".fixture-dispatch" 313 self.directory.mkdir() 314 self.binary = self.directory / "dispatch" 315 self.record = { 316 "interpreter": file_identity(Path(sys.executable)), 317 "roles": list(roles), 318 "tools": str(self.tools), 319 "environment_policy": "compiler inherits existing selected developer environment", 320 "body_policy": "actual interpreter argv is [interpreter, role.body.py, *original argument suffix]", 321 "image_policy": "actual canonical image parent equals first PATH; actual basename equals argv0 role", 322 } 323 self.compile() 324 325 def compile(self) -> None: 326 compiler = shutil.which("cc") 327 if compiler is None: 328 self.record["error"] = "selected native compiler unavailable" 329 self.save_record() 330 raise FixtureDispatchError( 331 "fixture requires an installed selected native compiler" 332 ) 333 source = self.directory / "dispatch.c" 334 source.write_text( 335 NATIVE_SOURCE.replace( 336 "ROLE_LITERALS", ",".join(map(c_literal, self.roles)) 337 ).replace("PYTHON_LITERAL", c_literal(sys.executable)) 338 ) 339 self.record.update( 340 source=file_identity(source), compiler=file_identity(Path(compiler)) 341 ) 342 argv = [ 343 compiler, 344 "-Wall", 345 "-Wextra", 346 "-Werror", 347 "-o", 348 str(self.binary), 349 str(source), 350 ] 351 command = CompilerCommand(argv, self.directory) 352 try: 353 result = command.execute() 354 if result.returncode != 0: 355 raise FixtureDispatchError("fixture native compilation failed") 356 if ( 357 not self.binary.is_file() 358 or self.binary.stat().st_size > 2 * 1024 * 1024 359 ): 360 raise FixtureDispatchError("fixture compiler output rejected") 361 self.record["binary"] = file_identity(self.binary) 362 except BaseException as error: 363 self.record["error"] = type(error).__name__ 364 raise 365 finally: 366 self.record["command"] = command.record 367 for name, raw in command.output.items(): 368 (self.directory / f"compiler.{name}").write_bytes(raw) 369 self.save_record() 370 371 def save_record(self) -> None: 372 (self.directory / "compilation.json").write_text( 373 json.dumps(self.record, indent=2) + "\n" 374 ) 375 376 def executable(self, name: str, body: str) -> None: 377 if name not in self.roles or len(body.encode()) > 2 * 1024 * 1024: 378 raise FixtureDispatchError("undeclared fixture tool or oversized body") 379 path = self.tools / name 380 if path.exists() or path.is_symlink(): 381 path.unlink() 382 body_path = self.tools / f"{name}.body.py" 383 body_path.write_text(body) 384 shutil.copyfile(self.binary, path) 385 path.chmod(0o700) 386 record = { 387 "tool": file_identity(path), 388 "body": file_identity(body_path), 389 "interpreter": self.record["interpreter"], 390 "actual_exec_argv_prefix": [sys.executable, str(body_path)], 391 "argument_suffix": "original native argv[1:] without transformation", 392 } 393 (self.directory / f"{name}.json").write_text( 394 json.dumps(record, indent=2) + "\n" 395 )