ffi_native.py (17880B)
1 """Closed Apple C input policy and execution bindings for cc 1.4.0/SQLite 0.37.0. 2 3 Inherited overrides are rejected by presence, including empty values. We never 4 flatten cc's priority-ordered tool selection or additive flags into one input. 5 CXX and CPPFLAGS are inactive in the current C graph. PATH is recorded by digest; 6 only public tool/SDK locations are exported, never arbitrary environment values. 7 """ 8 9 from __future__ import annotations 10 11 import hashlib 12 import json 13 import os 14 import shutil 15 import stat 16 import time 17 from pathlib import Path 18 from typing import Any 19 20 import ffi_source as source 21 22 TARGET_SDKS = { 23 "aarch64-apple-ios": "iphoneos", 24 "aarch64-apple-ios-sim": "iphonesimulator", 25 "aarch64-apple-darwin": "macosx", 26 } 27 HOST = "aarch64-apple-darwin" 28 TOOL_INPUTS = ("CC", "CFLAGS", "AR", "ARFLAGS", "RANLIB", "RANLIBFLAGS") 29 # cc 1.4.0 reads these global cargo_env_var_os controls with trim inheritance on. 30 TRIM_INPUTS = ("CARGO_TRIM_PATHS_SCOPE", "CARGO_TRIM_PATHS_REMAP") 31 PKG_INPUTS = ( 32 "PKG_CONFIG", 33 "PKG_CONFIG_PATH", 34 "PKG_CONFIG_LIBDIR", 35 "PKG_CONFIG_SYSROOT_DIR", 36 "PKG_CONFIG_ALLOW_CROSS", 37 ) 38 DIRECT_INPUTS = ( 39 *TRIM_INPUTS, 40 "CC_KNOWN_WRAPPER_CUSTOM", 41 "CC_SHELL_ESCAPED_FLAGS", 42 "CRATE_CC_NO_DEFAULTS", 43 "CC_FORCE_DISABLE", 44 "CC_ENABLE_DEBUG_OUTPUT", 45 "CROSS_COMPILE", 46 "LIBSQLITE3_FLAGS", 47 "RING_PREGENERATE_ASM", 48 "SQLITE_MAX_VARIABLE_NUMBER", 49 "SQLITE_MAX_EXPR_DEPTH", 50 "SQLITE_MAX_COLUMN", 51 "LIBSQLITE3_SYS_USE_PKG_CONFIG", 52 "SQLITE3_LIB_DIR", 53 "SQLITE3_INCLUDE_DIR", 54 "SQLITE3_STATIC", 55 "SQLITE3_NO_PKG_CONFIG", 56 "SQLITE3_DYNAMIC", 57 "PKG_CONFIG_ALL_STATIC", 58 "PKG_CONFIG_ALL_DYNAMIC", 59 "PKG_CONFIG_ALLOW_SYSTEM_CFLAGS", 60 "PKG_CONFIG_ALLOW_SYSTEM_LIBS", 61 "SDKROOT", 62 "DEVELOPER_DIR", 63 "TOOLCHAINS", 64 "SDK_DIR", 65 "IPHONEOS_DEPLOYMENT_TARGET", 66 "MACOSX_DEPLOYMENT_TARGET", 67 "CPATH", 68 "C_INCLUDE_PATH", 69 "OBJC_INCLUDE_PATH", 70 "CPLUS_INCLUDE_PATH", 71 "OBJCPLUS_INCLUDE_PATH", 72 "LIBRARY_PATH", 73 "COMPILER_PATH", 74 "GCC_EXEC_PREFIX", 75 "CCC_OVERRIDE_OPTIONS", 76 "RC_DEBUG_OPTIONS", 77 "RC_CFLAGS", 78 "ZERO_AR_DATE", 79 "CLANG_CONFIG_FILE_SYSTEM_DIR", 80 "CLANG_CONFIG_FILE_USER_DIR", 81 ) 82 PUBLIC_ROOTS = ( 83 Path("/Applications"), 84 Path("/Library/Developer"), 85 Path("/usr"), 86 Path("/opt/homebrew"), 87 ) 88 # Streaming identity caps, independent of the unchanged 2 MiB source-file cap. 89 MAX_NATIVE_NODES = 100_000 90 MAX_NATIVE_BYTES = 2_000_000_000 91 MAX_INSPECTION_SECONDS = 120 92 INSPECTION_LIMITS = { 93 "maximum_sdk_nodes": MAX_NATIVE_NODES, 94 "maximum_sdk_aggregate_bytes": MAX_NATIVE_BYTES, 95 "maximum_inspection_seconds": MAX_INSPECTION_SECONDS, 96 "stream_chunk_bytes": 65536, 97 } 98 99 100 def inherited_names() -> set[str]: 101 names = set(DIRECT_INPUTS) 102 for base in (*TOOL_INPUTS, *PKG_INPUTS): 103 names.update((base, "HOST_" + base, "TARGET_" + base)) 104 for target in TARGET_SDKS: 105 names.update((base + "_" + target, base + "_" + target.replace("-", "_"))) 106 return names 107 108 109 def reject_inherited(environment: dict[str, str]) -> None: 110 if inherited_names().intersection(environment): 111 raise source.ProvenanceError("ungoverned inherited native build input") 112 if any( 113 name.startswith("DYLD_") and name != "DYLD_FALLBACK_LIBRARY_PATH" 114 for name in environment 115 ): 116 raise source.ProvenanceError("ungoverned inherited Apple loader input") 117 118 119 def inherited_identity(environment: dict[str, str]) -> dict[str, Any]: 120 reject_inherited(environment) 121 path = environment.get("PATH", "") 122 if not path or len(path.encode()) > source.MAX_BYTES: 123 raise source.ProvenanceError("native search path is missing or exceeds bound") 124 result = { 125 "path_sha256": hashlib.sha256(path.encode()).hexdigest(), 126 "generator_profile_overrides": source.allowed_profile_overrides(), 127 } 128 fallback = environment.get("DYLD_FALLBACK_LIBRARY_PATH") 129 if fallback is not None: 130 result["removed_dyld_fallback_sha256"] = hashlib.sha256( 131 fallback.encode() 132 ).hexdigest() 133 return result 134 135 136 def inspection_environment() -> dict[str, str]: 137 environment = dict(os.environ) 138 reject_inherited(environment) 139 environment.pop("DYLD_FALLBACK_LIBRARY_PATH", None) 140 return environment 141 142 143 def public_path(value: str) -> Path: 144 path = Path(value) 145 if not path.is_absolute(): 146 raise source.ProvenanceError("native tool or SDK path is not absolute") 147 if ".." in path.parts or not any( 148 path.is_relative_to(root) for root in PUBLIC_ROOTS 149 ): 150 raise source.ProvenanceError("native tool or SDK path is not public") 151 try: 152 resolved = path.resolve(strict=True) 153 except (OSError, RuntimeError, ValueError): 154 raise source.ProvenanceError( 155 "native tool or SDK path cannot be resolved" 156 ) from None 157 if not any(resolved.is_relative_to(root) for root in PUBLIC_ROOTS): 158 raise source.ProvenanceError("native tool or SDK path is not public") 159 return resolved 160 161 162 def public_tool(value: str) -> Path: 163 public_path(value) 164 # Preserve the executable alias: Swift/ranlib dispatch by argv[0]. 165 return Path(value) 166 167 168 def check_deadline(deadline: float) -> None: 169 if time.monotonic() > deadline: 170 raise source.ProvenanceError("native identity exceeds inspection bound") 171 172 173 def file_stamp(path: Path) -> tuple[int, ...]: 174 return stat_stamp(path.stat(follow_symlinks=False)) 175 176 177 def stat_stamp(value: os.stat_result) -> tuple[int, ...]: 178 return ( 179 value.st_dev, 180 value.st_ino, 181 value.st_mode, 182 value.st_size, 183 value.st_mtime_ns, 184 value.st_ctime_ns, 185 ) 186 187 188 def file_identity(path: Path, deadline: float | None = None) -> dict[str, Any]: 189 if deadline is None: 190 deadline = time.monotonic() + MAX_INSPECTION_SECONDS 191 check_deadline(deadline) 192 descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) 193 with os.fdopen(descriptor, "rb") as handle: 194 before = os.fstat(handle.fileno()) 195 if not stat.S_ISREG(before.st_mode) or before.st_size > MAX_NATIVE_BYTES: 196 raise source.ProvenanceError( 197 "native identity input is not bounded regular data" 198 ) 199 digest = hashlib.sha256() 200 count = 0 201 while chunk := handle.read(65536): 202 check_deadline(deadline) 203 count += len(chunk) 204 if count > MAX_NATIVE_BYTES: 205 raise source.ProvenanceError("native identity input exceeds bound") 206 digest.update(chunk) 207 after = os.fstat(handle.fileno()) 208 if (after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns) != ( 209 before.st_ino, 210 before.st_size, 211 before.st_mtime_ns, 212 before.st_ctime_ns, 213 ) or file_stamp(path)[:4] != ( 214 before.st_dev, 215 before.st_ino, 216 before.st_mode, 217 before.st_size, 218 ): 219 raise source.ProvenanceError("native identity input changed while reading") 220 check_deadline(deadline) 221 return {"sha256": digest.hexdigest(), "bytes": count} 222 223 224 def sdk_directory( 225 root: Path, directory: Path, inventory: dict[Path, tuple[int, ...]], deadline: float 226 ) -> int: 227 check_deadline(deadline) 228 flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW 229 descriptor = os.open(root, flags) 230 current = root 231 try: 232 if stat_stamp(os.fstat(descriptor)) != inventory[current]: 233 raise source.ProvenanceError("native SDK directory changed during capture") 234 for name in directory.relative_to(root).parts: 235 check_deadline(deadline) 236 child = os.open(name, flags, dir_fd=descriptor) 237 os.close(descriptor) 238 descriptor = child 239 current = current / name 240 if stat_stamp(os.fstat(descriptor)) != inventory[current]: 241 raise source.ProvenanceError( 242 "native SDK directory changed during capture" 243 ) 244 return descriptor 245 except BaseException: 246 os.close(descriptor) 247 raise 248 249 250 def sdk_entries( 251 root: Path, directory: Path, inventory: dict[Path, tuple[int, ...]], deadline: float 252 ): 253 descriptor = sdk_directory(root, directory, inventory, deadline) 254 try: 255 with os.scandir(descriptor) as entries: 256 while True: 257 check_deadline(deadline) 258 try: 259 entry = next(entries) 260 except StopIteration: 261 return 262 check_deadline(deadline) 263 yield directory / entry.name 264 finally: 265 os.close(descriptor) 266 267 268 def sdk_inventory(path: Path, deadline: float) -> dict[Path, tuple[int, ...]]: 269 check_deadline(deadline) 270 if not stat.S_ISDIR(path.lstat().st_mode): 271 raise source.ProvenanceError("native SDK root is not a directory") 272 inventory = {path: file_stamp(path)} 273 pending = [path] 274 try: 275 while pending: 276 directory = pending.pop() 277 for entry in sdk_entries(path, directory, inventory, deadline): 278 if len(inventory) > MAX_NATIVE_NODES: 279 raise source.ProvenanceError( 280 "native SDK identity exceeds inspection bound" 281 ) 282 stamp = file_stamp(entry) 283 inventory[entry] = stamp 284 if stat.S_ISDIR(stamp[2]): 285 pending.append(entry) 286 except OSError: 287 raise source.ProvenanceError("native SDK enumeration failed") from None 288 return inventory 289 290 291 def sdk_identity(path: Path, deadline: float | None = None) -> dict[str, Any]: 292 digest = hashlib.sha256() 293 size = 0 294 if deadline is None: 295 deadline = time.monotonic() + MAX_INSPECTION_SECONDS 296 inventory = sdk_inventory(path, deadline) 297 for entry in sorted(inventory): 298 check_deadline(deadline) 299 identity, count = sdk_entry(path, entry, deadline) 300 size += count 301 if size > MAX_NATIVE_BYTES: 302 raise source.ProvenanceError("native SDK identity exceeds byte bound") 303 digest.update(entry.relative_to(path).as_posix().encode() + b"\0" + identity) 304 if sdk_inventory(path, deadline) != inventory: 305 raise source.ProvenanceError("native SDK inventory changed during capture") 306 return { 307 "path": str(path), 308 "sha256": digest.hexdigest(), 309 "nodes": len(inventory) - 1, 310 "bytes": size, 311 } 312 313 314 def sdk_entry(root: Path, path: Path, deadline: float) -> tuple[bytes, int]: 315 mode = path.lstat().st_mode 316 if stat.S_ISLNK(mode): 317 if not path.resolve().is_relative_to(root): 318 raise source.ProvenanceError("native SDK link escapes its root") 319 return b"link\0" + os.readlink(path).encode(), 0 320 if stat.S_ISDIR(mode): 321 return ("directory\0" + str(stat.S_IMODE(mode))).encode(), 0 322 identity = file_identity(path, deadline) 323 return str(identity).encode(), identity["bytes"] 324 325 326 def tool_identity(path: Path, deadline: float | None = None) -> dict[str, Any]: 327 resolved = public_path(str(path)) 328 if not os.access(path, os.X_OK): 329 raise source.ProvenanceError("native tool is not executable") 330 return { 331 "path": str(path), 332 "resolved_path": str(resolved), 333 **file_identity(resolved, deadline), 334 } 335 336 337 def query( 338 root: Path, argv: list[str], environment: dict[str, str], deadline: float 339 ) -> str: 340 check_deadline(deadline) 341 timeout = min(MAX_INSPECTION_SECONDS, deadline - time.monotonic()) 342 return ( 343 source.command(root, argv, environment=environment, timeout=timeout) 344 .decode() 345 .strip() 346 ) 347 348 349 def capture(root: Path, target: str) -> dict[str, Any]: 350 try: 351 return capture_apple(root, target) 352 except (OSError, UnicodeError, RuntimeError, ValueError): 353 raise source.ProvenanceError("native Apple input inspection failed") from None 354 355 356 def capture_apple(root: Path, target: str) -> dict[str, Any]: 357 deadline = time.monotonic() + MAX_INSPECTION_SECONDS 358 environment = dict(os.environ) 359 inherited = inherited_identity(environment) 360 environment.pop("DYLD_FALLBACK_LIBRARY_PATH", None) 361 if target not in TARGET_SDKS: 362 raise source.ProvenanceError("native target is not governed") 363 xcrun = public_tool(shutil.which("xcrun", path=environment["PATH"]) or "") 364 developer = public_path( 365 query(root, ["/usr/bin/xcode-select", "-p"], environment, deadline) 366 ) 367 environment["DEVELOPER_DIR"] = str(developer) 368 sdk = TARGET_SDKS[target] 369 tools = {"xcrun": tool_identity(xcrun, deadline)} 370 for key, name in ( 371 ("compiler", "clang"), 372 ("linker", "ld"), 373 ("archiver", "ar"), 374 ("ranlib", "ranlib"), 375 ("swiftc", "swiftc"), 376 ("swift_symbolgraph_extract", "swift-symbolgraph-extract"), 377 ("xcodebuild", "xcodebuild"), 378 ("swift_format", "swift-format"), 379 ("swift", "swift"), 380 ): 381 selected = query( 382 root, [str(xcrun), "--sdk", sdk, "--find", name], environment, deadline 383 ) 384 tools[key] = tool_identity(public_tool(selected), deadline) 385 formatter = public_tool(shutil.which("swiftformat", path=environment["PATH"]) or "") 386 tools["swiftformat"] = tool_identity(formatter, deadline) 387 sdk_path = public_path( 388 query( 389 root, [str(xcrun), "--sdk", sdk, "--show-sdk-path"], environment, deadline 390 ) 391 ) 392 resources = public_path( 393 query( 394 root, 395 [tools["compiler"]["path"], "-print-resource-dir"], 396 environment, 397 deadline, 398 ) 399 ) 400 result = { 401 "policy": "closed_apple_c_v1", 402 "target": target, 403 "sdk_name": sdk, 404 "inspection_limits": INSPECTION_LIMITS, 405 "inherited": inherited, 406 "developer_dir": str(developer), 407 "tools": tools, 408 "sdk": sdk_identity(sdk_path, deadline), 409 "compiler_resources": sdk_identity(resources, deadline), 410 "compiler_arguments": ["--no-default-config"], 411 "linker_driver_arguments": [ 412 "--no-default-config", 413 "-fuse-ld=" + tools["linker"]["path"], 414 ], 415 "sdk_version": query( 416 root, 417 [str(xcrun), "--sdk", sdk, "--show-sdk-version"], 418 environment, 419 deadline, 420 ), 421 "sdk_build": query( 422 root, 423 [str(xcrun), "--sdk", sdk, "--show-sdk-build-version"], 424 environment, 425 deadline, 426 ), 427 "deployment": {"IPHONEOS_DEPLOYMENT_TARGET": "18.0"} if target != HOST else {}, 428 } 429 if inherited_identity(dict(os.environ)) != inherited: 430 raise source.ProvenanceError("native environment changed during capture") 431 check_deadline(deadline) 432 return result 433 434 435 def verify(root: Path, context: dict[str, Any]) -> None: 436 if capture(root, context["target"]) != context: 437 raise source.ProvenanceError("native environment, tool or SDK drift") 438 439 440 def cache_identity(context: dict[str, Any], host: dict[str, Any]) -> str: 441 encoded = json.dumps( 442 {"target": context, "host": host}, sort_keys=True, separators=(",", ":") 443 ).encode() 444 return hashlib.sha256(encoded).hexdigest() 445 446 447 def execution_environment( 448 root: Path, 449 environment: dict[str, str], 450 context: dict[str, Any], 451 host: dict[str, Any], 452 ) -> dict[str, str]: 453 # The caller already supplies governed Rust flags/deployment; reject only 454 # the ungoverned trim controls here, including keys absent from os.environ. 455 if set(TRIM_INPUTS).intersection(environment): 456 raise source.ProvenanceError("ungoverned inherited native build input") 457 verify(root, context) 458 verify(root, host) 459 result = dict(environment) 460 result.pop("DYLD_FALLBACK_LIBRARY_PATH", None) 461 # PATH is copied before execution, and its inherited identity must match capture. 462 if ( 463 hashlib.sha256(result["PATH"].encode()).hexdigest() 464 != context["inherited"]["path_sha256"] 465 ): 466 raise source.ProvenanceError( 467 "native execution search path differs from capture" 468 ) 469 result.update( 470 {"DEVELOPER_DIR": context["developer_dir"], "SDKROOT": context["sdk"]["path"]} 471 ) 472 result.pop("IPHONEOS_DEPLOYMENT_TARGET", None) 473 result.update(context["deployment"]) 474 if context["tools"]["linker"] != host["tools"]["linker"]: 475 raise source.ProvenanceError("native split host/target linker is not governed") 476 identity = cache_identity(context, host) 477 # The captured clang driver must dispatch to the captured ld, not a PATH fallback. 478 result["CARGO_ENCODED_RUSTFLAGS"] = "\x1f".join( 479 filter( 480 None, 481 ( 482 result.get("CARGO_ENCODED_RUSTFLAGS", ""), 483 '--cfg=tera_native_context="' + identity + '"', 484 "\x1f".join( 485 "-Clink-arg=" + argument 486 for argument in context["linker_driver_arguments"] 487 ), 488 ), 489 ) 490 ) 491 for selected in (host, context): 492 target = selected["target"] 493 # cc tracks CFLAGS via rerun-if-env-changed. Rust flags also bind Cargo 494 # units, including builders such as ring that suppress cc metadata. 495 result["CFLAGS_" + target] = " ".join( 496 [ 497 *selected["compiler_arguments"], 498 "-DTERA_NATIVE_CONTEXT_" + identity + "=1", 499 ] 500 ) 501 for name, key in (("CC", "compiler"), ("AR", "archiver"), ("RANLIB", "ranlib")): 502 result[name + "_" + target] = selected["tools"][key]["path"] 503 result["CARGO_TARGET_" + target.upper().replace("-", "_") + "_LINKER"] = ( 504 selected["tools"]["compiler"]["path"] 505 ) 506 return result