ffi_source.py (15202B)
1 """Bounded source capture for the application-owned native producer.""" 2 3 from __future__ import annotations 4 5 import hashlib 6 import os 7 import re 8 import stat 9 import subprocess 10 from pathlib import Path 11 from typing import Any 12 13 import package_contract as contract 14 15 MAX_BYTES = 2 * 1024 * 1024 16 MAX_INPUTS = 2048 17 INPUTS = [ 18 "test-fixtures/legacy-identifiers.v1.json", 19 "Cargo.toml", 20 "Cargo.lock", 21 "rust-toolchain.toml", 22 "Makefile", 23 "core", 24 "scripts", 25 "TeraFFI/Makefile", 26 "TeraFFI/scripts", 27 "TeraFFI/producer.toml", 28 "radroots.lib.source-lock.v1.toml", 29 ] 30 PRODUCER_PATH = "TeraFFI/producer.toml" 31 32 33 class ProvenanceError(Exception): 34 """A source-free, fail-closed provenance rejection.""" 35 36 37 def command( 38 root: Path, 39 argv: list[str], 40 data: bytes | None = None, 41 *, 42 environment: dict[str, str] | None = None, 43 timeout: float = 120, 44 ) -> bytes: 45 if not 0 < timeout <= 120: 46 raise ProvenanceError("producer inspection deadline is expired or invalid") 47 if argv[0] == "git": 48 argv = ["git", "--no-replace-objects", *argv[1:]] 49 try: 50 result = subprocess.run( 51 argv, 52 cwd=root, 53 input=data, 54 env=environment, 55 capture_output=True, 56 check=False, 57 timeout=timeout, 58 ) 59 except (OSError, subprocess.TimeoutExpired) as error: 60 raise ProvenanceError("producer inspection command unavailable") from error 61 if result.returncode or len(result.stdout) > MAX_BYTES: 62 raise ProvenanceError("producer inspection command failed or exceeded bound") 63 return result.stdout 64 65 66 def read_source(root: Path, relative: str) -> bytes: 67 path = Path(relative) 68 if path.is_absolute() or str(path) != relative or ".." in path.parts: 69 raise ProvenanceError("producer input path is invalid") 70 current = root 71 for part in path.parts: 72 current = current / part 73 if current.is_symlink(): 74 raise ProvenanceError("producer input contains a symlink") 75 return contract._read_regular(current, maximum=MAX_BYTES) 76 77 78 def producer_contract(root: Path) -> dict[str, Any]: 79 value = contract._read_toml(root / PRODUCER_PATH) 80 expected = { 81 "schema", 82 "repository", 83 "foundation_lock", 84 "source_inputs", 85 "ffi", 86 "generator", 87 "build", 88 "native", 89 "redistribution", 90 } 91 contract._exact(set(value), expected, "producer contract fields") 92 contract._exact(value["schema"], "tera.native-producer.v2", "producer schema") 93 contract._exact( 94 value["repository"], 95 "https://github.com/radrootslabs/tera", 96 "producer repository", 97 ) 98 contract._exact(value["source_inputs"], INPUTS, "producer input inventory") 99 contract._exact( 100 value["foundation_lock"], 101 "radroots.lib.source-lock.v1.toml", 102 "foundation lock path", 103 ) 104 contract._exact( 105 value["ffi"], 106 { 107 "package": "tera_ffi", 108 "default_features": True, 109 "features": [], 110 "config": "core/crates/tera_ffi/uniffi.toml", 111 }, 112 "FFI producer selection", 113 ) 114 contract._exact( 115 value["generator"], 116 { 117 "package": "tera_bindgen", 118 "default_features": True, 119 "features": [], 120 }, 121 "generator selection", 122 ) 123 validate_build(value["build"]) 124 import ffi_native 125 import ffi_notices 126 127 ffi_notices.validate_contract(value["redistribution"]) 128 129 contract._exact( 130 value["native"], ffi_native.INSPECTION_LIMITS, "native inspection limits" 131 ) 132 cargo = contract._read_toml(root / "Cargo.toml") 133 lock = contract._read_toml(root / value["foundation_lock"]) 134 validate_foundation(cargo, lock) 135 contract._exact( 136 lock["lockfile_sha256"], 137 hashlib.sha256(read_source(root, "Cargo.lock")).hexdigest(), 138 "foundation consumer Cargo lock digest", 139 ) 140 toolchain = contract._read_toml(root / "rust-toolchain.toml") 141 contract._exact( 142 toolchain, 143 { 144 "toolchain": { 145 "channel": value["build"]["rust_version"], 146 "profile": "minimal", 147 "components": ["llvm-tools"], 148 } 149 }, 150 "producer toolchain", 151 ) 152 return value 153 154 155 def validate_build(value: Any) -> None: 156 value = contract._mapping(value, "producer build") 157 contract._exact( 158 set(value), 159 { 160 "rust_version", 161 "profile", 162 "ios_deployment_target", 163 "rust_flags", 164 "source_date_epoch", 165 "host", 166 "host_dylib_install_name", 167 "host_linker_reproducible", 168 "host_oso_prefix", 169 "targets", 170 "native_input_policy", 171 }, 172 "producer build fields", 173 ) 174 contract._exact(value["rust_version"], "1.99.0", "producer Rust version") 175 contract._exact(value["profile"], "release", "producer build profile") 176 contract._exact( 177 value["native_input_policy"], 178 "closed_apple_c_v1", 179 "producer native input policy", 180 ) 181 contract._exact( 182 value["ios_deployment_target"], "18.0", "producer deployment target" 183 ) 184 contract._exact(value["host"], "aarch64-apple-darwin", "producer host") 185 contract._exact( 186 value["host_dylib_install_name"], 187 "@rpath/libtera_ffi.dylib", 188 "producer host dylib install name", 189 ) 190 contract._exact( 191 value["host_linker_reproducible"] is True, 192 True, 193 "producer host linker reproducibility", 194 ) 195 contract._exact( 196 value["host_oso_prefix"], "{extbuild_root}", "producer host debug-map prefix" 197 ) 198 contract._exact( 199 value["rust_flags"], 200 [ 201 "--remap-path-prefix={producer_root}=/tera", 202 "--remap-path-prefix={cargo_home}=/cargo", 203 "--remap-path-prefix={extbuild_root}=/build", 204 ], 205 "producer Rust flags", 206 ) 207 contract._exact( 208 value["targets"], 209 [ 210 "aarch64-apple-ios", 211 "aarch64-apple-ios-sim", 212 "aarch64-apple-darwin", 213 ], 214 "producer targets", 215 ) 216 try: 217 contract.validate_source_date_epoch(value["source_date_epoch"]) 218 except contract.PackageContractError as error: 219 raise ProvenanceError("producer source epoch is invalid") from error 220 221 222 def library_rust_flags(build: dict[str, Any], target: str) -> list[str]: 223 if target == build["host"]: 224 return [ 225 f"-Clink-arg=-Wl,-install_name,{build['host_dylib_install_name']}", 226 "-Clink-arg=-Wl,-reproducible", 227 f"-Clink-arg=-Wl,-oso_prefix,{build['host_oso_prefix']}", 228 ] 229 return [] 230 231 232 def validate_foundation(cargo: dict[str, Any], lock: dict[str, Any]) -> None: 233 contract._exact( 234 lock.get("schema"), "radroots.lib.source-lock.v1", "foundation schema" 235 ) 236 contract._exact(lock.get("repository"), contract.LIB_REMOTE, "foundation remote") 237 if not contract.GIT_REVISION.fullmatch(str(lock.get("revision", ""))): 238 raise ProvenanceError("foundation revision is invalid") 239 for field in ( 240 "workspace_catalog_sha256", 241 "source_archive_sha256", 242 "lockfile_sha256", 243 ): 244 if not contract.SHA256.fullmatch(str(lock.get(field, ""))): 245 raise ProvenanceError("foundation digest is invalid") 246 dependencies = cargo["workspace"]["dependencies"] 247 selected = { 248 name: value 249 for name, value in dependencies.items() 250 if name.startswith("radroots_") 251 } 252 if not selected: 253 raise ProvenanceError("foundation dependency inventory is empty") 254 for value in selected.values(): 255 value = contract._mapping(value, "foundation Cargo dependency") 256 contract._exact(value.get("git"), lock["repository"], "foundation Cargo remote") 257 contract._exact(value.get("rev"), lock["revision"], "foundation Cargo revision") 258 contract._exact( 259 value.get("version"), "=" + lock["version"], "foundation Cargo version" 260 ) 261 262 263 def staged_diff_views(paths: list[str]) -> tuple[list[str], list[str]]: 264 arguments = [ 265 "diff", 266 "--cached", 267 "--raw", 268 "--no-abbrev", 269 "-z", 270 "--no-ext-diff", 271 "--no-textconv", 272 ] 273 return ( 274 [*arguments, "--ita-visible-in-index", *paths], 275 [*arguments, "--ita-invisible-in-index", *paths], 276 ) 277 278 279 def reject_intent_to_add(root: Path, inputs: list[str]) -> None: 280 visible, invisible = staged_diff_views(["--", *inputs]) 281 if command(root, ["git", *visible]) != command(root, ["git", *invisible]): 282 raise ProvenanceError("producer input is intent-to-add or index changed") 283 284 285 def source_snapshot(root: Path, inputs: list[str]) -> dict[str, Any]: 286 actual_root = ( 287 command(root, ["git", "rev-parse", "--show-toplevel"]).decode().strip() 288 ) 289 if actual_root != str(root): 290 raise ProvenanceError("producer source must use its own repository root") 291 reject_intent_to_add(root, inputs) 292 if command( 293 root, ["git", "ls-files", "--others", "--exclude-standard", "-z", "--", *inputs] 294 ): 295 raise ProvenanceError("producer inputs contain untracked source") 296 reject_ignored_source(root, inputs) 297 raw = command(root, ["git", "ls-files", "--stage", "-z", "--", *inputs]) 298 rows = raw.rstrip(b"\0").split(b"\0") if raw else [] 299 if not rows or len(rows) > MAX_INPUTS: 300 raise ProvenanceError("producer input inventory exceeds bounds") 301 files: dict[str, Any] = {} 302 tree: dict[str, Any] = {} 303 for row in rows: 304 relative, entry = staged_file(root, row) 305 files[relative] = entry 306 insert_tree(tree, relative, entry["mode"], entry["git_blob"]) 307 require_input_inventory(inputs, files) 308 return {"policy": "staged_inputs", "tree": tree_identity(tree), "files": files} 309 310 311 def require_input_inventory(inputs: list[str], files: dict[str, Any]) -> None: 312 for relative in inputs: 313 if not any( 314 name == relative or name.startswith(relative + "/") for name in files 315 ): 316 raise ProvenanceError("required producer input is missing from index") 317 318 319 def reject_ignored_source(root: Path, inputs: list[str]) -> None: 320 ignored = command( 321 root, 322 [ 323 "git", 324 "ls-files", 325 "--others", 326 "--ignored", 327 "--exclude-standard", 328 "-z", 329 "--", 330 *inputs, 331 ":(exclude)scripts/persona-verifier/.venv/**", 332 ":(exclude,glob)**/__pycache__/**", 333 ], 334 ) 335 if any(is_input(path, inputs) for path in ignored.decode().split("\0") if path): 336 raise ProvenanceError( 337 "producer inputs contain ignored source outside tool caches" 338 ) 339 340 341 def is_input(relative: str, inputs: list[str]) -> bool: 342 return any(relative == name or relative.startswith(name + "/") for name in inputs) 343 344 345 def staged_file(root: Path, row: bytes) -> tuple[str, dict[str, Any]]: 346 header, encoded_path = row.split(b"\t", 1) 347 mode, oid, stage = header.decode("ascii").split() 348 relative = encoded_path.decode("utf-8") 349 if stage != "0" or mode not in ("100644", "100755"): 350 raise ProvenanceError("producer input has unresolved or unsupported mode") 351 data = read_source(root, relative) 352 blob = command(root, ["git", "cat-file", "blob", oid]) 353 actual_mode = ( 354 "100755" if (root / relative).stat().st_mode & stat.S_IXUSR else "100644" 355 ) 356 if data != blob or mode != actual_mode: 357 raise ProvenanceError("producer worktree differs from staged source") 358 return relative, { 359 "mode": mode, 360 "git_blob": oid, 361 "bytes": len(data), 362 "sha256": hashlib.sha256(data).hexdigest(), 363 } 364 365 366 def reject_cargo_configuration(root: Path) -> None: 367 locations = [path / ".cargo" for path in (root, *root.parents)] 368 locations.append(Path(os.environ.get("CARGO_HOME", str(Path.home() / ".cargo")))) 369 for location in locations: 370 for name in ("config", "config.toml"): 371 path = location / name 372 if path.exists(): 373 value = contract._read_toml(path) 374 if {"build", "target", "env", "profile", "patch", "unstable"} & set( 375 value 376 ): 377 raise ProvenanceError( 378 "ungoverned Cargo configuration affects producer build" 379 ) 380 381 382 def insert_tree(tree: dict[str, Any], relative: str, mode: str, oid: str) -> None: 383 parts = relative.split("/") 384 for part in parts[:-1]: 385 tree = tree.setdefault(part, {}) 386 tree[parts[-1]] = (mode, oid) 387 388 389 def tree_identity(tree: dict[str, Any]) -> str: 390 rows = [] 391 for name, value in tree.items(): 392 if isinstance(value, dict): 393 mode, oid, key = "40000", tree_identity(value), name.encode() + b"/" 394 else: 395 mode, oid = value 396 key = name.encode() 397 rows.append((key, f"{mode} {name}".encode() + b"\0" + bytes.fromhex(oid))) 398 body = b"".join(row for _, row in sorted(rows)) 399 return hashlib.sha1( 400 b"tree " + str(len(body)).encode() + b"\0" + body, usedforsecurity=False 401 ).hexdigest() 402 403 404 def reject_build_overrides() -> None: 405 import ffi_native 406 407 ffi_native.reject_inherited(os.environ) 408 forbidden = ( 409 "RUSTFLAGS", 410 "RUSTC", 411 "RUSTC_WRAPPER", 412 "RUSTC_WORKSPACE_WRAPPER", 413 "CARGO_ENCODED_RUSTFLAGS", 414 "GIT_INDEX_FILE", 415 "GIT_DIR", 416 "GIT_WORK_TREE", 417 "GIT_OBJECT_DIRECTORY", 418 "GIT_ALTERNATE_OBJECT_DIRECTORIES", 419 "GIT_CONFIG_COUNT", 420 "GIT_CONFIG_PARAMETERS", 421 ) 422 if any(os.environ.get(name) for name in forbidden): 423 raise ProvenanceError("ungoverned Rust build override is active") 424 if any( 425 re.fullmatch( 426 r"CARGO_(BUILD_.*|PROFILE_.*|TARGET_.*_(RUSTFLAGS|LINKER|RUNNER))", name 427 ) 428 and name not in allowed_profile_overrides() 429 for name in os.environ 430 ): 431 raise ProvenanceError("ungoverned Cargo build override is active") 432 433 434 def allowed_profile_overrides() -> dict[str, str]: 435 # Extbuild's development debug policy affects the generator, not release libraries. 436 name = "CARGO_PROFILE_DEV_DEBUG" 437 if ( 438 os.environ.get("EXT_BUILD_RUN_ACTIVE") 439 and os.environ.get(name) == "line-tables-only" 440 ): 441 return {name: "line-tables-only"} 442 return {} 443 444 445 def feature_graph( 446 root: Path, package: str, target: str, *, environment: dict[str, str] | None = None 447 ) -> list[str]: 448 raw = command( 449 root, 450 [ 451 "cargo", 452 "tree", 453 "--locked", 454 "--offline", 455 "-p", 456 package, 457 "--target", 458 target, 459 "--edges", 460 "normal,build", 461 "--prefix", 462 "none", 463 "--format", 464 "{p}|{f}", 465 ], 466 environment=environment, 467 ).decode() 468 return sorted(set(raw.replace(str(root), "<producer-root>").splitlines()))