ffi_build.py (15006B)
1 """Build and verify native candidates from the sole owned Rust workspace.""" 2 3 from __future__ import annotations 4 5 import argparse 6 import hashlib 7 import json 8 import os 9 import plistlib 10 import re 11 import shutil 12 import subprocess 13 import sys 14 import tempfile 15 from pathlib import Path 16 from typing import Any 17 18 import ffi_artifacts as artifacts 19 import ffi_installed as installed 20 import ffi_native as native 21 import ffi_provenance as provenance 22 import ffi_notices as notices 23 import ffi_source as source 24 import package_contract as contract 25 26 27 def build_roots(root: Path) -> tuple[Path, Path]: 28 if not os.environ.get("EXT_BUILD_RUN_ACTIVE"): 29 raise source.ProvenanceError("native artifact commands require extbuild") 30 project = Path(os.environ["EXT_BUILD_PROJECT_DIR"]).resolve() 31 target = Path(os.environ["CARGO_TARGET_DIR"]).resolve() 32 if not target.is_relative_to(project) or project.is_relative_to(root): 33 raise source.ProvenanceError("native outputs must use the external build root") 34 return project, target 35 36 37 def build_environment( 38 root: Path, project: Path, config: dict[str, Any] 39 ) -> dict[str, str]: 40 source.reject_build_overrides() 41 values = { 42 "producer_root": str(root), 43 "extbuild_root": str(project), 44 "cargo_home": os.environ.get("CARGO_HOME", str(Path.home() / ".cargo")), 45 } 46 environment = dict(os.environ) 47 environment.pop("TERA_CONSUMER_REVISION", None) 48 environment.pop("DYLD_FALLBACK_LIBRARY_PATH", None) 49 environment.update( 50 { 51 "CARGO_ENCODED_RUSTFLAGS": "\x1f".join( 52 flag.format(**values) for flag in config["build"]["rust_flags"] 53 ), 54 "IPHONEOS_DEPLOYMENT_TARGET": config["build"]["ios_deployment_target"], 55 "SOURCE_DATE_EPOCH": str(config["build"]["source_date_epoch"]), 56 "RADROOTS_LIB_REVISION": contract._read_toml( 57 root / config["foundation_lock"] 58 )["revision"], 59 } 60 ) 61 return environment 62 63 64 def run( 65 root: Path, logs: Path, name: str, argv: list[str], environment: dict[str, str] 66 ) -> Path: 67 print(f"native candidate: {name}", flush=True) 68 log = logs / f"{name}.txt" 69 with log.open("wb") as output: 70 try: 71 result = subprocess.run( 72 argv, 73 cwd=root, 74 env=environment, 75 stdout=output, 76 stderr=subprocess.STDOUT, 77 timeout=1800, 78 check=False, 79 ) 80 except (OSError, subprocess.TimeoutExpired) as error: 81 raise source.ProvenanceError( 82 f"native command unavailable or timed out: {name}" 83 ) from error 84 if result.returncode or log.stat().st_size > 64 * 1024 * 1024: 85 raise source.ProvenanceError( 86 f"native command failed; retained diagnostic: {name}" 87 ) 88 return log 89 90 91 def library_command(root: Path, target: str, project: Path) -> list[str]: 92 config = source.producer_contract(root) 93 flags = [ 94 flag.format(extbuild_root=project) 95 for flag in source.library_rust_flags(config["build"], target) 96 ] 97 command = [ 98 "cargo", 99 "rustc" if flags else "build", 100 "--manifest-path", 101 str(root / "Cargo.toml"), 102 "-p", 103 "tera_ffi", 104 "--lib", 105 "--release", 106 "--locked", 107 "--target", 108 target, 109 ] 110 return command + (["--", *flags] if flags else []) 111 112 113 def build_libraries( 114 root: Path, 115 bundle: Path, 116 target_root: Path, 117 logs: Path, 118 env: dict[str, str], 119 records: dict[str, dict[str, Any]], 120 ) -> None: 121 for target in artifacts.TARGETS: 122 run_native( 123 root, 124 logs, 125 "build-" + target, 126 library_command(root, target, Path(env["EXT_BUILD_PROJECT_DIR"])), 127 env, 128 records[target]["build"]["native"], 129 records[target]["generator"]["native"], 130 ) 131 extension = "dylib" if target == artifacts.TARGETS[-1] else "a" 132 destination = bundle / "native" / target 133 destination.mkdir(parents=True) 134 shutil.copyfile( 135 target_root / target / "release" / f"libtera_ffi.{extension}", 136 destination / f"libtera_ffi.{extension}", 137 ) 138 139 140 def generate_bindings( 141 root: Path, 142 bundle: Path, 143 logs: Path, 144 env: dict[str, str], 145 host: dict[str, Any], 146 ) -> None: 147 generated = bundle / "generated" 148 generated.mkdir() 149 run_native( 150 root, 151 logs, 152 "generate-swift", 153 [ 154 "cargo", 155 "run", 156 "--manifest-path", 157 str(root / "Cargo.toml"), 158 "-p", 159 "tera_bindgen", 160 "--locked", 161 "--", 162 "generate", 163 str(bundle / "native/aarch64-apple-darwin/libtera_ffi.dylib"), 164 "--language", 165 "swift", 166 "--metadata-no-deps", 167 "--out-dir", 168 str(generated), 169 ], 170 env, 171 host, 172 host, 173 ) 174 headers = bundle / "headers" 175 headers.mkdir() 176 shutil.copyfile(generated / "TeraFFI.h", headers / "TeraFFI.h") 177 shutil.copyfile(generated / "TeraFFI.modulemap", headers / "module.modulemap") 178 179 180 def package_framework( 181 root: Path, 182 bundle: Path, 183 logs: Path, 184 env: dict[str, str], 185 context: dict[str, Any], 186 host: dict[str, Any], 187 ) -> None: 188 argv = [context["tools"]["xcodebuild"]["path"], "-create-xcframework"] 189 for target in artifacts.TARGETS[:2]: 190 argv += [ 191 "-library", 192 str(bundle / "native" / target / "libtera_ffi.a"), 193 "-headers", 194 str(bundle / "headers"), 195 ] 196 argv += ["-output", str(bundle / artifacts.FRAMEWORK)] 197 run_native(root, logs, "package-xcframework", argv, env, context, host) 198 canonicalize_framework_info(bundle / artifacts.FRAMEWORK / "Info.plist") 199 200 201 def canonicalize_framework_info(path: Path) -> None: 202 info = plistlib.loads(contract._read_regular(path)) 203 info["AvailableLibraries"].sort(key=lambda item: item["LibraryIdentifier"]) 204 path.write_bytes(plistlib.dumps(info, sort_keys=True)) 205 206 207 def generate_api( 208 root: Path, 209 bundle: Path, 210 work: Path, 211 logs: Path, 212 env: dict[str, str], 213 deployment: str, 214 context: dict[str, Any], 215 host: dict[str, Any], 216 ) -> None: 217 sdk = context["sdk"]["path"] 218 module = work / "module" 219 symbols = work / "symbols" 220 module.mkdir() 221 symbols.mkdir() 222 common = [ 223 "-module-name", 224 artifacts.MODULE, 225 "-target", 226 f"arm64-apple-ios{deployment}-simulator", 227 "-sdk", 228 sdk, 229 "-I", 230 str(bundle / "headers"), 231 "-module-cache-path", 232 str(work / "module-cache"), 233 ] 234 run_native( 235 root, 236 logs, 237 "compile-swift-module", 238 [ 239 context["tools"]["swiftc"]["path"], 240 "-emit-module", 241 "-parse-as-library", 242 *common, 243 "-emit-module-path", 244 str(module / f"{artifacts.MODULE}.swiftmodule"), 245 str(bundle / "generated/TeraKitBindings.swift"), 246 ], 247 env, 248 context, 249 host, 250 ) 251 run_native( 252 root, 253 logs, 254 "extract-swift-api", 255 [ 256 context["tools"]["swift_symbolgraph_extract"]["path"], 257 *common, 258 "-I", 259 str(module), 260 "-minimum-access-level", 261 "public", 262 "-skip-inherited-docs", 263 "-skip-synthesized-members", 264 "-output-dir", 265 str(symbols), 266 ], 267 env, 268 context, 269 host, 270 ) 271 graph = json.loads((symbols / f"{artifacts.MODULE}.symbols.json").read_text()) 272 output = normalize_api(graph) 273 (bundle / "api").mkdir() 274 (bundle / "api/TeraKitBindings.symbols.json").write_bytes( 275 (json.dumps(output, sort_keys=True, separators=(",", ":")) + "\n").encode() 276 ) 277 278 279 def normalize_api(graph: dict[str, Any]) -> dict[str, Any]: 280 symbols = [ 281 { 282 "kind": item["kind"]["identifier"], 283 "precise": item["identifier"]["precise"], 284 "path": item["pathComponents"], 285 "access": item["accessLevel"], 286 "declaration": item.get("declarationFragments"), 287 } 288 for item in graph["symbols"] 289 ] 290 relationships = [ 291 {key: item[key] for key in ("kind", "source", "target")} 292 for item in graph["relationships"] 293 ] 294 return { 295 "schema": "radroots.swift-api-snapshot.v1", 296 "generator": graph["metadata"]["generator"], 297 "module": graph["module"], 298 "symbols": sorted(symbols, key=lambda item: item["precise"]), 299 "relationships": sorted( 300 relationships, 301 key=lambda item: (item["kind"], item["source"], item["target"]), 302 ), 303 } 304 305 306 def record_abi( 307 root: Path, 308 bundle: Path, 309 logs: Path, 310 env: dict[str, str], 311 records: dict[str, dict[str, Any]], 312 ) -> None: 313 symbols = {} 314 reader = provenance.symbol_reader( 315 root, source.producer_contract(root)["build"]["host"] 316 ) 317 for target in artifacts.TARGETS: 318 expected_reader = records[target]["build"]["symbol_reader_identity"] 319 if native.file_identity(reader) != expected_reader: 320 raise source.ProvenanceError("native symbol reader drift before use") 321 extension = "dylib" if target == artifacts.TARGETS[-1] else "a" 322 log = run_native( 323 root, 324 logs, 325 "symbols-" + target, 326 [ 327 str(reader), 328 "--extern-only", 329 "--just-symbol-name", 330 "--defined-only", 331 str(bundle / "native" / target / f"libtera_ffi.{extension}"), 332 ], 333 env, 334 records[target]["build"]["native"], 335 records[target]["generator"]["native"], 336 ) 337 if native.file_identity(reader) != expected_reader: 338 raise source.ProvenanceError("native symbol reader drift after use") 339 symbols[target] = sorted( 340 set( 341 re.findall( 342 r"^_((?:ffi|uniffi)_tera_ffi_[A-Za-z0-9_]+)$", 343 log.read_text(), 344 re.MULTILINE, 345 ) 346 ) 347 ) 348 (bundle / "abi_symbols.json").write_bytes(provenance.encoded(symbols)) 349 350 351 def capture_sources(root: Path) -> dict[str, dict[str, Any]]: 352 return {target: provenance.capture(root, target) for target in artifacts.TARGETS} 353 354 355 def candidate_path(project: Path, records: dict[str, dict[str, Any]]) -> Path: 356 tree = records[artifacts.TARGETS[0]]["source"]["tree"] 357 identity = hashlib.sha256(provenance.encoded(records)).hexdigest() 358 return project / "target/tera_ffi/candidates" / (tree + "-" + identity) 359 360 361 def run_native( 362 root: Path, 363 logs: Path, 364 name: str, 365 argv: list[str], 366 environment: dict[str, str], 367 context: dict[str, Any], 368 host: dict[str, Any], 369 ) -> Path: 370 bound = native.execution_environment(root, environment, context, host) 371 try: 372 return run(root, logs, name, argv, bound) 373 finally: 374 native.verify(root, context) 375 native.verify(root, host) 376 377 378 def build( 379 root: Path, 380 project: Path, 381 target_root: Path, 382 records: dict[str, dict[str, Any]], 383 destination: Path, 384 ) -> dict[str, Any]: 385 config = source.producer_contract(root) 386 environment = build_environment(root, project, config) 387 host = records[native.HOST]["generator"]["native"] 388 simulator = records["aarch64-apple-ios-sim"]["build"]["native"] 389 destination.parent.mkdir(parents=True, exist_ok=True) 390 logs = Path(tempfile.mkdtemp(prefix="build-", dir=destination.parent)) 391 # Logs survive failure. Only this invocation's temporary workspace is cleaned. 392 with tempfile.TemporaryDirectory( 393 prefix="work-", dir=destination.parent 394 ) as temporary: 395 work = Path(temporary) 396 bundle = work / "bundle" 397 bundle.mkdir() 398 build_libraries(root, bundle, target_root, logs, environment, records) 399 generate_bindings(root, bundle, logs, environment, host) 400 package_framework(root, bundle, logs, environment, simulator, host) 401 generate_api( 402 root, 403 bundle, 404 work, 405 logs, 406 environment, 407 config["build"]["ios_deployment_target"], 408 simulator, 409 host, 410 ) 411 record_abi(root, bundle, logs, environment, records) 412 notices.install( 413 root, 414 bundle, 415 records[native.HOST]["redistribution"], 416 records[native.HOST]["foundation"], 417 ) 418 if capture_sources(root) != records: 419 raise source.ProvenanceError( 420 "producer changed during native artifact build" 421 ) 422 (bundle / "source").mkdir() 423 for target, record in records.items(): 424 (bundle / "source" / f"{target}.json").write_bytes( 425 provenance.encoded(record) 426 ) 427 result = artifacts.manifest(bundle, records) 428 (bundle / artifacts.MANIFEST).write_bytes(provenance.encoded(result)) 429 artifacts.check(bundle, records) 430 if destination.exists(): 431 raise source.ProvenanceError("native candidate destination already exists") 432 bundle.rename(destination) 433 return result 434 435 436 def main() -> int: 437 parser = argparse.ArgumentParser(description=__doc__) 438 parser.add_argument("mode", choices=("build", "check", "install", "clean")) 439 args = parser.parse_args() 440 root = Path(__file__).resolve().parent.parent 441 try: 442 project, target_root = build_roots(root) 443 if args.mode == "clean": 444 candidates = project / "target/tera_ffi/candidates" 445 if candidates.is_symlink() or candidates.resolve() != candidates: 446 raise source.ProvenanceError("candidate cache contains a symlink") 447 if candidates.exists(): 448 shutil.rmtree(candidates) 449 print("native candidate cache removed; Cargo incremental cache retained") 450 return 0 451 records = capture_sources(root) 452 tree = records[artifacts.TARGETS[0]]["source"]["tree"] 453 destination = candidate_path(project, records) 454 if args.mode in ("build", "install") and not destination.exists(): 455 build(root, project, target_root, records, destination) 456 artifacts.check(destination, records) 457 if args.mode == "install": 458 installed.install(root, destination, records) 459 except ( 460 source.ProvenanceError, 461 contract.PackageContractError, 462 OSError, 463 ValueError, 464 KeyError, 465 ) as error: 466 print(f"native candidate: {error}", file=sys.stderr) 467 return 1 468 print(f"native candidate {args.mode}: tree={tree}; {destination}") 469 return 0 470 471 472 if __name__ == "__main__": 473 raise SystemExit(main())