field_ios

In-the-field app for Radroots on iOS
git clone https://radroots.dev/git/field_ios.git
Log | Files | Refs | README | LICENSE

ffi_build.py (15006B)


      1 """Build and verify native candidates from the sole owned Rust workspace."""
      2 
      3 from __future__ import annotations
      4 
      5 import argparse
      6 import hashlib
      7 import json
      8 import os
      9 import plistlib
     10 import re
     11 import shutil
     12 import subprocess
     13 import sys
     14 import tempfile
     15 from pathlib import Path
     16 from typing import Any
     17 
     18 import ffi_artifacts as artifacts
     19 import ffi_installed as installed
     20 import ffi_native as native
     21 import ffi_provenance as provenance
     22 import ffi_notices as notices
     23 import ffi_source as source
     24 import package_contract as contract
     25 
     26 
     27 def build_roots(root: Path) -> tuple[Path, Path]:
     28     if not os.environ.get("EXT_BUILD_RUN_ACTIVE"):
     29         raise source.ProvenanceError("native artifact commands require extbuild")
     30     project = Path(os.environ["EXT_BUILD_PROJECT_DIR"]).resolve()
     31     target = Path(os.environ["CARGO_TARGET_DIR"]).resolve()
     32     if not target.is_relative_to(project) or project.is_relative_to(root):
     33         raise source.ProvenanceError("native outputs must use the external build root")
     34     return project, target
     35 
     36 
     37 def build_environment(
     38     root: Path, project: Path, config: dict[str, Any]
     39 ) -> dict[str, str]:
     40     source.reject_build_overrides()
     41     values = {
     42         "producer_root": str(root),
     43         "extbuild_root": str(project),
     44         "cargo_home": os.environ.get("CARGO_HOME", str(Path.home() / ".cargo")),
     45     }
     46     environment = dict(os.environ)
     47     environment.pop("TERA_CONSUMER_REVISION", None)
     48     environment.pop("DYLD_FALLBACK_LIBRARY_PATH", None)
     49     environment.update(
     50         {
     51             "CARGO_ENCODED_RUSTFLAGS": "\x1f".join(
     52                 flag.format(**values) for flag in config["build"]["rust_flags"]
     53             ),
     54             "IPHONEOS_DEPLOYMENT_TARGET": config["build"]["ios_deployment_target"],
     55             "SOURCE_DATE_EPOCH": str(config["build"]["source_date_epoch"]),
     56             "RADROOTS_LIB_REVISION": contract._read_toml(
     57                 root / config["foundation_lock"]
     58             )["revision"],
     59         }
     60     )
     61     return environment
     62 
     63 
     64 def run(
     65     root: Path, logs: Path, name: str, argv: list[str], environment: dict[str, str]
     66 ) -> Path:
     67     print(f"native candidate: {name}", flush=True)
     68     log = logs / f"{name}.txt"
     69     with log.open("wb") as output:
     70         try:
     71             result = subprocess.run(
     72                 argv,
     73                 cwd=root,
     74                 env=environment,
     75                 stdout=output,
     76                 stderr=subprocess.STDOUT,
     77                 timeout=1800,
     78                 check=False,
     79             )
     80         except (OSError, subprocess.TimeoutExpired) as error:
     81             raise source.ProvenanceError(
     82                 f"native command unavailable or timed out: {name}"
     83             ) from error
     84     if result.returncode or log.stat().st_size > 64 * 1024 * 1024:
     85         raise source.ProvenanceError(
     86             f"native command failed; retained diagnostic: {name}"
     87         )
     88     return log
     89 
     90 
     91 def library_command(root: Path, target: str, project: Path) -> list[str]:
     92     config = source.producer_contract(root)
     93     flags = [
     94         flag.format(extbuild_root=project)
     95         for flag in source.library_rust_flags(config["build"], target)
     96     ]
     97     command = [
     98         "cargo",
     99         "rustc" if flags else "build",
    100         "--manifest-path",
    101         str(root / "Cargo.toml"),
    102         "-p",
    103         "tera_ffi",
    104         "--lib",
    105         "--release",
    106         "--locked",
    107         "--target",
    108         target,
    109     ]
    110     return command + (["--", *flags] if flags else [])
    111 
    112 
    113 def build_libraries(
    114     root: Path,
    115     bundle: Path,
    116     target_root: Path,
    117     logs: Path,
    118     env: dict[str, str],
    119     records: dict[str, dict[str, Any]],
    120 ) -> None:
    121     for target in artifacts.TARGETS:
    122         run_native(
    123             root,
    124             logs,
    125             "build-" + target,
    126             library_command(root, target, Path(env["EXT_BUILD_PROJECT_DIR"])),
    127             env,
    128             records[target]["build"]["native"],
    129             records[target]["generator"]["native"],
    130         )
    131         extension = "dylib" if target == artifacts.TARGETS[-1] else "a"
    132         destination = bundle / "native" / target
    133         destination.mkdir(parents=True)
    134         shutil.copyfile(
    135             target_root / target / "release" / f"libtera_ffi.{extension}",
    136             destination / f"libtera_ffi.{extension}",
    137         )
    138 
    139 
    140 def generate_bindings(
    141     root: Path,
    142     bundle: Path,
    143     logs: Path,
    144     env: dict[str, str],
    145     host: dict[str, Any],
    146 ) -> None:
    147     generated = bundle / "generated"
    148     generated.mkdir()
    149     run_native(
    150         root,
    151         logs,
    152         "generate-swift",
    153         [
    154             "cargo",
    155             "run",
    156             "--manifest-path",
    157             str(root / "Cargo.toml"),
    158             "-p",
    159             "tera_bindgen",
    160             "--locked",
    161             "--",
    162             "generate",
    163             str(bundle / "native/aarch64-apple-darwin/libtera_ffi.dylib"),
    164             "--language",
    165             "swift",
    166             "--metadata-no-deps",
    167             "--out-dir",
    168             str(generated),
    169         ],
    170         env,
    171         host,
    172         host,
    173     )
    174     headers = bundle / "headers"
    175     headers.mkdir()
    176     shutil.copyfile(generated / "TeraFFI.h", headers / "TeraFFI.h")
    177     shutil.copyfile(generated / "TeraFFI.modulemap", headers / "module.modulemap")
    178 
    179 
    180 def package_framework(
    181     root: Path,
    182     bundle: Path,
    183     logs: Path,
    184     env: dict[str, str],
    185     context: dict[str, Any],
    186     host: dict[str, Any],
    187 ) -> None:
    188     argv = [context["tools"]["xcodebuild"]["path"], "-create-xcframework"]
    189     for target in artifacts.TARGETS[:2]:
    190         argv += [
    191             "-library",
    192             str(bundle / "native" / target / "libtera_ffi.a"),
    193             "-headers",
    194             str(bundle / "headers"),
    195         ]
    196     argv += ["-output", str(bundle / artifacts.FRAMEWORK)]
    197     run_native(root, logs, "package-xcframework", argv, env, context, host)
    198     canonicalize_framework_info(bundle / artifacts.FRAMEWORK / "Info.plist")
    199 
    200 
    201 def canonicalize_framework_info(path: Path) -> None:
    202     info = plistlib.loads(contract._read_regular(path))
    203     info["AvailableLibraries"].sort(key=lambda item: item["LibraryIdentifier"])
    204     path.write_bytes(plistlib.dumps(info, sort_keys=True))
    205 
    206 
    207 def generate_api(
    208     root: Path,
    209     bundle: Path,
    210     work: Path,
    211     logs: Path,
    212     env: dict[str, str],
    213     deployment: str,
    214     context: dict[str, Any],
    215     host: dict[str, Any],
    216 ) -> None:
    217     sdk = context["sdk"]["path"]
    218     module = work / "module"
    219     symbols = work / "symbols"
    220     module.mkdir()
    221     symbols.mkdir()
    222     common = [
    223         "-module-name",
    224         artifacts.MODULE,
    225         "-target",
    226         f"arm64-apple-ios{deployment}-simulator",
    227         "-sdk",
    228         sdk,
    229         "-I",
    230         str(bundle / "headers"),
    231         "-module-cache-path",
    232         str(work / "module-cache"),
    233     ]
    234     run_native(
    235         root,
    236         logs,
    237         "compile-swift-module",
    238         [
    239             context["tools"]["swiftc"]["path"],
    240             "-emit-module",
    241             "-parse-as-library",
    242             *common,
    243             "-emit-module-path",
    244             str(module / f"{artifacts.MODULE}.swiftmodule"),
    245             str(bundle / "generated/TeraKitBindings.swift"),
    246         ],
    247         env,
    248         context,
    249         host,
    250     )
    251     run_native(
    252         root,
    253         logs,
    254         "extract-swift-api",
    255         [
    256             context["tools"]["swift_symbolgraph_extract"]["path"],
    257             *common,
    258             "-I",
    259             str(module),
    260             "-minimum-access-level",
    261             "public",
    262             "-skip-inherited-docs",
    263             "-skip-synthesized-members",
    264             "-output-dir",
    265             str(symbols),
    266         ],
    267         env,
    268         context,
    269         host,
    270     )
    271     graph = json.loads((symbols / f"{artifacts.MODULE}.symbols.json").read_text())
    272     output = normalize_api(graph)
    273     (bundle / "api").mkdir()
    274     (bundle / "api/TeraKitBindings.symbols.json").write_bytes(
    275         (json.dumps(output, sort_keys=True, separators=(",", ":")) + "\n").encode()
    276     )
    277 
    278 
    279 def normalize_api(graph: dict[str, Any]) -> dict[str, Any]:
    280     symbols = [
    281         {
    282             "kind": item["kind"]["identifier"],
    283             "precise": item["identifier"]["precise"],
    284             "path": item["pathComponents"],
    285             "access": item["accessLevel"],
    286             "declaration": item.get("declarationFragments"),
    287         }
    288         for item in graph["symbols"]
    289     ]
    290     relationships = [
    291         {key: item[key] for key in ("kind", "source", "target")}
    292         for item in graph["relationships"]
    293     ]
    294     return {
    295         "schema": "radroots.swift-api-snapshot.v1",
    296         "generator": graph["metadata"]["generator"],
    297         "module": graph["module"],
    298         "symbols": sorted(symbols, key=lambda item: item["precise"]),
    299         "relationships": sorted(
    300             relationships,
    301             key=lambda item: (item["kind"], item["source"], item["target"]),
    302         ),
    303     }
    304 
    305 
    306 def record_abi(
    307     root: Path,
    308     bundle: Path,
    309     logs: Path,
    310     env: dict[str, str],
    311     records: dict[str, dict[str, Any]],
    312 ) -> None:
    313     symbols = {}
    314     reader = provenance.symbol_reader(
    315         root, source.producer_contract(root)["build"]["host"]
    316     )
    317     for target in artifacts.TARGETS:
    318         expected_reader = records[target]["build"]["symbol_reader_identity"]
    319         if native.file_identity(reader) != expected_reader:
    320             raise source.ProvenanceError("native symbol reader drift before use")
    321         extension = "dylib" if target == artifacts.TARGETS[-1] else "a"
    322         log = run_native(
    323             root,
    324             logs,
    325             "symbols-" + target,
    326             [
    327                 str(reader),
    328                 "--extern-only",
    329                 "--just-symbol-name",
    330                 "--defined-only",
    331                 str(bundle / "native" / target / f"libtera_ffi.{extension}"),
    332             ],
    333             env,
    334             records[target]["build"]["native"],
    335             records[target]["generator"]["native"],
    336         )
    337         if native.file_identity(reader) != expected_reader:
    338             raise source.ProvenanceError("native symbol reader drift after use")
    339         symbols[target] = sorted(
    340             set(
    341                 re.findall(
    342                     r"^_((?:ffi|uniffi)_tera_ffi_[A-Za-z0-9_]+)$",
    343                     log.read_text(),
    344                     re.MULTILINE,
    345                 )
    346             )
    347         )
    348     (bundle / "abi_symbols.json").write_bytes(provenance.encoded(symbols))
    349 
    350 
    351 def capture_sources(root: Path) -> dict[str, dict[str, Any]]:
    352     return {target: provenance.capture(root, target) for target in artifacts.TARGETS}
    353 
    354 
    355 def candidate_path(project: Path, records: dict[str, dict[str, Any]]) -> Path:
    356     tree = records[artifacts.TARGETS[0]]["source"]["tree"]
    357     identity = hashlib.sha256(provenance.encoded(records)).hexdigest()
    358     return project / "target/tera_ffi/candidates" / (tree + "-" + identity)
    359 
    360 
    361 def run_native(
    362     root: Path,
    363     logs: Path,
    364     name: str,
    365     argv: list[str],
    366     environment: dict[str, str],
    367     context: dict[str, Any],
    368     host: dict[str, Any],
    369 ) -> Path:
    370     bound = native.execution_environment(root, environment, context, host)
    371     try:
    372         return run(root, logs, name, argv, bound)
    373     finally:
    374         native.verify(root, context)
    375         native.verify(root, host)
    376 
    377 
    378 def build(
    379     root: Path,
    380     project: Path,
    381     target_root: Path,
    382     records: dict[str, dict[str, Any]],
    383     destination: Path,
    384 ) -> dict[str, Any]:
    385     config = source.producer_contract(root)
    386     environment = build_environment(root, project, config)
    387     host = records[native.HOST]["generator"]["native"]
    388     simulator = records["aarch64-apple-ios-sim"]["build"]["native"]
    389     destination.parent.mkdir(parents=True, exist_ok=True)
    390     logs = Path(tempfile.mkdtemp(prefix="build-", dir=destination.parent))
    391     # Logs survive failure. Only this invocation's temporary workspace is cleaned.
    392     with tempfile.TemporaryDirectory(
    393         prefix="work-", dir=destination.parent
    394     ) as temporary:
    395         work = Path(temporary)
    396         bundle = work / "bundle"
    397         bundle.mkdir()
    398         build_libraries(root, bundle, target_root, logs, environment, records)
    399         generate_bindings(root, bundle, logs, environment, host)
    400         package_framework(root, bundle, logs, environment, simulator, host)
    401         generate_api(
    402             root,
    403             bundle,
    404             work,
    405             logs,
    406             environment,
    407             config["build"]["ios_deployment_target"],
    408             simulator,
    409             host,
    410         )
    411         record_abi(root, bundle, logs, environment, records)
    412         notices.install(
    413             root,
    414             bundle,
    415             records[native.HOST]["redistribution"],
    416             records[native.HOST]["foundation"],
    417         )
    418         if capture_sources(root) != records:
    419             raise source.ProvenanceError(
    420                 "producer changed during native artifact build"
    421             )
    422         (bundle / "source").mkdir()
    423         for target, record in records.items():
    424             (bundle / "source" / f"{target}.json").write_bytes(
    425                 provenance.encoded(record)
    426             )
    427         result = artifacts.manifest(bundle, records)
    428         (bundle / artifacts.MANIFEST).write_bytes(provenance.encoded(result))
    429         artifacts.check(bundle, records)
    430         if destination.exists():
    431             raise source.ProvenanceError("native candidate destination already exists")
    432         bundle.rename(destination)
    433     return result
    434 
    435 
    436 def main() -> int:
    437     parser = argparse.ArgumentParser(description=__doc__)
    438     parser.add_argument("mode", choices=("build", "check", "install", "clean"))
    439     args = parser.parse_args()
    440     root = Path(__file__).resolve().parent.parent
    441     try:
    442         project, target_root = build_roots(root)
    443         if args.mode == "clean":
    444             candidates = project / "target/tera_ffi/candidates"
    445             if candidates.is_symlink() or candidates.resolve() != candidates:
    446                 raise source.ProvenanceError("candidate cache contains a symlink")
    447             if candidates.exists():
    448                 shutil.rmtree(candidates)
    449             print("native candidate cache removed; Cargo incremental cache retained")
    450             return 0
    451         records = capture_sources(root)
    452         tree = records[artifacts.TARGETS[0]]["source"]["tree"]
    453         destination = candidate_path(project, records)
    454         if args.mode in ("build", "install") and not destination.exists():
    455             build(root, project, target_root, records, destination)
    456         artifacts.check(destination, records)
    457         if args.mode == "install":
    458             installed.install(root, destination, records)
    459     except (
    460         source.ProvenanceError,
    461         contract.PackageContractError,
    462         OSError,
    463         ValueError,
    464         KeyError,
    465     ) as error:
    466         print(f"native candidate: {error}", file=sys.stderr)
    467         return 1
    468     print(f"native candidate {args.mode}: tree={tree}; {destination}")
    469     return 0
    470 
    471 
    472 if __name__ == "__main__":
    473     raise SystemExit(main())