test_ffi_native_cache.py (12040B)
1 """Real pinned Cargo/cc rebuild controls; disposable contexts are not admission.""" 2 3 from __future__ import annotations 4 5 import copy 6 import hashlib 7 import importlib 8 import json 9 import os 10 import shlex 11 import subprocess 12 import sys 13 import tempfile 14 import time 15 import unittest 16 from pathlib import Path 17 from unittest.mock import patch 18 19 SCRIPTS = Path(__file__).resolve().parent 20 if str(SCRIPTS) not in sys.path: 21 sys.path.insert(0, str(SCRIPTS)) 22 23 builder = importlib.import_module("ffi_build") 24 native = importlib.import_module("ffi_native") 25 26 27 class NativeCargoCacheTests(unittest.TestCase): 28 @classmethod 29 def setUpClass(cls) -> None: 30 if not os.environ.get("EXT_BUILD_RUN_ACTIVE"): 31 raise RuntimeError("real native cache tests require extbuild") 32 cls.capsule = SCRIPTS.parent 33 cls.platform = native.capture(cls.capsule, native.HOST) 34 35 def setUp(self) -> None: 36 project, self.target_root = builder.build_roots(self.capsule) 37 directory = project / "target/tera_ffi/native_cache_tests" 38 directory.mkdir(parents=True, exist_ok=True) 39 self.root = Path(tempfile.mkdtemp(prefix="fixture-", dir=directory)).resolve() 40 self.environment = native.inspection_environment() 41 self.target_before = self.environment["CARGO_TARGET_DIR"] 42 self.package = "tera_native_cache_" + self.root.name.replace("-", "_") 43 (self.root / "src").mkdir() 44 (self.root / "Cargo.toml").write_text( 45 f'[package]\nname="{self.package}"\nversion="0.0.0"\nedition="2024"\n' 46 '[build-dependencies]\ncc="=1.4.0"\nfind-msvc-tools="=0.1.9"\nshlex="=2.0.1"\n' 47 ) 48 (self.root / "build.rs").write_text( 49 'fn main() { println!("cargo:rerun-if-changed=native.c"); ' 50 'cc::Build::new().file("native.c").compile("native_cache_control"); }\n' 51 ) 52 (self.root / "native.c").write_text( 53 "int native_value(void) { return C137_COMPILER_VALUE + C137_SDK_VALUE; }\n" 54 ) 55 (self.root / "src/main.rs").write_text( 56 'unsafe extern "C" { fn native_value() -> i32; } ' 57 'fn main() { println!("{}", unsafe { native_value() }); }\n' 58 ) 59 self.wrapper = self.root / "clang" 60 self.invocations = self.root / "compiler_invocations.txt" 61 self.sdk = self.root / "MacOSX.sdk" 62 self.sdk.mkdir() 63 self.header = self.sdk / "native_value.h" 64 self.header.write_text("#define C137_SDK_VALUE 0\n") 65 self.write_compiler(1) 66 self.observations = [] 67 argv = [ 68 "cargo", 69 "generate-lockfile", 70 "--offline", 71 "--manifest-path", 72 str(self.root / "Cargo.toml"), 73 ] 74 with (self.root / "lock.txt").open("wb") as log: 75 subprocess.run( 76 argv, 77 cwd=self.root, 78 env=self.environment, 79 stdout=log, 80 stderr=subprocess.STDOUT, 81 check=True, 82 timeout=120, 83 ) 84 self.addCleanup(self.save_observations) 85 86 def write_compiler(self, value: int) -> None: 87 self.wrapper.write_text( 88 '#!/bin/sh\ncase " $* " in *native.c*) printf "compiled\\n" >> ' 89 + shlex.quote(str(self.invocations)) 90 + ";; esac\nexec " 91 + shlex.quote(self.platform["tools"]["compiler"]["path"]) 92 + f' -DC137_COMPILER_VALUE={value} "$@" -isysroot ' 93 + shlex.quote(self.platform["sdk"]["path"]) 94 + "\n" 95 ) 96 self.wrapper.chmod(0o755) 97 98 def context(self, target: str = native.HOST) -> dict: 99 context = copy.deepcopy(self.platform) 100 context["target"] = target 101 context["inherited"]["path_sha256"] = hashlib.sha256( 102 self.environment["PATH"].encode() 103 ).hexdigest() 104 context["tools"]["compiler"] = { 105 "path": str(self.wrapper), 106 "resolved_path": str(self.wrapper), 107 **native.file_identity(self.wrapper), 108 } 109 context["platform_sdk"] = context["sdk"] 110 context["sdk"] = native.sdk_identity(self.sdk) 111 context["compiler_arguments"] += [ 112 "-isysroot", 113 context["platform_sdk"]["path"], 114 "-include", 115 str(self.header), 116 ] 117 context["linker_driver_arguments"] += [ 118 "-isysroot", 119 context["platform_sdk"]["path"], 120 ] 121 return context 122 123 def fence(self, root: Path, context: dict) -> None: 124 self.assertEqual(root, self.root) 125 self.assertEqual( 126 hashlib.sha256(self.environment["PATH"].encode()).hexdigest(), 127 context["inherited"]["path_sha256"], 128 ) 129 compiler = context["tools"]["compiler"] 130 self.assertEqual( 131 native.file_identity(Path(compiler["resolved_path"])), 132 {key: compiler[key] for key in ("sha256", "bytes")}, 133 ) 134 self.assertEqual( 135 native.sdk_identity(Path(context["sdk"]["path"])), context["sdk"] 136 ) 137 self.assertEqual( 138 native.sdk_identity(Path(context["platform_sdk"]["path"])), 139 context["platform_sdk"], 140 ) 141 self.assertEqual( 142 native.sdk_identity(Path(context["compiler_resources"]["path"])), 143 context["compiler_resources"], 144 ) 145 for key in ("linker", "archiver", "ranlib"): 146 tool = context["tools"][key] 147 self.assertEqual( 148 native.file_identity(Path(tool["resolved_path"])), 149 {name: tool[name] for name in ("sha256", "bytes")}, 150 ) 151 152 def execute( 153 self, name: str, context: dict, host: dict, *, generator: bool = False 154 ) -> dict: 155 argv = [ 156 "cargo", 157 "run", 158 "--manifest-path", 159 str(self.root / "Cargo.toml"), 160 "--offline", 161 "--locked", 162 "--release", 163 "-vv", 164 ] 165 if not generator: 166 argv += ["--target", native.HOST] 167 with patch.object(native, "verify", side_effect=self.fence): 168 started = time.monotonic() 169 log = builder.run_native( 170 self.root, self.root, name, argv, self.environment, context, host 171 ) 172 elapsed = time.monotonic() - started 173 raw = log.read_text() 174 relative = "release" if generator else native.HOST + "/release" 175 binary = self.target_root / relative / self.package 176 observed = { 177 "name": name, 178 "argv": argv, 179 "exit_code": 0, 180 "value": int(raw.splitlines()[-1]), 181 "log": str(log), 182 "compiler_calls": len(self.invocations.read_text().splitlines()), 183 "binary_mtime_ns": binary.stat().st_mtime_ns, 184 "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), 185 "context": context, 186 "host": host, 187 "cargo_target_dir": self.environment["CARGO_TARGET_DIR"], 188 "elapsed_seconds": elapsed, 189 } 190 self.assertEqual(observed["cargo_target_dir"], self.target_before) 191 self.observations.append(observed) 192 return observed 193 194 def save_observations(self) -> None: 195 (self.root / "observations.json").write_text( 196 json.dumps( 197 { 198 "fixture": "Real clang/ld/ar/ranlib and platform SDK, pinned cc1.4.0 C archive and executed Rust binary. Disposable compiler wrapper and SDK include-header context replace public selection; real input bytes are fenced before/after each invocation. No production capture/public admission assertion or Tera artifact qualification.", 199 "root": str(self.root), 200 "observations": self.observations, 201 }, 202 sort_keys=True, 203 indent=2, 204 ) 205 + "\n" 206 ) 207 print("native cache observations: " + str(self.root / "observations.json")) 208 209 def assert_reused(self, first: dict, second: dict) -> None: 210 self.assertEqual(second["compiler_calls"], first["compiler_calls"]) 211 self.assertEqual(second["binary_mtime_ns"], first["binary_mtime_ns"]) 212 self.assertEqual(second["binary_sha256"], first["binary_sha256"]) 213 214 def test_same_path_compiler_bytes_rebuild_real_cargo_cc_and_reuse( 215 self, 216 ) -> None: 217 original = self.context() 218 first = self.execute("first", original, original) 219 self.assertEqual(first["value"], 1) 220 self.assert_reused(first, self.execute("stable-first", original, original)) 221 self.write_compiler(2) 222 compiler = self.context() 223 second = self.execute("changed-compiler", compiler, compiler) 224 self.assertEqual(second["value"], 2) 225 self.assertGreater(second["compiler_calls"], first["compiler_calls"]) 226 self.assert_reused(second, self.execute("stable-compiler", compiler, compiler)) 227 228 def test_same_path_sdk_header_bytes_rebuild_real_cargo_cc_and_reuse(self) -> None: 229 original = self.context() 230 first = self.execute("sdk-first", original, original) 231 self.assertEqual(first["value"], 1) 232 self.assert_reused(first, self.execute("sdk-stable-first", original, original)) 233 self.header.write_text("#define C137_SDK_VALUE 3\n") 234 sdk = self.context() 235 third = self.execute("changed-sdk", sdk, sdk) 236 self.assertEqual(third["value"], 4) 237 self.assertGreater(third["compiler_calls"], first["compiler_calls"]) 238 self.assert_reused(third, self.execute("stable-sdk", sdk, sdk)) 239 240 def test_metadata_suppressed_cc_builder_rebuilds_via_cargo_fingerprint( 241 self, 242 ) -> None: 243 (self.root / "build.rs").write_text( 244 'fn main() { println!("cargo:rerun-if-changed=native.c"); ' 245 'cc::Build::new().cargo_metadata(false).file("native.c").compile("native_cache_control"); ' 246 'println!("cargo:rustc-link-search=native={}", std::env::var("OUT_DIR").unwrap()); ' 247 'println!("cargo:rustc-link-lib=static=native_cache_control"); }\n' 248 ) 249 original = self.context() 250 first = self.execute("metadata-first", original, original) 251 self.assertEqual(first["value"], 1) 252 self.assert_reused(first, self.execute("metadata-stable", original, original)) 253 self.write_compiler(2) 254 changed = self.context() 255 second = self.execute("metadata-compiler-changed", changed, changed) 256 self.assertEqual(second["value"], 2) 257 self.assertGreater(second["compiler_calls"], first["compiler_calls"]) 258 self.assert_reused( 259 second, self.execute("metadata-rebuilt-stable", changed, changed) 260 ) 261 262 def test_independent_host_context_invalidates_real_generator_cache(self) -> None: 263 target = self.context("aarch64-apple-ios") 264 # Keep this independently selected target context unchanged while only 265 # the host compiler fixture changes. The executed fixture is Darwin. 266 target["tools"]["compiler"] = copy.deepcopy(self.platform["tools"]["compiler"]) 267 target["sdk"] = target["platform_sdk"] 268 target["compiler_arguments"] = self.platform["compiler_arguments"] 269 host = self.context() 270 first = self.execute("generator-first", target, host, generator=True) 271 self.assertEqual(first["value"], 1) 272 self.assert_reused( 273 first, self.execute("generator-stable", target, host, generator=True) 274 ) 275 self.write_compiler(2) 276 changed_host = self.context() 277 current = self.execute( 278 "generator-host-changed", target, changed_host, generator=True 279 ) 280 self.assertEqual(current["value"], 2) 281 self.assertGreater(current["compiler_calls"], first["compiler_calls"]) 282 self.assert_reused( 283 current, 284 self.execute("generator-host-stable", target, changed_host, generator=True), 285 ) 286 287 288 if __name__ == "__main__": 289 unittest.main()