app_source.py (11587B)
1 """Bounded staged authored-app identity, separate from the native graph.""" 2 3 from __future__ import annotations 4 5 import argparse 6 import hashlib 7 import json 8 import os 9 import selectors 10 import stat 11 import subprocess 12 import sys 13 import time 14 from contextlib import ExitStack 15 from pathlib import Path 16 from typing import Any 17 18 import ffi_source as source 19 import package_contract as contract 20 21 MAX_BYTES = source.MAX_BYTES 22 MAX_INPUTS = source.MAX_INPUTS 23 MAX_SECONDS = 120 24 INPUTS = ["Package.swift", "project.yml", "Tera"] 25 EXCLUDED = ["Tera/Generated", "Tera/Frameworks"] 26 GIT_OVERRIDES = { 27 "GIT_INDEX_FILE", 28 "GIT_DIR", 29 "GIT_WORK_TREE", 30 "GIT_COMMON_DIR", 31 "GIT_OBJECT_DIRECTORY", 32 "GIT_ALTERNATE_OBJECT_DIRECTORIES", 33 "GIT_NAMESPACE", 34 "GIT_CONFIG", 35 "GIT_CONFIG_SYSTEM", 36 "GIT_CONFIG_GLOBAL", 37 "GIT_CONFIG_COUNT", 38 "GIT_CONFIG_PARAMETERS", 39 "GIT_SHALLOW_FILE", 40 "GIT_REPLACE_REF_BASE", 41 } 42 43 44 def remaining(deadline: float) -> float: 45 value = deadline - time.monotonic() 46 if not 0 < value <= MAX_SECONDS: 47 raise source.ProvenanceError("authored app inspection deadline expired") 48 return value 49 50 51 def read_git_output(child: subprocess.Popen, deadline: float) -> bytes: 52 with selectors.DefaultSelector() as selection: 53 output = bytearray() 54 sizes = {"stdout": 0, "stderr": 0} 55 for name in sizes: 56 stream = getattr(child, name) 57 os.set_blocking(stream.fileno(), False) 58 selection.register(stream, selectors.EVENT_READ, name) 59 while selection.get_map(): 60 for key, _ in selection.select(remaining(deadline)): 61 chunk = os.read(key.fd, min(65536, MAX_BYTES + 1 - sizes[key.data])) 62 if not chunk: 63 selection.unregister(key.fileobj) 64 continue 65 sizes[key.data] += len(chunk) 66 if sizes[key.data] > MAX_BYTES: 67 raise source.ProvenanceError( 68 "authored app Git output exceeds byte bound" 69 ) 70 if key.data == "stdout": 71 output.extend(chunk) 72 return bytes(output) 73 74 75 def git(root: Path, arguments: list[str], deadline: float) -> bytes: 76 remaining(deadline) 77 try: 78 with subprocess.Popen( 79 ["git", "--no-replace-objects", *arguments], 80 cwd=root, 81 stdout=subprocess.PIPE, 82 stderr=subprocess.PIPE, 83 ) as child: 84 try: 85 output = read_git_output(child, deadline) 86 if child.wait(timeout=remaining(deadline)): 87 raise source.ProvenanceError("authored app Git inspection failed") 88 return output 89 finally: 90 if child.poll() is None: 91 child.kill() 92 child.wait() 93 except (OSError, subprocess.TimeoutExpired) as error: 94 raise source.ProvenanceError( 95 "authored app Git inspection unavailable" 96 ) from error 97 98 99 def reject_git_overrides() -> None: 100 if any( 101 name in GIT_OVERRIDES 102 or name.startswith(("GIT_CONFIG_KEY_", "GIT_CONFIG_VALUE_")) 103 for name in os.environ 104 ): 105 raise source.ProvenanceError("authored app Git selection override is active") 106 107 108 def reject_intent_to_add(root: Path, paths: list[str], deadline: float) -> None: 109 visible, invisible = source.staged_diff_views(paths) 110 if git(root, visible, deadline) != git(root, invisible, deadline): 111 raise source.ProvenanceError( 112 "authored app input is intent-to-add or index changed" 113 ) 114 115 116 def scoped_inventory(root: Path, deadline: float) -> bytes: 117 paths = ["--", *INPUTS, *(f":(exclude){path}" for path in EXCLUDED)] 118 reject_intent_to_add(root, paths, deadline) 119 for selection in ( 120 ["--others", "--exclude-standard"], 121 ["--others", "--ignored", "--exclude-standard"], 122 ): 123 if git(root, ["ls-files", *selection, "-z", *paths], deadline): 124 raise source.ProvenanceError( 125 "authored app contains untracked or ignored input" 126 ) 127 raw = git(root, ["ls-files", "--stage", "-z", *paths], deadline) 128 if not raw or raw.count(b"\0") > MAX_INPUTS: 129 raise source.ProvenanceError("authored app inventory exceeds bounds") 130 return raw 131 132 133 def identity(info: os.stat_result) -> tuple[int, ...]: 134 return ( 135 info.st_dev, 136 info.st_ino, 137 info.st_mode, 138 info.st_size, 139 info.st_mtime_ns, 140 info.st_ctime_ns, 141 ) 142 143 144 def admit_directory(stack: ExitStack, name: str | Path, parent: int | None) -> int: 145 descriptor = os.open( 146 name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent 147 ) 148 stack.callback(os.close, descriptor) 149 info = os.fstat(descriptor) 150 if not info.st_mode & 0o444 or not info.st_mode & 0o111: 151 raise source.ProvenanceError("authored app directory is unreadable") 152 return descriptor 153 154 155 def check_directories( 156 ancestors: list[tuple[int | None, str | Path, int, tuple]], 157 ) -> None: 158 for parent, name, descriptor, before in ancestors: 159 if ( 160 identity(os.fstat(descriptor)) != before 161 or identity(os.stat(name, dir_fd=parent, follow_symlinks=False)) != before 162 ): 163 raise source.ProvenanceError( 164 "authored app directory changed during capture" 165 ) 166 167 168 def read_bounded(member: int, expected_size: int, deadline: float) -> bytes: 169 chunks, size = [], 0 170 while True: 171 remaining(deadline) 172 chunk = os.read(member, min(65536, MAX_BYTES + 1 - size)) 173 if not chunk: 174 break 175 size += len(chunk) 176 if size > MAX_BYTES: 177 raise source.ProvenanceError("authored app input exceeds byte bound") 178 chunks.append(chunk) 179 if size != expected_size: 180 raise source.ProvenanceError("authored app input changed during capture") 181 return b"".join(chunks) 182 183 184 def read_member( 185 stack: ExitStack, parent: int, name: str, deadline: float 186 ) -> tuple[bytes, str]: 187 member = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent) 188 stack.callback(os.close, member) 189 info = os.fstat(member) 190 if not stat.S_ISREG(info.st_mode) or not info.st_mode & 0o444: 191 raise source.ProvenanceError( 192 "authored app input is not readable regular source" 193 ) 194 if info.st_size > MAX_BYTES: 195 raise source.ProvenanceError("authored app input exceeds byte bound") 196 data = read_bounded(member, info.st_size, deadline) 197 if identity(os.fstat(member)) != identity(info) or identity( 198 os.stat(name, dir_fd=parent, follow_symlinks=False) 199 ) != identity(info): 200 raise source.ProvenanceError("authored app input changed during capture") 201 mode = "100755" if info.st_mode & stat.S_IXUSR else "100644" 202 return data, mode 203 204 205 def read_authored_source( 206 root: Path, relative: str, deadline: float 207 ) -> tuple[bytes, str]: 208 path = Path(relative) 209 if path.is_absolute() or str(path) != relative or ".." in path.parts: 210 raise source.ProvenanceError("authored app input path is invalid") 211 try: 212 with ExitStack() as stack: 213 descriptor = admit_directory(stack, root, None) 214 ancestors = [(None, root, descriptor, identity(os.fstat(descriptor)))] 215 for name in path.parts[:-1]: 216 parent = descriptor 217 descriptor = admit_directory(stack, name, parent) 218 ancestors.append( 219 (parent, name, descriptor, identity(os.fstat(descriptor))) 220 ) 221 check_directories(ancestors) 222 result = read_member(stack, descriptor, path.name, deadline) 223 check_directories(ancestors) 224 return result 225 except OSError as error: 226 raise source.ProvenanceError( 227 "authored app input cannot be safely read" 228 ) from error 229 230 231 def staged_member( 232 root: Path, row: bytes, deadline: float 233 ) -> tuple[str, dict[str, Any]]: 234 header, encoded_path = row.split(b"\t", 1) 235 mode, oid, stage = header.decode("ascii").split() 236 relative = encoded_path.decode("utf-8") 237 if not source.is_input(relative, INPUTS) or source.is_input(relative, EXCLUDED): 238 raise source.ProvenanceError( 239 "authored app Git inventory selects an unowned input" 240 ) 241 if stage != "0" or mode not in ("100644", "100755"): 242 raise source.ProvenanceError( 243 "authored app input has unresolved or unsupported mode" 244 ) 245 data, actual_mode = read_authored_source(root, relative, deadline) 246 blob = git(root, ["cat-file", "blob", oid], deadline) 247 if data != blob or mode != actual_mode: 248 raise source.ProvenanceError("authored app worktree differs from staged input") 249 return relative, { 250 "mode": mode, 251 "git_blob": oid, 252 "bytes": len(data), 253 "sha256": hashlib.sha256(data).hexdigest(), 254 } 255 256 257 def encoded(value: dict[str, Any]) -> bytes: 258 raw = (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode() 259 if len(raw) > MAX_BYTES: 260 raise source.ProvenanceError("authored app record exceeds byte bound") 261 return raw 262 263 264 def capture(root: Path) -> dict[str, Any]: 265 reject_git_overrides() 266 deadline = time.monotonic() + MAX_SECONDS 267 if root.is_symlink() or git( 268 root, ["rev-parse", "--show-toplevel"], deadline 269 ).decode().strip() != str(root): 270 raise source.ProvenanceError( 271 "authored app capture requires its own repository root" 272 ) 273 raw = scoped_inventory(root, deadline) 274 files, tree = {}, {} 275 for row in raw.rstrip(b"\0").split(b"\0"): 276 relative, entry = staged_member(root, row, deadline) 277 if relative in files: 278 raise source.ProvenanceError("authored app input inventory is duplicated") 279 files[relative] = entry 280 source.insert_tree(tree, relative, entry["mode"], entry["git_blob"]) 281 source.require_input_inventory(INPUTS, files) 282 if scoped_inventory(root, deadline) != raw: 283 raise source.ProvenanceError( 284 "authored app index or inventory changed during capture" 285 ) 286 for relative, entry in files.items(): 287 data, mode = read_authored_source(root, relative, deadline) 288 if mode != entry["mode"] or hashlib.sha256(data).hexdigest() != entry["sha256"]: 289 raise source.ProvenanceError("authored app worktree changed during capture") 290 if scoped_inventory(root, deadline) != raw: 291 raise source.ProvenanceError( 292 "authored app index or inventory changed during capture" 293 ) 294 remaining(deadline) 295 result = { 296 "policy": "staged_inputs", 297 "tree": source.tree_identity(tree), 298 "files": dict(sorted(files.items())), 299 } 300 encoded(result) 301 remaining(deadline) 302 return result 303 304 305 def main() -> int: 306 parser = argparse.ArgumentParser() 307 parser.add_argument("--repo-root", type=Path, required=True) 308 args = parser.parse_args() 309 try: 310 value = { 311 "source": capture(args.repo_root), 312 "source_date_epoch": contract.producer_source_epoch(args.repo_root), 313 } 314 sys.stdout.buffer.write(encoded(value)) 315 return 0 316 except ( 317 source.ProvenanceError, 318 contract.PackageContractError, 319 ValueError, 320 UnicodeError, 321 ): 322 print("error: authored app source capture rejected", file=sys.stderr) 323 return 1 324 325 326 if __name__ == "__main__": 327 raise SystemExit(main())