field_ios

In-the-field app for Radroots on iOS
git clone https://radroots.dev/git/field_ios.git
Log | Files | Refs | README | LICENSE

commit 2a354cefe78c73605443404814f83b4dd5c7b4e3
parent 72e742b2b7fdd6b36056e1545420dc080a055ce6
Author: triesap <tyson@radroots.org>
Date:   Mon, 31 Aug 2026 19:55:06 +0000

ios: bind deterministic persona attempt evidence

- freeze strict v1 attempt and v2 aggregate schemas
- bind attachments to exact source build run and test identity
- reject unmeasured network evidence from aggregate success
- cover xcresult mutation bounds and secret-free evidence

Diffstat:
MREADME.md | 20+++++++++++++++-----
MRadrootsUITests/Info.plist | 8++++++++
MRadrootsUITests/RadrootsRemoteQualificationUITests.swift | 275++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
Mscripts/local-social-fixture.py | 627++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
Mscripts/test_local_social_fixture.py | 300+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mscripts/verify-package-contract.sh | 20++++++++++++++++++--
Mscripts/xcode.sh | 16+++++++++++++++-
Atest-fixtures/local-social-persona-attempt-evidence.v1.schema.json | 145+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Atest-fixtures/local-social-persona-results.v2.schema.json | 137+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
9 files changed, 1536 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md @@ -120,12 +120,22 @@ cargo extbuild run -- scripts/xcode.sh local-social-ui-test \ persona ``` -The persona fixture and result contracts are strict and deny unknown input. +The persona fixture and historical v1 result remain strict, deny-unknown +contracts. Each completed attempt now also emits one bounded canonical +`persona-attempt-evidence.v1` xcresult attachment bound to the exact XCUITest +target and identifier, test action and configuration, source commit and tree, +app-build digest, simulator, run, persona, attempt, endpoint policy, and visible +UI outcome. The attachment never retains a raw secret, signed authorization +event, or raw event content. Its closed `pending_step_258` network state is not +a passing network claim: the measured v2 aggregate rejects that state until the +observable connection and fixture counters are supplied by the next governed +remediation step. + The test uses ordinary visible controls, native-generated signing identities, -real app stores, and generated Rust FFI; it retains no raw secret or raw event -content. This is deterministic non-human conformance evidence. It does not -claim human usability, demographic or population validity, observed -VoiceOver-user experience, release readiness, or production qualification. +real app stores, and generated Rust FFI. This is deterministic non-human +conformance evidence. It does not claim human usability, demographic or +population validity, observed VoiceOver-user experience, release readiness, +or production qualification. ## Package surface diff --git a/RadrootsUITests/Info.plist b/RadrootsUITests/Info.plist @@ -24,5 +24,13 @@ <string>$(RADROOTS_IOS_UI_TEST_FIXTURE_CONTROL)</string> <key>RADROOTS_IOS_UI_TEST_NETWORK_PROFILE</key> <string>$(RADROOTS_IOS_UI_TEST_NETWORK_PROFILE)</string> + <key>RADROOTS_IOS_UI_TEST_SOURCE_COMMIT</key> + <string>$(RADROOTS_IOS_UI_TEST_SOURCE_COMMIT)</string> + <key>RADROOTS_IOS_UI_TEST_SOURCE_TREE</key> + <string>$(RADROOTS_IOS_UI_TEST_SOURCE_TREE)</string> + <key>RADROOTS_IOS_UI_TEST_APP_BUILD_SHA256</key> + <string>$(RADROOTS_IOS_UI_TEST_APP_BUILD_SHA256)</string> + <key>RADROOTS_IOS_UI_TEST_SIMULATOR_ID</key> + <string>$(RADROOTS_IOS_UI_TEST_SIMULATOR_ID)</string> </dict> </plist> diff --git a/RadrootsUITests/RadrootsRemoteQualificationUITests.swift b/RadrootsUITests/RadrootsRemoteQualificationUITests.swift @@ -215,6 +215,14 @@ final class RadrootsRemoteQualificationUITests: XCTestCase { app = launchPersona(configuration) try assertTodayContains(app, markers: markers) XCTAssertEqual(try readPublicKey(app), publicKey) + for attempt in persona.attempts { + try writePersonaAttemptAttachment( + configuration: configuration, + persona: persona, + attempt: attempt, + identityDigest: identityDigest + ) + } app.terminate() } XCTAssertEqual(identityDigests.count, 5) @@ -1439,6 +1447,93 @@ final class RadrootsRemoteQualificationUITests: XCTestCase { attachment.lifetime = .keepAlways add(attachment) } + + private func writePersonaAttemptAttachment( + configuration: QualificationConfiguration, + persona: Persona, + attempt: PersonaAttempt, + identityDigest: String + ) throws { + let binding = try XCTUnwrap(configuration.evidenceBinding) + let validation = attempt.expectedFailure == "validation_recovery" + let retry = attempt.expectedFailure == "transport_retry_relaunch" + let progressiveDisclosure = + persona.interactionProfile == "novice_progressive_disclosure" + let accessibilityKeyboard = + persona.interactionProfile == "novice_accessibility_keyboard" + let evidence = PersonaAttemptEvidence( + schema: "radroots.ios.local-social.persona-attempt-evidence.v1", + schemaVersion: 1, + testInvocation: .init( + target: "RadrootsUITests", + identifier: "RadrootsUITests/testLocalSocialDeterministicPersonas", + action: "test", + configuration: "Debug" + ), + source: .init(commit: binding.sourceCommit, tree: binding.sourceTree), + appBuildSHA256: binding.appBuildSHA256, + simulator: .init( + udid: binding.simulatorID, + os: Self.currentIOSVersion, + architecture: "arm64" + ), + runID: configuration.qualificationRunID, + personaRunID: configuration.runID, + personaAlias: persona.alias, + attemptID: attempt.id, + attemptOrder: attempt.order, + flow: attempt.flow.rawValue, + expectedFailure: attempt.expectedFailure, + publicIdentitySHA256: identityDigest, + endpointPolicySHA256: Self.endpointPolicyDigest(configuration), + uiObservation: .init( + validationAttempted: validation, + validationRejected: validation, + retryAttempts: retry ? 1 : 0, + relaunches: retry ? 1 : 0, + retentionVerified: true, + todayProjectionVerified: true + ), + networkObservation: .init(state: "pending_step_258"), + accessibility: .init( + locale: "en_US", + contentSize: "accessibility-extra-extra-extra-large", + reduceMotion: true, + progressiveDisclosure: progressiveDisclosure, + labelsValuesTraits: accessibilityKeyboard, + keyboardFocus: accessibilityKeyboard, + visibleActions: true, + voiceoverUserObserved: false + ), + artifactDigests: [] + ) + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + let data = try encoder.encode(evidence) + XCTAssertLessThanOrEqual(data.count, 64 * 1024) + let attachment = XCTAttachment(data: data, uniformTypeIdentifier: "public.json") + attachment.name = "radroots-local-social-\(attempt.id).json" + attachment.lifetime = .keepAlways + add(attachment) + } + + private static var currentIOSVersion: String { + let version = ProcessInfo.processInfo.operatingSystemVersion + return "iOS \(version.majorVersion).\(version.minorVersion).\(version.patchVersion)" + } + + private static func endpointPolicyDigest( + _ configuration: QualificationConfiguration + ) -> String { + var payload = Data("radroots.ios.local-social.endpoint-policy.v1\0".utf8) + for value in (configuration.relayURLs + configuration.blossomOrigins).sorted() { + let bytes = Data(value.utf8) + var length = UInt64(bytes.count).bigEndian + withUnsafeBytes(of: &length) { payload.append(contentsOf: $0) } + payload.append(bytes) + } + return SHA256.hash(data: payload).map { String(format: "%02x", $0) }.joined() + } } private struct QualificationConfiguration { @@ -1449,25 +1544,35 @@ private struct QualificationConfiguration { static let fixtureControlKey = "RADROOTS_IOS_UI_TEST_FIXTURE_CONTROL" static let mediaRelativePathKey = "RADROOTS_IOS_UI_TEST_MEDIA_RELATIVE_PATH" static let networkProfileKey = "RADROOTS_IOS_UI_TEST_NETWORK_PROFILE" + static let sourceCommitKey = "RADROOTS_IOS_UI_TEST_SOURCE_COMMIT" + static let sourceTreeKey = "RADROOTS_IOS_UI_TEST_SOURCE_TREE" + static let appBuildSHA256Key = "RADROOTS_IOS_UI_TEST_APP_BUILD_SHA256" + static let simulatorIDKey = "RADROOTS_IOS_UI_TEST_SIMULATOR_ID" let runID: String + let qualificationRunID: String let relayURLs: [String] let blossomOrigins: [String] let fixtureControl: String? let networkProfile: String + let evidenceBinding: PersonaEvidenceBinding? init( runID: String, relayURLs: [String], blossomOrigins: [String], fixtureControl: String? = nil, - networkProfile: String = "public" + networkProfile: String = "public", + qualificationRunID: String? = nil, + evidenceBinding: PersonaEvidenceBinding? = nil ) { self.runID = runID + self.qualificationRunID = qualificationRunID ?? runID self.relayURLs = relayURLs self.blossomOrigins = blossomOrigins self.fixtureControl = fixtureControl self.networkProfile = networkProfile + self.evidenceBinding = evidenceBinding } var launchEnvironment: [String: String] { @@ -1492,7 +1597,9 @@ private struct QualificationConfiguration { relayURLs: relayURLs, blossomOrigins: blossomOrigins, fixtureControl: fixtureControl, - networkProfile: networkProfile + networkProfile: networkProfile, + qualificationRunID: qualificationRunID, + evidenceBinding: evidenceBinding ) } @@ -1515,6 +1622,10 @@ private struct QualificationConfiguration { let networkProfile = values[networkProfileKey] ?? bundle.object(forInfoDictionaryKey: networkProfileKey) as? String + let sourceCommit = value(sourceCommitKey, values: values, bundle: bundle) + let sourceTree = value(sourceTreeKey, values: values, bundle: bundle) + let appBuildSHA256 = value(appBuildSHA256Key, values: values, bundle: bundle) + let simulatorID = value(simulatorIDKey, values: values, bundle: bundle) if runIDValue?.isEmpty != false, blossomValue?.isEmpty != false { throw XCTSkip("remote qualification inputs were not selected") } @@ -1527,20 +1638,178 @@ private struct QualificationConfiguration { else { throw QualificationError.missingEnvironment } + let evidenceBinding = PersonaEvidenceBinding( + sourceCommit: sourceCommit, + sourceTree: sourceTree, + appBuildSHA256: appBuildSHA256, + simulatorID: simulatorID + ) return Self( runID: runID, relayURLs: separated(relayValue), blossomOrigins: separated(blossom), fixtureControl: fixtureControl.flatMap { $0.isEmpty ? nil : $0 }, - networkProfile: networkProfile + networkProfile: networkProfile, + evidenceBinding: evidenceBinding ) } + private static func value( + _ key: String, + values: [String: String], + bundle: Bundle + ) -> String? { + (values[key] ?? bundle.object(forInfoDictionaryKey: key) as? String) + .flatMap { $0.isEmpty ? nil : $0 } + } + private static func separated(_ value: String?) -> [String] { (value ?? "").split(separator: ",").map(String.init).filter { !$0.isEmpty } } } +private struct PersonaEvidenceBinding { + let sourceCommit: String + let sourceTree: String + let appBuildSHA256: String + let simulatorID: String + + init?( + sourceCommit: String?, + sourceTree: String?, + appBuildSHA256: String?, + simulatorID: String? + ) { + guard let sourceCommit, + sourceCommit.range(of: "^[0-9a-f]{40}$", options: .regularExpression) != nil, + let sourceTree, + sourceTree.range(of: "^[0-9a-f]{40}$", options: .regularExpression) != nil, + let appBuildSHA256, + appBuildSHA256.range(of: "^[0-9a-f]{64}$", options: .regularExpression) != nil, + let simulatorID, + simulatorID.uppercased().range( + of: "^[A-F0-9-]{36}$", + options: .regularExpression + ) != nil + else { return nil } + self.sourceCommit = sourceCommit + self.sourceTree = sourceTree + self.appBuildSHA256 = appBuildSHA256 + self.simulatorID = simulatorID.uppercased() + } +} + +private struct PersonaAttemptTestInvocation: Encodable { + let target: String + let identifier: String + let action: String + let configuration: String +} + +private struct PersonaAttemptSource: Encodable { + let commit: String + let tree: String +} + +private struct PersonaAttemptSimulator: Encodable { + let udid: String + let os: String + let architecture: String +} + +private struct PersonaAttemptUIObservation: Encodable { + let validationAttempted: Bool + let validationRejected: Bool + let retryAttempts: Int + let relaunches: Int + let retentionVerified: Bool + let todayProjectionVerified: Bool + + enum CodingKeys: String, CodingKey { + case validationAttempted = "validation_attempted" + case validationRejected = "validation_rejected" + case retryAttempts = "retry_attempts" + case relaunches + case retentionVerified = "retention_verified" + case todayProjectionVerified = "today_projection_verified" + } +} + +private struct PersonaAttemptNetworkObservation: Encodable { + let state: String +} + +private struct PersonaAttemptAccessibility: Encodable { + let locale: String + let contentSize: String + let reduceMotion: Bool + let progressiveDisclosure: Bool + let labelsValuesTraits: Bool + let keyboardFocus: Bool + let visibleActions: Bool + let voiceoverUserObserved: Bool + + enum CodingKeys: String, CodingKey { + case locale + case contentSize = "content_size" + case reduceMotion = "reduce_motion" + case progressiveDisclosure = "progressive_disclosure" + case labelsValuesTraits = "labels_values_traits" + case keyboardFocus = "keyboard_focus" + case visibleActions = "visible_actions" + case voiceoverUserObserved = "voiceover_user_observed" + } +} + +private struct PersonaAttemptArtifactDigest: Encodable { + let role: String + let sha256: String +} + +private struct PersonaAttemptEvidence: Encodable { + let schema: String + let schemaVersion: UInt16 + let testInvocation: PersonaAttemptTestInvocation + let source: PersonaAttemptSource + let appBuildSHA256: String + let simulator: PersonaAttemptSimulator + let runID: String + let personaRunID: String + let personaAlias: String + let attemptID: String + let attemptOrder: Int + let flow: String + let expectedFailure: String + let publicIdentitySHA256: String + let endpointPolicySHA256: String + let uiObservation: PersonaAttemptUIObservation + let networkObservation: PersonaAttemptNetworkObservation + let accessibility: PersonaAttemptAccessibility + let artifactDigests: [PersonaAttemptArtifactDigest] + + enum CodingKeys: String, CodingKey { + case schema + case schemaVersion = "schema_version" + case testInvocation = "test_invocation" + case source + case appBuildSHA256 = "app_build_sha256" + case simulator + case runID = "run_id" + case personaRunID = "persona_run_id" + case personaAlias = "persona_alias" + case attemptID = "attempt_id" + case attemptOrder = "attempt_order" + case flow + case expectedFailure = "expected_failure" + case publicIdentitySHA256 = "public_identity_sha256" + case endpointPolicySHA256 = "endpoint_policy_sha256" + case uiObservation = "ui_observation" + case networkObservation = "network_observation" + case accessibility + case artifactDigests = "artifact_digests" + } +} + private struct BootstrapReceipt: Codable { let schema: String let schemaVersion: UInt16 diff --git a/scripts/local-social-fixture.py b/scripts/local-social-fixture.py @@ -16,6 +16,7 @@ import socket import socketserver import subprocess import struct +import tempfile import threading import time import unicodedata @@ -73,6 +74,30 @@ FLOW_KINDS = { } PHOTO_PERSONAS = frozenset(("P01", "P03", "P04")) PERSONA_CONTROL_SCHEMA = "radroots.ios.local-social.persona-control.v1" +PERSONA_ATTEMPT_SCHEMA = "radroots.ios.local-social.persona-attempt-evidence.v1" +PERSONA_RESULT_V2_SCHEMA = "radroots.ios.local-social.persona-results.v2" +PERSONA_TEST_TARGET = "RadrootsUITests" +PERSONA_TEST_IDENTIFIER = "RadrootsUITests/testLocalSocialDeterministicPersonas" +PERSONA_TEST_ACTION = "test" +PERSONA_TEST_CONFIGURATION = "Debug" +PERSONA_XCRESULT_NODE_IDENTIFIER = ( + "RadrootsRemoteQualificationUITests/testLocalSocialDeterministicPersonas()" +) +PERSONA_XCRESULT_NODE_URL = ( + "test://com.apple.xcode/Radroots/RadrootsUITests/" + "RadrootsRemoteQualificationUITests/testLocalSocialDeterministicPersonas" +) +MAX_XCRESULT_JSON_BYTES = 1024 * 1024 +MAX_PERSONA_ATTACHMENT_BYTES = 64 * 1024 +MAX_PERSONA_ATTACHMENTS_BYTES = 15 * MAX_PERSONA_ATTACHMENT_BYTES +PERSONA_ATTACHMENT_NAMES = tuple( + f"radroots-local-social-P{persona:02d}-A{attempt:02d}.json" + for persona in range(1, 6) + for attempt in range(1, 4) +) +FORBIDDEN_EVIDENCE_KEYS = frozenset( + ("private_key", "secret", "seed", "signed_event", "event_content", "raw_event") +) def strict_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: @@ -92,6 +117,15 @@ def read_json(path: Path, maximum: int = MAX_JSON_BYTES) -> tuple[bytes, Any]: return raw, value +def read_json_bounded(path: Path, maximum: int) -> tuple[bytes, Any]: + with path.open("rb") as stream: + raw = stream.read(maximum + 1) + if not raw or len(raw) > maximum: + raise ValueError("JSON input is empty or exceeds its byte bound") + value = json.loads(raw, object_pairs_hook=strict_object) + return raw, value + + def exact_keys(value: Any, keys: set[str], name: str) -> dict[str, Any]: if not isinstance(value, dict) or set(value) != keys: raise ValueError(f"{name} has an invalid field inventory") @@ -1208,11 +1242,21 @@ def verify_persona_fixture(arguments: argparse.Namespace) -> int: Path(arguments.result_schema).resolve(), "https://radroots.org/schemas/ios/local-social-persona-results.v1.schema.json", ) + attempt_schema = validate_schema_file( + Path(arguments.attempt_schema).resolve(), + "https://radroots.org/schemas/ios/local-social-persona-attempt-evidence.v1.schema.json", + ) + result_v2_schema = validate_schema_file( + Path(arguments.result_v2_schema).resolve(), + "https://radroots.org/schemas/ios/local-social-persona-results.v2.schema.json", + ) print( "local-social persona fixtures verified: " f"fixture={hashlib.sha256(raw).hexdigest()} " f"fixture_schema={hashlib.sha256(fixture_schema).hexdigest()} " - f"result_schema={hashlib.sha256(result_schema).hexdigest()}" + f"result_schema={hashlib.sha256(result_schema).hexdigest()} " + f"attempt_schema={hashlib.sha256(attempt_schema).hexdigest()} " + f"result_v2_schema={hashlib.sha256(result_v2_schema).hexdigest()}" ) return 0 @@ -1405,6 +1449,585 @@ def valid_ios_runtime(value: Any) -> bool: return int(value.split()[1].split(".", 1)[0]) >= 18 +def canonical_evidence_bytes(value: Any) -> bytes: + return json.dumps( + value, ensure_ascii=True, separators=(",", ":"), sort_keys=True + ).encode("utf-8") + + +def evidence_contains_forbidden_key(value: Any) -> bool: + if isinstance(value, dict): + return any( + key.lower() in FORBIDDEN_EVIDENCE_KEYS + or evidence_contains_forbidden_key(item) + for key, item in value.items() + ) + if isinstance(value, list): + return any(evidence_contains_forbidden_key(item) for item in value) + return False + + +def persona_invocation() -> dict[str, str]: + return { + "target": PERSONA_TEST_TARGET, + "identifier": PERSONA_TEST_IDENTIFIER, + "action": PERSONA_TEST_ACTION, + "configuration": PERSONA_TEST_CONFIGURATION, + } + + +def validate_persona_attempt_evidence( + value: Any, + suite: dict[str, Any], + *, + require_measured_network: bool, +) -> dict[str, Any]: + keys = { + "schema", + "schema_version", + "test_invocation", + "source", + "app_build_sha256", + "simulator", + "run_id", + "persona_run_id", + "persona_alias", + "attempt_id", + "attempt_order", + "flow", + "expected_failure", + "public_identity_sha256", + "endpoint_policy_sha256", + "ui_observation", + "network_observation", + "accessibility", + "artifact_digests", + } + attempt = exact_keys(value, keys, "persona attempt evidence") + if evidence_contains_forbidden_key(attempt): + raise ValueError("persona attempt evidence contains a forbidden field") + if ( + attempt["schema"] != PERSONA_ATTEMPT_SCHEMA + or attempt["schema_version"] != 1 + or attempt["test_invocation"] != persona_invocation() + or not lowercase_hex(attempt["app_build_sha256"], 64) + or not lowercase_hex(attempt["public_identity_sha256"], 64) + or attempt["public_identity_sha256"] == "0" * 64 + or not lowercase_hex(attempt["endpoint_policy_sha256"], 64) + or not re.fullmatch( + r"[a-z0-9][a-z0-9-]{6,62}[a-z0-9]", attempt["run_id"] + ) + ): + raise ValueError("persona attempt evidence header is invalid") + source = exact_keys(attempt["source"], {"commit", "tree"}, "attempt source") + if not lowercase_hex(source["commit"], 40) or not lowercase_hex( + source["tree"], 40 + ): + raise ValueError("persona attempt source identity is invalid") + simulator = exact_keys( + attempt["simulator"], {"udid", "os", "architecture"}, "attempt simulator" + ) + if ( + not isinstance(simulator["udid"], str) + or re.fullmatch(r"[A-F0-9-]{36}", simulator["udid"]) is None + or not valid_ios_runtime(simulator["os"]) + or simulator["architecture"] != "arm64" + ): + raise ValueError("persona attempt simulator is invalid") + expected_attempts = { + candidate["id"]: (persona, candidate) + for persona in suite["personas"] + for candidate in persona["attempts"] + } + expected = expected_attempts.get(attempt["attempt_id"]) + if expected is None: + raise ValueError("persona attempt identity is unknown") + expected_persona, expected_attempt = expected + alias = expected_persona["alias"] + if ( + attempt["persona_alias"] != alias + or attempt["attempt_order"] != expected_attempt["order"] + or attempt["flow"] != expected_attempt["flow"] + or attempt["expected_failure"] != expected_attempt["expected_failure"] + or re.fullmatch( + rf"persona-{alias.lower()}-[0-9a-f]{{24}}", attempt["persona_run_id"] + ) + is None + ): + raise ValueError("persona attempt binding is invalid") + ui = exact_keys( + attempt["ui_observation"], + { + "validation_attempted", + "validation_rejected", + "retry_attempts", + "relaunches", + "retention_verified", + "today_projection_verified", + }, + "attempt UI observation", + ) + validation = expected_attempt["expected_failure"] == "validation_recovery" + retry = expected_attempt["expected_failure"] == "transport_retry_relaunch" + if ( + ui["validation_attempted"] is not validation + or ui["validation_rejected"] is not validation + or ui["retry_attempts"] != int(retry) + or ui["relaunches"] != int(retry) + or ui["retention_verified"] is not True + or ui["today_projection_verified"] is not True + ): + raise ValueError("persona attempt UI observation is invalid") + network = attempt["network_observation"] + if network == {"state": "pending_step_258"}: + if require_measured_network: + raise ValueError("persona attempt network evidence is not measured") + else: + network = exact_keys( + network, + { + "state", + "accepted_connections", + "rejected_connections", + "non_loopback_attempts", + "subscriptions", + "accepted_events", + "accepted_uploads", + "retrievals", + "unintended_publications", + "events_accepted_during_expected_failure", + "final_candidate_data_loss", + }, + "attempt network observation", + ) + if network["state"] != "measured" or any( + type(network[key]) is not int or not 0 <= network[key] <= 4096 + for key in network + if key != "state" + ): + raise ValueError("persona attempt network observation is invalid") + expected_media = int(attempt["flow"] == "PhotoUpdate") + if ( + network["accepted_events"] != 1 + or network["accepted_uploads"] != expected_media + or network["retrievals"] != expected_media + or network["non_loopback_attempts"] != 0 + or network["unintended_publications"] != 0 + or network["events_accepted_during_expected_failure"] != 0 + or network["final_candidate_data_loss"] != 0 + ): + raise ValueError("persona attempt measured result is invalid") + accessibility = exact_keys( + attempt["accessibility"], + { + "locale", + "content_size", + "reduce_motion", + "progressive_disclosure", + "labels_values_traits", + "keyboard_focus", + "visible_actions", + "voiceover_user_observed", + }, + "attempt accessibility observation", + ) + if ( + accessibility["locale"] != "en_US" + or accessibility["content_size"] + != "accessibility-extra-extra-extra-large" + or accessibility["reduce_motion"] is not True + or any( + type(accessibility[key]) is not bool + for key in ( + "progressive_disclosure", + "labels_values_traits", + "keyboard_focus", + "visible_actions", + "voiceover_user_observed", + ) + ) + or accessibility["visible_actions"] is not True + or accessibility["voiceover_user_observed"] is not False + or accessibility["progressive_disclosure"] + is not (expected_persona["interaction_profile"] == "novice_progressive_disclosure") + or accessibility["keyboard_focus"] + is not (expected_persona["interaction_profile"] == "novice_accessibility_keyboard") + ): + raise ValueError("persona attempt accessibility evidence is invalid") + artifacts = attempt["artifact_digests"] + if not isinstance(artifacts, list) or len(artifacts) > 4: + raise ValueError("persona attempt artifact inventory is invalid") + roles: set[str] = set() + for artifact in artifacts: + artifact = exact_keys(artifact, {"role", "sha256"}, "attempt artifact") + if ( + artifact["role"] + not in {"media_input", "uploaded_blob", "retrieved_blob", "ui_snapshot"} + or artifact["role"] in roles + or not lowercase_hex(artifact["sha256"], 64) + ): + raise ValueError("persona attempt artifact is invalid") + roles.add(artifact["role"]) + return attempt + + +def exact_persona_test_node(value: Any) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != { + "devices", + "testNodes", + "testPlanConfigurations", + }: + raise ValueError("xcresult test inventory is invalid") + matches: list[dict[str, Any]] = [] + + def visit(node: Any) -> None: + if not isinstance(node, dict): + raise ValueError("xcresult test node is invalid") + if node.get("nodeType") == "Test Case" and ( + node.get("nodeIdentifier") == PERSONA_XCRESULT_NODE_IDENTIFIER + or node.get("nodeIdentifierURL") == PERSONA_XCRESULT_NODE_URL + ): + matches.append(node) + children = node.get("children", []) + if not isinstance(children, list): + raise ValueError("xcresult child inventory is invalid") + for child in children: + visit(child) + + nodes = value["testNodes"] + if not isinstance(nodes, list): + raise ValueError("xcresult test inventory is invalid") + for node in nodes: + visit(node) + if len(matches) != 1: + raise ValueError("xcresult exact persona test is unavailable") + match = matches[0] + if ( + match.get("nodeIdentifier") != PERSONA_XCRESULT_NODE_IDENTIFIER + or match.get("nodeIdentifierURL") != PERSONA_XCRESULT_NODE_URL + or match.get("name") != "testLocalSocialDeterministicPersonas()" + or match.get("nodeType") != "Test Case" + or match.get("result") != "Passed" + ): + raise ValueError("xcresult exact persona test did not pass") + return match + + +def run_json_command_bounded(command: list[str], maximum: int) -> Any: + with tempfile.TemporaryFile() as output: + subprocess.run(command, stdout=output, check=True) + size = output.tell() + if size <= 0 or size > maximum: + raise ValueError("command JSON output exceeds its byte bound") + output.seek(0) + return json.loads(output.read(), object_pairs_hook=strict_object) + + +def load_exported_persona_attachments( + export_directory: Path, + suite: dict[str, Any], + *, + require_measured_network: bool, +) -> list[tuple[bytes, dict[str, Any]]]: + manifest_raw, manifest_value = read_json_bounded( + export_directory / "manifest.json", MAX_XCRESULT_JSON_BYTES + ) + del manifest_raw + if not isinstance(manifest_value, list) or len(manifest_value) != 1: + raise ValueError("xcresult attachment manifest is invalid") + group = exact_keys( + manifest_value[0], + {"testIdentifier", "testIdentifierURL", "attachments"}, + "xcresult attachment group", + ) + if ( + group["testIdentifier"] != PERSONA_XCRESULT_NODE_IDENTIFIER + or group["testIdentifierURL"] != PERSONA_XCRESULT_NODE_URL + or not isinstance(group["attachments"], list) + or len(group["attachments"]) != 15 + ): + raise ValueError("xcresult attachment test binding is invalid") + attachments_by_name: dict[str, tuple[bytes, dict[str, Any]]] = {} + total_bytes = 0 + allowed_manifest_keys = { + "exportedFileName", + "suggestedHumanReadableName", + "isAssociatedWithFailure", + "configurationName", + "deviceName", + "deviceId", + "timestamp", + "repetitionNumber", + "arguments", + } + for row_value in group["attachments"]: + if not isinstance(row_value, dict) or not { + "exportedFileName", + "suggestedHumanReadableName", + "isAssociatedWithFailure", + "configurationName", + "deviceName", + "deviceId", + }.issubset(row_value) or not set(row_value).issubset(allowed_manifest_keys): + raise ValueError("xcresult attachment row is invalid") + exported = row_value["exportedFileName"] + name = row_value["suggestedHumanReadableName"] + if ( + name not in PERSONA_ATTACHMENT_NAMES + or name in attachments_by_name + or not isinstance(exported, str) + or not 1 <= len(exported.encode("utf-8")) <= 255 + or Path(exported).name != exported + or row_value["isAssociatedWithFailure"] is not False + or row_value["configurationName"] != "Test Scheme Action" + or not isinstance(row_value["deviceName"], str) + or not row_value["deviceName"] + or not isinstance(row_value["deviceId"], str) + ): + raise ValueError("xcresult attempt attachment identity is invalid") + path = export_directory / exported + if path.is_symlink() or not path.is_file(): + raise ValueError("xcresult attempt attachment is not a regular file") + raw, value = read_json_bounded(path, MAX_PERSONA_ATTACHMENT_BYTES) + if raw != canonical_evidence_bytes(value): + raise ValueError("persona attempt attachment is noncanonical") + attempt = validate_persona_attempt_evidence( + value, suite, require_measured_network=require_measured_network + ) + if name != f"radroots-local-social-{attempt['attempt_id']}.json": + raise ValueError("xcresult attachment name does not bind its attempt") + total_bytes += len(raw) + if total_bytes > MAX_PERSONA_ATTACHMENTS_BYTES: + raise ValueError("xcresult attempt attachments exceed their aggregate bound") + attachments_by_name[name] = (raw, attempt) + if tuple(sorted(attachments_by_name)) != tuple(sorted(PERSONA_ATTACHMENT_NAMES)): + raise ValueError("xcresult persona attachment inventory is incomplete") + return [attachments_by_name[name] for name in PERSONA_ATTACHMENT_NAMES] + + +def extract_persona_attempt_attachments( + result_bundle: Path, + suite: dict[str, Any], + *, + require_measured_network: bool, +) -> list[tuple[bytes, dict[str, Any]]]: + tests = run_json_command_bounded( + [ + "xcrun", + "xcresulttool", + "get", + "test-results", + "tests", + "--path", + str(result_bundle), + ], + MAX_XCRESULT_JSON_BYTES, + ) + exact_persona_test_node(tests) + with tempfile.TemporaryDirectory() as directory: + export_directory = Path(directory) + subprocess.run( + [ + "xcrun", + "xcresulttool", + "export", + "attachments", + "--test-id", + PERSONA_XCRESULT_NODE_URL, + "--path", + str(result_bundle), + "--output-path", + str(export_directory), + ], + check=True, + ) + return load_exported_persona_attachments( + export_directory, + suite, + require_measured_network=require_measured_network, + ) + + +def reconstruct_persona_result_v2( + suite: dict[str, Any], + attachments: list[tuple[bytes, dict[str, Any]]], + *, + fixture_sha256: str, + fixture_schema_sha256: str, + attempt_schema_sha256: str, + result_schema_sha256: str, + result_bundle_sha256: str, + forward_repairs: list[str], +) -> dict[str, Any]: + if len(attachments) != 15: + raise ValueError("persona result requires exactly 15 attempt attachments") + attempts = [ + validate_persona_attempt_evidence(value, suite, require_measured_network=True) + for _, value in attachments + ] + expected_ids = [ + candidate["id"] + for persona in suite["personas"] + for candidate in persona["attempts"] + ] + if [attempt["attempt_id"] for attempt in attempts] != expected_ids: + raise ValueError("persona result attempt inventory is not exact") + first = attempts[0] + shared_fields = ( + "test_invocation", + "source", + "app_build_sha256", + "simulator", + "run_id", + "endpoint_policy_sha256", + ) + if any( + attempt[field] != first[field] + for attempt in attempts[1:] + for field in shared_fields + ): + raise ValueError("persona attempt evidence spans multiple run identities") + for digest in ( + fixture_sha256, + fixture_schema_sha256, + attempt_schema_sha256, + result_schema_sha256, + result_bundle_sha256, + ): + if not lowercase_hex(digest, 64): + raise ValueError("persona result digest identity is invalid") + if ( + not isinstance(forward_repairs, list) + or len(forward_repairs) > 16 + or len(set(forward_repairs)) != len(forward_repairs) + or any(not lowercase_hex(revision, 40) for revision in forward_repairs) + ): + raise ValueError("persona result forward-repair inventory is invalid") + identity_by_persona: dict[str, str] = {} + persona_rows = [] + for persona in suite["personas"]: + alias = persona["alias"] + rows = [attempt for attempt in attempts if attempt["persona_alias"] == alias] + identities = {row["public_identity_sha256"] for row in rows} + if len(rows) != 3 or len(identities) != 1: + raise ValueError("persona result identity reuse is invalid") + identity = identities.pop() + identity_by_persona[alias] = identity + subscriptions = sum( + row["network_observation"]["subscriptions"] for row in rows + ) + if subscriptions < 1: + raise ValueError("persona result is missing a subscription") + persona_rows.append( + { + "alias": alias, + "public_identity_sha256": identity, + "subscriptions": subscriptions, + "attempt_ids": [row["attempt_id"] for row in rows], + } + ) + if len(set(identity_by_persona.values())) != 5: + raise ValueError("persona result identities are not distinct") + network_rows = [attempt["network_observation"] for attempt in attempts] + accepted_events = sum(row["accepted_events"] for row in network_rows) + flow_counts = { + flow: sum(attempt["flow"] == flow for attempt in attempts) + for flow in FLOW_KINDS + } + event_kind_counts = { + str(kind): sum( + FLOW_KINDS[attempt["flow"]] == kind + and attempt["network_observation"]["accepted_events"] == 1 + for attempt in attempts + ) + for kind in (1, 31923, 30402) + } + result = { + "schema": PERSONA_RESULT_V2_SCHEMA, + "schema_version": 2, + "run_id": first["run_id"], + "test_invocation": first["test_invocation"], + "source": first["source"], + "app_build_sha256": first["app_build_sha256"], + "endpoint_policy_sha256": first["endpoint_policy_sha256"], + "fixture_sha256": fixture_sha256, + "fixture_schema_sha256": fixture_schema_sha256, + "attempt_schema_sha256": attempt_schema_sha256, + "result_schema_sha256": result_schema_sha256, + "simulator": first["simulator"], + "result_bundle_sha256": result_bundle_sha256, + "attachments": [ + { + "attempt_id": attempt["attempt_id"], + "sha256": hashlib.sha256(raw).hexdigest(), + } + for raw, attempt in attachments + ], + "personas": persona_rows, + "flow_counts": flow_counts, + "accepted_events": accepted_events, + "event_kind_counts": event_kind_counts, + "accepted_uploads": sum(row["accepted_uploads"] for row in network_rows), + "retrievals": sum(row["retrievals"] for row in network_rows), + "distinct_identities": len(set(identity_by_persona.values())), + "unknown_attempts": len( + set(expected_ids) - {row["attempt_id"] for row in attempts} + ), + "duplicate_attempts": len(attempts) - len({row["attempt_id"] for row in attempts}), + "expected_failure_rejections": sum( + attempt["ui_observation"]["validation_rejected"] + or attempt["ui_observation"]["retry_attempts"] > 0 + for attempt in attempts + ), + "events_accepted_during_expected_failures": sum( + row["events_accepted_during_expected_failure"] for row in network_rows + ), + "accepted_connections": sum(row["accepted_connections"] for row in network_rows), + "rejected_connections": sum(row["rejected_connections"] for row in network_rows), + "non_loopback_attempts": sum(row["non_loopback_attempts"] for row in network_rows), + "unintended_publications": sum( + row["unintended_publications"] for row in network_rows + ), + "final_candidate_data_loss": sum( + row["final_candidate_data_loss"] for row in network_rows + ), + "accessibility": { + "locale": "en_US", + "content_size": "accessibility-extra-extra-extra-large", + "reduce_motion": True, + "semantic_attempts": sum( + attempt["accessibility"]["labels_values_traits"] for attempt in attempts + ), + "keyboard_focus_attempts": sum( + attempt["accessibility"]["keyboard_focus"] for attempt in attempts + ), + "voiceover_user_observed": False, + }, + "forward_repairs": forward_repairs, + "complete_matrix_rerun": True, + } + if ( + result["flow_counts"] != {flow: 3 for flow in FLOW_KINDS} + or result["accepted_events"] != 15 + or result["event_kind_counts"] != {"1": 9, "31923": 3, "30402": 3} + or result["accepted_uploads"] != 3 + or result["retrievals"] != 3 + or result["distinct_identities"] != 5 + or result["unknown_attempts"] != 0 + or result["duplicate_attempts"] != 0 + or result["expected_failure_rejections"] != 2 + or result["events_accepted_during_expected_failures"] != 0 + or result["accepted_connections"] < 1 + or result["non_loopback_attempts"] != 0 + or result["unintended_publications"] != 0 + or result["final_candidate_data_loss"] != 0 + or result["accessibility"]["semantic_attempts"] < 3 + or result["accessibility"]["keyboard_focus_attempts"] != 3 + ): + raise ValueError("persona result reconstructed outcome is invalid") + return result + + def validate_persona_result( value: Any, suite: dict[str, Any], @@ -1721,6 +2344,8 @@ def parser() -> argparse.ArgumentParser: fixture_command.add_argument("--fixture", required=True) fixture_command.add_argument("--fixture-schema", required=True) fixture_command.add_argument("--result-schema", required=True) + fixture_command.add_argument("--attempt-schema", required=True) + fixture_command.add_argument("--result-v2-schema", required=True) bud11_command = commands.add_parser("verify-bud11-corpus") bud11_command.add_argument("--corpus", required=True) bud11_command.add_argument("--schema", required=True) diff --git a/scripts/test_local_social_fixture.py b/scripts/test_local_social_fixture.py @@ -225,6 +225,85 @@ class LocalSocialFixtureTests(unittest.TestCase): } return suite, result + def persona_attempt_attachments( + self, + *, + measured: bool = True, + ) -> tuple[dict, list[tuple[bytes, dict]]]: + _, suite = fixture.load_persona_suite( + Path("test-fixtures/local-social-personas.v1.json") + ) + attachments = [] + for persona in suite["personas"]: + identity = hashlib.sha256(persona["alias"].encode("ascii")).hexdigest() + for index, attempt in enumerate(persona["attempts"]): + validation = attempt["expected_failure"] == "validation_recovery" + retry = attempt["expected_failure"] == "transport_retry_relaunch" + network = {"state": "pending_step_258"} + if measured: + network = { + "state": "measured", + "accepted_connections": 1, + "rejected_connections": int(retry), + "non_loopback_attempts": 0, + "subscriptions": int(index == 0), + "accepted_events": 1, + "accepted_uploads": int(attempt["flow"] == "PhotoUpdate"), + "retrievals": int(attempt["flow"] == "PhotoUpdate"), + "unintended_publications": 0, + "events_accepted_during_expected_failure": 0, + "final_candidate_data_loss": 0, + } + value = { + "schema": fixture.PERSONA_ATTEMPT_SCHEMA, + "schema_version": 1, + "test_invocation": fixture.persona_invocation(), + "source": {"commit": "1" * 40, "tree": "2" * 40}, + "app_build_sha256": "3" * 64, + "simulator": { + "udid": "11111111-2222-3333-4444-555555555555", + "os": "iOS 26.5", + "architecture": "arm64", + }, + "run_id": "persona-result-test-001", + "persona_run_id": ( + f"persona-{persona['alias'].lower()}-" + "a" * 24 + ), + "persona_alias": persona["alias"], + "attempt_id": attempt["id"], + "attempt_order": attempt["order"], + "flow": attempt["flow"], + "expected_failure": attempt["expected_failure"], + "public_identity_sha256": identity, + "endpoint_policy_sha256": "4" * 64, + "ui_observation": { + "validation_attempted": validation, + "validation_rejected": validation, + "retry_attempts": int(retry), + "relaunches": int(retry), + "retention_verified": True, + "today_projection_verified": True, + }, + "network_observation": network, + "accessibility": { + "locale": "en_US", + "content_size": "accessibility-extra-extra-extra-large", + "reduce_motion": True, + "progressive_disclosure": persona["interaction_profile"] + == "novice_progressive_disclosure", + "labels_values_traits": persona["interaction_profile"] + == "novice_accessibility_keyboard", + "keyboard_focus": persona["interaction_profile"] + == "novice_accessibility_keyboard", + "visible_actions": True, + "voiceover_user_observed": False, + }, + "artifact_digests": [], + } + raw = fixture.canonical_evidence_bytes(value) + attachments.append((raw, value)) + return suite, attachments + def test_bip340_reference_signature_and_mutation(self) -> None: public_key = bytes.fromhex( "f9308a019258c31049344f85f89d5229b531c845836f99b08601f113bce036f9" @@ -532,6 +611,227 @@ class LocalSocialFixtureTests(unittest.TestCase): changed, suite, "3" * 64, "4" * 64, "5" * 64 ) + def test_attempt_evidence_is_strict_bound_and_secret_free(self) -> None: + suite, attachments = self.persona_attempt_attachments(measured=False) + for raw, value in attachments: + self.assertLessEqual(len(raw), fixture.MAX_PERSONA_ATTACHMENT_BYTES) + self.assertIs( + fixture.validate_persona_attempt_evidence( + value, suite, require_measured_network=False + ), + value, + ) + self.assertNotRegex( + raw.decode("utf-8"), + r'"(?:private_key|secret|seed|signed_event|event_content|raw_event)"', + ) + with self.assertRaisesRegex(ValueError, "not measured"): + fixture.validate_persona_attempt_evidence( + value, suite, require_measured_network=True + ) + + def test_attempt_evidence_rejects_one_field_identity_and_outcome_drift(self) -> None: + suite, attachments = self.persona_attempt_attachments() + _, canonical = attachments[0] + mutations = ( + lambda value: value.update({"unknown": True}), + lambda value: value["test_invocation"].update( + {"identifier": "RadrootsUITests/testOther"} + ), + lambda value: value["source"].update({"tree": "A" * 40}), + lambda value: value.update({"app_build_sha256": "0" * 63}), + lambda value: value.update({"run_id": "x"}), + lambda value: value.update({"persona_alias": "P02"}), + lambda value: value.update({"attempt_order": 2}), + lambda value: value["ui_observation"].update( + {"today_projection_verified": False} + ), + lambda value: value["network_observation"].update( + {"non_loopback_attempts": 1} + ), + lambda value: value["accessibility"].update( + {"voiceover_user_observed": True} + ), + lambda value: value.update({"private_key": "canary"}), + ) + for mutation in mutations: + changed = copy.deepcopy(canonical) + mutation(changed) + with self.assertRaises(ValueError): + fixture.validate_persona_attempt_evidence( + changed, suite, require_measured_network=True + ) + + def test_persona_result_v2_is_reconstructed_only_from_measured_attempts(self) -> None: + suite, attachments = self.persona_attempt_attachments() + result = fixture.reconstruct_persona_result_v2( + suite, + attachments, + fixture_sha256="5" * 64, + fixture_schema_sha256="6" * 64, + attempt_schema_sha256="7" * 64, + result_schema_sha256="8" * 64, + result_bundle_sha256="9" * 64, + forward_repairs=[], + ) + self.assertEqual(result["schema"], fixture.PERSONA_RESULT_V2_SCHEMA) + self.assertEqual(result["accepted_events"], 15) + self.assertEqual(result["expected_failure_rejections"], 2) + self.assertEqual(result["accepted_uploads"], 3) + self.assertEqual(result["retrievals"], 3) + self.assertEqual(result["non_loopback_attempts"], 0) + self.assertEqual(len(result["attachments"]), 15) + + _, pending = self.persona_attempt_attachments(measured=False) + with self.assertRaisesRegex(ValueError, "not measured"): + fixture.reconstruct_persona_result_v2( + suite, + pending, + fixture_sha256="5" * 64, + fixture_schema_sha256="6" * 64, + attempt_schema_sha256="7" * 64, + result_schema_sha256="8" * 64, + result_bundle_sha256="9" * 64, + forward_repairs=[], + ) + + def test_xcresult_test_identity_and_attachment_inventory_are_exact(self) -> None: + suite, attachments = self.persona_attempt_attachments() + tests = { + "devices": [], + "testNodes": [ + { + "children": [ + { + "name": "testLocalSocialDeterministicPersonas()", + "nodeIdentifier": fixture.PERSONA_XCRESULT_NODE_IDENTIFIER, + "nodeIdentifierURL": fixture.PERSONA_XCRESULT_NODE_URL, + "nodeType": "Test Case", + "result": "Passed", + } + ], + "name": "Radroots", + "nodeType": "Test Plan", + "result": "Passed", + } + ], + "testPlanConfigurations": [], + } + self.assertEqual( + fixture.exact_persona_test_node(tests)["result"], "Passed" + ) + for mutation in ( + lambda value: value["testNodes"][0]["children"][0].update( + {"result": "Failed"} + ), + lambda value: value["testNodes"][0]["children"][0].update( + {"nodeIdentifierURL": "test://wrong"} + ), + lambda value: value["testNodes"].append( + copy.deepcopy(value["testNodes"][0]) + ), + ): + changed = copy.deepcopy(tests) + mutation(changed) + with self.assertRaises(ValueError): + fixture.exact_persona_test_node(changed) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + rows = [] + for name, (raw, _) in zip( + fixture.PERSONA_ATTACHMENT_NAMES, attachments, strict=True + ): + exported = "exported-" + name + (root / exported).write_bytes(raw) + rows.append( + { + "exportedFileName": exported, + "suggestedHumanReadableName": name, + "isAssociatedWithFailure": False, + "configurationName": "Test Scheme Action", + "deviceName": "iPhone 17 Pro", + "deviceId": "11111111-2222-3333-4444-555555555555", + } + ) + manifest = [ + { + "testIdentifier": fixture.PERSONA_XCRESULT_NODE_IDENTIFIER, + "testIdentifierURL": fixture.PERSONA_XCRESULT_NODE_URL, + "attachments": rows, + } + ] + (root / "manifest.json").write_text( + json.dumps(manifest), encoding="utf-8" + ) + loaded = fixture.load_exported_persona_attachments( + root, suite, require_measured_network=True + ) + self.assertEqual(len(loaded), 15) + self.assertEqual(loaded[0][1]["attempt_id"], "P01-A01") + + rows[0]["suggestedHumanReadableName"] = rows[1][ + "suggestedHumanReadableName" + ] + (root / "manifest.json").write_text( + json.dumps(manifest), encoding="utf-8" + ) + with self.assertRaises(ValueError): + fixture.load_exported_persona_attachments( + root, suite, require_measured_network=True + ) + + def test_xcresult_attachment_read_rejects_maximum_plus_one(self) -> None: + suite, attachments = self.persona_attempt_attachments() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + name = fixture.PERSONA_ATTACHMENT_NAMES[0] + exported = "oversized.json" + (root / exported).write_bytes( + b"{" + b" " * fixture.MAX_PERSONA_ATTACHMENT_BYTES + b"}" + ) + rows = [ + { + "exportedFileName": exported, + "suggestedHumanReadableName": name, + "isAssociatedWithFailure": False, + "configurationName": "Test Scheme Action", + "deviceName": "iPhone 17 Pro", + "deviceId": "11111111-2222-3333-4444-555555555555", + } + ] + for index, other_name in enumerate( + fixture.PERSONA_ATTACHMENT_NAMES[1:], 1 + ): + other_exported = f"exported-{index}.json" + (root / other_exported).write_bytes(attachments[index][0]) + rows.append( + { + "exportedFileName": other_exported, + "suggestedHumanReadableName": other_name, + "isAssociatedWithFailure": False, + "configurationName": "Test Scheme Action", + "deviceName": "iPhone 17 Pro", + "deviceId": "11111111-2222-3333-4444-555555555555", + } + ) + (root / "manifest.json").write_text( + json.dumps( + [ + { + "testIdentifier": fixture.PERSONA_XCRESULT_NODE_IDENTIFIER, + "testIdentifierURL": fixture.PERSONA_XCRESULT_NODE_URL, + "attachments": rows, + } + ] + ), + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "byte bound"): + fixture.load_exported_persona_attachments( + root, suite, require_measured_network=True + ) + def test_simulator_metadata_uses_exact_result_bundle_device(self) -> None: udid = "11111111-2222-3333-4444-555555555555" result_bundle = Path("result.xcresult") diff --git a/scripts/verify-package-contract.sh b/scripts/verify-package-contract.sh @@ -80,6 +80,14 @@ grep -Fq '<key>RADROOTS_IOS_UI_TEST_FIXTURE_CONTROL</key>' \ "$repo_root/RadrootsUITests/Info.plist" grep -Fq '<key>RADROOTS_IOS_UI_TEST_NETWORK_PROFILE</key>' \ "$repo_root/RadrootsUITests/Info.plist" +for evidence_key in \ + RADROOTS_IOS_UI_TEST_SOURCE_COMMIT \ + RADROOTS_IOS_UI_TEST_SOURCE_TREE \ + RADROOTS_IOS_UI_TEST_APP_BUILD_SHA256 \ + RADROOTS_IOS_UI_TEST_SIMULATOR_ID +do + grep -Fq "<key>$evidence_key</key>" "$repo_root/RadrootsUITests/Info.plist" +done grep -Fq 'local-social-ui-test)' "$repo_root/scripts/xcode.sh" grep -Fq 'RadrootsUITests/RadrootsRemoteQualificationUITests/testLocalSocialFiveFlowScenario' \ "$repo_root/scripts/xcode.sh" @@ -109,7 +117,9 @@ for fixture in \ bud11-upload-authorization-mutations.v1.schema.json \ local-social-personas.v1.json \ local-social-personas.v1.schema.json \ - local-social-persona-results.v1.schema.json + local-social-persona-results.v1.schema.json \ + local-social-persona-attempt-evidence.v1.schema.json \ + local-social-persona-results.v2.schema.json do test -f "$repo_root/test-fixtures/$fixture" done @@ -119,7 +129,13 @@ python3 "$repo_root/scripts/local-social-fixture.py" verify-bud11-corpus \ python3 "$repo_root/scripts/local-social-fixture.py" verify-persona-fixture \ --fixture "$repo_root/test-fixtures/local-social-personas.v1.json" \ --fixture-schema "$repo_root/test-fixtures/local-social-personas.v1.schema.json" \ - --result-schema "$repo_root/test-fixtures/local-social-persona-results.v1.schema.json" + --result-schema "$repo_root/test-fixtures/local-social-persona-results.v1.schema.json" \ + --attempt-schema "$repo_root/test-fixtures/local-social-persona-attempt-evidence.v1.schema.json" \ + --result-v2-schema "$repo_root/test-fixtures/local-social-persona-results.v2.schema.json" +grep -Fq 'RadrootsUITests/testLocalSocialDeterministicPersonas' \ + "$repo_root/scripts/local-social-fixture.py" +grep -Fq 'radroots.ios.local-social.persona-attempt-evidence.v1' \ + "$repo_root/RadrootsUITests/RadrootsRemoteQualificationUITests.swift" ( cd "$repo_root" python3 -m unittest scripts/test_local_social_fixture.py diff --git a/scripts/xcode.sh b/scripts/xcode.sh @@ -210,6 +210,14 @@ case "$operation" in source_commit=$(git rev-parse HEAD) source_tree=$(git rev-parse 'HEAD^{tree}') upstream=$(git rev-parse '@{upstream}') + xcode_identity=$(xcodebuild -version | tr '\n' ' ') + simulator_sdk_build=$(xcrun --sdk iphonesimulator --show-sdk-build-version) + app_build_sha256=$( + printf 'radroots.ios.local-social.app-build.v1\0%s\0%s\0%s\0%s\0%s\0%s\0%s\0' \ + "$source_commit" "$source_tree" RadrootsUITests test Debug \ + "$xcode_identity" "$simulator_sdk_build" \ + | shasum -a 256 | awk '{print $1}' + ) persona_repair_commit=${RADROOTS_IOS_UI_TEST_FORWARD_REPAIR_COMMIT:-} if [[ "$source_commit" != "$upstream" ]]; then echo "error: persona qualification source is not equal to its configured upstream" >&2 @@ -226,7 +234,9 @@ case "$operation" in python3 scripts/local-social-fixture.py verify-persona-fixture \ --fixture "$persona_fixture" \ --fixture-schema test-fixtures/local-social-personas.v1.schema.json \ - --result-schema test-fixtures/local-social-persona-results.v1.schema.json + --result-schema test-fixtures/local-social-persona-results.v1.schema.json \ + --attempt-schema test-fixtures/local-social-persona-attempt-evidence.v1.schema.json \ + --result-v2-schema test-fixtures/local-social-persona-results.v2.schema.json python3 scripts/local-social-fixture.py serve \ --relay-port "$relay_port" \ --blossom-port "$blossom_port" \ @@ -290,6 +300,10 @@ case "$operation" in "RADROOTS_IOS_UI_TEST_BLOSSOM_ORIGINS=http://127.0.0.1:$blossom_port" \ "RADROOTS_IOS_UI_TEST_FIXTURE_CONTROL=$control" \ "RADROOTS_IOS_UI_TEST_NETWORK_PROFILE=simulator" \ + "RADROOTS_IOS_UI_TEST_SOURCE_COMMIT=${source_commit:-}" \ + "RADROOTS_IOS_UI_TEST_SOURCE_TREE=${source_tree:-}" \ + "RADROOTS_IOS_UI_TEST_APP_BUILD_SHA256=${app_build_sha256:-}" \ + "RADROOTS_IOS_UI_TEST_SIMULATOR_ID=$simulator_id" \ test test_status=$? set -e diff --git a/test-fixtures/local-social-persona-attempt-evidence.v1.schema.json b/test-fixtures/local-social-persona-attempt-evidence.v1.schema.json @@ -0,0 +1,145 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://radroots.org/schemas/ios/local-social-persona-attempt-evidence.v1.schema.json", + "title": "Radroots iOS local-social persona attempt evidence", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", "schema_version", "test_invocation", "source", "app_build_sha256", + "simulator", "run_id", "persona_run_id", "persona_alias", "attempt_id", + "attempt_order", "flow", "expected_failure", "public_identity_sha256", + "endpoint_policy_sha256", "ui_observation", "network_observation", + "accessibility", "artifact_digests" + ], + "properties": { + "schema": {"const": "radroots.ios.local-social.persona-attempt-evidence.v1"}, + "schema_version": {"const": 1}, + "test_invocation": { + "type": "object", + "additionalProperties": false, + "required": ["target", "identifier", "action", "configuration"], + "properties": { + "target": {"const": "RadrootsUITests"}, + "identifier": {"const": "RadrootsUITests/testLocalSocialDeterministicPersonas"}, + "action": {"const": "test"}, + "configuration": {"const": "Debug"} + } + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["commit", "tree"], + "properties": { + "commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "tree": {"type": "string", "pattern": "^[0-9a-f]{40}$"} + } + }, + "app_build_sha256": {"$ref": "#/$defs/sha256"}, + "simulator": {"$ref": "#/$defs/simulator"}, + "run_id": {"$ref": "#/$defs/run_id"}, + "persona_run_id": {"$ref": "#/$defs/run_id"}, + "persona_alias": {"enum": ["P01", "P02", "P03", "P04", "P05"]}, + "attempt_id": {"type": "string", "pattern": "^P0[1-5]-A0[1-3]$"}, + "attempt_order": {"type": "integer", "minimum": 1, "maximum": 15}, + "flow": {"enum": ["Update", "PhotoUpdate", "Ask", "Event", "FoodAvailability"]}, + "expected_failure": {"enum": ["none", "validation_recovery", "transport_retry_relaunch"]}, + "public_identity_sha256": {"$ref": "#/$defs/nonzero_sha256"}, + "endpoint_policy_sha256": {"$ref": "#/$defs/sha256"}, + "ui_observation": { + "type": "object", + "additionalProperties": false, + "required": [ + "validation_attempted", "validation_rejected", "retry_attempts", + "relaunches", "retention_verified", "today_projection_verified" + ], + "properties": { + "validation_attempted": {"type": "boolean"}, + "validation_rejected": {"type": "boolean"}, + "retry_attempts": {"type": "integer", "minimum": 0, "maximum": 2}, + "relaunches": {"type": "integer", "minimum": 0, "maximum": 2}, + "retention_verified": {"type": "boolean"}, + "today_projection_verified": {"type": "boolean"} + } + }, + "network_observation": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["state"], + "properties": {"state": {"const": "pending_step_258"}} + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "state", "accepted_connections", "rejected_connections", + "non_loopback_attempts", "subscriptions", "accepted_events", + "accepted_uploads", "retrievals", "unintended_publications", + "events_accepted_during_expected_failure", "final_candidate_data_loss" + ], + "properties": { + "state": {"const": "measured"}, + "accepted_connections": {"type": "integer", "minimum": 0, "maximum": 4096}, + "rejected_connections": {"type": "integer", "minimum": 0, "maximum": 4096}, + "non_loopback_attempts": {"type": "integer", "minimum": 0, "maximum": 4096}, + "subscriptions": {"type": "integer", "minimum": 0, "maximum": 4096}, + "accepted_events": {"type": "integer", "minimum": 0, "maximum": 1}, + "accepted_uploads": {"type": "integer", "minimum": 0, "maximum": 1}, + "retrievals": {"type": "integer", "minimum": 0, "maximum": 1}, + "unintended_publications": {"type": "integer", "minimum": 0, "maximum": 1}, + "events_accepted_during_expected_failure": {"type": "integer", "minimum": 0, "maximum": 1}, + "final_candidate_data_loss": {"type": "integer", "minimum": 0, "maximum": 1} + } + } + ] + }, + "accessibility": { + "type": "object", + "additionalProperties": false, + "required": [ + "locale", "content_size", "reduce_motion", "progressive_disclosure", + "labels_values_traits", "keyboard_focus", "visible_actions", + "voiceover_user_observed" + ], + "properties": { + "locale": {"const": "en_US"}, + "content_size": {"const": "accessibility-extra-extra-extra-large"}, + "reduce_motion": {"const": true}, + "progressive_disclosure": {"type": "boolean"}, + "labels_values_traits": {"type": "boolean"}, + "keyboard_focus": {"type": "boolean"}, + "visible_actions": {"const": true}, + "voiceover_user_observed": {"const": false} + } + }, + "artifact_digests": { + "type": "array", + "maxItems": 4, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["role", "sha256"], + "properties": { + "role": {"enum": ["media_input", "uploaded_blob", "retrieved_blob", "ui_snapshot"]}, + "sha256": {"$ref": "#/$defs/sha256"} + } + } + } + }, + "$defs": { + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "nonzero_sha256": {"type": "string", "pattern": "^(?!0{64}$)[0-9a-f]{64}$"}, + "run_id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{6,62}[a-z0-9]$"}, + "simulator": { + "type": "object", + "additionalProperties": false, + "required": ["udid", "os", "architecture"], + "properties": { + "udid": {"type": "string", "pattern": "^[A-F0-9-]{36}$"}, + "os": {"type": "string", "pattern": "^iOS (1[89]|[2-9][0-9]|[1-9][0-9]{2,})(\\.[0-9]+){1,2}$", "maxLength": 32}, + "architecture": {"const": "arm64"} + } + } + } +} diff --git a/test-fixtures/local-social-persona-results.v2.schema.json b/test-fixtures/local-social-persona-results.v2.schema.json @@ -0,0 +1,137 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://radroots.org/schemas/ios/local-social-persona-results.v2.schema.json", + "title": "Radroots iOS local-social measured persona result", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", "schema_version", "run_id", "test_invocation", "source", + "app_build_sha256", "endpoint_policy_sha256", "fixture_sha256", "fixture_schema_sha256", + "attempt_schema_sha256", "result_schema_sha256", "simulator", + "result_bundle_sha256", "attachments", "personas", "flow_counts", + "accepted_events", "event_kind_counts", "accepted_uploads", "retrievals", + "distinct_identities", "unknown_attempts", "duplicate_attempts", + "expected_failure_rejections", "events_accepted_during_expected_failures", + "accepted_connections", "rejected_connections", "non_loopback_attempts", + "unintended_publications", "final_candidate_data_loss", "accessibility", + "forward_repairs", "complete_matrix_rerun" + ], + "properties": { + "schema": {"const": "radroots.ios.local-social.persona-results.v2"}, + "schema_version": {"const": 2}, + "run_id": {"$ref": "#/$defs/run_id"}, + "test_invocation": {"$ref": "#/$defs/test_invocation"}, + "source": {"$ref": "#/$defs/source"}, + "app_build_sha256": {"$ref": "#/$defs/sha256"}, + "endpoint_policy_sha256": {"$ref": "#/$defs/sha256"}, + "fixture_sha256": {"$ref": "#/$defs/sha256"}, + "fixture_schema_sha256": {"$ref": "#/$defs/sha256"}, + "attempt_schema_sha256": {"$ref": "#/$defs/sha256"}, + "result_schema_sha256": {"$ref": "#/$defs/sha256"}, + "simulator": {"$ref": "#/$defs/simulator"}, + "result_bundle_sha256": {"$ref": "#/$defs/sha256"}, + "attachments": { + "type": "array", "minItems": 15, "maxItems": 15, + "items": { + "type": "object", "additionalProperties": false, + "required": ["attempt_id", "sha256"], + "properties": { + "attempt_id": {"type": "string", "pattern": "^P0[1-5]-A0[1-3]$"}, + "sha256": {"$ref": "#/$defs/sha256"} + } + } + }, + "personas": { + "type": "array", "minItems": 5, "maxItems": 5, + "items": { + "type": "object", "additionalProperties": false, + "required": ["alias", "public_identity_sha256", "subscriptions", "attempt_ids"], + "properties": { + "alias": {"enum": ["P01", "P02", "P03", "P04", "P05"]}, + "public_identity_sha256": {"$ref": "#/$defs/nonzero_sha256"}, + "subscriptions": {"type": "integer", "minimum": 1, "maximum": 4096}, + "attempt_ids": { + "type": "array", "minItems": 3, "maxItems": 3, "uniqueItems": true, + "items": {"type": "string", "pattern": "^P0[1-5]-A0[1-3]$"} + } + } + } + }, + "flow_counts": {"$ref": "#/$defs/flow_counts"}, + "accepted_events": {"const": 15}, + "event_kind_counts": { + "type": "object", "additionalProperties": false, + "required": ["1", "31923", "30402"], + "properties": {"1": {"const": 9}, "31923": {"const": 3}, "30402": {"const": 3}} + }, + "accepted_uploads": {"const": 3}, + "retrievals": {"const": 3}, + "distinct_identities": {"const": 5}, + "unknown_attempts": {"const": 0}, + "duplicate_attempts": {"const": 0}, + "expected_failure_rejections": {"const": 2}, + "events_accepted_during_expected_failures": {"const": 0}, + "accepted_connections": {"type": "integer", "minimum": 1, "maximum": 61440}, + "rejected_connections": {"type": "integer", "minimum": 0, "maximum": 61440}, + "non_loopback_attempts": {"const": 0}, + "unintended_publications": {"const": 0}, + "final_candidate_data_loss": {"const": 0}, + "accessibility": { + "type": "object", "additionalProperties": false, + "required": ["locale", "content_size", "reduce_motion", "semantic_attempts", "keyboard_focus_attempts", "voiceover_user_observed"], + "properties": { + "locale": {"const": "en_US"}, + "content_size": {"const": "accessibility-extra-extra-extra-large"}, + "reduce_motion": {"const": true}, + "semantic_attempts": {"type": "integer", "minimum": 3, "maximum": 15}, + "keyboard_focus_attempts": {"const": 3}, + "voiceover_user_observed": {"const": false} + } + }, + "forward_repairs": { + "type": "array", "maxItems": 16, "uniqueItems": true, + "items": {"type": "string", "pattern": "^[0-9a-f]{40}$"} + }, + "complete_matrix_rerun": {"const": true} + }, + "$defs": { + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "nonzero_sha256": {"type": "string", "pattern": "^(?!0{64}$)[0-9a-f]{64}$"}, + "run_id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{6,62}[a-z0-9]$"}, + "test_invocation": { + "type": "object", "additionalProperties": false, + "required": ["target", "identifier", "action", "configuration"], + "properties": { + "target": {"const": "RadrootsUITests"}, + "identifier": {"const": "RadrootsUITests/testLocalSocialDeterministicPersonas"}, + "action": {"const": "test"}, + "configuration": {"const": "Debug"} + } + }, + "source": { + "type": "object", "additionalProperties": false, + "required": ["commit", "tree"], + "properties": { + "commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "tree": {"type": "string", "pattern": "^[0-9a-f]{40}$"} + } + }, + "simulator": { + "type": "object", "additionalProperties": false, + "required": ["udid", "os", "architecture"], + "properties": { + "udid": {"type": "string", "pattern": "^[A-F0-9-]{36}$"}, + "os": {"type": "string", "pattern": "^iOS (1[89]|[2-9][0-9]|[1-9][0-9]{2,})(\\.[0-9]+){1,2}$", "maxLength": 32}, + "architecture": {"const": "arm64"} + } + }, + "flow_counts": { + "type": "object", "additionalProperties": false, + "required": ["Update", "PhotoUpdate", "Ask", "Event", "FoodAvailability"], + "properties": { + "Update": {"const": 3}, "PhotoUpdate": {"const": 3}, "Ask": {"const": 3}, + "Event": {"const": 3}, "FoodAvailability": {"const": 3} + } + } + } +}