measurement_runner.mojo (6819B)
1 """Standalone governed H005A measurement entry point (ADR-0012 D29). 2 3 Runs one persistent HYF stdio process through warmup and measured frames and 4 prints the exact identity, per-phase timing, numeric RSS/FD samples, frame 5 counts and child exit. Exits nonzero on any failed mandatory guarantee. 6 7 ADR-0019 D39 MR04 additionally records the numeric direct-local-provider 8 elapsed time, request/connection counts and the deterministic client/schema 9 construction characterization, so the pre-migration baseline is source-backed 10 rather than a test duration or an unprinted assertion. 11 12 Usage (governed lane): ``cargo extbuild run -- pixi run --frozen measure-h005a`` 13 """ 14 15 from std.collections import List 16 17 from safe_tempdir import SafeTempDir 18 19 from parent_lifecycle import CleanupGuard, now_ms 20 from stdio_process_helper import ( 21 HYF_PATHS_PROFILE_ENV, 22 HYF_PATHS_REPO_LOCAL_ROOT_ENV, 23 ScopedEnvVar, 24 ) 25 from measurement_process_helper import ( 26 build_product_binary, 27 measure_persistent_process, 28 ) 29 from max_local_process_helper import spawn_max_local_stub 30 31 from json import Value, loads 32 33 from hyf_core.request_context import default_request_context 34 from hyf_provider.client import post_max_local_chat_completion 35 from hyf_provider.config import MaxLocalProviderConfig 36 from hyf_provider.result import parse_query_analysis_from_chat_completion 37 from hyf_provider.schema import build_query_rewrite_request_body 38 39 40 comptime MEASUREMENT_DEADLINE_MS = 180000 41 comptime WARMUP_FRAMES = 100 42 comptime MEASURED_FRAMES = 1000 43 44 comptime ANALYSIS_JSON_TEXT = ( 45 '{"original_text":"eggs near me",' 46 '"normalized_text":"eggs near me",' 47 '"rewritten_text":"eggs",' 48 '"query_terms":["eggs"],' 49 '"normalization_signals":["local_intent_detected"],' 50 '"ranking_hints":["prefer_local_results"],' 51 '"extracted_filters":{' 52 '"local_intent":true,' 53 '"fulfillment":"unspecified",' 54 '"time_window":"unspecified"' 55 "}}" 56 ) 57 58 59 def characterize_direct_provider(mut guard: CleanupGuard) raises: 60 """Numeric direct-local-provider and client/schema characterization. 61 62 One direct provider request over one verified connection against the local 63 stub: the elapsed stub-startup, schema-construction and request times and 64 the request/connection counts are printed, and the deterministic request 65 body/response parsing is proved. The Morph daemon-assisted path is out of 66 scope here and remains the explicit H024 obligation. 67 """ 68 var stub_start = now_ms() 69 with spawn_max_local_stub(0, "count_requests", 1, guard) as stub: 70 var stub_startup_ms = now_ms() - stub_start 71 var config = MaxLocalProviderConfig( 72 base_url="http://127.0.0.1:" + String(stub.port) + "/v1/", 73 health_url="http://127.0.0.1:" + String(stub.port) + "/health", 74 model="max-local-query-rewrite", 75 request_timeout_ms=15000, 76 ) 77 var context = default_request_context() 78 context.return_provenance = True 79 var construct_start = now_ms() 80 var body = build_query_rewrite_request_body( 81 config, "eggs near me", context 82 ) 83 var construct_ms = now_ms() - construct_start 84 var response_format = body["response_format"] 85 print( 86 "h005a.client_schema", 87 "fields=" + String(body.object_count()), 88 "messages=" + String(body["messages"].array_count()), 89 "response_format_type=" + response_format["type"].string_value(), 90 "json_schema_name=" 91 + response_format["json_schema"]["name"].string_value(), 92 "model=" + body["model"].string_value(), 93 ) 94 var request_start = now_ms() 95 var outcome = post_max_local_chat_completion(config, body) 96 var request_ms = now_ms() - request_start 97 stub.wait() 98 var failure_text = "false" 99 if outcome.failure: 100 failure_text = "true" 101 print( 102 "h005a.direct_provider", 103 "stub_startup_ms=" + String(stub_startup_ms), 104 "construct_ms=" + String(construct_ms), 105 "request_ms=" + String(request_ms), 106 "requests=" + String(stub.request_count()), 107 "connections=" + String(stub.connection_count()), 108 "failure=" + failure_text, 109 ) 110 if outcome.failure: 111 raise Error("measurement: direct provider request failed") 112 if stub.request_count() != 1: 113 raise Error("measurement: direct provider request count mismatch") 114 if stub.connection_count() != 1: 115 raise Error( 116 "measurement: direct provider connection count mismatch" 117 ) 118 var response = loads("{}") 119 var choices = loads("[]") 120 var choice = loads("{}") 121 var message = loads("{}") 122 message.set("content", Value(ANALYSIS_JSON_TEXT)) 123 choice.set("message", message) 124 choices.append(choice) 125 response.set("choices", choices) 126 var analysis = parse_query_analysis_from_chat_completion(response) 127 if analysis.original_text != "eggs near me": 128 raise Error("measurement: provider response parsing mismatch") 129 if len(analysis.query_terms) != 1: 130 raise Error("measurement: provider response term count mismatch") 131 132 133 def main() raises: 134 var guard = CleanupGuard() 135 var source_root = "." 136 with SafeTempDir() as temp_dir: 137 with ScopedEnvVar(HYF_PATHS_PROFILE_ENV, "repo_local"): 138 with ScopedEnvVar(HYF_PATHS_REPO_LOCAL_ROOT_ENV, temp_dir): 139 var built = build_product_binary(source_root, temp_dir, guard) 140 print( 141 "h005a.build", 142 "build_ms=" + String(built.build_ms), 143 built.source.describe(), 144 ) 145 var argv = List[String]() 146 var measured = measure_persistent_process( 147 source_root, 148 built.binary_path, 149 argv^, 150 WARMUP_FRAMES, 151 MEASURED_FRAMES, 152 MEASUREMENT_DEADLINE_MS, 153 guard, 154 "ps", 155 "lsof", 156 built.source.revision, 157 built.source.manifest_sha256, 158 built.binary_sha256, 159 ) 160 print("h005a.identity", measured.identity.describe()) 161 print("h005a.measurement", measured.summary()) 162 print("h005a.sampling_method", measured.sampling_method) 163 print("h005a.sampling_cadence", measured.sampling_cadence) 164 print( 165 "h005a.stderr_bytes", 166 measured.stderr_excerpt.byte_length(), 167 ) 168 characterize_direct_provider(guard) 169 guard.assert_clean() 170 print("h005a_measurement: ok")