test_provider_helpers.mojo (86097B)
1 """Strict fixture/script verification tests (ADR-0012 D29 / ADR-0014 D33). 2 3 Covers FX01-FX08 for both providers plus the in-memory mutation controls that 4 prove a negative control fails for the intended cause rather than for any 5 exception, signal or unrelated startup failure. 6 """ 7 8 from std.testing import TestSuite, assert_true, assert_equal 9 from std.ffi import ErrNo, c_int, external_call, get_errno 10 11 from flare.net import SocketAddr 12 from flare.net.socket import RawSocket 13 from flare.tcp import TcpListener, TcpStream 14 15 from parent_lifecycle import ( 16 CENSUS_MAX_FDS, 17 CleanupGuard, 18 PipedChildState, 19 ProcessStatus, 20 child_exit, 21 classify_census_errno, 22 classify_wait_errno, 23 close_fd, 24 descriptor_census, 25 descriptor_census_with_faults, 26 dup2_fd, 27 finalize_owned_failure, 28 fork_owned_or_close, 29 fork_owned_or_close3, 30 fork_pid, 31 make_pipe, 32 make_three_pipes, 33 now_ms, 34 open_fd_count, 35 open_fd_count_checked, 36 open_fd_count_checked_with_faults, 37 parse_ready_line, 38 parse_ready_or_cleanup, 39 pid_not_waitable, 40 pid_running, 41 piped_child_state, 42 read_all_bounded, 43 read_line_bounded, 44 sleep_ms, 45 wait_nohang, 46 write_fd_bounded, 47 write_raw, 48 write_raw_bytes, 49 ) 50 from strict_fixture import ( 51 STRICT_MAX_REPORT_BYTES, 52 ConnectionReader, 53 ExchangeScript, 54 FramedRequest, 55 authorization_reason, 56 classify_write_error_cause, 57 exchange_script, 58 is_peer_close_cause, 59 json_escape, 60 parse_report, 61 render_write_api_error, 62 report_status_matches_exit, 63 verify_exchange, 64 write_errno_class, 65 ) 66 from max_local_process_helper import ( 67 SpawnedMaxLocalStub, 68 reserve_loopback_port, 69 spawn_max_local_scripted, 70 spawn_max_local_stub, 71 ) 72 from jev_provider_helper import ( 73 SpawnedJevStub, 74 spawn_jev_scripted_auto, 75 spawn_jev_stub, 76 spawn_jev_stub_auto, 77 ) 78 79 80 # ── Client helpers ────────────────────────────────────────────────────────── 81 82 83 def _read_all(mut client: TcpStream) raises -> String: 84 var buffer = InlineArray[Byte, 4096](fill=0) 85 var response = String("") 86 while True: 87 var n = client.read(buffer.unsafe_ptr(), 4096) 88 if n <= 0: 89 break 90 response += String( 91 unsafe_from_utf8=Span(ptr=buffer.unsafe_ptr(), length=Int(n)) 92 ) 93 return response^ 94 95 96 def _write_all(mut client: TcpStream, text: String) raises: 97 client.write_all(Span[UInt8, _](text.as_bytes())) 98 99 100 def _raw_exchange(port: Int, raw: String) raises -> String: 101 var client = TcpStream.connect(SocketAddr.localhost(UInt16(port))) 102 _write_all(client, raw) 103 var response = _read_all(client) 104 client.close() 105 return response^ 106 107 108 def _raw_send_only(port: Int, raw: String) raises: 109 var client = TcpStream.connect(SocketAddr.localhost(UInt16(port))) 110 _write_all(client, raw) 111 client.close() 112 113 114 def _read_one_response(mut client: TcpStream) raises -> String: 115 var data = String("") 116 var buf = InlineArray[Byte, 1024](fill=0) 117 while data.find("\r\n\r\n") < 0: 118 var n = client.read(buf.unsafe_ptr(), 1024) 119 if n <= 0: 120 return data^ 121 data += String( 122 unsafe_from_utf8=Span(ptr=buf.unsafe_ptr(), length=Int(n)) 123 ) 124 var header_end = data.find("\r\n\r\n") 125 var length = 0 126 for line in data[byte=0:header_end].split("\r\n"): 127 var entry = String(line) 128 if entry.lower().startswith("content-length:"): 129 length = Int(String(entry[byte=15:]).strip()) 130 var body_start = header_end + 4 131 while data.byte_length() - body_start < length: 132 var n = client.read(buf.unsafe_ptr(), 1024) 133 if n <= 0: 134 break 135 data += String( 136 unsafe_from_utf8=Span(ptr=buf.unsafe_ptr(), length=Int(n)) 137 ) 138 return data^ 139 140 141 def _request( 142 port: Int, 143 method: String, 144 path: String, 145 body: String, 146 extra: String = "", 147 connection: String = "close", 148 ) raises -> String: 149 var raw = ( 150 method 151 + " " 152 + path 153 + " HTTP/1.1\r\nhost: 127.0.0.1\r\n" 154 + extra 155 + "content-type: application/json\r\ncontent-length: " 156 + String(body.byte_length()) 157 + "\r\nconnection: " 158 + connection 159 + "\r\n\r\n" 160 + body 161 ) 162 return _raw_exchange(port, raw) 163 164 165 def _default_script() -> ExchangeScript: 166 return exchange_script( 167 "expect_ok", "POST", "/v1/chat/completions", 200, '{"ok":true}' 168 ) 169 170 171 @fieldwise_init 172 struct FramingFailure(Movable): 173 """Bounded snapshot of a fixture read failure for post-scope asserts.""" 174 175 var ok: Bool 176 var phase_value: String 177 var case_value: String 178 var reason_value: String 179 var requests: Int 180 var connections: Int 181 182 def phase(self) -> String: 183 return String(self.phase_value) 184 185 def reason(self) -> String: 186 return String(self.reason_value) 187 188 def failure_case(self) -> String: 189 return String(self.case_value) 190 191 192 # ── Existing convenience-mode coverage ────────────────────────────────────── 193 194 195 def test_max_local_stub_reads_fragmented_large_body() raises: 196 var guard_1 = CleanupGuard() 197 with spawn_max_local_stub(0, "echo_body_bytes", 1, guard_1) as stub: 198 var body = String("") 199 for _ in range(9000): 200 body += "x" 201 var response = _request(stub.port, "POST", "/v1/chat/completions", body) 202 assert_true(response.find('"received_bytes":9000') >= 0) 203 stub.wait() 204 205 guard_1.assert_clean() 206 207 208 def test_max_local_stub_counts_every_wire_attempt() raises: 209 var requests = 3 210 var guard_2 = CleanupGuard() 211 with spawn_max_local_stub(0, "count_requests", requests, guard_2) as stub: 212 for index in range(requests): 213 var response = _request( 214 stub.port, "POST", "/v1/chat/completions", "{}" 215 ) 216 assert_true( 217 response.find('"request_index":' + String(index + 1)) >= 0 218 ) 219 stub.wait() 220 221 guard_2.assert_clean() 222 223 224 def test_max_local_stub_rejects_unknown_path() raises: 225 # FX02/FX04: an unexpected route must fail fixture verification, not be 226 # answered 404 and then reported as a successful stub run. 227 var guard_3 = CleanupGuard() 228 with spawn_max_local_stub(0, "query_rewrite_ok", 1, guard_3) as stub: 229 var response = _request(stub.port, "POST", "/not-a-route", "{}") 230 assert_true(response.find("404") < 0) 231 stub.reap() 232 assert_true(not stub.ok()) 233 assert_equal(stub.phase(), "exchange") 234 assert_equal(stub.reason(), "unexpected_path") 235 236 guard_3.assert_clean() 237 238 239 def test_max_local_stub_binds_and_reports_port() raises: 240 var guard_4 = CleanupGuard() 241 with spawn_max_local_stub(0, "count_requests", 1, guard_4) as stub: 242 assert_true(stub.port > 0) 243 var response = _request(stub.port, "POST", "/v1/chat/completions", "{}") 244 assert_true(response.find('"request_index":1') >= 0) 245 stub.wait() 246 247 guard_4.assert_clean() 248 249 250 def test_jev_stub_observes_bearer_sentinel_at_intended_origin() raises: 251 var guard_5 = CleanupGuard() 252 with spawn_jev_stub_auto("echo_authorization", 1, guard_5) as started: 253 var response = _request( 254 started.port, 255 "POST", 256 "/v1/systemone", 257 "{}", 258 "authorization: Bearer hyf-sentinel-token\r\n", 259 ) 260 assert_true(response.find("hyf-sentinel-token") >= 0) 261 started.stub.wait() 262 263 guard_5.assert_clean() 264 265 266 def test_max_local_stub_stalled_child_is_reaped() raises: 267 var guard_6 = CleanupGuard() 268 with spawn_max_local_stub(0, "stall", 1, guard_6) as stub: 269 _raw_send_only( 270 stub.port, 271 ( 272 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 273 "content-length: 2\r\nconnection: close\r\n\r\n{}" 274 ), 275 ) 276 stub.terminate() 277 assert_true(pid_not_waitable(stub.pid)) 278 279 # ── FX01: explicit ordered scripted exchanges ─────────────────────────────── 280 281 guard_6.assert_clean() 282 283 284 def test_max_local_scripted_matches_explicit_exchange() raises: 285 var scripts = List[ExchangeScript]() 286 var script = exchange_script( 287 "explicit_ok", "POST", "/v1/chat/completions", 201, '{"made":"yes"}' 288 ) 289 script.headers = "x-sentinel:explicit" 290 script.check_body = True 291 script.body = '{"q":"apples"}' 292 script.response_headers = "x-scripted: yes" 293 script.delay_ms = 20 294 scripts.append(script^) 295 var guard_7 = CleanupGuard() 296 with spawn_max_local_scripted(0, scripts^, guard_7) as stub: 297 var response = _request( 298 stub.port, 299 "POST", 300 "/v1/chat/completions", 301 '{"q":"apples"}', 302 "x-sentinel: explicit\r\n", 303 ) 304 assert_true(response.find("201") >= 0) 305 assert_true(response.find("x-scripted: yes") >= 0) 306 assert_true(response.find('{"made":"yes"}') >= 0) 307 stub.wait() 308 assert_equal(stub.request_count(), 1) 309 assert_equal(stub.connection_count(), 1) 310 311 guard_7.assert_clean() 312 313 314 def test_max_local_scripted_rejects_wrong_method() raises: 315 var scripts = List[ExchangeScript]() 316 scripts.append(_default_script()) 317 var guard_8 = CleanupGuard() 318 with spawn_max_local_scripted(0, scripts^, guard_8) as stub: 319 _raw_send_only( 320 stub.port, 321 ( 322 "GET /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 323 "content-length: 2\r\nconnection: close\r\n\r\n{}" 324 ), 325 ) 326 stub.reap() 327 assert_equal(stub.phase(), "exchange") 328 assert_equal(stub.reason(), "method_mismatch") 329 330 guard_8.assert_clean() 331 332 333 def test_max_local_scripted_rejects_wrong_path() raises: 334 var scripts = List[ExchangeScript]() 335 scripts.append(_default_script()) 336 var guard_9 = CleanupGuard() 337 with spawn_max_local_scripted(0, scripts^, guard_9) as stub: 338 _raw_send_only( 339 stub.port, 340 ( 341 "POST /v1/other HTTP/1.1\r\nhost: 127.0.0.1\r\n" 342 "content-length: 2\r\nconnection: close\r\n\r\n{}" 343 ), 344 ) 345 stub.reap() 346 assert_equal(stub.phase(), "exchange") 347 assert_equal(stub.reason(), "path_mismatch") 348 349 guard_9.assert_clean() 350 351 352 def test_max_local_scripted_rejects_wrong_selected_header() raises: 353 var scripts = List[ExchangeScript]() 354 var script = _default_script() 355 script.headers = "x-sentinel:expected" 356 scripts.append(script^) 357 var guard_10 = CleanupGuard() 358 with spawn_max_local_scripted(0, scripts^, guard_10) as stub: 359 _raw_send_only( 360 stub.port, 361 ( 362 "POST /v1/chat/completions HTTP/1.1\r\nhost:" 363 " 127.0.0.1\r\nx-sentinel: wrong\r\ncontent-length:" 364 " 2\r\nconnection: close\r\n\r\n{}" 365 ), 366 ) 367 stub.reap() 368 assert_equal(stub.phase(), "exchange") 369 assert_equal(stub.reason(), "header_mismatch:x-sentinel") 370 371 guard_10.assert_clean() 372 373 374 def test_max_local_scripted_rejects_wrong_body() raises: 375 var scripts = List[ExchangeScript]() 376 var script = _default_script() 377 script.check_body = True 378 script.body = '{"expected":true}' 379 scripts.append(script^) 380 var guard_11 = CleanupGuard() 381 with spawn_max_local_scripted(0, scripts^, guard_11) as stub: 382 _raw_send_only( 383 stub.port, 384 ( 385 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 386 'content-length: 15\r\nconnection: close\r\n\r\n{"wrong":true} ' 387 ), 388 ) 389 stub.reap() 390 assert_equal(stub.phase(), "exchange") 391 assert_equal(stub.reason(), "body_mismatch") 392 393 # ── FX02: unexpected/extra/missing/unconsumed accounting ──────────────────── 394 395 guard_11.assert_clean() 396 397 398 def test_scripted_rejects_extra_pipelined_exchange() raises: 399 var scripts = List[ExchangeScript]() 400 var script = _default_script() 401 script.close_connection = False 402 scripts.append(script^) 403 var guard_12 = CleanupGuard() 404 with spawn_max_local_scripted(0, scripts^, guard_12) as stub: 405 var first_frame = ( 406 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 407 "content-length: 2\r\nconnection: keep-alive\r\n\r\n{}" 408 ) 409 var second_frame = String(first_frame) 410 _raw_send_only(stub.port, first_frame + second_frame) 411 stub.reap() 412 assert_equal(stub.phase(), "accounting") 413 assert_equal(stub.reason(), "extra_exchange_after_completion") 414 415 guard_12.assert_clean() 416 417 418 def test_scripted_rejects_missing_exchange() raises: 419 var scripts = List[ExchangeScript]() 420 scripts.append(_default_script()) 421 var guard_13 = CleanupGuard() 422 with spawn_max_local_scripted(0, scripts^, guard_13) as stub: 423 var client = TcpStream.connect(SocketAddr.localhost(UInt16(stub.port))) 424 client.close() 425 stub.reap() 426 assert_equal(stub.phase(), "accounting") 427 assert_equal(stub.reason(), "missing_exchanges") 428 assert_equal(stub.request_count(), 0) 429 430 guard_13.assert_clean() 431 432 433 def test_scripted_reports_unconsumed_remaining_scripts() raises: 434 var scripts = List[ExchangeScript]() 435 var first = _default_script() 436 first.close_connection = False 437 scripts.append(first^) 438 scripts.append(_default_script()) 439 var guard_14 = CleanupGuard() 440 with spawn_max_local_scripted(0, scripts^, guard_14) as stub: 441 var client = TcpStream.connect(SocketAddr.localhost(UInt16(stub.port))) 442 _write_all( 443 client, 444 ( 445 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 446 "content-length: 2\r\nconnection: keep-alive\r\n\r\n{}" 447 ), 448 ) 449 var response = _read_one_response(client) 450 client.close() 451 assert_true(response.find('{"ok":true}') >= 0) 452 stub.reap() 453 assert_equal(stub.reason(), "missing_exchanges") 454 assert_equal(stub.request_count(), 1) 455 456 # ── FX03: strict lexical framing ──────────────────────────────────────────── 457 458 guard_14.assert_clean() 459 460 461 def _framing_failure(raw: String) raises -> FramingFailure: 462 var scripts = List[ExchangeScript]() 463 scripts.append(_default_script()) 464 var guard = CleanupGuard() 465 var snapshot = FramingFailure(False, "", "-", "", 0, 0) 466 with spawn_max_local_scripted(0, scripts^, guard) as stub: 467 _raw_send_only(stub.port, raw) 468 stub.reap() 469 snapshot = FramingFailure( 470 stub.ok(), 471 stub.phase(), 472 stub.failure_case(), 473 stub.reason(), 474 stub.request_count(), 475 stub.connection_count(), 476 ) 477 guard.assert_clean() 478 return snapshot^ 479 480 481 def test_strict_framing_lexical_content_length() raises: 482 var plus = _framing_failure( 483 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 484 "content-length: +2\r\nconnection: close\r\n\r\n{}" 485 ) 486 assert_equal(plus.phase(), "read") 487 assert_equal(plus.reason(), "malformed_content_length") 488 var spaced = _framing_failure( 489 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 490 "content-length: 2 3\r\nconnection: close\r\n\r\n{}" 491 ) 492 assert_equal(spaced.reason(), "malformed_content_length") 493 var empty = _framing_failure( 494 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 495 "content-length:\r\nconnection: close\r\n\r\n" 496 ) 497 assert_equal(empty.reason(), "malformed_content_length") 498 var non_digit = _framing_failure( 499 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 500 "content-length: 2x\r\nconnection: close\r\n\r\n{}" 501 ) 502 assert_equal(non_digit.reason(), "malformed_content_length") 503 var overflow = _framing_failure( 504 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 505 "content-length: 99999999999999999999\r\nconnection: close\r\n\r\n" 506 ) 507 assert_equal(overflow.reason(), "content_length_overflow") 508 509 510 def test_strict_framing_versions_and_header_syntax() raises: 511 var version = _framing_failure( 512 "POST /v1/chat/completions NONHTTP\r\nhost: 127.0.0.1\r\n" 513 "content-length: 2\r\nconnection: close\r\n\r\n{}" 514 ) 515 assert_equal(version.phase(), "read") 516 assert_equal(version.reason(), "malformed_version") 517 var space_name = _framing_failure( 518 "POST /v1/chat/completions HTTP/1.1\r\nhost : 127.0.0.1\r\n" 519 "content-length: 2\r\nconnection: close\r\n\r\n{}" 520 ) 521 assert_equal(space_name.reason(), "malformed_header") 522 var obs_fold = _framing_failure( 523 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 524 " content-length: 2\r\nconnection: close\r\n\r\n{}" 525 ) 526 assert_equal(obs_fold.reason(), "malformed_header") 527 528 529 def test_strict_framing_conflicting_and_transfer_modes() raises: 530 var conflicting = _framing_failure( 531 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 532 "content-length: 2\r\ntransfer-encoding: identity\r\n" 533 "connection: close\r\n\r\n{}" 534 ) 535 assert_equal(conflicting.reason(), "conflicting_framing") 536 var chunked = _framing_failure( 537 "POST /v1/chat/completions HTTP/1.1\r\nhost:" 538 " 127.0.0.1\r\ntransfer-encoding: chunked\r\nconnection:" 539 " close\r\n\r\n2\r\n{}\r\n0\r\n\r\n" 540 ) 541 assert_equal(chunked.reason(), "unsupported_transfer_encoding") 542 var duplicate = _framing_failure( 543 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 544 "transfer-encoding: identity\r\ntransfer-encoding: identity\r\n" 545 "connection: close\r\n\r\n" 546 ) 547 assert_equal(duplicate.reason(), "duplicate_transfer_encoding") 548 549 550 def test_strict_framing_premature_eof() raises: 551 var stub = _framing_failure( 552 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 553 "content-length: 100\r\nconnection: close\r\n\r\nshort" 554 ) 555 assert_equal(stub.phase(), "read") 556 assert_equal(stub.reason(), "premature_eof") 557 558 559 def test_strict_framing_duplicate_content_length() raises: 560 var stub = _framing_failure( 561 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 562 "content-length: 2\r\ncontent-length: 2\r\n" 563 "connection: close\r\n\r\n{}" 564 ) 565 assert_equal(stub.phase(), "read") 566 assert_equal(stub.reason(), "duplicate_content_length") 567 568 569 def test_strict_framing_body_cap_exceeded() raises: 570 var stub = _framing_failure( 571 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 572 "content-length: 1048577\r\nconnection: close\r\n\r\n" 573 ) 574 assert_equal(stub.phase(), "read") 575 assert_equal(stub.reason(), "body_too_large") 576 577 578 def test_strict_framing_header_cap_exceeded() raises: 579 var filler = String("") 580 for _ in range(70000): 581 filler += "a" 582 var stub = _framing_failure( 583 ( 584 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\nx-big: " 585 + filler 586 + "\r\n\r\n" 587 ) 588 ) 589 assert_equal(stub.phase(), "read") 590 assert_equal(stub.reason(), "header_too_large") 591 592 593 def test_jev_echo_authorization_ignores_x_authorization() raises: 594 var guard_16 = CleanupGuard() 595 with spawn_jev_stub_auto("echo_authorization", 1, guard_16) as started: 596 var response = _request( 597 started.port, 598 "POST", 599 "/v1/systemone", 600 "{}", 601 "x-authorization: Bearer spoof\r\n", 602 ) 603 assert_true(response.find("401") >= 0) 604 assert_true(response.find("spoof") < 0) 605 started.stub.wait() 606 607 guard_16.assert_clean() 608 609 610 def test_strict_framing_split_utf8_body() raises: 611 var scripts = List[ExchangeScript]() 612 var script = exchange_script( 613 "utf8", "POST", "/v1/chat/completions", 200, '{"ok":true}' 614 ) 615 script.check_body = True 616 var payload = String("") 617 for _ in range(800): 618 payload += "é" 619 script.body = payload 620 scripts.append(script^) 621 var guard_17 = CleanupGuard() 622 with spawn_max_local_scripted(0, scripts^, guard_17) as stub: 623 var client = TcpStream.connect(SocketAddr.localhost(UInt16(stub.port))) 624 var head = ( 625 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 626 "content-length: " 627 + String(payload.byte_length()) 628 + "\r\nconnection: close\r\n\r\n" 629 ) 630 _write_all(client, head) 631 var payload_bytes = payload.as_bytes() 632 var sent = 0 633 while sent < payload.byte_length(): 634 var end = sent + 3 635 if end > payload.byte_length(): 636 end = payload.byte_length() 637 client.write_all(payload_bytes[sent:end]) 638 sent = end 639 var response = _read_all(client) 640 client.close() 641 assert_true(response.find("200") >= 0) 642 stub.wait() 643 644 guard_17.assert_clean() 645 646 647 def test_strict_framing_surplus_retained_for_second_frame() raises: 648 var scripts = List[ExchangeScript]() 649 var first = exchange_script( 650 "first", "POST", "/v1/chat/completions", 200, '{"n":1}' 651 ) 652 first.close_connection = False 653 scripts.append(first^) 654 var second = exchange_script( 655 "second", "POST", "/v1/chat/completions", 200, '{"n":2}' 656 ) 657 scripts.append(second^) 658 var guard_18 = CleanupGuard() 659 with spawn_max_local_scripted(0, scripts^, guard_18) as stub: 660 var first_frame = ( 661 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 662 "content-length: 2\r\nconnection: keep-alive\r\n\r\n{}" 663 ) 664 var second_frame = String(first_frame) 665 var response = _raw_exchange(stub.port, first_frame + second_frame) 666 assert_true(response.find('{"n":1}') >= 0) 667 assert_true(response.find('{"n":2}') >= 0) 668 stub.wait() 669 assert_equal(stub.request_count(), 2) 670 assert_equal(stub.connection_count(), 1) 671 672 # ── FX04: route/method before auth, exact headers, safe escaping ──────────── 673 674 guard_18.assert_clean() 675 676 677 def test_jev_scripted_wrong_route_auth_not_bypassed() raises: 678 var scripts = List[ExchangeScript]() 679 var script = exchange_script( 680 "jev_expect", "POST", "/v1/systemone", 200, '{"ok":true}' 681 ) 682 script.require_bearer = True 683 scripts.append(script^) 684 var guard_19 = CleanupGuard() 685 with spawn_jev_scripted_auto(scripts^, guard_19) as started: 686 _raw_send_only( 687 started.port, 688 ( 689 "POST /not-a-route HTTP/1.1\r\nhost: 127.0.0.1\r\n" 690 "x-authorization: Bearer spoof\r\ncontent-length: 2\r\n" 691 "connection: close\r\n\r\n{}" 692 ), 693 ) 694 started.stub.reap() 695 assert_equal(started.stub.phase(), "exchange") 696 assert_equal(started.stub.reason(), "path_mismatch") 697 698 guard_19.assert_clean() 699 700 701 def test_scripted_rejects_duplicate_authorization() raises: 702 var scripts = List[ExchangeScript]() 703 var script = exchange_script( 704 "dup_auth", "POST", "/v1/chat/completions", 200, '{"ok":true}' 705 ) 706 script.require_bearer = True 707 scripts.append(script^) 708 var guard_20 = CleanupGuard() 709 with spawn_max_local_scripted(0, scripts^, guard_20) as stub: 710 _raw_send_only( 711 stub.port, 712 ( 713 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 714 "authorization: Bearer one\r\nauthorization: Bearer two\r\n" 715 "content-length: 2\r\nconnection: close\r\n\r\n{}" 716 ), 717 ) 718 stub.reap() 719 assert_equal(stub.phase(), "exchange") 720 assert_equal(stub.reason(), "auth_duplicate") 721 722 guard_20.assert_clean() 723 724 725 def test_json_escape_control_characters() raises: 726 assert_equal(json_escape("a\nb\tc"), '"a\\nb\\tc"') 727 assert_equal(json_escape('q"uote'), '"q\\"uote"') 728 assert_equal(json_escape("back\\slash"), '"back\\\\slash"') 729 assert_equal(json_escape("ctl\x01"), '"ctl\\u0001"') 730 731 732 def test_verify_exchange_mutation_controls() raises: 733 # Deliberate in-memory mutations: each must fail for its own bounded 734 # reason rather than any generic exception (FX07). 735 var script = exchange_script( 736 "control", "POST", "/v1/chat/completions", 200, "{}" 737 ) 738 script.headers = "x-sel:1" 739 script.check_body = True 740 script.body = '{"b":1}' 741 var ok = FramedRequest( 742 ok=True, 743 error="", 744 method="POST", 745 path="/v1/chat/completions", 746 version="HTTP/1.1", 747 headers_raw="host: h\r\nx-sel:1", 748 body='{"b":1}', 749 content_length=7, 750 keep_alive=False, 751 total_bytes=0, 752 ) 753 assert_equal(verify_exchange(script, ok), "") 754 var wrong_method = FramedRequest( 755 ok=True, 756 error="", 757 method="PUT", 758 path="/v1/chat/completions", 759 version="HTTP/1.1", 760 headers_raw="host: h\r\nx-sel:1", 761 body='{"b":1}', 762 content_length=7, 763 keep_alive=False, 764 total_bytes=0, 765 ) 766 assert_equal(verify_exchange(script, wrong_method), "method_mismatch") 767 var wrong_path = FramedRequest( 768 ok=True, 769 error="", 770 method="POST", 771 path="/v1/other", 772 version="HTTP/1.1", 773 headers_raw="host: h\r\nx-sel:1", 774 body='{"b":1}', 775 content_length=7, 776 keep_alive=False, 777 total_bytes=0, 778 ) 779 assert_equal(verify_exchange(script, wrong_path), "path_mismatch") 780 var missing_header = FramedRequest( 781 ok=True, 782 error="", 783 method="POST", 784 path="/v1/chat/completions", 785 version="HTTP/1.1", 786 headers_raw="host: h", 787 body='{"b":1}', 788 content_length=7, 789 keep_alive=False, 790 total_bytes=0, 791 ) 792 assert_equal( 793 verify_exchange(script, missing_header), "header_missing:x-sel" 794 ) 795 var wrong_body = FramedRequest( 796 ok=True, 797 error="", 798 method="POST", 799 path="/v1/chat/completions", 800 version="HTTP/1.1", 801 headers_raw="host: h\r\nx-sel:1", 802 body='{"b":2}', 803 content_length=7, 804 keep_alive=False, 805 total_bytes=0, 806 ) 807 assert_equal(verify_exchange(script, wrong_body), "body_mismatch") 808 assert_equal( 809 authorization_reason( 810 ( 811 "POST /v1/chat/completions HTTP/1.1\r\nx-authorization: Bearer" 812 " spoof" 813 ), 814 True, 815 ), 816 "auth_missing", 817 ) 818 assert_equal( 819 authorization_reason( 820 "POST /v1/chat/completions HTTP/1.1\r\nauthorization: Bearer t", 821 True, 822 ), 823 "", 824 ) 825 826 827 # ── FX05: persistent reusable/close connection semantics ──────────────────── 828 829 830 def test_scripted_persistent_counters_and_close_semantics() raises: 831 var scripts = List[ExchangeScript]() 832 var first = exchange_script( 833 "reusable", "POST", "/v1/systemone", 200, '{"step":1}' 834 ) 835 first.close_connection = False 836 first.response_headers = "x-step: one" 837 scripts.append(first^) 838 var second = exchange_script( 839 "close", "POST", "/v1/systemone", 200, '{"step":2}' 840 ) 841 second.response_headers = "x-step: two" 842 scripts.append(second^) 843 var guard_21 = CleanupGuard() 844 with spawn_jev_scripted_auto(scripts^, guard_21) as started: 845 var first_frame = ( 846 "POST /v1/systemone HTTP/1.1\r\nhost: 127.0.0.1\r\n" 847 "authorization: Bearer t\r\ncontent-length: 2\r\n" 848 "connection: keep-alive\r\n\r\n{}" 849 ) 850 var second_frame = String(first_frame) 851 var response = _raw_exchange(started.port, first_frame + second_frame) 852 assert_true(response.find("x-step: one") >= 0) 853 assert_true(response.find("x-step: two") >= 0) 854 assert_true(response.find("connection: keep-alive") >= 0) 855 assert_true(response.find("connection: close") >= 0) 856 started.stub.wait() 857 assert_equal(started.stub.request_count(), 2) 858 assert_equal(started.stub.connection_count(), 1) 859 860 # ── FX06/FX07/FX08: parent lifecycle and cause-specific failures ──────────── 861 862 guard_21.assert_clean() 863 864 865 def test_startup_failure_distinct_from_exchange_failure() raises: 866 # Occupy a port so the fixture child cannot bind; the parent must observe 867 # a bounded startup/serve_failed report, not a generic exception or a 868 # script/parser rejection (FX07 / R61). 869 var blocker = TcpListener.bind(SocketAddr.localhost(0)) 870 var port = Int(blocker.local_addr().port) 871 var message = "" 872 try: 873 var guard_22 = CleanupGuard() 874 with spawn_max_local_stub(port, "count_requests", 1, guard_22) as stub: 875 stub.terminate() 876 guard_22.assert_clean() 877 except e: 878 message = String(e) 879 blocker.close() 880 assert_true(message.find("phase=startup") >= 0) 881 assert_true(message.find("reason=serve_failed") >= 0) 882 assert_true(message.find("exited=") >= 0 or message.find("signal=") >= 0) 883 884 885 def test_provider_stub_parent_deadline_watchdog() raises: 886 # No client connects, so the child blocks in accept until the parent's own 887 # finite deadline fires and the owned child is terminated and reaped. 888 var guard_23 = CleanupGuard() 889 with spawn_max_local_stub(0, "count_requests", 1, guard_23, 800) as stub: 890 stub.reap() 891 assert_true(not stub.ok()) 892 assert_equal(stub.phase(), "watchdog") 893 assert_equal(stub.reason(), "timeout") 894 assert_true(pid_not_waitable(stub.pid)) 895 896 guard_23.assert_clean() 897 898 899 def test_jev_stub_parent_deadline_watchdog() raises: 900 var guard_24 = CleanupGuard() 901 with spawn_jev_stub_auto("ok", 1, guard_24, 800) as started: 902 started.stub.reap() 903 assert_true(not started.stub.ok()) 904 assert_equal(started.stub.phase(), "watchdog") 905 assert_equal(started.stub.reason(), "timeout") 906 assert_true(pid_not_waitable(started.stub.pid)) 907 908 guard_24.assert_clean() 909 910 911 def test_bounded_read_caps_fail_for_intended_cause() raises: 912 var ready_pipe = make_pipe() 913 write_raw(ready_pipe.write_fd, "no newline here") 914 var ready_message = "" 915 try: 916 _ = read_line_bounded(ready_pipe.read_fd, 8, 500) 917 except e: 918 ready_message = String(e) 919 close_fd(ready_pipe.read_fd) 920 close_fd(ready_pipe.write_fd) 921 assert_true(ready_message.find("ready_output_overflow") >= 0) 922 923 var stdout_pipe = make_pipe() 924 write_raw(stdout_pipe.write_fd, "0123456789") 925 close_fd(stdout_pipe.write_fd) 926 var stdout_message = "" 927 try: 928 _ = read_all_bounded(stdout_pipe.read_fd, 4, 500) 929 except e: 930 stdout_message = String(e) 931 close_fd(stdout_pipe.read_fd) 932 assert_true(stdout_message.find("stdout_overflow") >= 0) 933 934 935 def test_write_deadline_and_closed_pipe_causes() raises: 936 var closed = make_pipe() 937 close_fd(closed.read_fd) 938 var closed_reason = write_fd_bounded(closed.write_fd, "payload", 500) 939 close_fd(closed.write_fd) 940 assert_true( 941 closed_reason == "write_pipe_closed" or closed_reason == "write_failed" 942 ) 943 944 var full = make_pipe() 945 var payload = String("") 946 for _ in range(1024): 947 payload += "y" 948 for _ in range(7): 949 var doubled = String(payload) 950 payload = payload + doubled 951 var deadline_reason = write_fd_bounded(full.write_fd, payload, 200) 952 close_fd(full.read_fd) 953 close_fd(full.write_fd) 954 assert_equal(deadline_reason, "write_deadline_expired") 955 956 957 def test_real_write_to_closed_descriptor_is_ebadf() raises: 958 # RP03: a real (non-synthetic) descriptor/write failure. An exact-owned pipe 959 # write end is closed and then written to, so the OS returns a real EBADF. 960 # The raw errno is recorded and the actual classification function maps the 961 # rendered write-API cause to invalid_descriptor, which can never be a peer 962 # close. No product or fork source is involved. 963 var pipe = make_pipe() 964 close_fd(pipe.write_fd) 965 var written = write_raw(pipe.write_fd, "x") 966 assert_true(written < 0) 967 var errno_value = Int(get_errno().value) 968 assert_equal(errno_value, Int(ErrNo.EBADF.value)) 969 assert_equal(write_errno_class(errno_value), "invalid_descriptor") 970 var rendered = render_write_api_error(errno_value) 971 assert_true(rendered.find("Bad file descriptor") >= 0) 972 assert_equal(classify_write_error_cause(rendered), "invalid_descriptor") 973 assert_true(not is_peer_close_cause("invalid_descriptor")) 974 close_fd(pipe.read_fd) 975 976 977 def test_owned_child_reaped_after_early_terminate() raises: 978 var guard_25 = CleanupGuard() 979 with spawn_max_local_stub(0, "count_requests", 1, guard_25) as stub: 980 stub.terminate() 981 assert_true(pid_not_waitable(stub.pid)) 982 983 guard_25.assert_clean() 984 985 986 def test_repeated_failures_leave_no_owned_child() raises: 987 for _ in range(3): 988 var scripts = List[ExchangeScript]() 989 scripts.append(_default_script()) 990 var guard_26 = CleanupGuard() 991 with spawn_max_local_scripted(0, scripts^, guard_26) as stub: 992 _raw_send_only( 993 stub.port, 994 ( 995 "POST /v1/nope HTTP/1.1\r\nhost: 127.0.0.1\r\n" 996 "content-length: 2\r\nconnection: close\r\n\r\n{}" 997 ), 998 ) 999 stub.reap() 1000 assert_true(not stub.ok()) 1001 assert_equal(stub.reason(), "path_mismatch") 1002 assert_true(pid_not_waitable(stub.pid)) 1003 1004 guard_26.assert_clean() 1005 1006 1007 def test_repeated_teardown_does_not_leak_descriptors() raises: 1008 var before = open_fd_count() 1009 assert_true(before > 0) 1010 for _ in range(5): 1011 var guard_27 = CleanupGuard() 1012 with spawn_max_local_stub(0, "count_requests", 1, guard_27) as stub: 1013 stub.terminate() 1014 assert_true(pid_not_waitable(stub.pid)) 1015 guard_27.assert_clean() 1016 var after = open_fd_count() 1017 assert_true(after > 0) 1018 assert_true(after <= before) 1019 1020 1021 def test_timeout_terminates_and_reaps_stalled_child() raises: 1022 var guard_28 = CleanupGuard() 1023 with spawn_max_local_stub(0, "stall", 1, guard_28) as stub: 1024 _raw_send_only( 1025 stub.port, 1026 ( 1027 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 1028 "content-length: 2\r\nconnection: close\r\n\r\n{}" 1029 ), 1030 ) 1031 stub.terminate() 1032 assert_true(pid_not_waitable(stub.pid)) 1033 1034 guard_28.assert_clean() 1035 1036 1037 def _owned_report_child( 1038 mut guard: CleanupGuard, exit_code: Int, report: String 1039 ) raises -> SpawnedMaxLocalStub: 1040 """Fork a test-owned child that writes ``report`` to stdout and exits. 1041 1042 Lets LC02 prove real forged/empty/mismatched reports fail for their cause, 1043 independent of the fixture serve loop. The caller holds ``guard`` so the 1044 cleanup outcome stays observable. 1045 """ 1046 var pipe = make_pipe() 1047 var pid = fork_pid() 1048 if pid == 0: 1049 if dup2_fd(pipe.write_fd, 1) < 0: 1050 child_exit(126) 1051 close_fd(pipe.read_fd) 1052 close_fd(pipe.write_fd) 1053 if report != "": 1054 _ = write_raw(1, report) 1055 child_exit(exit_code) 1056 close_fd(pipe.write_fd) 1057 var state = piped_child_state( 1058 pid, pipe.read_fd, 2000, 1, UnsafePointer(to=guard) 1059 ) 1060 return SpawnedMaxLocalStub(pid, 0, state^) 1061 1062 1063 def _owned_jev_report_child( 1064 mut guard: CleanupGuard, exit_code: Int, report: String 1065 ) raises -> SpawnedJevStub: 1066 """Same controlled report child, reaped through the Jev provider path.""" 1067 var pipe = make_pipe() 1068 var pid = fork_pid() 1069 if pid == 0: 1070 if dup2_fd(pipe.write_fd, 1) < 0: 1071 child_exit(126) 1072 close_fd(pipe.read_fd) 1073 close_fd(pipe.write_fd) 1074 if report != "": 1075 _ = write_raw(1, report) 1076 child_exit(exit_code) 1077 close_fd(pipe.write_fd) 1078 var state = piped_child_state( 1079 pid, pipe.read_fd, 2000, 1, UnsafePointer(to=guard) 1080 ) 1081 return SpawnedJevStub(pid, 0, state^) 1082 1083 1084 # ── LC01: automatic scope ownership ───────────────────────────────────────── 1085 1086 1087 def test_scope_cleanup_on_assertion_failure() raises: 1088 var held_pid = 0 1089 var caught = False 1090 try: 1091 var guard_29 = CleanupGuard() 1092 with spawn_max_local_stub(0, "count_requests", 1, guard_29) as stub: 1093 held_pid = stub.pid 1094 assert_true(False) 1095 guard_29.assert_clean() 1096 except: 1097 caught = True 1098 assert_true(caught) 1099 assert_true(held_pid > 0) 1100 assert_true(pid_not_waitable(held_pid)) 1101 1102 1103 def test_scope_cleanup_on_generic_error() raises: 1104 var held_pid = 0 1105 var message = "" 1106 try: 1107 var guard_30 = CleanupGuard() 1108 with spawn_max_local_stub(0, "count_requests", 1, guard_30) as stub: 1109 held_pid = stub.pid 1110 raise Error("intentional scope error") 1111 guard_30.assert_clean() 1112 except e: 1113 message = String(e) 1114 assert_equal(message, "intentional scope error") 1115 assert_true(pid_not_waitable(held_pid)) 1116 1117 1118 def _early_return_owner(mut guard: CleanupGuard) raises -> Int: 1119 # The guard is caller-held so the cleanup outcome stays observable even 1120 # though this scope exits through a ``return`` before any post-scope line. 1121 with spawn_max_local_stub(0, "count_requests", 1, guard) as stub: 1122 return stub.pid 1123 return 0 1124 1125 1126 def test_scope_cleanup_on_early_return() raises: 1127 var guard = CleanupGuard() 1128 var held_pid = _early_return_owner(guard) 1129 assert_true(held_pid > 0) 1130 assert_true(pid_not_waitable(held_pid)) 1131 guard.assert_clean() 1132 1133 1134 def test_jev_scope_cleanup_on_assertion_failure() raises: 1135 var held_pid = 0 1136 var caught = False 1137 try: 1138 var guard_32 = CleanupGuard() 1139 with spawn_jev_stub_auto("ok", 1, guard_32) as started: 1140 held_pid = started.stub.pid 1141 assert_true(False) 1142 guard_32.assert_clean() 1143 except: 1144 caught = True 1145 assert_true(caught) 1146 assert_true(held_pid > 0) 1147 assert_true(pid_not_waitable(held_pid)) 1148 1149 1150 def test_jev_scope_cleanup_on_generic_error() raises: 1151 var held_pid = 0 1152 var message = "" 1153 try: 1154 var guard_33 = CleanupGuard() 1155 with spawn_jev_stub_auto("ok", 1, guard_33) as started: 1156 held_pid = started.stub.pid 1157 raise Error("intentional jev scope error") 1158 guard_33.assert_clean() 1159 except e: 1160 message = String(e) 1161 assert_equal(message, "intentional jev scope error") 1162 assert_true(pid_not_waitable(held_pid)) 1163 1164 1165 def _jev_early_return_owner(mut guard: CleanupGuard) raises -> Int: 1166 # The guard is caller-held so the cleanup outcome stays observable even 1167 # though this scope exits through a ``return`` before any post-scope line. 1168 with spawn_jev_stub_auto("ok", 1, guard) as started: 1169 return started.stub.pid 1170 return 0 1171 1172 1173 def test_jev_scope_cleanup_on_early_return() raises: 1174 var guard = CleanupGuard() 1175 var held_pid = _jev_early_return_owner(guard) 1176 assert_true(held_pid > 0) 1177 assert_true(pid_not_waitable(held_pid)) 1178 guard.assert_clean() 1179 1180 1181 def test_jev_early_return_cleanup_failure_is_observable() raises: 1182 # RA01: an early return that leaves cleanup unproved must still fail the 1183 # owning test through the caller-held guard, for the Jev provider path. 1184 var guard = CleanupGuard() 1185 var held_pid = 0 1186 with spawn_jev_stub_auto("ok", 1, guard) as started: 1187 held_pid = started.stub.pid 1188 started.stub.inject_cleanup_failure() 1189 var failure = "" 1190 try: 1191 guard.assert_clean() 1192 except e: 1193 failure = String(e) 1194 assert_true(failure.find("cleanup-unproved") >= 0) 1195 assert_true(held_pid > 0) 1196 # The retained exact-owned child is still recoverable, not a message only. 1197 assert_true(guard.retained() >= 1) 1198 assert_equal(guard.recover_all(), 0) 1199 guard.assert_clean() 1200 assert_true(pid_not_waitable(held_pid)) 1201 1202 1203 def test_jev_startup_failure_is_truthful_and_cause_specific() raises: 1204 # PC02/LC01: the Jev startup-readiness failure path executes against a real 1205 # owned child, reports its cause and exposes the finalizer's cleanup truth. 1206 # The caller-held guard is the enforcement point: a startup finalization 1207 # that could not prove cleanup fails this test instead of being discarded. 1208 var blocker = TcpListener.bind(SocketAddr.localhost(0)) 1209 var port = Int(blocker.local_addr().port) 1210 var message = "" 1211 var guard = CleanupGuard() 1212 try: 1213 var started = spawn_jev_stub(port, "ok", 1, guard) 1214 started.cleanup() 1215 except e: 1216 message = String(e) 1217 blocker.close() 1218 guard.assert_clean() 1219 assert_true(message.find("phase=startup") >= 0) 1220 assert_true(message.find("reason=serve_failed") >= 0) 1221 assert_true(message.find("cleanup=") >= 0) 1222 assert_true(message.find("pid=") >= 0) 1223 1224 1225 def test_wait_error_taxonomy_distinguishes_causes() raises: 1226 assert_equal(classify_wait_errno(Int(ErrNo.EINTR.value)), "interrupted") 1227 assert_equal(classify_wait_errno(Int(ErrNo.ECHILD.value)), "gone") 1228 assert_equal(classify_wait_errno(9999), "wait_error") 1229 assert_equal(wait_nohang(0).state, "wait_error") 1230 var live_pid = 0 1231 var guard_35 = CleanupGuard() 1232 with spawn_max_local_stub(0, "count_requests", 1, guard_35) as stub: 1233 live_pid = stub.pid 1234 assert_equal(wait_nohang(live_pid).state, "running") 1235 stub.terminate() 1236 assert_equal(wait_nohang(live_pid).state, "gone") 1237 guard_35.assert_clean() 1238 assert_true(pid_not_waitable(live_pid)) 1239 1240 1241 def test_repeated_reap_and_terminate_are_owned_and_idempotent() raises: 1242 var guard_36 = CleanupGuard() 1243 with spawn_max_local_stub(0, "count_requests", 1, guard_36) as stub: 1244 var owned_pid = stub.pid 1245 stub.terminate() 1246 stub.terminate() 1247 stub.reap() 1248 assert_true(pid_not_waitable(owned_pid)) 1249 var cached = stub.status() 1250 assert_true(cached.cleanup_proved()) 1251 assert_true(not stub.ok()) 1252 1253 # ── LC02: strict result truth ─────────────────────────────────────────────── 1254 1255 guard_36.assert_clean() 1256 1257 1258 def test_result_truth_rejects_empty_exit_zero_report() raises: 1259 var guard_101 = CleanupGuard() 1260 var stub = _owned_report_child(guard_101, 0, "") 1261 stub.reap() 1262 assert_true(not stub.ok()) 1263 assert_equal(stub.reason(), "missing_report") 1264 1265 guard_101.assert_clean() 1266 1267 1268 def test_result_truth_rejects_forged_success_with_nonzero_exit() raises: 1269 var guard_102 = CleanupGuard() 1270 var stub = _owned_report_child( 1271 guard_102, 1272 7, 1273 "result ok phase=complete case=- reason=ok requests=1 connections=1\n", 1274 ) 1275 stub.reap() 1276 assert_true(not stub.ok()) 1277 assert_true(stub.reason().startswith("report_status_mismatch")) 1278 1279 guard_102.assert_clean() 1280 1281 1282 def test_result_truth_accepts_matching_report_and_exit() raises: 1283 var guard_103 = CleanupGuard() 1284 var stub = _owned_report_child( 1285 guard_103, 1286 0, 1287 "result ok phase=complete case=- reason=ok requests=1 connections=1\n", 1288 ) 1289 stub.reap() 1290 assert_true(stub.ok()) 1291 assert_equal(stub.phase(), "complete") 1292 assert_equal(stub.request_count(), 1) 1293 assert_equal(stub.connection_count(), 1) 1294 1295 guard_103.assert_clean() 1296 1297 1298 def test_parse_report_rejects_malformed_inputs() raises: 1299 assert_equal(parse_report("").phase, "parse") 1300 assert_equal( 1301 parse_report( 1302 "result maybe phase=x case=- reason=y requests=1 connections=1" 1303 ).reason, 1304 "unknown_status", 1305 ) 1306 assert_equal( 1307 parse_report( 1308 "result ok phase=complete case=- reason=ok requests=1 connections=1" 1309 " requests=1" 1310 ).reason, 1311 "duplicate_field", 1312 ) 1313 assert_equal( 1314 parse_report( 1315 "result ok phase=complete case=- reason=ok requests=x connections=1" 1316 ).reason, 1317 "invalid_count", 1318 ) 1319 assert_equal( 1320 parse_report( 1321 "result ok phase=complete case=- reason=ok requests=1" 1322 ).reason, 1323 "missing_field", 1324 ) 1325 assert_equal( 1326 parse_report( 1327 "result ok phase=complete case=- reason=ok requests=1 connections=1" 1328 " extra=z" 1329 ).reason, 1330 "unknown_field", 1331 ) 1332 assert_equal( 1333 parse_report( 1334 "result ok phase=complete case=- reason=ok requests=1 connections=1" 1335 ).phase, 1336 "complete", 1337 ) 1338 1339 1340 def test_report_status_matches_exit() raises: 1341 assert_true(report_status_matches_exit(True, 0, True)) 1342 assert_true(report_status_matches_exit(True, 125, False)) 1343 assert_true(not report_status_matches_exit(True, 7, True)) 1344 assert_true(not report_status_matches_exit(True, 0, False)) 1345 assert_true(not report_status_matches_exit(False, 0, True)) 1346 1347 1348 # ── LC03: byte caps and surplus retention ─────────────────────────────────── 1349 1350 1351 def _pipe_line(text: String) raises -> String: 1352 var pipe = make_pipe() 1353 _ = write_raw(pipe.write_fd, text) 1354 var result = "" 1355 var raised = "" 1356 try: 1357 result = read_line_bounded(pipe.read_fd, 8, 500) 1358 except e: 1359 raised = String(e) 1360 close_fd(pipe.read_fd) 1361 close_fd(pipe.write_fd) 1362 if raised != "": 1363 return "raised:" + raised 1364 return result^ 1365 1366 1367 def test_line_cap_boundaries() raises: 1368 assert_equal(_pipe_line("1234567\n"), "1234567") 1369 assert_equal(_pipe_line("12345678\n"), "12345678") 1370 assert_equal(_pipe_line("123456789\n"), "raised:ready_output_overflow") 1371 1372 1373 def test_coalesced_ready_and_report_lines_retain_surplus() raises: 1374 # One write carries both lines; the report line must survive the ready read 1375 # rather than being discarded with the chunk. 1376 var pipe = make_pipe() 1377 _ = write_raw( 1378 pipe.write_fd, 1379 ( 1380 "ready 4242\nresult ok phase=complete case=- reason=ok requests=1" 1381 " connections=1\n" 1382 ), 1383 ) 1384 var guard = CleanupGuard() 1385 var state = piped_child_state( 1386 pid=0, 1387 report_fd=pipe.read_fd, 1388 deadline_ms=500, 1389 expected_requests=1, 1390 guard=UnsafePointer(to=guard), 1391 ) 1392 var ready = state.read_line(2048, 500) 1393 var report = state.read_line(2048, 500) 1394 state.close_reader() 1395 close_fd(pipe.write_fd) 1396 assert_equal(ready, "ready 4242") 1397 assert_true(report.startswith("result ok")) 1398 guard.assert_clean() 1399 1400 1401 # ── LC05: framing and descriptor census ───────────────────────────────────── 1402 1403 1404 def test_verify_exchange_rejects_malformed_script_header() raises: 1405 var script = exchange_script( 1406 "bad_decl", "POST", "/v1/chat/completions", 200, "{}" 1407 ) 1408 script.headers = "not-a-header" 1409 var framed = FramedRequest( 1410 ok=True, 1411 error="", 1412 method="POST", 1413 path="/v1/chat/completions", 1414 version="HTTP/1.1", 1415 headers_raw="host: h", 1416 body="", 1417 content_length=0, 1418 keep_alive=False, 1419 total_bytes=0, 1420 ) 1421 assert_equal(verify_exchange(script, framed), "malformed_script_header") 1422 1423 1424 def test_header_value_rejects_control_bytes_and_trims_ows() raises: 1425 # Raw control byte inside a value is rejected as malformed framing. 1426 var bad = _framing_failure( 1427 "POST /v1/chat/completions HTTP/1.1\r\nhost: 127.0.0.1\r\n" 1428 "x-ctl: a\x01b\r\ncontent-length: 2\r\nconnection: close\r\n\r\n{}" 1429 ) 1430 assert_equal(bad.phase(), "read") 1431 assert_equal(bad.reason(), "malformed_header") 1432 # Legal surrounding OWS on a selected header value is accepted. 1433 var scripts = List[ExchangeScript]() 1434 var script = exchange_script( 1435 "ows", "POST", "/v1/chat/completions", 200, "{}" 1436 ) 1437 script.headers = "x-ows:value" 1438 scripts.append(script^) 1439 var guard_37 = CleanupGuard() 1440 with spawn_max_local_scripted(0, scripts^, guard_37) as stub: 1441 var response = _request( 1442 stub.port, 1443 "POST", 1444 "/v1/chat/completions", 1445 "{}", 1446 "x-ows: value \r\n", 1447 ) 1448 assert_true(response.find("200") >= 0) 1449 stub.wait() 1450 1451 guard_37.assert_clean() 1452 1453 1454 def test_descriptor_census_detects_planted_high_fd() raises: 1455 var before = open_fd_count() 1456 assert_true(before > 0) 1457 assert_equal(descriptor_census(0), -1) 1458 assert_true(descriptor_census(3) > 0) 1459 var pipe = make_pipe() 1460 var planted = Int(dup2_fd(pipe.read_fd, 900)) 1461 var with_pipe = open_fd_count() 1462 assert_true(planted >= 0) 1463 assert_true(with_pipe > before) 1464 close_fd(planted) 1465 close_fd(pipe.read_fd) 1466 close_fd(pipe.write_fd) 1467 assert_equal(open_fd_count(), before) 1468 1469 1470 def test_partial_pipe_failure_rolls_back() raises: 1471 # LC01: a later pipe failure must close the pipes already created. 1472 var before = open_fd_count_checked() 1473 var message = "" 1474 try: 1475 _ = make_three_pipes(1) 1476 except e: 1477 message = String(e) 1478 assert_true(message.find("injected pipe creation failure") >= 0) 1479 assert_equal(open_fd_count_checked(), before) 1480 1481 1482 def test_fork_failure_closes_owned_pipes() raises: 1483 # LC01: a fork failure must close both ends of the owned pipe. 1484 var before = open_fd_count_checked() 1485 var pipe = make_pipe() 1486 var message = "" 1487 try: 1488 _ = fork_owned_or_close(pipe.copy(), True) 1489 except e: 1490 message = String(e) 1491 assert_true(message.find("injected fork failure") >= 0) 1492 assert_equal(open_fd_count_checked(), before) 1493 1494 1495 def test_stdio_fork_failure_closes_all_owned_pipes() raises: 1496 # LC01: the stdio helper's fork failure must close all three pipe pairs. 1497 var before = open_fd_count_checked() 1498 var pipes = make_three_pipes() 1499 var message = "" 1500 try: 1501 _ = fork_owned_or_close3(pipes.copy(), True) 1502 except e: 1503 message = String(e) 1504 assert_true(message.find("injected fork failure") >= 0) 1505 assert_equal(open_fd_count_checked(), before) 1506 1507 1508 def test_result_truth_rejects_duplicate_report_line() raises: 1509 var guard_104 = CleanupGuard() 1510 var stub = _owned_report_child( 1511 guard_104, 1512 0, 1513 ( 1514 "result ok phase=complete case=- reason=ok requests=1" 1515 " connections=1\nresult ok phase=complete case=- reason=ok" 1516 " requests=1 connections=1\n" 1517 ), 1518 ) 1519 stub.reap() 1520 assert_true(not stub.ok()) 1521 assert_equal(stub.reason(), "duplicate_report") 1522 1523 guard_104.assert_clean() 1524 1525 1526 def test_cleanup_failure_is_observable() raises: 1527 # RA01/RA02: a cleanup failure must be observable through the required 1528 # caller-held guard, must not claim the child was collected, and must retain 1529 # retryable ownership rather than only a message. 1530 var guard = CleanupGuard() 1531 var state = piped_child_state(0, -1, 100, 1, UnsafePointer(to=guard)) 1532 var stub = SpawnedMaxLocalStub(0, 0, state^) 1533 stub.cleanup() 1534 assert_true(stub.cleanup_error().find("unreaped") >= 0) 1535 assert_true(not stub.status().cleanup_proved()) 1536 assert_equal(guard.count(), 1) 1537 assert_true(guard.first().find("unproved") >= 0) 1538 assert_equal(guard.retained(), 1) 1539 # A second cleanup still retries the same owned identity rather than 1540 # short-circuiting on a false "reaped" flag. 1541 stub.cleanup() 1542 assert_equal(guard.count(), 2) 1543 var failure = "" 1544 try: 1545 guard.assert_clean() 1546 except e: 1547 failure = String(e) 1548 assert_true(failure.find("cleanup-unproved") >= 0) 1549 1550 1551 def test_result_truth_rejects_wrong_request_count() raises: 1552 var guard_105 = CleanupGuard() 1553 var stub = _owned_report_child( 1554 guard_105, 1555 0, 1556 "result ok phase=complete case=- reason=ok requests=2 connections=1\n", 1557 ) 1558 stub.reap() 1559 assert_true(not stub.ok()) 1560 assert_equal(stub.reason(), "request_count_mismatch") 1561 1562 guard_105.assert_clean() 1563 1564 1565 def test_result_truth_rejects_invalid_connection_count() raises: 1566 var guard_106 = CleanupGuard() 1567 var stub = _owned_report_child( 1568 guard_106, 1569 0, 1570 "result ok phase=complete case=- reason=ok requests=1 connections=5\n", 1571 ) 1572 stub.reap() 1573 assert_true(not stub.ok()) 1574 assert_equal(stub.reason(), "connection_count_invalid") 1575 1576 guard_106.assert_clean() 1577 1578 1579 def test_completion_probe_read_error_is_distinct_from_setup_and_timeout() raises: 1580 # LC05/PC04: a real socket read error after a successful timeout setup must 1581 # propagate as the exact read cause, not be read as success, a setup 1582 # failure or a timeout. 1583 # 1584 # 1. Successful setup on a real (unconnected) socket, then a real read 1585 # error (ENOTCONN): the probe must raise and never return success. 1586 var sock = RawSocket(c_int(2), c_int(1)) 1587 var stream = TcpStream(sock^, SocketAddr.localhost(UInt16(1))) 1588 var setup_ok = False 1589 try: 1590 stream.set_recv_timeout(20) 1591 setup_ok = True 1592 except e: 1593 _ = String(e) 1594 assert_true(setup_ok) 1595 var reader = ConnectionReader(stream^) 1596 var read_message = "" 1597 var read_result = "" 1598 try: 1599 read_result = reader.probe_completion(20) 1600 except e: 1601 read_message = String(e) 1602 assert_true(read_result == "") 1603 assert_true(read_message.find("recv") >= 0) 1604 assert_true(read_message.find("timeout") < 0) 1605 1606 # 2. A non-socket descriptor fails at setup, a distinct cause. 1607 var pipe = make_pipe() 1608 var pipe_sock = RawSocket(c_int(pipe.read_fd), c_int(2), c_int(1), True) 1609 var pipe_stream = TcpStream(pipe_sock^, SocketAddr.localhost(UInt16(1))) 1610 var pipe_reader = ConnectionReader(pipe_stream^) 1611 var setup_failure = "" 1612 try: 1613 _ = pipe_reader.probe_completion(20) 1614 except e: 1615 setup_failure = String(e) 1616 close_fd(pipe.write_fd) 1617 assert_true(setup_failure != "") 1618 assert_true(setup_failure.find("setsockopt") >= 0) 1619 1620 # 3. A quiet connected socket times out, which is the probe's success path 1621 # and stays distinct from the read error above. 1622 var listener = TcpListener.bind(SocketAddr.localhost(0)) 1623 var port = Int(listener.local_addr().port) 1624 var client = TcpStream.connect(SocketAddr.localhost(UInt16(port))) 1625 var server = listener.accept() 1626 var server_reader = ConnectionReader(server^) 1627 var quiet = server_reader.probe_completion(20) 1628 client.close() 1629 assert_equal(quiet, "") 1630 1631 1632 def test_descriptor_census_detects_planted_socket() raises: 1633 # The census must count sockets, not only regular files. 1634 var before = open_fd_count() 1635 var sock = Int(external_call["socket", c_int](c_int(2), c_int(1), c_int(0))) 1636 assert_true(sock >= 0) 1637 var with_socket = open_fd_count() 1638 assert_true(with_socket > before) 1639 close_fd(sock) 1640 assert_true(open_fd_count() <= with_socket) 1641 1642 1643 def test_ready_grammar_rejections() raises: 1644 var valid = 0 1645 var raised = "" 1646 try: 1647 valid = parse_ready_line("ready 65535", 256) 1648 except e: 1649 raised = String(e) 1650 assert_equal(valid, 65535) 1651 assert_equal(raised, "") 1652 var cases = List[String]() 1653 cases.append("") 1654 cases.append("ready") 1655 cases.append("ready ") 1656 cases.append("ready abc") 1657 cases.append("ready 12345x") 1658 cases.append("ready 70000") 1659 cases.append("ready 0") 1660 cases.append("not-ready") 1661 for probe in cases: 1662 var reason = "" 1663 try: 1664 _ = parse_ready_line(probe, 256) 1665 except e: 1666 reason = String(e) 1667 assert_true(reason != "") 1668 1669 1670 def test_malformed_ready_line_terminates_owned_child() raises: 1671 # LC03: malformed readiness must clean up the exact owned child. 1672 var pipe = make_pipe() 1673 var pid = fork_pid() 1674 if pid == 0: 1675 close_fd(pipe.read_fd) 1676 _ = write_raw(pipe.write_fd, "garbage-not-ready\n") 1677 for _ in range(400): 1678 sleep_ms(50) 1679 child_exit(0) 1680 close_fd(pipe.write_fd) 1681 var line = read_line_bounded(pipe.read_fd, 256, 500) 1682 close_fd(pipe.read_fd) 1683 var message = "" 1684 try: 1685 _ = parse_ready_or_cleanup(pid, line, 256) 1686 except e: 1687 message = String(e) 1688 assert_true(message.find("ready_invalid:ready_grammar") >= 0) 1689 assert_true(message.find("cleanup=proved") >= 0) 1690 assert_true(pid_not_waitable(pid)) 1691 1692 1693 def test_status_observation_preserves_ownership_and_report() raises: 1694 # LC01/LC02: observing an exited child must not consume the report. 1695 var guard_107 = CleanupGuard() 1696 var stub = _owned_report_child( 1697 guard_107, 1698 0, 1699 "result ok phase=complete case=- reason=ok requests=1 connections=1\n", 1700 ) 1701 var observed = "" 1702 for _ in range(200): 1703 observed = stub.status().state 1704 if observed != "running" and observed != "interrupted": 1705 break 1706 sleep_ms(20) 1707 assert_equal(observed, "reaped") 1708 var second = stub.status() 1709 assert_equal(second.state, "reaped") 1710 stub.reap() 1711 assert_true(stub.ok()) 1712 assert_equal(stub.request_count(), 1) 1713 assert_true(stub.status().cleanup_proved()) 1714 1715 guard_107.assert_clean() 1716 1717 1718 def test_status_observation_then_terminate_is_safe() raises: 1719 var guard_108 = CleanupGuard() 1720 var stub = _owned_report_child(guard_108, 0, "") 1721 sleep_ms(100) 1722 _ = stub.status() 1723 var owned_pid = stub.pid 1724 stub.terminate() 1725 stub.terminate() 1726 stub.reap() 1727 assert_true(pid_not_waitable(owned_pid)) 1728 assert_true(not stub.ok()) 1729 1730 # ── LC05: coalesced header cap accounting ─────────────────────────────────── 1731 1732 guard_108.assert_clean() 1733 1734 1735 def test_coalesced_large_body_does_not_charge_header_cap() raises: 1736 var scripts = List[ExchangeScript]() 1737 scripts.append(_default_script()) 1738 var guard_38 = CleanupGuard() 1739 with spawn_max_local_scripted(0, scripts^, guard_38) as stub: 1740 var header_filler = String("") 1741 for _ in range(20000): 1742 header_filler += "a" 1743 var body_filler = String("") 1744 for _ in range(50000): 1745 body_filler += "b" 1746 var raw = ( 1747 "POST /v1/chat/completions HTTP/1.1\r\nhost:" 1748 " 127.0.0.1\r\nx-filler: " 1749 + header_filler 1750 + "\r\ncontent-length: " 1751 + String(body_filler.byte_length()) 1752 + "\r\nconnection: close\r\n\r\n" 1753 + body_filler 1754 ) 1755 var response = _raw_exchange(stub.port, raw) 1756 assert_true(response.find("200") >= 0) 1757 stub.wait() 1758 1759 # ── PC01/PC02/PC04: complete report truth and retained ownership ───────────── 1760 1761 guard_38.assert_clean() 1762 1763 1764 comptime VALID_REPORT = ( 1765 "result ok phase=complete case=- reason=ok requests=1 connections=1\n" 1766 ) 1767 1768 1769 def _report_controls( 1770 report: String, exit_code: Int, expected_reason: String 1771 ) raises: 1772 """Every report-framing control must fail for its cause on BOTH providers. 1773 """ 1774 var guard_109 = CleanupGuard() 1775 var stub = _owned_report_child(guard_109, exit_code, report) 1776 var owned = stub.pid 1777 stub.reap() 1778 assert_true(not stub.ok()) 1779 assert_equal(stub.reason(), expected_reason) 1780 assert_true(pid_not_waitable(owned)) 1781 var guard_110 = CleanupGuard() 1782 var jev_stub = _owned_jev_report_child(guard_110, exit_code, report) 1783 var jev_owned = jev_stub.pid 1784 jev_stub.reap() 1785 assert_true(not jev_stub.ok()) 1786 assert_equal(jev_stub.reason(), expected_reason) 1787 assert_true(pid_not_waitable(jev_owned)) 1788 1789 guard_109.assert_clean() 1790 guard_110.assert_clean() 1791 1792 1793 def test_report_stream_controls_both_providers() raises: 1794 # PC01: the complete bounded report stream is validated through EOF; a 1795 # positive control and the aligned/split/coalesced duplicate, no-LF, 1796 # malformed and inconsistent-field controls all execute on both providers. 1797 var guard_111 = CleanupGuard() 1798 var stub = _owned_report_child(guard_111, 0, VALID_REPORT) 1799 stub.reap() 1800 assert_true(stub.ok()) 1801 assert_equal(stub.phase(), "complete") 1802 assert_equal(stub.request_count(), 1) 1803 var guard_112 = CleanupGuard() 1804 var jev_stub = _owned_jev_report_child(guard_112, 0, VALID_REPORT) 1805 jev_stub.reap() 1806 assert_true(jev_stub.ok()) 1807 assert_equal(jev_stub.request_count(), 1) 1808 1809 var first = "result ok phase=complete case=" 1810 var tail = " reason=ok requests=1 connections=1\n" 1811 while first.byte_length() + tail.byte_length() < 512: 1812 first += "x" 1813 first += tail 1814 assert_equal(first.byte_length(), 512) 1815 _report_controls(first + VALID_REPORT, 0, "duplicate_report") 1816 _report_controls(VALID_REPORT + VALID_REPORT, 0, "duplicate_report") 1817 var split_first = "result ok phase=complete case=" 1818 while split_first.byte_length() + tail.byte_length() < 700: 1819 split_first += "y" 1820 split_first += tail 1821 assert_equal(split_first.byte_length(), 700) 1822 _report_controls(split_first + VALID_REPORT, 0, "duplicate_report") 1823 var guard_113 = CleanupGuard() 1824 var split_positive = _owned_report_child(guard_113, 0, split_first) 1825 split_positive.reap() 1826 assert_true(split_positive.ok()) 1827 var guard_114 = CleanupGuard() 1828 var jev_split = _owned_jev_report_child(guard_114, 0, split_first) 1829 jev_split.reap() 1830 assert_true(jev_split.ok()) 1831 var unterminated = String( 1832 VALID_REPORT[byte = 0 : VALID_REPORT.byte_length() - 1] 1833 ) 1834 _report_controls(unterminated, 0, "unterminated_report") 1835 _report_controls( 1836 ( 1837 "result ok phase=read case=- reason=io_error requests=1" 1838 " connections=1\n" 1839 ), 1840 0, 1841 "inconsistent_status", 1842 ) 1843 _report_controls( 1844 "result ok phase= case=- reason=ok requests=1 connections=1\n", 1845 0, 1846 "empty_field", 1847 ) 1848 _report_controls( 1849 "result ok phase=complete case=- reason=ok requests=1\n", 1850 0, 1851 "missing_field", 1852 ) 1853 _report_controls( 1854 ( 1855 "result maybe phase=complete case=- reason=ok requests=1" 1856 " connections=1\n" 1857 ), 1858 0, 1859 "unknown_status", 1860 ) 1861 _report_controls( 1862 "result ok phase=complete case=- reason=ok requests=x connections=1\n", 1863 0, 1864 "invalid_count", 1865 ) 1866 _report_controls( 1867 ( 1868 "result ok phase=complete case=- reason=ok requests=1 connections=1" 1869 " extra=z\n" 1870 ), 1871 0, 1872 "unknown_field", 1873 ) 1874 _report_controls( 1875 ( 1876 "result ok phase=complete case=- reason=ok requests=1 requests=1" 1877 " connections=1\n" 1878 ), 1879 0, 1880 "duplicate_field", 1881 ) 1882 var oversize = "result ok phase=complete case=" 1883 while ( 1884 oversize.byte_length() + tail.byte_length() 1885 <= STRICT_MAX_REPORT_BYTES + 1 1886 ): 1887 oversize += "z" 1888 oversize += tail 1889 _report_controls(oversize, 0, "ready_output_overflow") 1890 1891 guard_111.assert_clean() 1892 guard_112.assert_clean() 1893 guard_113.assert_clean() 1894 guard_114.assert_clean() 1895 1896 1897 def test_startup_failure_cleanup_ownership_is_truthful() raises: 1898 # PC02/RA02: the shared startup-failure finalizer used by both provider 1899 # spawners must not claim an unproved termination as reaped; it records the 1900 # exact pid/reap status in the required guard and retains a usable 1901 # ownership handle for recovery. 1902 var guard = CleanupGuard() 1903 var state = piped_child_state(0, -1, 100, 1, UnsafePointer(to=guard)) 1904 var status = finalize_owned_failure(state, 0, "startup cleanup unproved") 1905 assert_true(not status.cleanup_proved()) 1906 assert_true(not state.reaped) 1907 assert_true(state.cleanup_error.startswith("unreaped")) 1908 assert_equal(guard.count(), 1) 1909 assert_equal(guard.retained(), 1) 1910 assert_true(guard.first().find("startup cleanup unproved") >= 0) 1911 assert_true(guard.first().find("pid=0") >= 0) 1912 1913 var stub = _owned_report_child(guard, 0, VALID_REPORT) 1914 var owned = stub.pid 1915 var proved = finalize_owned_failure(stub.state, owned, "startup cleanup") 1916 assert_true(proved.cleanup_proved()) 1917 assert_true(stub.state.reaped) 1918 assert_true(pid_not_waitable(owned)) 1919 assert_equal(guard.count(), 1) 1920 # The real child was collected, so only the synthetic entry above remains. 1921 assert_true(not guard.is_clean()) 1922 # The required check reports that unresolved entry truthfully rather than 1923 # silently discarding it. 1924 var synthetic = "" 1925 try: 1926 guard.assert_clean() 1927 except e: 1928 synthetic = String(e) 1929 assert_true(synthetic.find("cleanup-unproved") >= 0) 1930 1931 1932 def test_descriptor_read_error_is_distinct_from_eof() raises: 1933 # PC01/PC03: an unavailable descriptor is a bounded read error, never an 1934 # EOF/empty success, for both the shared reader and the owned-child state. 1935 var pipe = make_pipe() 1936 var closed_fd = pipe.read_fd 1937 close_fd(pipe.read_fd) 1938 close_fd(pipe.write_fd) 1939 var line_message = "" 1940 try: 1941 _ = read_line_bounded(closed_fd, 64, 200) 1942 except e: 1943 line_message = String(e) 1944 assert_equal(line_message, "read_error") 1945 var guard = CleanupGuard() 1946 var state = piped_child_state( 1947 closed_fd, closed_fd, 200, 1, UnsafePointer(to=guard) 1948 ) 1949 var state_message = "" 1950 try: 1951 _ = state.read_line(64, 200) 1952 except e: 1953 state_message = String(e) 1954 assert_equal(state_message, "read_error") 1955 assert_true(not state.last_terminated) 1956 guard.assert_clean() 1957 1958 1959 def test_multibyte_surplus_is_not_decoded_prematurely() raises: 1960 # PC01/LC03: a chunk that splits a multi-byte character after a newline must 1961 # be retained as bytes instead of raising a premature UTF-8 decode error. 1962 var pipe = make_pipe() 1963 _ = write_raw(pipe.write_fd, "ready 4242\n") 1964 var lead = List[UInt8]() 1965 lead.append(UInt8(0xC3)) 1966 _ = write_raw_bytes(pipe.write_fd, lead) 1967 var guard = CleanupGuard() 1968 var state = piped_child_state( 1969 pid=0, 1970 report_fd=pipe.read_fd, 1971 deadline_ms=500, 1972 expected_requests=1, 1973 guard=UnsafePointer(to=guard), 1974 ) 1975 var ready = state.read_line(64, 500) 1976 assert_equal(ready, "ready 4242") 1977 assert_true(state.last_terminated) 1978 var trail = List[UInt8]() 1979 trail.append(UInt8(0xA9)) 1980 trail.append(UInt8(10)) 1981 _ = write_raw_bytes(pipe.write_fd, trail) 1982 var letter = state.read_line(64, 500) 1983 state.close_reader() 1984 close_fd(pipe.write_fd) 1985 assert_equal(letter, "\u00e9") 1986 assert_true(state.last_terminated) 1987 guard.assert_clean() 1988 1989 1990 def test_cleanup_failure_preserves_retryable_ownership() raises: 1991 # RA02: a controlled cleanup failure on a real owned child (the fault seam 1992 # reports an unproved cleanup while the real forked child keeps running) 1993 # must not mark it reaped or discard ownership; the retry collects the exact 1994 # same identity and the earlier entry is resolved, not left as a message. 1995 var guard = CleanupGuard() 1996 var fd_before = open_fd_count_checked() 1997 var stub = spawn_max_local_stub(0, "count_requests", 1, guard, 2000) 1998 var actual = stub.pid 1999 stub.state.faults.cleanup_failures = 1 2000 stub.cleanup() 2001 assert_true(stub.cleanup_error().find("unreaped") >= 0) 2002 assert_true(not stub.status().cleanup_proved()) 2003 assert_equal(guard.count(), 1) 2004 assert_equal(guard.retained(), 1) 2005 assert_true(pid_running(actual)) 2006 stub.cleanup() 2007 assert_true(stub.status().cleanup_proved()) 2008 assert_true(pid_not_waitable(actual)) 2009 guard.assert_clean() 2010 assert_true(open_fd_count_checked() <= fd_before) 2011 2012 var jev_guard = CleanupGuard() 2013 var jev_fd_before = open_fd_count_checked() 2014 var jev_stub = spawn_jev_stub_auto("ok", 1, jev_guard, 2000) 2015 var jev_actual = jev_stub.stub.pid 2016 jev_stub.stub.state.faults.cleanup_failures = 1 2017 jev_stub.stub.cleanup() 2018 assert_true(jev_stub.stub.cleanup_error().find("unreaped") >= 0) 2019 assert_equal(jev_guard.retained(), 1) 2020 assert_true(pid_running(jev_actual)) 2021 jev_stub.stub.cleanup() 2022 assert_true(jev_stub.stub.status().cleanup_proved()) 2023 assert_true(pid_not_waitable(jev_actual)) 2024 jev_guard.assert_clean() 2025 assert_true(open_fd_count_checked() <= jev_fd_before) 2026 2027 2028 def test_cleanup_recovery_through_guard_both_providers() raises: 2029 # RA02: a startup/scope cleanup failure must retain a *usable* ownership 2030 # handle on the required guard, and recovery must collect the exact real 2031 # child rather than only reporting a string. 2032 var guard = CleanupGuard() 2033 var fd_before = open_fd_count_checked() 2034 var stub = spawn_max_local_stub(0, "count_requests", 1, guard, 2000) 2035 var actual = stub.pid 2036 stub.state.faults.cleanup_failures = 1 2037 stub.cleanup() 2038 assert_equal(guard.retained(), 1) 2039 assert_equal(guard.recover_all(), 0) 2040 assert_true(pid_not_waitable(actual)) 2041 guard.assert_clean() 2042 # Recovery closes the retained report descriptor exactly once. 2043 assert_true(open_fd_count_checked() <= fd_before) 2044 2045 var jev_guard = CleanupGuard() 2046 var jev_fd_before = open_fd_count_checked() 2047 var jev_stub = spawn_jev_stub_auto("ok", 1, jev_guard, 2000) 2048 var jev_actual = jev_stub.stub.pid 2049 jev_stub.stub.state.faults.cleanup_failures = 1 2050 jev_stub.stub.cleanup() 2051 assert_equal(jev_guard.retained(), 1) 2052 assert_equal(jev_guard.recover_all(), 0) 2053 assert_true(pid_not_waitable(jev_actual)) 2054 jev_guard.assert_clean() 2055 assert_true(open_fd_count_checked() <= jev_fd_before) 2056 2057 2058 def test_released_retained_descriptor_is_not_reclaimed_both_providers() raises: 2059 # MC03/RA02: once a retained report descriptor is released, a following 2060 # owned child that reuses that descriptor number must close its own 2061 # descriptor. The shared guard must not claim a reused number, which would 2062 # leak one descriptor per recovered failure (period-11 R73). 2063 var guard = CleanupGuard() 2064 var fd_before = open_fd_count_checked() 2065 var stub = spawn_max_local_stub(0, "count_requests", 1, guard, 2000) 2066 stub.state.faults.cleanup_failures = 1 2067 stub.cleanup() 2068 assert_equal(guard.retained(), 1) 2069 assert_equal(guard.recover_all(), 0) 2070 guard.assert_clean() 2071 var reused = spawn_max_local_stub(0, "count_requests", 1, guard, 2000) 2072 reused.cleanup() 2073 guard.assert_clean() 2074 assert_equal(open_fd_count_checked() - fd_before, 0) 2075 2076 var jev_guard = CleanupGuard() 2077 var jev_fd_before = open_fd_count_checked() 2078 var jev_stub = spawn_jev_stub_auto("ok", 1, jev_guard, 2000) 2079 jev_stub.stub.state.faults.cleanup_failures = 1 2080 jev_stub.stub.cleanup() 2081 assert_equal(jev_guard.retained(), 1) 2082 assert_equal(jev_guard.recover_all(), 0) 2083 jev_guard.assert_clean() 2084 var jev_reused = spawn_jev_stub_auto("ok", 1, jev_guard, 2000) 2085 jev_reused.stub.cleanup() 2086 jev_guard.assert_clean() 2087 assert_equal(open_fd_count_checked() - jev_fd_before, 0) 2088 2089 2090 def test_reap_wait_error_retains_ownership_both_providers() raises: 2091 # RA02: a transient wait error consumed by reap() must stay retryable and 2092 # must not become a cached terminal result; the retry reports success. 2093 var guard = CleanupGuard() 2094 var stub = _owned_report_child(guard, 0, VALID_REPORT) 2095 var actual = stub.pid 2096 stub.state.faults.wait_errors = 1 2097 stub.reap() 2098 assert_true(not stub.ok()) 2099 assert_equal(stub.reason(), "wait_error") 2100 assert_true(not stub.status().cleanup_proved()) 2101 assert_equal(guard.retained(), 1) 2102 # The transient error must not be cached as a terminal observation. 2103 assert_true(stub.status().state != "wait_error") 2104 # A retry must observe the real child and still decode its valid report. 2105 stub.reap() 2106 assert_true(stub.ok()) 2107 assert_equal(stub.phase(), "complete") 2108 assert_equal(stub.request_count(), 1) 2109 assert_true(stub.status().cleanup_proved()) 2110 assert_true(pid_not_waitable(actual)) 2111 guard.assert_clean() 2112 # A repeated reap is idempotent and does not re-wait or re-read. 2113 stub.reap() 2114 assert_true(stub.ok()) 2115 assert_equal(stub.request_count(), 1) 2116 2117 var jev_guard = CleanupGuard() 2118 var jev_stub = _owned_jev_report_child(jev_guard, 0, VALID_REPORT) 2119 var jev_actual = jev_stub.pid 2120 jev_stub.state.faults.wait_errors = 1 2121 jev_stub.reap() 2122 assert_true(not jev_stub.ok()) 2123 assert_equal(jev_stub.reason(), "wait_error") 2124 assert_equal(jev_guard.retained(), 1) 2125 jev_stub.reap() 2126 assert_true(jev_stub.ok()) 2127 assert_equal(jev_stub.request_count(), 1) 2128 assert_true(pid_not_waitable(jev_actual)) 2129 jev_guard.assert_clean() 2130 2131 2132 def test_unexpected_nonterminal_status_fails_closed_both_providers() raises: 2133 # RA02: an unexpected nonterminal wait status must fail closed and keep the 2134 # exact ownership instead of falling through to a report success. 2135 var guard = CleanupGuard() 2136 var stub = _owned_report_child(guard, 0, VALID_REPORT) 2137 var actual = stub.pid 2138 stub.state.faults.nonterminal = 1 2139 stub.reap() 2140 assert_true(not stub.ok()) 2141 assert_equal(stub.reason(), "unexpected_status") 2142 assert_true(not stub.status().cleanup_proved()) 2143 assert_equal(guard.retained(), 1) 2144 # Recovery still collects the exact owned child. 2145 guard.recover_all() 2146 guard.assert_clean() 2147 assert_true(pid_not_waitable(actual)) 2148 2149 var jev_guard = CleanupGuard() 2150 var jev_stub = _owned_jev_report_child(jev_guard, 0, VALID_REPORT) 2151 var jev_actual = jev_stub.pid 2152 jev_stub.state.faults.nonterminal = 1 2153 jev_stub.reap() 2154 assert_true(not jev_stub.ok()) 2155 assert_equal(jev_stub.reason(), "unexpected_status") 2156 jev_guard.recover_all() 2157 jev_guard.assert_clean() 2158 assert_true(pid_not_waitable(jev_actual)) 2159 2160 2161 def test_cleanup_failure_fails_normal_scope_exit_both_providers() raises: 2162 # RA01: an ordinary supported provider scope that exits normally must not 2163 # silently discard a cleanup failure. The exact-owned child is retained for 2164 # recovery, and the required guard check fails the owning test. 2165 var guard = CleanupGuard() 2166 var fd_before = open_fd_count_checked() 2167 var held_pid = 0 2168 with spawn_max_local_stub(0, "count_requests", 1, guard) as stub: 2169 held_pid = stub.pid 2170 stub.inject_cleanup_failure() 2171 assert_true(pid_running(held_pid)) 2172 var failure = "" 2173 try: 2174 guard.assert_clean() 2175 except e: 2176 failure = String(e) 2177 assert_true(failure.find("cleanup-unproved") >= 0) 2178 assert_equal(guard.recover_all(), 0) 2179 guard.assert_clean() 2180 assert_true(pid_not_waitable(held_pid)) 2181 assert_true(open_fd_count_checked() <= fd_before) 2182 2183 var jev_guard = CleanupGuard() 2184 var jev_fd_before = open_fd_count_checked() 2185 var jev_pid = 0 2186 with spawn_jev_stub_auto("ok", 1, jev_guard) as started: 2187 jev_pid = started.stub.pid 2188 started.stub.inject_cleanup_failure() 2189 var jev_failure = "" 2190 try: 2191 jev_guard.assert_clean() 2192 except e: 2193 jev_failure = String(e) 2194 assert_true(jev_failure.find("cleanup-unproved") >= 0) 2195 assert_equal(jev_guard.recover_all(), 0) 2196 jev_guard.assert_clean() 2197 assert_true(pid_not_waitable(jev_pid)) 2198 assert_true(open_fd_count_checked() <= jev_fd_before) 2199 2200 2201 def test_cleanup_failure_separately_exposed_with_body_cause() raises: 2202 # RA01: on the exception path the body/assertion cause must be preserved 2203 # exactly while the cleanup failure is separately exposed by the guard. 2204 var guard = CleanupGuard() 2205 var body_message = "" 2206 var held_pid = 0 2207 try: 2208 with spawn_max_local_stub(0, "count_requests", 1, guard) as stub: 2209 held_pid = stub.pid 2210 stub.inject_cleanup_failure() 2211 raise Error("intentional scope error") 2212 except e: 2213 body_message = String(e) 2214 # Body cause preserved, not replaced by the cleanup failure. 2215 assert_equal(body_message, "intentional scope error") 2216 assert_equal(guard.retained(), 1) 2217 var failure = "" 2218 try: 2219 guard.assert_clean() 2220 except e: 2221 failure = String(e) 2222 assert_true(failure.find("cleanup-unproved") >= 0) 2223 assert_equal(guard.recover_all(), 0) 2224 guard.assert_clean() 2225 assert_true(pid_not_waitable(held_pid)) 2226 2227 2228 def test_provider_reap_descriptor_read_error_both_paths() raises: 2229 # PC01: the descriptor-read control executes through BOTH provider reap 2230 # paths, not only the shared reader: a real owned child with an unavailable 2231 # report descriptor fails with read_error, never a silent EOF/empty report. 2232 var pipe = make_pipe() 2233 var closed_fd = pipe.read_fd 2234 close_fd(pipe.read_fd) 2235 close_fd(pipe.write_fd) 2236 var pid = fork_pid() 2237 if pid == 0: 2238 child_exit(0) 2239 var guard = CleanupGuard() 2240 var state = piped_child_state( 2241 pid, closed_fd, 2000, 1, UnsafePointer(to=guard) 2242 ) 2243 var stub = SpawnedMaxLocalStub(pid, 0, state^) 2244 stub.reap() 2245 assert_true(not stub.ok()) 2246 assert_equal(stub.reason(), "read_error") 2247 assert_true(pid_not_waitable(pid)) 2248 guard.assert_clean() 2249 2250 var jev_pipe = make_pipe() 2251 var jev_closed_fd = jev_pipe.read_fd 2252 close_fd(jev_pipe.read_fd) 2253 close_fd(jev_pipe.write_fd) 2254 var jev_pid = fork_pid() 2255 if jev_pid == 0: 2256 child_exit(0) 2257 var jev_guard = CleanupGuard() 2258 var jev_state = piped_child_state( 2259 jev_pid, jev_closed_fd, 2000, 1, UnsafePointer(to=jev_guard) 2260 ) 2261 var jev_stub = SpawnedJevStub(jev_pid, 0, jev_state^) 2262 jev_stub.reap() 2263 assert_true(not jev_stub.ok()) 2264 assert_equal(jev_stub.reason(), "read_error") 2265 assert_true(pid_not_waitable(jev_pid)) 2266 jev_guard.assert_clean() 2267 2268 2269 def test_cleanup_failure_survives_scope_exit() raises: 2270 # RA01/PC02: cleanup failure must remain observable after the owning handle 2271 # is destroyed at scope exit, for BOTH provider handles, because the guard 2272 # is owned by the calling test rather than the handle. 2273 var guard = CleanupGuard() 2274 var state = piped_child_state(0, -1, 100, 1, UnsafePointer(to=guard)) 2275 with SpawnedMaxLocalStub(0, 0, state^) as holder: 2276 _ = holder 2277 assert_equal(guard.count(), 1) 2278 assert_true(guard.first().find("unproved") >= 0) 2279 var jev_state = piped_child_state(0, -1, 100, 1, UnsafePointer(to=guard)) 2280 with SpawnedJevStub(0, 0, jev_state^) as jev_holder: 2281 _ = jev_holder 2282 assert_equal(guard.count(), 2) 2283 assert_true(guard.first().find("unproved") >= 0) 2284 var failure = "" 2285 try: 2286 guard.assert_clean() 2287 except e: 2288 failure = String(e) 2289 assert_true(failure.find("cleanup-unproved") >= 0) 2290 2291 2292 @fieldwise_init 2293 struct ForkedReportChild(Movable): 2294 """A real owned child whose bounded report emission is controlled by delay. 2295 """ 2296 2297 var pid: Int 2298 var read_fd: Int 2299 2300 2301 def _fork_budget_child( 2302 delay_ms: Int, report: String 2303 ) raises -> ForkedReportChild: 2304 """Fork a real owned child that writes ``report`` after ``delay_ms``.""" 2305 var pipe = make_pipe() 2306 var pid = fork_pid() 2307 if pid == 0: 2308 close_fd(pipe.read_fd) 2309 sleep_ms(delay_ms) 2310 if report != "": 2311 _ = write_raw(pipe.write_fd, report) 2312 close_fd(pipe.write_fd) 2313 child_exit(0) 2314 close_fd(pipe.write_fd) 2315 return ForkedReportChild(pid, pipe.read_fd) 2316 2317 2318 def test_report_after_work_budget_is_rejected_both_providers() raises: 2319 # RA03: the period-9 counterexample. The child emits a *valid* report after 2320 # 200 ms while the declared budget is 100 ms and the parent observes it at 2321 # 300 ms; the late report must be rejected and the exact-owned child 2322 # collected within the bounded cleanup allowance, never accepted with a 2323 # fresh success interval. 2324 for provider in range(2): 2325 var guard = CleanupGuard() 2326 var probe = _fork_budget_child(200, VALID_REPORT) 2327 var start = now_ms() 2328 var state = piped_child_state( 2329 probe.pid, probe.read_fd, 100, 1, UnsafePointer(to=guard) 2330 ) 2331 # The parent consumes its declared budget without observing the child, 2332 # exactly as in the period-9 counterexample. 2333 sleep_ms(300) 2334 if provider == 0: 2335 var stub = SpawnedMaxLocalStub(probe.pid, 0, state^) 2336 stub.reap() 2337 assert_true(not stub.ok()) 2338 assert_equal(stub.reason(), "work_budget_expired") 2339 assert_true(stub.status().cleanup_proved()) 2340 else: 2341 var stub = SpawnedJevStub(probe.pid, 0, state^) 2342 stub.reap() 2343 assert_true(not stub.ok()) 2344 assert_equal(stub.reason(), "work_budget_expired") 2345 assert_true(stub.status().cleanup_proved()) 2346 var elapsed = now_ms() - start 2347 # Only the bounded cleanup allowance is added after the budget expired. 2348 assert_true(elapsed <= 100 + 2000 + 500) 2349 assert_true(pid_not_waitable(probe.pid)) 2350 guard.assert_clean() 2351 2352 2353 def test_report_within_work_budget_succeeds_both_providers() raises: 2354 # RA03 positive control: the same budgeted path accepts a report that is 2355 # emitted and observed inside the declared budget. 2356 for provider in range(2): 2357 var guard = CleanupGuard() 2358 var probe = _fork_budget_child(20, VALID_REPORT) 2359 var state = piped_child_state( 2360 probe.pid, probe.read_fd, 4000, 1, UnsafePointer(to=guard) 2361 ) 2362 if provider == 0: 2363 var stub = SpawnedMaxLocalStub(probe.pid, 0, state^) 2364 stub.reap() 2365 assert_true(stub.ok()) 2366 assert_equal(stub.phase(), "complete") 2367 assert_equal(stub.request_count(), 1) 2368 else: 2369 var stub = SpawnedJevStub(probe.pid, 0, state^) 2370 stub.reap() 2371 assert_true(stub.ok()) 2372 assert_equal(stub.phase(), "complete") 2373 assert_equal(stub.request_count(), 1) 2374 assert_true(pid_not_waitable(probe.pid)) 2375 guard.assert_clean() 2376 2377 2378 def test_report_drain_deadline_is_bounded() raises: 2379 # RA03: the report EOF-drain stage honours a finite deadline instead of a 2380 # fresh unbounded interval; a report descriptor that never reaches EOF must 2381 # fail with the drain deadline cause, never hang or succeed. 2382 var pipe = make_pipe() 2383 _ = write_raw( 2384 pipe.write_fd, 2385 "result ok phase=complete case=- reason=ok requests=1 connections=1\n", 2386 ) 2387 var guard = CleanupGuard() 2388 var state = piped_child_state( 2389 0, pipe.read_fd, 500, 1, UnsafePointer(to=guard) 2390 ) 2391 var line = state.read_line(STRICT_MAX_REPORT_BYTES, 500) 2392 assert_true(state.last_terminated) 2393 assert_true(line.startswith("result ok")) 2394 var reason = "" 2395 try: 2396 _ = state.drain_surplus(STRICT_MAX_REPORT_BYTES, 60) 2397 except e: 2398 reason = String(e) 2399 state.close_reader() 2400 close_fd(pipe.write_fd) 2401 assert_equal(reason, "read_deadline_expired") 2402 guard.assert_clean() 2403 2404 2405 def test_work_budget_delay_rejects_before_report_both_providers() raises: 2406 # RA03: time consumed inside the wait phase counts against the same finite 2407 # budget, so an exhausted budget rejects even when a valid report is already 2408 # buffered on the owned descriptor. 2409 for provider in range(2): 2410 var guard = CleanupGuard() 2411 var probe = _fork_budget_child(0, VALID_REPORT) 2412 var state = piped_child_state( 2413 probe.pid, probe.read_fd, 400, 1, UnsafePointer(to=guard) 2414 ) 2415 state.faults.wait_delay_ms = 900 2416 if provider == 0: 2417 var stub = SpawnedMaxLocalStub(probe.pid, 0, state^) 2418 stub.reap() 2419 assert_true(not stub.ok()) 2420 assert_equal(stub.reason(), "work_budget_expired") 2421 assert_true(stub.status().cleanup_proved()) 2422 else: 2423 var stub = SpawnedJevStub(probe.pid, 0, state^) 2424 stub.reap() 2425 assert_true(not stub.ok()) 2426 assert_equal(stub.reason(), "work_budget_expired") 2427 assert_true(stub.status().cleanup_proved()) 2428 assert_true(pid_not_waitable(probe.pid)) 2429 guard.assert_clean() 2430 2431 2432 def test_descriptor_census_error_classes() raises: 2433 # RA04: EBADF is a closed slot, EINTR is a bounded retry, and any other 2434 # lookup error makes the census unavailable instead of silently lowering 2435 # the count. The seam changes no host limit. 2436 assert_equal(classify_census_errno(9), "closed") 2437 assert_equal(classify_census_errno(4), "retry") 2438 assert_equal(classify_census_errno(5), "unavailable") 2439 var before = open_fd_count_checked() 2440 assert_true(before > 0) 2441 # EBADF at an open slot is a closed slot: the count is simply lower. 2442 assert_equal(descriptor_census_with_faults(3, 0, 9), 2) 2443 # Any other lookup error makes the whole census unavailable. 2444 assert_equal(descriptor_census_with_faults(3, 0, 5), -1) 2445 var unavailable = "" 2446 try: 2447 _ = open_fd_count_checked_with_faults(3, 0, 5) 2448 except e: 2449 unavailable = String(e) 2450 assert_equal(unavailable, "descriptor_census_unavailable") 2451 assert_equal(open_fd_count_checked(), before) 2452 2453 2454 def test_descriptor_census_unavailable_propagates() raises: 2455 # PC04: an unavailable or incomplete-range census must fail explicitly 2456 # through the checked caller rather than look like a small passing count. 2457 assert_equal(descriptor_census(0), -1) 2458 var zero_reason = "" 2459 try: 2460 _ = open_fd_count_checked(0) 2461 except e: 2462 zero_reason = String(e) 2463 assert_equal(zero_reason, "descriptor_census_unavailable") 2464 var ceiling_reason = "" 2465 try: 2466 _ = open_fd_count_checked(CENSUS_MAX_FDS + 1) 2467 except e: 2468 ceiling_reason = String(e) 2469 assert_equal(ceiling_reason, "descriptor_census_unavailable") 2470 assert_true(open_fd_count_checked() > 0) 2471 2472 2473 def main() raises: 2474 TestSuite.discover_tests[__functions_in_module()]().run()