TeraRuntimeClient.swift (24906B)
1 import Foundation 2 3 actor TeraRuntimeClient { 4 private struct StartupOperation: Sendable { 5 let identity: TeraRuntimeOperationIdentity 6 let configuration: TeraRuntimeLaunchConfiguration 7 let task: TeraRuntimeResourceTask<TeraRuntimeBackendStart> 8 var waiters = 0 9 } 10 11 private struct ShutdownOperation: Sendable { 12 let identity: TeraRuntimeOperationIdentity 13 let work: TeraRuntimeShutdownWork 14 let task: TeraRuntimeShutdownTask 15 } 16 17 private struct ActiveOperation: Sendable { 18 let identity: TeraRuntimeOperationIdentity 19 let cancel: @Sendable () -> Void 20 let drain: @Sendable () async throws -> Void 21 } 22 23 private let factory: TeraRuntimeBackendFactory 24 private let deadlines: TeraRuntimeDeadlinePolicy 25 private var generation = TeraSessionGeneration.initial 26 private var operationSequence: UInt64 = 0 27 private var lifecycleState: TeraRuntimeLifecycle = .stopped 28 private var configuration: TeraRuntimeLaunchConfiguration? 29 private var backend: (any TeraRuntimeBackend)? 30 private var retryShutdownWork: TeraRuntimeShutdownWork? 31 private var abandonedStartups: [UInt64: TeraRuntimeResourceTask<TeraRuntimeBackendStart>] = [:] 32 private var startupOperation: StartupOperation? 33 private var shutdownOperation: ShutdownOperation? 34 private var activeOperations: [UInt64: ActiveOperation] = [:] 35 private var activeSubmission: TeraRuntimeOperationIdentity? 36 private var subscriptions: [UUID: TeraRuntimeSubscription] = [:] 37 38 init( 39 factory: @escaping TeraRuntimeBackendFactory, 40 deadlines: TeraRuntimeDeadlinePolicy = .production 41 ) { 42 self.factory = factory 43 self.deadlines = deadlines 44 } 45 46 func lifecycle() -> TeraRuntimeLifecycle { 47 lifecycleState 48 } 49 50 func start( 51 configuration requestedConfiguration: TeraRuntimeLaunchConfiguration 52 ) async throws -> TeraRuntimeSnapshot { 53 try await start(configuration: requestedConfiguration, kind: .startup) 54 } 55 56 func reconfigure( 57 configuration requestedConfiguration: TeraRuntimeLaunchConfiguration 58 ) async throws -> TeraRuntimeSnapshot { 59 try await start(configuration: requestedConfiguration, kind: .reconfiguration) 60 } 61 62 private func start( 63 configuration requestedConfiguration: TeraRuntimeLaunchConfiguration, 64 kind: TeraRuntimeOperationKind 65 ) async throws -> TeraRuntimeSnapshot { 66 if let shutdownOperation { 67 _ = try await finishShutdown(shutdownOperation) 68 } 69 70 if retryShutdownWork != nil || !abandonedStartups.isEmpty { 71 _ = try await finishShutdown(beginShutdown()) 72 } 73 74 if backend != nil, 75 configuration == requestedConfiguration, 76 case .running = lifecycleState 77 { 78 return try await snapshot() 79 } 80 81 if let startupOperation, 82 startupOperation.configuration == requestedConfiguration 83 { 84 return try await finishStartup(startupOperation) 85 } 86 87 if startupOperation != nil || backend != nil { 88 let operation = beginShutdown() 89 _ = try await finishShutdown(operation) 90 } 91 92 generation = try generation.next() 93 let operationGeneration = generation 94 lifecycleState = .starting(generation: operationGeneration) 95 96 let identity = nextIdentity(kind: kind) 97 let task = TeraRuntimeResourceCreation.startup( 98 configuration: requestedConfiguration, factory: factory, 99 identity: identity, deadlines: deadlines 100 ) 101 let operation = StartupOperation( 102 identity: identity, 103 configuration: requestedConfiguration, 104 task: task 105 ) 106 startupOperation = operation 107 return try await finishStartup(operation) 108 } 109 110 func retry( 111 configuration requestedConfiguration: TeraRuntimeLaunchConfiguration 112 ) async throws -> TeraRuntimeSnapshot { 113 _ = try await stop() 114 return try await start(configuration: requestedConfiguration) 115 } 116 117 func snapshot() async throws -> TeraRuntimeSnapshot { 118 do { 119 return try await runtimeOperation("runtime.status") { backend in 120 try await backend.snapshot() 121 } 122 } catch let error as TeraRuntimeClientError { 123 throw error 124 } catch { 125 throw TeraRuntimeClientError.status( 126 Self.failure(from: error, operation: "runtime.status") 127 ) 128 } 129 } 130 131 func search( 132 context: TeraLocalNetwork, 133 query: String, 134 limit: UInt16 = 50, 135 asOfUnixSeconds: UInt64 136 ) async throws -> [TeraSearchResult] { 137 try await supportOperation("runtime.support.search") { backend in 138 try await backend.search( 139 context: context, 140 query: query, 141 limit: limit, 142 asOfUnixSeconds: asOfUnixSeconds 143 ) 144 } 145 } 146 147 func me( 148 context: TeraLocalNetwork, 149 asOfUnixSeconds: UInt64 150 ) async throws -> TeraMeSnapshot { 151 try await supportOperation("runtime.support.me") { backend in 152 try await backend.me(context: context, asOfUnixSeconds: asOfUnixSeconds) 153 } 154 } 155 156 func retrieveMedia( 157 context: TeraLocalNetwork, 158 reference: TeraMediaReference 159 ) async throws -> TeraVerifiedMediaArtifact { 160 try await supportOperation("runtime.media.retrieve") { backend in 161 try await backend.retrieveMedia(context: context, reference: reference) 162 } 163 } 164 165 func verifiedMediaArtifact( 166 context: TeraLocalNetwork, 167 artifactID: String 168 ) async throws -> TeraVerifiedMediaArtifact? { 169 try await supportOperation("runtime.media.verified_artifact") { backend in 170 try await backend.verifiedMediaArtifact(context: context, artifactID: artifactID) 171 } 172 } 173 174 func invalidateMediaArtifact( 175 context: TeraLocalNetwork, 176 artifactID: String 177 ) async throws -> Bool { 178 try await supportOperation("runtime.media.invalidate") { backend in 179 try await backend.invalidateMediaArtifact(context: context, artifactID: artifactID) 180 } 181 } 182 183 func addSchemas() async throws -> [TeraAddSchema] { 184 try await addOperation("runtime.add.schemas") { backend in 185 try await backend.addSchemas() 186 } 187 } 188 189 func saveAddIntent( 190 input: TeraAddRuntimeInput, 191 existingDraftID: String?, 192 expectedRevision: UInt64? 193 ) async throws -> TeraDraftStatus { 194 try await addOperation("runtime.add.save") { backend in 195 try await backend.saveAddIntent( 196 input: input, 197 existingDraftID: existingDraftID, 198 expectedRevision: expectedRevision 199 ) 200 } 201 } 202 203 func saveRetractionDraft( 204 id: String, 205 input: TeraRetractionDraftInput, 206 authoredAtUnixSeconds: UInt64, 207 persistedAtUnixMilliseconds: UInt64 208 ) async throws -> TeraDraftStatus { 209 try await addOperation("runtime.add.retract") { backend in 210 try await backend.saveRetractionDraft( 211 id: id, 212 input: input, 213 authoredAtUnixSeconds: authoredAtUnixSeconds, 214 persistedAtUnixMilliseconds: persistedAtUnixMilliseconds 215 ) 216 } 217 } 218 219 func draftStatus(id: String) async throws -> TeraDraftStatus { 220 try await addOperation("runtime.add.status") { backend in 221 try await backend.draftStatus(id: id) 222 } 223 } 224 225 func draftHeads(limit: UInt16 = 100) async throws -> [TeraDraftStatus] { 226 try await addOperation("runtime.add.heads") { backend in 227 try await backend.draftHeads(limit: limit) 228 } 229 } 230 231 func queueAddIntent( 232 id: String, 233 expectedRevision: UInt64 234 ) async throws -> TeraDraftStatus { 235 try await addOperation("runtime.add.queue") { backend in 236 try await backend.queueAddIntent( 237 id: id, 238 expectedRevision: expectedRevision 239 ) 240 } 241 } 242 243 func recoverAddIntent(id: String) async throws -> TeraDraftStatus { 244 try await addOperation("runtime.add.recover") { backend in 245 try await backend.recoverAddIntent(id: id) 246 } 247 } 248 249 func uploadAddMediaIntent(input: TeraBlossomUploadIntent) async throws -> TeraDraftStatus { 250 try await addOperation("runtime.add.media") { backend in 251 try await backend.uploadAddMediaIntent(input: input) 252 } 253 } 254 255 func prepareAddMediaBackground( 256 input: TeraBlossomUploadIntent 257 ) async throws -> TeraNativeUploadJob { 258 try await addOperation("runtime.add.media.background.prepare") { backend in 259 try await backend.prepareAddMediaBackground(input: input) 260 } 261 } 262 263 func completeAddMediaBackground( 264 input: TeraNativeUploadCompletion 265 ) async throws -> TeraDraftStatus { 266 try await addOperation("runtime.add.media.background.complete") { backend in 267 try await backend.completeAddMediaBackground(input: input) 268 } 269 } 270 271 func probeBlossom() async throws -> TeraBlossomEvidence { 272 try await supportOperation("runtime.blossom.probe") { backend in 273 try await backend.probeBlossom() 274 } 275 } 276 277 func mobileSettings() async throws -> TeraMobileSettings { 278 try await supportOperation("runtime.settings.read") { backend in 279 try await backend.mobileSettings() 280 } 281 } 282 283 func replaceMobileSettings( 284 input: TeraReplaceSettings 285 ) async throws -> TeraSettingsTransition { 286 try await supportOperation("runtime.settings.replace") { backend in 287 try await backend.replaceMobileSettings(input: input) 288 } 289 } 290 291 func applyIdentityCommand( 292 expectedRevision: UInt64, 293 command: TeraIdentityCommand 294 ) async throws -> TeraSettingsTransition { 295 try await supportOperation("runtime.settings.identity") { backend in 296 try await backend.applyIdentityCommand( 297 expectedRevision: expectedRevision, 298 command: command 299 ) 300 } 301 } 302 303 func saveProfileMetadata(input: TeraProfileMetadataInput) async throws 304 -> TeraProfileStatus 305 { 306 try await supportOperation("runtime.profile.save") { backend in 307 try await backend.saveProfileMetadata(input: input) 308 } 309 } 310 311 func profileStatus(operationID: String) async throws -> TeraProfileStatus { 312 try await supportOperation("runtime.profile.status") { backend in 313 try await backend.profileStatus(operationID: operationID) 314 } 315 } 316 317 func advanceProfile(operationID: String) async throws -> TeraProfileStatus { 318 try await supportOperation("runtime.profile.advance") { backend in 319 try await backend.advanceProfile(operationID: operationID) 320 } 321 } 322 323 func cancelProfile( 324 operationID: String, 325 expectedRevision: UInt64 326 ) async throws -> TeraProfileStatus { 327 try await supportOperation("runtime.profile.cancel") { backend in 328 try await backend.cancelProfile( 329 operationID: operationID, 330 expectedRevision: expectedRevision 331 ) 332 } 333 } 334 335 func advanceDraft(id: String, expectedRevision: UInt64) async throws -> TeraDraftStatus { 336 try await addOperation("runtime.add.advance") { backend in 337 try await backend.advanceDraft(id: id, expectedRevision: expectedRevision) 338 } 339 } 340 341 func cancelAddIntent( 342 id: String, 343 expectedRevision: UInt64 344 ) async throws -> TeraDraftStatus { 345 try await addOperation("runtime.add.cancel") { backend in 346 try await backend.cancelAddIntent( 347 id: id, 348 expectedRevision: expectedRevision 349 ) 350 } 351 } 352 353 func changes(bufferCapacity: Int = 16) async throws -> AsyncStream<TeraRuntimeChange> { 354 guard (1 ... 64).contains(bufferCapacity) else { 355 throw TeraRuntimeClientError.invalidBufferCapacity 356 } 357 guard let backend, case .running = lifecycleState else { 358 throw TeraRuntimeClientError.notRunning 359 } 360 361 let id = UUID() 362 let subscriptionGeneration = generation 363 let identity = nextIdentity(kind: .subscription) 364 let pair = AsyncStream.makeStream( 365 of: TeraRuntimeChange.self, 366 bufferingPolicy: .bufferingNewest(bufferCapacity) 367 ) 368 pair.continuation.onTermination = { [weak self] _ in 369 Task { 370 await self?.cancelSubscription(id: id, generation: subscriptionGeneration) 371 } 372 } 373 subscriptions[id] = TeraRuntimeSubscription( 374 generation: subscriptionGeneration, 375 continuation: pair.continuation, 376 token: nil 377 ) 378 379 let task = TeraRuntimeResourceCreation.subscription( 380 backend: backend, bufferCapacity: bufferCapacity, 381 identity: identity, deadline: deadlines.subscriptionNanoseconds 382 ) { [weak self] change in 383 await self?.receive(change, subscriptionID: id, generation: subscriptionGeneration) 384 } 385 trackCreation(task, identity: identity) 386 let outcome = await task.value() 387 if case .completed = outcome { 388 removeActiveOperation(identity) 389 } 390 391 guard generation == subscriptionGeneration, 392 case .running = lifecycleState, 393 var subscription = subscriptions[id] 394 else { 395 subscriptions.removeValue(forKey: id)?.continuation.finish() 396 task.cancel() 397 throw TeraRuntimeClientError.superseded 398 } 399 400 if Task.isCancelled { 401 subscriptions.removeValue(forKey: id)?.continuation.finish() 402 task.cancel() 403 throw TeraRuntimeClientError.subscription(Self.cancellationFailure(identity: identity)) 404 } 405 406 switch outcome { 407 case let .completed(.success(token)): 408 guard task.adopt() else { 409 subscriptions.removeValue(forKey: id)?.continuation.finish() 410 throw TeraRuntimeClientError.superseded 411 } 412 subscription.token = token 413 subscriptions[id] = subscription 414 return pair.stream 415 case let .completed(.failure(failure)): 416 subscriptions.removeValue(forKey: id)?.continuation.finish() 417 throw TeraRuntimeClientError.subscription(failure) 418 case .timedOut: 419 subscriptions.removeValue(forKey: id)?.continuation.finish() 420 throw TeraRuntimeClientError.subscription( 421 Self.deadlineFailure(identity: identity) 422 ) 423 case .cancelled: 424 subscriptions.removeValue(forKey: id)?.continuation.finish() 425 throw TeraRuntimeClientError.subscription( 426 Self.cancellationFailure(identity: identity) 427 ) 428 } 429 } 430 431 func suspend() { 432 for operation in activeOperations.values { 433 operation.cancel() 434 } 435 436 let activeSubscriptions = Array(subscriptions.values) 437 subscriptions.removeAll() 438 for subscription in activeSubscriptions { 439 subscription.continuation.finish() 440 if let token = subscription.token { 441 cancelTokenDetached(token) 442 } 443 } 444 445 if let startupOperation { 446 generation = generation.invalidated() 447 abandonStartup(startupOperation) 448 self.startupOperation = nil 449 configuration = nil 450 lifecycleState = .stopped 451 } 452 } 453 454 func stop() async throws -> TeraRuntimeShutdownReceipt { 455 if let shutdownOperation { 456 return try await finishShutdown(shutdownOperation) 457 } 458 guard 459 startupOperation != nil || backend != nil || retryShutdownWork != nil || !abandonedStartups.isEmpty 460 || !subscriptions.isEmpty || !activeOperations.isEmpty 461 else { 462 lifecycleState = .stopped 463 return .alreadyStopped 464 } 465 return try await finishShutdown(beginShutdown()) 466 } 467 468 private func finishStartup( 469 _ operation: StartupOperation 470 ) async throws -> TeraRuntimeSnapshot { 471 if startupOperation?.identity == operation.identity { 472 startupOperation?.waiters += 1 473 } 474 defer { releaseStartupWaiter(operation) } 475 let outcome = await operation.task.value(cancelsOperationWhenWaiterCancelled: false) 476 guard generation == operation.identity.generation else { 477 operation.task.cancel() 478 throw TeraRuntimeClientError.superseded 479 } 480 481 if Task.isCancelled { 482 throw TeraRuntimeClientError.startup(Self.cancellationFailure(identity: operation.identity)) 483 } 484 485 if startupOperation?.identity == operation.identity { 486 startupOperation = nil 487 } else if case let .completed(.success(started)) = outcome, 488 configuration == operation.configuration, 489 case .running = lifecycleState 490 { 491 return started.snapshot 492 } else { 493 throw TeraRuntimeClientError.superseded 494 } 495 496 switch outcome { 497 case let .completed(.success(started)): 498 guard operation.task.adopt() else { throw TeraRuntimeClientError.superseded } 499 backend = started.backend 500 configuration = operation.configuration 501 lifecycleState = .running(generation: operation.identity.generation) 502 return started.snapshot 503 case let .completed(.failure(failure)): 504 lifecycleState = .failed(generation: operation.identity.generation, failure: failure) 505 throw TeraRuntimeClientError.startup(failure) 506 case .timedOut: 507 abandonStartup(operation) 508 let failure = Self.deadlineFailure(identity: operation.identity) 509 lifecycleState = .failed(generation: operation.identity.generation, failure: failure) 510 throw TeraRuntimeClientError.startup(failure) 511 case .cancelled: 512 abandonStartup(operation) 513 let failure = Self.cancellationFailure(identity: operation.identity) 514 lifecycleState = .failed(generation: operation.identity.generation, failure: failure) 515 throw TeraRuntimeClientError.startup(failure) 516 } 517 } 518 519 private func releaseStartupWaiter(_ operation: StartupOperation) { 520 guard var pending = startupOperation, pending.identity == operation.identity else { return } 521 pending.waiters -= 1 522 guard pending.waiters == 0 else { 523 startupOperation = pending 524 return 525 } 526 abandonStartup(pending) 527 startupOperation = nil 528 configuration = nil 529 lifecycleState = .stopped 530 } 531 532 private func trackCreation( 533 _ task: TeraRuntimeResourceTask<some Sendable>, identity: TeraRuntimeOperationIdentity 534 ) { 535 activeOperations[identity.sequence] = ActiveOperation( 536 identity: identity, 537 cancel: { task.cancel() }, 538 drain: { try await task.finishAbandonment() } 539 ) 540 } 541 542 private func abandonStartup(_ operation: StartupOperation) { 543 operation.task.cancel() 544 abandonedStartups[operation.identity.sequence] = operation.task 545 } 546 547 private func beginShutdown() -> ShutdownOperation { 548 generation = generation.invalidated() 549 if let startupOperation { 550 abandonStartup(startupOperation) 551 } 552 let activeSubscriptions = Array(subscriptions.values) 553 let activeRuntimeOperations = Array(activeOperations.values) 554 let startups = Array(abandonedStartups.values) 555 let activeBackend = backend 556 startupOperation = nil 557 backend = nil 558 configuration = nil 559 subscriptions.removeAll() 560 activeOperations.removeAll() 561 abandonedStartups.removeAll() 562 lifecycleState = .stopping(generation: generation) 563 564 for operation in activeRuntimeOperations { 565 operation.cancel() 566 } 567 for subscription in activeSubscriptions { 568 subscription.continuation.finish() 569 } 570 let drains: [@Sendable () async throws -> Void] = 571 startups.map { startup in { @Sendable in try await startup.finishAbandonment() } } 572 + activeRuntimeOperations.map(\.drain) 573 + activeSubscriptions.compactMap(\.token).map { token in { @Sendable in await token.cancel() } } 574 let work = retryShutdownWork ?? TeraRuntimeShutdownWork(backend: activeBackend, drains: drains) 575 retryShutdownWork = work 576 let identity = nextIdentity(kind: .shutdown) 577 let task = TeraRuntimeShutdownTask(work: work, deadline: deadlines.shutdownNanoseconds) { [weak self] result in 578 await self?.settleShutdown(identity: identity, result: result) 579 } 580 let operation = ShutdownOperation(identity: identity, work: work, task: task) 581 shutdownOperation = operation 582 return operation 583 } 584 585 private func finishShutdown(_ operation: ShutdownOperation) async throws -> TeraRuntimeShutdownReceipt { 586 let outcome = await operation.task.value() 587 guard generation == operation.identity.generation else { throw TeraRuntimeClientError.superseded } 588 switch outcome { 589 case let .completed(.success(receipt)): 590 return receipt 591 case let .completed(.failure(failure)): 592 throw TeraRuntimeClientError.shutdown(failure) 593 case .timedOut: 594 let failure = Self.deadlineFailure(identity: operation.identity) 595 if shutdownOperation?.identity == operation.identity { 596 lifecycleState = .failed(generation: operation.identity.generation, failure: failure) 597 } 598 throw TeraRuntimeClientError.shutdown(failure) 599 case .cancelled: 600 throw TeraRuntimeClientError.shutdown(Self.cancellationFailure(identity: operation.identity)) 601 } 602 } 603 604 func runtimeOperation<T: Sendable>( 605 _ operation: String, 606 submission: Bool = false, 607 _ body: @escaping @Sendable (any TeraRuntimeBackend) async throws -> T 608 ) async throws -> T { 609 guard let backend, case .running = lifecycleState else { 610 throw TeraRuntimeClientError.notRunning 611 } 612 if submission { 613 try Task.checkCancellation() 614 guard activeSubmission == nil else { 615 throw TeraRuntimeFailure.local(operation: operation, code: "operation_in_progress", 616 safeMessage: "The original submission call is still returning. Retry the same request when it finishes.") 617 } 618 } 619 let operationGeneration = generation 620 let identity = nextIdentity(kind: .operation) 621 if submission { 622 activeSubmission = identity 623 } 624 let task = TeraRuntimeBoundedTask<T>( 625 deadlineNanoseconds: deadlines.operationNanoseconds, 626 operation: { [weak self] in 627 let result: Result<T, TeraRuntimeFailure> 628 do { 629 result = try await .success(body(backend)) 630 } catch { 631 result = .failure(Self.failure(from: error, operation: operation)) 632 } 633 await self?.removeActiveOperation(identity) 634 return result 635 } 636 ) 637 activeOperations[identity.sequence] = ActiveOperation( 638 identity: identity, 639 cancel: { task.cancel() }, 640 drain: { _ = await task.settle() } 641 ) 642 let outcome = await task.value() 643 644 guard generation == operationGeneration, case .running = lifecycleState else { 645 throw TeraRuntimeClientError.superseded 646 } 647 switch outcome { 648 case let .completed(.success(value)): 649 return value 650 case let .completed(.failure(failure)): 651 throw failure 652 case .timedOut: 653 throw Self.deadlineFailure(identity: identity) 654 case .cancelled: 655 throw Self.cancellationFailure(identity: identity) 656 } 657 } 658 659 func supportOperation<T: Sendable>( 660 _ operation: String, 661 _ body: @escaping @Sendable (any TeraRuntimeBackend) async throws -> T 662 ) async throws -> T { 663 do { 664 return try await runtimeOperation(operation, body) 665 } catch let error as TeraRuntimeClientError { 666 throw error 667 } catch { 668 throw TeraRuntimeClientError.support( 669 Self.failure(from: error, operation: operation) 670 ) 671 } 672 } 673 674 private func receive( 675 _ change: TeraRuntimeChange, 676 subscriptionID: UUID, 677 generation subscriptionGeneration: TeraSessionGeneration 678 ) { 679 guard generation == subscriptionGeneration, 680 case .running = lifecycleState, 681 var subscription = subscriptions[subscriptionID], 682 subscription.generation == subscriptionGeneration, 683 change.matches(configuration), subscription.admission.accept(change) 684 else { 685 return 686 } 687 subscriptions[subscriptionID] = subscription 688 change.yield(to: subscription.continuation) 689 } 690 691 private func cancelSubscription(id: UUID, generation subscriptionGeneration: TeraSessionGeneration) { 692 guard let subscription = subscriptions[id], 693 subscription.generation == subscriptionGeneration 694 else { 695 return 696 } 697 subscriptions.removeValue(forKey: id) 698 subscription.continuation.finish() 699 if let token = subscription.token { 700 cancelTokenDetached(token) 701 } 702 } 703 704 private func cancelTokenDetached(_ token: any TeraRuntimeSubscriptionToken) { 705 let identity = nextIdentity(kind: .subscription) 706 let task = TeraRuntimeBoundedTask<Void>(deadlineNanoseconds: deadlines.subscriptionNanoseconds) { [weak self] in 707 await token.cancel() 708 await self?.removeActiveOperation(identity) 709 return .success(()) 710 } 711 activeOperations[identity.sequence] = ActiveOperation( 712 identity: identity, cancel: { task.cancel() }, drain: { _ = await task.settle() } 713 ) 714 } 715 716 private func settleShutdown( 717 identity: TeraRuntimeOperationIdentity, 718 result: Result<TeraRuntimeShutdownReceipt, TeraRuntimeFailure> 719 ) { 720 guard generation == identity.generation, shutdownOperation?.identity == identity else { return } 721 shutdownOperation = nil 722 switch result { 723 case .success: 724 retryShutdownWork = nil 725 lifecycleState = .stopped 726 case let .failure(failure): 727 lifecycleState = .failed(generation: identity.generation, failure: failure) 728 } 729 } 730 731 private func nextIdentity( 732 kind: TeraRuntimeOperationKind 733 ) -> TeraRuntimeOperationIdentity { 734 operationSequence &+= 1 735 return TeraRuntimeOperationIdentity( 736 generation: generation, 737 sequence: operationSequence, 738 kind: kind 739 ) 740 } 741 742 private func removeActiveOperation(_ identity: TeraRuntimeOperationIdentity) { 743 if activeSubmission == identity { 744 activeSubmission = nil 745 } 746 guard activeOperations[identity.sequence]?.identity == identity else { return } 747 activeOperations.removeValue(forKey: identity.sequence) 748 } 749 }