// KeyboardFlowCoordinator.swift // OSGKeyboard · Keyboard Extension // // Flow session start, recording, watchdogs, and result delivery handling. import UIKit import OSGKeyboardShared @MainActor final class KeyboardFlowCoordinator { private enum FlowWatchdog { static let pollIntervalNs: UInt64 = 200_000_000 /// Give the user time to manually open the host app when auto-jump fails. static let startTimeout: TimeInterval = 30 static func resultTimeout(engineMode: String) -> TimeInterval { FlowSessionKeys.keyboardResultTimeout(engineMode: engineMode) } } private let state: KeyboardState private let textInserter: KeyboardTextInserter private let hasFullAccess: () -> Bool private let wakeLockView: () -> UIView? private let openHostApp: (String) -> Void private let detectAndStoreAppContext: () -> Void private let scheduleAutoClearError: () -> Void private let refreshConfigFromAppGroup: () -> Void private var isPendingFlowStart = false private var flowStartDeadline: TimeInterval = 0 private var isFlowRecording = false private var flowWatchdogTask: Task? private var utteranceTimerTask: Task? private var utteranceStartedAt: TimeInterval = 0 private var wasFlowSessionActive = false private var flowSessionMonitorTask: Task? private var isAwaitingFlowResult = false private var lastFlowAutoStartAttempt: TimeInterval = 0 private static let flowAutoStartCooldown: TimeInterval = 20 init( state: KeyboardState, textInserter: KeyboardTextInserter, hasFullAccess: @escaping () -> Bool, wakeLockView: @escaping () -> UIView?, openHostApp: @escaping (String) -> Void, detectAndStoreAppContext: @escaping () -> Void, scheduleAutoClearError: @escaping () -> Void, refreshConfigFromAppGroup: @escaping () -> Void ) { self.state = state self.textInserter = textInserter self.hasFullAccess = hasFullAccess self.wakeLockView = wakeLockView self.openHostApp = openHostApp self.detectAndStoreAppContext = detectAndStoreAppContext self.scheduleAutoClearError = scheduleAutoClearError self.refreshConfigFromAppGroup = refreshConfigFromAppGroup } var preservesLifecycleOnDisappear: Bool { isPendingFlowStart || isFlowRecording || isAwaitingFlowResult } /// Session/transcription changes are pushed in real time by Darwin /// notifications (see `KeyboardConfigSync.installDarwinObservers`), so this /// loop is only a low-frequency safety net for coalesced/dropped Darwin /// signals — hence 3 s rather than 1 Hz to save battery while idle. private static let sessionMonitorIntervalNs: UInt64 = 3_000_000_000 func startSessionMonitor() { flowSessionMonitorTask?.cancel() flowSessionMonitorTask = Task { @MainActor [weak self] in while !Task.isCancelled { self?.refreshSessionState() try? await Task.sleep(nanoseconds: Self.sessionMonitorIntervalNs) } } } func stopSessionMonitor() { flowSessionMonitorTask?.cancel() flowSessionMonitorTask = nil } func refreshSessionState() { refreshConfigFromAppGroup() refreshFlowPartialIfNeeded() consumePendingFlowDeliveryIfNeeded() recoverFromDeadHostIfNeeded() if FlowSessionBridge.clearIfHostStale() { debug("cleared zombie Flow session from App Group") } let reachable = FlowSessionBridge.isHostReachable() state.flowSessionActive = reachable if wasFlowSessionActive && !reachable && !isFlowRecording && !isPendingFlowStart { switch state.phase { case .recording, .processing: break default: showFlowSessionExpiredHint() } } wasFlowSessionActive = reachable maybeAutoStartFlowSession() } func toggleRecording() { switch state.phase { case .recording: pressEnded() case .idle, .denied, .error: pressBegan() case .requestingPermissions, .processing: break } } func pressBegan() { switch state.phase { case .idle, .denied, .error: break default: return } guard !state.micDisabled else { return } guard hasFullAccess() else { let msg = ExtL10n.string("keyboard.error.fullAccessRequired") state.phase = .error(.fullAccessRequired, message: msg) scheduleAutoClearError() return } guard AppGroup.isAvailable else { let msg = ExtL10n.string("keyboard.error.appGroupCommunication") state.phase = .error(.appGroupUnavailable, message: msg) scheduleAutoClearError() return } detectAndStoreAppContext() let reachable = FlowSessionBridge.isHostReachable() debug( "pressBegan hostReachable=\(reachable) " + "staleness=\(FlowSessionBridge.heartbeatStaleness().map { String(format: "%.1f", $0) } ?? "nil") " + "container=\(AppGroup.containerPathForDiagnostics)" ) if reachable { startFlowRecording() } else { beginFlowStart() } } func pressEnded() { if isPendingFlowStart { cancelPendingFlowStart() return } guard isFlowRecording else { return } isFlowRecording = false stopUtteranceCountdown() ExtensionScreenWakeLock.release() FlowSessionBridge.setRecordingState(.stopped) debug("pressEnded wrote .stopped (readback=\(FlowSessionBridge.recordingState().rawValue))") state.phase = .processing state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing") startFlowResultWatchdog() } func beginFlowStart() { guard !isPendingFlowStart else { return } isPendingFlowStart = true isFlowRecording = false flowStartDeadline = Date().timeIntervalSince1970 + FlowWatchdog.startTimeout state.lastTranscript = ExtL10n.string("keyboard.flow.startingSession") state.phase = .processing openHostApp("startflow") startFlowStartWatchdog() debug("beginFlowStart") } func handleHostAppOpenResult(path: String, success: Bool) { debug("openHostApp path=\(path) success=\(success)") guard !success else { return } // The open genuinely failed (iOS blocked it / no Full Access). Don't // let the 30s watchdog spin — cancel the pending start immediately and // guide the user to open OSGKeyboard manually. if path == "startflow", isPendingFlowStart { isPendingFlowStart = false flowStartDeadline = 0 stopFlowWatchdog() showManualOpenHint(path: "startflow") return } showManualOpenHint(path: path) } func cancelPipelineUnlessAwaitingResult() { guard !isAwaitingFlowResult else { return } if isFlowRecording || isPendingFlowStart { if isFlowRecording { FlowSessionBridge.setRecordingState(.aborted) ExtensionScreenWakeLock.release() } isFlowRecording = false isPendingFlowStart = false stopUtteranceCountdown() stopFlowWatchdog() state.level = 0 } } // MARK: - Private private func consumePendingFlowDeliveryIfNeeded() { if isAwaitingFlowResult { if let delivery = FlowSessionBridge.consumeTranscriptionDelivery() { isAwaitingFlowResult = false stopFlowWatchdog() textInserter.handleFlowTranscript(delivery) return } if let error = FlowSessionBridge.consumeTranscriptionError() { isAwaitingFlowResult = false stopFlowWatchdog() state.phase = .error( .fromFlowTranscription(error), message: error.message ) scheduleAutoClearError() return } } if isPendingFlowStart, FlowSessionBridge.isHostReachable() { completeFlowStartHandoff() } } /// When the host process died mid-utterance, abort local recording / waiting /// so the user is not stuck until the long result watchdog fires. private func recoverFromDeadHostIfNeeded() { guard FlowSessionBridge.isHostStale() else { return } if isFlowRecording { isFlowRecording = false stopUtteranceCountdown() ExtensionScreenWakeLock.release() FlowSessionBridge.setRecordingState(.aborted) stopFlowWatchdog() state.level = 0 state.phase = .idle state.lastTranscript = "" debug("aborted recording — host heartbeat zombie") return } if isAwaitingFlowResult { failHostDisconnected() } } /// Restores the pre-ABCD behaviour: when the keyboard appears and the host /// is not reachable, automatically jump to the main app to start Flow. private func maybeAutoStartFlowSession() { guard !FlowSessionBridge.isHostReachable() else { return } guard !isPendingFlowStart, !isFlowRecording, !isAwaitingFlowResult else { return } guard hasFullAccess(), AppGroup.isAvailable else { return } guard case .idle = state.phase else { return } let now = Date().timeIntervalSince1970 guard now - lastFlowAutoStartAttempt >= Self.flowAutoStartCooldown else { return } lastFlowAutoStartAttempt = now beginFlowStart() } private func failHostDisconnected() { isAwaitingFlowResult = false isFlowRecording = false isPendingFlowStart = false stopUtteranceCountdown() ExtensionScreenWakeLock.release() FlowSessionBridge.setRecordingState(.aborted) stopFlowWatchdog() state.level = 0 let message = ExtL10n.string("keyboard.flow.hostDisconnected") state.phase = .error(.flowSessionExpired, message: message) scheduleAutoClearError() debug("host disconnected while awaiting Flow result") } private func showFlowSessionExpiredHint() { let message = ExtL10n.string("keyboard.flow.sessionExpired") state.phase = .error(.flowSessionExpired, message: message) scheduleAutoClearError() } private func showManualOpenHint(path: String) { let msg: String if !hasFullAccess() { msg = ExtL10n.string("keyboard.error.fullAccessForJump") } else if path == "settings" { msg = ExtL10n.string("keyboard.error.manualOpenSettings") } else if path == "startflow" { msg = ExtL10n.string("keyboard.error.manualOpenForFlow") } else { msg = ExtL10n.string("keyboard.error.manualOpenSettings") } state.phase = .error(.manualOpenRequired, message: msg) scheduleAutoClearError() } private func startFlowRecording() { guard FlowSessionBridge.isHostReachable() else { beginFlowStart() return } isPendingFlowStart = false flowStartDeadline = 0 stopFlowWatchdog() FlowSessionBridge.setTranscriptionLanguage(state.localeId) FlowSessionBridge.setRecordingState(.recording) isFlowRecording = true state.lastTranscript = "" state.phase = .recording if let view = wakeLockView() { ExtensionScreenWakeLock.acquire(from: view) } startUtteranceCountdown() startFlowLevelWatchdog() // Read back in-process to confirm the write landed before we rely on // the host polling it out cross-process. debug("startFlowRecording wrote .recording (readback=\(FlowSessionBridge.recordingState().rawValue))") } private func startUtteranceCountdown() { utteranceStartedAt = Date().timeIntervalSince1970 state.utteranceRemainingSeconds = Int(FlowSessionKeys.maxUtteranceDuration) utteranceTimerTask?.cancel() utteranceTimerTask = Task { @MainActor [weak self] in while let self, self.isFlowRecording, !Task.isCancelled { let elapsed = Date().timeIntervalSince1970 - self.utteranceStartedAt let remaining = max(0, Int(ceil(FlowSessionKeys.maxUtteranceDuration - elapsed))) self.state.utteranceRemainingSeconds = remaining if remaining <= 0 { self.pressEnded() return } try? await Task.sleep(nanoseconds: 200_000_000) } } } private func stopUtteranceCountdown() { utteranceTimerTask?.cancel() utteranceTimerTask = nil state.utteranceRemainingSeconds = Int(FlowSessionKeys.maxUtteranceDuration) } private func cancelPendingFlowStart() { isPendingFlowStart = false flowStartDeadline = 0 stopFlowWatchdog() state.phase = .idle state.lastTranscript = "" } private func startFlowStartWatchdog() { stopFlowWatchdog() flowWatchdogTask = Task { @MainActor [weak self] in while let self, !Task.isCancelled, self.isPendingFlowStart { if FlowSessionBridge.isHostReachable() { self.completeFlowStartHandoff() return } let now = Date().timeIntervalSince1970 if self.flowStartDeadline > 0, now > self.flowStartDeadline { self.isPendingFlowStart = false self.flowStartDeadline = 0 self.showManualOpenHint(path: "startflow") return } try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs) } } } private func completeFlowStartHandoff() { isPendingFlowStart = false flowStartDeadline = 0 stopFlowWatchdog() state.lastTranscript = "" refreshSessionState() startFlowRecording() debug("completeFlowStartHandoff → auto startFlowRecording") } private func startFlowLevelWatchdog() { stopFlowWatchdog() flowWatchdogTask = Task { @MainActor [weak self] in while let self, !Task.isCancelled, self.isFlowRecording { let levels = FlowSessionBridge.audioLevels() if let peak = levels.max(), peak > 0 { self.state.level = Double(peak) } self.refreshFlowPartialIfNeeded() try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs) } } } private func refreshFlowPartialIfNeeded() { guard isFlowRecording || isAwaitingFlowResult else { return } switch state.phase { case .recording, .processing: if let partial = FlowSessionBridge.transcriptionPartial() { state.lastTranscript = partial } default: break } } private func startFlowResultWatchdog() { stopFlowWatchdog() isAwaitingFlowResult = true let startedAt = Date().timeIntervalSince1970 let resultTimeout = FlowWatchdog.resultTimeout(engineMode: state.engineMode) debug("resultWatchdog started timeout=\(Int(resultTimeout))s engine=\(state.engineMode)") flowWatchdogTask = Task { @MainActor [weak self] in while let self, !Task.isCancelled { if let delivery = FlowSessionBridge.consumeTranscriptionDelivery() { self.isAwaitingFlowResult = false self.stopFlowWatchdog() self.debug("resultWatchdog consumed delivery len=\(delivery.text.count)") self.textInserter.handleFlowTranscript(delivery) return } if let error = FlowSessionBridge.consumeTranscriptionError() { self.isAwaitingFlowResult = false self.stopFlowWatchdog() self.debug("resultWatchdog consumed error kind=\(error.kind.rawValue)") self.state.phase = .error( .fromFlowTranscription(error), message: error.message ) self.scheduleAutoClearError() return } self.refreshFlowPartialIfNeeded() let now = Date().timeIntervalSince1970 let staleness = FlowSessionBridge.heartbeatStaleness() ?? .infinity if staleness > FlowSessionKeys.heartbeatZombieInterval { self.debug("resultWatchdog: host heartbeat zombie (staleness=\(String(format: "%.1f", staleness))s)") self.failHostDisconnected() return } if !FlowSessionBridge.isHostReachable(), now - startedAt > FlowSessionKeys.keyboardHostDisconnectFailFast { self.debug("resultWatchdog: host unreachable after \(String(format: "%.1f", now - startedAt))s") self.failHostDisconnected() return } if now - startedAt > resultTimeout { self.isAwaitingFlowResult = false self.stopFlowWatchdog() self.debug("resultWatchdog TIMEOUT after \(Int(resultTimeout))s — no result from host") let msg = ExtL10n.string("keyboard.flow.resultTimeout") self.state.phase = .error(.flowResultTimeout, message: msg) self.scheduleAutoClearError() return } try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs) } } } private func stopFlowWatchdog() { flowWatchdogTask?.cancel() flowWatchdogTask = nil } private func debug(_ message: String) { OSGLog.keyboardExt.info("\(message, privacy: .public)") } }