perf(asr): speed up local Flow dictation and land CLM/keyboard refactor
Reduce perceived latency from key release to final text: - Adaptive chunking: 2.5s first chunk + 5s follow-ups so short utterances start on-device recognition while still recording. - Session-level ASR warmup and audio-format cache reuse to remove per-utterance cold-start of SpeechAnalyzer. - Mirror live pipelined partials to the keyboard transcript line via a new flow.transcriptionPartial App Group key + Darwin ping. Also commits the accumulated custom language model, Flow session, keyboard extension restructure, and Xiaomi MiMo provider work in progress on this branch.
This commit is contained in:
Rocky
@@ -0,0 +1,412 @@
|
||||
// KeyboardFlowCoordinator.swift
|
||||
// OSGKeyboard · Keyboard Extension
|
||||
//
|
||||
// Flow session start, recording, watchdogs, and result delivery handling.
|
||||
|
||||
import UIKit
|
||||
import OSGKeyboardShared
|
||||
|
||||
@MainActor
|
||||
final class KeyboardFlowCoordinator {
|
||||
private enum FlowWatchdog {
|
||||
static let pollIntervalNs: UInt64 = 200_000_000
|
||||
/// Give the user time to manually open the host app when auto-jump fails.
|
||||
static let startTimeout: TimeInterval = 30
|
||||
|
||||
static func resultTimeout(engineMode: String) -> TimeInterval {
|
||||
FlowSessionKeys.keyboardResultTimeout(engineMode: engineMode)
|
||||
}
|
||||
}
|
||||
|
||||
private let state: KeyboardState
|
||||
private let textInserter: KeyboardTextInserter
|
||||
private let hasFullAccess: () -> Bool
|
||||
private let wakeLockView: () -> UIView?
|
||||
private let openHostApp: (String) -> Void
|
||||
private let detectAndStoreAppContext: () -> Void
|
||||
private let scheduleAutoClearError: () -> Void
|
||||
private let refreshConfigFromAppGroup: () -> Void
|
||||
|
||||
private var isPendingFlowStart = false
|
||||
private var flowStartDeadline: TimeInterval = 0
|
||||
private var isFlowRecording = false
|
||||
private var flowWatchdogTask: Task<Void, Never>?
|
||||
private var utteranceTimerTask: Task<Void, Never>?
|
||||
private var utteranceStartedAt: TimeInterval = 0
|
||||
private var wasFlowSessionActive = false
|
||||
private var flowSessionMonitorTask: Task<Void, Never>?
|
||||
private var isAwaitingFlowResult = false
|
||||
private var lastFlowAutoStartAttempt: TimeInterval = 0
|
||||
private static let flowAutoStartCooldown: TimeInterval = 20
|
||||
|
||||
init(
|
||||
state: KeyboardState,
|
||||
textInserter: KeyboardTextInserter,
|
||||
hasFullAccess: @escaping () -> Bool,
|
||||
wakeLockView: @escaping () -> UIView?,
|
||||
openHostApp: @escaping (String) -> Void,
|
||||
detectAndStoreAppContext: @escaping () -> Void,
|
||||
scheduleAutoClearError: @escaping () -> Void,
|
||||
refreshConfigFromAppGroup: @escaping () -> Void
|
||||
) {
|
||||
self.state = state
|
||||
self.textInserter = textInserter
|
||||
self.hasFullAccess = hasFullAccess
|
||||
self.wakeLockView = wakeLockView
|
||||
self.openHostApp = openHostApp
|
||||
self.detectAndStoreAppContext = detectAndStoreAppContext
|
||||
self.scheduleAutoClearError = scheduleAutoClearError
|
||||
self.refreshConfigFromAppGroup = refreshConfigFromAppGroup
|
||||
}
|
||||
|
||||
var preservesLifecycleOnDisappear: Bool {
|
||||
isPendingFlowStart || isFlowRecording || isAwaitingFlowResult
|
||||
}
|
||||
|
||||
func startSessionMonitor() {
|
||||
flowSessionMonitorTask?.cancel()
|
||||
flowSessionMonitorTask = Task { @MainActor [weak self] in
|
||||
while !Task.isCancelled {
|
||||
self?.refreshSessionState()
|
||||
try? await Task.sleep(nanoseconds: 1_000_000_000)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stopSessionMonitor() {
|
||||
flowSessionMonitorTask?.cancel()
|
||||
flowSessionMonitorTask = nil
|
||||
}
|
||||
|
||||
func refreshSessionState() {
|
||||
refreshConfigFromAppGroup()
|
||||
refreshFlowPartialIfNeeded()
|
||||
consumePendingFlowDeliveryIfNeeded()
|
||||
|
||||
let active = FlowSessionBridge.isSessionActive()
|
||||
state.flowSessionActive = active
|
||||
|
||||
if wasFlowSessionActive && !active && !isFlowRecording && !isPendingFlowStart {
|
||||
switch state.phase {
|
||||
case .recording, .processing:
|
||||
break
|
||||
default:
|
||||
showFlowSessionExpiredHint()
|
||||
}
|
||||
}
|
||||
wasFlowSessionActive = active
|
||||
|
||||
if !active {
|
||||
maybeAutoStartFlowSession()
|
||||
}
|
||||
}
|
||||
|
||||
func toggleRecording() {
|
||||
switch state.phase {
|
||||
case .recording:
|
||||
pressEnded()
|
||||
case .idle, .denied, .error:
|
||||
pressBegan()
|
||||
case .requestingPermissions, .processing:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
func pressBegan() {
|
||||
switch state.phase {
|
||||
case .idle, .denied, .error:
|
||||
break
|
||||
default:
|
||||
return
|
||||
}
|
||||
guard !state.micDisabled else { return }
|
||||
guard hasFullAccess() else {
|
||||
let msg = ExtL10n.string("keyboard.error.fullAccessRequired")
|
||||
state.phase = .error(.fullAccessRequired, message: msg)
|
||||
scheduleAutoClearError()
|
||||
return
|
||||
}
|
||||
guard AppGroup.isAvailable else {
|
||||
let msg = ExtL10n.string("keyboard.error.appGroupCommunication")
|
||||
state.phase = .error(.appGroupUnavailable, message: msg)
|
||||
scheduleAutoClearError()
|
||||
return
|
||||
}
|
||||
|
||||
detectAndStoreAppContext()
|
||||
|
||||
if FlowSessionBridge.isSessionActive() {
|
||||
startFlowRecording()
|
||||
} else {
|
||||
beginFlowStart()
|
||||
}
|
||||
}
|
||||
|
||||
func pressEnded() {
|
||||
if isPendingFlowStart {
|
||||
cancelPendingFlowStart()
|
||||
return
|
||||
}
|
||||
guard isFlowRecording else { return }
|
||||
|
||||
isFlowRecording = false
|
||||
stopUtteranceCountdown()
|
||||
ExtensionScreenWakeLock.release()
|
||||
FlowSessionBridge.setRecordingState(.stopped)
|
||||
state.phase = .processing
|
||||
state.lastTranscript = ExtL10n.string("keyboard.flow.transcribing")
|
||||
startFlowResultWatchdog()
|
||||
}
|
||||
|
||||
func beginFlowStart() {
|
||||
guard !isPendingFlowStart else { return }
|
||||
isPendingFlowStart = true
|
||||
isFlowRecording = false
|
||||
flowStartDeadline = Date().timeIntervalSince1970 + FlowWatchdog.startTimeout
|
||||
state.lastTranscript = ExtL10n.string("keyboard.flow.startingSession")
|
||||
state.phase = .processing
|
||||
openHostApp("startflow")
|
||||
startFlowStartWatchdog()
|
||||
debug("beginFlowStart")
|
||||
}
|
||||
|
||||
func handleHostAppOpenResult(path: String, success: Bool) {
|
||||
debug("openHostApp path=\(path) success=\(success)")
|
||||
guard !success else { return }
|
||||
|
||||
if path == "startflow", isPendingFlowStart {
|
||||
state.lastTranscript = ExtL10n.string("keyboard.flow.manualOpenHost")
|
||||
return
|
||||
}
|
||||
|
||||
showManualOpenHint(path: path)
|
||||
}
|
||||
|
||||
func cancelPipelineUnlessAwaitingResult() {
|
||||
guard !isAwaitingFlowResult else { return }
|
||||
if isFlowRecording || isPendingFlowStart {
|
||||
if isFlowRecording {
|
||||
FlowSessionBridge.setRecordingState(.aborted)
|
||||
ExtensionScreenWakeLock.release()
|
||||
}
|
||||
isFlowRecording = false
|
||||
isPendingFlowStart = false
|
||||
stopUtteranceCountdown()
|
||||
stopFlowWatchdog()
|
||||
state.level = 0
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Private
|
||||
|
||||
private func consumePendingFlowDeliveryIfNeeded() {
|
||||
if isAwaitingFlowResult {
|
||||
if let delivery = FlowSessionBridge.consumeTranscriptionDelivery() {
|
||||
isAwaitingFlowResult = false
|
||||
stopFlowWatchdog()
|
||||
textInserter.handleFlowTranscript(delivery)
|
||||
return
|
||||
}
|
||||
if let error = FlowSessionBridge.consumeTranscriptionError() {
|
||||
isAwaitingFlowResult = false
|
||||
stopFlowWatchdog()
|
||||
state.phase = .error(
|
||||
.fromFlowTranscription(error),
|
||||
message: error.message
|
||||
)
|
||||
scheduleAutoClearError()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if isPendingFlowStart, FlowSessionBridge.isSessionActive() {
|
||||
completeFlowStartHandoff()
|
||||
}
|
||||
}
|
||||
|
||||
private func maybeAutoStartFlowSession() {
|
||||
guard !FlowSessionBridge.isSessionActive() else { return }
|
||||
guard !isPendingFlowStart, !isFlowRecording, !isAwaitingFlowResult else { return }
|
||||
guard hasFullAccess(), AppGroup.isAvailable else { return }
|
||||
guard case .idle = state.phase else { return }
|
||||
|
||||
let now = Date().timeIntervalSince1970
|
||||
guard now - lastFlowAutoStartAttempt >= Self.flowAutoStartCooldown else { return }
|
||||
lastFlowAutoStartAttempt = now
|
||||
beginFlowStart()
|
||||
}
|
||||
|
||||
private func showFlowSessionExpiredHint() {
|
||||
let message = ExtL10n.string("keyboard.flow.sessionExpired")
|
||||
state.phase = .error(.flowSessionExpired, message: message)
|
||||
scheduleAutoClearError()
|
||||
}
|
||||
|
||||
private func showManualOpenHint(path: String) {
|
||||
let msg: String
|
||||
if !hasFullAccess() {
|
||||
msg = ExtL10n.string("keyboard.error.fullAccessForJump")
|
||||
} else if path == "settings" {
|
||||
msg = ExtL10n.string("keyboard.error.manualOpenSettings")
|
||||
} else if path == "startflow" {
|
||||
msg = ExtL10n.string("keyboard.error.manualOpenForFlow")
|
||||
} else {
|
||||
msg = ExtL10n.string("keyboard.error.manualOpenSettings")
|
||||
}
|
||||
state.phase = .error(.manualOpenRequired, message: msg)
|
||||
scheduleAutoClearError()
|
||||
}
|
||||
|
||||
private func startFlowRecording() {
|
||||
isPendingFlowStart = false
|
||||
flowStartDeadline = 0
|
||||
stopFlowWatchdog()
|
||||
|
||||
FlowSessionBridge.setTranscriptionLanguage(state.localeId)
|
||||
FlowSessionBridge.setRecordingState(.recording)
|
||||
isFlowRecording = true
|
||||
state.lastTranscript = ""
|
||||
state.phase = .recording
|
||||
if let view = wakeLockView() {
|
||||
ExtensionScreenWakeLock.acquire(from: view)
|
||||
}
|
||||
startUtteranceCountdown()
|
||||
startFlowLevelWatchdog()
|
||||
debug("startFlowRecording")
|
||||
}
|
||||
|
||||
private func startUtteranceCountdown() {
|
||||
utteranceStartedAt = Date().timeIntervalSince1970
|
||||
state.utteranceRemainingSeconds = Int(FlowSessionKeys.maxUtteranceDuration)
|
||||
utteranceTimerTask?.cancel()
|
||||
utteranceTimerTask = Task { @MainActor [weak self] in
|
||||
while let self, self.isFlowRecording, !Task.isCancelled {
|
||||
let elapsed = Date().timeIntervalSince1970 - self.utteranceStartedAt
|
||||
let remaining = max(0, Int(ceil(FlowSessionKeys.maxUtteranceDuration - elapsed)))
|
||||
self.state.utteranceRemainingSeconds = remaining
|
||||
if remaining <= 0 {
|
||||
self.pressEnded()
|
||||
return
|
||||
}
|
||||
try? await Task.sleep(nanoseconds: 200_000_000)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func stopUtteranceCountdown() {
|
||||
utteranceTimerTask?.cancel()
|
||||
utteranceTimerTask = nil
|
||||
state.utteranceRemainingSeconds = Int(FlowSessionKeys.maxUtteranceDuration)
|
||||
}
|
||||
|
||||
private func cancelPendingFlowStart() {
|
||||
isPendingFlowStart = false
|
||||
flowStartDeadline = 0
|
||||
stopFlowWatchdog()
|
||||
state.phase = .idle
|
||||
state.lastTranscript = ""
|
||||
}
|
||||
|
||||
private func startFlowStartWatchdog() {
|
||||
stopFlowWatchdog()
|
||||
flowWatchdogTask = Task { @MainActor [weak self] in
|
||||
while let self, !Task.isCancelled, self.isPendingFlowStart {
|
||||
if FlowSessionBridge.isSessionActive() {
|
||||
self.completeFlowStartHandoff()
|
||||
return
|
||||
}
|
||||
let now = Date().timeIntervalSince1970
|
||||
if self.flowStartDeadline > 0, now > self.flowStartDeadline {
|
||||
self.isPendingFlowStart = false
|
||||
self.flowStartDeadline = 0
|
||||
self.showManualOpenHint(path: "startflow")
|
||||
return
|
||||
}
|
||||
try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func completeFlowStartHandoff() {
|
||||
isPendingFlowStart = false
|
||||
flowStartDeadline = 0
|
||||
stopFlowWatchdog()
|
||||
state.lastTranscript = ""
|
||||
state.phase = .idle
|
||||
refreshSessionState()
|
||||
debug("completeFlowStartHandoff")
|
||||
}
|
||||
|
||||
private func startFlowLevelWatchdog() {
|
||||
stopFlowWatchdog()
|
||||
flowWatchdogTask = Task { @MainActor [weak self] in
|
||||
while let self, !Task.isCancelled, self.isFlowRecording {
|
||||
let levels = FlowSessionBridge.audioLevels()
|
||||
if let peak = levels.max(), peak > 0 {
|
||||
self.state.level = Double(peak)
|
||||
}
|
||||
self.refreshFlowPartialIfNeeded()
|
||||
try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func refreshFlowPartialIfNeeded() {
|
||||
guard isFlowRecording || isAwaitingFlowResult else { return }
|
||||
switch state.phase {
|
||||
case .recording, .processing:
|
||||
if let partial = FlowSessionBridge.transcriptionPartial() {
|
||||
state.lastTranscript = partial
|
||||
}
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
private func startFlowResultWatchdog() {
|
||||
stopFlowWatchdog()
|
||||
isAwaitingFlowResult = true
|
||||
let startedAt = Date().timeIntervalSince1970
|
||||
let resultTimeout = FlowWatchdog.resultTimeout(engineMode: state.engineMode)
|
||||
flowWatchdogTask = Task { @MainActor [weak self] in
|
||||
while let self, !Task.isCancelled {
|
||||
if let delivery = FlowSessionBridge.consumeTranscriptionDelivery() {
|
||||
self.isAwaitingFlowResult = false
|
||||
self.stopFlowWatchdog()
|
||||
self.textInserter.handleFlowTranscript(delivery)
|
||||
return
|
||||
}
|
||||
if let error = FlowSessionBridge.consumeTranscriptionError() {
|
||||
self.isAwaitingFlowResult = false
|
||||
self.stopFlowWatchdog()
|
||||
self.state.phase = .error(
|
||||
.fromFlowTranscription(error),
|
||||
message: error.message
|
||||
)
|
||||
self.scheduleAutoClearError()
|
||||
return
|
||||
}
|
||||
self.refreshFlowPartialIfNeeded()
|
||||
let now = Date().timeIntervalSince1970
|
||||
if now - startedAt > resultTimeout {
|
||||
self.isAwaitingFlowResult = false
|
||||
self.stopFlowWatchdog()
|
||||
let msg = ExtL10n.string("keyboard.flow.resultTimeout")
|
||||
self.state.phase = .error(.flowResultTimeout, message: msg)
|
||||
self.scheduleAutoClearError()
|
||||
return
|
||||
}
|
||||
try? await Task.sleep(nanoseconds: FlowWatchdog.pollIntervalNs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func stopFlowWatchdog() {
|
||||
flowWatchdogTask?.cancel()
|
||||
flowWatchdogTask = nil
|
||||
}
|
||||
|
||||
private func debug(_ message: String) {
|
||||
OSGLog.keyboardExt.info("\(message, privacy: .public)")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user