perf(asr): speed up local Flow dictation and land CLM/keyboard refactor

Reduce perceived latency from key release to final text:
- Adaptive chunking: 2.5s first chunk + 5s follow-ups so short
  utterances start on-device recognition while still recording.
- Session-level ASR warmup and audio-format cache reuse to remove
  per-utterance cold-start of SpeechAnalyzer.
- Mirror live pipelined partials to the keyboard transcript line via
  a new flow.transcriptionPartial App Group key + Darwin ping.

Also commits the accumulated custom language model, Flow session,
keyboard extension restructure, and Xiaomi MiMo provider work in
progress on this branch.
This commit is contained in:
Rocky
2026-07-06 00:00:19 +08:00
parent cfbfb542cc
commit 537a68552a
76 changed files with 3456 additions and 121086 deletions
+36 -27
View File
@@ -35,26 +35,37 @@ public final class KeyboardState: ObservableObject {
case asr(String)
case llm(LLMError)
case appGroupUnavailable
/// Keyboard extension lacks Full Access for host-app jumps.
case fullAccessRequired
/// Auto-jump to the host app failed; user must open it manually.
case manualOpenRequired
/// Host delivered raw transcript; polish step failed or was skipped.
case polishDegraded(String)
/// Host ASR finished with no usable speech.
case noSpeechDetected
/// Host ASR was interrupted before a final transcript arrived.
case recognitionInterrupted
/// Host could not start background audio capture.
case hostAudioUnavailable
/// Host ASR or pipeline failed with a user-facing message.
case hostTranscriptionFailed(String)
/// Flow result did not arrive before the keyboard watchdog expired.
case flowResultTimeout
/// Host Flow session ended while the keyboard was idle.
case flowSessionExpired
case unknown(String)
}
public enum Reason: Equatable { case mic, speech }
}
/// Voice input always runs through polish; legacy off/transcribe modes removed.
public enum InputMode: String, CaseIterable, Identifiable {
case off
case transcribe
case polish
public var id: String { rawValue }
public var labelKey: String {
switch self {
case .off: return "mode.off"
case .transcribe: return "mode.transcribe"
case .polish: return "mode.polish"
}
}
public var labelKey: String { "mode.polish" }
}
@Published public var phase: Phase = .idle
@@ -78,20 +89,6 @@ public final class KeyboardState: ObservableObject {
@Published public var micDisabledHint: String = ""
/// "local" on-device ASR only. "cloud" ASR + LLM polish.
@Published public var engineMode: String = "cloud"
/// Which on-device ASR engine to use when `engineMode == "local"`.
/// Mirrored from `ProviderConfig.localASRBackend` for UI display
/// and for `state` consumers that want a single source of truth.
@Published public var localASRBackend: LocalASRBackend = .speechAnalyzer
/// v0.2.0: kept for source compatibility with the previous Qwen3
/// CoreML local engine. Always `true` now iOS `SpeechAnalyzer`
/// ships with iOS 26 and has no per-user weights to download or
/// preload. Existing read sites will see `true` and behave the
/// same as the "stack ready" branch did.
@Published public var localModelsReady: Bool = true
/// v0.2.0: kept for source compatibility with the previous Qwen3
/// CoreML local engine. Always `false` now there are no weights
/// for the host app to preload.
@Published public var localModelsLoaded: Bool = false
/// v0.2.1 follow-up: derived translation is on iff a target
/// locale has been selected (mirrors `ProviderConfig.translationEnabled`
/// so the chip / pipeline read the same source of truth).
@@ -103,9 +100,6 @@ public final class KeyboardState: ObservableObject {
/// Defaults to `offLocaleId` so the keyboard boots in the "off"
/// state on first install.
@Published public var translationTargetLocaleId: String = TranslationLanguageCatalog.offLocaleId
/// v0.2.0: mirrored from App Group kept for source compatibility.
/// Local engine always runs built-in polish; the flag is ignored.
@Published public var localModeCloudPolishEnabled: Bool = true
/// Mirrored from App Group swaps delete / return on the bottom row.
@Published public var handednessPreference: HandednessPreference = .left
/// Press-and-drag pads beside the mic for four-way caret movement.
@@ -159,7 +153,6 @@ public final class KeyboardState: ObservableObject {
public var setMode: (InputMode) -> Void = { _ in }
public var setLocale: (String) -> Void = { _ in }
public var setEngineMode: (String) -> Void = { _ in }
public var setLocalASRBackend: (LocalASRBackend) -> Void = { _ in }
/// v0.2.1 follow-up: only the locale picker remains `enabled`
/// is derived from the locale id, so there's no separate toggle to
/// persist. Wired in `KeyboardViewController.installStateActions`.
@@ -209,4 +202,20 @@ public final class KeyboardState: ObservableObject {
return s
}
#endif
}
extension KeyboardState.Phase.ErrorKind {
/// Maps a host-app Flow transcription failure into a keyboard error kind.
public static func fromFlowTranscription(_ error: FlowTranscriptionError) -> Self {
switch error.kind {
case .noSpeech:
return .noSpeechDetected
case .recognitionInterrupted:
return .recognitionInterrupted
case .audioUnavailable:
return .hostAudioUnavailable
case .asrFailed, .generic:
return .hostTranscriptionFailed(error.message)
}
}
}