perf(asr): speed up local Flow dictation and land CLM/keyboard refactor

Reduce perceived latency from key release to final text:
- Adaptive chunking: 2.5s first chunk + 5s follow-ups so short
  utterances start on-device recognition while still recording.
- Session-level ASR warmup and audio-format cache reuse to remove
  per-utterance cold-start of SpeechAnalyzer.
- Mirror live pipelined partials to the keyboard transcript line via
  a new flow.transcriptionPartial App Group key + Darwin ping.

Also commits the accumulated custom language model, Flow session,
keyboard extension restructure, and Xiaomi MiMo provider work in
progress on this branch.
This commit is contained in:
Rocky
2026-07-06 00:00:19 +08:00
parent cfbfb542cc
commit 537a68552a
76 changed files with 3456 additions and 121086 deletions
@@ -13,9 +13,13 @@ public enum FlowSessionKeys {
public static let keyboardRecordingState = "flow.keyboardRecordingState"
public static let transcriptionLanguage = "flow.transcriptionLanguage"
public static let transcriptionResult = "flow.transcriptionResult"
/// Live pipelined ASR partial for the keyboard transcript line.
public static let transcriptionPartial = "flow.transcriptionPartial"
/// Soft warning when polish failed but raw transcript was delivered.
public static let transcriptionPolishWarning = "flow.transcriptionPolishWarning"
public static let transcriptionError = "flow.transcriptionError"
/// Structured kind paired with `transcriptionError` for keyboard UI.
public static let transcriptionErrorKind = "flow.transcriptionErrorKind"
public static let audioLevels = "flow.audioLevels"
/// Heartbeat older than this while the host is foreground likely killed.
@@ -36,15 +40,7 @@ public enum FlowSessionKeys {
/// Keyboard watchdog after the user stops recording (not utterance max length).
/// Must cover worst-case post-stop backlog: remaining SpeechAnalyzer chunks
/// plus cloud LLM polish (see `PolishingService.effectiveTimeout` cap).
///
/// As of v0.2.0 the local engine uses iOS `SpeechAnalyzer` only, so the
/// previous Qwen3-specific timeout (240 s) collapses into the shared
/// local path. We keep `localASRBackend` on the signature for symmetry
/// with other shared helpers.
public static func keyboardResultTimeout(
engineMode: String,
localASRBackend: LocalASRBackend
) -> TimeInterval {
public static func keyboardResultTimeout(engineMode: String) -> TimeInterval {
if engineMode == "local" {
return 180
}
@@ -58,4 +54,13 @@ public enum FlowSessionKeys {
case processing
case aborted
}
/// Structured host keyboard transcription failure kind.
public enum TranscriptionErrorKind: String, Sendable, Equatable {
case noSpeech
case recognitionInterrupted
case audioUnavailable
case asrFailed
case generic
}
}