perf(asr): speed up local Flow dictation and land CLM/keyboard refactor
Reduce perceived latency from key release to final text: - Adaptive chunking: 2.5s first chunk + 5s follow-ups so short utterances start on-device recognition while still recording. - Session-level ASR warmup and audio-format cache reuse to remove per-utterance cold-start of SpeechAnalyzer. - Mirror live pipelined partials to the keyboard transcript line via a new flow.transcriptionPartial App Group key + Darwin ping. Also commits the accumulated custom language model, Flow session, keyboard extension restructure, and Xiaomi MiMo provider work in progress on this branch.
This commit is contained in:
@@ -2,15 +2,31 @@
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// TypeWhisper-style Flow session bridge: keyboard writes recording
|
||||
// signals; host app writes transcription results. Legacy one-shot
|
||||
// dictation handoff remains in `DictationBridge`.
|
||||
// signals; host app writes transcription results.
|
||||
|
||||
import Foundation
|
||||
|
||||
public struct FlowTranscriptionError: Equatable, Sendable {
|
||||
public let message: String
|
||||
public let kind: FlowSessionKeys.TranscriptionErrorKind
|
||||
|
||||
public init(message: String, kind: FlowSessionKeys.TranscriptionErrorKind) {
|
||||
self.message = message
|
||||
self.kind = kind
|
||||
}
|
||||
}
|
||||
|
||||
public enum FlowSessionBridge {
|
||||
private static func resolvedDefaults(_ defaults: UserDefaults?) -> UserDefaults {
|
||||
if let defaults { return defaults }
|
||||
return AppGroup.isAvailable ? AppGroup.defaults : .standard
|
||||
guard let available = AppGroup.defaultsIfAvailable else {
|
||||
#if DEBUG
|
||||
fatalError("App Group unavailable — inject UserDefaults in tests or fix entitlements.")
|
||||
#else
|
||||
fatalError("App Group unavailable.")
|
||||
#endif
|
||||
}
|
||||
return available
|
||||
}
|
||||
|
||||
/// Force cross-process visibility. Must only be called on the main thread.
|
||||
@@ -70,7 +86,6 @@ public enum FlowSessionBridge {
|
||||
/// background while the continuous audio session is frozen.
|
||||
public static func isSessionActive(defaults: UserDefaults? = nil) -> Bool {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
guard store.bool(forKey: FlowSessionKeys.flowSessionActive) else { return false }
|
||||
|
||||
let expires = store.double(forKey: FlowSessionKeys.flowSessionExpires)
|
||||
@@ -81,7 +96,6 @@ public enum FlowSessionBridge {
|
||||
/// actively processing). Used for auto-start heuristics, not gating record.
|
||||
public static func isHostReachable(defaults: UserDefaults? = nil) -> Bool {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
guard isSessionActive(defaults: store) else { return false }
|
||||
|
||||
let heartbeat = store.double(forKey: FlowSessionKeys.flowHeartbeat)
|
||||
@@ -144,6 +158,7 @@ public enum FlowSessionBridge {
|
||||
let store = resolvedDefaults(defaults)
|
||||
store.set(trimmed, forKey: FlowSessionKeys.transcriptionResult)
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionError)
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionPartial)
|
||||
if let polishWarning, !polishWarning.isEmpty {
|
||||
store.set(polishWarning, forKey: FlowSessionKeys.transcriptionPolishWarning)
|
||||
} else {
|
||||
@@ -151,16 +166,46 @@ public enum FlowSessionBridge {
|
||||
}
|
||||
setRecordingState(.idle, defaults: store)
|
||||
flush(store)
|
||||
FlowSessionDarwin.postTranscriptionChanged()
|
||||
}
|
||||
|
||||
/// Host app: publish pipelined ASR partial while recording or finalizing.
|
||||
public static func storeTranscriptionPartial(
|
||||
_ text: String,
|
||||
defaults: UserDefaults? = nil
|
||||
) {
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let store = resolvedDefaults(defaults)
|
||||
if trimmed.isEmpty {
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionPartial)
|
||||
} else {
|
||||
store.set(trimmed, forKey: FlowSessionKeys.transcriptionPartial)
|
||||
}
|
||||
flush(store)
|
||||
FlowSessionDarwin.postTranscriptionChanged()
|
||||
}
|
||||
|
||||
/// Keyboard: read the latest partial without clearing it.
|
||||
public static func transcriptionPartial(defaults: UserDefaults? = nil) -> String? {
|
||||
let store = resolvedDefaults(defaults)
|
||||
guard let text = store.string(forKey: FlowSessionKeys.transcriptionPartial),
|
||||
!text.isEmpty else {
|
||||
return nil
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
public static func storeTranscriptionError(
|
||||
_ message: String,
|
||||
kind: FlowSessionKeys.TranscriptionErrorKind = .generic,
|
||||
defaults: UserDefaults? = nil
|
||||
) {
|
||||
let store = resolvedDefaults(defaults)
|
||||
store.set(message, forKey: FlowSessionKeys.transcriptionError)
|
||||
store.set(kind.rawValue, forKey: FlowSessionKeys.transcriptionErrorKind)
|
||||
setRecordingState(.idle, defaults: store)
|
||||
flush(store)
|
||||
FlowSessionDarwin.postTranscriptionChanged()
|
||||
}
|
||||
|
||||
/// Returns and clears a pending transcription result, if any.
|
||||
@@ -174,7 +219,6 @@ public enum FlowSessionBridge {
|
||||
defaults: UserDefaults? = nil
|
||||
) -> TranscriptionDelivery? {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
guard let text = store.string(forKey: FlowSessionKeys.transcriptionResult), !text.isEmpty else {
|
||||
return nil
|
||||
}
|
||||
@@ -186,20 +230,21 @@ public enum FlowSessionBridge {
|
||||
}
|
||||
|
||||
/// Returns and clears a pending transcription error, if any.
|
||||
public static func consumeTranscriptionError(defaults: UserDefaults? = nil) -> String? {
|
||||
public static func consumeTranscriptionError(defaults: UserDefaults? = nil) -> FlowTranscriptionError? {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
guard let message = store.string(forKey: FlowSessionKeys.transcriptionError), !message.isEmpty else {
|
||||
return nil
|
||||
}
|
||||
let kindRaw = store.string(forKey: FlowSessionKeys.transcriptionErrorKind)
|
||||
let kind = FlowSessionKeys.TranscriptionErrorKind(rawValue: kindRaw ?? "") ?? .generic
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionError)
|
||||
store.removeObject(forKey: FlowSessionKeys.transcriptionErrorKind)
|
||||
flush(store)
|
||||
return message
|
||||
return FlowTranscriptionError(message: message, kind: kind)
|
||||
}
|
||||
|
||||
public static func audioLevels(defaults: UserDefaults? = nil) -> [Float] {
|
||||
let store = resolvedDefaults(defaults)
|
||||
flush(store)
|
||||
if let levels = store.array(forKey: FlowSessionKeys.audioLevels) as? [Double], !levels.isEmpty {
|
||||
return levels.map { Float($0) }
|
||||
}
|
||||
@@ -240,7 +285,9 @@ public enum FlowSessionBridge {
|
||||
|
||||
private static func clearTranscription(defaults: UserDefaults) {
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionResult)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionPartial)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionPolishWarning)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionError)
|
||||
defaults.removeObject(forKey: FlowSessionKeys.transcriptionErrorKind)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user