perf(asr): speed up local Flow dictation and land CLM/keyboard refactor
Reduce perceived latency from key release to final text: - Adaptive chunking: 2.5s first chunk + 5s follow-ups so short utterances start on-device recognition while still recording. - Session-level ASR warmup and audio-format cache reuse to remove per-utterance cold-start of SpeechAnalyzer. - Mirror live pipelined partials to the keyboard transcript line via a new flow.transcriptionPartial App Group key + Darwin ping. Also commits the accumulated custom language model, Flow session, keyboard extension restructure, and Xiaomi MiMo provider work in progress on this branch.
This commit is contained in:
Rocky
@@ -22,9 +22,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
|
||||
private let capture = FlowContinuousCapture()
|
||||
private let store = AppGroupStore()
|
||||
/// Cloud-engine polish; local engine now ALSO runs through the
|
||||
/// polisher when `localModeCloudPolishEnabled` is on — the same
|
||||
/// `PolishingService` short-circuits to raw when the toggle is off.
|
||||
/// Cloud-engine polish; local engine runs through built-in DeepSeek polish.
|
||||
private var polisher: PolishingService {
|
||||
PolishingService()
|
||||
}
|
||||
@@ -33,12 +31,13 @@ final class FlowSessionManager: ObservableObject {
|
||||
/// factory-built service straight back without going through the
|
||||
/// old `OnDeviceModelWarmup` registry.
|
||||
private var sessionASR: ASRService?
|
||||
/// Tracks which engine mode `sessionASR` was created for.
|
||||
private var sessionASREngineMode: String?
|
||||
/// Locale id last passed to `warmup(locale:)`.
|
||||
private var sessionASRWarmedLocaleID: String?
|
||||
private var asr: ASRService {
|
||||
if let sessionASR { return sessionASR }
|
||||
let service = ASRServiceFactory.make(
|
||||
engineMode: store.engineMode,
|
||||
localBackend: store.localASRBackend
|
||||
)
|
||||
let service = ASRServiceFactory.make()
|
||||
sessionASR = service
|
||||
return service
|
||||
}
|
||||
@@ -155,7 +154,8 @@ final class FlowSessionManager: ObservableObject {
|
||||
// v0.2.0: iOS `SpeechAnalyzer` needs no warm-up. We still
|
||||
// re-bind the cached `sessionASR` so a config flip mid-session
|
||||
// (e.g. switching from cloud to local) is honoured.
|
||||
bindSessionASR()
|
||||
bindSessionASRIfNeeded()
|
||||
scheduleASRWarmup()
|
||||
|
||||
debug("Flow session restored (\(Int(remaining))s remaining)")
|
||||
}
|
||||
@@ -192,6 +192,8 @@ final class FlowSessionManager: ObservableObject {
|
||||
endBackgroundKeepAlive()
|
||||
ScreenWakeLock.release()
|
||||
sessionASR = nil
|
||||
sessionASREngineMode = nil
|
||||
sessionASRWarmedLocaleID = nil
|
||||
FlowSessionBridge.markSessionInactive()
|
||||
FlowSessionDarwin.postSessionChanged()
|
||||
isActive = false
|
||||
@@ -219,13 +221,11 @@ final class FlowSessionManager: ObservableObject {
|
||||
func handleScenePhase(_ phase: ScenePhase) {
|
||||
switch phase {
|
||||
case .active:
|
||||
FlowAppLifecycle.shared.setForeground(true)
|
||||
setAppForeground(true)
|
||||
resumeAfterForeground()
|
||||
case .inactive:
|
||||
writeHeartbeatIfActive()
|
||||
case .background:
|
||||
FlowAppLifecycle.shared.setForeground(false)
|
||||
setAppForeground(false)
|
||||
beginBackgroundKeepAlive()
|
||||
@unknown default:
|
||||
@@ -269,7 +269,8 @@ final class FlowSessionManager: ObservableObject {
|
||||
await self?.reactivateCaptureIfNeeded()
|
||||
// v0.2.0: iOS `SpeechAnalyzer` is bundled with the OS; no
|
||||
// on-device weights to reload after a background trip.
|
||||
self?.bindSessionASR()
|
||||
self?.bindSessionASRIfNeeded()
|
||||
self?.scheduleASRWarmup()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -336,16 +337,39 @@ final class FlowSessionManager: ObservableObject {
|
||||
// v0.2.0: iOS `SpeechAnalyzer` needs no warm-up; just refresh
|
||||
// the cached ASR service in case the user flipped engines
|
||||
// while the session was idle.
|
||||
bindSessionASR()
|
||||
bindSessionASRIfNeeded()
|
||||
scheduleASRWarmup()
|
||||
|
||||
debug("Flow session started (\(Int(duration))s), continuous capture running")
|
||||
}
|
||||
|
||||
private func bindSessionASR() {
|
||||
sessionASR = ASRServiceFactory.make(
|
||||
engineMode: store.engineMode,
|
||||
localBackend: store.localASRBackend
|
||||
)
|
||||
private func bindSessionASRIfNeeded(force: Bool = false) {
|
||||
let engineMode = store.engineMode
|
||||
if !force,
|
||||
let sessionASR,
|
||||
sessionASREngineMode == engineMode {
|
||||
return
|
||||
}
|
||||
sessionASR?.cancel()
|
||||
sessionASR = ASRServiceFactory.make()
|
||||
sessionASREngineMode = engineMode
|
||||
sessionASRWarmedLocaleID = nil
|
||||
}
|
||||
|
||||
private func scheduleASRWarmup() {
|
||||
Task { @MainActor [weak self] in
|
||||
await self?.warmupASRIfNeeded()
|
||||
}
|
||||
}
|
||||
|
||||
private func warmupASRIfNeeded() async {
|
||||
bindSessionASRIfNeeded()
|
||||
let locale = SpeechLocaleResolver.resolve(store.localeId)
|
||||
let localeID = locale.identifier(.bcp47)
|
||||
guard sessionASRWarmedLocaleID != localeID else { return }
|
||||
await asr.warmup(locale: locale)
|
||||
sessionASRWarmedLocaleID = localeID
|
||||
FlowDiagnostics.log("ASR warmup complete locale=\(localeID)")
|
||||
}
|
||||
|
||||
private func permissionWarningMessage() -> String {
|
||||
@@ -384,7 +408,10 @@ final class FlowSessionManager: ObservableObject {
|
||||
|
||||
private func beginUtterance() {
|
||||
guard capture.running else {
|
||||
failUtterance(message: AppL10n.string("flow.error.audioUnavailable"))
|
||||
failUtterance(
|
||||
message: AppL10n.string("flow.error.audioUnavailable"),
|
||||
kind: .audioUnavailable
|
||||
)
|
||||
return
|
||||
}
|
||||
guard !isUtteranceProcessing else {
|
||||
@@ -392,8 +419,8 @@ final class FlowSessionManager: ObservableObject {
|
||||
return
|
||||
}
|
||||
|
||||
// Honor engine / ASR backend changes without restarting the session.
|
||||
bindSessionASR()
|
||||
// Usually already warm from session start; refresh without blocking the mic gate.
|
||||
scheduleASRWarmup()
|
||||
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
@@ -411,7 +438,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
isUtteranceRecording = true
|
||||
utteranceRecordingStartedAt = Date()
|
||||
FlowDiagnostics.log(
|
||||
"beginUtterance engine=\(store.engineMode) asr=\(store.localASRBackend.rawValue) " +
|
||||
"beginUtterance engine=\(store.engineMode) " +
|
||||
"asrType=\(type(of: asr)) pipelined=true max=\(Int(FlowSessionKeys.maxUtteranceDuration))s"
|
||||
)
|
||||
|
||||
@@ -419,6 +446,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
let outcome = await pipeline.transcribe(stream: stream) { partial in
|
||||
Task { @MainActor in
|
||||
manager?.currentPartial = partial
|
||||
FlowSessionBridge.storeTranscriptionPartial(partial)
|
||||
}
|
||||
}
|
||||
// Re-bind `manager` inside the `@MainActor` block so the
|
||||
@@ -439,9 +467,9 @@ final class FlowSessionManager: ObservableObject {
|
||||
case .failure(let message):
|
||||
manager.debug("asr error: \(message)")
|
||||
if manager.isUtteranceRecording {
|
||||
manager.failUtterance(message: message)
|
||||
manager.failUtterance(message: message, kind: .asrFailed)
|
||||
} else if manager.isUtteranceProcessing {
|
||||
manager.finishProcessing(withError: message)
|
||||
manager.finishProcessing(withError: message, kind: .asrFailed)
|
||||
}
|
||||
case .cancelled:
|
||||
break
|
||||
@@ -487,11 +515,15 @@ final class FlowSessionManager: ObservableObject {
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
chunkWarnings = []
|
||||
FlowSessionBridge.storeTranscriptionPartial("")
|
||||
FlowSessionBridge.setRecordingState(.idle)
|
||||
debug("utterance aborted")
|
||||
}
|
||||
|
||||
private func failUtterance(message: String) {
|
||||
private func failUtterance(
|
||||
message: String,
|
||||
kind: FlowSessionKeys.TranscriptionErrorKind = .asrFailed
|
||||
) {
|
||||
isUtteranceRecording = false
|
||||
isUtteranceProcessing = false
|
||||
utteranceRecordingStartedAt = nil
|
||||
@@ -505,12 +537,16 @@ final class FlowSessionManager: ObservableObject {
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
chunkWarnings = []
|
||||
FlowSessionBridge.storeTranscriptionError(message)
|
||||
FlowSessionBridge.storeTranscriptionPartial("")
|
||||
FlowSessionBridge.storeTranscriptionError(message, kind: kind)
|
||||
FlowSessionBridge.setRecordingState(.idle)
|
||||
debug("utterance failed: \(message)")
|
||||
}
|
||||
|
||||
private func finishProcessing(withError message: String) {
|
||||
private func finishProcessing(
|
||||
withError message: String,
|
||||
kind: FlowSessionKeys.TranscriptionErrorKind = .asrFailed
|
||||
) {
|
||||
isUtteranceProcessing = false
|
||||
utteranceRecordingStartedAt = nil
|
||||
finalizeTask?.cancel()
|
||||
@@ -519,7 +555,8 @@ final class FlowSessionManager: ObservableObject {
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
chunkWarnings = []
|
||||
FlowSessionBridge.storeTranscriptionError(message)
|
||||
FlowSessionBridge.storeTranscriptionPartial("")
|
||||
FlowSessionBridge.storeTranscriptionError(message, kind: kind)
|
||||
FlowSessionBridge.setRecordingState(.idle)
|
||||
debug("utterance processing failed: \(message)")
|
||||
}
|
||||
@@ -533,8 +570,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
|
||||
let asrWait = asrWaitTimeout()
|
||||
FlowDiagnostics.log(
|
||||
"finalize start asrWait=\(Int(asrWait))s engine=\(store.engineMode) " +
|
||||
"backend=\(store.localASRBackend.rawValue)"
|
||||
"finalize start asrWait=\(Int(asrWait))s engine=\(store.engineMode)"
|
||||
)
|
||||
|
||||
let asrDeadline = Date().addingTimeInterval(asrWait)
|
||||
@@ -560,10 +596,13 @@ final class FlowSessionManager: ObservableObject {
|
||||
let key = (asrTask?.isCancelled == true)
|
||||
? "flow.error.recognitionInterrupted"
|
||||
: "flow.error.noSpeech"
|
||||
let kind: FlowSessionKeys.TranscriptionErrorKind =
|
||||
(asrTask?.isCancelled == true) ? .recognitionInterrupted : .noSpeech
|
||||
FlowDiagnostics.log("finalize failed: empty transcript after \(String(format: "%.1f", asrElapsed))s")
|
||||
utteranceRecordingStartedAt = nil
|
||||
FlowSessionBridge.storeTranscriptionError(
|
||||
AppL10n.string(key)
|
||||
AppL10n.string(key),
|
||||
kind: kind
|
||||
)
|
||||
return
|
||||
}
|
||||
@@ -576,33 +615,12 @@ final class FlowSessionManager: ObservableObject {
|
||||
// from the keyboard extension are visible before polish/translate.
|
||||
let pipelineStore = AppGroupStore()
|
||||
|
||||
if !pipelineStore.shouldRunCloudLLMStep {
|
||||
// Local engine with cloud polish off — ASR-only.
|
||||
FlowSessionBridge.storeTranscriptionResult(text, polishWarning: chunkNote)
|
||||
FlowDiagnostics.log(
|
||||
"finalize ASR-only total=\(String(format: "%.1f", Date().timeIntervalSince(pipelineStarted)))s " +
|
||||
"len=\(text.count)"
|
||||
)
|
||||
SpeechHistoryStore.shared.recordUtterance(
|
||||
text: text,
|
||||
engineMode: engineMode,
|
||||
duration: recordingDuration,
|
||||
wasTranslation: false
|
||||
)
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
chunkWarnings = []
|
||||
debug("utterance finalized length=\(text.count)")
|
||||
return
|
||||
}
|
||||
|
||||
var delivered = text
|
||||
let polishStarted = Date()
|
||||
let polishMode = pipelineStore.polishModeForPipeline
|
||||
FlowDiagnostics.log(
|
||||
"finalize LLM mode=\(Self.polishModeLogLabel(polishMode)) " +
|
||||
"translationTarget=\(pipelineStore.translationTargetLocaleId) " +
|
||||
"cloudPolish=\(pipelineStore.localModeCloudPolishEnabled)"
|
||||
"translationTarget=\(pipelineStore.translationTargetLocaleId)"
|
||||
)
|
||||
do {
|
||||
let polished = try await polisher.polish(
|
||||
@@ -640,6 +658,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
chunkWarnings = []
|
||||
FlowSessionBridge.storeTranscriptionPartial("")
|
||||
chunkedPipeline = nil
|
||||
debug("utterance finalized length=\(text.count)")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user