feat(flow): add whole-utterance batch ASR fallback (P1)

- Accumulate utterance PCM in FlowContinuousCapture for batch retry
- Run full-utterance transcribeChunk when stitched final lags partial
- Refactor Mac MLX tail drain to shared FlowUtteranceEndCoordinator
- Add FlowUtterancePCMStore, UtteranceBatchFallbackPolicy, and tests

Co-authored-by: Rocky <hkgood@users.noreply.github.com>
This commit is contained in:
Cursor Agent
2026-07-26 10:15:31 +00:00
parent c7ee891a90
commit f0240c3088
9 changed files with 256 additions and 3 deletions
@@ -66,6 +66,8 @@ final class FlowSessionManager: ObservableObject {
private var lastFinal = ""
/// Partial stitched text captured when the user stops recording.
private var bestPartialSnapshot = ""
/// Full utterance PCM for batch ASR fallback after pipelined chunking.
private var utterancePCMSamples: [Float] = []
private var chunkWarnings: [String] = []
private var lastReadyTraceSignature = ""
private var lastCommandFingerprint = ""
@@ -359,6 +361,7 @@ final class FlowSessionManager: ObservableObject {
currentPartial = ""
lastFinal = ""
bestPartialSnapshot = ""
utterancePCMSamples = []
chunkWarnings = []
FlowSessionBridge.setHostReady(false)
}
@@ -1105,6 +1108,7 @@ final class FlowSessionManager: ObservableObject {
currentPartial = ""
lastFinal = ""
bestPartialSnapshot = ""
utterancePCMSamples = []
chunkWarnings = []
let localeId = store.localeId
@@ -1214,6 +1218,7 @@ final class FlowSessionManager: ObservableObject {
guard let self else { return }
let drainReport = await self.capture.endUtteranceAndDrain()
FlowDiagnostics.logDrain(drainReport)
self.utterancePCMSamples = self.capture.consumeUtteranceSamples()
await self.finalizeUtterance(
sessionId: drainingSessionId,
utteranceId: drainingUtteranceId,
@@ -1239,6 +1244,7 @@ final class FlowSessionManager: ObservableObject {
currentPartial = ""
lastFinal = ""
bestPartialSnapshot = ""
utterancePCMSamples = []
chunkWarnings = []
currentUtteranceId = nil
currentCommandSeq = 0
@@ -1266,6 +1272,7 @@ final class FlowSessionManager: ObservableObject {
currentPartial = ""
lastFinal = ""
bestPartialSnapshot = ""
utterancePCMSamples = []
chunkWarnings = []
storeCurrentError(message, kind: kind)
currentUtteranceId = nil
@@ -1289,6 +1296,7 @@ final class FlowSessionManager: ObservableObject {
currentPartial = ""
lastFinal = ""
bestPartialSnapshot = ""
utterancePCMSamples = []
chunkWarnings = []
storeCurrentError(message, kind: kind)
currentUtteranceId = nil
@@ -1348,6 +1356,14 @@ final class FlowSessionManager: ObservableObject {
if text.isEmpty {
text = currentPartial.trimmingCharacters(in: .whitespacesAndNewlines)
}
if UtteranceBatchFallbackPolicy.shouldRunBatchFallback(
stitchedFinal: lastFinal,
partialSnapshot: bestPartialSnapshot
), !utterancePCMSamples.isEmpty {
text = await runBatchASRFallback(currentText: text)
}
utterancePCMSamples = []
guard !text.isEmpty else {
let key = (asrTask?.isCancelled == true || Task.isCancelled)
? "flow.error.recognitionInterrupted"
@@ -1441,6 +1457,7 @@ final class FlowSessionManager: ObservableObject {
currentPartial = ""
lastFinal = ""
bestPartialSnapshot = ""
utterancePCMSamples = []
chunkWarnings = []
chunkedPipeline = nil
debug("utterance finalized length=\(text.count)")
@@ -1569,6 +1586,49 @@ final class FlowSessionManager: ObservableObject {
)
}
/// Re-transcribe the full utterance PCM when pipelined chunking likely dropped tail text.
private func runBatchASRFallback(currentText: String) async -> String {
let samples = utterancePCMSamples
guard !samples.isEmpty else { return currentText }
let locale = SpeechLocaleResolver.resolve(store.localeId)
let stitched = lastFinal.trimmingCharacters(in: .whitespacesAndNewlines)
let partial = bestPartialSnapshot.trimmingCharacters(in: .whitespacesAndNewlines)
FlowDiagnostics.log(
"batch fallback start samples=\(samples.count) stitchedLen=\(stitched.count) partialLen=\(partial.count)"
)
let asrService = asr
let result = await Task.detached(priority: .userInitiated) { [asrService] in
await asrService.transcribeChunk(samples: samples, locale: locale)
}.value
switch result {
case .success(let batchText):
let trimmedBatch = batchText.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmedBatch.isEmpty else { return currentText }
let resolved = UtteranceBatchFallbackPolicy.preferredTranscript(
batch: trimmedBatch,
stitchedFinal: stitched,
partialSnapshot: partial,
current: currentText
)
FlowPipelineDiagnostics.logBatchFallback(
sampleCount: samples.count,
stitchedLength: stitched.count,
partialLength: partial.count,
batchLength: trimmedBatch.count
)
return resolved
case .failure(let message):
FlowDiagnostics.log("batch fallback failed: \(message)")
return currentText
case .cancelled:
return currentText
}
}
private func asrWaitTimeout() -> TimeInterval {
// v0.2.0: local engine is iOS `SpeechAnalyzer` only, so the
// previous Qwen3-specific timeout collapses into the shared