feat(flow): add whole-utterance batch ASR fallback (P1)
- Accumulate utterance PCM in FlowContinuousCapture for batch retry - Run full-utterance transcribeChunk when stitched final lags partial - Refactor Mac MLX tail drain to shared FlowUtteranceEndCoordinator - Add FlowUtterancePCMStore, UtteranceBatchFallbackPolicy, and tests Co-authored-by: Rocky <hkgood@users.noreply.github.com>
This commit is contained in:
@@ -66,6 +66,8 @@ final class FlowSessionManager: ObservableObject {
|
||||
private var lastFinal = ""
|
||||
/// Partial stitched text captured when the user stops recording.
|
||||
private var bestPartialSnapshot = ""
|
||||
/// Full utterance PCM for batch ASR fallback after pipelined chunking.
|
||||
private var utterancePCMSamples: [Float] = []
|
||||
private var chunkWarnings: [String] = []
|
||||
private var lastReadyTraceSignature = ""
|
||||
private var lastCommandFingerprint = ""
|
||||
@@ -359,6 +361,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
bestPartialSnapshot = ""
|
||||
utterancePCMSamples = []
|
||||
chunkWarnings = []
|
||||
FlowSessionBridge.setHostReady(false)
|
||||
}
|
||||
@@ -1105,6 +1108,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
bestPartialSnapshot = ""
|
||||
utterancePCMSamples = []
|
||||
chunkWarnings = []
|
||||
|
||||
let localeId = store.localeId
|
||||
@@ -1214,6 +1218,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
guard let self else { return }
|
||||
let drainReport = await self.capture.endUtteranceAndDrain()
|
||||
FlowDiagnostics.logDrain(drainReport)
|
||||
self.utterancePCMSamples = self.capture.consumeUtteranceSamples()
|
||||
await self.finalizeUtterance(
|
||||
sessionId: drainingSessionId,
|
||||
utteranceId: drainingUtteranceId,
|
||||
@@ -1239,6 +1244,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
bestPartialSnapshot = ""
|
||||
utterancePCMSamples = []
|
||||
chunkWarnings = []
|
||||
currentUtteranceId = nil
|
||||
currentCommandSeq = 0
|
||||
@@ -1266,6 +1272,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
bestPartialSnapshot = ""
|
||||
utterancePCMSamples = []
|
||||
chunkWarnings = []
|
||||
storeCurrentError(message, kind: kind)
|
||||
currentUtteranceId = nil
|
||||
@@ -1289,6 +1296,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
bestPartialSnapshot = ""
|
||||
utterancePCMSamples = []
|
||||
chunkWarnings = []
|
||||
storeCurrentError(message, kind: kind)
|
||||
currentUtteranceId = nil
|
||||
@@ -1348,6 +1356,14 @@ final class FlowSessionManager: ObservableObject {
|
||||
if text.isEmpty {
|
||||
text = currentPartial.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
}
|
||||
|
||||
if UtteranceBatchFallbackPolicy.shouldRunBatchFallback(
|
||||
stitchedFinal: lastFinal,
|
||||
partialSnapshot: bestPartialSnapshot
|
||||
), !utterancePCMSamples.isEmpty {
|
||||
text = await runBatchASRFallback(currentText: text)
|
||||
}
|
||||
utterancePCMSamples = []
|
||||
guard !text.isEmpty else {
|
||||
let key = (asrTask?.isCancelled == true || Task.isCancelled)
|
||||
? "flow.error.recognitionInterrupted"
|
||||
@@ -1441,6 +1457,7 @@ final class FlowSessionManager: ObservableObject {
|
||||
currentPartial = ""
|
||||
lastFinal = ""
|
||||
bestPartialSnapshot = ""
|
||||
utterancePCMSamples = []
|
||||
chunkWarnings = []
|
||||
chunkedPipeline = nil
|
||||
debug("utterance finalized length=\(text.count)")
|
||||
@@ -1569,6 +1586,49 @@ final class FlowSessionManager: ObservableObject {
|
||||
)
|
||||
}
|
||||
|
||||
/// Re-transcribe the full utterance PCM when pipelined chunking likely dropped tail text.
|
||||
private func runBatchASRFallback(currentText: String) async -> String {
|
||||
let samples = utterancePCMSamples
|
||||
guard !samples.isEmpty else { return currentText }
|
||||
|
||||
let locale = SpeechLocaleResolver.resolve(store.localeId)
|
||||
let stitched = lastFinal.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let partial = bestPartialSnapshot.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
|
||||
FlowDiagnostics.log(
|
||||
"batch fallback start samples=\(samples.count) stitchedLen=\(stitched.count) partialLen=\(partial.count)"
|
||||
)
|
||||
|
||||
let asrService = asr
|
||||
let result = await Task.detached(priority: .userInitiated) { [asrService] in
|
||||
await asrService.transcribeChunk(samples: samples, locale: locale)
|
||||
}.value
|
||||
|
||||
switch result {
|
||||
case .success(let batchText):
|
||||
let trimmedBatch = batchText.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmedBatch.isEmpty else { return currentText }
|
||||
let resolved = UtteranceBatchFallbackPolicy.preferredTranscript(
|
||||
batch: trimmedBatch,
|
||||
stitchedFinal: stitched,
|
||||
partialSnapshot: partial,
|
||||
current: currentText
|
||||
)
|
||||
FlowPipelineDiagnostics.logBatchFallback(
|
||||
sampleCount: samples.count,
|
||||
stitchedLength: stitched.count,
|
||||
partialLength: partial.count,
|
||||
batchLength: trimmedBatch.count
|
||||
)
|
||||
return resolved
|
||||
case .failure(let message):
|
||||
FlowDiagnostics.log("batch fallback failed: \(message)")
|
||||
return currentText
|
||||
case .cancelled:
|
||||
return currentText
|
||||
}
|
||||
}
|
||||
|
||||
private func asrWaitTimeout() -> TimeInterval {
|
||||
// v0.2.0: local engine is iOS `SpeechAnalyzer` only, so the
|
||||
// previous Qwen3-specific timeout collapses into the shared
|
||||
|
||||
Reference in New Issue
Block a user