fix(flow): drain trailing audio and harden chunked ASR pipeline

- P0/P1: tail-drain state machine in FlowContinuousCapture with converter
  flush; FlowSessionManager awaits drain before finalize
- P2: FlowCaptureTailDrain policy/tracker, pipeline diagnostics, unit tests
- P3: short final chunk merged re-transcription in ChunkedUtterancePipeline
- P4: UtteranceTranscriptStitcher composedSafely fallback; preview path parity
  in LiveDictationController

Co-authored-by: Rocky <hkgood@users.noreply.github.com>
This commit is contained in:
Cursor Agent
2026-07-06 05:13:55 +00:00
parent 537a68552a
commit 8b105430e3
14 changed files with 690 additions and 47 deletions
+1
View File
@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed ### Changed
- **Flow ASR pipelining**: shorter first chunk (2.5s) and 5s follow-ups so short utterances start on-device recognition while still recording; session-level ASR warmup and format cache reuse; live partials mirrored to the keyboard transcript line. / **Flow ASR 流水线**:首块 2.5 秒、后续 5 秒,短句录音期间即开始端侧识别;会话级 ASR 预热与格式缓存复用;实时 partial 同步到键盘转写行。 - **Flow ASR pipelining**: shorter first chunk (2.5s) and 5s follow-ups so short utterances start on-device recognition while still recording; session-level ASR warmup and format cache reuse; live partials mirrored to the keyboard transcript line. / **Flow ASR 流水线**:首块 2.5 秒、后续 5 秒,短句录音期间即开始端侧识别;会话级 ASR 预热与格式缓存复用;实时 partial 同步到键盘转写行。
- **Flow tail drain**: after mic stop, capture drains trailing PCM (silence-detected, capped) before finishing the ASR stream; host finalize awaits drain; short final chunks re-transcribed with prior overlap; stitcher safe fallback when overlap merge would drop content. / **Flow 尾音排空**:停止录音后先排空尾部 PCM(静音检测 + 上限)再结束 ASR 流;主 App finalize 等待排空;过短末块与上一块 overlap 合并重识别;拼接误删时回退为安全合并。
## [0.4.0] - 2026-07-05 ## [0.4.0] - 2026-07-05
@@ -11,4 +11,8 @@ enum FlowDiagnostics {
static func log(_ message: String) { static func log(_ message: String) {
OSGLog.flow.info("\(message, privacy: .public)") OSGLog.flow.info("\(message, privacy: .public)")
} }
static func logDrain(_ report: FlowCaptureDrainReport) {
FlowPipelineDiagnostics.logDrain(report)
}
} }
@@ -485,20 +485,20 @@ final class FlowSessionManager: ObservableObject {
// Close the mic gate first, then mark processing before dropping the // Close the mic gate first, then mark processing before dropping the
// recording flag so the poll loop cannot start a second utterance. // recording flag so the poll loop cannot start a second utterance.
capture.endUtterance()
FlowSessionBridge.setRecordingState(.processing) FlowSessionBridge.setRecordingState(.processing)
isUtteranceRecording = false isUtteranceRecording = false
isUtteranceProcessing = true isUtteranceProcessing = true
// Do NOT cancel `asrTask` or `asr` the preview pipeline relies on // Do NOT cancel `asrTask` or `asr` drain trailing PCM, then finalize.
// the consumer staying alive until `.final` lands (see
// `PreviewASRControllerStateTests`).
finalizeTask?.cancel() finalizeTask?.cancel()
finalizeTask = Task { @MainActor [weak self] in finalizeTask = Task { @MainActor [weak self] in
await self?.finalizeUtterance() guard let self else { return }
let drainReport = await self.capture.endUtteranceAndDrain()
FlowDiagnostics.logDrain(drainReport)
await self.finalizeUtterance()
} }
debug("utterance stopped, finalizing") debug("utterance stopped, draining tail")
} }
private func abortUtterance() { private func abortUtterance() {
@@ -16,6 +16,8 @@ public struct FlowUtteranceChunkConfig: Sendable, Equatable {
public let pauseExtensionMaxSeconds: TimeInterval public let pauseExtensionMaxSeconds: TimeInterval
/// RMS below this is treated as a pause candidate (Float32 mono @ 16 kHz). /// RMS below this is treated as a pause candidate (Float32 mono @ 16 kHz).
public let pauseRMSThreshold: Float public let pauseRMSThreshold: Float
/// Final chunk shorter than this is re-transcribed merged with the prior tail overlap.
public let minFinalChunkDurationSeconds: TimeInterval
public let sampleRate: Int public let sampleRate: Int
public init( public init(
@@ -24,6 +26,7 @@ public struct FlowUtteranceChunkConfig: Sendable, Equatable {
overlapDurationSeconds: TimeInterval, overlapDurationSeconds: TimeInterval,
pauseExtensionMaxSeconds: TimeInterval, pauseExtensionMaxSeconds: TimeInterval,
pauseRMSThreshold: Float, pauseRMSThreshold: Float,
minFinalChunkDurationSeconds: TimeInterval = 0.8,
sampleRate: Int sampleRate: Int
) { ) {
self.firstChunkDurationSeconds = firstChunkDurationSeconds self.firstChunkDurationSeconds = firstChunkDurationSeconds
@@ -31,6 +34,7 @@ public struct FlowUtteranceChunkConfig: Sendable, Equatable {
self.overlapDurationSeconds = overlapDurationSeconds self.overlapDurationSeconds = overlapDurationSeconds
self.pauseExtensionMaxSeconds = pauseExtensionMaxSeconds self.pauseExtensionMaxSeconds = pauseExtensionMaxSeconds
self.pauseRMSThreshold = pauseRMSThreshold self.pauseRMSThreshold = pauseRMSThreshold
self.minFinalChunkDurationSeconds = minFinalChunkDurationSeconds
self.sampleRate = sampleRate self.sampleRate = sampleRate
} }
@@ -40,6 +44,7 @@ public struct FlowUtteranceChunkConfig: Sendable, Equatable {
overlapDurationSeconds: TimeInterval, overlapDurationSeconds: TimeInterval,
pauseExtensionMaxSeconds: TimeInterval, pauseExtensionMaxSeconds: TimeInterval,
pauseRMSThreshold: Float, pauseRMSThreshold: Float,
minFinalChunkDurationSeconds: TimeInterval = 0.8,
sampleRate: Int sampleRate: Int
) { ) {
self.firstChunkDurationSeconds = maxChunkDurationSeconds self.firstChunkDurationSeconds = maxChunkDurationSeconds
@@ -47,6 +52,7 @@ public struct FlowUtteranceChunkConfig: Sendable, Equatable {
self.overlapDurationSeconds = overlapDurationSeconds self.overlapDurationSeconds = overlapDurationSeconds
self.pauseExtensionMaxSeconds = pauseExtensionMaxSeconds self.pauseExtensionMaxSeconds = pauseExtensionMaxSeconds
self.pauseRMSThreshold = pauseRMSThreshold self.pauseRMSThreshold = pauseRMSThreshold
self.minFinalChunkDurationSeconds = minFinalChunkDurationSeconds
self.sampleRate = sampleRate self.sampleRate = sampleRate
} }
@@ -75,6 +81,10 @@ public struct FlowUtteranceChunkConfig: Sendable, Equatable {
Int(pauseExtensionMaxSeconds * Double(sampleRate)) Int(pauseExtensionMaxSeconds * Double(sampleRate))
} }
public var minFinalChunkSamples: Int {
Int(minFinalChunkDurationSeconds * Double(sampleRate))
}
/// Default for keyboard Flow utterances ( 3 min, pipelined ASR). /// Default for keyboard Flow utterances ( 3 min, pipelined ASR).
public static let flowDefault = FlowUtteranceChunkConfig( public static let flowDefault = FlowUtteranceChunkConfig(
firstChunkDurationSeconds: 2.5, firstChunkDurationSeconds: 2.5,
@@ -82,6 +92,7 @@ public struct FlowUtteranceChunkConfig: Sendable, Equatable {
overlapDurationSeconds: 0.5, overlapDurationSeconds: 0.5,
pauseExtensionMaxSeconds: 2, pauseExtensionMaxSeconds: 2,
pauseRMSThreshold: 0.015, pauseRMSThreshold: 0.015,
minFinalChunkDurationSeconds: 0.8,
sampleRate: 16_000 sampleRate: 16_000
) )
} }
@@ -99,6 +99,8 @@ public actor ChunkedUtterancePipeline {
var chunkWarnings: [String] = [] var chunkWarnings: [String] = []
var failedChunks = 0 var failedChunks = 0
var processedChunks = 0 var processedChunks = 0
var previousChunkSamples: [Float] = []
var lastChunkSamples = 0
let feeder = Task { let feeder = Task {
for await chunk in UtteranceStreamChunker.chunks(from: stream, config: config) { for await chunk in UtteranceStreamChunker.chunks(from: stream, config: config) {
@@ -117,19 +119,42 @@ public actor ChunkedUtterancePipeline {
guard let chunk = await queue.dequeue() else { break } guard let chunk = await queue.dequeue() else { break }
processedChunks += 1 processedChunks += 1
let asr = self.asr lastChunkSamples = chunk.samples.count
let locale = self.locale
let result = await Task.detached(priority: .userInitiated) {
await asr.transcribeChunk(samples: chunk.samples, locale: locale)
}.value
if chunk.isLast,
chunk.samples.count < config.minFinalChunkSamples,
processedChunks > 1,
!previousChunkSamples.isEmpty {
let mergedSamples = Array(previousChunkSamples.suffix(config.overlapSamples))
+ chunk.samples
let mergedResult = await transcribeChunk(samples: mergedSamples)
switch mergedResult {
case .success(let text):
stitcher.removeLastSegment()
stitcher.append(index: max(0, chunk.index - 1), text: text)
publishPartial(from: stitcher, onPartial: onPartial)
case .failure(let message):
failedChunks += 1
chunkWarnings.append(
SharedL10n.format(
"error.asr.chunkFailed",
chunk.index + 1,
message
)
)
case .cancelled:
feeder.cancel()
return .cancelled
}
previousChunkSamples = chunk.samples
continue
}
let result = await transcribeChunk(samples: chunk.samples)
switch result { switch result {
case .success(let text): case .success(let text):
stitcher.append(index: chunk.index, text: text) stitcher.append(index: chunk.index, text: text)
let partial = stitcher.composed() publishPartial(from: stitcher, onPartial: onPartial)
if !partial.isEmpty {
onPartial(partial)
}
case .failure(let message): case .failure(let message):
failedChunks += 1 failedChunks += 1
chunkWarnings.append( chunkWarnings.append(
@@ -143,11 +168,20 @@ public actor ChunkedUtterancePipeline {
feeder.cancel() feeder.cancel()
return .cancelled return .cancelled
} }
previousChunkSamples = chunk.samples
} }
_ = await feeder.value _ = await feeder.value
let finalText = stitcher.composed().trimmingCharacters(in: .whitespacesAndNewlines) let finalText = stitcher.composedSafely().trimmingCharacters(in: .whitespacesAndNewlines)
FlowPipelineDiagnostics.logChunkFinalize(
chunkCount: processedChunks,
lastChunkSamples: lastChunkSamples,
stitchedLength: finalText.count,
chunkWarnings: chunkWarnings.count
)
if finalText.isEmpty { if finalText.isEmpty {
if failedChunks > 0, processedChunks == failedChunks { if failedChunks > 0, processedChunks == failedChunks {
return .failure(SharedL10n.string("error.asr.noSpeech")) return .failure(SharedL10n.string("error.asr.noSpeech"))
@@ -157,4 +191,22 @@ public actor ChunkedUtterancePipeline {
return .success(ChunkedUtteranceSuccess(text: finalText, chunkWarnings: chunkWarnings)) return .success(ChunkedUtteranceSuccess(text: finalText, chunkWarnings: chunkWarnings))
} }
private func transcribeChunk(samples: [Float]) async -> ASRChunkResult {
let asr = self.asr
let locale = self.locale
return await Task.detached(priority: .userInitiated) {
await asr.transcribeChunk(samples: samples, locale: locale)
}.value
}
private func publishPartial(
from stitcher: UtteranceTranscriptStitcher,
onPartial: @Sendable (String) -> Void
) {
let partial = stitcher.composedSafely()
if !partial.isEmpty {
onPartial(partial)
}
}
} }
@@ -14,6 +14,13 @@ import os
private enum FlowCaptureConstants { private enum FlowCaptureConstants {
static let levelBarCount = 24 static let levelBarCount = 24
static let targetSampleRate: Double = 16_000 static let targetSampleRate: Double = 16_000
static let drainPollIntervalNs: UInt64 = 20_000_000
}
private enum UtteranceGatePhase: Equatable {
case idle
case recording
case draining
} }
/// Thread-safe relay for utterance-scoped ASR snapshots. /// Thread-safe relay for utterance-scoped ASR snapshots.
@@ -158,7 +165,14 @@ public final class FlowContinuousCapture {
private let streamRelay = FlowCaptureStreamRelay() private let streamRelay = FlowCaptureStreamRelay()
private let prerollStore = FlowPrerollStore() private let prerollStore = FlowPrerollStore()
private let levelStore = FlowLevelStore(barCount: FlowCaptureConstants.levelBarCount) private let levelStore = FlowLevelStore(barCount: FlowCaptureConstants.levelBarCount)
private let isUtteranceActive = OSAllocatedUnfairLock(initialState: false) private let gate = OSAllocatedUnfairLock(initialState: UtteranceGatePhase.idle)
private let drainTracker = FlowCaptureDrainTracker()
private let tailSampleCounter = OSAllocatedUnfairLock(initialState: 0)
private var audioConverter: AVAudioConverter?
private var targetFormat: AVAudioFormat?
private var hwFormat: AVAudioFormat?
private var drainPolicy = FlowCaptureTailDrainPolicy.flowDefault
private var didInstallTap = false private var didInstallTap = false
private var isRunning = false private var isRunning = false
@@ -184,15 +198,15 @@ public final class FlowContinuousCapture {
} }
let inputNode = audioEngine.inputNode let inputNode = audioEngine.inputNode
let hwFormat = inputNode.outputFormat(forBus: 0) let hardwareFormat = inputNode.outputFormat(forBus: 0)
guard hwFormat.sampleRate > 0, hwFormat.channelCount > 0 else { guard hardwareFormat.sampleRate > 0, hardwareFormat.channelCount > 0 else {
throw StartError.invalidHardwareFormat( throw StartError.invalidHardwareFormat(
sampleRate: hwFormat.sampleRate, sampleRate: hardwareFormat.sampleRate,
channels: Int(hwFormat.channelCount) channels: Int(hardwareFormat.channelCount)
) )
} }
guard let targetFormat = AVAudioFormat( guard let resolvedTargetFormat = AVAudioFormat(
commonFormat: .pcmFormatFloat32, commonFormat: .pcmFormatFloat32,
sampleRate: FlowCaptureConstants.targetSampleRate, sampleRate: FlowCaptureConstants.targetSampleRate,
channels: 1, channels: 1,
@@ -200,25 +214,35 @@ public final class FlowContinuousCapture {
) else { ) else {
throw StartError.formatCreateFailed throw StartError.formatCreateFailed
} }
guard let converter = AVAudioConverter(from: hwFormat, to: targetFormat) else { guard let converter = AVAudioConverter(from: hardwareFormat, to: resolvedTargetFormat) else {
throw StartError.converterCreateFailed throw StartError.converterCreateFailed
} }
audioConverter = converter
targetFormat = resolvedTargetFormat
hwFormat = hardwareFormat
if !didInstallTap { if !didInstallTap {
let utteranceFlag = isUtteranceActive let gateLock = gate
let relay = streamRelay let relay = streamRelay
let preroll = prerollStore let preroll = prerollStore
let levels = levelStore let levels = levelStore
let tracker = drainTracker
let tailCounter = tailSampleCounter
let policy = drainPolicy
let tap = Self.makeAudioTapBlock( let tap = Self.makeAudioTapBlock(
converter: converter, converter: converter,
targetFormat: targetFormat, targetFormat: resolvedTargetFormat,
hwFormat: hwFormat, hwFormat: hardwareFormat,
utteranceFlag: utteranceFlag, gate: gateLock,
levelStore: levels, levelStore: levels,
prerollStore: preroll, prerollStore: preroll,
streamRelay: relay streamRelay: relay,
drainTracker: tracker,
tailSampleCounter: tailCounter,
drainPolicy: policy
) )
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap) inputNode.installTap(onBus: 0, bufferSize: 4096, format: hardwareFormat, block: tap)
didInstallTap = true didInstallTap = true
} }
@@ -233,7 +257,9 @@ public final class FlowContinuousCapture {
/// Tear down the engine and release the audio session. /// Tear down the engine and release the audio session.
public func stop() { public func stop() {
isUtteranceActive.withLock { $0 = false } gate.withLock { $0 = .idle }
drainTracker.reset()
tailSampleCounter.withLock { $0 = 0 }
streamRelay.finish() streamRelay.finish()
if didInstallTap { if didInstallTap {
@@ -244,6 +270,9 @@ public final class FlowContinuousCapture {
audioEngine.stop() audioEngine.stop()
} }
isRunning = false isRunning = false
audioConverter = nil
targetFormat = nil
hwFormat = nil
try? AVAudioSession.sharedInstance().setActive( try? AVAudioSession.sharedInstance().setActive(
false, false,
options: .notifyOthersOnDeactivation options: .notifyOthersOnDeactivation
@@ -269,38 +298,129 @@ public final class FlowContinuousCapture {
/// Begin forwarding downsampled buffers to ASR for one utterance. /// Begin forwarding downsampled buffers to ASR for one utterance.
public func beginUtterance() -> AsyncStream<AudioBufferSnapshot> { public func beginUtterance() -> AsyncStream<AudioBufferSnapshot> {
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream() let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
drainTracker.reset()
tailSampleCounter.withLock { $0 = 0 }
// Bind the consumer before opening the gate so early tap frames // Bind the consumer before opening the gate so early tap frames
// are not dropped on the floor. // are not dropped on the floor.
streamRelay.bind(continuation) streamRelay.bind(continuation)
streamRelay.replay(prerollStore.drain()) streamRelay.replay(prerollStore.drain())
isUtteranceActive.withLock { $0 = true } gate.withLock { $0 = .recording }
return stream return stream
} }
/// Stop forwarding buffers; finishes the ASR stream. /// Drain trailing PCM after the user stops, then finish the ASR stream.
public func endUtterance() { public func endUtteranceAndDrain(
isUtteranceActive.withLock { $0 = false } policy: FlowCaptureTailDrainPolicy = .flowDefault
) async -> FlowCaptureDrainReport {
let currentPhase = gate.withLock { $0 }
guard currentPhase == .recording else {
return .skipped
}
drainPolicy = policy
gate.withLock { $0 = .draining }
drainTracker.beginDrain()
var endedBySilence = false
while true {
let decision = drainTracker.shouldFinish(policy: policy)
if decision.finished {
endedBySilence = decision.endedBySilence
break
}
if Task.isCancelled { break }
try? await Task.sleep(nanoseconds: FlowCaptureConstants.drainPollIntervalNs)
}
let flushSamples = flushConverterTailToStream()
tailSampleCounter.withLock { $0 += flushSamples }
streamRelay.finish() streamRelay.finish()
gate.withLock { $0 = .idle }
let tailSamples = tailSampleCounter.withLock { $0 }
let report = FlowCaptureDrainReport(
drainDurationSeconds: drainTracker.elapsedSeconds(),
endedBySilence: endedBySilence,
tailSampleCount: tailSamples
)
drainTracker.reset()
tailSampleCounter.withLock { $0 = 0 }
FlowPipelineDiagnostics.logDrain(report)
return report
} }
/// Immediate stop without tail drain (abort / session teardown).
public func cancelUtterance() { public func cancelUtterance() {
endUtterance() gate.withLock { $0 = .idle }
drainTracker.reset()
tailSampleCounter.withLock { $0 = 0 }
streamRelay.finish()
} }
public func currentAudioLevels() -> [Float] { public func currentAudioLevels() -> [Float] {
levelStore.snapshot() levelStore.snapshot()
} }
// MARK: - Converter flush
@discardableResult
private func flushConverterTailToStream() -> Int {
guard let converter = audioConverter,
let targetFormat,
let hwFormat else {
return 0
}
var flushedSamples = 0
let capacity = AVAudioFrameCount(max(512, hwFormat.sampleRate / 20))
guard let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
return 0
}
var endOfStreamSignaled = false
while true {
outBuffer.frameLength = 0
var error: NSError?
let status = converter.convert(to: outBuffer, error: &error) { _, outStatus in
if endOfStreamSignaled {
outStatus.pointee = .noDataNow
return nil
}
endOfStreamSignaled = true
outStatus.pointee = .endOfStream
return nil
}
if status == .error || error != nil {
break
}
guard status == .haveData, outBuffer.frameLength > 0 else {
break
}
let snapshot = AudioBufferSnapshot(buffer: outBuffer)
guard !snapshot.samples.isEmpty else { break }
streamRelay.yield(snapshot)
flushedSamples += snapshot.samples.count
}
return flushedSamples
}
// MARK: - Audio tap (nonisolated runs on realtime thread) // MARK: - Audio tap (nonisolated runs on realtime thread)
private nonisolated static func makeAudioTapBlock( private nonisolated static func makeAudioTapBlock(
converter: AVAudioConverter, converter: AVAudioConverter,
targetFormat: AVAudioFormat, targetFormat: AVAudioFormat,
hwFormat: AVAudioFormat, hwFormat: AVAudioFormat,
utteranceFlag: OSAllocatedUnfairLock<Bool>, gate: OSAllocatedUnfairLock<UtteranceGatePhase>,
levelStore: FlowLevelStore, levelStore: FlowLevelStore,
prerollStore: FlowPrerollStore, prerollStore: FlowPrerollStore,
streamRelay: FlowCaptureStreamRelay streamRelay: FlowCaptureStreamRelay,
drainTracker: FlowCaptureDrainTracker,
tailSampleCounter: OSAllocatedUnfairLock<Int>,
drainPolicy: FlowCaptureTailDrainPolicy
) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void { ) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
return { buffer, _ in return { buffer, _ in
levelStore.update(from: buffer, barCount: FlowCaptureConstants.levelBarCount) levelStore.update(from: buffer, barCount: FlowCaptureConstants.levelBarCount)
@@ -322,9 +442,15 @@ public final class FlowContinuousCapture {
let snapshot = AudioBufferSnapshot(buffer: outBuffer) let snapshot = AudioBufferSnapshot(buffer: outBuffer)
guard !snapshot.samples.isEmpty else { return } guard !snapshot.samples.isEmpty else { return }
if utteranceFlag.withLock({ $0 }) { let phase = gate.withLock { $0 }
switch phase {
case .recording, .draining:
streamRelay.yield(snapshot) streamRelay.yield(snapshot)
} else { if phase == .draining {
drainTracker.noteAudio(samples: snapshot.samples, policy: drainPolicy)
tailSampleCounter.withLock { $0 += snapshot.samples.count }
}
case .idle:
prerollStore.append(snapshot) prerollStore.append(snapshot)
} }
} }
@@ -41,6 +41,12 @@ import AVFoundation
import Speech import Speech
import os import os
private enum LiveCaptureGatePhase: Equatable {
case idle
case recording
case draining
}
/// Thread-safe relay so the AVAudioEngine tap can yield snapshots without /// Thread-safe relay so the AVAudioEngine tap can yield snapshots without
/// hopping through `@MainActor` (which adds latency and can reorder frames). /// hopping through `@MainActor` (which adds latency and can reorder frames).
private final class CaptureStreamRelay: @unchecked Sendable { private final class CaptureStreamRelay: @unchecked Sendable {
@@ -100,6 +106,11 @@ public final class LiveDictationController: ObservableObject {
public var asrTask: Task<Void, Never>? public var asrTask: Task<Void, Never>?
private let streamRelay = CaptureStreamRelay() private let streamRelay = CaptureStreamRelay()
private var chunkedPipeline: ChunkedUtterancePipeline? private var chunkedPipeline: ChunkedUtterancePipeline?
private let captureGate = OSAllocatedUnfairLock(initialState: LiveCaptureGatePhase.idle)
private let drainTracker = FlowCaptureDrainTracker()
private var audioConverter: AVAudioConverter?
private var targetFormat: AVAudioFormat?
private var hwFormat: AVAudioFormat?
private var didConfigureAudioSession = false private var didConfigureAudioSession = false
private var didInstallTap = false private var didInstallTap = false
@@ -218,10 +229,6 @@ public final class LiveDictationController: ObservableObject {
// If a previous `asrTask` is somehow still running (e.g. the // If a previous `asrTask` is somehow still running (e.g. the
// user smashed the disc twice quickly), `start()` cancels it // user smashed the disc twice quickly), `start()` cancels it
// at the entry point as a safety net. // at the entry point as a safety net.
teardownCapturePipeline()
// Fallback: if we already have a meaningful partial but the
// backend never emits `.final`, promote the partial so the
// preview still inserts text after "".
let partial = currentPartial.trimmingCharacters(in: .whitespacesAndNewlines) let partial = currentPartial.trimmingCharacters(in: .whitespacesAndNewlines)
if !partial.isEmpty && lastFinal.isEmpty { if !partial.isEmpty && lastFinal.isEmpty {
lastFinal = partial lastFinal = partial
@@ -230,9 +237,10 @@ public final class LiveDictationController: ObservableObject {
if phase == .recording { if phase == .recording {
phase = .processing phase = .processing
} }
// Deactivate so the user's music resumes if the preview is
// dismissed mid-recording. Task { @MainActor [weak self] in
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) await self?.drainTailAndTeardownCapture()
}
// Safety net: if the ASR pipeline never produces a `.final` // Safety net: if the ASR pipeline never produces a `.final`
// (analyzer hang, system glitch, dropped continuation), force // (analyzer hang, system glitch, dropped continuation), force
@@ -311,6 +319,12 @@ public final class LiveDictationController: ObservableObject {
return return
} }
audioConverter = converter
self.targetFormat = targetFormat
self.hwFormat = hwFormat
drainTracker.reset()
captureGate.withLock { $0 = .recording }
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream() let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
streamRelay.bind(continuation) streamRelay.bind(continuation)
@@ -333,8 +347,16 @@ public final class LiveDictationController: ObservableObject {
} }
} }
let relay = streamRelay let relay = streamRelay
let gate = captureGate
let tracker = drainTracker
let policy = FlowCaptureTailDrainPolicy.flowDefault
let onSnapshot: @Sendable (AudioBufferSnapshot) -> Void = { snapshot in let onSnapshot: @Sendable (AudioBufferSnapshot) -> Void = { snapshot in
let phase = gate.withLock { $0 }
guard phase == .recording || phase == .draining else { return }
relay.yield(snapshot) relay.yield(snapshot)
if phase == .draining {
tracker.noteAudio(samples: snapshot.samples, policy: policy)
}
} }
let tap = Self.makeAudioTapBlock( let tap = Self.makeAudioTapBlock(
converter: converter, converter: converter,
@@ -521,7 +543,88 @@ public final class LiveDictationController: ObservableObject {
} }
} }
private func teardownCapturePipeline() { private func drainTailAndTeardownCapture() async {
let beganDrain = captureGate.withLock { phase -> Bool in
switch phase {
case .recording:
phase = .draining
return true
case .draining, .idle:
return false
}
}
guard beganDrain else { return }
drainTracker.beginDrain()
let policy = FlowCaptureTailDrainPolicy.flowDefault
while true {
let decision = drainTracker.shouldFinish(policy: policy)
if decision.finished { break }
if Task.isCancelled { break }
try? await Task.sleep(nanoseconds: 20_000_000)
}
_ = flushConverterTailToStream()
streamRelay.finish()
teardownCaptureEngine()
captureGate.withLock { $0 = .idle }
drainTracker.reset()
audioConverter = nil
targetFormat = nil
hwFormat = nil
try? AVAudioSession.sharedInstance().setActive(
false,
options: .notifyOthersOnDeactivation
)
}
@discardableResult
private func flushConverterTailToStream() -> Int {
guard let converter = audioConverter,
let targetFormat,
let hwFormat else {
return 0
}
var flushedSamples = 0
let capacity = AVAudioFrameCount(max(512, hwFormat.sampleRate / 20))
guard let outBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
return 0
}
var endOfStreamSignaled = false
while true {
outBuffer.frameLength = 0
var error: NSError?
let status = converter.convert(to: outBuffer, error: &error) { _, outStatus in
if endOfStreamSignaled {
outStatus.pointee = .noDataNow
return nil
}
endOfStreamSignaled = true
outStatus.pointee = .endOfStream
return nil
}
if status == .error || error != nil {
break
}
guard status == .haveData, outBuffer.frameLength > 0 else {
break
}
let snapshot = AudioBufferSnapshot(buffer: outBuffer)
guard !snapshot.samples.isEmpty else { break }
streamRelay.yield(snapshot)
flushedSamples += snapshot.samples.count
}
return flushedSamples
}
private func teardownCaptureEngine() {
if didInstallTap { if didInstallTap {
audioEngine.inputNode.removeTap(onBus: 0) audioEngine.inputNode.removeTap(onBus: 0)
didInstallTap = false didInstallTap = false
@@ -529,6 +632,10 @@ public final class LiveDictationController: ObservableObject {
if audioEngine.isRunning { if audioEngine.isRunning {
audioEngine.stop() audioEngine.stop()
} }
}
private func teardownCapturePipeline() {
teardownCaptureEngine()
streamRelay.finish() streamRelay.finish()
} }
@@ -0,0 +1,130 @@
// FlowCaptureTailDrain.swift
// OSGKeyboard · Shared
//
// Tail-drain policy and silence tracking for utterance end. After the user
// stops recording, capture keeps forwarding PCM until trailing speech drains
// or a safety timeout elapses (symmetric to pre-roll at utterance start).
import Foundation
/// Tunable tail-drain policy shared by Flow capture and preview dictation.
public struct FlowCaptureTailDrainPolicy: Sendable, Equatable {
/// RMS below this counts as silence while draining (16 kHz mono Float32).
public let silenceRMSThreshold: Float
/// Finish drain after this much continuous silence.
public let silenceDurationSeconds: TimeInterval
/// Hard cap so noisy environments cannot stall finalize forever.
public let maxDrainSeconds: TimeInterval
public init(
silenceRMSThreshold: Float,
silenceDurationSeconds: TimeInterval,
maxDrainSeconds: TimeInterval
) {
self.silenceRMSThreshold = silenceRMSThreshold
self.silenceDurationSeconds = silenceDurationSeconds
self.maxDrainSeconds = maxDrainSeconds
}
public static let flowDefault = FlowCaptureTailDrainPolicy(
silenceRMSThreshold: 0.015,
silenceDurationSeconds: 0.25,
maxDrainSeconds: 1.5
)
}
/// Metrics emitted when tail drain completes (for diagnostics and tests).
public struct FlowCaptureDrainReport: Sendable, Equatable {
public let drainDurationSeconds: Double
public let endedBySilence: Bool
public let tailSampleCount: Int
public init(
drainDurationSeconds: Double,
endedBySilence: Bool,
tailSampleCount: Int
) {
self.drainDurationSeconds = drainDurationSeconds
self.endedBySilence = endedBySilence
self.tailSampleCount = tailSampleCount
}
public static let skipped = FlowCaptureDrainReport(
drainDurationSeconds: 0,
endedBySilence: false,
tailSampleCount: 0
)
}
/// Thread-safe silence tracker used while draining trailing audio.
public final class FlowCaptureDrainTracker: @unchecked Sendable {
private let lock = OSAllocatedUnfairLock()
private var drainStartedAt: TimeInterval?
private var lastAudibleAt: TimeInterval?
public init() {}
public func reset() {
lock.withLock {
drainStartedAt = nil
lastAudibleAt = nil
}
}
public func beginDrain(now: TimeInterval = Date().timeIntervalSince1970) {
lock.withLock {
drainStartedAt = now
lastAudibleAt = now
}
}
public func noteAudio(
samples: [Float],
policy: FlowCaptureTailDrainPolicy,
now: TimeInterval = Date().timeIntervalSince1970
) {
guard !samples.isEmpty else { return }
let rms = Self.rms(of: samples)
lock.withLock {
guard drainStartedAt != nil else { return }
if rms >= policy.silenceRMSThreshold {
lastAudibleAt = now
}
}
}
public func shouldFinish(
policy: FlowCaptureTailDrainPolicy,
now: TimeInterval = Date().timeIntervalSince1970
) -> (finished: Bool, endedBySilence: Bool) {
lock.withLock {
guard let started = drainStartedAt else {
return (true, false)
}
let audible = lastAudibleAt ?? started
if now - started >= policy.maxDrainSeconds {
return (true, false)
}
if now - audible >= policy.silenceDurationSeconds {
return (true, true)
}
return (false, false)
}
}
public func elapsedSeconds(now: TimeInterval = Date().timeIntervalSince1970) -> Double {
lock.withLock {
guard let started = drainStartedAt else { return 0 }
return max(0, now - started)
}
}
public static func rms(of samples: [Float]) -> Float {
guard !samples.isEmpty else { return 0 }
var sum: Float = 0
for sample in samples {
sum += sample * sample
}
return sqrtf(sum / Float(samples.count))
}
}
@@ -0,0 +1,34 @@
// FlowPipelineDiagnostics.swift
// OSGKeyboard · Shared
//
// Structured Flow pipeline metrics for Console.app filtering.
import Foundation
import os
public enum FlowPipelineDiagnostics {
public static func logDrain(_ report: FlowCaptureDrainReport) {
OSGLog.flow.info(
"tailDrain duration=\(report.drainDurationSeconds, format: .fixed(precision: 2))s " +
"silenceEnd=\(report.endedBySilence) tailSamples=\(report.tailSampleCount)"
)
}
public static func logChunkFinalize(
chunkCount: Int,
lastChunkSamples: Int,
stitchedLength: Int,
chunkWarnings: Int
) {
OSGLog.flow.info(
"chunkPipeline chunks=\(chunkCount) lastChunkSamples=\(lastChunkSamples) " +
"stitchedLen=\(stitchedLength) warnings=\(chunkWarnings)"
)
}
public static func logStitcherSafeFallback(naiveLength: Int, mergedLength: Int) {
OSGLog.asr.warning(
"stitcher safe fallback naive=\(naiveLength) merged=\(mergedLength)"
)
}
}
@@ -21,6 +21,12 @@ public struct UtteranceTranscriptStitcher: Sendable {
} }
} }
/// Drop the highest-index segment (used when re-transcribing a merged tail chunk).
public mutating func removeLastSegment() {
guard !segments.isEmpty else { return }
segments.removeLast()
}
public func composed() -> String { public func composed() -> String {
guard let first = segments.first else { return "" } guard let first = segments.first else { return "" }
var result = first.text var result = first.text
@@ -30,6 +36,21 @@ public struct UtteranceTranscriptStitcher: Sendable {
return result return result
} }
/// Prefer overlap-aware merge, but fall back to naive join when dedup would drop real content.
public func composedSafely() -> String {
let merged = composed()
guard segments.count >= 2 else { return merged }
let naive = segments.map(\.text).joined(separator: " ")
if merged.count + 16 < naive.count {
FlowPipelineDiagnostics.logStitcherSafeFallback(
naiveLength: naive.count,
mergedLength: merged.count
)
return naive
}
return merged
}
/// Merge `next` onto `previous`, dropping duplicated suffix/prefix overlap. /// Merge `next` onto `previous`, dropping duplicated suffix/prefix overlap.
public static func mergeWithOverlap(previous: String, next: String) -> String { public static func mergeWithOverlap(previous: String, next: String) -> String {
let trimmedNext = next.trimmingCharacters(in: .whitespacesAndNewlines) let trimmedNext = next.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -87,6 +87,34 @@ final class ChunkedUtterancePipelineTests: XCTestCase {
XCTAssertFalse(success.text.isEmpty) XCTAssertFalse(success.text.isEmpty)
XCTAssertEqual(success.chunkWarnings.count, 1) XCTAssertEqual(success.chunkWarnings.count, 1)
} }
func testPipelineRetranscribesShortFinalChunkWithPriorOverlap() async {
let config = FlowUtteranceChunkConfig(
maxChunkDurationSeconds: 0.05,
overlapDurationSeconds: 10,
pauseExtensionMaxSeconds: 0,
pauseRMSThreshold: 0.02,
minFinalChunkDurationSeconds: 0.05,
sampleRate: 1_000
)
let asr = ShortFinalMergeStubASR()
let pipeline = ChunkedUtterancePipeline(
asr: asr,
locale: Locale(identifier: "zh-Hans"),
config: config
)
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
continuation.yield(AudioBufferSnapshot(samples: [Float](repeating: 0.1, count: 80), sampleRate: 1_000))
continuation.yield(AudioBufferSnapshot(samples: [Float](repeating: 0.1, count: 20), sampleRate: 1_000))
continuation.finish()
let outcome = await pipeline.transcribe(stream: stream) { _ in }
guard case .success(let success) = outcome else {
return XCTFail("expected success, got \(outcome)")
}
XCTAssertTrue(success.text.contains("merged"))
}
} }
private struct FailingSecondChunkASR: ASRService, @unchecked Sendable { private struct FailingSecondChunkASR: ASRService, @unchecked Sendable {
@@ -114,3 +142,32 @@ private struct FailingSecondChunkASR: ASRService, @unchecked Sendable {
return .success("seg\(samples.count)") return .success("seg\(samples.count)")
} }
} }
private struct ShortFinalMergeStubASR: ASRService, @unchecked Sendable {
private let callIndex = OSAllocatedUnfairLock(initialState: 0)
func transcribe(
stream: AsyncStream<AudioBufferSnapshot>,
locale: Locale
) -> AsyncStream<ASREvent> {
AsyncStream { $0.finish() }
}
func cancel() {}
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
_ = locale
let current = callIndex.withLock { state in
let value = state
state += 1
return value
}
if current == 0 {
return .success("head")
}
if samples.count > 20 {
return .success("merged-tail")
}
return .success("short")
}
}
@@ -0,0 +1,51 @@
// FlowCaptureTailDrainTests.swift
// OSGKeyboardTests
import XCTest
@testable import OSGKeyboardShared
final class FlowCaptureTailDrainTests: XCTestCase {
private let policy = FlowCaptureTailDrainPolicy(
silenceRMSThreshold: 0.02,
silenceDurationSeconds: 0.2,
maxDrainSeconds: 1.0
)
func testDrainFinishesAfterContinuousSilence() {
let tracker = FlowCaptureDrainTracker()
let start = Date().timeIntervalSince1970
tracker.beginDrain(now: start)
tracker.noteAudio(samples: [0.001, 0.001], policy: policy, now: start + 0.05)
let silentAt = start + 0.1
let decision = tracker.shouldFinish(policy: policy, now: silentAt + policy.silenceDurationSeconds)
XCTAssertTrue(decision.finished)
XCTAssertTrue(decision.endedBySilence)
}
func testDrainFinishesAtMaxDurationEvenWithoutSilence() {
let tracker = FlowCaptureDrainTracker()
let start = Date().timeIntervalSince1970
tracker.beginDrain(now: start)
tracker.noteAudio(samples: [0.5, 0.4], policy: policy, now: start + 0.05)
tracker.noteAudio(samples: [0.45, 0.42], policy: policy, now: start + 0.4)
let decision = tracker.shouldFinish(policy: policy, now: start + policy.maxDrainSeconds)
XCTAssertTrue(decision.finished)
XCTAssertFalse(decision.endedBySilence)
}
func testRMSDetectsAudibleSamples() {
XCTAssertGreaterThan(
FlowCaptureDrainTracker.rms(of: [0.2, 0.18, 0.15]),
policy.silenceRMSThreshold
)
XCTAssertLessThan(
FlowCaptureDrainTracker.rms(of: [0.001, 0.0005]),
policy.silenceRMSThreshold
)
}
}
@@ -63,4 +63,33 @@ final class UtteranceStreamChunkerTests: XCTestCase {
XCTAssertGreaterThanOrEqual(received.count, 2) XCTAssertGreaterThanOrEqual(received.count, 2)
XCTAssertTrue(received.last?.isLast == true) XCTAssertTrue(received.last?.isLast == true)
} }
func testFinalChunkIncludesLateArrivingTailSamples() async {
let config = FlowUtteranceChunkConfig(
firstChunkDurationSeconds: 0.5,
subsequentChunkDurationSeconds: 1.0,
overlapDurationSeconds: 0,
pauseExtensionMaxSeconds: 0,
pauseRMSThreshold: 0.02,
sampleRate: 1_000
)
let head = [Float](repeating: 0.05, count: 600)
let tail = [Float](repeating: 0.08, count: 250)
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
continuation.yield(AudioBufferSnapshot(samples: head, sampleRate: Double(config.sampleRate)))
continuation.yield(AudioBufferSnapshot(samples: tail, sampleRate: Double(config.sampleRate)))
continuation.finish()
var received: [UtteranceAudioChunk] = []
for await chunk in UtteranceStreamChunker.chunks(from: stream, config: config) {
received.append(chunk)
}
guard let last = received.last else {
return XCTFail("expected at least one chunk")
}
XCTAssertTrue(last.isLast)
XCTAssertGreaterThanOrEqual(last.samples.count, tail.count)
}
} }
@@ -20,4 +20,24 @@ final class UtteranceTranscriptStitcherTests: XCTestCase {
stitcher.append(index: 0, text: "第一段") stitcher.append(index: 0, text: "第一段")
XCTAssertEqual(stitcher.composed(), "第一段 第二段") XCTAssertEqual(stitcher.composed(), "第一段 第二段")
} }
func testComposedSafelyFallsBackWhenOverlapMergeShortensTooMuch() {
var stitcher = UtteranceTranscriptStitcher()
stitcher.append(index: 0, text: "今天天气很好我们")
stitcher.append(index: 1, text: "去公园")
let merged = stitcher.composed()
let safe = stitcher.composedSafely()
XCTAssertFalse(merged.isEmpty)
XCTAssertFalse(safe.isEmpty)
XCTAssertTrue(safe.contains("去公园"))
}
func testRemoveLastSegmentSupportsMergedTailRetranscription() {
var stitcher = UtteranceTranscriptStitcher()
stitcher.append(index: 0, text: "第一段")
stitcher.append(index: 1, text: "第二段")
stitcher.removeLastSegment()
stitcher.append(index: 1, text: "第二段合并")
XCTAssertEqual(stitcher.composed(), "第一段 第二段合并")
}
} }