merge: tail audio ASR fix (P0+P1) into feature/polish-style-packs

Co-authored-by: Rocky <hkgood@users.noreply.github.com>
2026-07-26 10:34:05 +00:00
parent 9d8914fbf3 f0240c3088
commit f108460302
20 changed files with 741 additions and 45 deletions
@@ -101,6 +101,7 @@ public actor ChunkedUtterancePipeline {
        var processedChunks = 0
        var previousChunkSamples: [Float] = []
        var lastChunkSamples = 0
+        var didRetryEmptyFinal = false

        let feeder = Task {
            for await chunk in UtteranceStreamChunker.chunks(from: stream, config: config) {
@@ -118,20 +119,28 @@ public actor ChunkedUtterancePipeline {

            guard let chunk = await queue.dequeue() else { break }

+            if chunk.isLast && chunk.samples.isEmpty {
+                continue
+            }
+
            processedChunks += 1
            lastChunkSamples = chunk.samples.count

-            if chunk.isLast,
-               chunk.samples.count < config.minFinalChunkSamples,
-               processedChunks > 1,
-               !previousChunkSamples.isEmpty {
-                let mergedSamples = Array(previousChunkSamples.suffix(config.overlapSamples))
-                    + chunk.samples
-                let mergedResult = await transcribeChunk(samples: mergedSamples)
+            if let preMerge = FinalChunkRecovery.preMergePlan(
+                chunk: chunk,
+                processedChunks: processedChunks,
+                previousChunkSamples: previousChunkSamples,
+                config: config
+            ) {
+                FlowPipelineDiagnostics.logFinalChunkRecovery(
+                    action: "preMerge",
+                    chunkIndex: chunk.index
+                )
+                let mergedResult = await transcribeChunk(samples: preMerge.samples)
                switch mergedResult {
                case .success(let text):
                    stitcher.removeLastSegment()
-                    stitcher.append(index: max(0, chunk.index - 1), text: text)
+                    stitcher.append(index: preMerge.stitchIndex, text: text)
                    publishPartial(from: stitcher, onPartial: onPartial)
                case .failure(let message):
                    failedChunks += 1
@@ -153,8 +162,52 @@ public actor ChunkedUtterancePipeline {
            let result = await transcribeChunk(samples: chunk.samples)
            switch result {
            case .success(let text):
-                stitcher.append(index: chunk.index, text: text)
-                publishPartial(from: stitcher, onPartial: onPartial)
+                if chunk.isLast,
+                   !didRetryEmptyFinal,
+                   let retry = FinalChunkRecovery.emptyResultRetryPlan(
+                       chunk: chunk,
+                       previousChunkSamples: previousChunkSamples,
+                       config: config,
+                       asrText: text
+                   ) {
+                    didRetryEmptyFinal = true
+                    FlowPipelineDiagnostics.logFinalChunkRecovery(
+                        action: "emptyRetry",
+                        chunkIndex: chunk.index
+                    )
+                    let retryResult = await transcribeChunk(samples: retry.samples)
+                    switch retryResult {
+                    case .success(let retryText):
+                        let trimmed = retryText.trimmingCharacters(in: .whitespacesAndNewlines)
+                        if !trimmed.isEmpty {
+                            if retry.stitchIndex < chunk.index {
+                                stitcher.removeLastSegment()
+                            }
+                            stitcher.append(index: retry.stitchIndex, text: retryText)
+                            publishPartial(from: stitcher, onPartial: onPartial)
+                        } else {
+                            stitcher.append(index: chunk.index, text: text)
+                            publishPartial(from: stitcher, onPartial: onPartial)
+                        }
+                    case .failure(let message):
+                        stitcher.append(index: chunk.index, text: text)
+                        publishPartial(from: stitcher, onPartial: onPartial)
+                        failedChunks += 1
+                        chunkWarnings.append(
+                            SharedL10n.format(
+                                "error.asr.chunkFailed",
+                                chunk.index + 1,
+                                message
+                            )
+                        )
+                    case .cancelled:
+                        feeder.cancel()
+                        return .cancelled
+                    }
+                } else {
+                    stitcher.append(index: chunk.index, text: text)
+                    publishPartial(from: stitcher, onPartial: onPartial)
+                }
            case .failure(let message):
                failedChunks += 1
                chunkWarnings.append(
@@ -281,6 +281,9 @@ public final class FlowContinuousCapture {
    private let gate = OSAllocatedUnfairLock(initialState: UtteranceGatePhase.idle)
    private let drainTracker = FlowCaptureDrainTracker()
    private let tailSampleCounter = OSAllocatedUnfairLock(initialState: 0)
+    private let utterancePCMStore = FlowUtterancePCMStore(
+        maxSampleCount: Int(FlowSessionKeys.maxUtteranceDuration) * 16_000
+    )

    private var downsampler: AdaptiveDownsampler?
    private var targetFormat: AVAudioFormat?
@@ -412,6 +415,7 @@ public final class FlowContinuousCapture {
        let proof = audioProofStore
        let tracker = drainTracker
        let tailCounter = tailSampleCounter
+        let pcmStore = utterancePCMStore
        let policy = drainPolicy
        let tap = Self.makeAudioTapBlock(
            downsampler: downsampler,
@@ -422,6 +426,7 @@ public final class FlowContinuousCapture {
            streamRelay: relay,
            drainTracker: tracker,
            tailSampleCounter: tailCounter,
+            utterancePCMStore: pcmStore,
            drainPolicy: policy
        )
        // `format: nil` binds the tap to the input node's *live* format. Passing
@@ -645,6 +650,7 @@ public final class FlowContinuousCapture {
        let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
        drainTracker.reset()
        tailSampleCounter.withLock { $0 = 0 }
+        utterancePCMStore.reset()
        // Bind the consumer before opening the gate so early tap frames
        // are not dropped on the floor.
        streamRelay.bind(continuation)
@@ -666,16 +672,11 @@ public final class FlowContinuousCapture {
        gate.withLock { $0 = .draining }
        drainTracker.beginDrain()

-        var endedBySilence = false
-        while true {
-            let decision = drainTracker.shouldFinish(policy: policy)
-            if decision.finished {
-                endedBySilence = decision.endedBySilence
-                break
-            }
-            if Task.isCancelled { break }
-            try? await Task.sleep(nanoseconds: FlowCaptureConstants.drainPollIntervalNs)
-        }
+        let timing = await FlowUtteranceEndCoordinator.awaitTailCapture(
+            tracker: drainTracker,
+            policy: policy,
+            pollIntervalNs: FlowCaptureConstants.drainPollIntervalNs
+        )

        // NOTE: We intentionally do NOT signal `.endOfStream` to the shared
        // downsampling converter here. `AVAudioConverter` is stateful: once its
@@ -692,8 +693,9 @@ public final class FlowContinuousCapture {
        let tailSamples = tailSampleCounter.withLock { $0 }
        let report = FlowCaptureDrainReport(
            drainDurationSeconds: drainTracker.elapsedSeconds(),
-            endedBySilence: endedBySilence,
-            tailSampleCount: tailSamples
+            endedBySilence: timing.endedBySilence,
+            tailSampleCount: tailSamples,
+            postRollDurationSeconds: timing.postRollDurationSeconds
        )
        drainTracker.reset()
        tailSampleCounter.withLock { $0 = 0 }
@@ -701,11 +703,17 @@ public final class FlowContinuousCapture {
        return report
    }

+    /// Returns the utterance PCM accumulated during the last recording cycle.
+    public func consumeUtteranceSamples() -> [Float] {
+        utterancePCMStore.consume()
+    }
+
    /// Immediate stop without tail drain (abort / session teardown).
    public func cancelUtterance() {
        gate.withLock { $0 = .idle }
        drainTracker.reset()
        tailSampleCounter.withLock { $0 = 0 }
+        utterancePCMStore.reset()
        streamRelay.finish()
    }

@@ -724,6 +732,7 @@ public final class FlowContinuousCapture {
        streamRelay: FlowCaptureStreamRelay,
        drainTracker: FlowCaptureDrainTracker,
        tailSampleCounter: OSAllocatedUnfairLock<Int>,
+        utterancePCMStore: FlowUtterancePCMStore,
        drainPolicy: FlowCaptureTailDrainPolicy
    ) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
        return { buffer, _ in
@@ -743,6 +752,7 @@ public final class FlowContinuousCapture {
            let phase = gate.withLock { $0 }
            switch phase {
            case .recording, .draining:
+                utterancePCMStore.append(snapshot.samples)
                streamRelay.yield(snapshot)
                if phase == .draining {
                    drainTracker.noteAudio(samples: snapshot.samples, policy: drainPolicy)
@@ -558,12 +558,10 @@ public final class LiveDictationController: ObservableObject {
        drainTracker.beginDrain()

        let policy = FlowCaptureTailDrainPolicy.flowDefault
-        while true {
-            let decision = drainTracker.shouldFinish(policy: policy)
-            if decision.finished { break }
-            if Task.isCancelled { break }
-            try? await Task.sleep(nanoseconds: 20_000_000)
-        }
+        _ = await FlowUtteranceEndCoordinator.awaitTailCapture(
+            tracker: drainTracker,
+            policy: policy
+        )

        // Trailing speech is preserved by the live `.draining` forwarding
        // loop above. We deliberately do NOT signal `.endOfStream` to the
@@ -0,0 +1,47 @@
+// FinalChunkRecovery.swift
+// OSGKeyboard · Shared
+//
+// Recovery plans for pipelined utterance ASR when the final chunk is short,
+// empty, or straddles a chunk boundary.
+
+import Foundation
+
+public enum FinalChunkRecovery {
+
+    /// Samples and stitch index when the final chunk should be merged with
+    /// prior overlap *before* the first ASR pass.
+    public static func preMergePlan(
+        chunk: UtteranceAudioChunk,
+        processedChunks: Int,
+        previousChunkSamples: [Float],
+        config: FlowUtteranceChunkConfig
+    ) -> (samples: [Float], stitchIndex: Int)? {
+        guard chunk.isLast, !chunk.samples.isEmpty, !previousChunkSamples.isEmpty else {
+            return nil
+        }
+        guard chunk.samples.count < config.minFinalChunkSamples else { return nil }
+
+        let merged = Array(previousChunkSamples.suffix(config.overlapSamples)) + chunk.samples
+        return (merged, max(0, chunk.index - 1))
+    }
+
+    /// Retry plan when the final chunk had audio but ASR returned empty text.
+    public static func emptyResultRetryPlan(
+        chunk: UtteranceAudioChunk,
+        previousChunkSamples: [Float],
+        config: FlowUtteranceChunkConfig,
+        asrText: String
+    ) -> (samples: [Float], stitchIndex: Int)? {
+        guard chunk.isLast, !chunk.samples.isEmpty else { return nil }
+        guard asrText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
+            return nil
+        }
+
+        if previousChunkSamples.isEmpty {
+            return (chunk.samples, chunk.index)
+        }
+
+        let merged = Array(previousChunkSamples.suffix(config.overlapSamples)) + chunk.samples
+        return (merged, max(0, chunk.index - 1))
+    }
+}
@@ -16,22 +16,39 @@ public struct FlowCaptureTailDrainPolicy: Sendable, Equatable {
    public let silenceDurationSeconds: TimeInterval
    /// Hard cap so noisy environments cannot stall finalize forever.
    public let maxDrainSeconds: TimeInterval
+    /// Fixed post-roll after silence drain; independent of RMS (captures weak tails).
+    public let postRollSeconds: TimeInterval

    public init(
        silenceRMSThreshold: Float,
        silenceDurationSeconds: TimeInterval,
-        maxDrainSeconds: TimeInterval
+        maxDrainSeconds: TimeInterval,
+        postRollSeconds: TimeInterval = 0
    ) {
        self.silenceRMSThreshold = silenceRMSThreshold
        self.silenceDurationSeconds = silenceDurationSeconds
        self.maxDrainSeconds = maxDrainSeconds
+        self.postRollSeconds = postRollSeconds
    }

-    public static let flowDefault = FlowCaptureTailDrainPolicy(
+    /// iOS Flow host + keyboard utterance capture.
+    public static let iosFlow = FlowCaptureTailDrainPolicy(
        silenceRMSThreshold: 0.015,
-        silenceDurationSeconds: 0.25,
-        maxDrainSeconds: 1.5
+        silenceDurationSeconds: 0.35,
+        maxDrainSeconds: 1.5,
+        postRollSeconds: 0.15
    )
+
+    /// Mac MLX streaming live capture.
+    public static let macMLX = FlowCaptureTailDrainPolicy(
+        silenceRMSThreshold: 0.015,
+        silenceDurationSeconds: 0.35,
+        maxDrainSeconds: 0.75,
+        postRollSeconds: 0.15
+    )
+
+    /// Backward-compatible alias for iOS Flow defaults.
+    public static let flowDefault = iosFlow
 }

 /// Metrics emitted when tail drain completes (for diagnostics and tests).
@@ -39,21 +56,25 @@ public struct FlowCaptureDrainReport: Sendable, Equatable {
    public let drainDurationSeconds: Double
    public let endedBySilence: Bool
    public let tailSampleCount: Int
+    public let postRollDurationSeconds: Double

    public init(
        drainDurationSeconds: Double,
        endedBySilence: Bool,
-        tailSampleCount: Int
+        tailSampleCount: Int,
+        postRollDurationSeconds: Double = 0
    ) {
        self.drainDurationSeconds = drainDurationSeconds
        self.endedBySilence = endedBySilence
        self.tailSampleCount = tailSampleCount
+        self.postRollDurationSeconds = postRollDurationSeconds
    }

    public static let skipped = FlowCaptureDrainReport(
        drainDurationSeconds: 0,
        endedBySilence: false,
-        tailSampleCount: 0
+        tailSampleCount: 0,
+        postRollDurationSeconds: 0
    )
 }

@@ -9,7 +9,31 @@ import os
 public enum FlowPipelineDiagnostics {
    public static func logDrain(_ report: FlowCaptureDrainReport) {
        OSGLog.flow.info(
-            "tailDrain duration=\(report.drainDurationSeconds, format: .fixed(precision: 2))s silenceEnd=\(report.endedBySilence) tailSamples=\(report.tailSampleCount)"
+            "tailDrain duration=\(report.drainDurationSeconds, format: .fixed(precision: 2))s " +
+            "postRoll=\(report.postRollDurationSeconds, format: .fixed(precision: 2))s " +
+            "silenceEnd=\(report.endedBySilence) tailSamples=\(report.tailSampleCount)"
+        )
+    }
+
+    public static func logTranscriptGuardUsedPartial(finalLength: Int, partialLength: Int) {
+        OSGLog.flow.warning(
+            "transcriptGuard partial preferred finalLen=\(finalLength) partialLen=\(partialLength)"
+        )
+    }
+
+    public static func logFinalChunkRecovery(action: String, chunkIndex: Int) {
+        OSGLog.asr.info("finalChunkRecovery \(action) chunk=\(chunkIndex)")
+    }
+
+    public static func logBatchFallback(
+        sampleCount: Int,
+        stitchedLength: Int,
+        partialLength: Int,
+        batchLength: Int
+    ) {
+        OSGLog.flow.info(
+            "batchFallback samples=\(sampleCount) stitchedLen=\(stitchedLength) " +
+            "partialLen=\(partialLength) batchLen=\(batchLength)"
        )
    }

@@ -0,0 +1,66 @@
+// FlowUtteranceEndCoordinator.swift
+// OSGKeyboard · Shared
+//
+// Unified tail-drain + post-roll orchestration after the user stops recording.
+// Keeps the mic gate open through silence detection, then a fixed post-roll
+// window that does not depend on RMS (captures weak trailing syllables).
+
+import Foundation
+
+/// Outcome of `FlowUtteranceEndCoordinator.awaitTailCapture`.
+public struct FlowUtteranceEndTiming: Sendable, Equatable {
+    public let endedBySilence: Bool
+    public let postRollDurationSeconds: Double
+
+    public init(endedBySilence: Bool, postRollDurationSeconds: Double) {
+        self.endedBySilence = endedBySilence
+        self.postRollDurationSeconds = postRollDurationSeconds
+    }
+}
+
+public enum FlowUtteranceEndCoordinator {
+    /// Poll interval while waiting for silence / max drain (20 ms).
+    public static let pollIntervalNs: UInt64 = 20_000_000
+
+    /// Waits until trailing speech drains (silence or max cap), then sleeps
+    /// through a fixed post-roll window so weak tail audio still reaches ASR.
+    ///
+    /// Callers must keep forwarding PCM from the audio tap while this runs
+    /// (gate `.draining` or equivalent).
+    public static func awaitTailCapture(
+        tracker: FlowCaptureDrainTracker,
+        policy: FlowCaptureTailDrainPolicy,
+        pollIntervalNs: UInt64 = pollIntervalNs
+    ) async -> FlowUtteranceEndTiming {
+        var endedBySilence = false
+        while true {
+            let decision = tracker.shouldFinish(policy: policy)
+            if decision.finished {
+                endedBySilence = decision.endedBySilence
+                break
+            }
+            if Task.isCancelled { break }
+            try? await Task.sleep(nanoseconds: pollIntervalNs)
+        }
+
+        let postRoll = await runPostRoll(policy: policy, pollIntervalNs: pollIntervalNs)
+        return FlowUtteranceEndTiming(
+            endedBySilence: endedBySilence,
+            postRollDurationSeconds: postRoll
+        )
+    }
+
+    private static func runPostRoll(
+        policy: FlowCaptureTailDrainPolicy,
+        pollIntervalNs: UInt64
+    ) async -> Double {
+        guard policy.postRollSeconds > 0 else { return 0 }
+        let started = Date()
+        let deadline = started.addingTimeInterval(policy.postRollSeconds)
+        while Date() < deadline {
+            if Task.isCancelled { break }
+            try? await Task.sleep(nanoseconds: pollIntervalNs)
+        }
+        return max(0, Date().timeIntervalSince(started))
+    }
+}
@@ -0,0 +1,46 @@
+// FlowUtterancePCMStore.swift
+// OSGKeyboard · Shared
+//
+// Thread-safe rolling buffer of 16 kHz mono utterance PCM for whole-utterance
+// batch ASR fallback when pipelined chunking drops weak tail segments.
+
+import Foundation
+
+public final class FlowUtterancePCMStore: @unchecked Sendable {
+    private let lock = OSAllocatedUnfairLock()
+    private var samples: [Float] = []
+    private let maxSampleCount: Int
+
+    public init(maxSampleCount: Int) {
+        self.maxSampleCount = max(1, maxSampleCount)
+    }
+
+    public func reset() {
+        lock.withLock {
+            samples.removeAll(keepingCapacity: false)
+        }
+    }
+
+    public func append(_ chunk: [Float]) {
+        guard !chunk.isEmpty else { return }
+        lock.withLock {
+            samples.append(contentsOf: chunk)
+            if samples.count > maxSampleCount {
+                samples.removeFirst(samples.count - maxSampleCount)
+            }
+        }
+    }
+
+    public var sampleCount: Int {
+        lock.withLock { samples.count }
+    }
+
+    /// Returns accumulated samples and clears the store.
+    public func consume() -> [Float] {
+        lock.withLock {
+            let out = samples
+            samples.removeAll(keepingCapacity: false)
+            return out
+        }
+    }
+}
@@ -0,0 +1,41 @@
+// UtteranceBatchFallbackPolicy.swift
+// OSGKeyboard · Shared
+//
+// Decides when to re-run ASR on the full utterance PCM after pipelined
+// chunking, and how to pick the best transcript among candidates.
+
+import Foundation
+
+public enum UtteranceBatchFallbackPolicy {
+    public static let defaultCharacterAdvantage = UtteranceTranscriptGuard.defaultPartialAdvantage
+
+    /// True when chunked output likely lost trailing content vs the live partial.
+    public static func shouldRunBatchFallback(
+        stitchedFinal: String,
+        partialSnapshot: String,
+        minimumCharacterAdvantage: Int = defaultCharacterAdvantage
+    ) -> Bool {
+        let final = stitchedFinal.trimmingCharacters(in: .whitespacesAndNewlines)
+        let partial = partialSnapshot.trimmingCharacters(in: .whitespacesAndNewlines)
+
+        if final.isEmpty, !partial.isEmpty { return true }
+        if partial.isEmpty { return false }
+        return partial.count >= final.count + minimumCharacterAdvantage
+    }
+
+    /// Prefer the longest non-empty transcript after batch ASR completes.
+    public static func preferredTranscript(
+        batch: String,
+        stitchedFinal: String,
+        partialSnapshot: String,
+        current: String
+    ) -> String {
+        let candidates = [batch, current, stitchedFinal, partialSnapshot]
+            .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
+            .filter { !$0.isEmpty }
+        guard let best = candidates.max(by: { $0.count < $1.count }) else {
+            return current.trimmingCharacters(in: .whitespacesAndNewlines)
+        }
+        return best
+    }
+}
@@ -56,7 +56,11 @@ public enum UtteranceStreamChunker {
                } else if chunkIndex == 0 {
                    // Empty utterance — no chunks.
                } else {
-                    // Stream ended exactly on boundary; mark prior path complete.
+                    // Stream ended exactly on a chunk boundary; prior emit holds
+                    // all tail audio. Marker so FinalChunkRecovery paths run.
+                    continuation.yield(
+                        UtteranceAudioChunk(index: chunkIndex, samples: [], isLast: true)
+                    )
                }

                continuation.finish()
@@ -0,0 +1,35 @@
+// UtteranceTranscriptGuard.swift
+// OSGKeyboard · Shared
+//
+// Chooses the best available transcript when pipelined final text may have
+// dropped a weak tail segment.
+
+import Foundation
+
+public enum UtteranceTranscriptGuard {
+    /// Partial must exceed final by at least this many characters to win.
+    public static let defaultPartialAdvantage = 8
+
+    /// Prefer `stitchedFinal` unless empty or clearly shorter than the live
+    /// partial snapshot taken at mic stop.
+    public static func resolve(
+        stitchedFinal: String,
+        partialSnapshot: String,
+        minimumPartialAdvantage: Int = defaultPartialAdvantage
+    ) -> String {
+        let final = stitchedFinal.trimmingCharacters(in: .whitespacesAndNewlines)
+        let partial = partialSnapshot.trimmingCharacters(in: .whitespacesAndNewlines)
+
+        if final.isEmpty { return partial }
+        if partial.isEmpty { return final }
+
+        if partial.count >= final.count + minimumPartialAdvantage {
+            FlowPipelineDiagnostics.logTranscriptGuardUsedPartial(
+                finalLength: final.count,
+                partialLength: partial.count
+            )
+            return partial
+        }
+        return final
+    }
+}