fix(flow): improve tail capture and final chunk ASR recovery
- Add FlowUtteranceEndCoordinator with 350ms silence drain and 150ms post-roll - Extend FinalChunkRecovery for short/empty final chunks in chunked pipeline - Snapshot partial at mic stop and guard final transcript in FlowSessionManager - Unify tail drain presets (iosFlow/macMLX) and expand diagnostics Co-authored-by: Rocky <hkgood@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
// FinalChunkRecovery.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Recovery plans for pipelined utterance ASR when the final chunk is short,
|
||||
// empty, or straddles a chunk boundary.
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum FinalChunkRecovery {
|
||||
|
||||
/// Samples and stitch index when the final chunk should be merged with
|
||||
/// prior overlap *before* the first ASR pass.
|
||||
public static func preMergePlan(
|
||||
chunk: UtteranceAudioChunk,
|
||||
processedChunks: Int,
|
||||
previousChunkSamples: [Float],
|
||||
config: FlowUtteranceChunkConfig
|
||||
) -> (samples: [Float], stitchIndex: Int)? {
|
||||
guard chunk.isLast, !chunk.samples.isEmpty, !previousChunkSamples.isEmpty else {
|
||||
return nil
|
||||
}
|
||||
guard chunk.samples.count < config.minFinalChunkSamples else { return nil }
|
||||
|
||||
let merged = Array(previousChunkSamples.suffix(config.overlapSamples)) + chunk.samples
|
||||
return (merged, max(0, chunk.index - 1))
|
||||
}
|
||||
|
||||
/// Retry plan when the final chunk had audio but ASR returned empty text.
|
||||
public static func emptyResultRetryPlan(
|
||||
chunk: UtteranceAudioChunk,
|
||||
previousChunkSamples: [Float],
|
||||
config: FlowUtteranceChunkConfig,
|
||||
asrText: String
|
||||
) -> (samples: [Float], stitchIndex: Int)? {
|
||||
guard chunk.isLast, !chunk.samples.isEmpty else { return nil }
|
||||
guard asrText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
|
||||
return nil
|
||||
}
|
||||
|
||||
if previousChunkSamples.isEmpty {
|
||||
return (chunk.samples, chunk.index)
|
||||
}
|
||||
|
||||
let merged = Array(previousChunkSamples.suffix(config.overlapSamples)) + chunk.samples
|
||||
return (merged, max(0, chunk.index - 1))
|
||||
}
|
||||
}
|
||||
@@ -16,22 +16,39 @@ public struct FlowCaptureTailDrainPolicy: Sendable, Equatable {
|
||||
public let silenceDurationSeconds: TimeInterval
|
||||
/// Hard cap so noisy environments cannot stall finalize forever.
|
||||
public let maxDrainSeconds: TimeInterval
|
||||
/// Fixed post-roll after silence drain; independent of RMS (captures weak tails).
|
||||
public let postRollSeconds: TimeInterval
|
||||
|
||||
public init(
|
||||
silenceRMSThreshold: Float,
|
||||
silenceDurationSeconds: TimeInterval,
|
||||
maxDrainSeconds: TimeInterval
|
||||
maxDrainSeconds: TimeInterval,
|
||||
postRollSeconds: TimeInterval = 0
|
||||
) {
|
||||
self.silenceRMSThreshold = silenceRMSThreshold
|
||||
self.silenceDurationSeconds = silenceDurationSeconds
|
||||
self.maxDrainSeconds = maxDrainSeconds
|
||||
self.postRollSeconds = postRollSeconds
|
||||
}
|
||||
|
||||
public static let flowDefault = FlowCaptureTailDrainPolicy(
|
||||
/// iOS Flow host + keyboard utterance capture.
|
||||
public static let iosFlow = FlowCaptureTailDrainPolicy(
|
||||
silenceRMSThreshold: 0.015,
|
||||
silenceDurationSeconds: 0.25,
|
||||
maxDrainSeconds: 1.5
|
||||
silenceDurationSeconds: 0.35,
|
||||
maxDrainSeconds: 1.5,
|
||||
postRollSeconds: 0.15
|
||||
)
|
||||
|
||||
/// Mac MLX streaming live capture.
|
||||
public static let macMLX = FlowCaptureTailDrainPolicy(
|
||||
silenceRMSThreshold: 0.015,
|
||||
silenceDurationSeconds: 0.35,
|
||||
maxDrainSeconds: 0.75,
|
||||
postRollSeconds: 0.15
|
||||
)
|
||||
|
||||
/// Backward-compatible alias for iOS Flow defaults.
|
||||
public static let flowDefault = iosFlow
|
||||
}
|
||||
|
||||
/// Metrics emitted when tail drain completes (for diagnostics and tests).
|
||||
@@ -39,21 +56,25 @@ public struct FlowCaptureDrainReport: Sendable, Equatable {
|
||||
public let drainDurationSeconds: Double
|
||||
public let endedBySilence: Bool
|
||||
public let tailSampleCount: Int
|
||||
public let postRollDurationSeconds: Double
|
||||
|
||||
public init(
|
||||
drainDurationSeconds: Double,
|
||||
endedBySilence: Bool,
|
||||
tailSampleCount: Int
|
||||
tailSampleCount: Int,
|
||||
postRollDurationSeconds: Double = 0
|
||||
) {
|
||||
self.drainDurationSeconds = drainDurationSeconds
|
||||
self.endedBySilence = endedBySilence
|
||||
self.tailSampleCount = tailSampleCount
|
||||
self.postRollDurationSeconds = postRollDurationSeconds
|
||||
}
|
||||
|
||||
public static let skipped = FlowCaptureDrainReport(
|
||||
drainDurationSeconds: 0,
|
||||
endedBySilence: false,
|
||||
tailSampleCount: 0
|
||||
tailSampleCount: 0,
|
||||
postRollDurationSeconds: 0
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -9,10 +9,22 @@ import os
|
||||
public enum FlowPipelineDiagnostics {
|
||||
public static func logDrain(_ report: FlowCaptureDrainReport) {
|
||||
OSGLog.flow.info(
|
||||
"tailDrain duration=\(report.drainDurationSeconds, format: .fixed(precision: 2))s silenceEnd=\(report.endedBySilence) tailSamples=\(report.tailSampleCount)"
|
||||
"tailDrain duration=\(report.drainDurationSeconds, format: .fixed(precision: 2))s " +
|
||||
"postRoll=\(report.postRollDurationSeconds, format: .fixed(precision: 2))s " +
|
||||
"silenceEnd=\(report.endedBySilence) tailSamples=\(report.tailSampleCount)"
|
||||
)
|
||||
}
|
||||
|
||||
public static func logTranscriptGuardUsedPartial(finalLength: Int, partialLength: Int) {
|
||||
OSGLog.flow.warning(
|
||||
"transcriptGuard partial preferred finalLen=\(finalLength) partialLen=\(partialLength)"
|
||||
)
|
||||
}
|
||||
|
||||
public static func logFinalChunkRecovery(action: String, chunkIndex: Int) {
|
||||
OSGLog.asr.info("finalChunkRecovery \(action) chunk=\(chunkIndex)")
|
||||
}
|
||||
|
||||
public static func logChunkFinalize(
|
||||
chunkCount: Int,
|
||||
lastChunkSamples: Int,
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
// FlowUtteranceEndCoordinator.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Unified tail-drain + post-roll orchestration after the user stops recording.
|
||||
// Keeps the mic gate open through silence detection, then a fixed post-roll
|
||||
// window that does not depend on RMS (captures weak trailing syllables).
|
||||
|
||||
import Foundation
|
||||
|
||||
/// Outcome of `FlowUtteranceEndCoordinator.awaitTailCapture`.
|
||||
public struct FlowUtteranceEndTiming: Sendable, Equatable {
|
||||
public let endedBySilence: Bool
|
||||
public let postRollDurationSeconds: Double
|
||||
|
||||
public init(endedBySilence: Bool, postRollDurationSeconds: Double) {
|
||||
self.endedBySilence = endedBySilence
|
||||
self.postRollDurationSeconds = postRollDurationSeconds
|
||||
}
|
||||
}
|
||||
|
||||
public enum FlowUtteranceEndCoordinator {
|
||||
/// Poll interval while waiting for silence / max drain (20 ms).
|
||||
public static let pollIntervalNs: UInt64 = 20_000_000
|
||||
|
||||
/// Waits until trailing speech drains (silence or max cap), then sleeps
|
||||
/// through a fixed post-roll window so weak tail audio still reaches ASR.
|
||||
///
|
||||
/// Callers must keep forwarding PCM from the audio tap while this runs
|
||||
/// (gate `.draining` or equivalent).
|
||||
public static func awaitTailCapture(
|
||||
tracker: FlowCaptureDrainTracker,
|
||||
policy: FlowCaptureTailDrainPolicy,
|
||||
pollIntervalNs: UInt64 = pollIntervalNs
|
||||
) async -> FlowUtteranceEndTiming {
|
||||
var endedBySilence = false
|
||||
while true {
|
||||
let decision = tracker.shouldFinish(policy: policy)
|
||||
if decision.finished {
|
||||
endedBySilence = decision.endedBySilence
|
||||
break
|
||||
}
|
||||
if Task.isCancelled { break }
|
||||
try? await Task.sleep(nanoseconds: pollIntervalNs)
|
||||
}
|
||||
|
||||
let postRoll = await runPostRoll(policy: policy, pollIntervalNs: pollIntervalNs)
|
||||
return FlowUtteranceEndTiming(
|
||||
endedBySilence: endedBySilence,
|
||||
postRollDurationSeconds: postRoll
|
||||
)
|
||||
}
|
||||
|
||||
private static func runPostRoll(
|
||||
policy: FlowCaptureTailDrainPolicy,
|
||||
pollIntervalNs: UInt64
|
||||
) async -> Double {
|
||||
guard policy.postRollSeconds > 0 else { return 0 }
|
||||
let started = Date()
|
||||
let deadline = started.addingTimeInterval(policy.postRollSeconds)
|
||||
while Date() < deadline {
|
||||
if Task.isCancelled { break }
|
||||
try? await Task.sleep(nanoseconds: pollIntervalNs)
|
||||
}
|
||||
return max(0, Date().timeIntervalSince(started))
|
||||
}
|
||||
}
|
||||
@@ -56,7 +56,11 @@ public enum UtteranceStreamChunker {
|
||||
} else if chunkIndex == 0 {
|
||||
// Empty utterance — no chunks.
|
||||
} else {
|
||||
// Stream ended exactly on boundary; mark prior path complete.
|
||||
// Stream ended exactly on a chunk boundary; prior emit holds
|
||||
// all tail audio. Marker so FinalChunkRecovery paths run.
|
||||
continuation.yield(
|
||||
UtteranceAudioChunk(index: chunkIndex, samples: [], isLast: true)
|
||||
)
|
||||
}
|
||||
|
||||
continuation.finish()
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
// UtteranceTranscriptGuard.swift
|
||||
// OSGKeyboard · Shared
|
||||
//
|
||||
// Chooses the best available transcript when pipelined final text may have
|
||||
// dropped a weak tail segment.
|
||||
|
||||
import Foundation
|
||||
|
||||
public enum UtteranceTranscriptGuard {
|
||||
/// Partial must exceed final by at least this many characters to win.
|
||||
public static let defaultPartialAdvantage = 8
|
||||
|
||||
/// Prefer `stitchedFinal` unless empty or clearly shorter than the live
|
||||
/// partial snapshot taken at mic stop.
|
||||
public static func resolve(
|
||||
stitchedFinal: String,
|
||||
partialSnapshot: String,
|
||||
minimumPartialAdvantage: Int = defaultPartialAdvantage
|
||||
) -> String {
|
||||
let final = stitchedFinal.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let partial = partialSnapshot.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
|
||||
if final.isEmpty { return partial }
|
||||
if partial.isEmpty { return final }
|
||||
|
||||
if partial.count >= final.count + minimumPartialAdvantage {
|
||||
FlowPipelineDiagnostics.logTranscriptGuardUsedPartial(
|
||||
finalLength: final.count,
|
||||
partialLength: partial.count
|
||||
)
|
||||
return partial
|
||||
}
|
||||
return final
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user