feat: flow session reliability, keyboard UX, and audio pre-roll fix
Renew voice sessions while the host app stays foreground, auto-start flow from the keyboard with a Start action, and prevent leading audio loss via pre-roll buffering and faster signal polling.
This commit is contained in:
@@ -25,6 +25,14 @@ private final class FlowCaptureStreamRelay: @unchecked Sendable {
|
||||
lock.withLock { self.continuation = continuation }
|
||||
}
|
||||
|
||||
func replay(_ snapshots: [AudioBufferSnapshot]) {
|
||||
lock.withLock {
|
||||
for snapshot in snapshots {
|
||||
continuation?.yield(snapshot)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func yield(_ snapshot: AudioBufferSnapshot) {
|
||||
lock.withLock { continuation?.yield(snapshot) }
|
||||
}
|
||||
@@ -37,6 +45,34 @@ private final class FlowCaptureStreamRelay: @unchecked Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
/// Rolling pre-roll while utterance gate is closed (~400 ms at typical tap rates).
|
||||
private final class FlowPrerollStore: @unchecked Sendable {
|
||||
private let lock = OSAllocatedUnfairLock()
|
||||
private var snapshots: [AudioBufferSnapshot] = []
|
||||
private let maxCount: Int
|
||||
|
||||
init(maxCount: Int = 6) {
|
||||
self.maxCount = maxCount
|
||||
}
|
||||
|
||||
func append(_ snapshot: AudioBufferSnapshot) {
|
||||
lock.withLock {
|
||||
snapshots.append(snapshot)
|
||||
if snapshots.count > maxCount {
|
||||
snapshots.removeFirst(snapshots.count - maxCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func drain() -> [AudioBufferSnapshot] {
|
||||
lock.withLock {
|
||||
let drained = snapshots
|
||||
snapshots.removeAll()
|
||||
return drained
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Rolling bar levels updated from the audio tap; read on the main actor.
|
||||
private final class FlowLevelStore: @unchecked Sendable {
|
||||
private let lock = OSAllocatedUnfairLock()
|
||||
@@ -120,6 +156,7 @@ public final class FlowContinuousCapture {
|
||||
|
||||
private let audioEngine = AVAudioEngine()
|
||||
private let streamRelay = FlowCaptureStreamRelay()
|
||||
private let prerollStore = FlowPrerollStore()
|
||||
private let levelStore = FlowLevelStore(barCount: FlowCaptureConstants.levelBarCount)
|
||||
private let isUtteranceActive = OSAllocatedUnfairLock(initialState: false)
|
||||
|
||||
@@ -170,6 +207,7 @@ public final class FlowContinuousCapture {
|
||||
if !didInstallTap {
|
||||
let utteranceFlag = isUtteranceActive
|
||||
let relay = streamRelay
|
||||
let preroll = prerollStore
|
||||
let levels = levelStore
|
||||
let tap = Self.makeAudioTapBlock(
|
||||
converter: converter,
|
||||
@@ -177,6 +215,7 @@ public final class FlowContinuousCapture {
|
||||
hwFormat: hwFormat,
|
||||
utteranceFlag: utteranceFlag,
|
||||
levelStore: levels,
|
||||
prerollStore: preroll,
|
||||
streamRelay: relay
|
||||
)
|
||||
inputNode.installTap(onBus: 0, bufferSize: 4096, format: hwFormat, block: tap)
|
||||
@@ -213,9 +252,12 @@ public final class FlowContinuousCapture {
|
||||
|
||||
/// Begin forwarding downsampled buffers to ASR for one utterance.
|
||||
public func beginUtterance() -> AsyncStream<AudioBufferSnapshot> {
|
||||
isUtteranceActive.withLock { $0 = true }
|
||||
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
|
||||
// Bind the consumer before opening the gate so early tap frames
|
||||
// are not dropped on the floor.
|
||||
streamRelay.bind(continuation)
|
||||
streamRelay.replay(prerollStore.drain())
|
||||
isUtteranceActive.withLock { $0 = true }
|
||||
return stream
|
||||
}
|
||||
|
||||
@@ -241,13 +283,12 @@ public final class FlowContinuousCapture {
|
||||
hwFormat: AVAudioFormat,
|
||||
utteranceFlag: OSAllocatedUnfairLock<Bool>,
|
||||
levelStore: FlowLevelStore,
|
||||
prerollStore: FlowPrerollStore,
|
||||
streamRelay: FlowCaptureStreamRelay
|
||||
) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void {
|
||||
return { buffer, _ in
|
||||
levelStore.update(from: buffer, barCount: FlowCaptureConstants.levelBarCount)
|
||||
|
||||
guard utteranceFlag.withLock({ $0 }) else { return }
|
||||
|
||||
let outFrames = AVAudioFrameCount(
|
||||
Double(buffer.frameLength) * targetFormat.sampleRate / hwFormat.sampleRate
|
||||
)
|
||||
@@ -264,7 +305,12 @@ public final class FlowContinuousCapture {
|
||||
|
||||
let snapshot = AudioBufferSnapshot(buffer: outBuffer)
|
||||
guard !snapshot.samples.isEmpty else { return }
|
||||
streamRelay.yield(snapshot)
|
||||
|
||||
if utteranceFlag.withLock({ $0 }) {
|
||||
streamRelay.yield(snapshot)
|
||||
} else {
|
||||
prerollStore.append(snapshot)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,6 +82,7 @@ public final class KeyboardState: ObservableObject {
|
||||
public var endRecording: () -> Void = {}
|
||||
public var tapMic: () -> Void = {}
|
||||
public var openSettings: () -> Void = {}
|
||||
public var startFlowSession: () -> Void = {}
|
||||
public var setMode: (InputMode) -> Void = { _ in }
|
||||
public var setLocale: (String) -> Void = { _ in }
|
||||
public var setEngineMode: (String) -> Void = { _ in }
|
||||
|
||||
Reference in New Issue
Block a user