// FlowContinuousCapture.swift // OSGKeyboard · HostSupport // // TypeWhisper-style continuous mic capture for Flow sessions: one // AVAudioEngine + input tap for the entire session. Utterances gate // whether buffers are forwarded to ASR; levels are always computed on // the audio thread and read from the main thread (never UserDefaults // from the realtime tap — that caused cross-process crashes). import Foundation import AVFoundation import os #if canImport(OSGKeyboardShared) import OSGKeyboardShared #endif private enum FlowCaptureConstants { static let levelBarCount = 24 static let targetSampleRate: Double = 16_000 static let drainPollIntervalNs: UInt64 = 20_000_000 } private enum UtteranceGatePhase: Equatable { case idle case recording case draining var label: String { switch self { case .idle: return "idle" case .recording: return "recording" case .draining: return "draining" } } } /// Thread-safe relay for utterance-scoped ASR snapshots. private final class FlowCaptureStreamRelay: @unchecked Sendable { private let lock = OSAllocatedUnfairLock() private var continuation: AsyncStream.Continuation? func bind(_ continuation: AsyncStream.Continuation) { lock.withLock { self.continuation = continuation } } func replay(_ snapshots: [AudioBufferSnapshot]) { lock.withLock { for snapshot in snapshots { continuation?.yield(snapshot) } } } func yield(_ snapshot: AudioBufferSnapshot) { _ = lock.withLock { continuation?.yield(snapshot) } } func finish() { lock.withLock { continuation?.finish() continuation = nil } } } /// Rolling pre-roll while utterance gate is closed. /// /// Sized by sample count (~3 s @ 16 kHz) so PiP mic spin-up between /// `capture.start()` and `beginUtterance` does not discard the user's /// opening words (the old 6-buffer cap was only ~400 ms). private final class FlowPrerollStore: @unchecked Sendable { private let lock = OSAllocatedUnfairLock() private var snapshots: [AudioBufferSnapshot] = [] private let maxSamples: Int init(maxSamples: Int = 48_000) { self.maxSamples = maxSamples } func append(_ snapshot: AudioBufferSnapshot) { lock.withLock { snapshots.append(snapshot) var total = snapshots.reduce(0) { $0 + $1.samples.count } while total > maxSamples, !snapshots.isEmpty { let removed = snapshots.removeFirst() total -= removed.samples.count } } } func drain() -> [AudioBufferSnapshot] { lock.withLock { let drained = snapshots snapshots.removeAll() return drained } } } /// Rolling bar levels updated from the audio tap; read on the main actor. private final class FlowLevelStore: @unchecked Sendable { private let lock = OSAllocatedUnfairLock() private var levels: [Float] init(barCount: Int) { levels = Array(repeating: 0, count: barCount) } func update(from buffer: AVAudioPCMBuffer, barCount: Int) { let computed = Self.calculateLevels(from: buffer, barCount: barCount) lock.withLock { levels = computed } } func snapshot() -> [Float] { lock.withLock { levels } } private static func calculateLevels(from buffer: AVAudioPCMBuffer, barCount: Int) -> [Float] { guard let channelData = buffer.floatChannelData else { return Array(repeating: 0, count: barCount) } let frameLength = Int(buffer.frameLength) guard frameLength > 0 else { return Array(repeating: 0, count: barCount) } let samplesPerBar = max(frameLength / barCount, 1) var result = [Float]() result.reserveCapacity(barCount) for barIndex in 0.. Bool { let timestamp = lock.withLock { $0 } guard timestamp > 0 else { return false } return Date().timeIntervalSince1970 - timestamp <= maxAge } } /// Why a tap buffer never reached the recogniser. /// /// Recorded as a plain integer on the realtime audio thread and rendered on the /// main actor — calling `Logger` inside the tap would allocate and risk /// priority inversion. Each of these was previously a bare `return`, which is /// what made "waveform moves but the transcript is empty" invisible: levels and /// the audio-proof timestamp are taken from the *raw* buffer, before /// conversion, so they keep looking healthy while ASR receives nothing. public enum FlowDownsampleFailure: Int, Sendable { case none = 0 case invalidSourceFormat case converterCreateFailed case scratchOverflow case converterError case emptyOutput public var label: String { switch self { case .none: return "none" case .invalidSourceFormat: return "invalidSourceFormat" case .converterCreateFailed: return "converterCreateFailed" case .scratchOverflow: return "scratchOverflow" case .converterError: return "converterError" case .emptyOutput: return "emptyOutput" } } } /// Tap accounting for one utterance (`beginUtterance()` resets it). public struct FlowCaptureFrameReport: Sendable, Equatable { public var framesReceived = 0 public var framesConverted = 0 public var framesDropped = 0 public var samplesToASR = 0 public var samplesToPreroll = 0 public var lastFailure = FlowDownsampleFailure.none public var lastFailureSourceRate = 0 public var lastFailureInputFrames = 0 public var lastFailureWantedFrames = 0 public init() {} /// The mic delivered frames but none survived conversion — i.e. the user /// saw a live waveform while the recogniser was fed silence. public var isFeedStarved: Bool { framesReceived > 0 && samplesToASR == 0 } public var summary: String { var text = "frames=\(framesReceived) converted=\(framesConverted) " + "dropped=\(framesDropped) asrSamples=\(samplesToASR) " + "asrSeconds=\(FlowTrace.seconds(samples: samplesToASR)) " + "prerollSamples=\(samplesToPreroll)" if lastFailure != .none { text += " lastFailure=\(lastFailure.label)" + " failSourceRate=\(lastFailureSourceRate)" + " failInFrames=\(lastFailureInputFrames)" + " failWantFrames=\(lastFailureWantedFrames)" } return text } } /// Realtime-safe counters behind an unfair lock (same discipline as the gate). private final class FlowCaptureFrameStats: @unchecked Sendable { private let lock = OSAllocatedUnfairLock(initialState: FlowCaptureFrameReport()) func noteFrameReceived() { lock.withLock { $0.framesReceived += 1 } } func noteConverted(samples: Int, reachedASR: Bool) { lock.withLock { $0.framesConverted += 1 if reachedASR { $0.samplesToASR += samples } else { $0.samplesToPreroll += samples } } } func noteDropped( failure: FlowDownsampleFailure, sourceRate: Double, inputFrames: Int, wantedFrames: Int ) { lock.withLock { $0.framesDropped += 1 $0.lastFailure = failure $0.lastFailureSourceRate = Int(sourceRate) $0.lastFailureInputFrames = inputFrames $0.lastFailureWantedFrames = wantedFrames } } func reset() { lock.withLock { $0 = FlowCaptureFrameReport() } } func snapshot() -> FlowCaptureFrameReport { lock.withLock { $0 } } } /// Outcome of one realtime conversion attempt. Carries the reason (and the /// formats involved) so the drop can be explained after the fact. private enum FlowDownsampleOutcome { case converted(AVAudioPCMBuffer) case failed( failure: FlowDownsampleFailure, sourceRate: Double, inputFrames: Int, wantedFrames: Int ) } /// Route-adaptive downsampling converter, safe to call from the realtime tap. /// /// `AVAudioEngine.installTap(format:)` traps with an **uncatchable** NSException /// when the format passed to it does not match the input node's *live* format. /// After an audio-route change — which the on-device `SpeechAnalyzer` triggers /// during warmup by reconfiguring the shared `AVAudioSession` — the value /// returned by `inputNode.outputFormat(forBus:)` can lag behind the real /// hardware rate (e.g. it reports 48 kHz while the node has already switched to /// 24 kHz). Installing a tap with that stale explicit format crashes the whole /// app (`Failed to create tap due to format mismatch`). /// /// We therefore install the tap with `format: nil` (which always uses the /// node's live format) and rebuild the sample-rate converter *here* whenever the /// incoming buffer's format actually changes, so downsampling to the ASR target /// rate is always valid regardless of route churn. private final class AdaptiveDownsampler: @unchecked Sendable { // `AVAudioConverter` / `AVAudioFormat` / `AVAudioPCMBuffer` are not // `Sendable`, so the state is guarded manually via the unchecked lock // APIs. The scratch output buffer is REUSED across tap callbacks — // allocating on the realtime audio thread risks priority inversion, and // taps on one bus are serialized, so a single scratch is safe as long as // callers copy its contents out before returning (AudioBufferSnapshot // does exactly that). private struct State { var converter: AVAudioConverter var source: AVAudioFormat var scratch: AVAudioPCMBuffer } private let lock = OSAllocatedUnfairLock(uncheckedState: nil) let targetFormat: AVAudioFormat /// Frame headroom for the reusable output buffer. Taps deliver ≤4096 /// input frames; output frames = input × (16k / hardwareRate), which /// exceeds input only for sub-16 kHz hardware (rare telephony routes), /// so 2× the tap size covers every realistic ratio. private static let scratchCapacity: AVAudioFrameCount = 8_192 init(targetFormat: AVAudioFormat) { self.targetFormat = targetFormat } /// Downsamples `buffer` into the reusable scratch buffer and returns it, /// rebuilding the converter lazily when the hardware route (and thus the /// source format) changes. The returned buffer is only valid until the /// next call — copy its samples out synchronously. func convertReusingScratch(_ buffer: AVAudioPCMBuffer) -> FlowDownsampleOutcome { let sourceFormat = buffer.format let sourceRate = sourceFormat.sampleRate let inputFrames = Int(buffer.frameLength) guard sourceRate > 0 else { return .failed( failure: .invalidSourceFormat, sourceRate: sourceRate, inputFrames: inputFrames, wantedFrames: 0 ) } return lock.withLockUnchecked { state -> FlowDownsampleOutcome in if state == nil || state!.source != sourceFormat { guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat), let scratch = AVAudioPCMBuffer( pcmFormat: targetFormat, frameCapacity: Self.scratchCapacity ) else { state = nil return .failed( failure: .converterCreateFailed, sourceRate: sourceRate, inputFrames: inputFrames, wantedFrames: 0 ) } state = State(converter: converter, source: sourceFormat, scratch: scratch) } guard let current = state else { return .failed( failure: .converterCreateFailed, sourceRate: sourceRate, inputFrames: inputFrames, wantedFrames: 0 ) } let wanted = AVAudioFrameCount( Double(buffer.frameLength) * targetFormat.sampleRate / sourceRate ) guard wanted > 0, wanted <= current.scratch.frameCapacity else { return .failed( failure: .scratchOverflow, sourceRate: sourceRate, inputFrames: inputFrames, wantedFrames: Int(wanted) ) } current.scratch.frameLength = 0 // ONE-SHOT input: the converter keeps pulling until the output // buffer's frameCapacity is full, and the scratch is deliberately // oversized — feeding the same tap buffer on every pull would // duplicate the audio ~6× (stuttering ASR input). After the // single feed we report "ran dry", so the expected status is // `.inputRanDry` (output not full), not `.haveData`. let provided = OSAllocatedUnfairLock(initialState: false) var error: NSError? let status = current.converter.convert(to: current.scratch, error: &error) { _, outStatus in if provided.withLock({ $0 }) { outStatus.pointee = .noDataNow return nil } provided.withLock { $0 = true } outStatus.pointee = .haveData return buffer } guard status != .error, error == nil else { return .failed( failure: .converterError, sourceRate: sourceRate, inputFrames: inputFrames, wantedFrames: Int(wanted) ) } guard current.scratch.frameLength > 0 else { return .failed( failure: .emptyOutput, sourceRate: sourceRate, inputFrames: inputFrames, wantedFrames: Int(wanted) ) } return .converted(current.scratch) } } } /// Owns the session-long capture graph on the main actor. The realtime tap /// must not touch UserDefaults, log, or invoke actor callbacks; it exchanges /// snapshots through lock-protected relays, and callbacks return on MainActor. @MainActor public final class FlowContinuousCapture { public enum StartError: LocalizedError { case invalidHardwareFormat(sampleRate: Double, channels: Int) case formatCreateFailed case converterCreateFailed case engineStartFailed(String) case audioSessionFailed(String) public var errorDescription: String? { switch self { case .invalidHardwareFormat(let sr, let ch): return String.localizedStringWithFormat( NSLocalizedString("preview.error.micUnavailable", comment: ""), sr, ch ) case .formatCreateFailed: return NSLocalizedString("preview.error.formatCreate", comment: "") case .converterCreateFailed: return NSLocalizedString("preview.error.converterCreate", comment: "") case .engineStartFailed(let detail): return String.localizedStringWithFormat( NSLocalizedString("preview.error.engineStart", comment: ""), detail ) case .audioSessionFailed(let detail): return String.localizedStringWithFormat( NSLocalizedString("preview.error.audioSession", comment: ""), detail ) } } } public static let levelBarCount = FlowCaptureConstants.levelBarCount /// Recreated after every hardware format transition. Reusing the same /// engine preserves a stale input node on some Bluetooth HFP changes. private var audioEngine = AVAudioEngine() private let streamRelay = FlowCaptureStreamRelay() private let prerollStore = FlowPrerollStore() private let levelStore = FlowLevelStore(barCount: FlowCaptureConstants.levelBarCount) private let audioProofStore = FlowAudioProofStore() private let gate = OSAllocatedUnfairLock(initialState: UtteranceGatePhase.idle) private let drainTracker = FlowCaptureDrainTracker() private let tailSampleCounter = OSAllocatedUnfairLock(initialState: 0) private let utterancePCMStore = FlowUtterancePCMStore( maxSampleCount: Int(FlowSessionKeys.maxUtteranceDuration) * 16_000 ) private let frameStats = FlowCaptureFrameStats() private var downsampler: AdaptiveDownsampler? private var targetFormat: AVAudioFormat? private var hwFormat: AVAudioFormat? private var activeRouteSnapshot: FlowAudioSessionSnapshot? private var drainPolicy = FlowCaptureTailDrainPolicy.flowDefault private var didInstallTap = false private var isRunning = false private var isRebuilding = false private var isActivatingEngine = false private var interrupted = false /// When the engine last (re)activated — a freshly started engine has /// produced no frames yet and must not be misclassified as a zombie. private var lastActivationAt = Date.distantPast private var routeObserver: NSObjectProtocol? private var interruptionObserver: NSObjectProtocol? private var mediaResetObserver: NSObjectProtocol? private var engineConfigurationObserver: NSObjectProtocol? private var routeRecoveryTask: Task? private let log = Logger(subsystem: "com.osgkeyboard.shared", category: "FlowCapture") public init() {} public var running: Bool { isRunning } public private(set) var engineActivationCount = 0 public private(set) var routeRecoveryCount = 0 /// True between interruption `.began` and `.ended` (phone call, Siri). /// While set, `setActive(true)` is guaranteed to fail — owners should /// wait for `.ended` (which rebuilds the engine) instead of retrying. public var isInterrupted: Bool { interrupted } /// True when the capture session flag, tap, and audio engine are all live. public var engineIsLive: Bool { isRunning && didInstallTap && audioEngine.isRunning } /// True only when the engine is live and the input tap has recently /// delivered an actual audio frame. /// /// NOTE: this is a *raw* mic signal (taken before downsampling), so it /// proves the microphone works — not that the recogniser is being fed. /// Use `frameReport()` for the latter. public func engineHasRecentAudio(maxAge: TimeInterval = 1) -> Bool { engineIsLive && audioProofStore.hasRecentFrame(maxAge: maxAge) } /// Tap accounting since the last `beginUtterance()`, i.e. how much audio /// actually survived conversion and reached the recogniser. public func frameReport() -> FlowCaptureFrameReport { frameStats.snapshot() } /// Called on the main actor when `engineIsLive` may have changed. public var onEngineLiveChanged: ((Bool) -> Void)? /// Called on the main actor when the system interrupted capture (phone /// call, Siri). The session owner should fail any mic-open utterance — /// audio frames stop arriving, so continuing to "record" only captures /// a silence gap the user cannot see. public var onInterruptionBegan: (() -> Void)? /// Configure `.playAndRecord`, install a permanent input tap, start the engine. /// /// Idempotent: "already running and healthy" is a warm-start fast path, /// while "already running but producing no audio" is a zombie state /// (force-quit relaunch, failed cold start, mediaserverd reset) that is /// torn down and rebuilt in place. It must never be a silent no-op — /// a `guard !isRunning` early-return here turned every cold-start retry /// into a guaranteed audio-proof timeout. public func start() async throws { if isRunning { let startedMomentsAgo = Date().timeIntervalSince(lastActivationAt) < 2 if engineIsLive && (engineHasRecentAudio(maxAge: 2) || startedMomentsAgo) { // Healthy warm engine — or one so fresh it simply hasn't // produced its first frame yet (interleaved start attempts // land here; rebuilding a 100 ms-old engine only multiplies // audio-session churn in the fragile post-relaunch window). FlowTrace.capture( "start.warmReuse", "engineLive=1 freshMs=\(Int(Date().timeIntervalSince(lastActivationAt) * 1000)) " + frameStats.snapshot().summary ) return } log.info("start(): zombie engine detected (running but no live audio) — forcing rebuild") FlowTrace.warn( "capture.start.zombieRebuild", "engineLive=\(engineIsLive ? 1 : 0) recentAudio=0 \(frameStats.snapshot().summary)" ) stop() } audioProofStore.reset() FlowTrace.capture("start.begin", "coldEngine=1") installSessionObservers() do { try await activateEngine() } catch { removeSessionObservers() FlowTrace.warn("capture.start.failed", "error=\(error.localizedDescription)") throw error } isRunning = true notifyEngineLiveChanged() FlowTrace.capture("start.done", "engineLive=\(engineIsLive ? 1 : 0)") } /// Bring up the audio session + engine for the *current* hardware route. /// Reused for route-change / interruption recovery, so it always rebuilds /// the tap against the live hardware format (which changes when the user /// plugs in AirPods or a wired headset mid-session). private func activateEngine() async throws { isActivatingEngine = true defer { isActivatingEngine = false } let sessionSnapshot: FlowAudioSessionSnapshot do { sessionSnapshot = try await FlowAudioSessionCoordinator.shared.activateCapture() } catch { FlowTrace.warn( "capture.audioSession.activateFailed", "error=\(error.localizedDescription)" ) throw StartError.audioSessionFailed(error.localizedDescription) } // AVAudioSession activation may replace the RemoteIO format. Build a // fresh engine only after activation so its input node cannot retain // the prior built-in-mic (48 kHz) format when HFP is now 24 kHz. // Retire the previous graph before publishing the fresh engine. if didInstallTap { audioEngine.inputNode.removeTap(onBus: 0) didInstallTap = false } await FlowAudioSessionCoordinator.shared.stopEngine( FlowAudioEngineHandle(audioEngine) ) // Voice Processing must be enabled while the engine is stopped, and // it can change the RemoteIO format — enable first, then sync rates. var candidateEngine = AVAudioEngine() var voiceProcessingOn = FlowCaptureVoiceProcessing.enableVoiceProcessing( on: candidateEngine ) var inputNode = candidateEngine.inputNode var hardwareFormat = inputNode.inputFormat(forBus: 0) var outputFormat = inputNode.outputFormat(forBus: 0) var resolvedSession = sessionSnapshot for _ in 0..<3 where abs(hardwareFormat.sampleRate - resolvedSession.sampleRate) >= 1 { try? await Task.sleep(nanoseconds: 50_000_000) // Re-enter capture so the session rate tracks VP / route shifts. resolvedSession = (try? await FlowAudioSessionCoordinator.shared.activateCapture()) ?? resolvedSession candidateEngine = AVAudioEngine() voiceProcessingOn = FlowCaptureVoiceProcessing.enableVoiceProcessing( on: candidateEngine ) inputNode = candidateEngine.inputNode hardwareFormat = inputNode.inputFormat(forBus: 0) outputFormat = inputNode.outputFormat(forBus: 0) } FlowTrace.capture( "audioSession.active", "hwRate=\(Int(hardwareFormat.sampleRate)) hwChannels=\(hardwareFormat.channelCount) " + "outputRate=\(Int(outputFormat.sampleRate)) " + "sessionRate=\(Int(resolvedSession.sampleRate)) " + "route=\(resolvedSession.inputPortType) " + "voiceProcessing=\(voiceProcessingOn ? 1 : 0)" ) guard hardwareFormat.sampleRate > 0, hardwareFormat.channelCount > 0 else { FlowTrace.warn( "capture.hardwareFormat.invalid", "hwRate=\(hardwareFormat.sampleRate) hwChannels=\(hardwareFormat.channelCount)" ) throw StartError.invalidHardwareFormat( sampleRate: hardwareFormat.sampleRate, channels: Int(hardwareFormat.channelCount) ) } guard resolvedSession.sampleRate <= 0 || abs(hardwareFormat.sampleRate - resolvedSession.sampleRate) < 1 else { throw StartError.invalidHardwareFormat( sampleRate: hardwareFormat.sampleRate, channels: Int(hardwareFormat.channelCount) ) } guard let resolvedTargetFormat = AVAudioFormat( commonFormat: .pcmFormatFloat32, sampleRate: FlowCaptureConstants.targetSampleRate, channels: 1, interleaved: false ) else { throw StartError.formatCreateFailed } // Route-adaptive converter: it rebuilds itself from the live buffer // format inside the tap, so it never assumes a fixed hardware rate. let downsampler = AdaptiveDownsampler(targetFormat: resolvedTargetFormat) self.downsampler = downsampler targetFormat = resolvedTargetFormat hwFormat = hardwareFormat let gateLock = gate let relay = streamRelay let preroll = prerollStore let levels = levelStore let proof = audioProofStore let tracker = drainTracker let tailCounter = tailSampleCounter let pcmStore = utterancePCMStore let policy = drainPolicy let tap = Self.makeAudioTapBlock( downsampler: downsampler, gate: gateLock, levelStore: levels, audioProofStore: proof, prerollStore: preroll, streamRelay: relay, drainTracker: tracker, tailSampleCounter: tailCounter, utterancePCMStore: pcmStore, frameStats: frameStats, drainPolicy: policy ) // `format: nil` binds the tap to the input node's *live* format. Passing // an explicit (possibly stale) format here is what crashed the app on a // route change (48 kHz client vs 24 kHz hardware); nil can never mismatch. inputNode.installTap(onBus: 0, bufferSize: 4096, format: nil, block: tap) didInstallTap = true FlowTrace.capture( "tap.installed", "hwRate=\(Int(hardwareFormat.sampleRate)) targetRate=\(Int(resolvedTargetFormat.sampleRate)) " + "bufferSize=4096 format=live" ) do { try await FlowAudioSessionCoordinator.shared.startEngine( FlowAudioEngineHandle(candidateEngine) ) } catch { inputNode.removeTap(onBus: 0) didInstallTap = false await FlowAudioSessionCoordinator.shared.stopEngine( FlowAudioEngineHandle(candidateEngine) ) FlowTrace.warn("capture.engine.startFailed", "error=\(error.localizedDescription)") throw StartError.engineStartFailed(error.localizedDescription) } audioEngine = candidateEngine activeRouteSnapshot = resolvedSession engineActivationCount += 1 lastActivationAt = Date() FlowTrace.capture("engine.started", "running=\(audioEngine.isRunning ? 1 : 0)") } /// Tear down the engine and release the audio session. public func stop(releaseSession: Bool = true) { // Logged before teardown: in PiP keep-alive every utterance ends with a // stop(), which also discards the converter — so this line marks the // point after which the next press must rebuild the whole audio path. FlowTrace.capture( "stop", "wasRunning=\(isRunning ? 1 : 0) engineLive=\(engineIsLive ? 1 : 0) " + frameStats.snapshot().summary ) removeSessionObservers() routeRecoveryTask?.cancel() routeRecoveryTask = nil gate.withLock { $0 = .idle } drainTracker.reset() tailSampleCounter.withLock { $0 = 0 } streamRelay.finish() if didInstallTap { audioEngine.inputNode.removeTap(onBus: 0) didInstallTap = false } FlowAudioSessionCoordinator.shared.enqueueStopEngine( FlowAudioEngineHandle(audioEngine) ) isRunning = false interrupted = false audioProofStore.reset() downsampler = nil targetFormat = nil hwFormat = nil activeRouteSnapshot = nil if releaseSession { FlowAudioSessionCoordinator.shared.deactivate() } notifyEngineLiveChanged() } /// Re-activate capture after returning from background without /// reinstalling the tap (iOS may deactivate the audio session). /// /// Doubles as the interruption-recovery probe: `setActive(true)` FAILS /// while a call/Siri interruption is live and succeeds once it ends, so a /// successful reassert proves the interruption is over. iOS does not /// guarantee delivery of `.ended` (commonly dropped when the app was /// suspended during the call), so this is the only reliable way to clear /// the `interrupted` latch in that case. @discardableResult public func reassertIfRunning() async -> Bool { guard isRunning else { return false } if routeFormatIsStable(), audioEngine.isRunning { do { _ = try await FlowAudioSessionCoordinator.shared.activateCapture() interrupted = false notifyEngineLiveChanged() FlowTrace.capture("reassert.ok", "engineLive=\(engineIsLive ? 1 : 0)") return engineIsLive } catch { FlowTrace.warn("capture.reassert.failed", "error=\(error.localizedDescription)") } } do { try await activateEngine() interrupted = false notifyEngineLiveChanged() FlowTrace.capture("reassert.rebuilt", "engineLive=\(engineIsLive ? 1 : 0)") return engineIsLive } catch { FlowTrace.warn("capture.reassert.failed", "error=\(error.localizedDescription)") notifyEngineLiveChanged() return false } } public func awaitAudioFlowing( timeout: TimeInterval, recentFrameMaxAge: TimeInterval = 1 ) async -> Bool { let deadline = Date().addingTimeInterval(timeout) while Date() < deadline { if engineHasRecentAudio(maxAge: recentFrameMaxAge) { return true } do { try await Task.sleep(nanoseconds: 50_000_000) } catch { // Cancelled — bail out instead of busy-spinning the main // actor for the rest of the window (a cancelled Task.sleep // returns immediately, starving concurrent start attempts). return false } } return engineHasRecentAudio(maxAge: recentFrameMaxAge) } // MARK: - Route / interruption recovery private func installSessionObservers() { let center = NotificationCenter.default if routeObserver == nil { routeObserver = center.addObserver( forName: AVAudioSession.routeChangeNotification, object: nil, queue: .main ) { [weak self] note in let reasonRaw = note.userInfo?[AVAudioSessionRouteChangeReasonKey] as? UInt MainActor.assumeIsolated { self?.handleRouteChange(reasonRaw: reasonRaw) } } } if interruptionObserver == nil { interruptionObserver = center.addObserver( forName: AVAudioSession.interruptionNotification, object: nil, queue: .main ) { [weak self] note in let typeRaw = note.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt let optionsRaw = note.userInfo?[AVAudioSessionInterruptionOptionKey] as? UInt MainActor.assumeIsolated { self?.handleInterruption(typeRaw: typeRaw, optionsRaw: optionsRaw) } } } // Apple QA1749: when the system media server resets, the engine, // converter and audio session all become orphaned and must be // rebuilt from scratch — otherwise capture silently produces no // audio (another cause of "waveform moves but ASR is empty"). if mediaResetObserver == nil { mediaResetObserver = center.addObserver( forName: AVAudioSession.mediaServicesWereResetNotification, object: nil, queue: .main ) { [weak self] _ in MainActor.assumeIsolated { self?.handleMediaServicesReset() } } } if engineConfigurationObserver == nil { engineConfigurationObserver = center.addObserver( forName: .AVAudioEngineConfigurationChange, object: nil, queue: .main ) { [weak self] _ in MainActor.assumeIsolated { guard let self else { return } guard !self.isActivatingEngine else { FlowTrace.capture( "engineConfigurationChange.ignored", "reason=activationInProgress" ) return } if !self.routeFormatIsStable() || !self.audioEngine.isRunning { self.scheduleRouteRecovery(reason: "engineConfigurationChange") } else { FlowTrace.capture( "engineConfigurationChange.ignored", "reason=stable" ) } } } } } private func removeSessionObservers() { let center = NotificationCenter.default if let routeObserver { center.removeObserver(routeObserver) } if let interruptionObserver { center.removeObserver(interruptionObserver) } if let mediaResetObserver { center.removeObserver(mediaResetObserver) } if let engineConfigurationObserver { center.removeObserver(engineConfigurationObserver) } routeObserver = nil interruptionObserver = nil mediaResetObserver = nil engineConfigurationObserver = nil } private func handleMediaServicesReset() { guard isRunning else { return } log.info("Media services were reset — rebuilding engine and converter") FlowTrace.warn("capture.mediaServicesReset", frameStats.snapshot().summary) Task { @MainActor [weak self] in await self?.rebuildEngine() } } private func handleRouteChange(reasonRaw: UInt?) { guard isRunning else { return } guard !isActivatingEngine else { FlowTrace.capture( "routeChange.ignored", "reason=\(reasonRaw ?? 0) activationInProgress=1" ) return } guard let reasonRaw, let reason = AVAudioSession.RouteChangeReason(rawValue: reasonRaw) else { return } let formatIsStable = routeFormatIsStable() if FlowAudioRouteRecoveryPolicy.shouldRebuild( reasonRaw: reasonRaw, formatIsStable: formatIsStable, engineIsRunning: audioEngine.isRunning ) { log.info("Audio route changed (\(reasonRaw, privacy: .public)) — rebuilding engine") FlowTrace.capture( "routeChange.rebuild", "reason=\(reasonRaw) gate=\(gate.withLock { $0 }.label) " + frameStats.snapshot().summary ) scheduleRouteRecovery(reason: "route=\(reasonRaw)") } else { FlowTrace.capture( "routeChange.ignored", "reason=\(reason.rawValue) stable=\(formatIsStable ? 1 : 0)" ) } } private func routeFormatIsStable() -> Bool { let session = AVAudioSession.sharedInstance() let sessionRate = session.sampleRate let inputRate = audioEngine.inputNode.inputFormat(forBus: 0).sampleRate guard sessionRate > 0, inputRate > 0 else { return false } guard abs(sessionRate - inputRate) < 1 else { return false } guard let activeRouteSnapshot else { return true } let currentInput = session.currentRoute.inputs.first return activeRouteSnapshot.inputPortUID == (currentInput?.uid ?? "") && activeRouteSnapshot.inputPortType == (currentInput?.portType.rawValue ?? "none") } private func scheduleRouteRecovery(reason: String) { guard isRunning else { return } routeRecoveryTask?.cancel() routeRecoveryTask = Task { @MainActor [weak self] in // Route notifications arrive in bursts while HFP negotiates its // final sample rate. Debounce before creating the new graph. try? await Task.sleep(nanoseconds: 150_000_000) guard let self, !Task.isCancelled, self.isRunning else { return } self.routeRecoveryCount += 1 FlowTrace.capture("routeRecovery.begin", "reason=\(reason)") await self.rebuildEngine() } } private func handleInterruption(typeRaw: UInt?, optionsRaw: UInt?) { guard let typeRaw, let type = AVAudioSession.InterruptionType(rawValue: typeRaw) else { return } switch type { case .began: log.info("Audio interruption began") FlowTrace.warn( "capture.interruption.began", "gate=\(gate.withLock { $0 }.label) \(frameStats.snapshot().summary)" ) interrupted = true notifyEngineLiveChanged() onInterruptionBegan?() case .ended: interrupted = false guard isRunning else { return } let shouldResume: Bool if let optionsRaw { shouldResume = AVAudioSession.InterruptionOptions(rawValue: optionsRaw).contains(.shouldResume) } else { shouldResume = true } FlowTrace.capture("interruption.ended", "shouldResume=\(shouldResume ? 1 : 0)") if shouldResume { log.info("Audio interruption ended — resuming capture") Task { @MainActor [weak self] in await self?.rebuildEngine() } } @unknown default: break } } /// Stop and rebuild the engine against the current route, keeping /// `isRunning` intact so the session survives the swap transparently. private func rebuildEngine() async { guard isRunning, !isRebuilding else { FlowTrace.capture( "rebuild.skipped", "running=\(isRunning ? 1 : 0) alreadyRebuilding=\(isRebuilding ? 1 : 0)" ) return } isRebuilding = true defer { isRebuilding = false } do { try await activateEngine() notifyEngineLiveChanged() FlowTrace.capture("rebuild.done", "engineLive=\(engineIsLive ? 1 : 0)") } catch { isRunning = false activeRouteSnapshot = nil if didInstallTap { audioEngine.inputNode.removeTap(onBus: 0) didInstallTap = false } FlowAudioSessionCoordinator.shared.enqueueStopEngine( FlowAudioEngineHandle(audioEngine) ) log.error("Engine rebuild failed: \(error.localizedDescription, privacy: .public)") FlowTrace.warn("capture.rebuild.failed", "error=\(error.localizedDescription)") notifyEngineLiveChanged() } } private func notifyEngineLiveChanged() { onEngineLiveChanged?(engineIsLive) } /// Begin forwarding downsampled buffers to ASR for one utterance. public func beginUtterance() -> AsyncStream { let (stream, continuation) = AsyncStream.makeStream() drainTracker.reset() tailSampleCounter.withLock { $0 = 0 } utterancePCMStore.reset() // Counters are per-utterance: reset here so the report emitted at drain // describes only this press. let priorReport = frameStats.snapshot() frameStats.reset() // Bind the consumer before opening the gate so early tap frames // are not dropped on the floor. streamRelay.bind(continuation) let preroll = prerollStore.drain() streamRelay.replay(preroll) gate.withLock { $0 = .recording } let prerollSamples = preroll.reduce(0) { $0 + $1.samples.count } FlowTrace.capture( "beginUtterance", "engineLive=\(engineIsLive ? 1 : 0) recentRawAudio=\(engineHasRecentAudio(maxAge: 2) ? 1 : 0) " + "prerollBuffers=\(preroll.count) prerollSamples=\(prerollSamples) " + "prerollSeconds=\(FlowTrace.seconds(samples: prerollSamples)) " + "sinceLastActivationMs=\(Int(Date().timeIntervalSince(lastActivationAt) * 1000)) " + "priorIdle[\(priorReport.summary)]" ) return stream } /// Drain trailing PCM after the user stops, then finish the ASR stream. public func endUtteranceAndDrain( policy: FlowCaptureTailDrainPolicy = .flowDefault ) async -> FlowCaptureDrainReport { let currentPhase = gate.withLock { $0 } guard currentPhase == .recording else { FlowTrace.warn( "capture.endUtterance.skipped", "gate=\(currentPhase.label) \(frameStats.snapshot().summary)" ) return .skipped } drainPolicy = policy gate.withLock { $0 = .draining } drainTracker.beginDrain() let timing = await FlowUtteranceEndCoordinator.awaitTailCapture( tracker: drainTracker, policy: policy, pollIntervalNs: FlowCaptureConstants.drainPollIntervalNs ) // NOTE: We intentionally do NOT signal `.endOfStream` to the shared // downsampling converter here. `AVAudioConverter` is stateful: once its // input block returns `.endOfStream`, the converter is permanently // finished and every subsequent `.haveData` conversion (from the live // tap) returns no data — which silently starved every utterance after // the first (Apple docs + AVAudioConverter reuse guidance). Trailing // speech is already preserved by the live `.draining` forwarding loop // above; the converter's sub-millisecond internal filter tail is not // worth poisoning a session-long converter for. streamRelay.finish() gate.withLock { $0 = .idle } let tailSamples = tailSampleCounter.withLock { $0 } let report = FlowCaptureDrainReport( drainDurationSeconds: drainTracker.elapsedSeconds(), endedBySilence: timing.endedBySilence, tailSampleCount: tailSamples, postRollDurationSeconds: timing.postRollDurationSeconds ) drainTracker.reset() tailSampleCounter.withLock { $0 = 0 } FlowPipelineDiagnostics.logDrain(report) // The decisive line for "waveform moved but no text": compare the raw // frame count the waveform was drawn from against the samples that // actually reached the recogniser. let frames = frameStats.snapshot() if frames.isFeedStarved { FlowTrace.warn( "capture.endUtterance.feedStarved", "micDeliveredFrames=\(frames.framesReceived) butASRGotSamples=0 \(frames.summary)" ) } else { FlowTrace.capture("endUtterance.done", frames.summary) } return report } /// Copy of utterance PCM so the host can decide empty-tap skip /// without consuming the buffer used by the normal drain path. public func utterancePCMSnapshot() -> [Float] { utterancePCMStore.snapshot() } /// Returns the utterance PCM accumulated during the last recording cycle. public func consumeUtteranceSamples() -> [Float] { utterancePCMStore.consume() } /// Immediate stop without tail drain (abort / session teardown). public func cancelUtterance() { FlowTrace.capture( "cancelUtterance", "gate=\(gate.withLock { $0 }.label) \(frameStats.snapshot().summary)" ) gate.withLock { $0 = .idle } drainTracker.reset() tailSampleCounter.withLock { $0 = 0 } utterancePCMStore.reset() streamRelay.finish() } public func currentAudioLevels() -> [Float] { levelStore.snapshot() } // MARK: - Audio tap (nonisolated — runs on realtime thread) private nonisolated static func makeAudioTapBlock( downsampler: AdaptiveDownsampler, gate: OSAllocatedUnfairLock, levelStore: FlowLevelStore, audioProofStore: FlowAudioProofStore, prerollStore: FlowPrerollStore, streamRelay: FlowCaptureStreamRelay, drainTracker: FlowCaptureDrainTracker, tailSampleCounter: OSAllocatedUnfairLock, utterancePCMStore: FlowUtterancePCMStore, frameStats: FlowCaptureFrameStats, drainPolicy: FlowCaptureTailDrainPolicy ) -> @Sendable (AVAudioPCMBuffer, AVAudioTime) -> Void { return { buffer, _ in // Levels and the audio-proof timestamp come from the RAW buffer, // everything downstream from the converted one. `frameStats` bridges // the two so a mismatch (waveform alive, ASR starved) is reportable // instead of invisible — counters only, no logging on this thread. audioProofStore.markFrameReceived() frameStats.noteFrameReceived() levelStore.update(from: buffer, barCount: FlowCaptureConstants.levelBarCount) // The downsampler derives its converter from the *live* buffer // format (mid-session route changes handled transparently) and // returns a REUSED scratch buffer — no per-callback allocation // on the realtime thread. The snapshot below copies the samples // out before the next tap callback can overwrite the scratch. let outcome = downsampler.convertReusingScratch(buffer) guard case .converted(let outBuffer) = outcome else { if case .failed(let failure, let sourceRate, let inFrames, let wanted) = outcome { frameStats.noteDropped( failure: failure, sourceRate: sourceRate, inputFrames: inFrames, wantedFrames: wanted ) } return } let snapshot = AudioBufferSnapshot(buffer: outBuffer) guard !snapshot.samples.isEmpty else { frameStats.noteDropped( failure: .emptyOutput, sourceRate: buffer.format.sampleRate, inputFrames: Int(buffer.frameLength), wantedFrames: 0 ) return } let phase = gate.withLock { $0 } switch phase { case .recording, .draining: frameStats.noteConverted(samples: snapshot.samples.count, reachedASR: true) utterancePCMStore.append(snapshot.samples) streamRelay.yield(snapshot) if phase == .draining { drainTracker.noteAudio(samples: snapshot.samples, policy: drainPolicy) tailSampleCounter.withLock { $0 += snapshot.samples.count } } case .idle: frameStats.noteConverted(samples: snapshot.samples.count, reachedASR: false) prerollStore.append(snapshot) } } } }