// ASRService.swift // OSGKeyboard · Shared // // Speech-to-text abstraction. As of iOS 26 being the minimum // deployment target, the only ASR backend is `SpeechAnalyzer` + // `DictationTranscriber` — always on-device, no cloud fallback, no // `requiresOnDevice` toggle. The previous legacy recognizer path is // gone; if a future platform ever needs it back, // reintroduce as a sibling class in `ASRServiceFactory.make()`. // // Lives in `OSGKeyboardShared` (not the keyboard extension target) so // that the host app's `KeyboardPreviewSheet` can run the same ASR // pipeline against real iOS audio — without it, the in-app preview // was a static mock that never actually called `SFSpeechRecognizer`, // and "did you actually wire up ASR?" was a fair review note. import Foundation import AVFoundation import CoreMedia import Speech import os // MARK: - Sendable conformance // `AVAudioPCMBuffer` and `SpeechAnalyzer` are not Sendable. We only // ever access them serially — the PCM buffer is built and consumed // inside a single Task, and the analyzer is cancelled but never // shared concurrently — so an unchecked conformance is sound here. extension AVAudioPCMBuffer: @unchecked @retroactive Sendable {} // MARK: - Protocol public protocol ASRService: ASRChunkTranscribing, Sendable { /// Start a transcription session. The returned stream emits `.partial` /// updates and exactly one `.final` (or `.error`) before finishing. /// `SpeechAnalyzer` is always fully on-device, so there is no /// `requiresOnDevice` flag — that legacy cloud-fallback control /// doesn't apply to the iOS 26 `SpeechAnalyzer` path. func transcribe( stream: AsyncStream, locale: Locale ) -> AsyncStream /// Cancel any in-flight recognition and tear down its tasks. func cancel() /// Clears cancellation / cached session state before a new utterance. func resetForNewUtterance() /// Pre-load locale assets and analyzer format for lower first-chunk latency. func warmup(locale: Locale) async /// Transcribe one PCM chunk (Flow pipelined path). Default wraps `transcribe(stream:)`. func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult } extension ASRService { public func resetForNewUtterance() {} public func warmup(locale: Locale) async {} public func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult { guard !samples.isEmpty else { return .success("") } if Task.isCancelled { return .cancelled } let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000) let (stream, continuation) = AsyncStream.makeStream() continuation.yield(snapshot) continuation.finish() var lastPartial = "" var finalText = "" var failure: String? for await event in transcribe(stream: stream, locale: locale) { if Task.isCancelled { return .cancelled } switch event { case .capability: break case .partial(let text): lastPartial = text case .final(let text): finalText = text case .error(let message): failure = message } } if let failure { return .failure(failure) } let trimmed = finalText.trimmingCharacters(in: .whitespacesAndNewlines) if !trimmed.isEmpty { return .success(trimmed) } let partial = lastPartial.trimmingCharacters(in: .whitespacesAndNewlines) if !partial.isEmpty { return .success(partial) } return .success("") } } public enum ASREvent: Sendable, Equatable { /// Emitted exactly once at the start of every `transcribe` call, so /// the UI can flag non-on-device locales (e.g. ja-JP on devices that /// only ship on-device ASR for en/zh). The ASR session continues /// either way — we fall back to cloud automatically. case capability(onDeviceSupported: Bool) case partial(String) case final(String) case error(String) } // MARK: - Factory public enum ASRServiceFactory { /// Returns on-device SpeechAnalyzer for `local`, or the user's cloud /// ASR provider when `engineMode == "cloud"`. public static func make(store: any ConfigurationStore = AppGroupStore()) -> ASRService { if store.engineMode == "cloud" { return CloudASRService(store: store) } return SpeechAnalyzerASR() } } // MARK: - PCM format conversion (testable helpers) // // Extracted from the audio-thread hot path so the scaling + clipping // math can be exercised in unit tests without instantiating the // full ASR pipeline. See `OSGKeyboardTests/ASRConversionTests.swift`. extension ASRServiceFactory { /// Convert a Float32 PCM buffer (`-1.0...1.0`) to an Int16 PCM /// buffer (`-32768...32767`). /// /// - Parameters: /// - source: Pointer to `sourceCount` `Float` samples. May be /// `nil` when `sourceCount == 0`. /// - sourceCount: Number of samples to convert. A `0` count /// turns the call into a no-op regardless of the pointers. /// - destination: Pointer to at least `sourceCount` slots of /// `Int16`. May be `nil` when `sourceCount == 0`. /// /// Per-sample: `Int16(round(clamp(s * 32767, -32768, 32767)))`. /// The explicit clip matters: without it, `s == 1.0` would map /// to `+32767` (fine) but `s == 1.5` (which can show up at the /// audio engine boundary under gain) would wrap to a negative /// value after the implicit Float→Int16 conversion. The /// `round()` (rather than truncate) preserves DC balance — `0.5` /// quantises to `+16384`, not `+16383`, matching what most audio /// DAW round-trips expect. static func convertFloat32ToInt16( source: UnsafePointer?, sourceCount: Int, destination: UnsafeMutablePointer? ) { guard sourceCount > 0, let source, let destination else { return } for i in 0..? private var analyzerFinished = false /// Reused across pipelined chunks within one utterance (assets + format). private var chunkPreparedLocaleID: String? private var chunkAnalyzerFormat: AVAudioFormat? func resetForNewUtterance() { // Keep chunk format / asset cache warm across utterances in one Flow session. } func invalidateChunkPreparationCache() { lock.withLock { chunkPreparedLocaleID = nil chunkAnalyzerFormat = nil } } func warmup(locale: Locale) async { let warmupStartedAt = Date() guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else { Self.debug("warmup locale unsupported requested=\(locale.identifier(.bcp47))") FlowTrace.warn( "asr.local.warmup.localeUnsupported", "requested=\(locale.identifier(.bcp47))" ) return } let localeID = resolvedLocale.identifier(.bcp47) let cachedLocaleID = lock.withLock { chunkPreparedLocaleID } if cachedLocaleID == localeID, lock.withLock({ chunkAnalyzerFormat != nil }) { Self.debug("warmup cache hit locale=\(localeID)") FlowTrace.asr("local.warmup.cacheHit", "locale=\(localeID)") return } let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale) Self.debug( "warmup start locale=\(localeID) customLMEnabled=\(setup.customLanguageModelEnabled) " + "customLMAttached=\(setup.usesCustomLanguageModel) " + "clmState=\(Self.describeCLMState(setup.clmState))" ) FlowTrace.asr( "local.warmup.begin", "locale=\(localeID) customLM=\(setup.usesCustomLanguageModel ? 1 : 0) " + "clmState=\(Self.describeCLMState(setup.clmState))" ) do { try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale) guard let format = await SpeechAnalyzer.bestAvailableAudioFormat( compatibleWith: [setup.transcriber], considering: Self.captureFormat ) else { Self.debug("warmup format unsupported locale=\(localeID)") FlowTrace.warn("asr.local.warmup.formatUnsupported", "locale=\(localeID)") return } lock.withLock { chunkPreparedLocaleID = localeID chunkAnalyzerFormat = format } Self.debug("warmup ready locale=\(localeID)") FlowTrace.asr( "local.warmup.ready", "locale=\(localeID) analyzerRate=\(Int(format.sampleRate)) " + "elapsed=\(FlowTrace.seconds(since: warmupStartedAt))s" ) } catch { Self.debug("warmup failed: \(error.localizedDescription)") FlowTrace.warn( "asr.local.warmup.failed", "locale=\(localeID) elapsed=\(FlowTrace.seconds(since: warmupStartedAt))s " + "error=\(error.localizedDescription)" ) } } func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult { guard !samples.isEmpty else { return .success("") } if Task.isCancelled { return .cancelled } let startedAt = Date() let rms = Self.rms(of: samples) Self.debug( "chunk start samples=\(samples.count) rms=\(String(format: "%.4f", rms)) " + "locale=\(locale.identifier(.bcp47))" ) do { let text = try await transcribeSamples(samples, locale: locale, reuseChunkPrep: true) let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) Self.debug( "chunk success textLen=\(trimmed.count) elapsed=\(Self.elapsed(startedAt))s " + "empty=\(trimmed.isEmpty)" ) FlowTrace.transcript( "asr.local.chunk", trimmed, "engine=local samples=\(samples.count) rms=\(String(format: "%.4f", rms)) " + "elapsed=\(Self.elapsed(startedAt))s locale=\(locale.identifier(.bcp47))" ) return trimmed.isEmpty ? .success("") : .success(trimmed) } catch is CancellationError { Self.debug("chunk cancelled elapsed=\(Self.elapsed(startedAt))s") FlowTrace.asr("local.chunk.cancelled", "samples=\(samples.count)") return .cancelled } catch { Self.debug("chunk failed elapsed=\(Self.elapsed(startedAt))s error=\(error.localizedDescription)") FlowTrace.warn( "asr.local.chunk.failed", "samples=\(samples.count) rms=\(String(format: "%.4f", rms)) " + "error=\(error.localizedDescription)" ) return .failure(error.localizedDescription) } } /// Analyze a single PCM buffer without the streaming `transcribe` wrapper. private func transcribeSamples( _ samples: [Float], locale: Locale, reuseChunkPrep: Bool ) async throws -> String { guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else { throw ASRChunkError.localeUnsupported } let localeID = resolvedLocale.identifier(.bcp47) let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale) Self.debug( "chunk setup locale=\(localeID) customLMEnabled=\(setup.customLanguageModelEnabled) " + "customLMAttached=\(setup.usesCustomLanguageModel) " + "clmState=\(Self.describeCLMState(setup.clmState))" ) let analyzerFormat: AVAudioFormat let cachedPrep = lock.withLock { (chunkPreparedLocaleID, chunkAnalyzerFormat) } if reuseChunkPrep, cachedPrep.0 == localeID, let cached = cachedPrep.1 { analyzerFormat = cached Self.debug( "chunk using cached analyzer format sr=\(Int(cached.sampleRate)) " + "channels=\(cached.channelCount) common=\(cached.commonFormat.rawValue)" ) } else { try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale) guard let format = await SpeechAnalyzer.bestAvailableAudioFormat( compatibleWith: [setup.transcriber], considering: Self.captureFormat ) else { throw ASRChunkError.formatUnsupported } analyzerFormat = format lock.withLock { chunkPreparedLocaleID = localeID chunkAnalyzerFormat = format } Self.debug( "chunk prepared analyzer format sr=\(Int(format.sampleRate)) " + "channels=\(format.channelCount) common=\(format.commonFormat.rawValue)" ) } let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000) guard let pcm = Self.makeAnalyzerPCMBuffer(from: snapshot, format: analyzerFormat) else { throw ASRChunkError.formatUnsupported } let analyzer = SpeechAnalyzer(modules: [setup.transcriber]) try await analyzer.prepareToAnalyze(in: analyzerFormat) let resultsTask = Task { var accumulator = ProgressiveDictationTranscriptAccumulator() for try await result in setup.transcriber.results { if Task.isCancelled { break } let text = String(result.text.characters) _ = accumulator.ingest(range: result.range, text: text) } return accumulator.finalize() } let inputStream = AsyncStream { continuation in continuation.yield(AnalyzerInput(buffer: pcm)) continuation.finish() } let lastSampleTime = try await analyzer.analyzeSequence(inputStream) if let lastSampleTime { try await analyzer.finalizeAndFinish(through: lastSampleTime) } else { await analyzer.cancelAndFinishNow() } return try await resultsTask.value } private enum ASRChunkError: LocalizedError { case localeUnsupported case formatUnsupported var errorDescription: String? { switch self { case .localeUnsupported: return SharedL10n.string("error.asr.localeUnsupported") case .formatUnsupported: return SharedL10n.string("error.asr.formatUnsupported") } } } /// Canonical capture format: 16 kHz mono Float32 from `AudioCaptureService` /// / `PreviewASRController` before it reaches SpeechAnalyzer. private static let captureFormat = AVAudioFormat( commonFormat: .pcmFormatFloat32, sampleRate: 16_000, channels: 1, interleaved: false )! func transcribe( stream: AsyncStream, locale: Locale ) -> AsyncStream { AsyncStream { continuation in continuation.yield(.capability(onDeviceSupported: true)) let task = Task { [weak self] in guard let self else { return } self.lock.withLock { self.analyzerFinished = false } defer { self.lock.withLock { self.analyzer = nil self.analyzerTask = nil self.analyzerFinished = true } } do { guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else { Self.debug("locale unsupported: \(locale.identifier(.bcp47))") continuation.yield(.error(SharedL10n.string("error.asr.localeUnsupported"))) continuation.finish() return } // Each pipelined chunk is ≤ 30 s; long dictation preset keeps a // single chunk coherent (Flow utterances run up to 3 min). let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale) Self.debug( "stream setup locale=\(resolvedLocale.identifier(.bcp47)) " + "customLMEnabled=\(setup.customLanguageModelEnabled) " + "customLMAttached=\(setup.usesCustomLanguageModel) " + "clmState=\(Self.describeCLMState(setup.clmState))" ) do { try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale) } catch { Self.debug("asset prepare failed: \(error.localizedDescription)") FlowTrace.warn( "asr.local.stream.assetsNotReady", "locale=\(resolvedLocale.identifier(.bcp47)) " + "error=\(error.localizedDescription)" ) continuation.yield(.error(SharedL10n.string("error.asr.assetsNotReady"))) continuation.finish() return } let newAnalyzer = SpeechAnalyzer(modules: [setup.transcriber]) self.lock.withLock { self.analyzer = newAnalyzer } guard let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat( compatibleWith: [setup.transcriber], considering: Self.captureFormat ) else { continuation.yield(.error(SharedL10n.string("error.asr.formatUnsupported"))) continuation.finish() return } try await newAnalyzer.prepareToAnalyze(in: analyzerFormat) let inputStream = self.makeInputStream(from: stream, analyzerFormat: analyzerFormat) // Apple recommends consuming `transcriber.results` concurrently // while `analyzeSequence` drains the input stream. let resultsTask = Task { var accumulator = ProgressiveDictationTranscriptAccumulator() for try await result in setup.transcriber.results { if Task.isCancelled { break } let text = String(result.text.characters) guard let full = accumulator.ingest(range: result.range, text: text) else { continue } FlowTrace.transcript("asr.local.partial", full, "engine=local") continuation.yield(.partial(full)) } return accumulator.finalize() } let lastSampleTime = try await newAnalyzer.analyzeSequence(inputStream) if let lastSampleTime { try await newAnalyzer.finalizeAndFinish(through: lastSampleTime) } else { await newAnalyzer.cancelAndFinishNow() } let lastText: String do { lastText = try await resultsTask.value } catch { Self.debug("transcriber results failed: \(error.localizedDescription)") continuation.yield(.error(error.localizedDescription)) continuation.finish() return } let trimmed = lastText.trimmingCharacters(in: .whitespacesAndNewlines) if trimmed.isEmpty { FlowTrace.warn( "asr.local.stream.emptyFinal", "locale=\(resolvedLocale.identifier(.bcp47))" ) continuation.yield(.error(SharedL10n.string("error.asr.noSpeech"))) } else { FlowTrace.transcript("asr.local.final", trimmed, "engine=local") continuation.yield(.final(trimmed)) } continuation.finish() } catch is CancellationError { continuation.finish() } catch { Self.debug("SpeechAnalyzer failed: \(error.localizedDescription)") continuation.yield(.error(error.localizedDescription)) continuation.finish() } } self.lock.withLock { self.analyzerTask = task } continuation.onTermination = { @Sendable [weak self] _ in self?.cancel() } } } private struct DiagnosticTranscriber { let transcriber: DictationTranscriber let customLanguageModelEnabled: Bool let usesCustomLanguageModel: Bool let clmState: CustomLanguageModelManager.PrepareState } private static func makeDiagnosticTranscriber(locale: Locale) -> DiagnosticTranscriber { let defaults = AppGroup.defaultsIfAvailable let clmKey = AppGroupConfiguration.Keys.localASRCustomLanguageModelEnabled let clmEnabled = defaults?.object(forKey: clmKey) == nil ? true : (defaults?.bool(forKey: clmKey) ?? true) let clmState = CustomLanguageModelManager.shared.currentState() let lmConfiguration = clmEnabled ? CustomLanguageModelManager.shared.configurationForTranscription(locale: locale) : nil let transcriber = CustomLanguageModelManager.makeDictationTranscriber( locale: locale, lmConfiguration: lmConfiguration ) return DiagnosticTranscriber( transcriber: transcriber, customLanguageModelEnabled: clmEnabled, usesCustomLanguageModel: lmConfiguration != nil, clmState: clmState ) } private static func describeCLMState(_ state: CustomLanguageModelManager.PrepareState) -> String { switch state { case .idle: return "idle" case .preparing: return "preparing" case .ready: return "ready" case .failed(let message): return "failed(\(message))" } } private static func rms(of samples: [Float]) -> Float { guard !samples.isEmpty else { return 0 } var sum: Float = 0 for sample in samples { sum += sample * sample } return sqrtf(sum / Float(samples.count)) } private static func elapsed(_ start: Date) -> String { String(format: "%.2f", Date().timeIntervalSince(start)) } private static func debug(_ message: String) { OSGLog.asr.info("\(message, privacy: .public)") } private static func prepareAssetsIfNeeded( for transcriber: DictationTranscriber, locale: Locale ) async throws { let localeID = locale.identifier(.bcp47) let startedAt = Date() do { _ = try await AssetInventory.reserve(locale: locale) Self.debug("asset reserve ok locale=\(localeID)") } catch { // Reservation may already exist or slots are full; continue, but // log it so local-ASR setup failures are not hidden behind a later // "no speech" timeout. Self.debug("asset reserve non-fatal locale=\(localeID) error=\(error.localizedDescription)") } do { if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { Self.debug("asset install required locale=\(localeID)") try await request.downloadAndInstall() Self.debug("asset install done locale=\(localeID) elapsed=\(elapsed(startedAt))s") } else { Self.debug("asset already installed locale=\(localeID) elapsed=\(elapsed(startedAt))s") } } catch { Self.debug("asset prepare failed locale=\(localeID) elapsed=\(elapsed(startedAt))s error=\(error.localizedDescription)") throw error } } func cancel() { let (task, currentAnalyzer, finished) = lock.withLock { () -> (Task?, SpeechAnalyzer?, Bool) in let t = analyzerTask let a = analyzer let f = analyzerFinished analyzerTask = nil analyzer = nil return (t, a, f) } task?.cancel() guard !finished, let currentAnalyzer else { return } Task { await currentAnalyzer.cancelAndFinishNow() } } /// Maps 16 kHz Float32 snapshots into `AnalyzerInput` using the format /// returned by `bestAvailableAudioFormat(compatibleWith:considering:)`. private func makeInputStream( from stream: AsyncStream, analyzerFormat: AVAudioFormat ) -> AsyncStream { AsyncStream { continuation in Task { for await snap in stream { guard !snap.samples.isEmpty else { continue } guard let pcm = Self.makeAnalyzerPCMBuffer(from: snap, format: analyzerFormat) else { continue } continuation.yield(AnalyzerInput(buffer: pcm)) } continuation.finish() } } } private static func makeAnalyzerPCMBuffer( from snap: AudioBufferSnapshot, format: AVAudioFormat ) -> AVAudioPCMBuffer? { let capacity = AVAudioFrameCount(snap.samples.count) guard capacity > 0, let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity) else { return nil } pcm.frameLength = capacity switch format.commonFormat { case .pcmFormatInt16: guard let dst = pcm.int16ChannelData?[0] else { return nil } snap.samples.withUnsafeBufferPointer { src in ASRServiceFactory.convertFloat32ToInt16( source: src.baseAddress, sourceCount: src.count, destination: dst ) } case .pcmFormatFloat32: guard let dst = pcm.floatChannelData?[0] else { return nil } snap.samples.withUnsafeBufferPointer { src in guard let base = src.baseAddress else { return } memcpy(dst, base, src.count * MemoryLayout.stride) } default: return nil } return pcm } }