// ASRService.swift // OSGKeyboard · Shared // // Speech-to-text abstraction. As of iOS 26 being the minimum // deployment target, the only ASR backend is `SpeechAnalyzer` + // `DictationTranscriber` — always on-device, no cloud fallback, no // `requiresOnDevice` toggle. The previous legacy recognizer path is // gone; if a future platform ever needs it back, // reintroduce as a sibling class in `ASRServiceFactory.make()`. // // Lives in `OSGKeyboardShared` (not the keyboard extension target) so // that the host app's `KeyboardPreviewSheet` can run the same ASR // pipeline against real iOS audio — without it, the in-app preview // was a static mock that never actually called `SFSpeechRecognizer`, // and "did you actually wire up ASR?" was a fair review note. import Foundation import AVFoundation import Speech import os // MARK: - Sendable conformance // `AVAudioPCMBuffer` and `SpeechAnalyzer` are not Sendable. We only // ever access them serially — the PCM buffer is built and consumed // inside a single Task, and the analyzer is cancelled but never // shared concurrently — so an unchecked conformance is sound here. extension AVAudioPCMBuffer: @unchecked @retroactive Sendable {} // MARK: - Protocol public protocol ASRService: Sendable { /// Start a transcription session. The returned stream emits `.partial` /// updates and exactly one `.final` (or `.error`) before finishing. /// `SpeechAnalyzer` is always fully on-device, so there is no /// `requiresOnDevice` flag — that legacy cloud-fallback control /// doesn't apply to the iOS 26 `SpeechAnalyzer` path. func transcribe( stream: AsyncStream, locale: Locale ) -> AsyncStream /// Cancel any in-flight recognition and tear down its tasks. func cancel() } public enum ASREvent: Sendable, Equatable { /// Emitted exactly once at the start of every `transcribe` call, so /// the UI can flag non-on-device locales (e.g. ja-JP on devices that /// only ship on-device ASR for en/zh). The ASR session continues /// either way — we fall back to cloud automatically. case capability(onDeviceSupported: Bool) case partial(String) case final(String) case error(String) } // MARK: - Factory public enum ASRServiceFactory { /// Returns the ASR backend. With iOS 26 as the deployment target, /// there is exactly one backend (`SpeechAnalyzer`). public static func make() -> ASRService { SpeechAnalyzerASR() } } // MARK: - PCM format conversion (testable helpers) // // Extracted from the audio-thread hot path so the scaling + clipping // math can be exercised in unit tests without instantiating the // full ASR pipeline. See `OSGKeyboardTests/ASRConversionTests.swift`. extension ASRServiceFactory { /// Convert a Float32 PCM buffer (`-1.0...1.0`) to an Int16 PCM /// buffer (`-32768...32767`). /// /// - Parameters: /// - source: Pointer to `sourceCount` `Float` samples. May be /// `nil` when `sourceCount == 0`. /// - sourceCount: Number of samples to convert. A `0` count /// turns the call into a no-op regardless of the pointers. /// - destination: Pointer to at least `sourceCount` slots of /// `Int16`. May be `nil` when `sourceCount == 0`. /// /// Per-sample: `Int16(round(clamp(s * 32767, -32768, 32767)))`. /// The explicit clip matters: without it, `s == 1.0` would map /// to `+32767` (fine) but `s == 1.5` (which can show up at the /// audio engine boundary under gain) would wrap to a negative /// value after the implicit Float→Int16 conversion. The /// `round()` (rather than truncate) preserves DC balance — `0.5` /// quantises to `+16384`, not `+16383`, matching what most audio /// DAW round-trips expect. static func convertFloat32ToInt16( source: UnsafePointer?, sourceCount: Int, destination: UnsafeMutablePointer? ) { guard sourceCount > 0, let source, let destination else { return } for i in 0..? func transcribe( stream: AsyncStream, locale: Locale ) -> AsyncStream { AsyncStream { continuation in // SpeechAnalyzer is always fully on-device. continuation.yield(.capability(onDeviceSupported: true)) let transcriber = DictationTranscriber(locale: locale, preset: .progressiveShortDictation) let newAnalyzer = SpeechAnalyzer(modules: [transcriber]) self.lock.withLock { self.analyzer = newAnalyzer } // iOS 26's `DictationTranscriber` requires **Int16** PCM // (precondition `"Audio sample data must be 16-bit signed // integers"` — the legacy recognizer used Float32 at this // boundary; `SpeechAnalyzer` is strict Int16). 16 kHz mono, Int16, // interleaved — the canonical layout Apple's Speech // framework examples use. let audioFormat = AVAudioFormat( commonFormat: .pcmFormatInt16, sampleRate: 16_000, channels: 1, interleaved: true )! let task = Task { [weak self] in guard let self else { return } do { try await newAnalyzer.prepareToAnalyze(in: audioFormat) let inputStream = self.makeInputStream(from: stream, format: audioFormat) // Feed audio in a child task so we can concurrently // iterate `transcriber.results` on the outer task. // After the audio stream ends, finalize so the results // sequence can drain and complete. let feedTask = Task { do { try await newAnalyzer.start(inputSequence: inputStream) try await newAnalyzer.finalizeAndFinishThroughEndOfInput() } catch {} } defer { feedTask.cancel() } var lastText = "" do { for try await result in transcriber.results { if Task.isCancelled { break } // `result.text` is an AttributedString; extract plain text. let text = result.text.characters.map(String.init).joined() guard !text.isEmpty, text != lastText else { continue } lastText = text continuation.yield(.partial(text)) } } catch { // Results sequence threw — likely cancellation. } if !Task.isCancelled { continuation.yield(.final(lastText)) } continuation.finish() } catch is CancellationError { continuation.finish() } catch { continuation.yield(.error(error.localizedDescription)) continuation.finish() } } self.lock.withLock { self.analyzerTask = task } continuation.onTermination = { @Sendable [weak self] _ in self?.cancel() } } } func cancel() { let (task, currentAnalyzer) = lock.withLock { () -> (Task?, SpeechAnalyzer?) in let t = analyzerTask let a = analyzer analyzerTask = nil analyzer = nil return (t, a) } task?.cancel() if let a = currentAnalyzer { Task { await a.cancelAndFinishNow() } } } /// Maps the `AudioBufferSnapshot` stream into the `AnalyzerInput` stream /// that `SpeechAnalyzer` consumes. /// /// `AudioBufferSnapshot.samples` is `[Float]` (the transport format /// both `AudioCaptureService` and `PreviewASRController` produce — /// Float32 is what `AVAudioEngine` gives us at the hardware rate /// and we already downsample to 16 kHz mono before this point). /// iOS 26's `DictationTranscriber` requires **Int16** PCM at the /// `AnalyzerInput` boundary, so we convert per-snapshot here. /// /// The conversion is the textbook `[-1.0, 1.0]` × 32767 + clip + /// cast. For a 16 kHz mono feed the loop is ~16k iters/sec — /// well under any audio-thread budget — so a simple scalar loop /// beats pulling in `vDSP` (which would also need a scratch /// buffer the audio thread can't easily allocate). private func makeInputStream( from stream: AsyncStream, format: AVAudioFormat ) -> AsyncStream { AsyncStream { continuation in Task { for await snap in stream { guard !snap.samples.isEmpty else { continue } let capacity = AVAudioFrameCount(snap.samples.count) guard let pcm = AVAudioPCMBuffer( pcmFormat: format, frameCapacity: capacity ) else { continue } pcm.frameLength = capacity // For a 1-channel Int16 buffer (interleaved or not — // single channel, so the data layout is identical), // `int16ChannelData?[0]` gives us the raw sample // pointer. Clip on overflow to avoid wraparound // (a Float like 1.5 would otherwise become a // negative Int16 after the implicit truncation). if let dst = pcm.int16ChannelData?[0] { snap.samples.withUnsafeBufferPointer { src in ASRServiceFactory.convertFloat32ToInt16( source: src.baseAddress, sourceCount: src.count, destination: dst ) } } continuation.yield(AnalyzerInput(buffer: pcm)) } continuation.finish() } } } }