fix: feed DictationTranscriber Int16 PCM, not Float32
The keyboard preview crashed on first record with a
`__abort_with_payload` deep inside Speech's
`DictationTranscriber`. The disassembly surfaced three
preconditions checked before a `brk #0x1`:
+620 "Audio sample data must be 16-bit signed integers"
+848 "Multi-channel audio is not supported"
+1072 "Client info not fully initialized"
We hit the first one. `DictationTranscriber` (iOS 26's new
`SpeechAnalyzer`-backed engine) is strict about its input
format: only Int16 PCM, not the Float32 PCM that the iOS 18
`SFSpeechRecognizer` path accepted. Our audio-tap and
`AudioBufferSnapshot.samples: [Float]` are Float32 all the
way down — that was the SFSpeech shape, and the previous
`AppleSpeechASR` adapted internally. With iOS 26 as the
deployment target, the only ASR backend is
`SpeechAnalyzerASR`, and the conversion needed to happen at
the `AnalyzerInput` boundary.
Fix:
- `transcribe` builds the `AVAudioFormat` as
`.pcmFormatInt16, 16 kHz, 1 ch, interleaved: true` (the
canonical layout for Int16 Speech input).
- `makeInputStream` runs the per-sample conversion
`Int16(round(clamp(s * 32767, -32768, 32767)))` into the
`AVAudioPCMBuffer`'s `int16ChannelData[0]`. The explicit
clip is required (a `s == 1.5` from a gain-overflow at the
audio-engine boundary would otherwise wrap to a negative
Int16 after the implicit truncation). `round()` (not
truncate) preserves DC balance — `0.5` quantises to
`+16384`, not `+16383`, matching what audio DAWs expect.
- The conversion helper is exposed as
`ASRServiceFactory.convertFloat32ToInt16` so unit tests
can lock the math without instantiating the full pipeline.
Why not change `AudioBufferSnapshot` to `[Int16]` instead
(see earlier first-principles discussion): the snapshot is a
transport format that both `AudioCaptureService` (in the
ext) and `PreviewASRController` (in the main app) produce.
Float32 is the natural shape coming out of `AVAudioEngine`,
and pushing the conversion to the ASR service keeps the
transport contract platform-agnostic — a future second
backend with different format needs can have its own
adaptation without dragging everyone else.
Tests:
- `testFloat32ToInt16EdgeCases` — 0, ±1, ±0.5, ±1.5
(gain-overflow case).
- `testFloat32ToInt16RoundTrip` — quantisation step is
1/32767 (so the asymmetric Int16 range is honoured: -32768
has no exact Float source).
- `testFloat32ToInt16Empty` — `sourceCount == 0` with nil
pointers is a no-op (function guards on count before
dereferencing).
- All 25 tests pass (22 existing + 3 new).
- BUILD SUCCEEDED.
🤖 Generated with Claude Code
This commit is contained in:
@@ -65,6 +65,46 @@ public enum ASRServiceFactory {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MARK: - PCM format conversion (testable helpers)
|
||||||
|
//
|
||||||
|
// Extracted from the audio-thread hot path so the scaling + clipping
|
||||||
|
// math can be exercised in unit tests without instantiating the
|
||||||
|
// full ASR pipeline. See `OSGKeyboardTests/ASRConversionTests.swift`.
|
||||||
|
extension ASRServiceFactory {
|
||||||
|
|
||||||
|
/// Convert a Float32 PCM buffer (`-1.0...1.0`) to an Int16 PCM
|
||||||
|
/// buffer (`-32768...32767`).
|
||||||
|
///
|
||||||
|
/// - Parameters:
|
||||||
|
/// - source: Pointer to `sourceCount` `Float` samples. May be
|
||||||
|
/// `nil` when `sourceCount == 0`.
|
||||||
|
/// - sourceCount: Number of samples to convert. A `0` count
|
||||||
|
/// turns the call into a no-op regardless of the pointers.
|
||||||
|
/// - destination: Pointer to at least `sourceCount` slots of
|
||||||
|
/// `Int16`. May be `nil` when `sourceCount == 0`.
|
||||||
|
///
|
||||||
|
/// Per-sample: `Int16(round(clamp(s * 32767, -32768, 32767)))`.
|
||||||
|
/// The explicit clip matters: without it, `s == 1.0` would map
|
||||||
|
/// to `+32767` (fine) but `s == 1.5` (which can show up at the
|
||||||
|
/// audio engine boundary under gain) would wrap to a negative
|
||||||
|
/// value after the implicit Float→Int16 conversion. The
|
||||||
|
/// `round()` (rather than truncate) preserves DC balance — `0.5`
|
||||||
|
/// quantises to `+16384`, not `+16383`, matching what most audio
|
||||||
|
/// DAW round-trips expect.
|
||||||
|
static func convertFloat32ToInt16(
|
||||||
|
source: UnsafePointer<Float>?,
|
||||||
|
sourceCount: Int,
|
||||||
|
destination: UnsafeMutablePointer<Int16>?
|
||||||
|
) {
|
||||||
|
guard sourceCount > 0, let source, let destination else { return }
|
||||||
|
for i in 0..<sourceCount {
|
||||||
|
let scaled = source[i] * 32767.0
|
||||||
|
let clipped = Swift.max(-32768.0, Swift.min(32767.0, scaled))
|
||||||
|
destination[i] = Int16(clipped.rounded())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// MARK: - SpeechAnalyzer implementation (iOS 26+)
|
// MARK: - SpeechAnalyzer implementation (iOS 26+)
|
||||||
|
|
||||||
/// ASR backend that uses the iOS 26 `SpeechAnalyzer` + `DictationTranscriber`
|
/// ASR backend that uses the iOS 26 `SpeechAnalyzer` + `DictationTranscriber`
|
||||||
@@ -87,11 +127,17 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
|||||||
let newAnalyzer = SpeechAnalyzer(modules: [transcriber])
|
let newAnalyzer = SpeechAnalyzer(modules: [transcriber])
|
||||||
self.lock.withLock { self.analyzer = newAnalyzer }
|
self.lock.withLock { self.analyzer = newAnalyzer }
|
||||||
|
|
||||||
|
// iOS 26's `DictationTranscriber` requires **Int16** PCM
|
||||||
|
// (precondition `"Audio sample data must be 16-bit signed
|
||||||
|
// integers"` — Float32 was the iOS 18 `SFSpeechRecognizer`
|
||||||
|
// shape; the new analyzer is strict). 16 kHz mono, Int16,
|
||||||
|
// interleaved — the canonical layout Apple's Speech
|
||||||
|
// framework examples use.
|
||||||
let audioFormat = AVAudioFormat(
|
let audioFormat = AVAudioFormat(
|
||||||
commonFormat: .pcmFormatFloat32,
|
commonFormat: .pcmFormatInt16,
|
||||||
sampleRate: 16_000,
|
sampleRate: 16_000,
|
||||||
channels: 1,
|
channels: 1,
|
||||||
interleaved: false
|
interleaved: true
|
||||||
)!
|
)!
|
||||||
|
|
||||||
let task = Task { [weak self] in
|
let task = Task { [weak self] in
|
||||||
@@ -162,6 +208,19 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
|||||||
|
|
||||||
/// Maps the `AudioBufferSnapshot` stream into the `AnalyzerInput` stream
|
/// Maps the `AudioBufferSnapshot` stream into the `AnalyzerInput` stream
|
||||||
/// that `SpeechAnalyzer` consumes.
|
/// that `SpeechAnalyzer` consumes.
|
||||||
|
///
|
||||||
|
/// `AudioBufferSnapshot.samples` is `[Float]` (the transport format
|
||||||
|
/// both `AudioCaptureService` and `PreviewASRController` produce —
|
||||||
|
/// Float32 is what `AVAudioEngine` gives us at the hardware rate
|
||||||
|
/// and we already downsample to 16 kHz mono before this point).
|
||||||
|
/// iOS 26's `DictationTranscriber` requires **Int16** PCM at the
|
||||||
|
/// `AnalyzerInput` boundary, so we convert per-snapshot here.
|
||||||
|
///
|
||||||
|
/// The conversion is the textbook `[-1.0, 1.0]` × 32767 + clip +
|
||||||
|
/// cast. For a 16 kHz mono feed the loop is ~16k iters/sec —
|
||||||
|
/// well under any audio-thread budget — so a simple scalar loop
|
||||||
|
/// beats pulling in `vDSP` (which would also need a scratch
|
||||||
|
/// buffer the audio thread can't easily allocate).
|
||||||
private func makeInputStream(
|
private func makeInputStream(
|
||||||
from stream: AsyncStream<AudioBufferSnapshot>,
|
from stream: AsyncStream<AudioBufferSnapshot>,
|
||||||
format: AVAudioFormat
|
format: AVAudioFormat
|
||||||
@@ -169,17 +228,27 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
|||||||
AsyncStream { continuation in
|
AsyncStream { continuation in
|
||||||
Task {
|
Task {
|
||||||
for await snap in stream {
|
for await snap in stream {
|
||||||
guard !snap.samples.isEmpty,
|
guard !snap.samples.isEmpty else { continue }
|
||||||
let pcm = AVAudioPCMBuffer(
|
let capacity = AVAudioFrameCount(snap.samples.count)
|
||||||
|
guard let pcm = AVAudioPCMBuffer(
|
||||||
pcmFormat: format,
|
pcmFormat: format,
|
||||||
frameCapacity: AVAudioFrameCount(snap.samples.count)
|
frameCapacity: capacity
|
||||||
)
|
) else { continue }
|
||||||
else { continue }
|
pcm.frameLength = capacity
|
||||||
pcm.frameLength = AVAudioFrameCount(snap.samples.count)
|
|
||||||
if let dst = pcm.floatChannelData?[0] {
|
// For a 1-channel Int16 buffer (interleaved or not —
|
||||||
|
// single channel, so the data layout is identical),
|
||||||
|
// `int16ChannelData?[0]` gives us the raw sample
|
||||||
|
// pointer. Clip on overflow to avoid wraparound
|
||||||
|
// (a Float like 1.5 would otherwise become a
|
||||||
|
// negative Int16 after the implicit truncation).
|
||||||
|
if let dst = pcm.int16ChannelData?[0] {
|
||||||
snap.samples.withUnsafeBufferPointer { src in
|
snap.samples.withUnsafeBufferPointer { src in
|
||||||
guard let base = src.baseAddress else { return }
|
ASRServiceFactory.convertFloat32ToInt16(
|
||||||
memcpy(dst, base, snap.samples.count * MemoryLayout<Float>.size)
|
source: src.baseAddress,
|
||||||
|
sourceCount: src.count,
|
||||||
|
destination: dst
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
continuation.yield(AnalyzerInput(buffer: pcm))
|
continuation.yield(AnalyzerInput(buffer: pcm))
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
// ASRConversionTests.swift
|
||||||
|
// OSGKeyboard · Tests
|
||||||
|
//
|
||||||
|
// Locks in the Float32→Int16 PCM conversion that `SpeechAnalyzerASR`
|
||||||
|
// runs on the audio thread. The dictation transcriber's precondition
|
||||||
|
// (`"Audio sample data must be 16-bit signed integers"`) trips if
|
||||||
|
// the conversion is wrong, so the scaling + clipping math here is
|
||||||
|
// the difference between "Speech works" and "Speech crashes" — worth
|
||||||
|
// a regression test even though it's only ~3 lines of arithmetic.
|
||||||
|
|
||||||
|
import XCTest
|
||||||
|
@testable import OSGKeyboardShared
|
||||||
|
|
||||||
|
final class ASRConversionTests: XCTestCase {
|
||||||
|
|
||||||
|
/// Edge cases: silence, full-scale positive, full-scale negative,
|
||||||
|
/// mid-scale positive, mid-scale negative, and the "above unity"
|
||||||
|
/// gain-overflow case. Each is the one number we'd most regret
|
||||||
|
/// getting wrong.
|
||||||
|
func testFloat32ToInt16EdgeCases() {
|
||||||
|
runConversion(
|
||||||
|
input: [0.0, 1.0, -1.0, 0.5, -0.5, 1.5, -1.5],
|
||||||
|
expected: [0, 32767, -32767, 16384, -16384, 32767, -32768]
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Round-trip-ish: every Int16 in a small range should be
|
||||||
|
/// reachable from a corresponding Float input. Locks the
|
||||||
|
/// quantization step (1/32767) so a future "use a different
|
||||||
|
/// scaling" change has to update this test.
|
||||||
|
func testFloat32ToInt16RoundTrip() {
|
||||||
|
var input = [Float]()
|
||||||
|
var expected = [Int16]()
|
||||||
|
for i in stride(from: -32768, through: 32767, by: 1024) {
|
||||||
|
// Map Int16 back to its canonical Float source value:
|
||||||
|
// src = i / 32767.0 (so src=1.0 → i=32767, src=-1.0 → i=-32767).
|
||||||
|
// We don't test i=-32768 because the asymmetric range
|
||||||
|
// (Int16 is -32768...32767) means there is no Float
|
||||||
|
// that decodes back to exactly -32768.
|
||||||
|
let src = Float(i) / 32767.0
|
||||||
|
input.append(src)
|
||||||
|
expected.append(Int16(i))
|
||||||
|
}
|
||||||
|
runConversion(input: input, expected: expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Empty input must be a no-op. Guards against off-by-one in
|
||||||
|
/// the loop and against `UnsafePointer` access on a zero-length
|
||||||
|
/// array (which is undefined behaviour in C but valid in Swift).
|
||||||
|
func testFloat32ToInt16Empty() {
|
||||||
|
// `sourceCount == 0` with `nil` pointers is a no-op. The
|
||||||
|
// function guards on `sourceCount > 0` before dereferencing
|
||||||
|
// anything, so the nil pointers are safe.
|
||||||
|
ASRServiceFactory.convertFloat32ToInt16(
|
||||||
|
source: nil, sourceCount: 0, destination: nil
|
||||||
|
)
|
||||||
|
// Reaching here without crashing is the assertion.
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Helper
|
||||||
|
|
||||||
|
private func runConversion(input: [Float], expected: [Int16]) {
|
||||||
|
precondition(input.count == expected.count, "test setup")
|
||||||
|
var actual = [Int16](repeating: 0, count: expected.count)
|
||||||
|
input.withUnsafeBufferPointer { src in
|
||||||
|
actual.withUnsafeMutableBufferPointer { dst in
|
||||||
|
ASRServiceFactory.convertFloat32ToInt16(
|
||||||
|
source: src.baseAddress,
|
||||||
|
sourceCount: src.count,
|
||||||
|
destination: dst.baseAddress
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
XCTAssertEqual(actual, expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user