fix: feed DictationTranscriber Int16 PCM, not Float32
The keyboard preview crashed on first record with a
`__abort_with_payload` deep inside Speech's
`DictationTranscriber`. The disassembly surfaced three
preconditions checked before a `brk #0x1`:
+620 "Audio sample data must be 16-bit signed integers"
+848 "Multi-channel audio is not supported"
+1072 "Client info not fully initialized"
We hit the first one. `DictationTranscriber` (iOS 26's new
`SpeechAnalyzer`-backed engine) is strict about its input
format: only Int16 PCM, not the Float32 PCM that the iOS 18
`SFSpeechRecognizer` path accepted. Our audio-tap and
`AudioBufferSnapshot.samples: [Float]` are Float32 all the
way down — that was the SFSpeech shape, and the previous
`AppleSpeechASR` adapted internally. With iOS 26 as the
deployment target, the only ASR backend is
`SpeechAnalyzerASR`, and the conversion needed to happen at
the `AnalyzerInput` boundary.
Fix:
- `transcribe` builds the `AVAudioFormat` as
`.pcmFormatInt16, 16 kHz, 1 ch, interleaved: true` (the
canonical layout for Int16 Speech input).
- `makeInputStream` runs the per-sample conversion
`Int16(round(clamp(s * 32767, -32768, 32767)))` into the
`AVAudioPCMBuffer`'s `int16ChannelData[0]`. The explicit
clip is required (a `s == 1.5` from a gain-overflow at the
audio-engine boundary would otherwise wrap to a negative
Int16 after the implicit truncation). `round()` (not
truncate) preserves DC balance — `0.5` quantises to
`+16384`, not `+16383`, matching what audio DAWs expect.
- The conversion helper is exposed as
`ASRServiceFactory.convertFloat32ToInt16` so unit tests
can lock the math without instantiating the full pipeline.
Why not change `AudioBufferSnapshot` to `[Int16]` instead
(see earlier first-principles discussion): the snapshot is a
transport format that both `AudioCaptureService` (in the
ext) and `PreviewASRController` (in the main app) produce.
Float32 is the natural shape coming out of `AVAudioEngine`,
and pushing the conversion to the ASR service keeps the
transport contract platform-agnostic — a future second
backend with different format needs can have its own
adaptation without dragging everyone else.
Tests:
- `testFloat32ToInt16EdgeCases` — 0, ±1, ±0.5, ±1.5
(gain-overflow case).
- `testFloat32ToInt16RoundTrip` — quantisation step is
1/32767 (so the asymmetric Int16 range is honoured: -32768
has no exact Float source).
- `testFloat32ToInt16Empty` — `sourceCount == 0` with nil
pointers is a no-op (function guards on count before
dereferencing).
- All 25 tests pass (22 existing + 3 new).
- BUILD SUCCEEDED.
🤖 Generated with Claude Code
This commit is contained in:
Rocky
@@ -65,6 +65,46 @@ public enum ASRServiceFactory {
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - PCM format conversion (testable helpers)
|
||||
//
|
||||
// Extracted from the audio-thread hot path so the scaling + clipping
|
||||
// math can be exercised in unit tests without instantiating the
|
||||
// full ASR pipeline. See `OSGKeyboardTests/ASRConversionTests.swift`.
|
||||
extension ASRServiceFactory {
|
||||
|
||||
/// Convert a Float32 PCM buffer (`-1.0...1.0`) to an Int16 PCM
|
||||
/// buffer (`-32768...32767`).
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - source: Pointer to `sourceCount` `Float` samples. May be
|
||||
/// `nil` when `sourceCount == 0`.
|
||||
/// - sourceCount: Number of samples to convert. A `0` count
|
||||
/// turns the call into a no-op regardless of the pointers.
|
||||
/// - destination: Pointer to at least `sourceCount` slots of
|
||||
/// `Int16`. May be `nil` when `sourceCount == 0`.
|
||||
///
|
||||
/// Per-sample: `Int16(round(clamp(s * 32767, -32768, 32767)))`.
|
||||
/// The explicit clip matters: without it, `s == 1.0` would map
|
||||
/// to `+32767` (fine) but `s == 1.5` (which can show up at the
|
||||
/// audio engine boundary under gain) would wrap to a negative
|
||||
/// value after the implicit Float→Int16 conversion. The
|
||||
/// `round()` (rather than truncate) preserves DC balance — `0.5`
|
||||
/// quantises to `+16384`, not `+16383`, matching what most audio
|
||||
/// DAW round-trips expect.
|
||||
static func convertFloat32ToInt16(
|
||||
source: UnsafePointer<Float>?,
|
||||
sourceCount: Int,
|
||||
destination: UnsafeMutablePointer<Int16>?
|
||||
) {
|
||||
guard sourceCount > 0, let source, let destination else { return }
|
||||
for i in 0..<sourceCount {
|
||||
let scaled = source[i] * 32767.0
|
||||
let clipped = Swift.max(-32768.0, Swift.min(32767.0, scaled))
|
||||
destination[i] = Int16(clipped.rounded())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - SpeechAnalyzer implementation (iOS 26+)
|
||||
|
||||
/// ASR backend that uses the iOS 26 `SpeechAnalyzer` + `DictationTranscriber`
|
||||
@@ -87,11 +127,17 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
let newAnalyzer = SpeechAnalyzer(modules: [transcriber])
|
||||
self.lock.withLock { self.analyzer = newAnalyzer }
|
||||
|
||||
// iOS 26's `DictationTranscriber` requires **Int16** PCM
|
||||
// (precondition `"Audio sample data must be 16-bit signed
|
||||
// integers"` — Float32 was the iOS 18 `SFSpeechRecognizer`
|
||||
// shape; the new analyzer is strict). 16 kHz mono, Int16,
|
||||
// interleaved — the canonical layout Apple's Speech
|
||||
// framework examples use.
|
||||
let audioFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatFloat32,
|
||||
commonFormat: .pcmFormatInt16,
|
||||
sampleRate: 16_000,
|
||||
channels: 1,
|
||||
interleaved: false
|
||||
interleaved: true
|
||||
)!
|
||||
|
||||
let task = Task { [weak self] in
|
||||
@@ -162,6 +208,19 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
|
||||
/// Maps the `AudioBufferSnapshot` stream into the `AnalyzerInput` stream
|
||||
/// that `SpeechAnalyzer` consumes.
|
||||
///
|
||||
/// `AudioBufferSnapshot.samples` is `[Float]` (the transport format
|
||||
/// both `AudioCaptureService` and `PreviewASRController` produce —
|
||||
/// Float32 is what `AVAudioEngine` gives us at the hardware rate
|
||||
/// and we already downsample to 16 kHz mono before this point).
|
||||
/// iOS 26's `DictationTranscriber` requires **Int16** PCM at the
|
||||
/// `AnalyzerInput` boundary, so we convert per-snapshot here.
|
||||
///
|
||||
/// The conversion is the textbook `[-1.0, 1.0]` × 32767 + clip +
|
||||
/// cast. For a 16 kHz mono feed the loop is ~16k iters/sec —
|
||||
/// well under any audio-thread budget — so a simple scalar loop
|
||||
/// beats pulling in `vDSP` (which would also need a scratch
|
||||
/// buffer the audio thread can't easily allocate).
|
||||
private func makeInputStream(
|
||||
from stream: AsyncStream<AudioBufferSnapshot>,
|
||||
format: AVAudioFormat
|
||||
@@ -169,17 +228,27 @@ final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
||||
AsyncStream { continuation in
|
||||
Task {
|
||||
for await snap in stream {
|
||||
guard !snap.samples.isEmpty,
|
||||
let pcm = AVAudioPCMBuffer(
|
||||
pcmFormat: format,
|
||||
frameCapacity: AVAudioFrameCount(snap.samples.count)
|
||||
)
|
||||
else { continue }
|
||||
pcm.frameLength = AVAudioFrameCount(snap.samples.count)
|
||||
if let dst = pcm.floatChannelData?[0] {
|
||||
guard !snap.samples.isEmpty else { continue }
|
||||
let capacity = AVAudioFrameCount(snap.samples.count)
|
||||
guard let pcm = AVAudioPCMBuffer(
|
||||
pcmFormat: format,
|
||||
frameCapacity: capacity
|
||||
) else { continue }
|
||||
pcm.frameLength = capacity
|
||||
|
||||
// For a 1-channel Int16 buffer (interleaved or not —
|
||||
// single channel, so the data layout is identical),
|
||||
// `int16ChannelData?[0]` gives us the raw sample
|
||||
// pointer. Clip on overflow to avoid wraparound
|
||||
// (a Float like 1.5 would otherwise become a
|
||||
// negative Int16 after the implicit truncation).
|
||||
if let dst = pcm.int16ChannelData?[0] {
|
||||
snap.samples.withUnsafeBufferPointer { src in
|
||||
guard let base = src.baseAddress else { return }
|
||||
memcpy(dst, base, snap.samples.count * MemoryLayout<Float>.size)
|
||||
ASRServiceFactory.convertFloat32ToInt16(
|
||||
source: src.baseAddress,
|
||||
sourceCount: src.count,
|
||||
destination: dst
|
||||
)
|
||||
}
|
||||
}
|
||||
continuation.yield(AnalyzerInput(buffer: pcm))
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
// ASRConversionTests.swift
|
||||
// OSGKeyboard · Tests
|
||||
//
|
||||
// Locks in the Float32→Int16 PCM conversion that `SpeechAnalyzerASR`
|
||||
// runs on the audio thread. The dictation transcriber's precondition
|
||||
// (`"Audio sample data must be 16-bit signed integers"`) trips if
|
||||
// the conversion is wrong, so the scaling + clipping math here is
|
||||
// the difference between "Speech works" and "Speech crashes" — worth
|
||||
// a regression test even though it's only ~3 lines of arithmetic.
|
||||
|
||||
import XCTest
|
||||
@testable import OSGKeyboardShared
|
||||
|
||||
final class ASRConversionTests: XCTestCase {
|
||||
|
||||
/// Edge cases: silence, full-scale positive, full-scale negative,
|
||||
/// mid-scale positive, mid-scale negative, and the "above unity"
|
||||
/// gain-overflow case. Each is the one number we'd most regret
|
||||
/// getting wrong.
|
||||
func testFloat32ToInt16EdgeCases() {
|
||||
runConversion(
|
||||
input: [0.0, 1.0, -1.0, 0.5, -0.5, 1.5, -1.5],
|
||||
expected: [0, 32767, -32767, 16384, -16384, 32767, -32768]
|
||||
)
|
||||
}
|
||||
|
||||
/// Round-trip-ish: every Int16 in a small range should be
|
||||
/// reachable from a corresponding Float input. Locks the
|
||||
/// quantization step (1/32767) so a future "use a different
|
||||
/// scaling" change has to update this test.
|
||||
func testFloat32ToInt16RoundTrip() {
|
||||
var input = [Float]()
|
||||
var expected = [Int16]()
|
||||
for i in stride(from: -32768, through: 32767, by: 1024) {
|
||||
// Map Int16 back to its canonical Float source value:
|
||||
// src = i / 32767.0 (so src=1.0 → i=32767, src=-1.0 → i=-32767).
|
||||
// We don't test i=-32768 because the asymmetric range
|
||||
// (Int16 is -32768...32767) means there is no Float
|
||||
// that decodes back to exactly -32768.
|
||||
let src = Float(i) / 32767.0
|
||||
input.append(src)
|
||||
expected.append(Int16(i))
|
||||
}
|
||||
runConversion(input: input, expected: expected)
|
||||
}
|
||||
|
||||
/// Empty input must be a no-op. Guards against off-by-one in
|
||||
/// the loop and against `UnsafePointer` access on a zero-length
|
||||
/// array (which is undefined behaviour in C but valid in Swift).
|
||||
func testFloat32ToInt16Empty() {
|
||||
// `sourceCount == 0` with `nil` pointers is a no-op. The
|
||||
// function guards on `sourceCount > 0` before dereferencing
|
||||
// anything, so the nil pointers are safe.
|
||||
ASRServiceFactory.convertFloat32ToInt16(
|
||||
source: nil, sourceCount: 0, destination: nil
|
||||
)
|
||||
// Reaching here without crashing is the assertion.
|
||||
}
|
||||
|
||||
// MARK: - Helper
|
||||
|
||||
private func runConversion(input: [Float], expected: [Int16]) {
|
||||
precondition(input.count == expected.count, "test setup")
|
||||
var actual = [Int16](repeating: 0, count: expected.count)
|
||||
input.withUnsafeBufferPointer { src in
|
||||
actual.withUnsafeMutableBufferPointer { dst in
|
||||
ASRServiceFactory.convertFloat32ToInt16(
|
||||
source: src.baseAddress,
|
||||
sourceCount: src.count,
|
||||
destination: dst.baseAddress
|
||||
)
|
||||
}
|
||||
}
|
||||
XCTAssertEqual(actual, expected)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user