Rocky
498f407585
Introduce optional Apple account-backed credits with scoped gateway access while preserving local and BYOK paths. Refresh assistant behavior, tests, privacy disclosures, docs, and the website for the 2.0 experience.
672 lines
27 KiB
Swift
672 lines
27 KiB
Swift
// ASRService.swift
|
|
// OSGKeyboard · HostSupport
|
|
//
|
|
// Speech-to-text abstraction for foreground host flows. Local mode uses
|
|
// iOS 26 `SpeechAnalyzer` + `DictationTranscriber`; cloud mode uses the
|
|
// provider selected in the user's configuration.
|
|
//
|
|
// Lives in `OSGKeyboardHostSupport` because the foreground host owns
|
|
// audio capture and recognition. The keyboard extension receives
|
|
// completed results through the Flow bridge instead of running ASR.
|
|
|
|
import AVFoundation
|
|
import CoreMedia
|
|
import Foundation
|
|
import os
|
|
import Speech
|
|
#if canImport(OSGKeyboardShared)
|
|
import OSGKeyboardShared
|
|
#endif
|
|
|
|
// MARK: - Sendable conformance
|
|
|
|
// `AVAudioPCMBuffer` and `SpeechAnalyzer` are not Sendable. We only
|
|
// ever access them serially — the PCM buffer is built and consumed
|
|
// inside a single Task, and the analyzer is cancelled but never
|
|
// shared concurrently — so an unchecked conformance is sound here.
|
|
extension AVAudioPCMBuffer: @unchecked @retroactive Sendable {}
|
|
|
|
// MARK: - Protocol
|
|
|
|
public protocol ASRService: ASRChunkTranscribing, Sendable {
|
|
/// Start a transcription session. The returned stream emits `.partial`
|
|
/// updates and exactly one `.final` (or `.error`) before finishing.
|
|
/// The local `SpeechAnalyzer` path is fully on-device, so this
|
|
/// abstraction does not expose the legacy `requiresOnDevice` flag.
|
|
func transcribe(
|
|
stream: AsyncStream<AudioBufferSnapshot>,
|
|
locale: Locale
|
|
) -> AsyncStream<ASREvent>
|
|
|
|
/// Cancel any in-flight recognition and tear down its tasks.
|
|
func cancel()
|
|
|
|
/// Clears cancellation / cached session state before a new utterance.
|
|
func resetForNewUtterance()
|
|
|
|
/// Pre-load locale assets and analyzer format for lower first-chunk latency.
|
|
func warmup(locale: Locale) async
|
|
|
|
/// Transcribe one PCM chunk (Flow pipelined path). Default wraps `transcribe(stream:)`.
|
|
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult
|
|
}
|
|
|
|
extension ASRService {
|
|
public func resetForNewUtterance() {}
|
|
|
|
public func warmup(locale: Locale) async {}
|
|
|
|
public func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
|
|
guard !samples.isEmpty else { return .success("") }
|
|
if Task.isCancelled { return .cancelled }
|
|
|
|
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
|
|
let (stream, continuation) = AsyncStream<AudioBufferSnapshot>.makeStream()
|
|
continuation.yield(snapshot)
|
|
continuation.finish()
|
|
|
|
var lastPartial = ""
|
|
var finalText = ""
|
|
var failure: String?
|
|
|
|
for await event in transcribe(stream: stream, locale: locale) {
|
|
if Task.isCancelled { return .cancelled }
|
|
switch event {
|
|
case .capability:
|
|
break
|
|
case .partial(let text):
|
|
lastPartial = text
|
|
case .final(let text):
|
|
finalText = text
|
|
case .error(let message):
|
|
failure = message
|
|
}
|
|
}
|
|
|
|
if let failure {
|
|
return .failure(failure)
|
|
}
|
|
let trimmed = finalText.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
if !trimmed.isEmpty {
|
|
return .success(trimmed)
|
|
}
|
|
let partial = lastPartial.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
if !partial.isEmpty {
|
|
return .success(partial)
|
|
}
|
|
return .success("")
|
|
}
|
|
}
|
|
|
|
public enum ASREvent: Sendable, Equatable {
|
|
/// Emitted exactly once at the start of every `transcribe` call, so
|
|
/// the UI can flag non-on-device locales (e.g. ja-JP on devices that
|
|
/// only ship on-device ASR for en/zh). The ASR session continues
|
|
/// either way — we fall back to cloud automatically.
|
|
case capability(onDeviceSupported: Bool)
|
|
case partial(String)
|
|
case final(String)
|
|
case error(String)
|
|
}
|
|
|
|
// MARK: - Factory
|
|
|
|
public enum ASRServiceFactory {
|
|
/// Returns on-device SpeechAnalyzer for `local`, or the user's cloud
|
|
/// ASR provider when `engineMode == "cloud"`.
|
|
public static func make(store: any ConfigurationStore = AppGroupStore()) -> ASRService {
|
|
if store.engineMode == "cloud" {
|
|
return CloudASRService(store: store)
|
|
}
|
|
return SpeechAnalyzerASR()
|
|
}
|
|
}
|
|
|
|
// MARK: - PCM format conversion (testable helpers)
|
|
//
|
|
// Extracted from the audio-thread hot path so the scaling + clipping
|
|
// math can be exercised in unit tests without instantiating the
|
|
// full ASR pipeline. See `OSGKeyboardTests/ASRConversionTests.swift`.
|
|
extension ASRServiceFactory {
|
|
|
|
/// Convert a Float32 PCM buffer (`-1.0...1.0`) to an Int16 PCM
|
|
/// buffer (`-32768...32767`).
|
|
///
|
|
/// - Parameters:
|
|
/// - source: Pointer to `sourceCount` `Float` samples. May be
|
|
/// `nil` when `sourceCount == 0`.
|
|
/// - sourceCount: Number of samples to convert. A `0` count
|
|
/// turns the call into a no-op regardless of the pointers.
|
|
/// - destination: Pointer to at least `sourceCount` slots of
|
|
/// `Int16`. May be `nil` when `sourceCount == 0`.
|
|
///
|
|
/// Per-sample: `Int16(round(clamp(s * 32767, -32768, 32767)))`.
|
|
/// The explicit clip matters: without it, `s == 1.0` would map
|
|
/// to `+32767` (fine) but `s == 1.5` (which can show up at the
|
|
/// audio engine boundary under gain) would wrap to a negative
|
|
/// value after the implicit Float→Int16 conversion. The
|
|
/// `round()` (rather than truncate) preserves DC balance — `0.5`
|
|
/// quantises to `+16384`, not `+16383`, matching what most audio
|
|
/// DAW round-trips expect.
|
|
static func convertFloat32ToInt16(
|
|
source: UnsafePointer<Float>?,
|
|
sourceCount: Int,
|
|
destination: UnsafeMutablePointer<Int16>?
|
|
) {
|
|
guard sourceCount > 0, let source, let destination else { return }
|
|
for i in 0..<sourceCount {
|
|
let scaled = source[i] * 32767.0
|
|
let clipped = Swift.max(-32768.0, Swift.min(32767.0, scaled))
|
|
destination[i] = Int16(clipped.rounded())
|
|
}
|
|
}
|
|
}
|
|
|
|
// MARK: - SpeechAnalyzer implementation (iOS 26+)
|
|
|
|
/// ASR backend that uses the iOS 26 `SpeechAnalyzer` + `DictationTranscriber`
|
|
/// APIs. This engine is always fully on-device.
|
|
final class SpeechAnalyzerASR: ASRService, @unchecked Sendable {
|
|
|
|
private let lock = OSAllocatedUnfairLock()
|
|
private var analyzer: SpeechAnalyzer?
|
|
private var analyzerTask: Task<Void, Never>?
|
|
private var analyzerFinished = false
|
|
/// Reused across pipelined chunks within one utterance (assets + format).
|
|
private var chunkPreparedLocaleID: String?
|
|
private var chunkAnalyzerFormat: AVAudioFormat?
|
|
|
|
func resetForNewUtterance() {
|
|
// Keep chunk format / asset cache warm across utterances in one Flow session.
|
|
}
|
|
|
|
func invalidateChunkPreparationCache() {
|
|
lock.withLock {
|
|
chunkPreparedLocaleID = nil
|
|
chunkAnalyzerFormat = nil
|
|
}
|
|
}
|
|
|
|
func warmup(locale: Locale) async {
|
|
let warmupStartedAt = Date()
|
|
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
|
|
Self.debug("warmup locale unsupported requested=\(locale.identifier(.bcp47))")
|
|
FlowTrace.warn(
|
|
"asr.local.warmup.localeUnsupported",
|
|
"requested=\(locale.identifier(.bcp47))"
|
|
)
|
|
return
|
|
}
|
|
let localeID = resolvedLocale.identifier(.bcp47)
|
|
let cachedLocaleID = lock.withLock { chunkPreparedLocaleID }
|
|
if cachedLocaleID == localeID, lock.withLock({ chunkAnalyzerFormat != nil }) {
|
|
Self.debug("warmup cache hit locale=\(localeID)")
|
|
FlowTrace.asr("local.warmup.cacheHit", "locale=\(localeID)")
|
|
return
|
|
}
|
|
|
|
let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale)
|
|
Self.debug(
|
|
"warmup start locale=\(localeID) customLMEnabled=\(setup.customLanguageModelEnabled) " +
|
|
"customLMAttached=\(setup.usesCustomLanguageModel) " +
|
|
"clmState=\(Self.describeCLMState(setup.clmState))"
|
|
)
|
|
|
|
FlowTrace.asr(
|
|
"local.warmup.begin",
|
|
"locale=\(localeID) customLM=\(setup.usesCustomLanguageModel ? 1 : 0) "
|
|
+ "clmState=\(Self.describeCLMState(setup.clmState))"
|
|
)
|
|
do {
|
|
try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale)
|
|
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(
|
|
compatibleWith: [setup.transcriber],
|
|
considering: Self.captureFormat
|
|
) else {
|
|
Self.debug("warmup format unsupported locale=\(localeID)")
|
|
FlowTrace.warn("asr.local.warmup.formatUnsupported", "locale=\(localeID)")
|
|
return
|
|
}
|
|
lock.withLock {
|
|
chunkPreparedLocaleID = localeID
|
|
chunkAnalyzerFormat = format
|
|
}
|
|
Self.debug("warmup ready locale=\(localeID)")
|
|
FlowTrace.asr(
|
|
"local.warmup.ready",
|
|
"locale=\(localeID) analyzerRate=\(Int(format.sampleRate)) "
|
|
+ "elapsed=\(FlowTrace.seconds(since: warmupStartedAt))s"
|
|
)
|
|
} catch {
|
|
Self.debug("warmup failed: \(error.localizedDescription)")
|
|
FlowTrace.warn(
|
|
"asr.local.warmup.failed",
|
|
"locale=\(localeID) elapsed=\(FlowTrace.seconds(since: warmupStartedAt))s "
|
|
+ "error=\(error.localizedDescription)"
|
|
)
|
|
}
|
|
}
|
|
|
|
func transcribeChunk(samples: [Float], locale: Locale) async -> ASRChunkResult {
|
|
guard !samples.isEmpty else { return .success("") }
|
|
if Task.isCancelled { return .cancelled }
|
|
|
|
let startedAt = Date()
|
|
let rms = Self.rms(of: samples)
|
|
Self.debug(
|
|
"chunk start samples=\(samples.count) rms=\(String(format: "%.4f", rms)) " +
|
|
"locale=\(locale.identifier(.bcp47))"
|
|
)
|
|
do {
|
|
let text = try await transcribeSamples(samples, locale: locale, reuseChunkPrep: true)
|
|
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
Self.debug(
|
|
"chunk success textLen=\(trimmed.count) elapsed=\(Self.elapsed(startedAt))s " +
|
|
"empty=\(trimmed.isEmpty)"
|
|
)
|
|
FlowTrace.transcript(
|
|
"asr.local.chunk",
|
|
trimmed,
|
|
"engine=local samples=\(samples.count) rms=\(String(format: "%.4f", rms)) "
|
|
+ "elapsed=\(Self.elapsed(startedAt))s locale=\(locale.identifier(.bcp47))"
|
|
)
|
|
return trimmed.isEmpty ? .success("") : .success(trimmed)
|
|
} catch is CancellationError {
|
|
Self.debug("chunk cancelled elapsed=\(Self.elapsed(startedAt))s")
|
|
FlowTrace.asr("local.chunk.cancelled", "samples=\(samples.count)")
|
|
return .cancelled
|
|
} catch {
|
|
Self.debug("chunk failed elapsed=\(Self.elapsed(startedAt))s error=\(error.localizedDescription)")
|
|
FlowTrace.warn(
|
|
"asr.local.chunk.failed",
|
|
"samples=\(samples.count) rms=\(String(format: "%.4f", rms)) "
|
|
+ "error=\(error.localizedDescription)"
|
|
)
|
|
return .failure(error.localizedDescription)
|
|
}
|
|
}
|
|
|
|
/// Analyze a single PCM buffer without the streaming `transcribe` wrapper.
|
|
private func transcribeSamples(
|
|
_ samples: [Float],
|
|
locale: Locale,
|
|
reuseChunkPrep: Bool
|
|
) async throws -> String {
|
|
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
|
|
throw ASRChunkError.localeUnsupported
|
|
}
|
|
let localeID = resolvedLocale.identifier(.bcp47)
|
|
let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale)
|
|
Self.debug(
|
|
"chunk setup locale=\(localeID) customLMEnabled=\(setup.customLanguageModelEnabled) " +
|
|
"customLMAttached=\(setup.usesCustomLanguageModel) " +
|
|
"clmState=\(Self.describeCLMState(setup.clmState))"
|
|
)
|
|
|
|
let analyzerFormat: AVAudioFormat
|
|
let cachedPrep = lock.withLock { (chunkPreparedLocaleID, chunkAnalyzerFormat) }
|
|
if reuseChunkPrep,
|
|
cachedPrep.0 == localeID,
|
|
let cached = cachedPrep.1 {
|
|
analyzerFormat = cached
|
|
Self.debug(
|
|
"chunk using cached analyzer format sr=\(Int(cached.sampleRate)) " +
|
|
"channels=\(cached.channelCount) common=\(cached.commonFormat.rawValue)"
|
|
)
|
|
} else {
|
|
try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale)
|
|
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(
|
|
compatibleWith: [setup.transcriber],
|
|
considering: Self.captureFormat
|
|
) else {
|
|
throw ASRChunkError.formatUnsupported
|
|
}
|
|
analyzerFormat = format
|
|
lock.withLock {
|
|
chunkPreparedLocaleID = localeID
|
|
chunkAnalyzerFormat = format
|
|
}
|
|
Self.debug(
|
|
"chunk prepared analyzer format sr=\(Int(format.sampleRate)) " +
|
|
"channels=\(format.channelCount) common=\(format.commonFormat.rawValue)"
|
|
)
|
|
}
|
|
|
|
let snapshot = AudioBufferSnapshot(samples: samples, sampleRate: 16_000)
|
|
guard let pcm = Self.makeAnalyzerPCMBuffer(from: snapshot, format: analyzerFormat) else {
|
|
throw ASRChunkError.formatUnsupported
|
|
}
|
|
|
|
let analyzer = SpeechAnalyzer(modules: [setup.transcriber])
|
|
try await analyzer.prepareToAnalyze(in: analyzerFormat)
|
|
|
|
let resultsTask = Task<String, Error> {
|
|
var accumulator = ProgressiveDictationTranscriptAccumulator()
|
|
for try await result in setup.transcriber.results {
|
|
if Task.isCancelled { break }
|
|
let text = String(result.text.characters)
|
|
_ = accumulator.ingest(range: result.range, text: text)
|
|
}
|
|
return accumulator.finalize()
|
|
}
|
|
|
|
let inputStream = AsyncStream<AnalyzerInput> { continuation in
|
|
continuation.yield(AnalyzerInput(buffer: pcm))
|
|
continuation.finish()
|
|
}
|
|
|
|
let lastSampleTime = try await analyzer.analyzeSequence(inputStream)
|
|
if let lastSampleTime {
|
|
try await analyzer.finalizeAndFinish(through: lastSampleTime)
|
|
} else {
|
|
await analyzer.cancelAndFinishNow()
|
|
}
|
|
|
|
return try await resultsTask.value
|
|
}
|
|
|
|
private enum ASRChunkError: LocalizedError {
|
|
case localeUnsupported
|
|
case formatUnsupported
|
|
|
|
var errorDescription: String? {
|
|
switch self {
|
|
case .localeUnsupported:
|
|
return SharedL10n.string("error.asr.localeUnsupported")
|
|
case .formatUnsupported:
|
|
return SharedL10n.string("error.asr.formatUnsupported")
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Canonical 16 kHz mono Float32 format produced by the host capture
|
|
/// pipelines before samples reach SpeechAnalyzer.
|
|
private static let captureFormat = AVAudioFormat(
|
|
commonFormat: .pcmFormatFloat32,
|
|
sampleRate: 16_000,
|
|
channels: 1,
|
|
interleaved: false
|
|
)!
|
|
|
|
func transcribe(
|
|
stream: AsyncStream<AudioBufferSnapshot>,
|
|
locale: Locale
|
|
) -> AsyncStream<ASREvent> {
|
|
AsyncStream { continuation in
|
|
continuation.yield(.capability(onDeviceSupported: true))
|
|
|
|
let task = Task { [weak self] in
|
|
guard let self else { return }
|
|
self.lock.withLock { self.analyzerFinished = false }
|
|
defer {
|
|
self.lock.withLock {
|
|
self.analyzer = nil
|
|
self.analyzerTask = nil
|
|
self.analyzerFinished = true
|
|
}
|
|
}
|
|
|
|
do {
|
|
guard let resolvedLocale = await DictationTranscriber.supportedLocale(equivalentTo: locale) else {
|
|
Self.debug("locale unsupported: \(locale.identifier(.bcp47))")
|
|
continuation.yield(.error(SharedL10n.string("error.asr.localeUnsupported")))
|
|
continuation.finish()
|
|
return
|
|
}
|
|
// Each pipelined chunk is ≤ 30 s; long dictation preset keeps a
|
|
// single chunk coherent (Flow utterances run up to 3 min).
|
|
let setup = Self.makeDiagnosticTranscriber(locale: resolvedLocale)
|
|
Self.debug(
|
|
"stream setup locale=\(resolvedLocale.identifier(.bcp47)) " +
|
|
"customLMEnabled=\(setup.customLanguageModelEnabled) " +
|
|
"customLMAttached=\(setup.usesCustomLanguageModel) " +
|
|
"clmState=\(Self.describeCLMState(setup.clmState))"
|
|
)
|
|
do {
|
|
try await Self.prepareAssetsIfNeeded(for: setup.transcriber, locale: resolvedLocale)
|
|
} catch {
|
|
Self.debug("asset prepare failed: \(error.localizedDescription)")
|
|
FlowTrace.warn(
|
|
"asr.local.stream.assetsNotReady",
|
|
"locale=\(resolvedLocale.identifier(.bcp47)) "
|
|
+ "error=\(error.localizedDescription)"
|
|
)
|
|
continuation.yield(.error(SharedL10n.string("error.asr.assetsNotReady")))
|
|
continuation.finish()
|
|
return
|
|
}
|
|
|
|
let newAnalyzer = SpeechAnalyzer(modules: [setup.transcriber])
|
|
self.lock.withLock { self.analyzer = newAnalyzer }
|
|
|
|
guard let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat(
|
|
compatibleWith: [setup.transcriber],
|
|
considering: Self.captureFormat
|
|
) else {
|
|
continuation.yield(.error(SharedL10n.string("error.asr.formatUnsupported")))
|
|
continuation.finish()
|
|
return
|
|
}
|
|
|
|
try await newAnalyzer.prepareToAnalyze(in: analyzerFormat)
|
|
|
|
let inputStream = self.makeInputStream(from: stream, analyzerFormat: analyzerFormat)
|
|
|
|
// Apple recommends consuming `transcriber.results` concurrently
|
|
// while `analyzeSequence` drains the input stream.
|
|
let resultsTask = Task<String, Error> {
|
|
var accumulator = ProgressiveDictationTranscriptAccumulator()
|
|
for try await result in setup.transcriber.results {
|
|
if Task.isCancelled { break }
|
|
let text = String(result.text.characters)
|
|
guard let full = accumulator.ingest(range: result.range, text: text) else {
|
|
continue
|
|
}
|
|
FlowTrace.transcript("asr.local.partial", full, "engine=local")
|
|
continuation.yield(.partial(full))
|
|
}
|
|
return accumulator.finalize()
|
|
}
|
|
|
|
let lastSampleTime = try await newAnalyzer.analyzeSequence(inputStream)
|
|
|
|
if let lastSampleTime {
|
|
try await newAnalyzer.finalizeAndFinish(through: lastSampleTime)
|
|
} else {
|
|
await newAnalyzer.cancelAndFinishNow()
|
|
}
|
|
|
|
let lastText: String
|
|
do {
|
|
lastText = try await resultsTask.value
|
|
} catch {
|
|
Self.debug("transcriber results failed: \(error.localizedDescription)")
|
|
continuation.yield(.error(error.localizedDescription))
|
|
continuation.finish()
|
|
return
|
|
}
|
|
|
|
let trimmed = lastText.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
if trimmed.isEmpty {
|
|
FlowTrace.warn(
|
|
"asr.local.stream.emptyFinal",
|
|
"locale=\(resolvedLocale.identifier(.bcp47))"
|
|
)
|
|
continuation.yield(.error(SharedL10n.string("error.asr.noSpeech")))
|
|
} else {
|
|
FlowTrace.transcript("asr.local.final", trimmed, "engine=local")
|
|
continuation.yield(.final(trimmed))
|
|
}
|
|
continuation.finish()
|
|
} catch is CancellationError {
|
|
continuation.finish()
|
|
} catch {
|
|
Self.debug("SpeechAnalyzer failed: \(error.localizedDescription)")
|
|
continuation.yield(.error(error.localizedDescription))
|
|
continuation.finish()
|
|
}
|
|
}
|
|
self.lock.withLock { self.analyzerTask = task }
|
|
|
|
continuation.onTermination = { @Sendable [weak self] _ in
|
|
self?.cancel()
|
|
}
|
|
}
|
|
}
|
|
|
|
private struct DiagnosticTranscriber {
|
|
let transcriber: DictationTranscriber
|
|
let customLanguageModelEnabled: Bool
|
|
let usesCustomLanguageModel: Bool
|
|
let clmState: CustomLanguageModelManager.PrepareState
|
|
}
|
|
|
|
private static func makeDiagnosticTranscriber(locale: Locale) -> DiagnosticTranscriber {
|
|
let defaults = AppGroup.defaultsIfAvailable
|
|
let clmKey = AppGroupConfiguration.Keys.localASRCustomLanguageModelEnabled
|
|
let clmEnabled = defaults?.object(forKey: clmKey) == nil
|
|
? true
|
|
: (defaults?.bool(forKey: clmKey) ?? true)
|
|
let clmState = CustomLanguageModelManager.shared.currentState()
|
|
let lmConfiguration = clmEnabled
|
|
? CustomLanguageModelManager.shared.configurationForTranscription(locale: locale)
|
|
: nil
|
|
let transcriber = CustomLanguageModelManager.makeDictationTranscriber(
|
|
locale: locale,
|
|
lmConfiguration: lmConfiguration
|
|
)
|
|
return DiagnosticTranscriber(
|
|
transcriber: transcriber,
|
|
customLanguageModelEnabled: clmEnabled,
|
|
usesCustomLanguageModel: lmConfiguration != nil,
|
|
clmState: clmState
|
|
)
|
|
}
|
|
|
|
private static func describeCLMState(_ state: CustomLanguageModelManager.PrepareState) -> String {
|
|
switch state {
|
|
case .idle:
|
|
return "idle"
|
|
case .preparing:
|
|
return "preparing"
|
|
case .ready:
|
|
return "ready"
|
|
case .failed(let message):
|
|
return "failed(\(message))"
|
|
}
|
|
}
|
|
|
|
private static func rms(of samples: [Float]) -> Float {
|
|
guard !samples.isEmpty else { return 0 }
|
|
var sum: Float = 0
|
|
for sample in samples {
|
|
sum += sample * sample
|
|
}
|
|
return sqrtf(sum / Float(samples.count))
|
|
}
|
|
|
|
private static func elapsed(_ start: Date) -> String {
|
|
String(format: "%.2f", Date().timeIntervalSince(start))
|
|
}
|
|
|
|
private static func debug(_ message: String) {
|
|
OSGLog.asr.info("\(message, privacy: .public)")
|
|
}
|
|
|
|
private static func prepareAssetsIfNeeded(
|
|
for transcriber: DictationTranscriber,
|
|
locale: Locale
|
|
) async throws {
|
|
let localeID = locale.identifier(.bcp47)
|
|
let startedAt = Date()
|
|
do {
|
|
_ = try await AssetInventory.reserve(locale: locale)
|
|
Self.debug("asset reserve ok locale=\(localeID)")
|
|
} catch {
|
|
// Reservation may already exist or slots are full; continue, but
|
|
// log it so local-ASR setup failures are not hidden behind a later
|
|
// "no speech" timeout.
|
|
Self.debug("asset reserve non-fatal locale=\(localeID) error=\(error.localizedDescription)")
|
|
}
|
|
do {
|
|
if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
|
|
Self.debug("asset install required locale=\(localeID)")
|
|
try await request.downloadAndInstall()
|
|
Self.debug("asset install done locale=\(localeID) elapsed=\(elapsed(startedAt))s")
|
|
} else {
|
|
Self.debug("asset already installed locale=\(localeID) elapsed=\(elapsed(startedAt))s")
|
|
}
|
|
} catch {
|
|
Self.debug("asset prepare failed locale=\(localeID) elapsed=\(elapsed(startedAt))s error=\(error.localizedDescription)")
|
|
throw error
|
|
}
|
|
}
|
|
|
|
func cancel() {
|
|
let (task, currentAnalyzer, finished) = lock.withLock { () -> (Task<Void, Never>?, SpeechAnalyzer?, Bool) in
|
|
let t = analyzerTask
|
|
let a = analyzer
|
|
let f = analyzerFinished
|
|
analyzerTask = nil
|
|
analyzer = nil
|
|
return (t, a, f)
|
|
}
|
|
task?.cancel()
|
|
guard !finished, let currentAnalyzer else { return }
|
|
Task { await currentAnalyzer.cancelAndFinishNow() }
|
|
}
|
|
|
|
/// Maps 16 kHz Float32 snapshots into `AnalyzerInput` using the format
|
|
/// returned by `bestAvailableAudioFormat(compatibleWith:considering:)`.
|
|
private func makeInputStream(
|
|
from stream: AsyncStream<AudioBufferSnapshot>,
|
|
analyzerFormat: AVAudioFormat
|
|
) -> AsyncStream<AnalyzerInput> {
|
|
AsyncStream { continuation in
|
|
Task {
|
|
for await snap in stream {
|
|
guard !snap.samples.isEmpty else { continue }
|
|
guard let pcm = Self.makeAnalyzerPCMBuffer(from: snap, format: analyzerFormat) else {
|
|
continue
|
|
}
|
|
continuation.yield(AnalyzerInput(buffer: pcm))
|
|
}
|
|
continuation.finish()
|
|
}
|
|
}
|
|
}
|
|
|
|
private static func makeAnalyzerPCMBuffer(
|
|
from snap: AudioBufferSnapshot,
|
|
format: AVAudioFormat
|
|
) -> AVAudioPCMBuffer? {
|
|
let capacity = AVAudioFrameCount(snap.samples.count)
|
|
guard capacity > 0,
|
|
let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity) else {
|
|
return nil
|
|
}
|
|
pcm.frameLength = capacity
|
|
|
|
switch format.commonFormat {
|
|
case .pcmFormatInt16:
|
|
guard let dst = pcm.int16ChannelData?[0] else { return nil }
|
|
snap.samples.withUnsafeBufferPointer { src in
|
|
ASRServiceFactory.convertFloat32ToInt16(
|
|
source: src.baseAddress,
|
|
sourceCount: src.count,
|
|
destination: dst
|
|
)
|
|
}
|
|
case .pcmFormatFloat32:
|
|
guard let dst = pcm.floatChannelData?[0] else { return nil }
|
|
snap.samples.withUnsafeBufferPointer { src in
|
|
guard let base = src.baseAddress else { return }
|
|
memcpy(dst, base, src.count * MemoryLayout<Float>.stride)
|
|
}
|
|
default:
|
|
return nil
|
|
}
|
|
return pcm
|
|
}
|
|
}
|